ChangeLogTag:Fri Apr 12 19:49:50 UTC 2013 Steve Huston <shuston@riverace.com>

3 files changed, 155 insertions, 177 deletions

diff --git a/ACE/ChangeLog b/ACE/ChangeLog
index aa7ac387360..561657353d9 100644
--- a/ACE/ChangeLog
+++ b/ACE/ChangeLog
@@ -1,3 +1,16 @@
+Fri Apr 12 19:49:50 UTC 2013  Steve Huston  <shuston@riverace.com>
+
+        * ace/Dev_Poll_Reactor.cpp:
+        * ace/Dev_Poll_Reactor.h: Take advantage of the thread-safe behavior
+          of epoll_ctl/epoll_wait and allow changes to the wait set to be made
+          without interrupting the waiting leader thread. To make timers and
+          notifies more predictable, still allow only one thread to wait for
+          events at a time, but allow registrations, removals, etc. to the
+          wait set to be made by any thread. The "reactor token" now controls
+          the leader/followers for event dispatching; a new repository token
+          is added to synchronize access to the handler repository, independent
+          of waiting for events.
+
 Fri Apr 12 19:43:34 UTC 2013  Huang-Ming Huang  <huangh@ociweb.com>
 
         * ace/Log_Category.cpp:
diff --git a/ACE/ace/Dev_Poll_Reactor.cpp b/ACE/ace/Dev_Poll_Reactor.cpp
index 5a8419ff22f..d2091a22de8 100644
--- a/ACE/ace/Dev_Poll_Reactor.cpp
+++ b/ACE/ace/Dev_Poll_Reactor.cpp
@@ -582,17 +582,14 @@ ACE_Dev_Poll_Reactor::ACE_Dev_Poll_Reactor (ACE_Sig_Handler *sh,
   : initialized_ (false)
   , poll_fd_ (ACE_INVALID_HANDLE)
   // , ready_set_ ()
-#if defined (ACE_HAS_EVENT_POLL)
-  , epoll_wait_in_progress_ (false)
-#endif /* ACE_HAS_EVENT_POLL */
 #if defined (ACE_HAS_DEV_POLL)
   , dp_fds_ (0)
   , start_pfds_ (0)
   , end_pfds_ (0)
 #endif  /* ACE_HAS_DEV_POLL */
-  , deactivated_ (0)
   , token_ (*this, s_queue)
   , lock_adapter_ (token_)
+  , deactivated_ (0)
   , timer_queue_ (0)
   , delete_timer_queue_ (false)
   , signal_handler_ (0)
@@ -633,9 +630,9 @@ ACE_Dev_Poll_Reactor::ACE_Dev_Poll_Reactor (size_t size,
   , start_pfds_ (0)
   , end_pfds_ (0)
 #endif  /* ACE_HAS_DEV_POLL */
-  , deactivated_ (0)
   , token_ (*this, s_queue)
   , lock_adapter_ (token_)
+  , deactivated_ (0)
   , timer_queue_ (0)
   , delete_timer_queue_ (false)
   , signal_handler_ (0)
@@ -941,33 +938,11 @@ ACE_Dev_Poll_Reactor::work_pending_i (ACE_Time_Value * max_wait_time)
 
 #if defined (ACE_HAS_EVENT_POLL)
 
-  // See if there are handlers that have to be resumed before waiting.
-  {
-    ACE_GUARD_RETURN (ACE_SYNCH_MUTEX, grd, this->to_be_resumed_lock_, -1);
-    this->epoll_wait_in_progress_ = true;
-    for (Resume_Map::iterator i = this->to_be_resumed_.begin ();
-         i != this->to_be_resumed_.end ();
-         ++i)
-      {
-        // Make sure that 1) the handle is still registered,
-        // 2) the registered handler is the one we're waiting to resume.
-        Event_Tuple *info = this->handler_rep_.find (i->first);
-        if (info != 0 && info->event_handler == i->second)
-          {
-            this->resume_handler_i (i->first);
-          }
-      }
-    this->to_be_resumed_.clear ();
-  }
-
   // Wait for an event.
   int const nfds = ::epoll_wait (this->poll_fd_,
                                  &this->event_,
                                  1,
                                  static_cast<int> (timeout));
-  // Count on this being an atomic update; at worst, we may get an
-  // extraneous notify() from dispatch_io_event.
-  this->epoll_wait_in_progress_ = false;
 
 #else
 
@@ -1188,86 +1163,90 @@ ACE_Dev_Poll_Reactor::dispatch_io_event (Token_Guard &guard)
          handler together with descriptor, instead of looking
          it up in a repository ? Could it boost performance ?
       */
-      Event_Tuple *info = this->handler_rep_.find (handle);
-      if (info == 0)   // No registered handler any longer
-        {
-#ifdef ACE_HAS_EVENT_POLL
-          this->event_.data.fd = ACE_INVALID_HANDLE;  // Dump the event
-#endif /* ACE_HAS_EVENT_POLL */
-          return 0;
-        }
 
-      // Figure out what to do first in order to make it easier to manage
-      // the bit twiddling and possible pfds increment before releasing
-      // the token for dispatch.
-      // Note that if there's an error (such as the handle was closed
-      // without being removed from the event set) the EPOLLHUP and/or
-      // EPOLLERR bits will be set in revents.
+      // Going to access handler repo, so lock it. If the lock is
+      // unobtainable, something is very wrong so bail out.
+      Event_Tuple *info = 0;
       ACE_Reactor_Mask disp_mask = 0;
-      ACE_Event_Handler *eh = info->event_handler;
+      ACE_Event_Handler *eh = 0;
       int (ACE_Event_Handler::*callback)(ACE_HANDLE) = 0;
-      if (ACE_BIT_ENABLED (revents, out_event))
-        {
-          disp_mask = ACE_Event_Handler::WRITE_MASK;
-          callback = &ACE_Event_Handler::handle_output;
-          ACE_CLR_BITS (revents, out_event);
-        }
-      else if (ACE_BIT_ENABLED (revents, exc_event))
-        {
-          disp_mask = ACE_Event_Handler::EXCEPT_MASK;
-          callback = &ACE_Event_Handler::handle_exception;
-          ACE_CLR_BITS (revents, exc_event);
-        }
-      else if (ACE_BIT_ENABLED (revents, in_event))
-        {
-          disp_mask = ACE_Event_Handler::READ_MASK;
-          callback = &ACE_Event_Handler::handle_input;
-          ACE_CLR_BITS (revents, in_event);
-        }
-      else if (ACE_BIT_ENABLED (revents, err_event))
-        {
-          this->remove_handler_i (handle,
-                                  ACE_Event_Handler::ALL_EVENTS_MASK,
-                                  info->event_handler);
+      bool reactor_resumes_eh = false;
+      {
+        ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1);
+        info = this->handler_rep_.find (handle);
+        if (info == 0)   // No registered handler any longer
+          return 0;
+
+        // Figure out what to do first in order to make it easier to manage
+        // the bit twiddling and possible pfds increment before releasing
+        // the token for dispatch.
+        // Note that if there's an error (such as the handle was closed
+        // without being removed from the event set) the EPOLLHUP and/or
+        // EPOLLERR bits will be set in revents.
+        eh = info->event_handler;
+        if (ACE_BIT_ENABLED (revents, out_event))
+          {
+            disp_mask = ACE_Event_Handler::WRITE_MASK;
+            callback = &ACE_Event_Handler::handle_output;
+            ACE_CLR_BITS (revents, out_event);
+          }
+        else if (ACE_BIT_ENABLED (revents, exc_event))
+          {
+            disp_mask = ACE_Event_Handler::EXCEPT_MASK;
+            callback = &ACE_Event_Handler::handle_exception;
+            ACE_CLR_BITS (revents, exc_event);
+          }
+        else if (ACE_BIT_ENABLED (revents, in_event))
+          {
+            disp_mask = ACE_Event_Handler::READ_MASK;
+            callback = &ACE_Event_Handler::handle_input;
+            ACE_CLR_BITS (revents, in_event);
+          }
+        else if (ACE_BIT_ENABLED (revents, err_event))
+          {
+            this->remove_handler_i (handle,
+                                    ACE_Event_Handler::ALL_EVENTS_MASK,
+                                    grd,
+                                    info->event_handler);
 #ifdef ACE_HAS_DEV_POLL
-          ++pfds;
+            ++pfds;
 #endif /* ACE_HAS_DEV_POLL */
-          return 1;
-        }
-      else
-        {
-          ACELIB_ERROR ((LM_ERROR,
-                      ACE_TEXT ("(%t) dispatch_io h %d unknown events 0x%x\n"),
-                      handle, revents));
-        }
+            return 1;
+          }
+        else
+          {
+            ACELIB_ERROR ((LM_ERROR,
+                           ACE_TEXT ("(%t) dispatch_io h %d unknown events 0x%x\n"),
+                           handle, revents));
+          }
 
 #ifdef ACE_HAS_DEV_POLL
-      // Increment the pointer to the next element before we
-      // release the token.  Otherwise event handlers end up being
-      // dispatched multiple times for the same poll.
-      if (revents == 0)
-        ++pfds;
+        // Increment the pointer to the next element before we
+        // release the token.  Otherwise event handlers end up being
+        // dispatched multiple times for the same poll.
+        if (revents == 0)
+          ++pfds;
 #else
-      // With epoll, events are registered with oneshot, so the handle is
-      // effectively suspended; future calls to epoll_wait() will select
-      // the next event, so they're not managed here.
-      // The hitch to this is that the notify handler is always registered
-      // WITHOUT oneshot and is never suspended/resumed. This avoids endless
-      // notify loops caused by the notify handler requiring a resumption
-      // which requires the token, which requires a notify, etc. described
-      // in Bugzilla 3714. So, never suspend the notify handler.
-
-      bool reactor_resumes_eh = false;
-      if (eh != this->notify_handler_)
-        {
-          info->suspended = true;
+        // With epoll, events are registered with oneshot, so the handle is
+        // effectively suspended; future calls to epoll_wait() will select
+        // the next event, so they're not managed here.
+        // The hitch to this is that the notify handler is always registered
+        // WITHOUT oneshot and is never suspended/resumed. This avoids endless
+        // notify loops caused by the notify handler requiring a resumption
+        // which requires the token, which requires a notify, etc. described
+        // in Bugzilla 3714. So, never suspend the notify handler.
+        if (eh != this->notify_handler_)
+          {
+            info->suspended = true;
 
-          reactor_resumes_eh =
-            eh->resume_handler () ==
-            ACE_Event_Handler::ACE_REACTOR_RESUMES_HANDLER;
-        }
+            reactor_resumes_eh =
+              eh->resume_handler () ==
+              ACE_Event_Handler::ACE_REACTOR_RESUMES_HANDLER;
+          }
 #endif /* ACE_HAS_DEV_POLL */
 
+      }     // End scope for ACE_GUARD holding repo lock
+
       int status = 0;   // gets callback status, below.
 
       // Dispatch notifies directly. The notify dispatcher locates a
@@ -1305,10 +1284,6 @@ ACE_Dev_Poll_Reactor::dispatch_io_event (Token_Guard &guard)
 
         // If the callback returned 0, epoll-based needs to resume the
         // suspended handler but dev/poll doesn't.
-        // The epoll case is optimized to not acquire the token in order
-        // to resume the handler; the handler is added to a list of those
-        // that need to be resumed and is handled by the next leader
-        // that does an epoll_wait().
         // In both epoll and dev/poll cases, if the callback returns <0,
         // the token needs to be acquired and the handler checked and
         // removed if it hasn't already been.
@@ -1316,34 +1291,30 @@ ACE_Dev_Poll_Reactor::dispatch_io_event (Token_Guard &guard)
           {
 #ifdef ACE_HAS_EVENT_POLL
             // epoll-based effectively suspends handlers around the upcall.
-            // If the handler must be resumed, add it to the list.
+            // If the handler must be resumed, check to be sure it's the
+            // same handle/handler combination still.
             if (reactor_resumes_eh)
               {
-                ACE_GUARD_RETURN (ACE_SYNCH_MUTEX,
-                                  grd,
-                                  this->to_be_resumed_lock_,
-                                  -1);
-                bool map_was_empty = this->to_be_resumed_.empty();
-                this->to_be_resumed_.insert
-                  (Resume_Map::value_type (handle, eh));
-                if (this->epoll_wait_in_progress_ && map_was_empty)
-                  this->notify();
+                ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1);
+                info = this->handler_rep_.find (handle);
+                if (info != 0 && info->event_handler == eh)
+                  this->resume_handler_i (handle);
               }
 #endif /* ACE_HAS_EVENT_POLL */
             return 1;
           }
 
         // All state in the handler repository may have changed during the
-        // upcall while other threads had the token. Thus, reacquire the
-        // token and evaluate what's needed. If the upcalled handler is still
-        // the handler of record for handle, continue with checking whether
-        // or not to remove or resume the handler.
-        guard.acquire ();
+        // upcall. Thus, reacquire the repo lock and evaluate what's needed.
+        // If the upcalled handler is still the handler of record for handle,
+        // continue with checking whether or not to remove or resume the
+        // handler.
+        ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, 1);
         info = this->handler_rep_.find (handle);
         if (info != 0 && info->event_handler == eh)
           {
             if (status < 0)
-              this->remove_handler_i (handle, disp_mask);
+              this->remove_handler_i (handle, disp_mask, grd);
           }
       }
       // Scope close handles eh ref count decrement, if needed.
@@ -1397,7 +1368,7 @@ ACE_Dev_Poll_Reactor::register_handler (ACE_Event_Handler *handler,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::register_handler");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->register_handler_i (handler->get_handle (),
                                    handler,
@@ -1411,7 +1382,7 @@ ACE_Dev_Poll_Reactor::register_handler (ACE_HANDLE handle,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::register_handler");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->register_handler_i (handle,
                                    event_handler,
@@ -1517,7 +1488,7 @@ ACE_Dev_Poll_Reactor::register_handler (const ACE_Handle_Set &handle_set,
 
   ACE_Handle_Set_Iterator handle_iter (handle_set);
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   // @@ It might be more efficient to construct a pollfd array and
   //    pass it to the write () call in register_handler_i () only once,
@@ -1584,9 +1555,8 @@ ACE_Dev_Poll_Reactor::remove_handler (ACE_Event_Handler *handler,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::remove_handler");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
-
-  return this->remove_handler_i (handler->get_handle (), mask);
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
+  return this->remove_handler_i (handler->get_handle (), mask, grd);
 }
 
 int
@@ -1595,14 +1565,15 @@ ACE_Dev_Poll_Reactor::remove_handler (ACE_HANDLE handle,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::remove_handler");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
-  return this->remove_handler_i (handle, mask);
+  return this->remove_handler_i (handle, mask, grd);
 }
 
 int
 ACE_Dev_Poll_Reactor::remove_handler_i (ACE_HANDLE handle,
                                         ACE_Reactor_Mask mask,
+                                        ACE_Guard<ACE_DEV_POLL_TOKEN> &repo_guard,
                                         ACE_Event_Handler *eh)
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::remove_handler_i");
@@ -1627,7 +1598,12 @@ ACE_Dev_Poll_Reactor::remove_handler_i (ACE_HANDLE handle,
     ACE_Event_Handler::Reference_Counting_Policy::ENABLED;
 
   if (ACE_BIT_DISABLED (mask, ACE_Event_Handler::DONT_CALL))
-    (void) eh->handle_close (handle, mask);
+    {
+      // It would be great if ACE_Reverse_Lock worked with ACE_Guard.
+      repo_guard.release ();
+      eh->handle_close (handle, mask);
+      repo_guard.acquire ();
+    }
 
   // If there are no longer any outstanding events on the given handle
   // then remove it from the handler repository.
@@ -1644,20 +1620,14 @@ ACE_Dev_Poll_Reactor::remove_handler (const ACE_Handle_Set &handle_set,
   ACE_TRACE ("ACE_Dev_Poll_Reactor::remove_handler");
 
   ACE_Handle_Set_Iterator handle_iter (handle_set);
-
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
-
-  // @@ It might be more efficient to construct a pollfd array and
-  //    pass it to the write () call in register_handler_i () only once,
-  //    instead of calling write () (a system call) once for each file
-  //    descriptor.
-
   for (ACE_HANDLE h = handle_iter ();
        h != ACE_INVALID_HANDLE;
        h = handle_iter ())
-    if (this->remove_handler_i (h, mask) == -1)
-      return -1;
-
+    {
+      ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
+      if (this->remove_handler_i (h, mask, grd) == -1)
+        return -1;
+    }
   return 0;
 }
 
@@ -1711,7 +1681,7 @@ ACE_Dev_Poll_Reactor::suspend_handler (ACE_Event_Handler *event_handler)
 
   ACE_HANDLE handle = event_handler->get_handle ();
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->suspend_handler_i (handle);
 }
@@ -1721,7 +1691,7 @@ ACE_Dev_Poll_Reactor::suspend_handler (ACE_HANDLE handle)
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::suspend_handler");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->suspend_handler_i (handle);
 }
@@ -1734,7 +1704,7 @@ ACE_Dev_Poll_Reactor::suspend_handler (const ACE_Handle_Set &handles)
   ACE_Handle_Set_Iterator handle_iter (handles);
   ACE_HANDLE h;
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   while ((h = handle_iter ()) != ACE_INVALID_HANDLE)
     if (this->suspend_handler_i (h) == -1)
@@ -1748,7 +1718,7 @@ ACE_Dev_Poll_Reactor::suspend_handlers (void)
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::suspend_handlers");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   size_t const len = this->handler_rep_.max_size ();
 
@@ -1822,7 +1792,7 @@ ACE_Dev_Poll_Reactor::resume_handler (ACE_Event_Handler *event_handler)
 
   ACE_HANDLE handle = event_handler->get_handle ();
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->resume_handler_i (handle);
 }
@@ -1832,7 +1802,7 @@ ACE_Dev_Poll_Reactor::resume_handler (ACE_HANDLE handle)
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::resume_handler");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->resume_handler_i (handle);
 }
@@ -1845,7 +1815,7 @@ ACE_Dev_Poll_Reactor::resume_handler (const ACE_Handle_Set &handles)
   ACE_Handle_Set_Iterator handle_iter (handles);
   ACE_HANDLE h;
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   while ((h = handle_iter ()) != ACE_INVALID_HANDLE)
     if (this->resume_handler_i (h) == -1)
@@ -1859,7 +1829,7 @@ ACE_Dev_Poll_Reactor::resume_handlers (void)
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::resume_handlers");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   size_t const len = this->handler_rep_.max_size ();
 
@@ -1986,8 +1956,8 @@ ACE_Dev_Poll_Reactor::cancel_timer (ACE_Event_Handler *event_handler,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::cancel_timer");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
-
+  // Don't bother waking the poll - the worse that will happen is it will
+  // wake up for a timer that doesn't exist then go back to waiting.
   return (this->timer_queue_ == 0
           ? 0
           : this->timer_queue_->cancel (event_handler,
@@ -2001,8 +1971,8 @@ ACE_Dev_Poll_Reactor::cancel_timer (long timer_id,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::cancel_timer");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
-
+  // Don't bother waking the poll - the worse that will happen is it will
+  // wake up for a timer that doesn't exist then go back to waiting.
   return (this->timer_queue_ == 0
           ? 0
           : this->timer_queue_->cancel (timer_id,
@@ -2097,7 +2067,7 @@ ACE_Dev_Poll_Reactor::purge_pending_notifications (ACE_Event_Handler * eh,
 ACE_Event_Handler *
 ACE_Dev_Poll_Reactor::find_handler (ACE_HANDLE handle)
 {
-  ACE_MT (ACE_READ_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, 0));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, 0));
 
   Event_Tuple *info = this->handler_rep_.find (handle);
   if (info)
@@ -2118,7 +2088,7 @@ ACE_Dev_Poll_Reactor::handler (ACE_HANDLE handle,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::handler");
 
-  ACE_MT (ACE_READ_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   Event_Tuple *info = this->handler_rep_.find (handle);
 
@@ -2185,7 +2155,7 @@ ACE_Dev_Poll_Reactor::wakeup_all_threads (void)
   // it.
   this->notify (0,
                 ACE_Event_Handler::NULL_MASK,
- (ACE_Time_Value *) &ACE_Time_Value::zero);
+                (ACE_Time_Value *) &ACE_Time_Value::zero);
 }
 
 int
@@ -2254,7 +2224,7 @@ ACE_Dev_Poll_Reactor::mask_ops (ACE_Event_Handler *event_handler,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::mask_ops");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->mask_ops_i (event_handler->get_handle (), mask, ops);
 }
@@ -2266,7 +2236,7 @@ ACE_Dev_Poll_Reactor::mask_ops (ACE_HANDLE handle,
 {
   ACE_TRACE ("ACE_Dev_Poll_Reactor::mask_ops");
 
-  ACE_MT (ACE_GUARD_RETURN (ACE_Dev_Poll_Reactor_Token, mon, this->token_, -1));
+  ACE_MT (ACE_GUARD_RETURN (ACE_DEV_POLL_TOKEN, grd, this->repo_token_, -1));
 
   return this->mask_ops_i (handle, mask, ops);
 }
diff --git a/ACE/ace/Dev_Poll_Reactor.h b/ACE/ace/Dev_Poll_Reactor.h
index 064a278fa7b..47e67593a18 100644
--- a/ACE/ace/Dev_Poll_Reactor.h
+++ b/ACE/ace/Dev_Poll_Reactor.h
@@ -310,7 +310,7 @@ class ACE_Export ACE_Dev_Poll_Reactor : public ACE_Reactor_Impl
    *
    * @note Calls to any method in this class, and any modification to a
    *       Event_Tuple returned from this class's methods, must be made
-   *       while holding the reactor token.
+   *       while holding the repository lock.
    */
   class Handler_Repository
   {
@@ -809,7 +809,7 @@ public:
   /// table.
   virtual size_t size (void) const;
 
-  /// Returns a reference to the Reactor's internal lock.
+  /// Returns a reference to the Reactor's internal repository lock.
   virtual ACE_Lock &lock (void);
 
   /// Wake up all threads waiting in the event loop.
@@ -967,11 +967,15 @@ protected:
                           ACE_Reactor_Mask mask);
 
   /// Remove the event handler associated with the given handle and
-  /// event mask from the "interest set." If @a eh is supplied, only
-  /// do the remove if @eh matches the event handler that's registered
-  /// for @a handle.
+  /// event mask from the "interest set." If @a eh is supplied, only do the
+  /// remove if @eh matches the event handler that's registered for @a handle.
+  /// The caller is expected to be holding the repo token on entry and have
+  /// @repo_guard referencing that token. It will be temporarily released
+  /// during a handle_close() callback if needed; if it is released for the
+  //// callback it will be reacquired before return.
   int remove_handler_i (ACE_HANDLE handle,
                         ACE_Reactor_Mask mask,
+                        ACE_Guard<ACE_DEV_POLL_TOKEN> &repo_guard,
                         ACE_Event_Handler *eh = 0);
 
   /// Temporarily remove the given handle from the "interest set."
@@ -1015,17 +1019,6 @@ protected:
   /// epoll_wait() but not yet processed.
   struct epoll_event event_;
 
-  /// Event handlers that are suspended/resumed around upcalls are not
-  /// immediately resumed; they're added to this list for resumption at
-  /// the next epoll_wait() call. This avoids always needing to acquire the
-  /// token just to resume a handler. Of course, if there are no other
-  /// handlers in the to-be-resumed list and an epoll_wait is already in
-  /// progress, the reactor needs to be notified to force another run around
-  /// the epoll_wait() call.
-  typedef ACE_Array_Map<ACE_HANDLE, ACE_Event_Handler *> Resume_Map;
-  Resume_Map to_be_resumed_;
-  volatile bool epoll_wait_in_progress_;
-  ACE_SYNCH_MUTEX to_be_resumed_lock_;
 #else
   /// The pollfd array that `/dev/poll' will feed its results to.
   struct pollfd *dp_fds_;
@@ -1043,16 +1036,20 @@ protected:
   struct pollfd *end_pfds_;
 #endif  /* ACE_HAS_EVENT_POLL */
 
-  /// This flag is used to keep track of whether we are actively handling
-  /// events or not.
-  sig_atomic_t deactivated_;
-
-  /// Lock used for synchronization of reactor state.
+  /// Token serializing event waiter threads.
   ACE_Dev_Poll_Reactor_Token token_;
 
   /// Adapter used to return internal lock to outside world.
   ACE_Lock_Adapter<ACE_Dev_Poll_Reactor_Token> lock_adapter_;
 
+  /// This flag is used to keep track of whether we are actively handling
+  /// events or not.
+  sig_atomic_t deactivated_;
+
+  /// Token used to protect manipulation of the handler repository.
+  /// No need to hold the waiter token to change the repo.
+  ACE_DEV_POLL_TOKEN repo_token_;
+
   /// The repository that contains all registered event handlers.
   Handler_Repository handler_rep_;
 
@@ -1123,8 +1120,7 @@ protected:
     /// 2) wait quietly for the token, not waking another thread. This
     /// is appropriate for cases where a thread wants to wait for and
     /// dispatch an event, not causing an existing waiter to relinquish the
-    /// token, and also queuing up behind other threads waiting to modify
-    /// event records.
+    /// token.
     int acquire_quietly (ACE_Time_Value *max_wait = 0);
 
     /// A helper method that acquires the token at a high priority, and
@@ -1147,7 +1143,6 @@ protected:
     int owner_;
 
   };
-
 };