- add new event PoolEvents.invalidate(). allows interception of invalidation

events including auto-invalidation, which is useful both for tests here as well as
detecting failure conditions within the "reset" or "close" cases.
- rename the argument for PoolEvents.reset() to dbapi_connection and connection_record
to be consistent with everything else.
- add new documentation sections on invalidation, including auto-invalidation
and the invalidation process within the pool.
- add _ConnectionFairy and _ConnectionRecord to the pool documentation.  Establish
docs for common _ConnectionFairy/_ConnectionRecord methods and accessors and
have PoolEvents docs refer to _ConnectionRecord,
since it is passed to all events.  Rename a few _ConnectionFairy methods that are actually
private to pool such as _checkout(), _checkin() and _checkout_existing(); there should not
be any external code calling these

6 files changed, 309 insertions, 55 deletions

diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst
index eed126c2b..e6a77378a 100644
--- a/doc/build/changelog/changelog_09.rst
+++ b/doc/build/changelog/changelog_09.rst
@@ -15,6 +15,23 @@
     :version: 0.9.2
 
     .. change::
+        :tags: feature, pool, engine
+
+        Added a new pool event :meth:`.PoolEvents.invalidate`.  Called when
+        a DBAPI connection is to be marked as "invaldated" and discarded
+        from the pool.
+
+    .. change::
+        :tags: bug, pool
+
+        The argument names for the :meth:`.PoolEvents.reset` event have been
+        renamed to ``dbapi_connection`` and ``connection_record`` in order
+        to maintain consistency with all the other pool events.  It is expected
+        that any existing listeners for this relatively new and
+        seldom-used event are using positional style to receive arguments in
+        any case.
+
+    .. change::
         :tags: bug, py3k, cextensions
         :pullreq: github:55
 
diff --git a/doc/build/core/pooling.rst b/doc/build/core/pooling.rst
index 550fb3527..fcd8fd55c 100644
--- a/doc/build/core/pooling.rst
+++ b/doc/build/core/pooling.rst
@@ -282,6 +282,51 @@ server at the point at which the script pauses for input::
         print c.execute("select 1").fetchall()
         c.close()
 
+.. _pool_connection_invalidation:
+
+More on Invalidation
+^^^^^^^^^^^^^^^^^^^^
+
+The :class:`.Pool` provides "connection invalidation" services which allow
+both explicit invalidation of a connection as well as automatic invalidation
+in response to conditions that are determined to render a connection unusable.
+
+"Invalidation" means that a particular DBAPI connection is removed from the
+pool and discarded.  The ``.close()`` method is called on this connection
+if it is not clear that the connection itself might not be closed, however
+if this method fails, the exception is logged but the operation still proceeds.
+
+When using a :class:`.Engine`, the :meth:`.Connection.invalidate` method is
+the usual entrypoint to explicit invalidation.   Other conditions by which
+a DBAPI connection might be invalidated include:
+
+* a DBAPI exception such as :class:`.OperationalError`, raised when a
+  method like ``connection.execute()`` is called, is detected as indicating
+  a so-called "disconnect" condition.   As the Python DBAPI provides no
+  standard system for determining the nature of an exception, all SQLAlchemy
+  dialects include a system called ``is_disconnect()`` which will examine
+  the contents of an exception object, including the string message and
+  any potential error codes included with it, in order to determine if this
+  exception indicates that the connection is no longer usable.  If this is the
+  case, the :meth:`._ConnectionFairy.invalidate` method is called and the
+  DBAPI connection is then discarded.
+
+* When the connection is returned to the pool, and
+  calling the ``connection.rollback()`` or ``connection.commit()`` methods,
+  as dictated by the pool's "reset on return" behavior, throws an exception.
+  A final attempt at calling ``.close()`` on the connection will be made,
+  and it is then discarded.
+
+* When a listener implementing :meth:`.PoolEvents.checkout` raises the
+  :class:`~sqlalchemy.exc.DisconnectionError` exception, indicating that the connection
+  won't be usable and a new connection attempt needs to be made.
+
+All invalidations which occur will invoke the :meth:`.PoolEvents.invalidate`
+event.
+
+
+
+
 API Documentation - Available Pool Implementations
 ---------------------------------------------------
 
@@ -301,7 +346,6 @@ API Documentation - Available Pool Implementations
 
 .. autoclass:: SingletonThreadPool
 
-
    .. automethod:: __init__
 
 .. autoclass:: AssertionPool
@@ -312,6 +356,13 @@ API Documentation - Available Pool Implementations
 
 .. autoclass:: StaticPool
 
+.. autoclass:: _ConnectionFairy
+    :members:
+
+    .. autoattribute:: _connection_record
+
+.. autoclass:: _ConnectionRecord
+    :members:
 
 
 Pooling Plain DB-API Connections
diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py
index 0e888ca4a..ff2e6e282 100644
--- a/lib/sqlalchemy/engine/base.py
+++ b/lib/sqlalchemy/engine/base.py
@@ -303,20 +303,40 @@ class Connection(Connectable):
 
     def invalidate(self, exception=None):
         """Invalidate the underlying DBAPI connection associated with
-        this Connection.
+        this :class:`.Connection`.
 
-        The underlying DB-API connection is literally closed (if
+        The underlying DBAPI connection is literally closed (if
         possible), and is discarded.  Its source connection pool will
         typically lazily create a new connection to replace it.
 
-        Upon the next usage, this Connection will attempt to reconnect
-        to the pool with a new connection.
+        Upon the next use (where "use" typically means using the
+        :meth:`.Connection.execute` method or similar),
+        this :class:`.Connection` will attempt to
+        procure a new DBAPI connection using the services of the
+        :class:`.Pool` as a source of connectivty (e.g. a "reconnection").
+
+        If a transaction was in progress (e.g. the
+        :meth:`.Connection.begin` method has been called) when
+        :meth:`.Connection.invalidate` method is called, at the DBAPI
+        level all state associated with this transaction is lost, as
+        the DBAPI connection is closed.  The :class:`.Connection`
+        will not allow a reconnection to proceed until the :class:`.Transaction`
+        object is ended, by calling the :meth:`.Transaction.rollback`
+        method; until that point, any attempt at continuing to use the
+        :class:`.Connection` will raise an
+        :class:`~sqlalchemy.exc.InvalidRequestError`.
+        This is to prevent applications from accidentally
+        continuing an ongoing transactional operations despite the
+        fact that the transaction has been lost due to an
+        invalidation.
+
+        The :meth:`.Connection.invalidate` method, just like auto-invalidation,
+        will at the connection pool level invoke the :meth:`.PoolEvents.invalidate`
+        event.
 
-        Transactions in progress remain in an "opened" state (even though the
-        actual transaction is gone); these must be explicitly rolled back
-        before a reconnect on this Connection can proceed. This is to prevent
-        applications from accidentally continuing their transactional
-        operations in a non-transactional state.
+        .. seealso::
+
+            :ref:`pool_connection_invalidation`
 
         """
         if self.invalidated:
diff --git a/lib/sqlalchemy/events.py b/lib/sqlalchemy/events.py
index cf77bbb7d..9f05c8b5b 100644
--- a/lib/sqlalchemy/events.py
+++ b/lib/sqlalchemy/events.py
@@ -266,41 +266,52 @@ class PoolEvents(event.Events):
             return target
 
     def connect(self, dbapi_connection, connection_record):
-        """Called once for each new DB-API connection or Pool's ``creator()``.
+        """Called at the moment a particular DBAPI connection is first
+        created for a given :class:`.Pool`.
 
-        :param dbapi_con:
-          A newly connected raw DB-API connection (not a SQLAlchemy
-          ``Connection`` wrapper).
+        This event allows one to capture the point directly after which
+        the DBAPI module-level ``.connect()`` method has been used in order
+        to produce a new DBAPI connection.
 
-        :param con_record:
-          The ``_ConnectionRecord`` that persistently manages the connection
+        :param dbapi_connection: a DBAPI connection.
+
+        :param connection_record: the :class:`._ConnectionRecord` managing the
+         DBAPI connection.
 
         """
 
     def first_connect(self, dbapi_connection, connection_record):
-        """Called exactly once for the first DB-API connection.
+        """Called exactly once for the first time a DBAPI connection is
+        checked out from a particular :class:`.Pool`.
+
+        The rationale for :meth:`.PoolEvents.first_connect` is to determine
+        information about a particular series of database connections based
+        on the settings used for all connections.  Since a particular
+        :class:`.Pool` refers to a single "creator" function (which in terms
+        of a :class:`.Engine` refers to the URL and connection options used),
+        it is typically valid to make observations about a single connection
+        that can be safely assumed to be valid about all subsequent connections,
+        such as the database version, the server and client encoding settings,
+        collation settings, and many others.
 
-        :param dbapi_con:
-          A newly connected raw DB-API connection (not a SQLAlchemy
-          ``Connection`` wrapper).
+        :param dbapi_connection: a DBAPI connection.
 
-        :param con_record:
-          The ``_ConnectionRecord`` that persistently manages the connection
+        :param connection_record: the :class:`._ConnectionRecord` managing the
+         DBAPI connection.
 
         """
 
     def checkout(self, dbapi_connection, connection_record, connection_proxy):
         """Called when a connection is retrieved from the Pool.
 
-        :param dbapi_con:
-          A raw DB-API connection
+        :param dbapi_connection: a DBAPI connection.
 
-        :param con_record:
-          The ``_ConnectionRecord`` that persistently manages the connection
+        :param connection_record: the :class:`._ConnectionRecord` managing the
+         DBAPI connection.
 
-        :param con_proxy:
-          The ``_ConnectionFairy`` which manages the connection for the span of
-          the current checkout.
+        :param connection_proxy: the :class:`._ConnectionFairy` object which
+          will proxy the public interface of the DBAPI connection for the lifespan
+          of the checkout.
 
         If you raise a :class:`~sqlalchemy.exc.DisconnectionError`, the current
         connection will be disposed and a fresh connection retrieved.
@@ -319,15 +330,14 @@ class PoolEvents(event.Events):
         connection has been invalidated.  ``checkin`` will not be called
         for detached connections.  (They do not return to the pool.)
 
-        :param dbapi_con:
-          A raw DB-API connection
+        :param dbapi_connection: a DBAPI connection.
 
-        :param con_record:
-          The ``_ConnectionRecord`` that persistently manages the connection
+        :param connection_record: the :class:`._ConnectionRecord` managing the
+         DBAPI connection.
 
         """
 
-    def reset(self, dbapi_con, con_record):
+    def reset(self, dbapi_connnection, connection_record):
         """Called before the "reset" action occurs for a pooled connection.
 
         This event represents
@@ -341,11 +351,10 @@ class PoolEvents(event.Events):
         the :meth:`.PoolEvents.checkin` event is called, except in those
         cases where the connection is discarded immediately after reset.
 
-        :param dbapi_con:
-          A raw DB-API connection
+        :param dbapi_connection: a DBAPI connection.
 
-        :param con_record:
-          The ``_ConnectionRecord`` that persistently manages the connection
+        :param connection_record: the :class:`._ConnectionRecord` managing the
+         DBAPI connection.
 
         .. versionadded:: 0.8
 
@@ -357,6 +366,30 @@ class PoolEvents(event.Events):
 
         """
 
+    def invalidate(self, dbapi_connection, connection_record, exception):
+        """Called when a DBAPI connection is to be "invalidated".
+
+        This event is called any time the :meth:`._ConnectionRecord.invalidate`
+        method is invoked, either from API usage or via "auto-invalidation".
+        The event occurs before a final attempt to call ``.close()`` on the connection
+        occurs.
+
+        :param dbapi_connection: a DBAPI connection.
+
+        :param connection_record: the :class:`._ConnectionRecord` managing the
+         DBAPI connection.
+
+        :param exception: the exception object corresponding to the reason
+         for this invalidation, if any.  May be ``None``.
+
+        .. versionadded:: 0.9.2 Added support for connection invalidation
+           listening.
+
+        .. seealso::
+
+            :ref:`pool_connection_invalidation`
+
+        """
 
 
 class ConnectionEvents(event.Events):
diff --git a/lib/sqlalchemy/pool.py b/lib/sqlalchemy/pool.py
index 3adfb320b..0f0a2ac10 100644
--- a/lib/sqlalchemy/pool.py
+++ b/lib/sqlalchemy/pool.py
@@ -216,7 +216,7 @@ class Pool(log.Identified):
 
         """
 
-        return _ConnectionFairy.checkout(self)
+        return _ConnectionFairy._checkout(self)
 
     def _create_connection(self):
         """Called by subclasses to create a new ConnectionRecord."""
@@ -268,7 +268,7 @@ class Pool(log.Identified):
 
         """
         if not self._use_threadlocal:
-            return _ConnectionFairy.checkout(self)
+            return _ConnectionFairy._checkout(self)
 
         try:
             rec = self._threadconns.current()
@@ -276,9 +276,9 @@ class Pool(log.Identified):
             pass
         else:
             if rec is not None:
-                return rec.checkout_existing()
+                return rec._checkout_existing()
 
-        return _ConnectionFairy.checkout(self, self._threadconns)
+        return _ConnectionFairy._checkout(self, self._threadconns)
 
     def _return_conn(self, record):
         """Given a _ConnectionRecord, return it to the :class:`.Pool`.
@@ -309,6 +309,34 @@ class Pool(log.Identified):
 
 
 class _ConnectionRecord(object):
+    """Internal object which maintains an individual DBAPI connection
+    referenced by a :class:`.Pool`.
+
+    The :class:`._ConnectionRecord` object always exists for any particular
+    DBAPI connection whether or not that DBAPI connection has been
+    "checked out".  This is in contrast to the :class:`._ConnectionFairy`
+    which is only a public facade to the DBAPI connection while it is checked
+    out.
+
+    A :class:`._ConnectionRecord` may exist for a span longer than that
+    of a single DBAPI connection.  For example, if the
+    :meth:`._ConnectionRecord.invalidate`
+    method is called, the DBAPI connection associated with this
+    :class:`._ConnectionRecord`
+    will be discarded, but the :class:`._ConnectionRecord` may be used again,
+    in which case a new DBAPI connection is produced when the :class:`.Pool`
+    next uses this record.
+
+    The :class:`._ConnectionRecord` is delivered along with connection
+    pool events, including :meth:`.PoolEvents.connect` and
+    :meth:`.PoolEvents.checkout`, however :class:`._ConnectionRecord` still
+    remains an internal object whose API and internals may change.
+
+    .. seealso::
+
+        :class:`._ConnectionFairy`
+
+    """
 
     def __init__(self, pool):
         self.__pool = pool
@@ -320,8 +348,23 @@ class _ConnectionRecord(object):
                     exec_once(self.connection, self)
         pool.dispatch.connect(self.connection, self)
 
+    connection = None
+    """A reference to the actual DBAPI connection being tracked.
+
+    May be ``None`` if this :class:`._ConnectionRecord` has been marked
+    as invalidated; a new DBAPI connection may replace it if the owning
+    pool calls upon this :class:`._ConnectionRecord` to reconnect.
+
+    """
+
     @util.memoized_property
     def info(self):
+        """The ``.info`` dictionary associated with the DBAPI connection.
+
+        This dictionary is shared among the :attr:`._ConnectionFairy.info`
+        and :attr:`.Connection.info` accessors.
+
+        """
         return {}
 
     @classmethod
@@ -360,9 +403,22 @@ class _ConnectionRecord(object):
 
     def close(self):
         if self.connection is not None:
-            self.__pool._close_connection(self.connection)
+            self.__close()
 
     def invalidate(self, e=None):
+        """Invalidate the DBAPI connection held by this :class:`._ConnectionRecord`.
+
+        This method is called for all connection invalidations, including
+        when the :meth:`._ConnectionFairy.invalidate` or :meth:`.Connection.invalidate`
+        methods are called, as well as when any so-called "automatic invalidation"
+        condition occurs.
+
+        .. seealso::
+
+            :ref:`pool_connection_invalidation`
+
+        """
+        self.__pool.dispatch.invalidate(self.connection, self, e)
         if e is not None:
             self.__pool.logger.info(
                 "Invalidate connection %r (reason: %s:%s)",
@@ -453,15 +509,41 @@ _refs = set()
 
 
 class _ConnectionFairy(object):
-    """Proxies a DB-API connection and provides return-on-dereference
-    support."""
+    """Proxies a DBAPI connection and provides return-on-dereference
+    support.
+
+    This is an internal object used by the :class:`.Pool` implementation
+    to provide context management to a DBAPI connection delivered by
+    that :class:`.Pool`.
+
+    The name "fairy" is inspired by the fact that the :class:`._ConnectionFairy`
+    object's lifespan is transitory, as it lasts only for the length of a
+    specific DBAPI connection being checked out from the pool, and additionally
+    that as a transparent proxy, it is mostly invisible.
+
+    .. seealso::
+
+        :class:`._ConnectionRecord`
+
+    """
 
     def __init__(self, dbapi_connection, connection_record):
         self.connection = dbapi_connection
         self._connection_record = connection_record
 
+    connection = None
+    """A reference to the actual DBAPI connection being tracked."""
+
+    _connection_record = None
+    """A reference to the :class:`._ConnectionRecord` object associated
+    with the DBAPI connection.
+
+    This is currently an internal accessor which is subject to change.
+
+    """
+
     @classmethod
-    def checkout(cls, pool, threadconns=None, fairy=None):
+    def _checkout(cls, pool, threadconns=None, fairy=None):
         if not fairy:
             fairy = _ConnectionRecord.checkout(pool)
 
@@ -498,16 +580,16 @@ class _ConnectionFairy(object):
         fairy.invalidate()
         raise exc.InvalidRequestError("This connection is closed")
 
-    def checkout_existing(self):
-        return _ConnectionFairy.checkout(self._pool, fairy=self)
+    def _checkout_existing(self):
+        return _ConnectionFairy._checkout(self._pool, fairy=self)
 
-    def checkin(self):
+    def _checkin(self):
         _finalize_fairy(self.connection, self._connection_record,
                             self._pool, None, self._echo, fairy=self)
         self.connection = None
         self._connection_record = None
 
-    _close = checkin
+    _close = _checkin
 
     @property
     def _logger(self):
@@ -515,6 +597,9 @@ class _ConnectionFairy(object):
 
     @property
     def is_valid(self):
+        """Return True if this :class:`._ConnectionFairy` still refers
+        to an active DBAPI connection."""
+
         return self.connection is not None
 
     @util.memoized_property
@@ -525,7 +610,9 @@ class _ConnectionFairy(object):
 
         The data here will follow along with the DBAPI connection including
         after it is returned to the connection pool and used again
-        in subsequent instances of :class:`.ConnectionFairy`.
+        in subsequent instances of :class:`._ConnectionFairy`.  It is shared
+        with the :attr:`._ConnectionRecord.info` and :attr:`.Connection.info`
+        accessors.
 
         """
         return self._connection_record.info
@@ -533,8 +620,16 @@ class _ConnectionFairy(object):
     def invalidate(self, e=None):
         """Mark this connection as invalidated.
 
-        The connection will be immediately closed.  The containing
-        ConnectionRecord will create a new connection when next used.
+        This method can be called directly, and is also called as a result
+        of the :meth:`.Connection.invalidate` method.   When invoked,
+        the DBAPI connection is immediately closed and discarded from
+        further use by the pool.  The invalidation mechanism proceeds
+        via the :meth:`._ConnectionRecord.invalidate` internal method.
+
+        .. seealso::
+
+            :ref:`pool_connection_invalidation`
+
         """
 
         if self.connection is None:
@@ -542,9 +637,15 @@ class _ConnectionFairy(object):
         if self._connection_record:
             self._connection_record.invalidate(e=e)
         self.connection = None
-        self.checkin()
+        self._checkin()
 
     def cursor(self, *args, **kwargs):
+        """Return a new DBAPI cursor for the underlying connection.
+
+        This method is a proxy for the ``connection.cursor()`` DBAPI
+        method.
+
+        """
         return self.connection.cursor(*args, **kwargs)
 
     def __getattr__(self, key):
@@ -576,7 +677,7 @@ class _ConnectionFairy(object):
     def close(self):
         self._counter -= 1
         if self._counter == 0:
-            self.checkin()
+            self._checkin()
 
 
 
diff --git a/test/engine/test_pool.py b/test/engine/test_pool.py
index eb70bdf7f..10f490b48 100644
--- a/test/engine/test_pool.py
+++ b/test/engine/test_pool.py
@@ -308,6 +308,13 @@ class PoolEventsTest(PoolTestBase):
 
         return p, canary
 
+    def _invalidate_event_fixture(self):
+        p = self._queuepool_fixture()
+        canary = Mock()
+        event.listen(p, 'invalidate', canary)
+
+        return p, canary
+
     def test_first_connect_event(self):
         p, canary = self._first_connect_event_fixture()
 
@@ -411,6 +418,31 @@ class PoolEventsTest(PoolTestBase):
         c1.close()
         eq_(canary, ['reset'])
 
+    def test_invalidate_event_no_exception(self):
+        p, canary = self._invalidate_event_fixture()
+
+        c1 = p.connect()
+        c1.close()
+        assert not canary.called
+        c1 = p.connect()
+        dbapi_con = c1.connection
+        c1.invalidate()
+        assert canary.call_args_list[0][0][0] is dbapi_con
+        assert canary.call_args_list[0][0][2] is None
+
+    def test_invalidate_event_exception(self):
+        p, canary = self._invalidate_event_fixture()
+
+        c1 = p.connect()
+        c1.close()
+        assert not canary.called
+        c1 = p.connect()
+        dbapi_con = c1.connection
+        exc = Exception("hi")
+        c1.invalidate(exc)
+        assert canary.call_args_list[0][0][0] is dbapi_con
+        assert canary.call_args_list[0][0][2] is exc
+
     def test_checkin_event_gc(self):
         p, canary = self._checkin_event_fixture()