Convert remaining ORM APIs to support 2.0 style

This is kind of a mixed bag of all kinds to help get us
to 1.4 betas.    The documentation stuff is a work in
progress.    Lots of other relatively small changes to
APIs and things.    More commits will follow to continue
improving the documentation and transitioning to the
1.4/2.0 hybrid documentation.  In particular some refinements
to Session usage models so that it can match Engine's
scoping / transactional patterns, and a decision to
start moving away from "subtransactions" completely.

* add select().from_statement() to produce FromStatement in an
  ORM context

* begin referring to select() that has "plugins" for the few edge
  cases where select() will have ORM-only behaviors

* convert dynamic.AppenderQuery to its own object that can use
  select(), though at the moment it uses Query to support legacy
  join calling forms.

* custom query classes for AppenderQuery are replaced by
  do_orm_execute() hooks for custom actions, a separate gerrit
  will document this

* add Session.get() to replace query.get()

* Deprecate session.begin->subtransaction.  propose within the
  test suite a hypothetical recipe for apps that rely on this
  pattern

* introduce Session construction level context manager,
  sessionmaker context manager, rewrite the whole top of the
  session_transaction.rst documentation.   Establish context manager
  patterns for Session that are identical to engine

* ensure same begin_nested() / commit() behavior as engine

* devise all new "join into an external transaction" recipe,
  add test support for it, add rules into Session so it
  just works, write new docs.  need to ensure this doesn't
  break anything

* vastly reduce the verbosity of lots of session docs as
  I dont think people read this stuff and it's difficult
  to keep current in any case

* constructs like case(), with_only_columns() really need
  to move to *columns, add a coercion rule to just change
  these.

* docs need changes everywhere I look.  in_() is not in
  the Core tutorial?  how do people even know about it?
  Remove tons of cruft from Select docs, etc.

* build a system for common ORM options like populate_existing
  and autoflush to populate from execution options.

* others?

Change-Id: Ia4bea0f804250e54d90b3884cf8aab8b66b82ecf

1 files changed, 236 insertions, 68 deletions

diff --git a/lib/sqlalchemy/orm/dynamic.py b/lib/sqlalchemy/orm/dynamic.py
index d15127563..7832152a2 100644
--- a/lib/sqlalchemy/orm/dynamic.py
+++ b/lib/sqlalchemy/orm/dynamic.py
@@ -23,7 +23,12 @@ from . import util as orm_util
 from .query import Query
 from .. import exc
 from .. import log
+from .. import sql
 from .. import util
+from ..engine import result as _result
+from ..sql import selectable
+from ..sql.base import _generative
+from ..sql.base import Generative
 
 
 @log.class_logger
@@ -74,7 +79,6 @@ class DynamicAttributeImpl(attributes.AttributeImpl):
         dispatch,
         target_mapper,
         order_by,
-        query_class=None,
         **kw
     ):
         super(DynamicAttributeImpl, self).__init__(
@@ -82,12 +86,7 @@ class DynamicAttributeImpl(attributes.AttributeImpl):
         )
         self.target_mapper = target_mapper
         self.order_by = order_by
-        if not query_class:
-            self.query_class = AppenderQuery
-        elif AppenderMixin in query_class.mro():
-            self.query_class = query_class
-        else:
-            self.query_class = mixin_user_query(query_class)
+        self.query_class = AppenderQuery
 
     def get(self, state, dict_, passive=attributes.PASSIVE_OFF):
         if not passive & attributes.SQL_OK:
@@ -259,15 +258,26 @@ class DynamicAttributeImpl(attributes.AttributeImpl):
         self.remove(state, dict_, value, initiator, passive=passive)
 
 
-class AppenderMixin(object):
-    query_class = None
+class AppenderQuery(Generative):
+    """A dynamic query that supports basic collection storage operations."""
 
     def __init__(self, attr, state):
-        super(AppenderMixin, self).__init__(attr.target_mapper, None)
+
+        # this can be select() except for aliased=True flag on join()
+        # and corresponding behaviors on select().
+        self._is_core = False
+        self._statement = Query([attr.target_mapper], None)
+
+        # self._is_core = True
+        # self._statement = sql.select(attr.target_mapper)._set_label_style(
+        #    selectable.LABEL_STYLE_TABLENAME_PLUS_COL
+        # )
+
+        self._autoflush = True
         self.instance = instance = state.obj()
         self.attr = attr
 
-        mapper = object_mapper(instance)
+        self.mapper = mapper = object_mapper(instance)
         prop = mapper._props[self.attr.key]
 
         if prop.secondary is not None:
@@ -277,29 +287,154 @@ class AppenderMixin(object):
             # is in the FROM.  So we purposely put the mapper selectable
             # in _from_obj[0] to ensure a user-defined join() later on
             # doesn't fail, and secondary is then in _from_obj[1].
-            self._from_obj = (prop.mapper.selectable, prop.secondary)
+            self._statement = self._statement.select_from(
+                prop.mapper.selectable, prop.secondary
+            )
 
-        self._where_criteria += (
+        self._statement = self._statement.where(
             prop._with_parent(instance, alias_secondary=False),
         )
 
         if self.attr.order_by:
 
-            if (
-                self._order_by_clauses is False
-                or self._order_by_clauses is None
-            ):
-                self._order_by_clauses = tuple(self.attr.order_by)
-            else:
-                self._order_by_clauses = self._order_by_clauses + tuple(
-                    self.attr.order_by
-                )
+            self._statement = self._statement.order_by(*self.attr.order_by)
+
+    @_generative
+    def autoflush(self, setting):
+        """Set autoflush to a specific setting.
+
+        Note that a Session with autoflush=False will
+        not autoflush, even if this flag is set to True at the
+        Query level.  Therefore this flag is usually used only
+        to disable autoflush for a specific Query.
+
+        """
+        self._autoflush = setting
+
+    @property
+    def statement(self):
+        """Return the Core statement represented by this
+        :class:`.AppenderQuery`.
+
+        """
+        if self._is_core:
+            return self._statement._set_label_style(
+                selectable.LABEL_STYLE_DISAMBIGUATE_ONLY
+            )
+
+        else:
+            return self._statement.statement
+
+    def filter(self, *criteria):
+        """A synonym for the :meth:`_orm.AppenderQuery.where` method."""
+
+        return self.where(*criteria)
+
+    @_generative
+    def where(self, *criteria):
+        r"""Apply the given WHERE criterion, using SQL expressions.
+
+        Equivalent to :meth:`.Select.where`.
+
+        """
+        self._statement = self._statement.where(*criteria)
+
+    @_generative
+    def order_by(self, *criteria):
+        r"""Apply the given ORDER BY criterion, using SQL expressions.
+
+        Equivalent to :meth:`.Select.order_by`.
+
+        """
+        self._statement = self._statement.order_by(*criteria)
+
+    @_generative
+    def filter_by(self, **kwargs):
+        r"""Apply the given filtering criterion using keyword expressions.
+
+        Equivalent to :meth:`.Select.filter_by`.
+
+        """
+        self._statement = self._statement.filter_by(**kwargs)
+
+    @_generative
+    def join(self, target, *props, **kwargs):
+        r"""Create a SQL JOIN against this
+        object's criterion.
+
+        Equivalent to :meth:`.Select.join`.
+        """
+
+        self._statement = self._statement.join(target, *props, **kwargs)
+
+    @_generative
+    def outerjoin(self, target, *props, **kwargs):
+        r"""Create a SQL LEFT OUTER JOIN against this
+        object's criterion.
+
+        Equivalent to :meth:`.Select.outerjoin`.
+
+        """
+
+        self._statement = self._statement.outerjoin(target, *props, **kwargs)
+
+    def scalar(self):
+        """Return the first element of the first result or None
+        if no rows present.  If multiple rows are returned,
+        raises MultipleResultsFound.
+
+        Equivalent to :meth:`_query.Query.scalar`.
+
+        .. versionadded:: 1.1.6
+
+        """
+        return self._iter().scalar()
+
+    def first(self):
+        """Return the first row.
+
+        Equivalent to :meth:`_query.Query.first`.
+
+        """
+
+        # replicates limit(1) behavior
+        if self._statement is not None:
+            return self._iter().first()
+        else:
+            return self.limit(1)._iter().first()
+
+    def one(self):
+        """Return exactly one result or raise an exception.
+
+        Equivalent to :meth:`_query.Query.one`.
+
+        """
+        return self._iter().one()
+
+    def one_or_none(self):
+        """Return one or zero results, or raise an exception for multiple
+        rows.
+
+        Equivalent to :meth:`_query.Query.one_or_none`.
+
+        .. versionadded:: 1.0.9
+
+        """
+        return self._iter().one_or_none()
+
+    def all(self):
+        """Return all rows.
+
+        Equivalent to :meth:`_query.Query.all`.
+
+        """
+        return self._iter().all()
 
     def session(self):
         sess = object_session(self.instance)
         if (
             sess is not None
-            and self.autoflush
+            and self._autoflush
             and sess.autoflush
             and self.instance in sess
         ):
@@ -311,17 +446,60 @@ class AppenderMixin(object):
 
     session = property(session, lambda s, x: None)
 
-    def __iter__(self):
+    def _execute(self, sess=None):
+        # note we're returning an entirely new Query class instance
+        # here without any assignment capabilities; the class of this
+        # query is determined by the session.
+        instance = self.instance
+        if sess is None:
+            sess = object_session(instance)
+            if sess is None:
+                raise orm_exc.DetachedInstanceError(
+                    "Parent instance %s is not bound to a Session, and no "
+                    "contextual session is established; lazy load operation "
+                    "of attribute '%s' cannot proceed"
+                    % (orm_util.instance_str(instance), self.attr.key)
+                )
+
+        result = sess.execute(self._statement, future=True)
+        result = result.scalars()
+
+        if result._attributes.get("filtered", False):
+            result = result.unique()
+
+        return result
+
+    def _iter(self):
         sess = self.session
         if sess is None:
-            return iter(
-                self.attr._get_collection_history(
-                    attributes.instance_state(self.instance),
-                    attributes.PASSIVE_NO_INITIALIZE,
-                ).added_items
-            )
+            instance = self.instance
+            state = attributes.instance_state(instance)
+
+            if state.detached:
+                raise orm_exc.DetachedInstanceError(
+                    "Parent instance %s is not bound to a Session, and no "
+                    "contextual session is established; lazy load operation "
+                    "of attribute '%s' cannot proceed"
+                    % (orm_util.instance_str(instance), self.attr.key)
+                )
+            else:
+                iterator = (
+                    (item,)
+                    for item in self.attr._get_collection_history(
+                        state, attributes.PASSIVE_NO_INITIALIZE,
+                    ).added_items
+                )
+
+                row_metadata = _result.SimpleResultMetaData(
+                    (self.mapper.class_.__name__,), [], _unique_filters=[id],
+                )
+
+                return _result.IteratorResult(row_metadata, iterator).scalars()
         else:
-            return iter(self._generate(sess))
+            return self._execute(sess)
+
+    def __iter__(self):
+        return iter(self._iter())
 
     def __getitem__(self, index):
         sess = self.session
@@ -331,9 +509,32 @@ class AppenderMixin(object):
                 attributes.PASSIVE_NO_INITIALIZE,
             ).indexed(index)
         else:
-            return self._generate(sess).__getitem__(index)
+            return orm_util._getitem(self, index)
+
+    def slice(self, start, stop):
+        """Computes the "slice" represented by
+        the given indices and apply as LIMIT/OFFSET.
+
+
+        """
+        limit_clause, offset_clause = orm_util._make_slice(
+            self._statement._limit_clause,
+            self._statement._offset_clause,
+            start,
+            stop,
+        )
+        self._statement = self._statement.limit(limit_clause).offset(
+            offset_clause
+        )
 
     def count(self):
+        """return the 'count'.
+
+        Equivalent to :meth:`_query.Query.count`.
+
+
+        """
+
         sess = self.session
         if sess is None:
             return len(
@@ -343,33 +544,10 @@ class AppenderMixin(object):
                 ).added_items
             )
         else:
-            return self._generate(sess).count()
-
-    def _generate(self, sess=None):
-        # note we're returning an entirely new Query class instance
-        # here without any assignment capabilities; the class of this
-        # query is determined by the session.
-        instance = self.instance
-        if sess is None:
-            sess = object_session(instance)
-            if sess is None:
-                raise orm_exc.DetachedInstanceError(
-                    "Parent instance %s is not bound to a Session, and no "
-                    "contextual session is established; lazy load operation "
-                    "of attribute '%s' cannot proceed"
-                    % (orm_util.instance_str(instance), self.attr.key)
-                )
-
-        if self.query_class:
-            query = self.query_class(self.attr.target_mapper, session=sess)
-        else:
-            query = sess.query(self.attr.target_mapper)
-
-        query._where_criteria = self._where_criteria
-        query._from_obj = self._from_obj
-        query._order_by_clauses = self._order_by_clauses
+            col = sql.func.count(sql.literal_column("*"))
 
-        return query
+            stmt = sql.select(col).select_from(self._statement.subquery())
+            return self.session.execute(stmt).scalar()
 
     def extend(self, iterator):
         for item in iterator:
@@ -397,16 +575,6 @@ class AppenderMixin(object):
         )
 
 
-class AppenderQuery(AppenderMixin, Query):
-    """A dynamic query that supports basic collection storage operations."""
-
-
-def mixin_user_query(cls):
-    """Return a new class with AppenderQuery functionality layered over."""
-    name = "Appender" + cls.__name__
-    return type(name, (AppenderMixin, cls), {"query_class": cls})
-
-
 class CollectionHistory(object):
     """Overrides AttributeHistory to receive append/remove events directly."""