Merge branch 'ticket_2746'

Conflicts:
	doc/build/changelog/changelog_08.rst
	doc/build/changelog/changelog_09.rst

3 files changed, 186 insertions, 66 deletions

diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py
index a302bf401..272bd1740 100644
--- a/lib/sqlalchemy/dialects/oracle/base.py
+++ b/lib/sqlalchemy/dialects/oracle/base.py
@@ -553,12 +553,8 @@ class OracleCompiler(compiler.SQLCompiler):
 
         if not getattr(select, '_oracle_visit', None):
             if not self.dialect.use_ansi:
-                if self.stack and 'from' in self.stack[-1]:
-                    existingfroms = self.stack[-1]['from']
-                else:
-                    existingfroms = None
-
-                froms = select._get_display_froms(existingfroms)
+                froms = self._display_froms_for_select(
+                                    select, kwargs.get('asfrom', False))
                 whereclause = self._get_nonansi_join_whereclause(froms)
                 if whereclause is not None:
                     select = select.where(whereclause)
diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index f1fe53b73..0afcdfaec 100644
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -621,9 +621,15 @@ class SQLCompiler(engine.Compiled):
 
     def visit_compound_select(self, cs, asfrom=False,
                             parens=True, compound_index=0, **kwargs):
-        entry = self.stack and self.stack[-1] or {}
-        self.stack.append({'from': entry.get('from', None),
-                    'iswrapper': not entry})
+        toplevel = not self.stack
+        entry = self._default_stack_entry if toplevel else self.stack[-1]
+
+        self.stack.append(
+                    {
+                        'correlate_froms': entry['correlate_froms'],
+                        'iswrapper': toplevel,
+                        'asfrom_froms': entry['asfrom_froms']
+                    })
 
         keyword = self.compound_keywords.get(cs.keyword)
 
@@ -644,7 +650,7 @@ class SQLCompiler(engine.Compiled):
                         self.limit_clause(cs) or ""
 
         if self.ctes and \
-            compound_index == 0 and not entry:
+            compound_index == 0 and toplevel:
             text = self._render_cte_clause() + text
 
         self.stack.pop(-1)
@@ -1197,12 +1203,42 @@ class SQLCompiler(engine.Compiled):
             objs = tuple([d.get(col, col) for col in objs])
             self.result_map[key] = (name, objs, typ)
 
+
+    _default_stack_entry = util.immutabledict([
+                                        ('iswrapper', False),
+                                        ('correlate_froms', frozenset()),
+                                        ('asfrom_froms', frozenset())
+                                    ])
+
+    def _display_froms_for_select(self, select, asfrom):
+        # utility method to help external dialects
+        # get the correct from list for a select.
+        # specifically the oracle dialect needs this feature
+        # right now.
+        toplevel = not self.stack
+        entry = self._default_stack_entry if toplevel else self.stack[-1]
+
+        correlate_froms = entry['correlate_froms']
+        asfrom_froms = entry['asfrom_froms']
+
+        if asfrom:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=\
+                                correlate_froms.difference(asfrom_froms),
+                            implicit_correlate_froms=())
+        else:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=correlate_froms,
+                            implicit_correlate_froms=asfrom_froms)
+        return froms
+
     def visit_select(self, select, asfrom=False, parens=True,
                             iswrapper=False, fromhints=None,
                             compound_index=0,
                             force_result_map=False,
                             positional_names=None,
-                            nested_join_translation=False, **kwargs):
+                            nested_join_translation=False,
+                            **kwargs):
 
         needs_nested_translation = \
                             select.use_labels and \
@@ -1221,12 +1257,14 @@ class SQLCompiler(engine.Compiled):
                             nested_join_translation=True, **kwargs
                         )
 
-        entry = self.stack and self.stack[-1] or {}
+        toplevel = not self.stack
+        entry = self._default_stack_entry if toplevel else self.stack[-1]
+
 
         populate_result_map = force_result_map or (
                                         compound_index == 0 and (
-                                            not entry or \
-                                            entry.get('iswrapper', False)
+                                            toplevel or \
+                                            entry['iswrapper']
                                         )
                                     )
 
@@ -1236,15 +1274,28 @@ class SQLCompiler(engine.Compiled):
                                                 select, transformed_select)
             return text
 
-        existingfroms = entry.get('from', None)
+        correlate_froms = entry['correlate_froms']
+        asfrom_froms = entry['asfrom_froms']
 
-        froms = select._get_display_froms(existingfroms, asfrom=asfrom)
-
-        correlate_froms = set(sql._from_objects(*froms))
+        if asfrom:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=
+                                correlate_froms.difference(asfrom_froms),
+                            implicit_correlate_froms=())
+        else:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=correlate_froms,
+                            implicit_correlate_froms=asfrom_froms)
 
+        new_correlate_froms = set(sql._from_objects(*froms))
+        all_correlate_froms = new_correlate_froms.union(correlate_froms)
 
-        self.stack.append({'from': correlate_froms,
-                            'iswrapper': iswrapper})
+        new_entry = {
+                    'asfrom_froms': new_correlate_froms,
+                    'iswrapper': iswrapper,
+                    'correlate_froms': all_correlate_froms
+                }
+        self.stack.append(new_entry)
 
         column_clause_args = kwargs.copy()
         column_clause_args.update({
@@ -1333,7 +1384,7 @@ class SQLCompiler(engine.Compiled):
             text += self.for_update_clause(select)
 
         if self.ctes and \
-            compound_index == 0 and not entry:
+            compound_index == 0 and toplevel:
             text = self._render_cte_clause() + text
 
         self.stack.pop(-1)
@@ -1546,7 +1597,10 @@ class SQLCompiler(engine.Compiled):
                     for t in extra_froms)
 
     def visit_update(self, update_stmt, **kw):
-        self.stack.append({'from': set([update_stmt.table])})
+        self.stack.append(
+                        {'correlate_froms': set([update_stmt.table]),
+                        "iswrapper": False,
+                        "asfrom_froms": set([update_stmt.table])})
 
         self.isupdate = True
 
@@ -1880,7 +1934,9 @@ class SQLCompiler(engine.Compiled):
         return values
 
     def visit_delete(self, delete_stmt, **kw):
-        self.stack.append({'from': set([delete_stmt.table])})
+        self.stack.append({'correlate_froms': set([delete_stmt.table]),
+                            "iswrapper": False,
+                            "asfrom_froms": set([delete_stmt.table])})
         self.isdelete = True
 
         text = "DELETE "
diff --git a/lib/sqlalchemy/sql/expression.py b/lib/sqlalchemy/sql/expression.py
index 46a08379b..37c0ac65c 100644
--- a/lib/sqlalchemy/sql/expression.py
+++ b/lib/sqlalchemy/sql/expression.py
@@ -1468,6 +1468,10 @@ def _cloned_intersection(a, b):
     return set(elem for elem in a
                if all_overlap.intersection(elem._cloned_set))
 
+def _cloned_difference(a, b):
+    all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b))
+    return set(elem for elem in a
+                if not all_overlap.intersection(elem._cloned_set))
 
 def _from_objects(*elements):
     return itertools.chain(*[element._from_objects for element in elements])
@@ -5262,7 +5266,7 @@ class Select(HasPrefixes, SelectBase):
     _distinct = False
     _from_cloned = None
     _correlate = ()
-    _correlate_except = ()
+    _correlate_except = None
     _memoized_property = SelectBase._memoized_property
 
     def __init__(self,
@@ -5357,7 +5361,8 @@ class Select(HasPrefixes, SelectBase):
 
         return froms
 
-    def _get_display_froms(self, existing_froms=None, asfrom=False):
+    def _get_display_froms(self, explicit_correlate_froms=None,
+                                    implicit_correlate_froms=None):
         """Return the full list of 'from' clauses to be displayed.
 
         Takes into account a set of existing froms which may be
@@ -5368,7 +5373,9 @@ class Select(HasPrefixes, SelectBase):
         """
         froms = self._froms
 
-        toremove = set(itertools.chain(*[f._hide_froms for f in froms]))
+        toremove = set(itertools.chain(*[
+                            _expand_cloned(f._hide_froms)
+                            for f in froms]))
         if toremove:
             # if we're maintaining clones of froms,
             # add the copies out to the toremove list.  only include
@@ -5383,36 +5390,42 @@ class Select(HasPrefixes, SelectBase):
             # using a list to maintain ordering
             froms = [f for f in froms if f not in toremove]
 
-        if not asfrom:
-            if self._correlate:
+        if self._correlate:
+            to_correlate = self._correlate
+            if to_correlate:
                 froms = [
                     f for f in froms if f not in
                     _cloned_intersection(
-                        _cloned_intersection(froms, existing_froms or ()),
-                        self._correlate
-                    )
-                ]
-            if self._correlate_except:
-                froms = [
-                    f for f in froms if f in
-                    _cloned_intersection(
-                        froms,
-                        self._correlate_except
+                        _cloned_intersection(froms, explicit_correlate_froms or ()),
+                        to_correlate
                     )
                 ]
 
-            if self._auto_correlate and existing_froms and len(froms) > 1:
-                froms = [
-                    f for f in froms if f not in
-                    _cloned_intersection(froms, existing_froms)
-                ]
+        if self._correlate_except is not None:
+
+            froms = [
+                f for f in froms if f not in
+                _cloned_difference(
+                    _cloned_intersection(froms, explicit_correlate_froms or ()),
+                    self._correlate_except
+                )
+            ]
 
-                if not len(froms):
-                    raise exc.InvalidRequestError("Select statement '%s"
-                            "' returned no FROM clauses due to "
-                            "auto-correlation; specify "
-                            "correlate(<tables>) to control "
-                            "correlation manually." % self)
+        if self._auto_correlate and \
+            implicit_correlate_froms and \
+            len(froms) > 1:
+
+            froms = [
+                f for f in froms if f not in
+                _cloned_intersection(froms, implicit_correlate_froms)
+            ]
+
+            if not len(froms):
+                raise exc.InvalidRequestError("Select statement '%s"
+                        "' returned no FROM clauses due to "
+                        "auto-correlation; specify "
+                        "correlate(<tables>) to control "
+                        "correlation manually." % self)
 
         return froms
 
@@ -5757,19 +5770,52 @@ class Select(HasPrefixes, SelectBase):
 
     @_generative
     def correlate(self, *fromclauses):
-        """return a new select() construct which will correlate the given FROM
-        clauses to that of an enclosing select(), if a match is found.
-
-        By "match", the given fromclause must be present in this select's
-        list of FROM objects and also present in an enclosing select's list of
-        FROM objects.
-
-        Calling this method turns off the select's default behavior of
-        "auto-correlation". Normally, select() auto-correlates all of its FROM
-        clauses to those of an embedded select when compiled.
-
-        If the fromclause is None, correlation is disabled for the returned
-        select().
+        """return a new :class:`.Select` which will correlate the given FROM
+        clauses to that of an enclosing :class:`.Select`.
+
+        Calling this method turns off the :class:`.Select` object's
+        default behavior of "auto-correlation".  Normally, FROM elements
+        which appear in a :class:`.Select` that encloses this one via
+        its :term:`WHERE clause`, ORDER BY, HAVING or
+        :term:`columns clause` will be omitted from this :class:`.Select`
+        object's :term:`FROM clause`.
+        Setting an explicit correlation collection using the
+        :meth:`.Select.correlate` method provides a fixed list of FROM objects
+        that can potentially take place in this process.
+
+        When :meth:`.Select.correlate` is used to apply specific FROM clauses
+        for correlation, the FROM elements become candidates for
+        correlation regardless of how deeply nested this :class:`.Select`
+        object is, relative to an enclosing :class:`.Select` which refers to
+        the same FROM object.  This is in contrast to the behavior of
+        "auto-correlation" which only correlates to an immediate enclosing
+        :class:`.Select`.   Multi-level correlation ensures that the link
+        between enclosed and enclosing :class:`.Select` is always via
+        at least one WHERE/ORDER BY/HAVING/columns clause in order for
+        correlation to take place.
+
+        If ``None`` is passed, the :class:`.Select` object will correlate
+        none of its FROM entries, and all will render unconditionally
+        in the local FROM clause.
+
+        :param \*fromclauses: a list of one or more :class:`.FromClause`
+         constructs, or other compatible constructs (i.e. ORM-mapped
+         classes) to become part of the correlate collection.
+
+         .. versionchanged:: 0.8.0 ORM-mapped classes are accepted by
+            :meth:`.Select.correlate`.
+
+        .. versionchanged:: 0.8.0 The :meth:`.Select.correlate` method no
+           longer unconditionally removes entries from the FROM clause; instead,
+           the candidate FROM entries must also be matched by a FROM entry
+           located in an enclosing :class:`.Select`, which ultimately encloses
+           this one as present in the WHERE clause, ORDER BY clause, HAVING
+           clause, or columns clause of an enclosing :meth:`.Select`.
+
+        .. versionchanged:: 0.8.2 explicit correlation takes place
+           via any level of nesting of :class:`.Select` objects; in previous
+           0.8 versions, correlation would only occur relative to the immediate
+           enclosing :class:`.Select` construct.
 
         .. seealso::
 
@@ -5787,9 +5833,30 @@ class Select(HasPrefixes, SelectBase):
 
     @_generative
     def correlate_except(self, *fromclauses):
-        """"Return a new select() construct which will auto-correlate
-        on FROM clauses of enclosing selectables, except for those FROM
-        clauses specified here.
+        """return a new :class:`.Select` which will omit the given FROM
+        clauses from the auto-correlation process.
+
+        Calling :meth:`.Select.correlate_except` turns off the
+        :class:`.Select` object's default behavior of
+        "auto-correlation" for the given FROM elements.  An element
+        specified here will unconditionally appear in the FROM list, while
+        all other FROM elements remain subject to normal auto-correlation
+        behaviors.
+
+        .. versionchanged:: 0.8.2 The :meth:`.Select.correlate_except`
+           method was improved to fully prevent FROM clauses specified here
+           from being omitted from the immediate FROM clause of this
+           :class:`.Select`.
+
+        If ``None`` is passed, the :class:`.Select` object will correlate
+        all of its FROM entries.
+
+        .. versionchanged:: 0.8.2 calling ``correlate_except(None)`` will
+           correctly auto-correlate all FROM clauses.
+
+        :param \*fromclauses: a list of one or more :class:`.FromClause`
+         constructs, or other compatible constructs (i.e. ORM-mapped
+         classes) to become part of the correlate-exception collection.
 
         .. seealso::
 
@@ -5798,11 +5865,12 @@ class Select(HasPrefixes, SelectBase):
             :ref:`correlated_subqueries`
 
         """
+
         self._auto_correlate = False
         if fromclauses and fromclauses[0] is None:
             self._correlate_except = ()
         else:
-            self._correlate_except = set(self._correlate_except).union(
+            self._correlate_except = set(self._correlate_except or ()).union(
                     _interpret_as_from(f) for f in fromclauses)
 
     def append_correlation(self, fromclause):