- rework of correlation, continuing on #2668, #2746

- add support for correlations to propagate all the way in; because
correlations require context now, need to make sure a select enclosure
of any level takes effect any number of levels deep.
- fix what we said correlate_except() was supposed to do when we first
released #2668 - "the FROM clause is left intact if the correlated SELECT
is not used in the context of an enclosing SELECT..." - it was not
considering the "existing_froms" collection at all, and prohibited
additional FROMs from being placed in an any() or has().
- add test for multilevel any()
- lots of docs, including glossary entries as we really need to define
"WHERE clause", "columns clause" etc. so that we can explain correlation better
- based on the insight that a SELECT can correlate anything that ultimately
came from an enclosing SELECT that links to this one via WHERE/columns/HAVING/ORDER BY,
have the compiler keep track of the FROM lists that correspond in this way,
link it to the asfrom flag, so that we send to _get_display_froms() the exact
list of candidate FROMs to correlate.  no longer need any asfrom logic in the
Select() itself
- preserve 0.8.1's behavior for correlation when no correlate options are given, not
to mention 0.7 and prior's behavior of not propagating implicit correlation more than one level..
this is to reduce surprises/hard-to-debug situations when a user isn't trying
to correlate anything.

Conflicts:
	doc/build/changelog/changelog_08.rst
	doc/build/changelog/changelog_09.rst
	lib/sqlalchemy/sql/compiler.py

1 files changed, 78 insertions, 22 deletions

diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index b902f9ffc..3de59ca90 100644
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -587,9 +587,15 @@ class SQLCompiler(engine.Compiled):
 
     def visit_compound_select(self, cs, asfrom=False,
                             parens=True, compound_index=0, **kwargs):
-        entry = self.stack and self.stack[-1] or {}
-        self.stack.append({'from': entry.get('from', None),
-                    'iswrapper': not entry})
+        toplevel = not self.stack
+        entry = self._default_stack_entry if toplevel else self.stack[-1]
+
+        self.stack.append(
+                    {
+                        'correlate_froms': entry['correlate_froms'],
+                        'iswrapper': toplevel,
+                        'asfrom_froms': entry['asfrom_froms']
+                    })
 
         keyword = self.compound_keywords.get(cs.keyword)
 
@@ -610,7 +616,7 @@ class SQLCompiler(engine.Compiled):
                         self.limit_clause(cs) or ""
 
         if self.ctes and \
-            compound_index == 0 and not entry:
+            compound_index == 0 and toplevel:
             text = self._render_cte_clause() + text
 
         self.stack.pop(-1)
@@ -1079,33 +1085,78 @@ class SQLCompiler(engine.Compiled):
     def get_crud_hint_text(self, table, text):
         return None
 
+
+
+    _default_stack_entry = util.immutabledict([
+                                        ('iswrapper', False),
+                                        ('correlate_froms', frozenset()),
+                                        ('asfrom_froms', frozenset())
+                                    ])
+
+    def _display_froms_for_select(self, select, asfrom):
+        # utility method to help external dialects
+        # get the correct from list for a select.
+        # specifically the oracle dialect needs this feature
+        # right now.
+        toplevel = not self.stack
+        entry = self._default_stack_entry if toplevel else self.stack[-1]
+
+        correlate_froms = entry['correlate_froms']
+        asfrom_froms = entry['asfrom_froms']
+
+        if asfrom:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=\
+                                correlate_froms.difference(asfrom_froms),
+                            implicit_correlate_froms=())
+        else:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=correlate_froms,
+                            implicit_correlate_froms=asfrom_froms)
+        return froms
+
     def visit_select(self, select, asfrom=False, parens=True,
                             iswrapper=False, fromhints=None,
                             compound_index=0,
                             force_result_map=False,
-                            positional_names=None, **kwargs):
-        entry = self.stack and self.stack[-1] or {}
+                            positional_names=None,
+                            **kwargs):
 
-        existingfroms = entry.get('from', None)
+        toplevel = not self.stack
+        entry = self._default_stack_entry if toplevel else self.stack[-1]
 
-        froms = select._get_display_froms(existingfroms, asfrom=asfrom)
-
-        correlate_froms = set(sql._from_objects(*froms))
-
-        # TODO: might want to propagate existing froms for
-        # select(select(select)) where innermost select should correlate
-        # to outermost if existingfroms: correlate_froms =
-        # correlate_froms.union(existingfroms)
 
         populate_result_map = force_result_map or (
                                         compound_index == 0 and (
-                                            not entry or \
-                                            entry.get('iswrapper', False)
+                                            toplevel or \
+                                            entry['iswrapper']
                                         )
                                     )
 
-        self.stack.append({'from': correlate_froms,
-                            'iswrapper': iswrapper})
+
+        correlate_froms = entry['correlate_froms']
+        asfrom_froms = entry['asfrom_froms']
+
+        if asfrom:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=
+                                correlate_froms.difference(asfrom_froms),
+                            implicit_correlate_froms=())
+        else:
+            froms = select._get_display_froms(
+                            explicit_correlate_froms=correlate_froms,
+                            implicit_correlate_froms=asfrom_froms)
+
+
+        new_correlate_froms = set(sql._from_objects(*froms))
+        all_correlate_froms = new_correlate_froms.union(correlate_froms)
+
+        new_entry = {
+                    'asfrom_froms': new_correlate_froms,
+                    'iswrapper': iswrapper,
+                    'correlate_froms': all_correlate_froms
+                }
+        self.stack.append(new_entry)
 
         column_clause_args = kwargs.copy()
         column_clause_args.update({
@@ -1188,7 +1239,7 @@ class SQLCompiler(engine.Compiled):
             text += self.for_update_clause(select)
 
         if self.ctes and \
-            compound_index == 0 and not entry:
+            compound_index == 0 and toplevel:
             text = self._render_cte_clause() + text
 
         self.stack.pop(-1)
@@ -1401,7 +1452,10 @@ class SQLCompiler(engine.Compiled):
                     for t in extra_froms)
 
     def visit_update(self, update_stmt, **kw):
-        self.stack.append({'from': set([update_stmt.table])})
+        self.stack.append(
+                        {'correlate_froms': set([update_stmt.table]),
+                        "iswrapper": False,
+                        "asfrom_froms": set([update_stmt.table])})
 
         self.isupdate = True
 
@@ -1735,7 +1789,9 @@ class SQLCompiler(engine.Compiled):
         return values
 
     def visit_delete(self, delete_stmt, **kw):
-        self.stack.append({'from': set([delete_stmt.table])})
+        self.stack.append({'correlate_froms': set([delete_stmt.table]),
+                            "iswrapper": False,
+                            "asfrom_froms": set([delete_stmt.table])})
         self.isdelete = True
 
         text = "DELETE "