- rework all of autogenerate to build directly on alembic.operations.ops

objects; the "diffs" is now a legacy system that is exported from
the ops.  A new model of comparison/rendering/ upgrade/downgrade
composition that is cleaner and much more extensible is introduced.
- autogenerate is now extensible as far as database objects compared
and rendered into scripts; any new operation directive can also be
registered into a series of hooks that allow custom database/model
comparison functions to run as well as to render new operation
directives into autogenerate scripts.
- write all new docs for the new system
fixes #306

1 files changed, 103 insertions, 100 deletions

diff --git a/tests/test_autogen_diffs.py b/tests/test_autogen_diffs.py
index f32fd84..d176b91 100644
--- a/tests/test_autogen_diffs.py
+++ b/tests/test_autogen_diffs.py
@@ -6,6 +6,7 @@ from sqlalchemy import MetaData, Column, Table, Integer, String, Text, \
 from sqlalchemy.types import NULLTYPE
 from sqlalchemy.engine.reflection import Inspector
 
+from alembic.operations import ops
 from alembic import autogenerate
 from alembic.migration import MigrationContext
 from alembic.testing import TestBase
@@ -14,8 +15,7 @@ from alembic.testing import assert_raises_message
 from alembic.testing.mock import Mock
 from alembic.testing import eq_
 from alembic.util import CommandError
-from ._autogen_fixtures import \
-    AutogenTest, AutogenFixtureTest, _default_object_filters
+from ._autogen_fixtures import AutogenTest, AutogenFixtureTest
 
 py3k = sys.version_info >= (3, )
 
@@ -63,25 +63,24 @@ class AutogenCrossSchemaTest(AutogenTest, TestBase):
         return m
 
     def test_default_schema_omitted_upgrade(self):
-        diffs = []
 
         def include_object(obj, name, type_, reflected, compare_to):
             if type_ == "table":
                 return name == "t3"
             else:
                 return True
-        self.autogen_context.update({
-            'object_filters': [include_object],
-            'include_schemas': True,
-            'metadata': self.m2
-        })
-        autogenerate._produce_net_changes(self.autogen_context, diffs)
+        self._update_context(
+            object_filters=include_object,
+            include_schemas=True,
+        )
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(self.autogen_context, uo)
 
+        diffs = uo.as_diffs()
         eq_(diffs[0][0], "add_table")
         eq_(diffs[0][1].schema, None)
 
     def test_alt_schema_included_upgrade(self):
-        diffs = []
 
         def include_object(obj, name, type_, reflected, compare_to):
             if type_ == "table":
@@ -89,48 +88,48 @@ class AutogenCrossSchemaTest(AutogenTest, TestBase):
             else:
                 return True
 
-        self.autogen_context.update({
-            'object_filters': [include_object],
-            'include_schemas': True,
-            'metadata': self.m2
-        })
-        autogenerate._produce_net_changes(self.autogen_context, diffs)
+        self._update_context(
+            object_filters=include_object,
+            include_schemas=True,
+        )
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(self.autogen_context, uo)
 
+        diffs = uo.as_diffs()
         eq_(diffs[0][0], "add_table")
         eq_(diffs[0][1].schema, config.test_schema)
 
     def test_default_schema_omitted_downgrade(self):
-        diffs = []
-
         def include_object(obj, name, type_, reflected, compare_to):
             if type_ == "table":
                 return name == "t1"
             else:
                 return True
-        self.autogen_context.update({
-            'object_filters': [include_object],
-            'include_schemas': True,
-            'metadata': self.m2
-        })
-        autogenerate._produce_net_changes(self.autogen_context, diffs)
+        self._update_context(
+            object_filters=include_object,
+            include_schemas=True,
+        )
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(self.autogen_context, uo)
 
+        diffs = uo.as_diffs()
         eq_(diffs[0][0], "remove_table")
         eq_(diffs[0][1].schema, None)
 
     def test_alt_schema_included_downgrade(self):
-        diffs = []
 
         def include_object(obj, name, type_, reflected, compare_to):
             if type_ == "table":
                 return name == "t2"
             else:
                 return True
-        self.autogen_context.update({
-            'object_filters': [include_object],
-            'include_schemas': True,
-            'metadata': self.m2
-        })
-        autogenerate._produce_net_changes(self.autogen_context, diffs)
+        self._update_context(
+            object_filters=include_object,
+            include_schemas=True,
+        )
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(self.autogen_context, uo)
+        diffs = uo.as_diffs()
         eq_(diffs[0][0], "remove_table")
         eq_(diffs[0][1].schema, config.test_schema)
 
@@ -268,14 +267,14 @@ class AutogenerateDiffTest(ModelOne, AutogenTest, TestBase):
         """test generation of diff rules"""
 
         metadata = self.m2
-        diffs = []
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
-        ctx['object_filters'] = _default_object_filters
+        uo = ops.UpgradeOps(ops=[])
+        ctx = self.autogen_context
+
         autogenerate._produce_net_changes(
-            ctx, diffs
+            ctx, uo
         )
 
+        diffs = uo.as_diffs()
         eq_(
             diffs[0],
             ('add_table', metadata.tables['item'])
@@ -396,21 +395,25 @@ class AutogenerateDiffTest(ModelOne, AutogenTest, TestBase):
         eq_(alter_cols, set(['user_id', 'order', 'user']))
 
     def test_skip_null_type_comparison_reflected(self):
-        diff = []
-        autogenerate.compare._compare_type(None, "sometable", "somecol",
-                                           Column("somecol", NULLTYPE),
-                                           Column("somecol", Integer()),
-                                           diff, self.autogen_context
-                                           )
+        ac = ops.AlterColumnOp("sometable", "somecol")
+        autogenerate.compare._compare_type(
+            self.autogen_context, ac,
+            None, "sometable", "somecol",
+            Column("somecol", NULLTYPE),
+            Column("somecol", Integer()),
+        )
+        diff = ac.to_diff_tuple()
         assert not diff
 
     def test_skip_null_type_comparison_local(self):
-        diff = []
-        autogenerate.compare._compare_type(None, "sometable", "somecol",
-                                           Column("somecol", Integer()),
-                                           Column("somecol", NULLTYPE),
-                                           diff, self.autogen_context
-                                           )
+        ac = ops.AlterColumnOp("sometable", "somecol")
+        autogenerate.compare._compare_type(
+            self.autogen_context, ac,
+            None, "sometable", "somecol",
+            Column("somecol", Integer()),
+            Column("somecol", NULLTYPE),
+        )
+        diff = ac.to_diff_tuple()
         assert not diff
 
     def test_custom_type_compare(self):
@@ -420,20 +423,24 @@ class AutogenerateDiffTest(ModelOne, AutogenTest, TestBase):
             def compare_against_backend(self, dialect, conn_type):
                 return isinstance(conn_type, Integer)
 
-        diff = []
-        autogenerate.compare._compare_type(None, "sometable", "somecol",
-                                           Column("somecol", INTEGER()),
-                                           Column("somecol", MyType()),
-                                           diff, self.autogen_context
-                                           )
-        assert not diff
+        ac = ops.AlterColumnOp("sometable", "somecol")
+        autogenerate.compare._compare_type(
+            self.autogen_context, ac,
+            None, "sometable", "somecol",
+            Column("somecol", INTEGER()),
+            Column("somecol", MyType()),
+        )
+
+        assert not ac.has_changes()
 
-        diff = []
-        autogenerate.compare._compare_type(None, "sometable", "somecol",
-                                           Column("somecol", String()),
-                                           Column("somecol", MyType()),
-                                           diff, self.autogen_context
-                                           )
+        ac = ops.AlterColumnOp("sometable", "somecol")
+        autogenerate.compare._compare_type(
+            self.autogen_context, ac,
+            None, "sometable", "somecol",
+            Column("somecol", String()),
+            Column("somecol", MyType()),
+        )
+        diff = ac.to_diff_tuple()
         eq_(
             diff[0][0:4],
             ('modify_type', None, 'sometable', 'somecol')
@@ -449,26 +456,26 @@ class AutogenerateDiffTest(ModelOne, AutogenTest, TestBase):
                 else:
                     return dialect.type_descriptor(CHAR(32))
 
-        diff = []
+        uo = ops.AlterColumnOp('sometable', 'somecol')
         autogenerate.compare._compare_type(
+            self.autogen_context, uo,
             None, "sometable", "somecol",
             Column("somecol", Integer, nullable=True),
-            Column("somecol", MyType()),
-            diff, self.autogen_context
+            Column("somecol", MyType())
         )
-        assert not diff
+        assert not uo.has_changes()
 
     def test_dont_barf_on_already_reflected(self):
-        diffs = []
         from sqlalchemy.util import OrderedSet
         inspector = Inspector.from_engine(self.bind)
+        uo = ops.UpgradeOps(ops=[])
         autogenerate.compare._compare_tables(
             OrderedSet([(None, 'extra'), (None, 'user')]),
-            OrderedSet(), [], inspector,
-            MetaData(), diffs, self.autogen_context
+            OrderedSet(), inspector,
+            MetaData(), uo, self.autogen_context
         )
         eq_(
-            [(rec[0], rec[1].name) for rec in diffs],
+            [(rec[0], rec[1].name) for rec in uo.as_diffs()],
             [('remove_table', 'extra'), ('remove_table', 'user')]
         )
 
@@ -481,14 +488,14 @@ class AutogenerateDiffTestWSchema(ModelOne, AutogenTest, TestBase):
         """test generation of diff rules"""
 
         metadata = self.m2
-        diffs = []
 
-        self.autogen_context.update({
-            'object_filters': _default_object_filters,
-            'include_schemas': True,
-            'metadata': self.m2
-        })
-        autogenerate._produce_net_changes(self.autogen_context, diffs)
+        self._update_context(
+            include_schemas=True,
+        )
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(self.autogen_context, uo)
+
+        diffs = uo.as_diffs()
 
         eq_(
             diffs[0],
@@ -567,10 +574,10 @@ class AutogenerateCustomCompareTypeTest(AutogenTest, TestBase):
         my_compare_type = Mock()
         self.context._user_compare_type = my_compare_type
 
-        diffs = []
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
-        autogenerate._produce_net_changes(ctx, diffs)
+        uo = ops.UpgradeOps(ops=[])
+
+        ctx = self.autogen_context
+        autogenerate._produce_net_changes(ctx, uo)
 
         first_table = self.m2.tables['sometable']
         first_column = first_table.columns['id']
@@ -593,8 +600,7 @@ class AutogenerateCustomCompareTypeTest(AutogenTest, TestBase):
         self.context._user_compare_type = my_compare_type
 
         diffs = []
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
+        ctx = self.autogen_context
         diffs = []
         autogenerate._produce_net_changes(ctx, diffs)
 
@@ -605,10 +611,10 @@ class AutogenerateCustomCompareTypeTest(AutogenTest, TestBase):
         my_compare_type.return_value = True
         self.context._user_compare_type = my_compare_type
 
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
-        diffs = []
-        autogenerate._produce_net_changes(ctx, diffs)
+        ctx = self.autogen_context
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(ctx, uo)
+        diffs = uo.as_diffs()
 
         eq_(diffs[0][0][0], 'modify_type')
         eq_(diffs[1][0][0], 'modify_type')
@@ -636,8 +642,7 @@ class PKConstraintUpgradesIgnoresNullableTest(AutogenTest, TestBase):
 
     def test_no_change(self):
         diffs = []
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
+        ctx = self.autogen_context
         autogenerate._produce_net_changes(ctx, diffs)
         eq_(diffs, [])
 
@@ -674,11 +679,11 @@ class AutogenKeyTest(AutogenTest, TestBase):
 
     def test_autogen(self):
 
-        diffs = []
+        uo = ops.UpgradeOps(ops=[])
 
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
-        autogenerate._produce_net_changes(ctx, diffs)
+        ctx = self.autogen_context
+        autogenerate._produce_net_changes(ctx, uo)
+        diffs = uo.as_diffs()
         eq_(diffs[0][0], "add_table")
         eq_(diffs[0][1].name, "sometable")
         eq_(diffs[1][0], "add_column")
@@ -705,8 +710,7 @@ class AutogenVersionTableTest(AutogenTest, TestBase):
 
     def test_no_version_table(self):
         diffs = []
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
+        ctx = self.autogen_context
 
         autogenerate._produce_net_changes(ctx, diffs)
         eq_(diffs, [])
@@ -717,8 +721,7 @@ class AutogenVersionTableTest(AutogenTest, TestBase):
             self.version_table_name,
             self.m2, Column('x', Integer), schema=self.version_table_schema)
 
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
+        ctx = self.autogen_context
         autogenerate._produce_net_changes(ctx, diffs)
         eq_(diffs, [])
 
@@ -769,10 +772,10 @@ class AutogenerateDiffOrderTest(AutogenTest, TestBase):
         before their parent tables
         """
 
-        ctx = self.autogen_context.copy()
-        ctx['metadata'] = self.m2
-        diffs = []
-        autogenerate._produce_net_changes(ctx, diffs)
+        ctx = self.autogen_context
+        uo = ops.UpgradeOps(ops=[])
+        autogenerate._produce_net_changes(ctx, uo)
+        diffs = uo.as_diffs()
 
         eq_(diffs[0][0], 'add_table')
         eq_(diffs[0][1].name, "parent")