diff options
| -rw-r--r-- | CHANGELOG.rst | 5 | ||||
| -rw-r--r-- | doc/Makefile | 1 | ||||
| -rw-r--r-- | doc/conf.py | 1 | ||||
| -rw-r--r-- | doc/index.rst | 234 | ||||
| -rwxr-xr-x | tox.ini | 2 | ||||
| -rwxr-xr-x | yoyo/__init__.py | 2 | ||||
| -rw-r--r-- | yoyo/config.py | 14 | ||||
| -rw-r--r-- | yoyo/scripts/init.py | 55 | ||||
| -rwxr-xr-x | yoyo/scripts/main.py | 28 | ||||
| -rwxr-xr-x | yoyo/scripts/migrate.py | 24 | ||||
| -rw-r--r-- | yoyo/scripts/newmigration.py | 5 | ||||
| -rw-r--r-- | yoyo/tests/test_cli_script.py | 23 |
12 files changed, 322 insertions, 72 deletions
diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1b3c9b9..67d3983 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +8.1.0 (released 2022-11-03) +--------------------------- + +* Add a new ``yoyo init`` command + 8.0.0 (released 2022-10-05) --------------------------- diff --git a/doc/Makefile b/doc/Makefile index 2070e97..c97ae1c 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,5 +1,6 @@ # Minimal makefile for Sphinx documentation # +export PATH:=$(PATH):../.tox/py310-sphinx/bin/ # You can set these variables from the command line. SPHINXOPTS = diff --git a/doc/conf.py b/doc/conf.py index 2819070..ef9e4d8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -48,6 +48,7 @@ extensions = [ "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinxcontrib.programoutput", ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/index.rst b/doc/index.rst index f3874a5..5937bc1 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -19,88 +19,184 @@ They can be as simple as this: ] -Installation -================== +Installation and project setup +============================== -Install yoyo-migrations using from the PyPI, for example: +Install yoyo-migrations from PyPI: .. code:: shell pip install yoyo-migrations +Initialize yoyo for your project, supplying a database connection string and migrations directory name, for example: -Command line usage -================== +.. code:: shell + + yoyo init --database sqlite:///mydb.sqlite3 migrations -Start a new migration: +This will create a new, empty directory called ``migrations`` and install a +``yoyo.ini`` configuration file in the current directory. The configuration file +will contain any database credentials supplied on the command line. If you do +not wish this to happen, then omit the ``--database`` argument from the +command. + +Create a new migration by running ``yoyo new``. By default, a Python format file is generated, use ``--sql`` if you prefer SQL format: .. code:: shell - yoyo new ./migrations -m "Add column to foo" + yoyo new --sql -Apply migrations from directory ``migrations`` to a PostgreSQL database: +An editor will open with a template migration file. +Add a comment explaining what the migration does followed by the SQL commands, +for example: -.. code:: shell +.. code:: sql - yoyo apply --database postgresql://scott:tiger@localhost/db ./migrations + -- Create table foo + -- depends: -Rollback migrations previously applied to a MySQL database: + CREATE TABLE foo ( + a int + ); + + +Save and exit, and the new migration file will be created. +Check your migration has been created with ``yoyo list`` and apply it with +``yoyo apply``: .. code:: shell - yoyo rollback --database mysql://scott:tiger@localhost/database ./migrations + $ yoyo list + $ yoyo apply -Reapply (ie rollback then apply again) migrations to a SQLite database at -location ``/home/sheila/important.db``: -.. code:: shell - yoyo reapply --database sqlite:////home/sheila/important.db ./migrations +Command line usage +================== -List available migrations: +You can see the list of available commands by running: + +.. command-output:: yoyo --help -.. code:: shell - yoyo list --database sqlite:////home/sheila/important.db ./migrations +You can check options for any command with ``yoyo <command> --help`` +yoyo new +-------- -During development, the ``yoyo develop`` command can be used to apply any -unapplied migrations without further prompting: +Start a new migration. ``yoyo new`` will create a new migration file and opens it your configured editor. + +By default a Python formation migration will be created. To use the simpler SQL format, specify ``--sql``. .. code:: shell - $ yoyo develop --database postgresql://localhost/mydb migrations - Applying 3 migrations: - [00000000_initial-schema] - [00000001_add-table-foo] - [00000002_add-table-bar] + yoyo new -m "Add column to foo" + yoyo new --sql -If there are no migrations waiting to be applied the ``develop`` command will -instead roll back and reapply the last migration: +yoyo list +---------- -.. code:: shell +List available migrations. Each migration will be prefixed with one of ``U`` +(unapplied) or ``A`` (applied). + +yoyo apply +---------- + +Apply migrations to the target database. By default this will prompt you for each unapplied migration. To turn off prompting use ``--batch`` or specify ``batch_mode = on`` in ``yoyo.ini``. + + +yoyo rollback +------------- + +By default this will prompt you for each applied migration, starting with the most recently applied. + +If you wish to rollback a single migration, specify the migration with the ``-r``/``--revision`` flag. Note that this will also cause any migrations that depend on the selected migration to be rolled back. + + +yoyo reapply +------------- + +Reapply (ie rollback then apply again) migrations. As with `yoyo rollback`_, you can select a target migration with ``-r``/``--revision`` + + +yoyo develop +------------ + +Apply any unapplied migrations without prompting. + +If there are no unapplied migrations, rollback and reapply the most recent +migration. Use ``yoyo develop -n <n>`` to act on just the *n* most recently +applied migrations. + +yoyo mark +--------- + +Mark one or more migrations as applied, without actually applying them. + +yoyo unmark +----------- + +Unmark one or more migrations as unapplied, without actually rolling them back. - $ yoyo develop --database postgresql://localhost/dev ./migrations - Reapplying 1 migration: - [00000002_add-table-bar] Connecting to a database ------------------------- +======================== -Database connections are specified using a URL. Examples: +Database connections are specified using a URL, for example: -.. code:: ini +.. code:: shell + + yoyo list --database postgresql://scott:tiger@localhost/mydatabase + +The protocol part of the URL (the part before ``://``) is used to specify the backend. +Yoyo provides the following core backends: + +- ``postgresql`` (psycopg2_) +- ``postgresql+psycopg`` (psycopg3_) +- ``mysql`` (pymysql_) +- ``mysql+mysqldb`` (mysqlclient_) +- ``sqlite`` (sqlite3_) + +And these backends have been contributed and are bundled with yoyo: + +- ``odbc`` (pyodbc_) +- ``oracle`` (`cx_Oracle`_) +- ``snowflake`` (snowflake_) +- ``redshift`` (psycopg2_) + +How other parts of the URL are interpreted depends on the underlying backend +and the DB-API driver used. The host part especially tends to be interpreted +differently by drivers. A few of the more important differences are listed below. + +MySQL connections +----------------- + +mysqlclient_ and pymysql_ have +different ways to interpret the ``host`` part of the connection URL: + +- With mysqlclient_ (``mysql+mysqldb://``), + setting the host to ``localhost`` or leaving it empty causes the + driver to attempt a local unix socket connection. +- In pymysql_ (``mysql://``), + the driver will attempt a tcp connection in both cases. + Specify a unix socket connection + with the ``unix_socket`` option (eg ``?unix_socket=/tmp/mysql.sock``) + +To enable SSL, specify ``?ssl=1`` and the following options as required: + +- ``sslca`` +- ``sslcapath`` +- ``sslcert`` +- ``sslkey`` +- ``sslcipher`` - # SQLite: use 4 slashes for an absolute database path on unix like platforms - database = sqlite:////home/user/mydb.sqlite +These options correspond to the ``ca``, ``capath``, ``cert``, ``key`` and ``cipher`` options used by `mysql_ssl_set <https://dev.mysql.com/doc/c-api/8.0/en/mysql-ssl-set.html>`_. - # SQLite: use 3 slashes for a relative path - database = sqlite:///mydb.sqlite +Example configurations: - # SQLite: absolute path on Windows. - database = sqlite:///c:\home\user\mydb.sqlite +.. code:: ini # MySQL: Network database connection database = mysql://scott:tiger@localhost/mydatabase @@ -114,18 +210,51 @@ Database connections are specified using a URL. Examples: # MySQL with SSL/TLS enabled database = mysql+mysqldb://scott:tiger@localhost/mydatabase?ssl=yes&sslca=/path/to/cert - # PostgreSQL: database connection +PostgreSQL connections +---------------------- + +The psycopg family of drivers will use a unix socket if the host is left empty +(or the value of ``PGHOST`` if this is set in your environment). Otherwise it will attempt a tcp connection to the specified host. + +To force a unix socket connection leave the host part of the URL +empty and provide a ``host`` option that points to the directory containing the socket +(eg ``postgresql:///mydb?host=/path/to/socket/``). + +The postgresql backends also allow a custom schema to be selected by specifying a ``schema`` option, eg ``postgresql://…/mydatabase?schema=myschema``. + +Example configurations: + +.. code:: ini + database = postgresql://scott:tiger@localhost/mydatabase - # PostgreSQL: unix socket connection + # unix socket connection database = postgresql://scott:tiger@/mydatabase - # PostgreSQL with the newer psycopg 3 driver + # unix socket at a non-standard location and port number + database = postgresql://scott:tiger@/mydatabase?host=/var/run/postgresql&port=5434 + + # PostgreSQL with psycopg 3 driver database = postgresql+psycopg://scott:tiger@localhost/mydatabase - # PostgreSQL: changing the schema (via set search_path) + # Changing the default schema database = postgresql://scott:tiger@/mydatabase?schema=some_schema +SQLite connections +------------------ + +The SQLite backend ignores everything in the connection URL except the database +name, which should be a filename, or the special value ``:memory:`` for an in-memory database. + +3 slashes are required to specify a relative path:: + + sqlite:///mydb.sqlite + +and 4 for an absolute path on unix-like platforms:: + + sqlite:////home/user/mydb.sqlite + + Password security ----------------- @@ -241,7 +370,7 @@ Migrations may declare dependencies on other migrations via the ] -If you use the ``yoyo new`` command the ``_depends__`` attribute will be auto +If you use the ``yoyo new`` command the ``__depends__`` attribute will be auto populated for you. @@ -451,7 +580,7 @@ rollbacks happen. For example: Disabling transactions ---------------------- -You can disable transaction handling within a migration by setting +Disable transaction handling within a migration by setting ``__transactional__ = False``, eg: .. code:: python @@ -536,8 +665,7 @@ in the package metadata (typically in ``setup.cfg``), for example: mybackend = mypackage:MyBackend -You can then use the backend by specifying ``'mybackend'`` as the driver -protocol:: +Use the backend by specifying ``'mybackend'`` as the driver protocol:: .. code:: sh @@ -611,3 +739,11 @@ Changelog ========= .. include:: ../CHANGELOG.rst +.. _mysqlclient: https://pypi.org/project/mysqlclient/ +.. _pymysql: https://pypi.org/project/pymysql/ +.. _psycopg2: https://pypi.org/project/psycopg2/ +.. _psycopg3: https://pypi.org/project/psycopg/ +.. _sqlite3: https://docs.python.org/3/library/sqlite3.html +.. _pyodbc: https://pypi.org/project/pyodbc/ +.. _cx_Oracle: https://pypi.org/project/cx-Oracle/ +.. _snowflake: https://pypi.org/project/snowflake-connector-python/ @@ -15,8 +15,8 @@ commands=pytest [] deps= hg+http://hg.sr.ht/~olly/fresco-sphinx-theme#egg=fresco_sphinx_theme sphinx + sphinxcontrib-programoutput {[testenv]deps} - commands= sphinx-build -M doctest "{toxinidir}/doc/" "{toxinidir}_build" diff --git a/yoyo/__init__.py b/yoyo/__init__.py index ef784dd..bf7c658 100755 --- a/yoyo/__init__.py +++ b/yoyo/__init__.py @@ -34,4 +34,4 @@ from yoyo.migrations import read_migrations from yoyo.migrations import step from yoyo.migrations import transaction -__version__ = "8.0.1.dev0" +__version__ = "8.1.1.dev0" diff --git a/yoyo/config.py b/yoyo/config.py index 9010cac..0569a12 100644 --- a/yoyo/config.py +++ b/yoyo/config.py @@ -84,8 +84,9 @@ def update_argparser_defaults(parser, defaults): def read_config(src: Optional[str]) -> ConfigParser: """ - Read the configuration file at ``path``, or return an empty - ConfigParse object if ``path`` is ``None``. + Read the configuration file at ``src`` and construct a ConfigParser instance. + + If ``src`` is None a new, empty ConfigParse object will be created. """ if src is None: return get_configparser(get_interpolation_defaults()) @@ -205,3 +206,12 @@ def find_config(): return path d = os.path.dirname(d) return None + + +def config_changed(config: ConfigParser, path: str) -> bool: + def to_dict(config: ConfigParser): + return {k: dict(section.items()) for k, section in config.items()} + + if Path(path).exists(): + return to_dict(read_config(path)) != to_dict(config) + return True diff --git a/yoyo/scripts/init.py b/yoyo/scripts/init.py new file mode 100644 index 0000000..96b04e9 --- /dev/null +++ b/yoyo/scripts/init.py @@ -0,0 +1,55 @@ +# Copyright 2022 Oliver Cope +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import logging +import sys + +from yoyo.config import CONFIG_FILENAME +from yoyo.scripts.main import save_config + +logger = logging.getLogger("yoyo.migrations") + + +def install_argparsers(global_parser, subparsers): + parser = subparsers.add_parser( + "init", parents=[global_parser], help="Initialize a new project" + ) + parser.set_defaults(func=init) + parser.add_argument("sources", nargs=1, help="Directory for migration scripts") + parser.add_argument( + "-d", + "--database", + help="Database, eg 'sqlite:///path/to/sqlite.db' " + "or 'postgresql://user@host/db'", + ) + + +def init(args, config) -> int: + + if args.config: + configpath = pathlib.Path(args.config) + else: + configpath = pathlib.Path.cwd() / CONFIG_FILENAME + migrations_path = pathlib.Path(args.sources[0]).resolve() + + if configpath.exists(): + print("Existing configuration file found, exiting", file=sys.stderr) + return 1 + save_config(config, configpath) + print(f"Saved configuration to {configpath}") + if not migrations_path.exists(): + migrations_path.mkdir(exist_ok=True, parents=True) + print(f"Created migrations directory {migrations_path}") + return 0 diff --git a/yoyo/scripts/main.py b/yoyo/scripts/main.py index e6bae2b..3e9f76d 100755 --- a/yoyo/scripts/main.py +++ b/yoyo/scripts/main.py @@ -18,6 +18,7 @@ import configparser import logging import os import sys +import typing as t from yoyo import connections from yoyo import default_migration_table @@ -27,6 +28,7 @@ from yoyo.config import CONFIG_FILENAME from yoyo.config import find_config from yoyo.config import read_config from yoyo.config import save_config +from yoyo.config import config_changed from yoyo.config import update_argparser_defaults verbosity_levels = { @@ -46,11 +48,13 @@ class InvalidArgument(Exception): pass -def parse_args(argv=None): +def parse_args( + argv=None, +) -> t.Tuple[configparser.ConfigParser, argparse.ArgumentParser, argparse.Namespace]: """ Parse the config file and command line args. - :return: tuple of (argparser, parsed_args) + :return: tuple of ``(parsed config file, argument parser, parsed arguments)`` """ #: List of arguments whose defaults should be read from the config file config_args = { @@ -67,9 +71,8 @@ def parse_args(argv=None): global_args, _ = globalparser.parse_known_args(argv) # Read the config file and create a dictionary of defaults for argparser - config = read_config( - (global_args.config or find_config()) if global_args.use_config_file else None - ) + configfile = global_args.config or find_config() + config = read_config(configfile if global_args.use_config_file else None) defaults = {} for argname, getter in config_args.items(): @@ -137,8 +140,11 @@ def make_argparser(): subparsers = argparser.add_subparsers(help="Commands help") - from . import migrate, newmigration + from . import migrate + from . import newmigration + from . import init + init.install_argparsers(global_parser, subparsers) migrate.install_argparsers(global_parser, subparsers) newmigration.install_argparsers(global_parser, subparsers) @@ -301,13 +307,17 @@ def main(argv=None): try: if vars(args).get("func"): - args.func(args, config) + exitcode = args.func(args, config) except InvalidArgument as e: argparser.error(e.args[0]) if config_is_empty and args.use_config_file and not args.batch_mode: - prompt_save_config(config, args.config or CONFIG_FILENAME) + config_file = args.config or CONFIG_FILENAME + if config_changed(config, config_file): + prompt_save_config(config, config_file) + + return exitcode if __name__ == "__main__": - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/yoyo/scripts/migrate.py b/yoyo/scripts/migrate.py index 15de7b0..060499a 100755 --- a/yoyo/scripts/migrate.py +++ b/yoyo/scripts/migrate.py @@ -256,30 +256,33 @@ def get_migrations(args, backend, direction=None): return migrations -def apply(args, config): +def apply(args, config) -> int: backend = get_backend(args, config) with backend.lock(): migrations = get_migrations(args, backend) backend.apply_migrations(migrations, args.force) + return 0 -def reapply(args, config): +def reapply(args, config) -> int: backend = get_backend(args, config) with backend.lock(): migrations = get_migrations(args, backend) backend.rollback_migrations(migrations, args.force) migrations = backend.to_apply(migrations) backend.apply_migrations(migrations, args.force) + return 0 -def rollback(args, config): +def rollback(args, config) -> int: backend = get_backend(args, config) with backend.lock(): migrations = get_migrations(args, backend) backend.rollback_migrations(migrations, args.force) + return 0 -def develop(args, config): +def develop(args, config) -> int: args.batch_mode = True backend = get_backend(args, config) with backend.lock(): @@ -303,28 +306,32 @@ def develop(args, config): backend.rollback_migrations(migrations, args.force) migrations = get_migrations(args, backend, "apply") backend.apply_migrations(migrations, args.force) + return 0 -def mark(args, config): +def mark(args, config) -> int: backend = get_backend(args, config) with backend.lock(): migrations = get_migrations(args, backend) backend.mark_migrations(migrations) + return 0 -def unmark(args, config): +def unmark(args, config) -> int: backend = get_backend(args, config) with backend.lock(): migrations = get_migrations(args, backend) backend.unmark_migrations(migrations) + return 0 -def break_lock(args, config): +def break_lock(args, config) -> int: backend = get_backend(args, config) backend.break_lock() + return 0 -def list_migrations(args, config): +def list_migrations(args, config) -> int: backend = get_backend(args, config) migrations = read_migrations(*args.sources) migrations = filter_migrations(migrations, args.match) @@ -343,6 +350,7 @@ def list_migrations(args, config): for m in migrations ) print(tabulate.tabulate(data, headers=("STATUS", "ID", "SOURCE"))) + return 0 def prompt_migrations(backend, migrations, direction): diff --git a/yoyo/scripts/newmigration.py b/yoyo/scripts/newmigration.py index 0629441..13b2986 100644 --- a/yoyo/scripts/newmigration.py +++ b/yoyo/scripts/newmigration.py @@ -89,7 +89,7 @@ def install_argparsers(global_parser, subparsers): ) -def new_migration(args, config): +def new_migration(args, config) -> int: try: directory = args.sources[0] @@ -115,7 +115,7 @@ def new_migration(args, config): else: p = create_with_editor(config, directory, migration_source, extension) if p is None: - return + return 1 try: command = config.get("DEFAULT", CONFIG_NEW_MIGRATION_COMMAND_KEY) @@ -126,6 +126,7 @@ def new_migration(args, config): pass print("Created file", p) + return 0 def slugify(message): diff --git a/yoyo/tests/test_cli_script.py b/yoyo/tests/test_cli_script.py index 8199ee1..1160667 100644 --- a/yoyo/tests/test_cli_script.py +++ b/yoyo/tests/test_cli_script.py @@ -17,6 +17,7 @@ from datetime import datetime from tempfile import mkdtemp from functools import partial from itertools import count +import pathlib import io import os import os.path @@ -596,3 +597,25 @@ class TestDevelopCommand(TestInteractiveScript): ("m1", "apply"), ("m2", "apply"), ] + + +class TestInitCommand(TestInteractiveScript): + def test_it_creates_project(self): + main(["-b", "init", "migrations", "--database", "sqlite://foo"]) + config = (pathlib.Path(self.tmpdir) / "yoyo.ini").read_text() + assert "database = sqlite://foo" in config + assert "sources = migrations" in config + assert (pathlib.Path(self.tmpdir) / "migrations").is_dir() + + def test_it_doesnt_overwrite(self): + configfile = pathlib.Path(self.tmpdir) / "yoyo.ini" + configfile.write_text("[existing file]") + main(["-b", "init", "migrations", "--database", "sqlite://foo"]) + assert "database = sqlite://foo" not in configfile.read_text() + assert "existing file" in configfile.read_text() + assert not (pathlib.Path(self.tmpdir) / "migrations").exists() + + def test_it_doesnt_prompt(self): + with patch("yoyo.scripts.main.prompt_save_config") as prompt_save_config: + main(["-b", "init", "migrations", "--database", "sqlite://foo"]) + assert prompt_save_config.called is False |