Remove submodule, just put Dependencies in ./libs

libs/migrate/versioning/__init__.py

@@ -0,0 +1,5 @@
+"""
+   This package provides functionality to create and manage
+   repositories of database schema changesets and to apply these
+   changesets to databases.
+"""

libs/migrate/versioning/api.py

@@ -0,0 +1,383 @@
+"""
+   This module provides an external API to the versioning system.
+
+   .. versionchanged:: 0.6.0
+    :func:`migrate.versioning.api.test` and schema diff functions
+    changed order of positional arguments so all accept `url` and `repository`
+    as first arguments.
+
+   .. versionchanged:: 0.5.4
+    ``--preview_sql`` displays source file when using SQL scripts.
+    If Python script is used, it runs the action with mocked engine and
+    returns captured SQL statements.
+
+   .. versionchanged:: 0.5.4
+    Deprecated ``--echo`` parameter in favour of new
+    :func:`migrate.versioning.util.construct_engine` behavior.
+"""
+
+# Dear migrate developers,
+#
+# please do not comment this module using sphinx syntax because its
+# docstrings are presented as user help and most users cannot
+# interpret sphinx annotated ReStructuredText.
+#
+# Thanks,
+# Jan Dittberner
+
+import sys
+import inspect
+import logging
+
+from migrate import exceptions
+from migrate.versioning import (repository, schema, version,
+    script as script_) # command name conflict
+from migrate.versioning.util import catch_known_errors, with_engine
+
+
+log = logging.getLogger(__name__)
+command_desc = {
+    'help': 'displays help on a given command',
+    'create': 'create an empty repository at the specified path',
+    'script': 'create an empty change Python script',
+    'script_sql': 'create empty change SQL scripts for given database',
+    'version': 'display the latest version available in a repository',
+    'db_version': 'show the current version of the repository under version control',
+    'source': 'display the Python code for a particular version in this repository',
+    'version_control': 'mark a database as under this repository\'s version control',
+    'upgrade': 'upgrade a database to a later version',
+    'downgrade': 'downgrade a database to an earlier version',
+    'drop_version_control': 'removes version control from a database',
+    'manage': 'creates a Python script that runs Migrate with a set of default values',
+    'test': 'performs the upgrade and downgrade command on the given database',
+    'compare_model_to_db': 'compare MetaData against the current database state',
+    'create_model': 'dump the current database as a Python model to stdout',
+    'make_update_script_for_model': 'create a script changing the old MetaData to the new (current) MetaData',
+    'update_db_from_model': 'modify the database to match the structure of the current MetaData',
+}
+__all__ = command_desc.keys()
+
+Repository = repository.Repository
+ControlledSchema = schema.ControlledSchema
+VerNum = version.VerNum
+PythonScript = script_.PythonScript
+SqlScript = script_.SqlScript
+
+
+# deprecated
+def help(cmd=None, **opts):
+    """%prog help COMMAND
+
+    Displays help on a given command.
+    """
+    if cmd is None:
+        raise exceptions.UsageError(None)
+    try:
+        func = globals()[cmd]
+    except:
+        raise exceptions.UsageError(
+            "'%s' isn't a valid command. Try 'help COMMAND'" % cmd)
+    ret = func.__doc__
+    if sys.argv[0]:
+        ret = ret.replace('%prog', sys.argv[0])
+    return ret
+
+@catch_known_errors
+def create(repository, name, **opts):
+    """%prog create REPOSITORY_PATH NAME [--table=TABLE]
+
+    Create an empty repository at the specified path.
+
+    You can specify the version_table to be used; by default, it is
+    'migrate_version'.  This table is created in all version-controlled
+    databases.
+    """
+    repo_path = Repository.create(repository, name, **opts)
+
+
+@catch_known_errors
+def script(description, repository, **opts):
+    """%prog script DESCRIPTION REPOSITORY_PATH
+
+    Create an empty change script using the next unused version number
+    appended with the given description.
+
+    For instance, manage.py script "Add initial tables" creates:
+    repository/versions/001_Add_initial_tables.py
+    """
+    repo = Repository(repository)
+    repo.create_script(description, **opts)
+
+
+@catch_known_errors
+def script_sql(database, repository, **opts):
+    """%prog script_sql DATABASE REPOSITORY_PATH
+
+    Create empty change SQL scripts for given DATABASE, where DATABASE
+    is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
+    or generic ('default').
+
+    For instance, manage.py script_sql postgres creates:
+    repository/versions/001_postgres_upgrade.sql and
+    repository/versions/001_postgres_postgres.sql
+    """
+    repo = Repository(repository)
+    repo.create_script_sql(database, **opts)
+
+
+def version(repository, **opts):
+    """%prog version REPOSITORY_PATH
+
+    Display the latest version available in a repository.
+    """
+    repo = Repository(repository)
+    return repo.latest
+
+
+@with_engine
+def db_version(url, repository, **opts):
+    """%prog db_version URL REPOSITORY_PATH
+
+    Show the current version of the repository with the given
+    connection string, under version control of the specified
+    repository.
+
+    The url should be any valid SQLAlchemy connection string.
+    """
+    engine = opts.pop('engine')
+    schema = ControlledSchema(engine, repository)
+    return schema.version
+
+
+def source(version, dest=None, repository=None, **opts):
+    """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH
+
+    Display the Python code for a particular version in this
+    repository.  Save it to the file at DESTINATION or, if omitted,
+    send to stdout.
+    """
+    if repository is None:
+        raise exceptions.UsageError("A repository must be specified")
+    repo = Repository(repository)
+    ret = repo.version(version).script().source()
+    if dest is not None:
+        dest = open(dest, 'w')
+        dest.write(ret)
+        dest.close()
+        ret = None
+    return ret
+
+
+def upgrade(url, repository, version=None, **opts):
+    """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql]
+
+    Upgrade a database to a later version.
+
+    This runs the upgrade() function defined in your change scripts.
+
+    By default, the database is updated to the latest available
+    version. You may specify a version instead, if you wish.
+
+    You may preview the Python or SQL code to be executed, rather than
+    actually executing it, using the appropriate 'preview' option.
+    """
+    err = "Cannot upgrade a database of version %s to version %s. "\
+        "Try 'downgrade' instead."
+    return _migrate(url, repository, version, upgrade=True, err=err, **opts)
+
+
+def downgrade(url, repository, version, **opts):
+    """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql]
+
+    Downgrade a database to an earlier version.
+
+    This is the reverse of upgrade; this runs the downgrade() function
+    defined in your change scripts.
+
+    You may preview the Python or SQL code to be executed, rather than
+    actually executing it, using the appropriate 'preview' option.
+    """
+    err = "Cannot downgrade a database of version %s to version %s. "\
+        "Try 'upgrade' instead."
+    return _migrate(url, repository, version, upgrade=False, err=err, **opts)
+
+@with_engine
+def test(url, repository, **opts):
+    """%prog test URL REPOSITORY_PATH [VERSION]
+
+    Performs the upgrade and downgrade option on the given
+    database. This is not a real test and may leave the database in a
+    bad state. You should therefore better run the test on a copy of
+    your database.
+    """
+    engine = opts.pop('engine')
+    repos = Repository(repository)
+    script = repos.version(None).script()
+
+    # Upgrade
+    log.info("Upgrading...")
+    script.run(engine, 1)
+    log.info("done")
+
+    log.info("Downgrading...")
+    script.run(engine, -1)
+    log.info("done")
+    log.info("Success")
+
+
+@with_engine
+def version_control(url, repository, version=None, **opts):
+    """%prog version_control URL REPOSITORY_PATH [VERSION]
+
+    Mark a database as under this repository's version control.
+
+    Once a database is under version control, schema changes should
+    only be done via change scripts in this repository.
+
+    This creates the table version_table in the database.
+
+    The url should be any valid SQLAlchemy connection string.
+
+    By default, the database begins at version 0 and is assumed to be
+    empty.  If the database is not empty, you may specify a version at
+    which to begin instead. No attempt is made to verify this
+    version's correctness - the database schema is expected to be
+    identical to what it would be if the database were created from
+    scratch.
+    """
+    engine = opts.pop('engine')
+    ControlledSchema.create(engine, repository, version)
+
+
+@with_engine
+def drop_version_control(url, repository, **opts):
+    """%prog drop_version_control URL REPOSITORY_PATH
+
+    Removes version control from a database.
+    """
+    engine = opts.pop('engine')
+    schema = ControlledSchema(engine, repository)
+    schema.drop()
+
+
+def manage(file, **opts):
+    """%prog manage FILENAME [VARIABLES...]
+
+    Creates a script that runs Migrate with a set of default values.
+
+    For example::
+
+        %prog manage manage.py --repository=/path/to/repository \
+--url=sqlite:///project.db
+
+    would create the script manage.py. The following two commands
+    would then have exactly the same results::
+
+        python manage.py version
+        %prog version --repository=/path/to/repository
+    """
+    Repository.create_manage_file(file, **opts)
+
+
+@with_engine
+def compare_model_to_db(url, repository, model, **opts):
+    """%prog compare_model_to_db URL REPOSITORY_PATH MODEL
+
+    Compare the current model (assumed to be a module level variable
+    of type sqlalchemy.MetaData) against the current database.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    return ControlledSchema.compare_model_to_db(engine, model, repository)
+
+
+@with_engine
+def create_model(url, repository, **opts):
+    """%prog create_model URL REPOSITORY_PATH [DECLERATIVE=True]
+
+    Dump the current database as a Python model to stdout.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    declarative = opts.get('declarative', False)
+    return ControlledSchema.create_model(engine, repository, declarative)
+
+
+@catch_known_errors
+@with_engine
+def make_update_script_for_model(url, repository, oldmodel, model, **opts):
+    """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH
+
+    Create a script changing the old Python model to the new (current)
+    Python model, sending to stdout.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    return PythonScript.make_update_script_for_model(
+        engine, oldmodel, model, repository, **opts)
+
+
+@with_engine
+def update_db_from_model(url, repository, model, **opts):
+    """%prog update_db_from_model URL REPOSITORY_PATH MODEL
+
+    Modify the database to match the structure of the current Python
+    model. This also sets the db_version number to the latest in the
+    repository.
+
+    NOTE: This is EXPERIMENTAL.
+    """  # TODO: get rid of EXPERIMENTAL label
+    engine = opts.pop('engine')
+    schema = ControlledSchema(engine, repository)
+    schema.update_db_from_model(model)
+
+@with_engine
+def _migrate(url, repository, version, upgrade, err, **opts):
+    engine = opts.pop('engine')
+    url = str(engine.url)
+    schema = ControlledSchema(engine, repository)
+    version = _migrate_version(schema, version, upgrade, err)
+
+    changeset = schema.changeset(version)
+    for ver, change in changeset:
+        nextver = ver + changeset.step
+        log.info('%s -> %s... ', ver, nextver)
+
+        if opts.get('preview_sql'):
+            if isinstance(change, PythonScript):
+                log.info(change.preview_sql(url, changeset.step, **opts))
+            elif isinstance(change, SqlScript):
+                log.info(change.source())
+
+        elif opts.get('preview_py'):
+            if not isinstance(change, PythonScript):
+                raise exceptions.UsageError("Python source can be only displayed"
+                    " for python migration files")
+            source_ver = max(ver, nextver)
+            module = schema.repository.version(source_ver).script().module
+            funcname = upgrade and "upgrade" or "downgrade"
+            func = getattr(module, funcname)
+            log.info(inspect.getsource(func))
+        else:
+            schema.runchange(ver, change, changeset.step)
+            log.info('done')
+
+
+def _migrate_version(schema, version, upgrade, err):
+    if version is None:
+        return version
+    # Version is specified: ensure we're upgrading in the right direction
+    # (current version < target version for upgrading; reverse for down)
+    version = VerNum(version)
+    cur = schema.version
+    if upgrade is not None:
+        if upgrade:
+            direction = cur <= version
+        else:
+            direction = cur >= version
+        if not direction:
+            raise exceptions.KnownError(err % (cur, version))
+    return version

libs/migrate/versioning/cfgparse.py

@@ -0,0 +1,27 @@
+"""
+   Configuration parser module.
+"""
+
+from ConfigParser import ConfigParser
+
+from migrate.versioning.config import *
+from migrate.versioning import pathed
+
+
+class Parser(ConfigParser):
+    """A project configuration file."""
+
+    def to_dict(self, sections=None):
+        """It's easier to access config values like dictionaries"""
+        return self._sections
+
+
+class Config(pathed.Pathed, Parser):
+    """Configuration class."""
+
+    def __init__(self, path, *p, **k):
+        """Confirm the config file exists; read it."""
+        self.require_found(path)
+        pathed.Pathed.__init__(self, path)
+        Parser.__init__(self, *p, **k)
+        self.read(path)

libs/migrate/versioning/config.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from sqlalchemy.util import OrderedDict
+
+
+__all__ = ['databases', 'operations']
+
+databases = ('sqlite', 'postgres', 'mysql', 'oracle', 'mssql', 'firebird')
+
+# Map operation names to function names
+operations = OrderedDict()
+operations['upgrade'] = 'upgrade'
+operations['downgrade'] = 'downgrade'

libs/migrate/versioning/genmodel.py

@@ -0,0 +1,254 @@
+"""
+   Code to generate a Python model from a database or differences
+   between a model and database.
+
+   Some of this is borrowed heavily from the AutoCode project at:
+   http://code.google.com/p/sqlautocode/
+"""
+
+import sys
+import logging
+
+import sqlalchemy
+
+import migrate
+import migrate.changeset
+
+
+log = logging.getLogger(__name__)
+HEADER = """
+## File autogenerated by genmodel.py
+
+from sqlalchemy import *
+meta = MetaData()
+"""
+
+DECLARATIVE_HEADER = """
+## File autogenerated by genmodel.py
+
+from sqlalchemy import *
+from sqlalchemy.ext import declarative
+
+Base = declarative.declarative_base()
+"""
+
+
+class ModelGenerator(object):
+
+    def __init__(self, diff, engine, declarative=False):
+        self.diff = diff
+        self.engine = engine
+        self.declarative = declarative
+
+    def column_repr(self, col):
+        kwarg = []
+        if col.key != col.name:
+            kwarg.append('key')
+        if col.primary_key:
+            col.primary_key = True  # otherwise it dumps it as 1
+            kwarg.append('primary_key')
+        if not col.nullable:
+            kwarg.append('nullable')
+        if col.onupdate:
+            kwarg.append('onupdate')
+        if col.default:
+            if col.primary_key:
+                # I found that PostgreSQL automatically creates a
+                # default value for the sequence, but let's not show
+                # that.
+                pass
+            else:
+                kwarg.append('default')
+        ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg)
+
+        # crs: not sure if this is good idea, but it gets rid of extra
+        # u''
+        name = col.name.encode('utf8')
+
+        type_ = col.type
+        for cls in col.type.__class__.__mro__:
+            if cls.__module__ == 'sqlalchemy.types' and \
+                not cls.__name__.isupper():
+                if cls is not type_.__class__:
+                    type_ = cls()
+                break
+
+        data = {
+            'name': name,
+            'type': type_,
+            'constraints': ', '.join([repr(cn) for cn in col.constraints]),
+            'args': ks and ks or ''}
+
+        if data['constraints']:
+            if data['args']:
+                data['args'] = ',' + data['args']
+
+        if data['constraints'] or data['args']:
+            data['maybeComma'] = ','
+        else:
+            data['maybeComma'] = ''
+
+        commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data
+        commonStuff = commonStuff.strip()
+        data['commonStuff'] = commonStuff
+        if self.declarative:
+            return """%(name)s = Column(%(type)r%(commonStuff)s""" % data
+        else:
+            return """Column(%(name)r, %(type)r%(commonStuff)s""" % data
+
+    def getTableDefn(self, table):
+        out = []
+        tableName = table.name
+        if self.declarative:
+            out.append("class %(table)s(Base):" % {'table': tableName})
+            out.append("  __tablename__ = '%(table)s'" % {'table': tableName})
+            for col in table.columns:
+                out.append("  %s" % self.column_repr(col))
+        else:
+            out.append("%(table)s = Table('%(table)s', meta," % \
+                           {'table': tableName})
+            for col in table.columns:
+                out.append("  %s," % self.column_repr(col))
+            out.append(")")
+        return out
+
+    def _get_tables(self,missingA=False,missingB=False,modified=False):
+        to_process = []
+        for bool_,names,metadata in (
+            (missingA,self.diff.tables_missing_from_A,self.diff.metadataB),
+            (missingB,self.diff.tables_missing_from_B,self.diff.metadataA),
+            (modified,self.diff.tables_different,self.diff.metadataA),
+                ):
+            if bool_:
+                for name in names:
+                    yield metadata.tables.get(name)
+        
+    def toPython(self):
+        """Assume database is current and model is empty."""
+        out = []
+        if self.declarative:
+            out.append(DECLARATIVE_HEADER)
+        else:
+            out.append(HEADER)
+        out.append("")
+        for table in self._get_tables(missingA=True):
+            out.extend(self.getTableDefn(table))
+            out.append("")
+        return '\n'.join(out)
+
+    def toUpgradeDowngradePython(self, indent='    '):
+        ''' Assume model is most current and database is out-of-date. '''
+        decls = ['from migrate.changeset import schema',
+                 'meta = MetaData()']
+        for table in self._get_tables(
+            missingA=True,missingB=True,modified=True
+            ):
+            decls.extend(self.getTableDefn(table))
+
+        upgradeCommands, downgradeCommands = [], []
+        for tableName in self.diff.tables_missing_from_A:
+            upgradeCommands.append("%(table)s.drop()" % {'table': tableName})
+            downgradeCommands.append("%(table)s.create()" % \
+                                         {'table': tableName})
+        for tableName in self.diff.tables_missing_from_B:
+            upgradeCommands.append("%(table)s.create()" % {'table': tableName})
+            downgradeCommands.append("%(table)s.drop()" % {'table': tableName})
+
+        for tableName in self.diff.tables_different:
+            dbTable = self.diff.metadataB.tables[tableName]
+            missingInDatabase, missingInModel, diffDecl = \
+                self.diff.colDiffs[tableName]
+            for col in missingInDatabase:
+                upgradeCommands.append('%s.columns[%r].create()' % (
+                        modelTable, col.name))
+                downgradeCommands.append('%s.columns[%r].drop()' % (
+                        modelTable, col.name))
+            for col in missingInModel:
+                upgradeCommands.append('%s.columns[%r].drop()' % (
+                        modelTable, col.name))
+                downgradeCommands.append('%s.columns[%r].create()' % (
+                        modelTable, col.name))
+            for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl:
+                upgradeCommands.append(
+                    'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
+                    modelTable, modelCol.name, databaseCol.name))
+                downgradeCommands.append(
+                    'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
+                    modelTable, modelCol.name, databaseCol.name))
+        pre_command = '    meta.bind = migrate_engine'
+
+        return (
+            '\n'.join(decls),
+            '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in upgradeCommands]),
+            '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in downgradeCommands]))
+
+    def _db_can_handle_this_change(self,td):
+        if (td.columns_missing_from_B
+            and not td.columns_missing_from_A
+            and not td.columns_different):
+            # Even sqlite can handle this.
+            return True
+        else:
+            return not self.engine.url.drivername.startswith('sqlite')
+
+    def applyModel(self):
+        """Apply model to current database."""
+
+        meta = sqlalchemy.MetaData(self.engine)
+
+        for table in self._get_tables(missingA=True):
+            table = table.tometadata(meta)
+            table.drop()
+        for table in self._get_tables(missingB=True):
+            table = table.tometadata(meta)
+            table.create()
+        for modelTable in self._get_tables(modified=True):
+            tableName = modelTable.name
+            modelTable = modelTable.tometadata(meta)
+            dbTable = self.diff.metadataB.tables[tableName]
+
+            td = self.diff.tables_different[tableName]
+            
+            if self._db_can_handle_this_change(td):
+                
+                for col in td.columns_missing_from_B:
+                    modelTable.columns[col].create()
+                for col in td.columns_missing_from_A:
+                    dbTable.columns[col].drop()
+                # XXX handle column changes here.
+            else:
+                # Sqlite doesn't support drop column, so you have to
+                # do more: create temp table, copy data to it, drop
+                # old table, create new table, copy data back.
+                #
+                # I wonder if this is guaranteed to be unique?
+                tempName = '_temp_%s' % modelTable.name
+
+                def getCopyStatement():
+                    preparer = self.engine.dialect.preparer
+                    commonCols = []
+                    for modelCol in modelTable.columns:
+                        if modelCol.name in dbTable.columns:
+                            commonCols.append(modelCol.name)
+                    commonColsStr = ', '.join(commonCols)
+                    return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \
+                        (tableName, commonColsStr, commonColsStr, tempName)
+
+                # Move the data in one transaction, so that we don't
+                # leave the database in a nasty state.
+                connection = self.engine.connect()
+                trans = connection.begin()
+                try:
+                    connection.execute(
+                        'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \
+                            (tempName, modelTable.name))
+                    # make sure the drop takes place inside our
+                    # transaction with the bind parameter
+                    modelTable.drop(bind=connection)
+                    modelTable.create(bind=connection)
+                    connection.execute(getCopyStatement())
+                    connection.execute('DROP TABLE %s' % tempName)
+                    trans.commit()
+                except:
+                    trans.rollback()
+                    raise

libs/migrate/versioning/migrate_repository.py

@@ -0,0 +1,100 @@
+"""
+   Script to migrate repository from sqlalchemy <= 0.4.4 to the new
+   repository schema. This shouldn't use any other migrate modules, so
+   that it can work in any version.
+"""
+
+import os
+import sys
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def usage():
+    """Gives usage information."""
+    print """Usage: %(prog)s repository-to-migrate
+
+    Upgrade your repository to the new flat format.
+
+    NOTE: You should probably make a backup before running this.
+    """ % {'prog': sys.argv[0]}
+
+    sys.exit(1)
+
+
+def delete_file(filepath):
+    """Deletes a file and prints a message."""
+    log.info('Deleting file: %s' % filepath)
+    os.remove(filepath)
+
+
+def move_file(src, tgt):
+    """Moves a file and prints a message."""
+    log.info('Moving file %s to %s' % (src, tgt))
+    if os.path.exists(tgt):
+        raise Exception(
+            'Cannot move file %s because target %s already exists' % \
+                (src, tgt))
+    os.rename(src, tgt)
+
+
+def delete_directory(dirpath):
+    """Delete a directory and print a message."""
+    log.info('Deleting directory: %s' % dirpath)
+    os.rmdir(dirpath)
+
+
+def migrate_repository(repos):
+    """Does the actual migration to the new repository format."""
+    log.info('Migrating repository at: %s to new format' % repos)
+    versions = '%s/versions' % repos
+    dirs = os.listdir(versions)
+    # Only use int's in list.
+    numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()]
+    numdirs.sort()  # Sort list.
+    for dirname in numdirs:
+        origdir = '%s/%s' % (versions, dirname)
+        log.info('Working on directory: %s' % origdir)
+        files = os.listdir(origdir)
+        files.sort()
+        for filename in files:
+            # Delete compiled Python files.
+            if filename.endswith('.pyc') or filename.endswith('.pyo'):
+                delete_file('%s/%s' % (origdir, filename))
+
+            # Delete empty __init__.py files.
+            origfile = '%s/__init__.py' % origdir
+            if os.path.exists(origfile) and len(open(origfile).read()) == 0:
+                delete_file(origfile)
+
+            # Move sql upgrade scripts.
+            if filename.endswith('.sql'):
+                version, dbms, operation = filename.split('.', 3)[0:3]
+                origfile = '%s/%s' % (origdir, filename)
+                # For instance:  2.postgres.upgrade.sql ->
+                #  002_postgres_upgrade.sql
+                tgtfile = '%s/%03d_%s_%s.sql' % (
+                    versions, int(version), dbms, operation)
+                move_file(origfile, tgtfile)
+
+        # Move Python upgrade script.
+        pyfile = '%s.py' % dirname
+        pyfilepath = '%s/%s' % (origdir, pyfile)
+        if os.path.exists(pyfilepath):
+            tgtfile = '%s/%03d.py' % (versions, int(dirname))
+            move_file(pyfilepath, tgtfile)
+
+        # Try to remove directory. Will fail if it's not empty.
+        delete_directory(origdir)
+
+
+def main():
+    """Main function to be called when using this script."""
+    if len(sys.argv) != 2:
+        usage()
+    migrate_repository(sys.argv[1])
+
+
+if __name__ == '__main__':
+    main()

libs/migrate/versioning/pathed.py

@@ -0,0 +1,75 @@
+"""
+   A path/directory class.
+"""
+
+import os
+import shutil
+import logging
+
+from migrate import exceptions
+from migrate.versioning.config import *
+from migrate.versioning.util import KeyedInstance
+
+
+log = logging.getLogger(__name__)
+
+class Pathed(KeyedInstance):
+    """
+    A class associated with a path/directory tree.
+
+    Only one instance of this class may exist for a particular file;
+    __new__ will return an existing instance if possible
+    """
+    parent = None
+
+    @classmethod
+    def _key(cls, path):
+        return str(path)
+
+    def __init__(self, path):
+        self.path = path
+        if self.__class__.parent is not None:
+            self._init_parent(path)
+
+    def _init_parent(self, path):
+        """Try to initialize this object's parent, if it has one"""
+        parent_path = self.__class__._parent_path(path)
+        self.parent = self.__class__.parent(parent_path)
+        log.debug("Getting parent %r:%r" % (self.__class__.parent, parent_path))
+        self.parent._init_child(path, self)
+
+    def _init_child(self, child, path):
+        """Run when a child of this object is initialized.
+
+        Parameters: the child object; the path to this object (its
+        parent)
+        """
+
+    @classmethod
+    def _parent_path(cls, path):
+        """
+        Fetch the path of this object's parent from this object's path.
+        """
+        # os.path.dirname(), but strip directories like files (like
+        # unix basename)
+        #
+        # Treat directories like files...
+        if path[-1] == '/':
+            path = path[:-1]
+        ret = os.path.dirname(path)
+        return ret
+
+    @classmethod
+    def require_notfound(cls, path):
+        """Ensures a given path does not already exist"""
+        if os.path.exists(path):
+            raise exceptions.PathFoundError(path)
+
+    @classmethod
+    def require_found(cls, path):
+        """Ensures a given path already exists"""
+        if not os.path.exists(path):
+            raise exceptions.PathNotFoundError(path)
+
+    def __str__(self):
+        return self.path

libs/migrate/versioning/repository.py

@@ -0,0 +1,231 @@
+"""
+   SQLAlchemy migrate repository management.
+"""
+import os
+import shutil
+import string
+import logging
+
+from pkg_resources import resource_filename
+from tempita import Template as TempitaTemplate
+
+from migrate import exceptions
+from migrate.versioning import version, pathed, cfgparse
+from migrate.versioning.template import Template
+from migrate.versioning.config import *
+
+
+log = logging.getLogger(__name__)
+
+class Changeset(dict):
+    """A collection of changes to be applied to a database.
+
+    Changesets are bound to a repository and manage a set of
+    scripts from that repository.
+
+    Behaves like a dict, for the most part. Keys are ordered based on step value.
+    """
+
+    def __init__(self, start, *changes, **k):
+        """
+        Give a start version; step must be explicitly stated.
+        """
+        self.step = k.pop('step', 1)
+        self.start = version.VerNum(start)
+        self.end = self.start
+        for change in changes:
+            self.add(change)
+
+    def __iter__(self):
+        return iter(self.items())
+
+    def keys(self):
+        """
+        In a series of upgrades x -> y, keys are version x. Sorted.
+        """
+        ret = super(Changeset, self).keys()
+        # Reverse order if downgrading
+        ret.sort(reverse=(self.step < 1))
+        return ret
+
+    def values(self):
+        return [self[k] for k in self.keys()]
+
+    def items(self):
+        return zip(self.keys(), self.values())
+
+    def add(self, change):
+        """Add new change to changeset"""
+        key = self.end
+        self.end += self.step
+        self[key] = change
+
+    def run(self, *p, **k):
+        """Run the changeset scripts"""
+        for version, script in self:
+            script.run(*p, **k)
+
+
+class Repository(pathed.Pathed):
+    """A project's change script repository"""
+
+    _config = 'migrate.cfg'
+    _versions = 'versions'
+
+    def __init__(self, path):
+        log.debug('Loading repository %s...' % path)
+        self.verify(path)
+        super(Repository, self).__init__(path)
+        self.config = cfgparse.Config(os.path.join(self.path, self._config))
+        self.versions = version.Collection(os.path.join(self.path,
+                                                      self._versions))
+        log.debug('Repository %s loaded successfully' % path)
+        log.debug('Config: %r' % self.config.to_dict())
+
+    @classmethod
+    def verify(cls, path):
+        """
+        Ensure the target path is a valid repository.
+
+        :raises: :exc:`InvalidRepositoryError <migrate.exceptions.InvalidRepositoryError>`
+        """
+        # Ensure the existence of required files
+        try:
+            cls.require_found(path)
+            cls.require_found(os.path.join(path, cls._config))
+            cls.require_found(os.path.join(path, cls._versions))
+        except exceptions.PathNotFoundError, e:
+            raise exceptions.InvalidRepositoryError(path)
+
+    @classmethod
+    def prepare_config(cls, tmpl_dir, name, options=None):
+        """
+        Prepare a project configuration file for a new project.
+
+        :param tmpl_dir: Path to Repository template
+        :param config_file: Name of the config file in Repository template
+        :param name: Repository name
+        :type tmpl_dir: string
+        :type config_file: string
+        :type name: string
+        :returns: Populated config file
+        """
+        if options is None:
+            options = {}
+        options.setdefault('version_table', 'migrate_version')
+        options.setdefault('repository_id', name)
+        options.setdefault('required_dbs', [])
+
+        tmpl = open(os.path.join(tmpl_dir, cls._config)).read()
+        ret = TempitaTemplate(tmpl).substitute(options)
+
+        # cleanup
+        del options['__template_name__']
+
+        return ret
+
+    @classmethod
+    def create(cls, path, name, **opts):
+        """Create a repository at a specified path"""
+        cls.require_notfound(path)
+        theme = opts.pop('templates_theme', None)
+        t_path = opts.pop('templates_path', None)
+
+        # Create repository
+        tmpl_dir = Template(t_path).get_repository(theme=theme)
+        shutil.copytree(tmpl_dir, path)
+
+        # Edit config defaults
+        config_text = cls.prepare_config(tmpl_dir, name, options=opts)
+        fd = open(os.path.join(path, cls._config), 'w')
+        fd.write(config_text)
+        fd.close()
+
+        opts['repository_name'] = name
+
+        # Create a management script
+        manager = os.path.join(path, 'manage.py')
+        Repository.create_manage_file(manager, templates_theme=theme,
+            templates_path=t_path, **opts)
+
+        return cls(path)
+
+    def create_script(self, description, **k):
+        """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
+        self.versions.create_new_python_version(description, **k)
+
+    def create_script_sql(self, database, **k):
+        """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
+        self.versions.create_new_sql_version(database, **k)
+
+    @property
+    def latest(self):
+        """API to :attr:`migrate.versioning.version.Collection.latest`"""
+        return self.versions.latest
+
+    @property
+    def version_table(self):
+        """Returns version_table name specified in config"""
+        return self.config.get('db_settings', 'version_table')
+
+    @property
+    def id(self):
+        """Returns repository id specified in config"""
+        return self.config.get('db_settings', 'repository_id')
+
+    def version(self, *p, **k):
+        """API to :attr:`migrate.versioning.version.Collection.version`"""
+        return self.versions.version(*p, **k)
+
+    @classmethod
+    def clear(cls):
+        # TODO: deletes repo
+        super(Repository, cls).clear()
+        version.Collection.clear()
+
+    def changeset(self, database, start, end=None):
+        """Create a changeset to migrate this database from ver. start to end/latest.
+
+        :param database: name of database to generate changeset
+        :param start: version to start at
+        :param end: version to end at (latest if None given)
+        :type database: string
+        :type start: int
+        :type end: int
+        :returns: :class:`Changeset instance <migration.versioning.repository.Changeset>`
+        """
+        start = version.VerNum(start)
+
+        if end is None:
+            end = self.latest
+        else:
+            end = version.VerNum(end)
+
+        if start <= end:
+            step = 1
+            range_mod = 1
+            op = 'upgrade'
+        else:
+            step = -1
+            range_mod = 0
+            op = 'downgrade'
+
+        versions = range(start + range_mod, end + range_mod, step)
+        changes = [self.version(v).script(database, op) for v in versions]
+        ret = Changeset(start, step=step, *changes)
+        return ret
+
+    @classmethod
+    def create_manage_file(cls, file_, **opts):
+        """Create a project management script (manage.py)
+        
+        :param file_: Destination file to be written
+        :param opts: Options that are passed to :func:`migrate.versioning.shell.main`
+        """
+        mng_file = Template(opts.pop('templates_path', None))\
+            .get_manage(theme=opts.pop('templates_theme', None))
+
+        tmpl = open(mng_file).read()
+        fd = open(file_, 'w')
+        fd.write(TempitaTemplate(tmpl).substitute(opts))
+        fd.close()

libs/migrate/versioning/schema.py

@@ -0,0 +1,213 @@
+"""
+   Database schema version management.
+"""
+import sys
+import logging
+
+from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
+    create_engine)
+from sqlalchemy.sql import and_
+from sqlalchemy import exceptions as sa_exceptions
+from sqlalchemy.sql import bindparam
+
+from migrate import exceptions
+from migrate.versioning import genmodel, schemadiff
+from migrate.versioning.repository import Repository
+from migrate.versioning.util import load_model
+from migrate.versioning.version import VerNum
+
+
+log = logging.getLogger(__name__)
+
+class ControlledSchema(object):
+    """A database under version control"""
+
+    def __init__(self, engine, repository):
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+        self.engine = engine
+        self.repository = repository
+        self.meta = MetaData(engine)
+        self.load()
+
+    def __eq__(self, other):
+        """Compare two schemas by repositories and versions"""
+        return (self.repository is other.repository \
+            and self.version == other.version)
+
+    def load(self):
+        """Load controlled schema version info from DB"""
+        tname = self.repository.version_table
+        try:
+            if not hasattr(self, 'table') or self.table is None:
+                    self.table = Table(tname, self.meta, autoload=True)
+
+            result = self.engine.execute(self.table.select(
+                self.table.c.repository_id == str(self.repository.id)))
+
+            data = list(result)[0]
+        except:
+            cls, exc, tb = sys.exc_info()
+            raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
+
+        self.version = data['version']
+        return data
+
+    def drop(self):
+        """
+        Remove version control from a database.
+        """
+        try:
+            self.table.drop()
+        except (sa_exceptions.SQLError):
+            raise exceptions.DatabaseNotControlledError(str(self.table))
+
+    def changeset(self, version=None):
+        """API to Changeset creation.
+        
+        Uses self.version for start version and engine.name
+        to get database name.
+        """
+        database = self.engine.name
+        start_ver = self.version
+        changeset = self.repository.changeset(database, start_ver, version)
+        return changeset
+
+    def runchange(self, ver, change, step):
+        startver = ver
+        endver = ver + step
+        # Current database version must be correct! Don't run if corrupt!
+        if self.version != startver:
+            raise exceptions.InvalidVersionError("%s is not %s" % \
+                                                     (self.version, startver))
+        # Run the change
+        change.run(self.engine, step)
+
+        # Update/refresh database version
+        self.update_repository_table(startver, endver)
+        self.load()
+
+    def update_repository_table(self, startver, endver):
+        """Update version_table with new information"""
+        update = self.table.update(and_(self.table.c.version == int(startver),
+             self.table.c.repository_id == str(self.repository.id)))
+        self.engine.execute(update, version=int(endver))
+
+    def upgrade(self, version=None):
+        """
+        Upgrade (or downgrade) to a specified version, or latest version.
+        """
+        changeset = self.changeset(version)
+        for ver, change in changeset:
+            self.runchange(ver, change, changeset.step)
+
+    def update_db_from_model(self, model):
+        """
+        Modify the database to match the structure of the current Python model.
+        """
+        model = load_model(model)
+
+        diff = schemadiff.getDiffOfModelAgainstDatabase(
+            model, self.engine, excludeTables=[self.repository.version_table]
+            )
+        genmodel.ModelGenerator(diff,self.engine).applyModel()
+
+        self.update_repository_table(self.version, int(self.repository.latest))
+
+        self.load()
+
+    @classmethod
+    def create(cls, engine, repository, version=None):
+        """
+        Declare a database to be under a repository's version control.
+
+        :raises: :exc:`DatabaseAlreadyControlledError`
+        :returns: :class:`ControlledSchema`
+        """
+        # Confirm that the version # is valid: positive, integer,
+        # exists in repos
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+        version = cls._validate_version(repository, version)
+        table = cls._create_table_version(engine, repository, version)
+        # TODO: history table
+        # Load repository information and return
+        return cls(engine, repository)
+
+    @classmethod
+    def _validate_version(cls, repository, version):
+        """
+        Ensures this is a valid version number for this repository.
+
+        :raises: :exc:`InvalidVersionError` if invalid
+        :return: valid version number
+        """
+        if version is None:
+            version = 0
+        try:
+            version = VerNum(version) # raises valueerror
+            if version < 0 or version > repository.latest:
+                raise ValueError()
+        except ValueError:
+            raise exceptions.InvalidVersionError(version)
+        return version
+
+    @classmethod
+    def _create_table_version(cls, engine, repository, version):
+        """
+        Creates the versioning table in a database.
+
+        :raises: :exc:`DatabaseAlreadyControlledError`
+        """
+        # Create tables
+        tname = repository.version_table
+        meta = MetaData(engine)
+
+        table = Table(
+            tname, meta,
+            Column('repository_id', String(250), primary_key=True),
+            Column('repository_path', Text),
+            Column('version', Integer), )
+
+        # there can be multiple repositories/schemas in the same db
+        if not table.exists():
+            table.create()
+
+        # test for existing repository_id
+        s = table.select(table.c.repository_id == bindparam("repository_id"))
+        result = engine.execute(s, repository_id=repository.id)
+        if result.fetchone():
+            raise exceptions.DatabaseAlreadyControlledError
+
+        # Insert data
+        engine.execute(table.insert().values(
+                           repository_id=repository.id,
+                           repository_path=repository.path,
+                           version=int(version)))
+        return table
+
+    @classmethod
+    def compare_model_to_db(cls, engine, model, repository):
+        """
+        Compare the current model against the current database.
+        """
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+        model = load_model(model)
+
+        diff = schemadiff.getDiffOfModelAgainstDatabase(
+            model, engine, excludeTables=[repository.version_table])
+        return diff
+
+    @classmethod
+    def create_model(cls, engine, repository, declarative=False):
+        """
+        Dump the current database as a Python model.
+        """
+        if isinstance(repository, basestring):
+            repository = Repository(repository)
+
+        diff = schemadiff.getDiffOfModelAgainstDatabase(
+            MetaData(), engine, excludeTables=[repository.version_table]
+            )
+        return genmodel.ModelGenerator(diff, engine, declarative).toPython()

libs/migrate/versioning/schemadiff.py

@@ -0,0 +1,285 @@
+"""
+   Schema differencing support.
+"""
+
+import logging
+import sqlalchemy
+
+from migrate.changeset import SQLA_06
+from sqlalchemy.types import Float
+
+log = logging.getLogger(__name__)
+
+def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None):
+    """
+    Return differences of model against database.
+
+    :return: object which will evaluate to :keyword:`True` if there \
+      are differences else :keyword:`False`.
+    """
+    return SchemaDiff(metadata,
+                      sqlalchemy.MetaData(engine, reflect=True),
+                      labelA='model',
+                      labelB='database',
+                      excludeTables=excludeTables)
+
+
+def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None):
+    """
+    Return differences of model against another model.
+
+    :return: object which will evaluate to :keyword:`True` if there \
+      are differences else :keyword:`False`.
+    """
+    return SchemaDiff(metadataA, metadataB, excludeTables)
+
+
+class ColDiff(object):
+    """
+    Container for differences in one :class:`~sqlalchemy.schema.Column`
+    between two :class:`~sqlalchemy.schema.Table` instances, ``A``
+    and ``B``.
+    
+    .. attribute:: col_A
+
+      The :class:`~sqlalchemy.schema.Column` object for A.
+      
+    .. attribute:: col_B
+
+      The :class:`~sqlalchemy.schema.Column` object for B.
+
+    .. attribute:: type_A
+
+      The most generic type of the :class:`~sqlalchemy.schema.Column`
+      object in A. 
+      
+    .. attribute:: type_B
+
+      The most generic type of the :class:`~sqlalchemy.schema.Column`
+      object in A. 
+      
+    """
+    
+    diff = False
+
+    def __init__(self,col_A,col_B):
+        self.col_A = col_A
+        self.col_B = col_B
+
+        self.type_A = col_A.type
+        self.type_B = col_B.type
+
+        self.affinity_A = self.type_A._type_affinity
+        self.affinity_B = self.type_B._type_affinity
+
+        if self.affinity_A is not self.affinity_B:
+            self.diff = True
+            return
+
+        if isinstance(self.type_A,Float) or isinstance(self.type_B,Float):
+            if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)):
+                self.diff=True
+                return
+
+        for attr in ('precision','scale','length'):
+            A = getattr(self.type_A,attr,None)
+            B = getattr(self.type_B,attr,None)
+            if not (A is None or B is None) and A!=B:
+                self.diff=True
+                return
+        
+    def __nonzero__(self):
+        return self.diff
+    
+class TableDiff(object):
+    """
+    Container for differences in one :class:`~sqlalchemy.schema.Table`
+    between two :class:`~sqlalchemy.schema.MetaData` instances, ``A``
+    and ``B``.
+
+    .. attribute:: columns_missing_from_A
+
+      A sequence of column names that were found in B but weren't in
+      A.
+      
+    .. attribute:: columns_missing_from_B
+
+      A sequence of column names that were found in A but weren't in
+      B.
+      
+    .. attribute:: columns_different
+
+      A dictionary containing information about columns that were
+      found to be different.
+      It maps column names to a :class:`ColDiff` objects describing the
+      differences found.
+    """
+    __slots__ = (
+        'columns_missing_from_A',
+        'columns_missing_from_B',
+        'columns_different',
+        )
+
+    def __nonzero__(self):
+        return bool(
+            self.columns_missing_from_A or
+            self.columns_missing_from_B or
+            self.columns_different
+            )
+    
+class SchemaDiff(object):
+    """
+    Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
+    objects.
+
+    The string representation of a :class:`SchemaDiff` will summarise
+    the changes found between the two
+    :class:`~sqlalchemy.schema.MetaData` objects.
+
+    The length of a :class:`SchemaDiff` will give the number of
+    changes found, enabling it to be used much like a boolean in
+    expressions.
+        
+    :param metadataA:
+      First :class:`~sqlalchemy.schema.MetaData` to compare.
+      
+    :param metadataB:
+      Second :class:`~sqlalchemy.schema.MetaData` to compare.
+      
+    :param labelA:
+      The label to use in messages about the first
+      :class:`~sqlalchemy.schema.MetaData`. 
+    
+    :param labelB: 
+      The label to use in messages about the second
+      :class:`~sqlalchemy.schema.MetaData`. 
+    
+    :param excludeTables:
+      A sequence of table names to exclude.
+      
+    .. attribute:: tables_missing_from_A
+
+      A sequence of table names that were found in B but weren't in
+      A.
+      
+    .. attribute:: tables_missing_from_B
+
+      A sequence of table names that were found in A but weren't in
+      B.
+      
+    .. attribute:: tables_different
+
+      A dictionary containing information about tables that were found
+      to be different.
+      It maps table names to a :class:`TableDiff` objects describing the
+      differences found.
+    """
+
+    def __init__(self,
+                 metadataA, metadataB,
+                 labelA='metadataA',
+                 labelB='metadataB',
+                 excludeTables=None):
+
+        self.metadataA, self.metadataB = metadataA, metadataB
+        self.labelA, self.labelB = labelA, labelB
+        self.label_width = max(len(labelA),len(labelB))
+        excludeTables = set(excludeTables or [])
+
+        A_table_names = set(metadataA.tables.keys())
+        B_table_names = set(metadataB.tables.keys())
+
+        self.tables_missing_from_A = sorted(
+            B_table_names - A_table_names - excludeTables
+            )
+        self.tables_missing_from_B = sorted(
+            A_table_names - B_table_names - excludeTables
+            )
+        
+        self.tables_different = {}
+        for table_name in A_table_names.intersection(B_table_names):
+
+            td = TableDiff()
+            
+            A_table = metadataA.tables[table_name]
+            B_table = metadataB.tables[table_name]
+            
+            A_column_names = set(A_table.columns.keys())
+            B_column_names = set(B_table.columns.keys())
+
+            td.columns_missing_from_A = sorted(
+                B_column_names - A_column_names
+                )
+            
+            td.columns_missing_from_B = sorted(
+                A_column_names - B_column_names
+                )
+            
+            td.columns_different = {}
+
+            for col_name in A_column_names.intersection(B_column_names):
+
+                cd = ColDiff(
+                    A_table.columns.get(col_name),
+                    B_table.columns.get(col_name)
+                    )
+
+                if cd:
+                    td.columns_different[col_name]=cd
+                
+            # XXX - index and constraint differences should
+            #       be checked for here
+
+            if td:
+                self.tables_different[table_name]=td
+
+    def __str__(self):
+        ''' Summarize differences. '''
+        out = []
+        column_template ='      %%%is: %%r' % self.label_width
+        
+        for names,label in (
+            (self.tables_missing_from_A,self.labelA),
+            (self.tables_missing_from_B,self.labelB),
+            ):
+            if names:
+                out.append(
+                    '  tables missing from %s: %s' % (
+                        label,', '.join(sorted(names))
+                        )
+                    )
+                
+        for name,td in sorted(self.tables_different.items()):
+            out.append(
+               '  table with differences: %s' % name
+               )
+            for names,label in (
+                (td.columns_missing_from_A,self.labelA),
+                (td.columns_missing_from_B,self.labelB),
+                ):
+                if names:
+                    out.append(
+                        '    %s missing these columns: %s' % (
+                            label,', '.join(sorted(names))
+                            )
+                        )
+            for name,cd in td.columns_different.items():
+                out.append('    column with differences: %s' % name)
+                out.append(column_template % (self.labelA,cd.col_A))
+                out.append(column_template % (self.labelB,cd.col_B))
+                
+        if out:
+            out.insert(0, 'Schema diffs:')
+            return '\n'.join(out)
+        else:
+            return 'No schema diffs'
+
+    def __len__(self):
+        """
+        Used in bool evaluation, return of 0 means no diffs.
+        """
+        return (
+            len(self.tables_missing_from_A) +
+            len(self.tables_missing_from_B) +
+            len(self.tables_different)
+            )

libs/migrate/versioning/script/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from migrate.versioning.script.base import BaseScript
+from migrate.versioning.script.py import PythonScript
+from migrate.versioning.script.sql import SqlScript

libs/migrate/versioning/script/base.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import logging
+
+from migrate import exceptions
+from migrate.versioning.config import operations
+from migrate.versioning import pathed
+
+
+log = logging.getLogger(__name__)
+
+class BaseScript(pathed.Pathed):
+    """Base class for other types of scripts.
+    All scripts have the following properties:
+
+    source (script.source())
+      The source code of the script
+    version (script.version())
+      The version number of the script
+    operations (script.operations())
+      The operations defined by the script: upgrade(), downgrade() or both.
+      Returns a tuple of operations.
+      Can also check for an operation with ex. script.operation(Script.ops.up)
+    """ # TODO: sphinxfy this and implement it correctly
+
+    def __init__(self, path):
+        log.debug('Loading script %s...' % path)
+        self.verify(path)
+        super(BaseScript, self).__init__(path)
+        log.debug('Script %s loaded successfully' % path)
+    
+    @classmethod
+    def verify(cls, path):
+        """Ensure this is a valid script
+        This version simply ensures the script file's existence
+
+        :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>`
+        """
+        try:
+            cls.require_found(path)
+        except:
+            raise exceptions.InvalidScriptError(path)
+
+    def source(self):
+        """:returns: source code of the script.
+        :rtype: string
+        """
+        fd = open(self.path)
+        ret = fd.read()
+        fd.close()
+        return ret
+
+    def run(self, engine):
+        """Core of each BaseScript subclass.
+        This method executes the script.
+        """
+        raise NotImplementedError()

libs/migrate/versioning/script/py.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import shutil
+import warnings
+import logging
+import inspect
+from StringIO import StringIO
+
+import migrate
+from migrate.versioning import genmodel, schemadiff
+from migrate.versioning.config import operations
+from migrate.versioning.template import Template
+from migrate.versioning.script import base
+from migrate.versioning.util import import_path, load_model, with_engine
+from migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError
+
+log = logging.getLogger(__name__)
+__all__ = ['PythonScript']
+
+
+class PythonScript(base.BaseScript):
+    """Base for Python scripts"""
+
+    @classmethod
+    def create(cls, path, **opts):
+        """Create an empty migration script at specified path
+        
+        :returns: :class:`PythonScript instance <migrate.versioning.script.py.PythonScript>`"""
+        cls.require_notfound(path)
+
+        src = Template(opts.pop('templates_path', None)).get_script(theme=opts.pop('templates_theme', None))
+        shutil.copy(src, path)
+
+        return cls(path)
+
+    @classmethod
+    def make_update_script_for_model(cls, engine, oldmodel,
+                                     model, repository, **opts):
+        """Create a migration script based on difference between two SA models.
+        
+        :param repository: path to migrate repository
+        :param oldmodel: dotted.module.name:SAClass or SAClass object
+        :param model: dotted.module.name:SAClass or SAClass object
+        :param engine: SQLAlchemy engine
+        :type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>`
+        :type oldmodel: string or Class
+        :type model: string or Class
+        :type engine: Engine instance
+        :returns: Upgrade / Downgrade script
+        :rtype: string
+        """
+        
+        if isinstance(repository, basestring):
+            # oh dear, an import cycle!
+            from migrate.versioning.repository import Repository
+            repository = Repository(repository)
+
+        oldmodel = load_model(oldmodel)
+        model = load_model(model)
+
+        # Compute differences.
+        diff = schemadiff.getDiffOfModelAgainstModel(
+            oldmodel,
+            model,
+            excludeTables=[repository.version_table])
+        # TODO: diff can be False (there is no difference?)
+        decls, upgradeCommands, downgradeCommands = \
+            genmodel.ModelGenerator(diff,engine).toUpgradeDowngradePython()
+
+        # Store differences into file.
+        src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None))
+        f = open(src)
+        contents = f.read()
+        f.close()
+
+        # generate source
+        search = 'def upgrade(migrate_engine):'
+        contents = contents.replace(search, '\n\n'.join((decls, search)), 1)
+        if upgradeCommands:
+            contents = contents.replace('    pass', upgradeCommands, 1)
+        if downgradeCommands:
+            contents = contents.replace('    pass', downgradeCommands, 1)
+        return contents
+
+    @classmethod
+    def verify_module(cls, path):
+        """Ensure path is a valid script
+        
+        :param path: Script location
+        :type path: string
+        :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>`
+        :returns: Python module
+        """
+        # Try to import and get the upgrade() func
+        module = import_path(path)
+        try:
+            assert callable(module.upgrade)
+        except Exception, e:
+            raise InvalidScriptError(path + ': %s' % str(e))
+        return module
+
+    def preview_sql(self, url, step, **args):
+        """Mocks SQLAlchemy Engine to store all executed calls in a string 
+        and runs :meth:`PythonScript.run <migrate.versioning.script.py.PythonScript.run>`
+
+        :returns: SQL file
+        """
+        buf = StringIO()
+        args['engine_arg_strategy'] = 'mock'
+        args['engine_arg_executor'] = lambda s, p = '': buf.write(str(s) + p)
+
+        @with_engine
+        def go(url, step, **kw):
+            engine = kw.pop('engine')
+            self.run(engine, step)
+            return buf.getvalue()
+
+        return go(url, step, **args)
+
+    def run(self, engine, step):
+        """Core method of Script file. 
+        Exectues :func:`update` or :func:`downgrade` functions
+
+        :param engine: SQLAlchemy Engine
+        :param step: Operation to run
+        :type engine: string
+        :type step: int
+        """
+        if step > 0:
+            op = 'upgrade'
+        elif step < 0:
+            op = 'downgrade'
+        else:
+            raise ScriptError("%d is not a valid step" % step)
+
+        funcname = base.operations[op]
+        script_func = self._func(funcname)
+
+        # check for old way of using engine
+        if not inspect.getargspec(script_func)[0]:
+            raise TypeError("upgrade/downgrade functions must accept engine"
+                " parameter (since version 0.5.4)")
+
+        script_func(engine)
+
+    @property
+    def module(self):
+        """Calls :meth:`migrate.versioning.script.py.verify_module`
+        and returns it.
+        """
+        if not hasattr(self, '_module'):
+            self._module = self.verify_module(self.path)
+        return self._module
+
+    def _func(self, funcname):
+        if not hasattr(self.module, funcname):
+            msg = "Function '%s' is not defined in this script"
+            raise ScriptError(msg % funcname)
+        return getattr(self.module, funcname)

libs/migrate/versioning/script/sql.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import logging
+import shutil
+
+from migrate.versioning.script import base
+from migrate.versioning.template import Template
+
+
+log = logging.getLogger(__name__)
+
+class SqlScript(base.BaseScript):
+    """A file containing plain SQL statements."""
+
+    @classmethod
+    def create(cls, path, **opts):
+        """Create an empty migration script at specified path
+        
+        :returns: :class:`SqlScript instance <migrate.versioning.script.sql.SqlScript>`"""
+        cls.require_notfound(path)
+
+        src = Template(opts.pop('templates_path', None)).get_sql_script(theme=opts.pop('templates_theme', None))
+        shutil.copy(src, path)
+        return cls(path)
+
+    # TODO: why is step parameter even here?
+    def run(self, engine, step=None, executemany=True):
+        """Runs SQL script through raw dbapi execute call"""
+        text = self.source()
+        # Don't rely on SA's autocommit here
+        # (SA uses .startswith to check if a commit is needed. What if script
+        # starts with a comment?)
+        conn = engine.connect()
+        try:
+            trans = conn.begin()
+            try:
+                # HACK: SQLite doesn't allow multiple statements through
+                # its execute() method, but it provides executescript() instead
+                dbapi = conn.engine.raw_connection()
+                if executemany and getattr(dbapi, 'executescript', None):
+                    dbapi.executescript(text)
+                else:
+                    conn.execute(text)
+                trans.commit()
+            except:
+                trans.rollback()
+                raise
+        finally:
+            conn.close()

libs/migrate/versioning/shell.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""The migrate command-line tool."""
+
+import sys
+import inspect
+import logging
+from optparse import OptionParser, BadOptionError
+
+from migrate import exceptions
+from migrate.versioning import api
+from migrate.versioning.config import *
+from migrate.versioning.util import asbool
+
+
+alias = dict(
+    s=api.script,
+    vc=api.version_control,
+    dbv=api.db_version,
+    v=api.version,
+)
+
+def alias_setup():
+    global alias
+    for key, val in alias.iteritems():
+        setattr(api, key, val)
+alias_setup()
+
+
+class PassiveOptionParser(OptionParser):
+
+    def _process_args(self, largs, rargs, values):
+        """little hack to support all --some_option=value parameters"""
+
+        while rargs:
+            arg = rargs[0]
+            if arg == "--":
+                del rargs[0]
+                return
+            elif arg[0:2] == "--":
+                # if parser does not know about the option
+                # pass it along (make it anonymous)
+                try:
+                    opt = arg.split('=', 1)[0]
+                    self._match_long_opt(opt)
+                except BadOptionError:
+                    largs.append(arg)
+                    del rargs[0]
+                else:
+                    self._process_long_opt(rargs, values)
+            elif arg[:1] == "-" and len(arg) > 1:
+                self._process_short_opts(rargs, values)
+            elif self.allow_interspersed_args:
+                largs.append(arg)
+                del rargs[0]
+
+def main(argv=None, **kwargs):
+    """Shell interface to :mod:`migrate.versioning.api`.
+
+    kwargs are default options that can be overriden with passing
+    --some_option as command line option
+
+    :param disable_logging: Let migrate configure logging
+    :type disable_logging: bool
+    """
+    if argv is not None:
+        argv = argv
+    else:
+        argv = list(sys.argv[1:])
+    commands = list(api.__all__)
+    commands.sort()
+
+    usage = """%%prog COMMAND ...
+
+    Available commands:
+        %s
+
+    Enter "%%prog help COMMAND" for information on a particular command.
+    """ % '\n\t'.join(["%s - %s" % (command.ljust(28), api.command_desc.get(command)) for command in commands])
+
+    parser = PassiveOptionParser(usage=usage)
+    parser.add_option("-d", "--debug",
+                     action="store_true",
+                     dest="debug",
+                     default=False,
+                     help="Shortcut to turn on DEBUG mode for logging")
+    parser.add_option("-q", "--disable_logging",
+                      action="store_true",
+                      dest="disable_logging",
+                      default=False,
+                      help="Use this option to disable logging configuration")
+    help_commands = ['help', '-h', '--help']
+    HELP = False
+
+    try:
+        command = argv.pop(0)
+        if command in help_commands:
+            HELP = True
+            command = argv.pop(0)
+    except IndexError:
+        parser.print_help()
+        return
+
+    command_func = getattr(api, command, None)
+    if command_func is None or command.startswith('_'):
+        parser.error("Invalid command %s" % command)
+
+    parser.set_usage(inspect.getdoc(command_func))
+    f_args, f_varargs, f_kwargs, f_defaults = inspect.getargspec(command_func)
+    for arg in f_args:
+        parser.add_option(
+            "--%s" % arg,
+            dest=arg,
+            action='store',
+            type="string")
+
+    # display help of the current command
+    if HELP:
+        parser.print_help()
+        return
+
+    options, args = parser.parse_args(argv)
+
+    # override kwargs with anonymous parameters
+    override_kwargs = dict()
+    for arg in list(args):
+        if arg.startswith('--'):
+            args.remove(arg)
+            if '=' in arg:
+                opt, value = arg[2:].split('=', 1)
+            else:
+                opt = arg[2:]
+                value = True
+            override_kwargs[opt] = value
+
+    # override kwargs with options if user is overwriting
+    for key, value in options.__dict__.iteritems():
+        if value is not None:
+            override_kwargs[key] = value
+
+    # arguments that function accepts without passed kwargs
+    f_required = list(f_args)
+    candidates = dict(kwargs)
+    candidates.update(override_kwargs)
+    for key, value in candidates.iteritems():
+        if key in f_args:
+            f_required.remove(key)
+
+    # map function arguments to parsed arguments
+    for arg in args:
+        try:
+            kw = f_required.pop(0)
+        except IndexError:
+            parser.error("Too many arguments for command %s: %s" % (command,
+                                                                    arg))
+        kwargs[kw] = arg
+
+    # apply overrides
+    kwargs.update(override_kwargs)
+
+    # configure options
+    for key, value in options.__dict__.iteritems():
+        kwargs.setdefault(key, value)
+
+    # configure logging
+    if not asbool(kwargs.pop('disable_logging', False)):
+        # filter to log =< INFO into stdout and rest to stderr
+        class SingleLevelFilter(logging.Filter):
+            def __init__(self, min=None, max=None):
+                self.min = min or 0
+                self.max = max or 100
+
+            def filter(self, record):
+                return self.min <= record.levelno <= self.max
+
+        logger = logging.getLogger()
+        h1 = logging.StreamHandler(sys.stdout)
+        f1 = SingleLevelFilter(max=logging.INFO)
+        h1.addFilter(f1)
+        h2 = logging.StreamHandler(sys.stderr)
+        f2 = SingleLevelFilter(min=logging.WARN)
+        h2.addFilter(f2)
+        logger.addHandler(h1)
+        logger.addHandler(h2)
+
+        if options.debug:
+            logger.setLevel(logging.DEBUG)
+        else:
+            logger.setLevel(logging.INFO)
+
+    log = logging.getLogger(__name__)
+
+    # check if all args are given
+    try:
+        num_defaults = len(f_defaults)
+    except TypeError:
+        num_defaults = 0
+    f_args_default = f_args[len(f_args) - num_defaults:]
+    required = list(set(f_required) - set(f_args_default))
+    if required:
+        parser.error("Not enough arguments for command %s: %s not specified" \
+            % (command, ', '.join(required)))
+
+    # handle command
+    try:
+        ret = command_func(**kwargs)
+        if ret is not None:
+            log.info(ret)
+    except (exceptions.UsageError, exceptions.KnownError), e:
+        parser.error(e.args[0])
+
+if __name__ == "__main__":
+    main()

libs/migrate/versioning/template.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import shutil
+import sys
+
+from pkg_resources import resource_filename
+
+from migrate.versioning.config import *
+from migrate.versioning import pathed
+
+
+class Collection(pathed.Pathed):
+    """A collection of templates of a specific type"""
+    _mask = None
+
+    def get_path(self, file):
+        return os.path.join(self.path, str(file))
+
+
+class RepositoryCollection(Collection):
+    _mask = '%s'
+
+class ScriptCollection(Collection):
+    _mask = '%s.py_tmpl'
+
+class ManageCollection(Collection):
+    _mask = '%s.py_tmpl'
+
+class SQLScriptCollection(Collection):
+    _mask = '%s.py_tmpl'
+
+class Template(pathed.Pathed):
+    """Finds the paths/packages of various Migrate templates.
+    
+    :param path: Templates are loaded from migrate package
+    if `path` is not provided.
+    """
+    pkg = 'migrate.versioning.templates'
+    _manage = 'manage.py_tmpl'
+
+    def __new__(cls, path=None):
+        if path is None:
+            path = cls._find_path(cls.pkg)
+        return super(Template, cls).__new__(cls, path)
+
+    def __init__(self, path=None):
+        if path is None:
+            path = Template._find_path(self.pkg)
+        super(Template, self).__init__(path)
+        self.repository = RepositoryCollection(os.path.join(path, 'repository'))
+        self.script = ScriptCollection(os.path.join(path, 'script'))
+        self.manage = ManageCollection(os.path.join(path, 'manage'))
+        self.sql_script = SQLScriptCollection(os.path.join(path, 'sql_script'))
+
+    @classmethod
+    def _find_path(cls, pkg):
+        """Returns absolute path to dotted python package."""
+        tmp_pkg = pkg.rsplit('.', 1)
+
+        if len(tmp_pkg) != 1:
+            return resource_filename(tmp_pkg[0], tmp_pkg[1])
+        else:
+            return resource_filename(tmp_pkg[0], '')
+
+    def _get_item(self, collection, theme=None):
+        """Locates and returns collection.
+        
+        :param collection: name of collection to locate
+        :param type_: type of subfolder in collection (defaults to "_default")
+        :returns: (package, source)
+        :rtype: str, str
+        """
+        item = getattr(self, collection)
+        theme_mask = getattr(item, '_mask')
+        theme = theme_mask % (theme or 'default')
+        return item.get_path(theme)
+
+    def get_repository(self, *a, **kw):
+        """Calls self._get_item('repository', *a, **kw)"""
+        return self._get_item('repository', *a, **kw)
+    
+    def get_script(self, *a, **kw):
+        """Calls self._get_item('script', *a, **kw)"""
+        return self._get_item('script', *a, **kw)
+
+    def get_sql_script(self, *a, **kw):
+        """Calls self._get_item('sql_script', *a, **kw)"""
+        return self._get_item('sql_script', *a, **kw)
+
+    def get_manage(self, *a, **kw):
+        """Calls self._get_item('manage', *a, **kw)"""
+        return self._get_item('manage', *a, **kw)

libs/migrate/versioning/templates/manage.py_tmpl

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(%(defaults)s)

libs/migrate/versioning/templates/manage/default.py_tmpl

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+{{py:
+_vars = locals().copy()
+del _vars['__template_name__']
+_vars.pop('repository_name', None)
+defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
+}}
+main({{ defaults }})

libs/migrate/versioning/templates/manage/pylons.py_tmpl

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import sys
+
+from sqlalchemy import engine_from_config
+from paste.deploy.loadwsgi import ConfigLoader
+
+from migrate.versioning.shell import main
+from {{ locals().pop('repository_name') }}.model import migrations
+
+
+if '-c' in sys.argv:
+    pos = sys.argv.index('-c')
+    conf_path = sys.argv[pos + 1]
+    del sys.argv[pos:pos + 2]
+else:
+    conf_path = 'development.ini'
+
+{{py:
+_vars = locals().copy()
+del _vars['__template_name__']
+defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
+}}
+
+conf_dict = ConfigLoader(conf_path).parser._sections['app:main']
+
+# migrate supports passing url as an existing Engine instance (since 0.6.0)
+# usage: migrate -c path/to/config.ini COMMANDS
+main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }})

libs/migrate/versioning/templates/repository/default/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

libs/migrate/versioning/templates/repository/default/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id={{ locals().pop('repository_id') }}
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table={{ locals().pop('version_table') }}
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs={{ locals().pop('required_dbs') }}

libs/migrate/versioning/templates/repository/pylons/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

libs/migrate/versioning/templates/repository/pylons/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id={{ locals().pop('repository_id') }}
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table={{ locals().pop('version_table') }}
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs={{ locals().pop('required_dbs') }}

libs/migrate/versioning/templates/script/default.py_tmpl

@@ -0,0 +1,11 @@
+from sqlalchemy import *
+from migrate import *
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind migrate_engine
+    # to your metadata
+    pass
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    pass

libs/migrate/versioning/templates/script/pylons.py_tmpl

@@ -0,0 +1,11 @@
+from sqlalchemy import *
+from migrate import *
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind migrate_engine
+    # to your metadata
+    pass
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    pass

libs/migrate/versioning/util/__init__.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+""".. currentmodule:: migrate.versioning.util"""
+
+import warnings
+import logging
+from decorator import decorator
+from pkg_resources import EntryPoint
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+from sqlalchemy.pool import StaticPool
+
+from migrate import exceptions
+from migrate.versioning.util.keyedinstance import KeyedInstance
+from migrate.versioning.util.importpath import import_path
+
+
+log = logging.getLogger(__name__)
+
+def load_model(dotted_name):
+    """Import module and use module-level variable".
+
+    :param dotted_name: path to model in form of string: ``some.python.module:Class``
+    
+    .. versionchanged:: 0.5.4
+
+    """
+    if isinstance(dotted_name, basestring):
+        if ':' not in dotted_name:
+            # backwards compatibility
+            warnings.warn('model should be in form of module.model:User '
+                'and not module.model.User', exceptions.MigrateDeprecationWarning)
+            dotted_name = ':'.join(dotted_name.rsplit('.', 1))
+        return EntryPoint.parse('x=%s' % dotted_name).load(False)
+    else:
+        # Assume it's already loaded.
+        return dotted_name
+
+def asbool(obj):
+    """Do everything to use object as bool"""
+    if isinstance(obj, basestring):
+        obj = obj.strip().lower()
+        if obj in ['true', 'yes', 'on', 'y', 't', '1']:
+            return True
+        elif obj in ['false', 'no', 'off', 'n', 'f', '0']:
+            return False
+        else:
+            raise ValueError("String is not true/false: %r" % obj)
+    if obj in (True, False):
+        return bool(obj)
+    else:
+        raise ValueError("String is not true/false: %r" % obj)
+
+def guess_obj_type(obj):
+    """Do everything to guess object type from string
+    
+    Tries to convert to `int`, `bool` and finally returns if not succeded.
+    
+    .. versionadded: 0.5.4
+    """
+
+    result = None
+
+    try:
+        result = int(obj)
+    except:
+        pass
+
+    if result is None:
+        try:
+            result = asbool(obj)
+        except:
+            pass
+
+    if result is not None:
+        return result
+    else:
+        return obj
+
+@decorator
+def catch_known_errors(f, *a, **kw):
+    """Decorator that catches known api errors
+    
+    .. versionadded: 0.5.4
+    """
+
+    try:
+        return f(*a, **kw)
+    except exceptions.PathFoundError, e:
+        raise exceptions.KnownError("The path %s already exists" % e.args[0])
+
+def construct_engine(engine, **opts):
+    """.. versionadded:: 0.5.4
+
+    Constructs and returns SQLAlchemy engine.
+
+    Currently, there are 2 ways to pass create_engine options to :mod:`migrate.versioning.api` functions:
+
+    :param engine: connection string or a existing engine
+    :param engine_dict: python dictionary of options to pass to `create_engine`
+    :param engine_arg_*: keyword parameters to pass to `create_engine` (evaluated with :func:`migrate.versioning.util.guess_obj_type`)
+    :type engine_dict: dict
+    :type engine: string or Engine instance
+    :type engine_arg_*: string
+    :returns: SQLAlchemy Engine
+
+    .. note::
+
+        keyword parameters override ``engine_dict`` values.
+
+    """
+    if isinstance(engine, Engine):
+        return engine
+    elif not isinstance(engine, basestring):
+        raise ValueError("you need to pass either an existing engine or a database uri")
+
+    # get options for create_engine
+    if opts.get('engine_dict') and isinstance(opts['engine_dict'], dict):
+        kwargs = opts['engine_dict']
+    else:
+        kwargs = dict()
+
+    # DEPRECATED: handle echo the old way
+    echo = asbool(opts.get('echo', False))
+    if echo:
+        warnings.warn('echo=True parameter is deprecated, pass '
+            'engine_arg_echo=True or engine_dict={"echo": True}',
+            exceptions.MigrateDeprecationWarning)
+        kwargs['echo'] = echo
+
+    # parse keyword arguments
+    for key, value in opts.iteritems():
+        if key.startswith('engine_arg_'):
+            kwargs[key[11:]] = guess_obj_type(value)
+
+    log.debug('Constructing engine')
+    # TODO: return create_engine(engine, poolclass=StaticPool, **kwargs)
+    # seems like 0.5.x branch does not work with engine.dispose and staticpool
+    return create_engine(engine, **kwargs)
+
+@decorator
+def with_engine(f, *a, **kw):
+    """Decorator for :mod:`migrate.versioning.api` functions
+    to safely close resources after function usage.
+
+    Passes engine parameters to :func:`construct_engine` and
+    resulting parameter is available as kw['engine'].
+
+    Engine is disposed after wrapped function is executed.
+
+    .. versionadded: 0.6.0
+    """
+    url = a[0]
+    engine = construct_engine(url, **kw)
+
+    try:
+        kw['engine'] = engine
+        return f(*a, **kw)
+    finally:
+        if isinstance(engine, Engine):
+            log.debug('Disposing SQLAlchemy engine %s', engine)
+            engine.dispose()
+
+
+class Memoize:
+    """Memoize(fn) - an instance which acts like fn but memoizes its arguments
+       Will only work on functions with non-mutable arguments
+
+       ActiveState Code 52201
+    """
+    def __init__(self, fn):
+        self.fn = fn
+        self.memo = {}
+
+    def __call__(self, *args):
+        if not self.memo.has_key(args):
+            self.memo[args] = self.fn(*args)
+        return self.memo[args]

libs/migrate/versioning/util/importpath.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+def import_path(fullpath):
+    """ Import a file with full path specification. Allows one to
+        import from anywhere, something __import__ does not do. 
+    """
+    # http://zephyrfalcon.org/weblog/arch_d7_2002_08_31.html
+    path, filename = os.path.split(fullpath)
+    filename, ext = os.path.splitext(filename)
+    sys.path.append(path)
+    module = __import__(filename)
+    reload(module) # Might be out of date during tests
+    del sys.path[-1]
+    return module
+

libs/migrate/versioning/util/keyedinstance.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+class KeyedInstance(object):
+    """A class whose instances have a unique identifier of some sort
+    No two instances with the same unique ID should exist - if we try to create
+    a second instance, the first should be returned. 
+    """
+
+    _instances = dict()
+
+    def __new__(cls, *p, **k):
+        instances = cls._instances
+        clskey = str(cls)
+        if clskey not in instances:
+            instances[clskey] = dict()
+        instances = instances[clskey]
+
+        key = cls._key(*p, **k)
+        if key not in instances:
+            instances[key] = super(KeyedInstance, cls).__new__(cls)
+        return instances[key]
+
+    @classmethod
+    def _key(cls, *p, **k):
+        """Given a unique identifier, return a dictionary key
+        This should be overridden by child classes, to specify which parameters 
+        should determine an object's uniqueness
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def clear(cls):
+        # Allow cls.clear() as well as uniqueInstance.clear(cls)
+        if str(cls) in cls._instances:
+            del cls._instances[str(cls)]

libs/migrate/versioning/version.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import shutil
+import logging
+
+from migrate import exceptions
+from migrate.versioning import pathed, script
+
+
+log = logging.getLogger(__name__)
+
+class VerNum(object):
+    """A version number that behaves like a string and int at the same time"""
+
+    _instances = dict()
+
+    def __new__(cls, value):
+        val = str(value)
+        if val not in cls._instances:
+            cls._instances[val] = super(VerNum, cls).__new__(cls)
+        ret = cls._instances[val]
+        return ret
+
+    def __init__(self,value):
+        self.value = str(int(value))
+        if self < 0:
+            raise ValueError("Version number cannot be negative")
+
+    def __add__(self, value):
+        ret = int(self) + int(value)
+        return VerNum(ret)
+
+    def __sub__(self, value):
+        return self + (int(value) * -1)
+
+    def __cmp__(self, value):
+        return int(self) - int(value)
+
+    def __repr__(self):
+        return "<VerNum(%s)>" % self.value
+
+    def __str__(self):
+        return str(self.value)
+
+    def __int__(self):
+        return int(self.value)
+
+
+class Collection(pathed.Pathed):
+    """A collection of versioning scripts in a repository"""
+
+    FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
+
+    def __init__(self, path):
+        """Collect current version scripts in repository
+        and store them in self.versions
+        """
+        super(Collection, self).__init__(path)
+        
+        # Create temporary list of files, allowing skipped version numbers.
+        files = os.listdir(path)
+        if '1' in files:
+            # deprecation
+            raise Exception('It looks like you have a repository in the old '
+                'format (with directories for each version). '
+                'Please convert repository before proceeding.')
+
+        tempVersions = dict()
+        for filename in files:
+            match = self.FILENAME_WITH_VERSION.match(filename)
+            if match:
+                num = int(match.group(1))
+                tempVersions.setdefault(num, []).append(filename)
+            else:
+                pass  # Must be a helper file or something, let's ignore it.
+
+        # Create the versions member where the keys
+        # are VerNum's and the values are Version's.
+        self.versions = dict()
+        for num, files in tempVersions.items():
+            self.versions[VerNum(num)] = Version(num, path, files)
+
+    @property
+    def latest(self):
+        """:returns: Latest version in Collection"""
+        return max([VerNum(0)] + self.versions.keys())
+
+    def create_new_python_version(self, description, **k):
+        """Create Python files for new version"""
+        ver = self.latest + 1
+        extra = str_to_filename(description)
+
+        if extra:
+            if extra == '_':
+                extra = ''
+            elif not extra.startswith('_'):
+                extra = '_%s' % extra
+
+        filename = '%03d%s.py' % (ver, extra)
+        filepath = self._version_path(filename)
+
+        script.PythonScript.create(filepath, **k)
+        self.versions[ver] = Version(ver, self.path, [filename])
+        
+    def create_new_sql_version(self, database, **k):
+        """Create SQL files for new version"""
+        ver = self.latest + 1
+        self.versions[ver] = Version(ver, self.path, [])
+
+        # Create new files.
+        for op in ('upgrade', 'downgrade'):
+            filename = '%03d_%s_%s.sql' % (ver, database, op)
+            filepath = self._version_path(filename)
+            script.SqlScript.create(filepath, **k)
+            self.versions[ver].add_script(filepath)
+        
+    def version(self, vernum=None):
+        """Returns latest Version if vernum is not given.
+        Otherwise, returns wanted version"""
+        if vernum is None:
+            vernum = self.latest
+        return self.versions[VerNum(vernum)]
+
+    @classmethod
+    def clear(cls):
+        super(Collection, cls).clear()
+
+    def _version_path(self, ver):
+        """Returns path of file in versions repository"""
+        return os.path.join(self.path, str(ver))
+
+
+class Version(object):
+    """A single version in a collection
+    :param vernum: Version Number 
+    :param path: Path to script files
+    :param filelist: List of scripts
+    :type vernum: int, VerNum
+    :type path: string
+    :type filelist: list
+    """
+
+    def __init__(self, vernum, path, filelist):
+        self.version = VerNum(vernum)
+
+        # Collect scripts in this folder
+        self.sql = dict()
+        self.python = None
+
+        for script in filelist:
+            self.add_script(os.path.join(path, script))
+    
+    def script(self, database=None, operation=None):
+        """Returns SQL or Python Script"""
+        for db in (database, 'default'):
+            # Try to return a .sql script first
+            try:
+                return self.sql[db][operation]
+            except KeyError:
+                continue  # No .sql script exists
+
+        # TODO: maybe add force Python parameter?
+        ret = self.python
+
+        assert ret is not None, \
+            "There is no script for %d version" % self.version
+        return ret
+
+    def add_script(self, path):
+        """Add script to Collection/Version"""
+        if path.endswith(Extensions.py):
+            self._add_script_py(path)
+        elif path.endswith(Extensions.sql):
+            self._add_script_sql(path)
+
+    SQL_FILENAME = re.compile(r'^(\d+)_([^_]+)_([^_]+).sql')
+
+    def _add_script_sql(self, path):
+        basename = os.path.basename(path)
+        match = self.SQL_FILENAME.match(basename)
+
+        if match:
+            version, dbms, op = match.group(1), match.group(2), match.group(3)
+        else:
+            raise exceptions.ScriptError(
+                "Invalid SQL script name %s " % basename + \
+                "(needs to be ###_database_operation.sql)")
+
+        # File the script into a dictionary
+        self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
+
+    def _add_script_py(self, path):
+        if self.python is not None:
+            raise exceptions.ScriptError('You can only have one Python script '
+                'per version, but you have: %s and %s' % (self.python, path))
+        self.python = script.PythonScript(path)
+
+
+class Extensions:
+    """A namespace for file extensions"""
+    py = 'py'
+    sql = 'sql'
+
+def str_to_filename(s):
+    """Replaces spaces, (double and single) quotes
+    and double underscores to underscores
+    """
+
+    s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
+    while '__' in s:
+        s = s.replace('__', '_')
+    return s