From 1120b4ab510f1ce11b3fb923f235eaa26f884937 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 26 Jan 2014 16:29:16 +0100 Subject: [PATCH] Remove Elixir library Update SQLAlchemy --- couchpotato/core/settings/model.py | 454 +- couchpotato/runner.py | 2 +- libs/elixir/__init__.py | 114 - libs/elixir/collection.py | 125 - libs/elixir/entity.py | 1039 --- libs/elixir/events.py | 27 - libs/elixir/ext/__init__.py | 5 - libs/elixir/ext/associable.py | 234 - libs/elixir/ext/encrypted.py | 124 - libs/elixir/ext/perform_ddl.py | 106 - libs/elixir/ext/versioned.py | 288 - libs/elixir/fields.py | 191 - libs/elixir/options.py | 274 - libs/elixir/properties.py | 244 - libs/elixir/relationships.py | 1247 ---- libs/elixir/statements.py | 59 - libs/sqlalchemy/__init__.py | 41 +- libs/sqlalchemy/cextension/processors.c | 245 +- libs/sqlalchemy/cextension/resultproxy.c | 116 +- libs/sqlalchemy/cextension/utils.c | 225 + libs/sqlalchemy/connectors/__init__.py | 3 +- libs/sqlalchemy/connectors/mxodbc.py | 43 +- libs/sqlalchemy/connectors/mysqldb.py | 29 +- libs/sqlalchemy/connectors/pyodbc.py | 53 +- libs/sqlalchemy/connectors/zxJDBC.py | 5 +- libs/sqlalchemy/databases/__init__.py | 24 +- libs/sqlalchemy/dialects/__init__.py | 33 +- libs/sqlalchemy/dialects/access/__init__.py | 0 libs/sqlalchemy/dialects/access/base.py | 451 -- libs/sqlalchemy/dialects/drizzle/base.py | 27 +- libs/sqlalchemy/dialects/drizzle/mysqldb.py | 17 +- libs/sqlalchemy/dialects/firebird/__init__.py | 8 +- libs/sqlalchemy/dialects/firebird/base.py | 172 +- libs/sqlalchemy/dialects/firebird/fdb.py | 115 + .../dialects/firebird/kinterbasdb.py | 96 +- libs/sqlalchemy/dialects/informix/__init__.py | 9 - libs/sqlalchemy/dialects/informix/base.py | 596 -- .../dialects/informix/informixdb.py | 73 - libs/sqlalchemy/dialects/maxdb/__init__.py | 9 - libs/sqlalchemy/dialects/maxdb/base.py | 1117 ---- libs/sqlalchemy/dialects/maxdb/sapdb.py | 23 - libs/sqlalchemy/dialects/mssql/__init__.py | 4 +- libs/sqlalchemy/dialects/mssql/adodbapi.py | 24 +- libs/sqlalchemy/dialects/mssql/base.py | 689 +- .../dialects/mssql/information_schema.py | 30 +- libs/sqlalchemy/dialects/mssql/mxodbc.py | 82 +- libs/sqlalchemy/dialects/mssql/pymssql.py | 44 +- libs/sqlalchemy/dialects/mssql/pyodbc.py | 48 +- libs/sqlalchemy/dialects/mssql/zxjdbc.py | 36 +- libs/sqlalchemy/dialects/mysql/__init__.py | 8 +- libs/sqlalchemy/dialects/mysql/base.py | 738 ++- libs/sqlalchemy/dialects/mysql/cymysql.py | 69 + libs/sqlalchemy/dialects/mysql/gaerdbms.py | 58 +- .../dialects/mysql/mysqlconnector.py | 47 +- libs/sqlalchemy/dialects/mysql/mysqldb.py | 26 +- libs/sqlalchemy/dialects/mysql/oursql.py | 127 +- libs/sqlalchemy/dialects/mysql/pymysql.py | 33 +- libs/sqlalchemy/dialects/mysql/pyodbc.py | 26 +- libs/sqlalchemy/dialects/mysql/zxjdbc.py | 32 +- libs/sqlalchemy/dialects/oracle/__init__.py | 4 +- libs/sqlalchemy/dialects/oracle/base.py | 479 +- libs/sqlalchemy/dialects/oracle/cx_oracle.py | 302 +- libs/sqlalchemy/dialects/oracle/zxjdbc.py | 38 +- libs/sqlalchemy/dialects/postgres.py | 2 +- .../dialects/postgresql/__init__.py | 27 +- libs/sqlalchemy/dialects/postgresql/base.py | 1319 ++-- .../dialects/postgresql/constraints.py | 73 + libs/sqlalchemy/dialects/postgresql/hstore.py | 369 ++ libs/sqlalchemy/dialects/postgresql/json.py | 199 + libs/sqlalchemy/dialects/postgresql/pg8000.py | 43 +- .../dialects/postgresql/psycopg2.py | 295 +- .../dialects/postgresql/pypostgresql.py | 31 +- libs/sqlalchemy/dialects/postgresql/ranges.py | 160 + libs/sqlalchemy/dialects/postgresql/zxjdbc.py | 21 +- libs/sqlalchemy/dialects/sqlite/__init__.py | 9 +- libs/sqlalchemy/dialects/sqlite/base.py | 365 +- libs/sqlalchemy/dialects/sqlite/pysqlite.py | 127 +- libs/sqlalchemy/dialects/sybase/__init__.py | 29 +- libs/sqlalchemy/dialects/sybase/base.py | 504 +- libs/sqlalchemy/dialects/sybase/mxodbc.py | 19 +- libs/sqlalchemy/dialects/sybase/pyodbc.py | 25 +- libs/sqlalchemy/dialects/sybase/pysybase.py | 22 +- libs/sqlalchemy/engine/__init__.py | 149 +- libs/sqlalchemy/engine/base.py | 2455 +------ libs/sqlalchemy/engine/ddl.py | 182 - libs/sqlalchemy/engine/default.py | 276 +- libs/sqlalchemy/engine/interfaces.py | 846 +++ libs/sqlalchemy/engine/reflection.py | 280 +- libs/sqlalchemy/engine/result.py | 1018 +++ libs/sqlalchemy/engine/strategies.py | 75 +- libs/sqlalchemy/engine/threadlocal.py | 34 +- libs/sqlalchemy/engine/url.py | 135 +- libs/sqlalchemy/engine/util.py | 72 + libs/sqlalchemy/event.py | 460 -- libs/sqlalchemy/event/__init__.py | 10 + libs/sqlalchemy/event/api.py | 107 + libs/sqlalchemy/event/attr.py | 376 ++ libs/sqlalchemy/event/base.py | 217 + libs/sqlalchemy/event/legacy.py | 156 + libs/sqlalchemy/event/registry.py | 236 + libs/sqlalchemy/events.py | 472 +- libs/sqlalchemy/exc.py | 113 +- libs/sqlalchemy/ext/__init__.py | 3 +- libs/sqlalchemy/ext/associationproxy.py | 203 +- libs/sqlalchemy/ext/automap.py | 840 +++ libs/sqlalchemy/ext/compiler.py | 148 +- .../__init__.py} | 1148 +--- libs/sqlalchemy/ext/declarative/api.py | 512 ++ libs/sqlalchemy/ext/declarative/base.py | 506 ++ .../sqlalchemy/ext/declarative/clsregistry.py | 305 + libs/sqlalchemy/ext/horizontal_shard.py | 42 +- libs/sqlalchemy/ext/hybrid.py | 251 +- libs/sqlalchemy/ext/instrumentation.py | 407 ++ libs/sqlalchemy/ext/mutable.py | 194 +- libs/sqlalchemy/ext/orderinglist.py | 14 +- libs/sqlalchemy/ext/serializer.py | 69 +- libs/sqlalchemy/ext/sqlsoup.py | 811 --- libs/sqlalchemy/inspection.py | 92 + libs/sqlalchemy/interfaces.py | 23 +- libs/sqlalchemy/log.py | 18 +- libs/sqlalchemy/orm/__init__.py | 1602 +---- libs/sqlalchemy/orm/attributes.py | 658 +- libs/sqlalchemy/orm/base.py | 453 ++ libs/sqlalchemy/orm/collections.py | 596 +- libs/sqlalchemy/orm/dependency.py | 92 +- libs/sqlalchemy/orm/deprecated_interfaces.py | 24 +- libs/sqlalchemy/orm/descriptor_props.py | 317 +- libs/sqlalchemy/orm/dynamic.py | 212 +- libs/sqlalchemy/orm/evaluator.py | 40 +- libs/sqlalchemy/orm/events.py | 752 ++- libs/sqlalchemy/orm/exc.py | 63 +- libs/sqlalchemy/orm/identity.py | 90 +- libs/sqlalchemy/orm/instrumentation.py | 507 +- libs/sqlalchemy/orm/interfaces.py | 672 +- libs/sqlalchemy/orm/loading.py | 610 ++ libs/sqlalchemy/orm/mapper.py | 1694 +++-- libs/sqlalchemy/orm/path_registry.py | 261 + libs/sqlalchemy/orm/persistence.py | 432 +- libs/sqlalchemy/orm/properties.py | 1554 +---- libs/sqlalchemy/orm/query.py | 2040 +++--- libs/sqlalchemy/orm/relationships.py | 2511 +++++++ libs/sqlalchemy/orm/scoping.py | 105 +- libs/sqlalchemy/orm/session.py | 1196 ++-- libs/sqlalchemy/orm/shard.py | 15 - libs/sqlalchemy/orm/state.py | 536 +- libs/sqlalchemy/orm/strategies.py | 935 +-- libs/sqlalchemy/orm/strategy_options.py | 924 +++ libs/sqlalchemy/orm/sync.py | 58 +- libs/sqlalchemy/orm/unitofwork.py | 163 +- libs/sqlalchemy/orm/util.py | 991 +-- libs/sqlalchemy/pool.py | 418 +- libs/sqlalchemy/processors.py | 24 +- libs/sqlalchemy/schema.py | 3242 +-------- libs/sqlalchemy/sql/__init__.py | 30 +- libs/sqlalchemy/sql/annotation.py | 182 + libs/sqlalchemy/sql/base.py | 348 + libs/sqlalchemy/sql/compiler.py | 1651 +++-- libs/sqlalchemy/sql/ddl.py | 864 +++ libs/sqlalchemy/sql/default_comparator.py | 278 + libs/sqlalchemy/sql/dml.py | 770 +++ libs/sqlalchemy/sql/elements.py | 2880 ++++++++ libs/sqlalchemy/sql/expression.py | 5821 +---------------- libs/sqlalchemy/sql/functions.py | 482 +- libs/sqlalchemy/sql/operators.py | 364 +- libs/sqlalchemy/sql/schema.py | 3150 +++++++++ libs/sqlalchemy/sql/selectable.py | 3001 +++++++++ libs/sqlalchemy/sql/sqltypes.py | 1628 +++++ libs/sqlalchemy/sql/type_api.py | 1053 +++ libs/sqlalchemy/sql/util.py | 587 +- libs/sqlalchemy/sql/visitors.py | 96 +- libs/sqlalchemy/types.py | 2467 +------ libs/sqlalchemy/util/__init__.py | 39 +- libs/sqlalchemy/util/_collections.py | 366 +- libs/sqlalchemy/util/compat.py | 338 +- libs/sqlalchemy/util/deprecations.py | 51 +- libs/sqlalchemy/util/langhelpers.py | 723 +- libs/sqlalchemy/util/queue.py | 43 +- libs/sqlalchemy/util/topological.py | 14 +- 178 files changed, 43610 insertions(+), 35304 deletions(-) delete mode 100644 libs/elixir/__init__.py delete mode 100644 libs/elixir/collection.py delete mode 100644 libs/elixir/entity.py delete mode 100644 libs/elixir/events.py delete mode 100644 libs/elixir/ext/__init__.py delete mode 100644 libs/elixir/ext/associable.py delete mode 100644 libs/elixir/ext/encrypted.py delete mode 100644 libs/elixir/ext/perform_ddl.py delete mode 100644 libs/elixir/ext/versioned.py delete mode 100644 libs/elixir/fields.py delete mode 100644 libs/elixir/options.py delete mode 100644 libs/elixir/properties.py delete mode 100644 libs/elixir/relationships.py delete mode 100644 libs/elixir/statements.py create mode 100644 libs/sqlalchemy/cextension/utils.c delete mode 100644 libs/sqlalchemy/dialects/access/__init__.py delete mode 100644 libs/sqlalchemy/dialects/access/base.py create mode 100644 libs/sqlalchemy/dialects/firebird/fdb.py delete mode 100644 libs/sqlalchemy/dialects/informix/__init__.py delete mode 100644 libs/sqlalchemy/dialects/informix/base.py delete mode 100644 libs/sqlalchemy/dialects/informix/informixdb.py delete mode 100644 libs/sqlalchemy/dialects/maxdb/__init__.py delete mode 100644 libs/sqlalchemy/dialects/maxdb/base.py delete mode 100644 libs/sqlalchemy/dialects/maxdb/sapdb.py create mode 100644 libs/sqlalchemy/dialects/mysql/cymysql.py create mode 100644 libs/sqlalchemy/dialects/postgresql/constraints.py create mode 100644 libs/sqlalchemy/dialects/postgresql/hstore.py create mode 100644 libs/sqlalchemy/dialects/postgresql/json.py create mode 100644 libs/sqlalchemy/dialects/postgresql/ranges.py delete mode 100644 libs/sqlalchemy/engine/ddl.py create mode 100644 libs/sqlalchemy/engine/interfaces.py create mode 100644 libs/sqlalchemy/engine/result.py create mode 100644 libs/sqlalchemy/engine/util.py delete mode 100644 libs/sqlalchemy/event.py create mode 100644 libs/sqlalchemy/event/__init__.py create mode 100644 libs/sqlalchemy/event/api.py create mode 100644 libs/sqlalchemy/event/attr.py create mode 100644 libs/sqlalchemy/event/base.py create mode 100644 libs/sqlalchemy/event/legacy.py create mode 100644 libs/sqlalchemy/event/registry.py create mode 100644 libs/sqlalchemy/ext/automap.py rename libs/sqlalchemy/ext/{declarative.py => declarative/__init__.py} (51%) mode change 100755 => 100644 create mode 100644 libs/sqlalchemy/ext/declarative/api.py create mode 100644 libs/sqlalchemy/ext/declarative/base.py create mode 100644 libs/sqlalchemy/ext/declarative/clsregistry.py create mode 100644 libs/sqlalchemy/ext/instrumentation.py delete mode 100644 libs/sqlalchemy/ext/sqlsoup.py create mode 100644 libs/sqlalchemy/inspection.py create mode 100644 libs/sqlalchemy/orm/base.py create mode 100644 libs/sqlalchemy/orm/loading.py create mode 100644 libs/sqlalchemy/orm/path_registry.py create mode 100644 libs/sqlalchemy/orm/relationships.py delete mode 100644 libs/sqlalchemy/orm/shard.py create mode 100644 libs/sqlalchemy/orm/strategy_options.py create mode 100644 libs/sqlalchemy/sql/annotation.py create mode 100644 libs/sqlalchemy/sql/base.py create mode 100644 libs/sqlalchemy/sql/ddl.py create mode 100644 libs/sqlalchemy/sql/default_comparator.py create mode 100644 libs/sqlalchemy/sql/dml.py create mode 100644 libs/sqlalchemy/sql/elements.py create mode 100644 libs/sqlalchemy/sql/schema.py create mode 100644 libs/sqlalchemy/sql/selectable.py create mode 100644 libs/sqlalchemy/sql/sqltypes.py create mode 100644 libs/sqlalchemy/sql/type_api.py diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index ef6e8a5c..09d46afa 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -1,23 +1,20 @@ +import uuid +import datetime + +from sqlalchemy import Column, ForeignKey, Table, Index +from sqlalchemy.ext.associationproxy import AssociationProxy +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import relationship, object_mapper, ColumnProperty, class_mapper +from sqlalchemy.orm.exc import UnmappedInstanceError +from sqlalchemy.orm.query import Query +from sqlalchemy.ext.declarative import declarative_base from couchpotato.core.helpers.encoding import toUnicode -from elixir.entity import Entity -from elixir.fields import Field -from elixir.options import options_defaults, using_options -from elixir.relationships import ManyToMany, OneToMany, ManyToOne from sqlalchemy.ext.mutable import Mutable from sqlalchemy.types import Integer, Unicode, UnicodeText, Boolean, String, \ TypeDecorator import json import time -options_defaults["shortnames"] = True - -# We would like to be able to create this schema in a specific database at -# will, so we can test it easily. -# Make elixir not bind to any session to make this possible. -# -# http://elixir.ematia.de/trac/wiki/Recipes/MultipleDatabasesOneMetadata -__session__ = None - class SetEncoder(json.JSONEncoder): def default(self, obj): @@ -73,74 +70,227 @@ class MutableDict(Mutable, dict): MutableDict.associate_with(JsonType) +Base = declarative_base() + +COLUMN_BLACKLIST = ('_sa_polymorphic_on', ) + +def is_mapped_class(cls): + try: + class_mapper(cls) + return True + except: + return False + +def is_like_list(instance, relation): + """Returns ``True`` if and only if the relation of `instance` whose name is +`relation` is list-like. + +A relation may be like a list if, for example, it is a non-lazy one-to-many +relation, or it is a dynamically loaded one-to-many. + +""" + if relation in instance._sa_class_manager: + return instance._sa_class_manager[relation].property.uselist + related_value = getattr(type(instance), relation, None) + return isinstance(related_value, AssociationProxy) + +class TableHelper(): + def to_dict(self, deep = None, exclude = None, include = None, + exclude_relations = None, include_relations = None, + include_methods = None): + instance = self + + if (exclude is not None or exclude_relations is not None) and \ + (include is not None or include_relations is not None): + raise ValueError('Cannot specify both include and exclude.') + # create a list of names of columns, including hybrid properties + try: + columns = [p.key for p in object_mapper(instance).iterate_properties + if isinstance(p, ColumnProperty)] + except UnmappedInstanceError: + return instance + for parent in type(instance).mro(): + columns += [key for key, value in parent.__dict__.items() + if isinstance(value, hybrid_property)] + # filter the columns based on exclude and include values + if exclude is not None: + columns = (c for c in columns if c not in exclude) + elif include is not None: + columns = (c for c in columns if c in include) + # create a dictionary mapping column name to value + result = dict((col, getattr(instance, col)) for col in columns + if not (col.startswith('__') or col in COLUMN_BLACKLIST)) + # add any included methods + if include_methods is not None: + result.update(dict((method, getattr(instance, method)()) for method in include_methods if not '.' in method)) + # Check for objects in the dictionary that may not be serializable by + # default. Specifically, convert datetime and date objects to ISO 8601 + # format, and convert UUID objects to hexadecimal strings. + for key, value in result.items(): + # TODO We can get rid of this when issue #33 is resolved. + if isinstance(value, datetime.date): + result[key] = value.isoformat() + elif isinstance(value, uuid.UUID): + result[key] = str(value) + elif is_mapped_class(type(value)): + result[key] = value.to_dict() + # recursively call _to_dict on each of the `deep` relations + deep = deep or {} + for relation, rdeep in deep.items(): + # Get the related value so we can see if it is None, a list, a query + # (as specified by a dynamic relationship loader), or an actual + # instance of a model. + relatedvalue = getattr(instance, relation) + if relatedvalue is None: + result[relation] = None + continue + # Determine the included and excluded fields for the related model. + newexclude = None + newinclude = None + if exclude_relations is not None and relation in exclude_relations: + newexclude = exclude_relations[relation] + elif (include_relations is not None and + relation in include_relations): + newinclude = include_relations[relation] + # Determine the included methods for the related model. + newmethods = None + if include_methods is not None: + newmethods = [method.split('.', 1)[1] for method in include_methods + if method.split('.', 1)[0] == relation] + if is_like_list(instance, relation): + result[relation] = [inst.to_dict(rdeep, exclude = newexclude, + include = newinclude, + include_methods = newmethods) + for inst in relatedvalue] + continue + # If the related value is dynamically loaded, resolve the query to get + # the single instance. + if isinstance(relatedvalue, Query): + relatedvalue = relatedvalue.one() + result[relation] = relatedvalue.to_dict(rdeep, exclude = newexclude, + include = newinclude, + include_methods = newmethods) + + return result + + +movie_files = Table('movie_files__file_movie', Base.metadata, + Column('movie_id', Integer, ForeignKey('movie.id'), nullable = False), + Column('file_id', Integer, ForeignKey('file.id'), nullable = False), + Index('movie_files_idx', 'movie_id', 'file_id', unique = True) +) + +release_files = Table('release_files__file_release', Base.metadata, + Column('release_id', Integer, ForeignKey('release.id'), nullable = False), + Column('file_id', Integer, ForeignKey('file.id'), nullable = False), + Index('release_files_idx', 'release_id', 'file_id', unique = True) +) + +library_files = Table('library_files__file_library', Base.metadata, + Column('library_id', Integer, ForeignKey('library.id'), nullable = False), + Column('file_id', Integer, ForeignKey('file.id'), nullable = False), + Index('library_files_idx', 'library_id', 'file_id', unique = True) +) + +class Movie(Base, TableHelper): + __tablename__ = 'movie' + id = Column(Integer, primary_key = True) -class Movie(Entity): """Movie Resource a movie could have multiple releases The files belonging to the movie object are global for the whole movie such as trailers, nfo, thumbnails""" - last_edit = Field(Integer, default = lambda: int(time.time()), index = True) + last_edit = Column(Integer, default = lambda: int(time.time()), index = True) type = 'movie' # Compat tv branch - library = ManyToOne('Library', cascade = 'delete, delete-orphan', single_parent = True) - status = ManyToOne('Status') - profile = ManyToOne('Profile') - category = ManyToOne('Category') - releases = OneToMany('Release', cascade = 'all, delete-orphan') - files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) + library_id = Column(Integer, ForeignKey('library.id'), index = True) + status_id = Column(Integer, ForeignKey('status.id'), index = True) + profile_id = Column(Integer, ForeignKey('profile.id'), index = True) + category_id = Column(Integer, ForeignKey('category.id'), index = True) + + library = relationship('Library') #cascade = 'delete, delete-orphan', single_parent = True) + status = relationship('Status') + profile = relationship('Profile') + category = relationship('Category') + releases = relationship('Release') #, cascade = 'all, delete-orphan') + files = relationship('File', secondary = movie_files) #, cascade = 'all, delete-orphan', single_parent = True) Media = Movie # Compat tv branch -class Library(Entity): +class Library(Base, TableHelper): + __tablename__ = 'library' + id = Column(Integer, primary_key = True) + """""" - year = Field(Integer) - identifier = Field(String(20), index = True) + year = Column(Integer) + identifier = Column(String(20), index = True) - plot = Field(UnicodeText) - tagline = Field(UnicodeText(255)) - info = Field(JsonType) + plot = Column(UnicodeText) + tagline = Column(UnicodeText(255)) + info = Column(JsonType) - status = ManyToOne('Status') - movies = OneToMany('Movie', cascade = 'all, delete-orphan') - titles = OneToMany('LibraryTitle', cascade = 'all, delete-orphan') - files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) + status_id = Column(Integer, ForeignKey('status.id'), index = True) + status = relationship('Status') + + movies = relationship('Movie') #, cascade = 'all, delete-orphan') + titles = relationship('LibraryTitle', order_by="desc(LibraryTitle.default)") #, cascade = 'all, delete-orphan') + files = relationship('File', secondary = library_files) #, cascade = 'all, delete-orphan', single_parent = True) -class LibraryTitle(Entity): - """""" - using_options(order_by = '-default') +class LibraryTitle(Base, TableHelper): + __tablename__ = 'librarytitle' + id = Column(Integer, primary_key = True) - title = Field(Unicode) - simple_title = Field(Unicode, index = True) - default = Field(Boolean, default = False, index = True) - - language = OneToMany('Language') - libraries = ManyToOne('Library') - - -class Language(Entity): """""" - identifier = Field(String(20), index = True) - label = Field(Unicode) + #using_options(order_by = '-default') - titles = ManyToOne('LibraryTitle') + title = Column(Unicode) + simple_title = Column(Unicode, index = True) + default = Column(Boolean, default = False, index = True) + + language = relationship('Language') + + libraries_id = Column(Integer, ForeignKey('library.id'), index = True) + libraries = relationship('Library') -class Release(Entity): +class Language(Base, TableHelper): + __tablename__ = 'language' + id = Column(Integer, primary_key = True) + + """""" + + identifier = Column(String(20), index = True) + label = Column(Unicode) + + titles_id = Column(Integer, ForeignKey('librarytitle.id'), index = True) + titles = relationship('LibraryTitle') + + +class Release(Base, TableHelper): + __tablename__ = 'release' + id = Column(Integer, primary_key = True) + """Logically groups all files that belong to a certain release, such as parts of a movie, subtitles.""" - last_edit = Field(Integer, default = lambda: int(time.time()), index = True) - identifier = Field(String(100), index = True) + last_edit = Column(Integer, default = lambda: int(time.time()), index = True) + identifier = Column(String(100), index = True) - movie = ManyToOne('Movie') - status = ManyToOne('Status') - quality = ManyToOne('Quality') - files = ManyToMany('File') - info = OneToMany('ReleaseInfo', cascade = 'all, delete-orphan') + movie_id = Column(Integer, ForeignKey('movie.id'), index = True) + movie = relationship('Movie') + + status_id = Column(Integer, ForeignKey('status.id'), index = True) + status = relationship('Status') + + quality_id = Column(Integer, ForeignKey('quality.id'), index = True) + quality = relationship('Quality') + + files = relationship('File', secondary = release_files) + info = relationship('ReleaseInfo') #, cascade = 'all, delete-orphan') def to_dict(self, deep = None, exclude = None): if not exclude: exclude = [] @@ -162,51 +312,66 @@ class Release(Entity): return orig_dict -class ReleaseInfo(Entity): +class ReleaseInfo(Base, TableHelper): + __tablename__ = 'releaseinfo' + id = Column(Integer, primary_key = True) + """Properties that can be bound to a file for off-line usage""" - identifier = Field(String(50), index = True) - value = Field(Unicode(255), nullable = False) + identifier = Column(String(50), index = True) + value = Column(Unicode(255), nullable = False) - release = ManyToOne('Release') + release_id = Column(Integer, ForeignKey('release.id'), index = True) + release = relationship('Release') -class Status(Entity): +class Status(Base, TableHelper): + __tablename__ = 'status' + id = Column(Integer, primary_key = True) + """The status of a release, such as Downloaded, Deleted, Wanted etc""" - identifier = Field(String(20), unique = True) - label = Field(Unicode(20)) + identifier = Column(String(20), unique = True) + label = Column(Unicode(20)) - releases = OneToMany('Release') - movies = OneToMany('Movie') + releases = relationship('Release') + movies = relationship('Movie') -class Quality(Entity): +class Quality(Base, TableHelper): + __tablename__ = 'quality' + id = Column(Integer, primary_key = True) + """Quality name of a release, DVD, 720p, DVD-Rip etc""" - using_options(order_by = 'order') - identifier = Field(String(20), unique = True) - label = Field(Unicode(20)) - order = Field(Integer, default = 0, index = True) + #using_options(order_by = 'order') - size_min = Field(Integer) - size_max = Field(Integer) + identifier = Column(String(20), unique = True) + label = Column(Unicode(20)) + order = Column(Integer, default = 0, index = True) - releases = OneToMany('Release') - profile_types = OneToMany('ProfileType') + size_min = Column(Integer) + size_max = Column(Integer) + + releases = relationship('Release') + profile_types = relationship('ProfileType', order_by="asc(ProfileType.order)") -class Profile(Entity): +class Profile(Base, TableHelper): + __tablename__ = 'profile' + id = Column(Integer, primary_key = True) + """""" - using_options(order_by = 'order') - label = Field(Unicode(50)) - order = Field(Integer, default = 0, index = True) - core = Field(Boolean, default = False) - hide = Field(Boolean, default = False) + #using_options(order_by = 'order') - movie = OneToMany('Movie') - types = OneToMany('ProfileType', cascade = 'all, delete-orphan') + label = Column(Unicode(50)) + order = Column(Integer, default = 0, index = True) + core = Column(Boolean, default = False) + hide = Column(Boolean, default = False) + + movie = relationship('Movie') + types = relationship('ProfileType', order_by="asc(ProfileType.order)") #, cascade = 'all, delete-orphan') def to_dict(self, deep = None, exclude = None): if not exclude: exclude = [] @@ -219,100 +384,119 @@ class Profile(Entity): return orig_dict -class Category(Entity): +class Category(Base, TableHelper): + __tablename__ = 'category' + id = Column(Integer, primary_key = True) + """""" - using_options(order_by = 'order') - label = Field(Unicode(50)) - order = Field(Integer, default = 0, index = True) - required = Field(Unicode(255)) - preferred = Field(Unicode(255)) - ignored = Field(Unicode(255)) - destination = Field(Unicode(255)) + #using_options(order_by = 'order') - movie = OneToMany('Movie') + label = Column(Unicode(50)) + order = Column(Integer, default = 0, index = True) + required = Column(Unicode(255)) + preferred = Column(Unicode(255)) + ignored = Column(Unicode(255)) + destination = Column(Unicode(255)) + + movie = relationship('Movie') -class ProfileType(Entity): +class ProfileType(Base, TableHelper): + __tablename__ = 'profiletype' + id = Column(Integer, primary_key = True) + """""" - using_options(order_by = 'order') - order = Field(Integer, default = 0, index = True) - finish = Field(Boolean, default = True) - wait_for = Field(Integer, default = 0) + #using_options(order_by = 'order') - quality = ManyToOne('Quality') - profile = ManyToOne('Profile') + order = Column(Integer, default = 0, index = True) + finish = Column(Boolean, default = True) + wait_for = Column(Integer, default = 0) + + quality_id = Column(Integer, ForeignKey('quality.id'), index = True) + quality = relationship('Quality') + + profile_id = Column(Integer, ForeignKey('profile.id'), index = True) + profile = relationship('Profile') -class File(Entity): +class File(Base, TableHelper): + __tablename__ = 'file' + id = Column(Integer, primary_key = True) + """File that belongs to a release.""" - path = Field(Unicode(255), nullable = False, unique = True) - part = Field(Integer, default = 1) - available = Field(Boolean, default = True) + path = Column(Unicode(255), nullable = False, unique = True) + part = Column(Integer, default = 1) + available = Column(Boolean, default = True) - type = ManyToOne('FileType') - properties = OneToMany('FileProperty') + type_id = Column(Integer, ForeignKey('filetype.id'), index = True) + type = relationship('FileType') - history = OneToMany('RenameHistory') - movie = ManyToMany('Movie') - release = ManyToMany('Release') - library = ManyToMany('Library') + properties = relationship('FileProperty') + + movie = relationship('Movie', secondary = movie_files) + release = relationship('Release', secondary = release_files) + library = relationship('Library', secondary = library_files) -class FileType(Entity): +class FileType(Base, TableHelper): + __tablename__ = 'filetype' + id = Column(Integer, primary_key = True) + """Types could be trailer, subtitle, movie, partial movie etc.""" - identifier = Field(String(20), unique = True) - type = Field(Unicode(20)) - name = Field(Unicode(50), nullable = False) + identifier = Column(String(20), unique = True) + type = Column(Unicode(20)) + name = Column(Unicode(50), nullable = False) - files = OneToMany('File') + files = relationship('File') -class FileProperty(Entity): +class FileProperty(Base, TableHelper): + __tablename__ = 'fileproperty' + id = Column(Integer, primary_key = True) + """Properties that can be bound to a file for off-line usage""" - identifier = Field(String(20), index = True) - value = Field(Unicode(255), nullable = False) + identifier = Column(String(20), index = True) + value = Column(Unicode(255), nullable = False) - file = ManyToOne('File') + file_id = Column(Integer, ForeignKey('file.id'), index = True) + file = relationship('File') -class RenameHistory(Entity): - """Remembers from where to where files have been moved.""" +class Notification(Base, TableHelper): + __tablename__ = 'notification' + id = Column(Integer, primary_key = True) - old = Field(Unicode(255)) - new = Field(Unicode(255)) + """""" - file = ManyToOne('File') + #using_options(order_by = 'added') + + added = Column(Integer, default = lambda: int(time.time()), index = True) + read = Column(Boolean, default = False, index = True) + message = Column(Unicode(255)) + data = Column(JsonType) -class Notification(Entity): - using_options(order_by = 'added') +class Properties(Base, TableHelper): + __tablename__ = 'properties' + id = Column(Integer, primary_key = True) - added = Field(Integer, default = lambda: int(time.time())) - read = Field(Boolean, default = False) - message = Field(Unicode(255)) - data = Field(JsonType) + """""" - -class Properties(Entity): - - identifier = Field(String(50), index = True) - value = Field(Unicode(255), nullable = False) + identifier = Column(String(50), index = True) + value = Column(Unicode(255), nullable = False) def setup(): """Setup the database and create the tables that don't exists yet""" - from elixir import setup_all, create_all from couchpotato.environment import Env engine = Env.getEngine() - - setup_all() - create_all(engine) + Base.metadata.create_all(engine) try: engine.execute("PRAGMA journal_mode = WAL") diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 5c175201..bd033983 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -188,7 +188,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En db_exists = os.path.isfile(toUnicode(db_path)) # Load migrations - if db_exists: + if False and db_exists: from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') diff --git a/libs/elixir/__init__.py b/libs/elixir/__init__.py deleted file mode 100644 index a242b538..00000000 --- a/libs/elixir/__init__.py +++ /dev/null @@ -1,114 +0,0 @@ -''' -Elixir package - -A declarative layer on top of the `SQLAlchemy library -`_. It is a fairly thin wrapper, which provides -the ability to create simple Python classes that map directly to relational -database tables (this pattern is often referred to as the Active Record design -pattern), providing many of the benefits of traditional databases -without losing the convenience of Python objects. - -Elixir is intended to replace the ActiveMapper SQLAlchemy extension, and the -TurboEntity project but does not intend to replace SQLAlchemy's core features, -and instead focuses on providing a simpler syntax for defining model objects -when you do not need the full expressiveness of SQLAlchemy's manual mapper -definitions. -''' - -try: - set -except NameError: - from sets import Set as set - -import sqlalchemy -from sqlalchemy.types import * - -from elixir.options import using_options, using_table_options, \ - using_mapper_options, options_defaults, \ - using_options_defaults -from elixir.entity import Entity, EntityBase, EntityMeta, EntityDescriptor, \ - setup_entities, cleanup_entities -from elixir.fields import has_field, Field -from elixir.relationships import belongs_to, has_one, has_many, \ - has_and_belongs_to_many, \ - ManyToOne, OneToOne, OneToMany, ManyToMany -from elixir.properties import has_property, GenericProperty, ColumnProperty, \ - Synonym -from elixir.statements import Statement -from elixir.collection import EntityCollection, GlobalEntityCollection - - -__version__ = '0.8.0dev' - -__all__ = ['Entity', 'EntityBase', 'EntityMeta', 'EntityCollection', - 'entities', - 'Field', 'has_field', - 'has_property', 'GenericProperty', 'ColumnProperty', 'Synonym', - 'belongs_to', 'has_one', 'has_many', 'has_and_belongs_to_many', - 'ManyToOne', 'OneToOne', 'OneToMany', 'ManyToMany', - 'using_options', 'using_table_options', 'using_mapper_options', - 'options_defaults', 'using_options_defaults', - 'metadata', 'session', - 'create_all', 'drop_all', - 'setup_all', 'cleanup_all', - 'setup_entities', 'cleanup_entities'] + \ - sqlalchemy.types.__all__ - -__doc_all__ = ['create_all', 'drop_all', - 'setup_all', 'cleanup_all', - 'metadata', 'session'] - -# default session -session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker()) - -# default metadata -metadata = sqlalchemy.MetaData() - -metadatas = set() - -# default entity collection -entities = GlobalEntityCollection() - - -def create_all(*args, **kwargs): - '''Create the necessary tables for all declared entities''' - for md in metadatas: - md.create_all(*args, **kwargs) - - -def drop_all(*args, **kwargs): - '''Drop tables for all declared entities''' - for md in metadatas: - md.drop_all(*args, **kwargs) - - -def setup_all(create_tables=False, *args, **kwargs): - '''Setup the table and mapper of all entities in the default entity - collection. - ''' - setup_entities(entities) - - # issue the "CREATE" SQL statements - if create_tables: - create_all(*args, **kwargs) - - -def cleanup_all(drop_tables=False, *args, **kwargs): - '''Clear all mappers, clear the session, and clear all metadatas. - Optionally drops the tables. - ''' - session.close() - - cleanup_entities(entities) - - sqlalchemy.orm.clear_mappers() - entities.clear() - - if drop_tables: - drop_all(*args, **kwargs) - - for md in metadatas: - md.clear() - metadatas.clear() - - diff --git a/libs/elixir/collection.py b/libs/elixir/collection.py deleted file mode 100644 index 78127e3e..00000000 --- a/libs/elixir/collection.py +++ /dev/null @@ -1,125 +0,0 @@ -''' -Default entity collection implementation -''' -import sys -import re - -class BaseCollection(list): - def __init__(self, entities=None): - list.__init__(self) - if entities is not None: - self.extend(entities) - - def extend(self, entities): - for e in entities: - self.append(e) - - def clear(self): - del self[:] - - def resolve_absolute(self, key, full_path, entity=None, root=None): - if root is None: - root = entity._descriptor.resolve_root - if root: - full_path = '%s.%s' % (root, full_path) - module_path, classname = full_path.rsplit('.', 1) - module = sys.modules[module_path] - res = getattr(module, classname, None) - if res is None: - if entity is not None: - raise Exception("Couldn't resolve target '%s' <%s> in '%s'!" - % (key, full_path, entity.__name__)) - else: - raise Exception("Couldn't resolve target '%s' <%s>!" - % (key, full_path)) - return res - - def __getattr__(self, key): - return self.resolve(key) - -# default entity collection -class GlobalEntityCollection(BaseCollection): - def __init__(self, entities=None): - # _entities is a dict of entities keyed on their name. - self._entities = {} - super(GlobalEntityCollection, self).__init__(entities) - - def append(self, entity): - ''' - Add an entity to the collection. - ''' - super(EntityCollection, self).append(entity) - - existing_entities = self._entities.setdefault(entity.__name__, []) - existing_entities.append(entity) - - def resolve(self, key, entity=None): - ''' - Resolve a key to an Entity. The optional `entity` argument is the - "source" entity when resolving relationship targets. - ''' - # Do we have a fully qualified entity name? - if '.' in key: - return self.resolve_absolute(key, key, entity) - else: - # Otherwise we look in the entities of this collection - res = self._entities.get(key, None) - if res is None: - if entity: - raise Exception("Couldn't resolve target '%s' in '%s'" - % (key, entity.__name__)) - else: - raise Exception("This collection does not contain any " - "entity corresponding to the key '%s'!" - % key) - elif len(res) > 1: - raise Exception("'%s' resolves to several entities, you should" - " use the full path (including the full module" - " name) to that entity." % key) - else: - return res[0] - - def clear(self): - self._entities = {} - super(GlobalEntityCollection, self).clear() - -# backward compatible name -EntityCollection = GlobalEntityCollection - -_leading_dots = re.compile('^([.]*).*$') - -class RelativeEntityCollection(BaseCollection): - # the entity=None does not make any sense with a relative entity collection - def resolve(self, key, entity): - ''' - Resolve a key to an Entity. The optional `entity` argument is the - "source" entity when resolving relationship targets. - ''' - full_path = key - - if '.' not in key or key.startswith('.'): - # relative target - - # any leading dot is stripped and with each dot removed, - # the entity_module is stripped of one more chunk (starting with - # the last one). - num_dots = _leading_dots.match(full_path).end(1) - full_path = full_path[num_dots:] - chunks = entity.__module__.split('.') - chunkstokeep = len(chunks) - num_dots - if chunkstokeep < 0: - raise Exception("Couldn't resolve relative target " - "'%s' relative to '%s'" % (key, entity.__module__)) - entity_module = '.'.join(chunks[:chunkstokeep]) - - if entity_module and entity_module is not '__main__': - full_path = '%s.%s' % (entity_module, full_path) - - root = '' - else: - root = None - return self.resolve_absolute(key, full_path, entity, root=root) - - def __getattr__(self, key): - raise NotImplementedError - diff --git a/libs/elixir/entity.py b/libs/elixir/entity.py deleted file mode 100644 index 87f5154c..00000000 --- a/libs/elixir/entity.py +++ /dev/null @@ -1,1039 +0,0 @@ -''' -This module provides the ``Entity`` base class, as well as its metaclass -``EntityMeta``. -''' - -import sys -import types -import warnings - -from copy import deepcopy - -import sqlalchemy -from sqlalchemy import Table, Column, Integer, desc, ForeignKey, and_, \ - ForeignKeyConstraint -from sqlalchemy.orm import MapperExtension, mapper, object_session, \ - EXT_CONTINUE, polymorphic_union, ScopedSession, \ - ColumnProperty -from sqlalchemy.sql import ColumnCollection - -import elixir -from elixir.statements import process_mutators, MUTATORS -from elixir import options -from elixir.properties import Property - -DEBUG = False - -__doc_all__ = ['Entity', 'EntityMeta'] - - -def session_mapper_factory(scoped_session): - def session_mapper(cls, *args, **kwargs): - if kwargs.pop('save_on_init', True): - old_init = cls.__init__ - def __init__(self, *args, **kwargs): - old_init(self, *args, **kwargs) - scoped_session.add(self) - cls.__init__ = __init__ - cls.query = scoped_session.query_property() - return mapper(cls, *args, **kwargs) - return session_mapper - - -class EntityDescriptor(object): - ''' - EntityDescriptor describes fields and options needed for table creation. - ''' - - def __init__(self, entity): - self.entity = entity - self.parent = None - - bases = [] - for base in entity.__bases__: - if isinstance(base, EntityMeta): - if is_entity(base) and not is_abstract_entity(base): - if self.parent: - raise Exception( - '%s entity inherits from several entities, ' - 'and this is not supported.' - % self.entity.__name__) - else: - self.parent = base - bases.extend(base._descriptor.bases) - self.parent._descriptor.children.append(entity) - else: - bases.append(base) - self.bases = bases - if not is_entity(entity) or is_abstract_entity(entity): - return - - # entity.__module__ is not always reliable (eg in mod_python) - self.module = sys.modules.get(entity.__module__) - - self.builders = [] - - #XXX: use entity.__subclasses__ ? - self.children = [] - - # used for multi-table inheritance - self.join_condition = None - self.has_pk = False - self._pk_col_done = False - - # columns and constraints waiting for a table to exist - self._columns = ColumnCollection() - self.constraints = [] - - # properties (it is only useful for checking dupe properties at the - # moment, and when adding properties before the mapper is created, - # which shouldn't happen). - self.properties = {} - - # - self.relationships = [] - - # set default value for options - self.table_args = [] - - # base class(es) options_defaults - options_defaults = self.options_defaults() - - complete_defaults = options.options_defaults.copy() - complete_defaults.update({ - 'metadata': elixir.metadata, - 'session': elixir.session, - 'collection': elixir.entities - }) - - # set default value for other options - for key in options.valid_options: - value = options_defaults.get(key, complete_defaults[key]) - if isinstance(value, dict): - value = value.copy() - setattr(self, key, value) - - # override options with module-level defaults defined - for key in ('metadata', 'session', 'collection'): - attr = '__%s__' % key - if hasattr(self.module, attr): - setattr(self, key, getattr(self.module, attr)) - - def options_defaults(self): - base_defaults = {} - for base in self.bases: - base_defaults.update(base._descriptor.options_defaults()) - base_defaults.update(getattr(self.entity, 'options_defaults', {})) - return base_defaults - - def setup_options(self): - ''' - Setup any values that might depend on the "using_options" class - mutator. For example, the tablename or the metadata. - ''' - elixir.metadatas.add(self.metadata) - if self.collection is not None: - self.collection.append(self.entity) - - entity = self.entity - if self.parent: - if self.inheritance == 'single': - self.tablename = self.parent._descriptor.tablename - - if not self.tablename: - if self.shortnames: - self.tablename = entity.__name__.lower() - else: - modulename = entity.__module__.replace('.', '_') - tablename = "%s_%s" % (modulename, entity.__name__) - self.tablename = tablename.lower() - elif hasattr(self.tablename, '__call__'): - self.tablename = self.tablename(entity) - - if not self.identity: - if 'polymorphic_identity' in self.mapper_options: - self.identity = self.mapper_options['polymorphic_identity'] - else: - #TODO: include module name (We could have b.Account inherit - # from a.Account) - self.identity = entity.__name__.lower() - elif 'polymorphic_identity' in self.mapper_options: - raise Exception('You cannot use the "identity" option and the ' - 'polymorphic_identity mapper option at the same ' - 'time.') - elif hasattr(self.identity, '__call__'): - self.identity = self.identity(entity) - - if self.polymorphic: - if not isinstance(self.polymorphic, basestring): - self.polymorphic = options.DEFAULT_POLYMORPHIC_COL_NAME - - #--------------------- - # setup phase methods - - def setup_autoload_table(self): - self.setup_table(True) - - def create_pk_cols(self): - """ - Create primary_key columns. That is, call the 'create_pk_cols' - builders then add a primary key to the table if it hasn't already got - one and needs one. - - This method is "semi-recursive" in some cases: it calls the - create_keys method on ManyToOne relationships and those in turn call - create_pk_cols on their target. It shouldn't be possible to have an - infinite loop since a loop of primary_keys is not a valid situation. - """ - if self._pk_col_done: - return - - self.call_builders('create_pk_cols') - - if not self.autoload: - if self.parent: - if self.inheritance == 'multi': - # Add columns with foreign keys to the parent's primary - # key columns - parent_desc = self.parent._descriptor - tablename = parent_desc.table_fullname - join_clauses = [] - fk_columns = [] - for pk_col in parent_desc.primary_keys: - colname = options.MULTIINHERITANCECOL_NAMEFORMAT % \ - {'entity': self.parent.__name__.lower(), - 'key': pk_col.key} - - # It seems like SA ForeignKey is not happy being given - # a real column object when said column is not yet - # attached to a table - pk_col_name = "%s.%s" % (tablename, pk_col.key) - col = Column(colname, pk_col.type, primary_key=True) - fk_columns.append(col) - self.add_column(col) - join_clauses.append(col == pk_col) - self.join_condition = and_(*join_clauses) - self.add_constraint( - ForeignKeyConstraint(fk_columns, - parent_desc.primary_keys, ondelete='CASCADE')) - elif self.inheritance == 'concrete': - # Copy primary key columns from the parent. - for col in self.parent._descriptor.columns: - if col.primary_key: - self.add_column(col.copy()) - elif not self.has_pk and self.auto_primarykey: - if isinstance(self.auto_primarykey, basestring): - colname = self.auto_primarykey - else: - colname = options.DEFAULT_AUTO_PRIMARYKEY_NAME - - self.add_column( - Column(colname, options.DEFAULT_AUTO_PRIMARYKEY_TYPE, - primary_key=True)) - self._pk_col_done = True - - def setup_relkeys(self): - self.call_builders('create_non_pk_cols') - - def before_table(self): - self.call_builders('before_table') - - def setup_table(self, only_autoloaded=False): - ''' - Create a SQLAlchemy table-object with all columns that have been - defined up to this point. - ''' - if self.entity.table is not None: - return - - if self.autoload != only_autoloaded: - return - - kwargs = self.table_options - if self.autoload: - args = self.table_args - kwargs['autoload'] = True - else: - if self.parent: - if self.inheritance == 'single': - # we know the parent is setup before the child - self.entity.table = self.parent.table - - # re-add the entity columns to the parent entity so that - # they are added to the parent's table (whether the - # parent's table is already setup or not). - for col in self._columns: - self.parent._descriptor.add_column(col) - for constraint in self.constraints: - self.parent._descriptor.add_constraint(constraint) - return - elif self.inheritance == 'concrete': - #TODO: we should also copy columns from the parent table - # if the parent is a base (abstract?) entity (whatever the - # inheritance type -> elif will need to be changed) - - # Copy all non-primary key columns from parent table - # (primary key columns have already been copied earlier). - for col in self.parent._descriptor.columns: - if not col.primary_key: - self.add_column(col.copy()) - - for con in self.parent._descriptor.constraints: - self.add_constraint( - ForeignKeyConstraint( - [e.parent.key for e in con.elements], - [e.target_fullname for e in con.elements], - name=con.name, #TODO: modify it - onupdate=con.onupdate, ondelete=con.ondelete, - use_alter=con.use_alter)) - - if self.polymorphic and \ - self.inheritance in ('single', 'multi') and \ - self.children and not self.parent: - self.add_column(Column(self.polymorphic, - options.POLYMORPHIC_COL_TYPE)) - - if self.version_id_col: - if not isinstance(self.version_id_col, basestring): - self.version_id_col = options.DEFAULT_VERSION_ID_COL_NAME - self.add_column(Column(self.version_id_col, Integer)) - - args = list(self.columns) + self.constraints + self.table_args - self.entity.table = Table(self.tablename, self.metadata, - *args, **kwargs) - if DEBUG: - print self.entity.table.repr2() - - def setup_reltables(self): - self.call_builders('create_tables') - - def after_table(self): - self.call_builders('after_table') - - def setup_events(self): - def make_proxy_method(methods): - def proxy_method(self, mapper, connection, instance): - for func in methods: - ret = func(instance) - # I couldn't commit myself to force people to - # systematicaly return EXT_CONTINUE in all their event - # methods. - # But not doing that diverge to how SQLAlchemy works. - # I should try to convince Mike to do EXT_CONTINUE by - # default, and stop processing as the special case. -# if ret != EXT_CONTINUE: - if ret is not None and ret != EXT_CONTINUE: - return ret - return EXT_CONTINUE - return proxy_method - - # create a list of callbacks for each event - methods = {} - - all_methods = getmembers(self.entity, - lambda a: isinstance(a, types.MethodType)) - - for name, method in all_methods: - for event in getattr(method, '_elixir_events', []): - event_methods = methods.setdefault(event, []) - event_methods.append(method) - - if not methods: - return - - # transform that list into methods themselves - for event in methods: - methods[event] = make_proxy_method(methods[event]) - - # create a custom mapper extension class, tailored to our entity - ext = type('EventMapperExtension', (MapperExtension,), methods)() - - # then, make sure that the entity's mapper has our mapper extension - self.add_mapper_extension(ext) - - def before_mapper(self): - self.call_builders('before_mapper') - - def _get_children(self): - children = self.children[:] - for child in self.children: - children.extend(child._descriptor._get_children()) - return children - - def translate_order_by(self, order_by): - if isinstance(order_by, basestring): - order_by = [order_by] - - order = [] - for colname in order_by: - #FIXME: get_column uses self.columns[key] instead of property - # names. self.columns correspond to the columns of the table if - # the table was already created and to self._columns otherwise, - # which is a ColumnCollection indexed on columns.key - # See ticket #108. - col = self.get_column(colname.strip('-')) - if colname.startswith('-'): - col = desc(col) - order.append(col) - return order - - def setup_mapper(self): - ''' - Initializes and assign a mapper to the entity. - At this point the mapper will usually have no property as they are - added later. - ''' - if self.entity.mapper: - return - - # for now we don't support the "abstract" parent class in a concrete - # inheritance scenario as demonstrated in - # sqlalchemy/test/orm/inheritance/concrete.py - # this should be added along other - kwargs = {} - if self.order_by: - kwargs['order_by'] = self.translate_order_by(self.order_by) - - if self.version_id_col: - kwargs['version_id_col'] = self.get_column(self.version_id_col) - - if self.inheritance in ('single', 'concrete', 'multi'): - if self.parent and \ - (self.inheritance != 'concrete' or self.polymorphic): - # non-polymorphic concrete doesn't need this - kwargs['inherits'] = self.parent.mapper - - if self.inheritance == 'multi' and self.parent: - kwargs['inherit_condition'] = self.join_condition - - if self.polymorphic: - if self.children: - if self.inheritance == 'concrete': - keys = [(self.identity, self.entity.table)] - keys.extend([(child._descriptor.identity, child.table) - for child in self._get_children()]) - # Having the same alias name for an entity and one of - # its child (which is a parent itself) shouldn't cause - # any problem because the join shouldn't be used at - # the same time. But in reality, some versions of SA - # do misbehave on this. Since it doesn't hurt to have - # different names anyway, here they go. - pjoin = polymorphic_union( - dict(keys), self.polymorphic, - 'pjoin_%s' % self.identity) - - kwargs['with_polymorphic'] = ('*', pjoin) - kwargs['polymorphic_on'] = \ - getattr(pjoin.c, self.polymorphic) - elif not self.parent: - kwargs['polymorphic_on'] = \ - self.get_column(self.polymorphic) - - if self.children or self.parent: - kwargs['polymorphic_identity'] = self.identity - - if self.parent and self.inheritance == 'concrete': - kwargs['concrete'] = True - - if self.parent and self.inheritance == 'single': - args = [] - else: - args = [self.entity.table] - - # let user-defined kwargs override Elixir-generated ones, though that's - # not very usefull since most of them expect Column instances. - kwargs.update(self.mapper_options) - - #TODO: document this! - if 'primary_key' in kwargs: - cols = self.entity.table.c - kwargs['primary_key'] = [getattr(cols, colname) for - colname in kwargs['primary_key']] - - # do the mapping - if self.session is None: - self.entity.mapper = mapper(self.entity, *args, **kwargs) - elif isinstance(self.session, ScopedSession): - session_mapper = session_mapper_factory(self.session) - self.entity.mapper = session_mapper(self.entity, *args, **kwargs) - else: - raise Exception("Failed to map entity '%s' with its table or " - "selectable. You can only bind an Entity to a " - "ScopedSession object or None for manual session " - "management." - % self.entity.__name__) - - def after_mapper(self): - self.call_builders('after_mapper') - - def setup_properties(self): - self.call_builders('create_properties') - - def finalize(self): - self.call_builders('finalize') - self.entity._setup_done = True - - #---------------- - # helper methods - - def call_builders(self, what): - for builder in self.builders: - if hasattr(builder, what): - getattr(builder, what)() - - def add_column(self, col, check_duplicate=None): - '''when check_duplicate is None, the value of the allowcoloverride - option of the entity is used. - ''' - if check_duplicate is None: - check_duplicate = not self.allowcoloverride - - if col.key in self._columns: - if check_duplicate: - raise Exception("Column '%s' already exist in '%s' ! " % - (col.key, self.entity.__name__)) - else: - del self._columns[col.key] - # are indexed on col.key - self._columns.add(col) - - if col.primary_key: - self.has_pk = True - - table = self.entity.table - if table is not None: - if check_duplicate and col.key in table.columns.keys(): - raise Exception("Column '%s' already exist in table '%s' ! " % - (col.key, table.name)) - table.append_column(col) - if DEBUG: - print "table.append_column(%s)" % col - - def add_constraint(self, constraint): - self.constraints.append(constraint) - - table = self.entity.table - if table is not None: - table.append_constraint(constraint) - - def add_property(self, name, property, check_duplicate=True): - if check_duplicate and name in self.properties: - raise Exception("property '%s' already exist in '%s' ! " % - (name, self.entity.__name__)) - self.properties[name] = property - -#FIXME: something like this is needed to propagate the relationships from -# parent entities to their children in a concrete inheritance scenario. But -# this doesn't work because of the backref matching code. In most case -# (test_concrete.py) it doesn't even happen at all. -# if self.children and self.inheritance == 'concrete': -# for child in self.children: -# child._descriptor.add_property(name, property) - - mapper = self.entity.mapper - if mapper: - mapper.add_property(name, property) - if DEBUG: - print "mapper.add_property('%s', %s)" % (name, repr(property)) - - def add_mapper_extension(self, extension): - extensions = self.mapper_options.get('extension', []) - if not isinstance(extensions, list): - extensions = [extensions] - extensions.append(extension) - self.mapper_options['extension'] = extensions - - def get_column(self, key, check_missing=True): - #TODO: this needs to work whether the table is already setup or not - #TODO: support SA table/autoloaded entity - try: - return self.columns[key] - except KeyError: - if check_missing: - raise Exception("No column named '%s' found in the table of " - "the '%s' entity!" - % (key, self.entity.__name__)) - - def get_inverse_relation(self, rel, check_reverse=True): - ''' - Return the inverse relation of rel, if any, None otherwise. - ''' - - matching_rel = None - for other_rel in self.relationships: - if rel.is_inverse(other_rel): - if matching_rel is None: - matching_rel = other_rel - else: - raise Exception( - "Several relations match as inverse of the '%s' " - "relation in entity '%s'. You should specify " - "inverse relations manually by using the inverse " - "keyword." - % (rel.name, rel.entity.__name__)) - # When a matching inverse is found, we check that it has only - # one relation matching as its own inverse. We don't need the result - # of the method though. But we do need to be careful not to start an - # infinite recursive loop. - if matching_rel and check_reverse: - rel.entity._descriptor.get_inverse_relation(matching_rel, False) - - return matching_rel - - def find_relationship(self, name): - for rel in self.relationships: - if rel.name == name: - return rel - if self.parent: - return self.parent._descriptor.find_relationship(name) - else: - return None - - #------------------------ - # some useful properties - - @property - def table_fullname(self): - ''' - Complete name of the table for the related entity. - Includes the schema name if there is one specified. - ''' - schema = self.table_options.get('schema', None) - if schema is not None: - return "%s.%s" % (schema, self.tablename) - else: - return self.tablename - - @property - def columns(self): - if self.entity.table is not None: - return self.entity.table.columns - else: - #FIXME: depending on the type of inheritance, we should also - # return the parent entity's columns (for example for order_by - # using a column defined in the parent. - return self._columns - - @property - def primary_keys(self): - """ - Returns the list of primary key columns of the entity. - - This property isn't valid before the "create_pk_cols" phase. - """ - if self.autoload: - return [col for col in self.entity.table.primary_key.columns] - else: - if self.parent and self.inheritance == 'single': - return self.parent._descriptor.primary_keys - else: - return [col for col in self.columns if col.primary_key] - - @property - def table(self): - if self.entity.table is not None: - return self.entity.table - else: - return FakeTable(self) - - @property - def primary_key_properties(self): - """ - Returns the list of (mapper) properties corresponding to the primary - key columns of the table of the entity. - - This property caches its value, so it shouldn't be called before the - entity is fully set up. - """ - if not hasattr(self, '_pk_props'): - col_to_prop = {} - mapper = self.entity.mapper - for prop in mapper.iterate_properties: - if isinstance(prop, ColumnProperty): - for col in prop.columns: - #XXX: Why is this extra loop necessary? What is this - # "proxy_set" supposed to mean? - for col in col.proxy_set: - col_to_prop[col] = prop - pk_cols = [c for c in mapper.mapped_table.c if c.primary_key] - self._pk_props = [col_to_prop[c] for c in pk_cols] - return self._pk_props - -class FakePK(object): - def __init__(self, descriptor): - self.descriptor = descriptor - - @property - def columns(self): - return self.descriptor.primary_keys - -class FakeTable(object): - def __init__(self, descriptor): - self.descriptor = descriptor - self.primary_key = FakePK(descriptor) - - @property - def columns(self): - return self.descriptor.columns - - @property - def fullname(self): - ''' - Complete name of the table for the related entity. - Includes the schema name if there is one specified. - ''' - schema = self.descriptor.table_options.get('schema', None) - if schema is not None: - return "%s.%s" % (schema, self.descriptor.tablename) - else: - return self.descriptor.tablename - - -def is_entity(cls): - """ - Scan the bases classes of `cls` to see if any is an instance of - EntityMeta. If we don't find any, it means it is either an unrelated class - or an entity base class (like the 'Entity' class). - """ - for base in cls.__bases__: - if isinstance(base, EntityMeta): - return True - return False - - -# Note that we don't use inspect.getmembers because of -# http://bugs.python.org/issue1785 -# See also http://elixir.ematia.de/trac/changeset/262 -def getmembers(object, predicate=None): - base_props = [] - for key in dir(object): - try: - value = getattr(object, key) - except AttributeError: - continue - if not predicate or predicate(value): - base_props.append((key, value)) - return base_props - -def is_abstract_entity(dict_or_cls): - if not isinstance(dict_or_cls, dict): - dict_or_cls = dict_or_cls.__dict__ - for mutator, args, kwargs in dict_or_cls.get(MUTATORS, []): - if 'abstract' in kwargs: - return kwargs['abstract'] - - return False - -def instrument_class(cls): - """ - Instrument a class as an Entity. This is usually done automatically through - the EntityMeta metaclass. - """ - # Create the entity descriptor - desc = cls._descriptor = EntityDescriptor(cls) - - # Process mutators - # We *do* want mutators to be processed for base/abstract classes - # (so that statements like using_options_defaults work). - process_mutators(cls) - - # We do not want to do any more processing for base/abstract classes - # (Entity et al.). - if not is_entity(cls) or is_abstract_entity(cls): - return - - cls.table = None - cls.mapper = None - - # Copy the properties ('Property' instances) of the entity base class(es). - # We use getmembers (instead of __dict__) so that we also get the - # properties from the parents of the base class if any. - base_props = [] - for base in cls.__bases__: - if isinstance(base, EntityMeta) and \ - (not is_entity(base) or is_abstract_entity(base)): - base_props += [(name, deepcopy(attr)) for name, attr in - getmembers(base, lambda a: isinstance(a, Property))] - - # Process attributes (using the assignment syntax), looking for - # 'Property' instances and attaching them to this entity. - properties = [(name, attr) for name, attr in cls.__dict__.iteritems() - if isinstance(attr, Property)] - sorted_props = sorted(base_props + properties, - key=lambda i: i[1]._counter) - for name, prop in sorted_props: - prop.attach(cls, name) - - # setup misc options here (like tablename etc.) - desc.setup_options() - - -class EntityMeta(type): - """ - Entity meta class. - You should only use it directly if you want to define your own base class - for your entities (ie you don't want to use the provided 'Entity' class). - """ - - def __init__(cls, name, bases, dict_): - instrument_class(cls) - - def __setattr__(cls, key, value): - if isinstance(value, Property): - if hasattr(cls, '_setup_done'): - raise Exception('Cannot set attribute on a class after ' - 'setup_all') - else: - value.attach(cls, key) - else: - type.__setattr__(cls, key, value) - - -def setup_entities(entities): - '''Setup all entities in the list passed as argument''' - - for entity in entities: - # delete all Elixir properties so that it doesn't interfere with - # SQLAlchemy. At this point they should have be converted to - # builders. - for name, attr in entity.__dict__.items(): - if isinstance(attr, Property): - delattr(entity, name) - - for method_name in ( - 'setup_autoload_table', 'create_pk_cols', 'setup_relkeys', - 'before_table', 'setup_table', 'setup_reltables', 'after_table', - 'setup_events', - 'before_mapper', 'setup_mapper', 'after_mapper', - 'setup_properties', - 'finalize'): -# if DEBUG: -# print "=" * 40 -# print method_name -# print "=" * 40 - for entity in entities: -# print entity.__name__, "...", - if hasattr(entity, '_setup_done'): -# print "already done" - continue - method = getattr(entity._descriptor, method_name) - method() -# print "ok" - - -def cleanup_entities(entities): - """ - Try to revert back the list of entities passed as argument to the state - they had just before their setup phase. - - As of now, this function is *not* functional in that it doesn't revert to - the exact same state the entities were before setup. For example, the - properties do not work yet as those would need to be regenerated (since the - columns they are based on are regenerated too -- and as such the - corresponding joins are not correct) but this doesn't happen because of - the way relationship setup is designed to be called only once (especially - the backref stuff in create_properties). - """ - for entity in entities: - desc = entity._descriptor - - if hasattr(entity, '_setup_done'): - del entity._setup_done - - entity.table = None - entity.mapper = None - - desc._pk_col_done = False - desc.has_pk = False - desc._columns = ColumnCollection() - desc.constraints = [] - desc.properties = {} - -class EntityBase(object): - """ - This class holds all methods of the "Entity" base class, but does not act - as a base class itself (it does not use the EntityMeta metaclass), but - rather as a parent class for Entity. This is meant so that people who want - to provide their own base class but don't want to loose or copy-paste all - the methods of Entity can do so by inheriting from EntityBase: - - .. sourcecode:: python - - class MyBase(EntityBase): - __metaclass__ = EntityMeta - - def myCustomMethod(self): - # do something great - """ - - def __init__(self, **kwargs): - self.set(**kwargs) - - def set(self, **kwargs): - for key, value in kwargs.iteritems(): - setattr(self, key, value) - - @classmethod - def update_or_create(cls, data, surrogate=True): - pk_props = cls._descriptor.primary_key_properties - - # if all pk are present and not None - if not [1 for p in pk_props if data.get(p.key) is None]: - pk_tuple = tuple([data[prop.key] for prop in pk_props]) - record = cls.query.get(pk_tuple) - if record is None: - if surrogate: - raise Exception("Cannot create surrogate with pk") - else: - record = cls() - else: - if surrogate: - record = cls() - else: - raise Exception("Cannot create non surrogate without pk") - record.from_dict(data) - return record - - def from_dict(self, data): - """ - Update a mapped class with data from a JSON-style nested dict/list - structure. - """ - # surrogate can be guessed from autoincrement/sequence but I guess - # that's not 100% reliable, so we'll need an override - - mapper = sqlalchemy.orm.object_mapper(self) - - for key, value in data.iteritems(): - if isinstance(value, dict): - dbvalue = getattr(self, key) - rel_class = mapper.get_property(key).mapper.class_ - pk_props = rel_class._descriptor.primary_key_properties - - # If the data doesn't contain any pk, and the relationship - # already has a value, update that record. - if not [1 for p in pk_props if p.key in data] and \ - dbvalue is not None: - dbvalue.from_dict(value) - else: - record = rel_class.update_or_create(value) - setattr(self, key, record) - elif isinstance(value, list) and \ - value and isinstance(value[0], dict): - - rel_class = mapper.get_property(key).mapper.class_ - new_attr_value = [] - for row in value: - if not isinstance(row, dict): - raise Exception( - 'Cannot send mixed (dict/non dict) data ' - 'to list relationships in from_dict data.') - record = rel_class.update_or_create(row) - new_attr_value.append(record) - setattr(self, key, new_attr_value) - else: - setattr(self, key, value) - - def to_dict(self, deep={}, exclude=[]): - """Generate a JSON-style nested dict/list structure from an object.""" - col_prop_names = [p.key for p in self.mapper.iterate_properties \ - if isinstance(p, ColumnProperty)] - data = dict([(name, getattr(self, name)) - for name in col_prop_names if name not in exclude]) - for rname, rdeep in deep.iteritems(): - dbdata = getattr(self, rname) - #FIXME: use attribute names (ie coltoprop) instead of column names - fks = self.mapper.get_property(rname).remote_side - exclude = [c.name for c in fks] - if dbdata is None: - data[rname] = None - elif isinstance(dbdata, list): - data[rname] = [o.to_dict(rdeep, exclude) for o in dbdata] - else: - data[rname] = dbdata.to_dict(rdeep, exclude) - return data - - # session methods - def flush(self, *args, **kwargs): - return object_session(self).flush([self], *args, **kwargs) - - def delete(self, *args, **kwargs): - return object_session(self).delete(self, *args, **kwargs) - - def expire(self, *args, **kwargs): - return object_session(self).expire(self, *args, **kwargs) - - def refresh(self, *args, **kwargs): - return object_session(self).refresh(self, *args, **kwargs) - - def expunge(self, *args, **kwargs): - return object_session(self).expunge(self, *args, **kwargs) - - # This bunch of session methods, along with all the query methods below - # only make sense when using a global/scoped/contextual session. - @property - def _global_session(self): - return self._descriptor.session.registry() - - #FIXME: remove all deprecated methods, possibly all of these - def merge(self, *args, **kwargs): - return self._global_session.merge(self, *args, **kwargs) - - def save(self, *args, **kwargs): - return self._global_session.save(self, *args, **kwargs) - - def update(self, *args, **kwargs): - return self._global_session.update(self, *args, **kwargs) - - # only exist in SA < 0.5 - # IMO, the replacement (session.add) doesn't sound good enough to be added - # here. For example: "o = Order(); o.add()" is not very telling. It's - # better to leave it as "session.add(o)" - def save_or_update(self, *args, **kwargs): - return self._global_session.save_or_update(self, *args, **kwargs) - - # query methods - @classmethod - def get_by(cls, *args, **kwargs): - """ - Returns the first instance of this class matching the given criteria. - This is equivalent to: - session.query(MyClass).filter_by(...).first() - """ - return cls.query.filter_by(*args, **kwargs).first() - - @classmethod - def get(cls, *args, **kwargs): - """ - Return the instance of this class based on the given identifier, - or None if not found. This is equivalent to: - session.query(MyClass).get(...) - """ - return cls.query.get(*args, **kwargs) - - -class Entity(EntityBase): - ''' - The base class for all entities - - All Elixir model objects should inherit from this class. Statements can - appear within the body of the definition of an entity to define its - fields, relationships, and other options. - - Here is an example: - - .. sourcecode:: python - - class Person(Entity): - name = Field(Unicode(128)) - birthdate = Field(DateTime, default=datetime.now) - - Please note, that if you don't specify any primary keys, Elixir will - automatically create one called ``id``. - - For further information, please refer to the provided examples or - tutorial. - ''' - __metaclass__ = EntityMeta - - diff --git a/libs/elixir/events.py b/libs/elixir/events.py deleted file mode 100644 index 293a8a4a..00000000 --- a/libs/elixir/events.py +++ /dev/null @@ -1,27 +0,0 @@ -from sqlalchemy.orm import reconstructor - -__all__ = [ - 'before_insert', - 'after_insert', - 'before_update', - 'after_update', - 'before_delete', - 'after_delete', - 'reconstructor' -] - -def create_decorator(event_name): - def decorator(func): - if not hasattr(func, '_elixir_events'): - func._elixir_events = [] - func._elixir_events.append(event_name) - return func - return decorator - -before_insert = create_decorator('before_insert') -after_insert = create_decorator('after_insert') -before_update = create_decorator('before_update') -after_update = create_decorator('after_update') -before_delete = create_decorator('before_delete') -after_delete = create_decorator('after_delete') - diff --git a/libs/elixir/ext/__init__.py b/libs/elixir/ext/__init__.py deleted file mode 100644 index c8708f25..00000000 --- a/libs/elixir/ext/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -''' -Ext package - -Additional Elixir statements and functionality. -''' diff --git a/libs/elixir/ext/associable.py b/libs/elixir/ext/associable.py deleted file mode 100644 index b31c5a74..00000000 --- a/libs/elixir/ext/associable.py +++ /dev/null @@ -1,234 +0,0 @@ -''' -Associable Elixir Statement Generator - -========== -Associable -========== - -About Polymorphic Associations ------------------------------- - -A frequent pattern in database schemas is the has_and_belongs_to_many, or a -many-to-many table. Quite often multiple tables will refer to a single one -creating quite a few many-to-many intermediate tables. - -Polymorphic associations lower the amount of many-to-many tables by setting up -a table that allows relations to any other table in the database, and relates -it to the associable table. In some implementations, this layout does not -enforce referential integrity with database foreign key constraints, this -implementation uses an additional many-to-many table with foreign key -constraints to avoid this problem. - -.. note: - SQLite does not support foreign key constraints, so referential integrity - can only be enforced using database backends with such support. - -Elixir Statement Generator for Polymorphic Associations -------------------------------------------------------- - -The ``associable`` function generates the intermediary tables for an Elixir -entity that should be associable with other Elixir entities and returns an -Elixir Statement for use with them. This automates the process of creating the -polymorphic association tables and ensuring their referential integrity. - -Matching select_XXX and select_by_XXX are also added to the associated entity -which allow queries to be run for the associated objects. - -Example usage: - -.. sourcecode:: python - - class Tag(Entity): - name = Field(Unicode) - - acts_as_taggable = associable(Tag) - - class Entry(Entity): - title = Field(Unicode) - acts_as_taggable('tags') - - class Article(Entity): - title = Field(Unicode) - acts_as_taggable('tags') - -Or if one of the entities being associated should only have a single member of -the associated table: - -.. sourcecode:: python - - class Address(Entity): - street = Field(String(130)) - city = Field(String(100)) - - is_addressable = associable(Address, 'addresses') - - class Person(Entity): - name = Field(Unicode) - orders = OneToMany('Order') - is_addressable() - - class Order(Entity): - order_num = Field(primary_key=True) - item_count = Field(Integer) - person = ManyToOne('Person') - is_addressable('address', uselist=False) - - home = Address(street='123 Elm St.', city='Spooksville') - user = Person(name='Jane Doe') - user.addresses.append(home) - - neworder = Order(item_count=4) - neworder.address = home - user.orders.append(neworder) - - # Queries using the added helpers - Person.select_by_addresses(city='Cupertino') - Person.select_addresses(and_(Address.c.street=='132 Elm St', - Address.c.city=='Smallville')) - -Statement Options ------------------ - -The generated Elixir Statement has several options available: - -+---------------+-------------------------------------------------------------+ -| Option Name | Description | -+===============+=============================================================+ -| ``name`` | Specify a custom name for the Entity attribute. This is | -| | used to declare the attribute used to access the associated | -| | table values. Otherwise, the name will use the plural_name | -| | provided to the associable call. | -+---------------+-------------------------------------------------------------+ -| ``uselist`` | Whether or not the associated table should be represented | -| | as a list, or a single property. It should be set to False | -| | when the entity should only have a single associated | -| | entity. Defaults to True. | -+---------------+-------------------------------------------------------------+ -| ``lazy`` | Determines eager loading of the associated entity objects. | -| | Defaults to False, to indicate that they should not be | -| | lazily loaded. | -+---------------+-------------------------------------------------------------+ -''' -from elixir.statements import Statement -import sqlalchemy as sa - -__doc_all__ = ['associable'] - - -def associable(assoc_entity, plural_name=None, lazy=True): - ''' - Generate an associable Elixir Statement - ''' - interface_name = assoc_entity._descriptor.tablename - able_name = interface_name + 'able' - - if plural_name: - attr_name = "%s_rel" % plural_name - else: - plural_name = interface_name - attr_name = "%s_rel" % interface_name - - class GenericAssoc(object): - - def __init__(self, tablename): - self.type = tablename - - #TODO: inherit from entity builder - class Associable(object): - """An associable Elixir Statement object""" - - def __init__(self, entity, name=None, uselist=True, lazy=True): - self.entity = entity - self.lazy = lazy - self.uselist = uselist - - if name is None: - self.name = plural_name - else: - self.name = name - - def after_table(self): - col = sa.Column('%s_assoc_id' % interface_name, sa.Integer, - sa.ForeignKey('%s.id' % able_name)) - self.entity._descriptor.add_column(col) - - if not hasattr(assoc_entity, '_assoc_table'): - metadata = assoc_entity._descriptor.metadata - association_table = sa.Table("%s" % able_name, metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('type', sa.String(40), nullable=False), - ) - tablename = "%s_to_%s" % (able_name, interface_name) - association_to_table = sa.Table(tablename, metadata, - sa.Column('assoc_id', sa.Integer, - sa.ForeignKey(association_table.c.id, - ondelete="CASCADE"), - primary_key=True), - #FIXME: this assumes a single id col - sa.Column('%s_id' % interface_name, sa.Integer, - sa.ForeignKey(assoc_entity.table.c.id, - ondelete="RESTRICT"), - primary_key=True), - ) - - assoc_entity._assoc_table = association_table - assoc_entity._assoc_to_table = association_to_table - - def after_mapper(self): - if not hasattr(assoc_entity, '_assoc_mapper'): - assoc_entity._assoc_mapper = sa.orm.mapper( - GenericAssoc, assoc_entity._assoc_table, properties={ - 'targets': sa.orm.relation( - assoc_entity, - secondary=assoc_entity._assoc_to_table, - lazy=lazy, backref='associations', - order_by=assoc_entity.mapper.order_by) - }) - - entity = self.entity - entity.mapper.add_property( - attr_name, - sa.orm.relation(GenericAssoc, lazy=self.lazy, - backref='_backref_%s' % entity.table.name) - ) - - if self.uselist: - def get(self): - if getattr(self, attr_name) is None: - setattr(self, attr_name, - GenericAssoc(entity.table.name)) - return getattr(self, attr_name).targets - setattr(entity, self.name, property(get)) - else: - # scalar based property decorator - def get(self): - attr = getattr(self, attr_name) - if attr is not None: - return attr.targets[0] - else: - return None - def set(self, value): - if getattr(self, attr_name) is None: - setattr(self, attr_name, - GenericAssoc(entity.table.name)) - getattr(self, attr_name).targets = [value] - setattr(entity, self.name, property(get, set)) - - # self.name is both set via mapper synonym and the python - # property, but that's how synonym properties work. - # adding synonym property after "real" property otherwise it - # breaks when using SQLAlchemy > 0.4.1 - entity.mapper.add_property(self.name, sa.orm.synonym(attr_name)) - - # add helper methods - def select_by(cls, **kwargs): - return cls.query.join(attr_name, 'targets') \ - .filter_by(**kwargs).all() - setattr(entity, 'select_by_%s' % self.name, classmethod(select_by)) - - def select(cls, *args, **kwargs): - return cls.query.join(attr_name, 'targets') \ - .filter(*args, **kwargs).all() - setattr(entity, 'select_%s' % self.name, classmethod(select)) - - return Statement(Associable) diff --git a/libs/elixir/ext/encrypted.py b/libs/elixir/ext/encrypted.py deleted file mode 100644 index 410855d2..00000000 --- a/libs/elixir/ext/encrypted.py +++ /dev/null @@ -1,124 +0,0 @@ -''' -An encryption plugin for Elixir utilizing the excellent PyCrypto library, which -can be downloaded here: http://www.amk.ca/python/code/crypto - -Values for columns that are specified to be encrypted will be transparently -encrypted and safely encoded for storage in a unicode column using the powerful -and secure Blowfish Cipher using a specified "secret" which can be passed into -the plugin at class declaration time. - -Example usage: - -.. sourcecode:: python - - from elixir import * - from elixir.ext.encrypted import acts_as_encrypted - - class Person(Entity): - name = Field(Unicode) - password = Field(Unicode) - ssn = Field(Unicode) - acts_as_encrypted(for_fields=['password', 'ssn'], - with_secret='secret') - -The above Person entity will automatically encrypt and decrypt the password and -ssn columns on save, update, and load. Different secrets can be specified on -an entity by entity basis, for added security. - -**Important note**: instance attributes are encrypted in-place. This means that -if one of the encrypted attributes of an instance is accessed after the -instance has been flushed to the database (and thus encrypted), the value for -that attribute will be crypted in the in-memory object in addition to the -database row. -''' - -from Crypto.Cipher import Blowfish -from elixir.statements import Statement -from sqlalchemy.orm import MapperExtension, EXT_CONTINUE, EXT_STOP - -try: - from sqlalchemy.orm import EXT_PASS - SA05orlater = False -except ImportError: - SA05orlater = True - -__all__ = ['acts_as_encrypted'] -__doc_all__ = [] - - -# -# encryption and decryption functions -# - -def encrypt_value(value, secret): - return Blowfish.new(secret, Blowfish.MODE_CFB) \ - .encrypt(value).encode('string_escape') - -def decrypt_value(value, secret): - return Blowfish.new(secret, Blowfish.MODE_CFB) \ - .decrypt(value.decode('string_escape')) - - -# -# acts_as_encrypted statement -# - -class ActsAsEncrypted(object): - - def __init__(self, entity, for_fields=[], with_secret='abcdef'): - - def perform_encryption(instance, encrypt=True): - encrypted = getattr(instance, '_elixir_encrypted', None) - if encrypted is encrypt: - # skipping encryption or decryption, as it is already done - return - else: - # marking instance as already encrypted/decrypted - instance._elixir_encrypted = encrypt - - if encrypt: - func = encrypt_value - else: - func = decrypt_value - - for column_name in for_fields: - current_value = getattr(instance, column_name) - if current_value: - setattr(instance, column_name, - func(current_value, with_secret)) - - def perform_decryption(instance): - perform_encryption(instance, encrypt=False) - - class EncryptedMapperExtension(MapperExtension): - - def before_insert(self, mapper, connection, instance): - perform_encryption(instance) - return EXT_CONTINUE - - def before_update(self, mapper, connection, instance): - perform_encryption(instance) - return EXT_CONTINUE - - if SA05orlater: - def reconstruct_instance(self, mapper, instance): - perform_decryption(instance) - # no special return value is required for - # reconstruct_instance, but you never know... - return EXT_CONTINUE - else: - def populate_instance(self, mapper, selectcontext, row, - instance, *args, **kwargs): - mapper.populate_instance(selectcontext, instance, row, - *args, **kwargs) - perform_decryption(instance) - # EXT_STOP because we already did populate the instance and - # the normal processing should not happen - return EXT_STOP - - # make sure that the entity's mapper has our mapper extension - entity._descriptor.add_mapper_extension(EncryptedMapperExtension()) - - -acts_as_encrypted = Statement(ActsAsEncrypted) - diff --git a/libs/elixir/ext/perform_ddl.py b/libs/elixir/ext/perform_ddl.py deleted file mode 100644 index bb8528df..00000000 --- a/libs/elixir/ext/perform_ddl.py +++ /dev/null @@ -1,106 +0,0 @@ -''' -DDL statements for Elixir. - -Entities having the perform_ddl statement, will automatically execute the -given DDL statement, at the given moment: ether before or after the table -creation in SQL. - -The 'when' argument can be either 'before-create' or 'after-create'. -The 'statement' argument can be one of: - -- a single string statement -- a list of string statements, in which case, each of them will be executed - in turn. -- a callable which should take no argument and return either a single string - or a list of strings. - -In each string statement, you may use the special '%(fullname)s' construct, -that will be replaced with the real table name including schema, if unknown -to you. Also, self explained '%(table)s' and '%(schema)s' may be used here. - -You would use this extension to handle non elixir sql statemts, like triggers -etc. - -.. sourcecode:: python - - class Movie(Entity): - title = Field(Unicode(30), primary_key=True) - year = Field(Integer) - - perform_ddl('after-create', - "insert into %(fullname)s values ('Alien', 1979)") - -preload_data is a more specific statement meant to preload data in your -entity table from a list of tuples (of fields values for each row). - -.. sourcecode:: python - - class Movie(Entity): - title = Field(Unicode(30), primary_key=True) - year = Field(Integer) - - preload_data(('title', 'year'), - [(u'Alien', 1979), (u'Star Wars', 1977)]) - preload_data(('year', 'title'), - [(1982, u'Blade Runner')]) - preload_data(data=[(u'Batman', 1966)]) -''' - -from elixir.statements import Statement -from elixir.properties import EntityBuilder -from sqlalchemy import DDL - -__all__ = ['perform_ddl', 'preload_data'] -__doc_all__ = [] - -# -# the perform_ddl statement -# -class PerformDDLEntityBuilder(EntityBuilder): - - def __init__(self, entity, when, statement, on=None, context=None): - self.entity = entity - self.when = when - self.statement = statement - self.on = on - self.context = context - - def after_table(self): - statement = self.statement - if hasattr(statement, '__call__'): - statement = statement() - if not isinstance(statement, list): - statement = [statement] - for s in statement: - ddl = DDL(s, self.on, self.context) - ddl.execute_at(self.when, self.entity.table) - -perform_ddl = Statement(PerformDDLEntityBuilder) - -# -# the preload_data statement -# -class PreloadDataEntityBuilder(EntityBuilder): - - def __init__(self, entity, columns=None, data=None): - self.entity = entity - self.columns = columns - self.data = data - - def after_table(self): - all_columns = [col.name for col in self.entity.table.columns] - def onload(event, schema_item, connection): - columns = self.columns - if columns is None: - columns = all_columns - data = self.data - if hasattr(data, '__call__'): - data = data() - insert = schema_item.insert() - connection.execute(insert, - [dict(zip(columns, values)) for values in data]) - - self.entity.table.append_ddl_listener('after-create', onload) - -preload_data = Statement(PreloadDataEntityBuilder) - diff --git a/libs/elixir/ext/versioned.py b/libs/elixir/ext/versioned.py deleted file mode 100644 index 75f406b0..00000000 --- a/libs/elixir/ext/versioned.py +++ /dev/null @@ -1,288 +0,0 @@ -''' -A versioning plugin for Elixir. - -Entities that are marked as versioned with the `acts_as_versioned` statement -will automatically have a history table created and a timestamp and version -column added to their tables. In addition, versioned entities are provided -with four new methods: revert, revert_to, compare_with and get_as_of, and one -new attribute: versions. Entities with compound primary keys are supported. - -The `versions` attribute will contain a list of previous versions of the -instance, in increasing version number order. - -The `get_as_of` method will retrieve a previous version of the instance "as of" -a specified datetime. If the current version is the most recent, it will be -returned. - -The `revert` method will rollback the current instance to its previous version, -if possible. Once reverted, the current instance will be expired from the -session, and you will need to fetch it again to retrieve the now reverted -instance. - -The `revert_to` method will rollback the current instance to the specified -version number, if possibe. Once reverted, the current instance will be expired -from the session, and you will need to fetch it again to retrieve the now -reverted instance. - -The `compare_with` method will compare the instance with a previous version. A -dictionary will be returned with each field difference as an element in the -dictionary where the key is the field name and the value is a tuple of the -format (current_value, version_value). Version instances also have a -`compare_with` method so that two versions can be compared. - -Also included in the module is a `after_revert` decorator that can be used to -decorate methods on the versioned entity that will be called following that -instance being reverted. - -The acts_as_versioned statement also accepts an optional `ignore` argument -that consists of a list of strings, specifying names of fields. Changes in -those fields will not result in a version increment. In addition, you can -pass in an optional `check_concurrent` argument, which will use SQLAlchemy's -built-in optimistic concurrency mechanisms. - -Note that relationships that are stored in mapping tables will not be included -as part of the versioning process, and will need to be handled manually. Only -values within the entity's main table will be versioned into the history table. -''' - -from datetime import datetime -import inspect - -from sqlalchemy import Table, Column, and_, desc -from sqlalchemy.orm import mapper, MapperExtension, EXT_CONTINUE, \ - object_session - -from elixir import Integer, DateTime -from elixir.statements import Statement -from elixir.properties import EntityBuilder -from elixir.entity import getmembers - -__all__ = ['acts_as_versioned', 'after_revert'] -__doc_all__ = [] - -# -# utility functions -# - -def get_entity_where(instance): - clauses = [] - for column in instance.table.primary_key.columns: - instance_value = getattr(instance, column.name) - clauses.append(column==instance_value) - return and_(*clauses) - - -def get_history_where(instance): - clauses = [] - history_columns = instance.__history_table__.primary_key.columns - for column in instance.table.primary_key.columns: - instance_value = getattr(instance, column.name) - history_column = getattr(history_columns, column.name) - clauses.append(history_column==instance_value) - return and_(*clauses) - - -# -# a mapper extension to track versions on insert, update, and delete -# - -class VersionedMapperExtension(MapperExtension): - def before_insert(self, mapper, connection, instance): - version_colname, timestamp_colname = \ - instance.__class__.__versioned_column_names__ - setattr(instance, version_colname, 1) - setattr(instance, timestamp_colname, datetime.now()) - return EXT_CONTINUE - - def before_update(self, mapper, connection, instance): - old_values = instance.table.select(get_entity_where(instance)) \ - .execute().fetchone() - - # SA might've flagged this for an update even though it didn't change. - # This occurs when a relation is updated, thus marking this instance - # for a save/update operation. We check here against the last version - # to ensure we really should save this version and update the version - # data. - ignored = instance.__class__.__ignored_fields__ - version_colname, timestamp_colname = \ - instance.__class__.__versioned_column_names__ - for key in instance.table.c.keys(): - if key in ignored: - continue - if getattr(instance, key) != old_values[key]: - # the instance was really updated, so we create a new version - dict_values = dict(old_values.items()) - connection.execute( - instance.__class__.__history_table__.insert(), dict_values) - old_version = getattr(instance, version_colname) - setattr(instance, version_colname, old_version + 1) - setattr(instance, timestamp_colname, datetime.now()) - break - - return EXT_CONTINUE - - def before_delete(self, mapper, connection, instance): - connection.execute(instance.__history_table__.delete( - get_history_where(instance) - )) - return EXT_CONTINUE - - -versioned_mapper_extension = VersionedMapperExtension() - - -# -# the acts_as_versioned statement -# - -class VersionedEntityBuilder(EntityBuilder): - - def __init__(self, entity, ignore=None, check_concurrent=False, - column_names=None): - self.entity = entity - self.add_mapper_extension(versioned_mapper_extension) - #TODO: we should rather check that the version_id_col isn't set - # externally - self.check_concurrent = check_concurrent - - # Changes in these fields will be ignored - if column_names is None: - column_names = ['version', 'timestamp'] - entity.__versioned_column_names__ = column_names - if ignore is None: - ignore = [] - ignore.extend(column_names) - entity.__ignored_fields__ = ignore - - def create_non_pk_cols(self): - # add a version column to the entity, along with a timestamp - version_colname, timestamp_colname = \ - self.entity.__versioned_column_names__ - #XXX: fail in case the columns already exist? - #col_names = [col.name for col in self.entity._descriptor.columns] - #if version_colname not in col_names: - self.add_table_column(Column(version_colname, Integer)) - #if timestamp_colname not in col_names: - self.add_table_column(Column(timestamp_colname, DateTime)) - - # add a concurrent_version column to the entity, if required - if self.check_concurrent: - self.entity._descriptor.version_id_col = 'concurrent_version' - - # we copy columns from the main entity table, so we need it to exist first - def after_table(self): - entity = self.entity - version_colname, timestamp_colname = \ - entity.__versioned_column_names__ - - # look for events - after_revert_events = [] - for name, func in getmembers(entity, inspect.ismethod): - if getattr(func, '_elixir_after_revert', False): - after_revert_events.append(func) - - # create a history table for the entity - skipped_columns = [version_colname] - if self.check_concurrent: - skipped_columns.append('concurrent_version') - - columns = [ - column.copy() for column in entity.table.c - if column.name not in skipped_columns - ] - columns.append(Column(version_colname, Integer, primary_key=True)) - table = Table(entity.table.name + '_history', entity.table.metadata, - *columns - ) - entity.__history_table__ = table - - # create an object that represents a version of this entity - class Version(object): - pass - - # map the version class to the history table for this entity - Version.__name__ = entity.__name__ + 'Version' - Version.__versioned_entity__ = entity - mapper(Version, entity.__history_table__) - - version_col = getattr(table.c, version_colname) - timestamp_col = getattr(table.c, timestamp_colname) - - # attach utility methods and properties to the entity - def get_versions(self): - v = object_session(self).query(Version) \ - .filter(get_history_where(self)) \ - .order_by(version_col) \ - .all() - # history contains all the previous records. - # Add the current one to the list to get all the versions - v.append(self) - return v - - def get_as_of(self, dt): - # if the passed in timestamp is older than our current version's - # time stamp, then the most recent version is our current version - if getattr(self, timestamp_colname) < dt: - return self - - # otherwise, we need to look to the history table to get our - # older version - sess = object_session(self) - query = sess.query(Version) \ - .filter(and_(get_history_where(self), - timestamp_col <= dt)) \ - .order_by(desc(timestamp_col)).limit(1) - return query.first() - - def revert_to(self, to_version): - if isinstance(to_version, Version): - to_version = getattr(to_version, version_colname) - - old_version = table.select(and_( - get_history_where(self), - version_col == to_version - )).execute().fetchone() - - entity.table.update(get_entity_where(self)).execute( - dict(old_version.items()) - ) - - table.delete(and_(get_history_where(self), - version_col >= to_version)).execute() - self.expire() - for event in after_revert_events: - event(self) - - def revert(self): - assert getattr(self, version_colname) > 1 - self.revert_to(getattr(self, version_colname) - 1) - - def compare_with(self, version): - differences = {} - for column in self.table.c: - if column.name in (version_colname, 'concurrent_version'): - continue - this = getattr(self, column.name) - that = getattr(version, column.name) - if this != that: - differences[column.name] = (this, that) - return differences - - entity.versions = property(get_versions) - entity.get_as_of = get_as_of - entity.revert_to = revert_to - entity.revert = revert - entity.compare_with = compare_with - Version.compare_with = compare_with - -acts_as_versioned = Statement(VersionedEntityBuilder) - - -def after_revert(func): - """ - Decorator for watching for revert events. - """ - func._elixir_after_revert = True - return func - - diff --git a/libs/elixir/fields.py b/libs/elixir/fields.py deleted file mode 100644 index 8659cdd8..00000000 --- a/libs/elixir/fields.py +++ /dev/null @@ -1,191 +0,0 @@ -''' -This module provides support for defining the fields (columns) of your -entities. Elixir currently supports two syntaxes to do so: the default -`Attribute-based syntax`_ as well as the has_field_ DSL statement. - -Attribute-based syntax ----------------------- - -Here is a quick example of how to use the object-oriented syntax. - -.. sourcecode:: python - - class Person(Entity): - id = Field(Integer, primary_key=True) - name = Field(String(50), required=True) - ssn = Field(String(50), unique=True) - biography = Field(Text) - join_date = Field(DateTime, default=datetime.datetime.now) - photo = Field(Binary, deferred=True) - _email = Field(String(20), colname='email', synonym='email') - - def _set_email(self, email): - self._email = email - def _get_email(self): - return self._email - email = property(_get_email, _set_email) - - -The Field class takes one mandatory argument, which is its type. Please refer -to SQLAlchemy documentation for a list of `types supported by SQLAlchemy -`_. - -Following that first mandatory argument, fields can take any number of -optional keyword arguments. Please note that all the **arguments** that are -**not specifically processed by Elixir**, as mentioned in the documentation -below **are passed on to the SQLAlchemy ``Column`` object**. Please refer to -the `SQLAlchemy Column object's documentation -`_ for more details about other -supported keyword arguments. - -The following Elixir-specific arguments are supported: - -+-------------------+---------------------------------------------------------+ -| Argument Name | Description | -+===================+=========================================================+ -| ``required`` | Specify whether or not this field can be set to None | -| | (left without a value). Defaults to ``False``, unless | -| | the field is a primary key. | -+-------------------+---------------------------------------------------------+ -| ``colname`` | Specify a custom name for the column of this field. By | -| | default the column will have the same name as the | -| | attribute. | -+-------------------+---------------------------------------------------------+ -| ``deferred`` | Specify whether this particular column should be | -| | fetched by default (along with the other columns) when | -| | an instance of the entity is fetched from the database | -| | or rather only later on when this particular column is | -| | first referenced. This can be useful when one wants to | -| | avoid loading a large text or binary field into memory | -| | when its not needed. Individual columns can be lazy | -| | loaded by themselves (by using ``deferred=True``) | -| | or placed into groups that lazy-load together (by using | -| | ``deferred`` = `"group_name"`). | -+-------------------+---------------------------------------------------------+ -| ``synonym`` | Specify a synonym name for this field. The field will | -| | also be usable under that name in keyword-based Query | -| | functions such as filter_by. The Synonym class (see the | -| | `properties` module) provides a similar functionality | -| | with an (arguably) nicer syntax, but a limited scope. | -+-------------------+---------------------------------------------------------+ - -has_field ---------- - -The `has_field` statement allows you to define fields one at a time. - -The first argument is the name of the field, the second is its type. Following -these, any number of keyword arguments can be specified for additional -behavior. The following arguments are supported: - -+-------------------+---------------------------------------------------------+ -| Argument Name | Description | -+===================+=========================================================+ -| ``through`` | Specify a relation name to go through. This field will | -| | not exist as a column on the database but will be a | -| | property which automatically proxy values to the | -| | ``attribute`` attribute of the object pointed to by the | -| | relation. If the ``attribute`` argument is not present, | -| | the name of the current field will be used. In an | -| | has_field statement, you can only proxy through a | -| | belongs_to or an has_one relationship. | -+-------------------+---------------------------------------------------------+ -| ``attribute`` | Name of the "endpoint" attribute to proxy to. This | -| | should only be used in combination with the ``through`` | -| | argument. | -+-------------------+---------------------------------------------------------+ - - -Here is a quick example of how to use ``has_field``. - -.. sourcecode:: python - - class Person(Entity): - has_field('id', Integer, primary_key=True) - has_field('name', String(50)) -''' -from sqlalchemy import Column -from sqlalchemy.orm import deferred, synonym -from sqlalchemy.ext.associationproxy import association_proxy - -from elixir.statements import ClassMutator -from elixir.properties import Property - -__doc_all__ = ['Field'] - - -class Field(Property): - ''' - Represents the definition of a 'field' on an entity. - - This class represents a column on the table where the entity is stored. - ''' - - def __init__(self, type, *args, **kwargs): - super(Field, self).__init__() - - self.colname = kwargs.pop('colname', None) - self.synonym = kwargs.pop('synonym', None) - self.deferred = kwargs.pop('deferred', False) - if 'required' in kwargs: - kwargs['nullable'] = not kwargs.pop('required') - self.type = type - self.primary_key = kwargs.get('primary_key', False) - - self.column = None - self.property = None - - self.args = args - self.kwargs = kwargs - - def attach(self, entity, name): - # If no colname was defined (through the 'colname' kwarg), set - # it to the name of the attr. - if self.colname is None: - self.colname = name - super(Field, self).attach(entity, name) - - def create_pk_cols(self): - if self.primary_key: - self.create_col() - - def create_non_pk_cols(self): - if not self.primary_key: - self.create_col() - - def create_col(self): - self.column = Column(self.colname, self.type, - *self.args, **self.kwargs) - self.add_table_column(self.column) - - def create_properties(self): - if self.deferred: - group = None - if isinstance(self.deferred, basestring): - group = self.deferred - self.property = deferred(self.column, group=group) - elif self.name != self.colname: - # if the property name is different from the column name, we need - # to add an explicit property (otherwise nothing is needed as it's - # done automatically by SA) - self.property = self.column - - if self.property is not None: - self.add_mapper_property(self.name, self.property) - - if self.synonym: - self.add_mapper_property(self.synonym, synonym(self.name)) - - -def has_field_handler(entity, name, *args, **kwargs): - if 'through' in kwargs: - setattr(entity, name, - association_proxy(kwargs.pop('through'), - kwargs.pop('attribute', name), - **kwargs)) - return - field = Field(*args, **kwargs) - field.attach(entity, name) - -has_field = ClassMutator(has_field_handler) diff --git a/libs/elixir/options.py b/libs/elixir/options.py deleted file mode 100644 index 27d7d195..00000000 --- a/libs/elixir/options.py +++ /dev/null @@ -1,274 +0,0 @@ -''' -This module provides support for defining several options on your Elixir -entities. There are three different kinds of options that can be set -up, and for this there are three different statements: using_options_, -using_table_options_ and using_mapper_options_. - -Alternatively, these options can be set on all Elixir entities by modifying -the `options_defaults` dictionary before defining any entity. - -`using_options` ---------------- -The 'using_options' DSL statement allows you to set up some additional -behaviors on your model objects, including table names, ordering, and -more. To specify an option, simply supply the option as a keyword -argument onto the statement, as follows: - -.. sourcecode:: python - - class Person(Entity): - name = Field(Unicode(64)) - - using_options(shortnames=True, order_by='name') - -The list of supported arguments are as follows: - -+---------------------+-------------------------------------------------------+ -| Option Name | Description | -+=====================+=======================================================+ -| ``inheritance`` | Specify the type of inheritance this entity must use. | -| | It can be one of ``single``, ``concrete`` or | -| | ``multi``. Defaults to ``single``. | -| | Note that polymorphic concrete inheritance is | -| | currently not implemented. See: | -| | http://www.sqlalchemy.org/docs/05/mappers.html | -| | #mapping-class-inheritance-hierarchies for an | -| | explanation of the different kinds of inheritances. | -+---------------------+-------------------------------------------------------+ -| ``abstract`` | Set 'abstract'=True to declare abstract entity. | -| | Abstract base classes are useful when you want to put | -| | some common information into a number of other | -| | entities. Abstract entity will not be used to create | -| | any database table. Instead, when it is used as a base| -| | class for other entity, its fields will be added to | -| | those of the child class. | -+---------------------+-------------------------------------------------------+ -| ``polymorphic`` | Whether the inheritance should be polymorphic or not. | -| | Defaults to ``True``. The column used to store the | -| | type of each row is named "row_type" by default. You | -| | can change this by passing the desired name for the | -| | column to this argument. | -+---------------------+-------------------------------------------------------+ -| ``identity`` | Specify a custom polymorphic identity. When using | -| | polymorphic inheritance, this value (usually a | -| | string) will represent this particular entity (class) | -| | . It will be used to differentiate it from other | -| | entities (classes) in your inheritance hierarchy when | -| | loading from the database instances of different | -| | entities in that hierarchy at the same time. | -| | This value will be stored by default in the | -| | "row_type" column of the entity's table (see above). | -| | You can either provide a | -| | plain string or a callable. The callable will be | -| | given the entity (ie class) as argument and must | -| | return a value (usually a string) representing the | -| | polymorphic identity of that entity. | -| | By default, this value is automatically generated: it | -| | is the name of the entity lower-cased. | -+---------------------+-------------------------------------------------------+ -| ``metadata`` | Specify a custom MetaData for this entity. | -| | By default, entities uses the global | -| | ``elixir.metadata``. | -| | This option can also be set for all entities of a | -| | module by setting the ``__metadata__`` attribute of | -| | that module. | -+---------------------+-------------------------------------------------------+ -| ``autoload`` | Automatically load column definitions from the | -| | existing database table. | -+---------------------+-------------------------------------------------------+ -| ``tablename`` | Specify a custom tablename. You can either provide a | -| | plain string or a callable. The callable will be | -| | given the entity (ie class) as argument and must | -| | return a string representing the name of the table | -| | for that entity. By default, the tablename is | -| | automatically generated: it is a concatenation of the | -| | full module-path to the entity and the entity (class) | -| | name itself. The result is lower-cased and separated | -| | by underscores ("_"), eg.: for an entity named | -| | "MyEntity" in the module "project1.model", the | -| | generated table name will be | -| | "project1_model_myentity". | -+---------------------+-------------------------------------------------------+ -| ``shortnames`` | Specify whether or not the automatically generated | -| | table names include the full module-path | -| | to the entity. If ``shortnames`` is ``True``, only | -| | the entity name is used. Defaults to ``False``. | -+---------------------+-------------------------------------------------------+ -| ``auto_primarykey`` | If given as string, it will represent the | -| | auto-primary-key's column name. If this option | -| | is True, it will allow auto-creation of a primary | -| | key if there's no primary key defined for the | -| | corresponding entity. If this option is False, | -| | it will disallow auto-creation of a primary key. | -| | Defaults to ``True``. | -+---------------------+-------------------------------------------------------+ -| ``version_id_col`` | If this option is True, it will create a version | -| | column automatically using the default name. If given | -| | as string, it will create the column using that name. | -| | This can be used to prevent concurrent modifications | -| | to the entity's table rows (i.e. it will raise an | -| | exception if it happens). Defaults to ``False``. | -+---------------------+-------------------------------------------------------+ -| ``order_by`` | How to order select results. Either a string or a | -| | list of strings, composed of the field name, | -| | optionally lead by a minus (for descending order). | -+---------------------+-------------------------------------------------------+ -| ``session`` | Specify a custom contextual session for this entity. | -| | By default, entities uses the global | -| | ``elixir.session``. | -| | This option takes a ``ScopedSession`` object or | -| | ``None``. In the later case your entity will be | -| | mapped using a non-contextual mapper which requires | -| | manual session management, as seen in pure SQLAlchemy.| -| | This option can also be set for all entities of a | -| | module by setting the ``__session__`` attribute of | -| | that module. | -+---------------------+-------------------------------------------------------+ -| ``allowcoloverride``| Specify whether it is allowed to override columns. | -| | By default, Elixir forbids you to add a column to an | -| | entity's table which already exist in that table. If | -| | you set this option to ``True`` it will skip that | -| | check. Use with care as it is easy to shoot oneself | -| | in the foot when overriding columns. | -+---------------------+-------------------------------------------------------+ - -For examples, please refer to the examples and unit tests. - -`using_table_options` ---------------------- -The 'using_table_options' DSL statement allows you to set up some -additional options on your entity table. It is meant only to handle the -options which are not supported directly by the 'using_options' statement. -By opposition to the 'using_options' statement, these options are passed -directly to the underlying SQLAlchemy Table object (both non-keyword arguments -and keyword arguments) without any processing. - -For further information, please refer to the `SQLAlchemy table's documentation -`_. - -You might also be interested in the section about `constraints -`_. - -`using_mapper_options` ----------------------- -The 'using_mapper_options' DSL statement allows you to set up some -additional options on your entity mapper. It is meant only to handle the -options which are not supported directly by the 'using_options' statement. -By opposition to the 'using_options' statement, these options are passed -directly to the underlying SQLAlchemy mapper (as keyword arguments) -without any processing. - -For further information, please refer to the `SQLAlchemy mapper -function's documentation -`_. - -`using_options_defaults` ------------------------- -The 'using_options_defaults' DSL statement allows you to set up some -default options on a custom base class. These will be used as the default value -for options of all its subclasses. Note that any option not set within the -using_options_defaults (nor specifically on a particular Entity) will use the -global defaults, so you don't have to provide a default value for all options, -but only those you want to change. Please also note that this statement does -not work on normal entities, and the normal using_options statement does not -work on base classes (because normal options do not and should not propagate to -the children classes). -''' - -from sqlalchemy import Integer, String - -from elixir.statements import ClassMutator - -__doc_all__ = ['options_defaults'] - -OLD_M2MCOL_NAMEFORMAT = "%(tablename)s_%(key)s%(numifself)s" -ALTERNATE_M2MCOL_NAMEFORMAT = "%(inversename)s_%(key)s" - -def default_m2m_column_formatter(data): - if data['selfref']: - return ALTERNATE_M2MCOL_NAMEFORMAT % data - else: - return OLD_M2MCOL_NAMEFORMAT % data - -NEW_M2MCOL_NAMEFORMAT = default_m2m_column_formatter - -# format constants -FKCOL_NAMEFORMAT = "%(relname)s_%(key)s" -M2MCOL_NAMEFORMAT = NEW_M2MCOL_NAMEFORMAT -CONSTRAINT_NAMEFORMAT = "%(tablename)s_%(colnames)s_fk" -MULTIINHERITANCECOL_NAMEFORMAT = "%(entity)s_%(key)s" - -# other global constants -DEFAULT_AUTO_PRIMARYKEY_NAME = "id" -DEFAULT_AUTO_PRIMARYKEY_TYPE = Integer -DEFAULT_VERSION_ID_COL_NAME = "row_version" -DEFAULT_POLYMORPHIC_COL_NAME = "row_type" -POLYMORPHIC_COL_SIZE = 40 -POLYMORPHIC_COL_TYPE = String(POLYMORPHIC_COL_SIZE) - -# debugging/migration help -MIGRATION_TO_07_AID = False - -# -options_defaults = dict( - abstract=False, - inheritance='single', - polymorphic=True, - identity=None, - autoload=False, - tablename=None, - shortnames=False, - auto_primarykey=True, - version_id_col=False, - allowcoloverride=False, - order_by=None, - resolve_root=None, - mapper_options={}, - table_options={} -) - -valid_options = options_defaults.keys() + [ - 'metadata', - 'session', - 'collection' -] - - -def using_options_defaults_handler(entity, **kwargs): - for kwarg in kwargs: - if kwarg not in valid_options: - raise Exception("'%s' is not a valid option for Elixir entities." - % kwarg) - - # We use __dict__ instead of hasattr to not check its presence within the - # parent, and thus update the parent dict instead of creating a local dict. - if not entity.__dict__.get('options_defaults'): - entity.options_defaults = {} - entity.options_defaults.update(kwargs) - - -def using_options_handler(entity, *args, **kwargs): - for kwarg in kwargs: - if kwarg in valid_options: - setattr(entity._descriptor, kwarg, kwargs[kwarg]) - else: - raise Exception("'%s' is not a valid option for Elixir entities." - % kwarg) - - -def using_table_options_handler(entity, *args, **kwargs): - entity._descriptor.table_args.extend(list(args)) - entity._descriptor.table_options.update(kwargs) - - -def using_mapper_options_handler(entity, *args, **kwargs): - entity._descriptor.mapper_options.update(kwargs) - - -using_options_defaults = ClassMutator(using_options_defaults_handler) -using_options = ClassMutator(using_options_handler) -using_table_options = ClassMutator(using_table_options_handler) -using_mapper_options = ClassMutator(using_mapper_options_handler) diff --git a/libs/elixir/properties.py b/libs/elixir/properties.py deleted file mode 100644 index 68ff8fab..00000000 --- a/libs/elixir/properties.py +++ /dev/null @@ -1,244 +0,0 @@ -''' -This module provides support for defining properties on your entities. It both -provides, the `Property` class which acts as a building block for common -properties such as fields and relationships (for those, please consult the -corresponding modules), but also provides some more specialized properties, -such as `ColumnProperty` and `Synonym`. It also provides the GenericProperty -class which allows you to wrap any SQLAlchemy property, and its DSL-syntax -equivalent: has_property_. - -`has_property` --------------- -The ``has_property`` statement allows you to define properties which rely on -their entity's table (and columns) being defined before they can be declared -themselves. The `has_property` statement takes two arguments: first the name of -the property to be defined and second a function (often given as an anonymous -lambda) taking one argument and returning the desired SQLAlchemy property. That -function will be called whenever the entity table is completely defined, and -will be given the .c attribute of the entity as argument (as a way to access -the entity columns). - -Here is a quick example of how to use ``has_property``. - -.. sourcecode:: python - - class OrderLine(Entity): - has_field('quantity', Float) - has_field('unit_price', Float) - has_property('price', - lambda c: column_property( - (c.quantity * c.unit_price).label('price'))) -''' - -from elixir.statements import PropertyStatement -from sqlalchemy.orm import column_property, synonym - -__doc_all__ = ['EntityBuilder', 'Property', 'GenericProperty', - 'ColumnProperty'] - -class EntityBuilder(object): - ''' - Abstract base class for all entity builders. An Entity builder is a class - of objects which can be added to an Entity (usually by using special - properties or statements) to "build" that entity. Building an entity, - meaning to add columns to its "main" table, create other tables, add - properties to its mapper, ... To do so an EntityBuilder must override the - corresponding method(s). This is to ensure the different operations happen - in the correct order (for example, that the table is fully created before - the mapper that use it is defined). - ''' - def create_pk_cols(self): - pass - - def create_non_pk_cols(self): - pass - - def before_table(self): - pass - - def create_tables(self): - ''' - Subclasses may override this method to create tables. - ''' - - def after_table(self): - pass - - def create_properties(self): - ''' - Subclasses may override this method to add properties to the involved - entity. - ''' - - def before_mapper(self): - pass - - def after_mapper(self): - pass - - def finalize(self): - pass - - # helper methods - def add_table_column(self, column): - self.entity._descriptor.add_column(column) - - def add_mapper_property(self, name, prop): - self.entity._descriptor.add_property(name, prop) - - def add_mapper_extension(self, ext): - self.entity._descriptor.add_mapper_extension(ext) - - -class CounterMeta(type): - ''' - A simple meta class which adds a ``_counter`` attribute to the instances of - the classes it is used on. This counter is simply incremented for each new - instance. - ''' - counter = 0 - - def __call__(self, *args, **kwargs): - instance = type.__call__(self, *args, **kwargs) - instance._counter = CounterMeta.counter - CounterMeta.counter += 1 - return instance - - -class Property(EntityBuilder): - ''' - Abstract base class for all properties of an Entity. - ''' - __metaclass__ = CounterMeta - - def __init__(self, *args, **kwargs): - self.entity = None - self.name = None - - def attach(self, entity, name): - """Attach this property to its entity, using 'name' as name. - - Properties will be attached in the order they were declared. - """ - self.entity = entity - self.name = name - - # register this property as a builder - entity._descriptor.builders.append(self) - - def __repr__(self): - return "Property(%s, %s)" % (self.name, self.entity) - - -class GenericProperty(Property): - ''' - Generic catch-all class to wrap an SQLAlchemy property. - - .. sourcecode:: python - - class OrderLine(Entity): - quantity = Field(Float) - unit_price = Field(Numeric) - price = GenericProperty(lambda c: column_property( - (c.quantity * c.unit_price).label('price'))) - ''' - - def __init__(self, prop, *args, **kwargs): - super(GenericProperty, self).__init__(*args, **kwargs) - self.prop = prop - #XXX: move this to Property? - self.args = args - self.kwargs = kwargs - - def create_properties(self): - if hasattr(self.prop, '__call__'): - prop_value = self.prop(self.entity.table.c) - else: - prop_value = self.prop - prop_value = self.evaluate_property(prop_value) - self.add_mapper_property(self.name, prop_value) - - def evaluate_property(self, prop): - if self.args or self.kwargs: - raise Exception('superfluous arguments passed to GenericProperty') - return prop - - -class ColumnProperty(GenericProperty): - ''' - A specialized form of the GenericProperty to generate SQLAlchemy - ``column_property``'s. - - It takes a function (often given as an anonymous lambda) as its first - argument. Other arguments and keyword arguments are forwarded to the - column_property construct. That first-argument function must accept exactly - one argument and must return the desired (scalar-returning) SQLAlchemy - ClauseElement. - - The function will be called whenever the entity table is completely - defined, and will be given - the .c attribute of the table of the entity as argument (as a way to - access the entity columns). The ColumnProperty will first wrap your - ClauseElement in an - "empty" label (ie it will be labelled automatically during queries), - then wrap that in a column_property. - - .. sourcecode:: python - - class OrderLine(Entity): - quantity = Field(Float) - unit_price = Field(Numeric) - price = ColumnProperty(lambda c: c.quantity * c.unit_price, - deferred=True) - - Please look at the `corresponding SQLAlchemy - documentation `_ for details. - ''' - - def evaluate_property(self, prop): - return column_property(prop.label(None), *self.args, **self.kwargs) - - -class Synonym(GenericProperty): - ''' - This class represents a synonym property of another property (column, ...) - of an entity. As opposed to the `synonym` kwarg to the Field class (which - share the same goal), this class can be used to define a synonym of a - property defined in a parent class (of the current class). On the other - hand, it cannot define a synonym for the purpose of using a standard python - property in queries. See the Field class for details on that usage. - - .. sourcecode:: python - - class Person(Entity): - name = Field(String(30)) - primary_email = Field(String(100)) - email_address = Synonym('primary_email') - - class User(Person): - user_name = Synonym('name') - password = Field(String(20)) - ''' - - def evaluate_property(self, prop): - return synonym(prop, *self.args, **self.kwargs) - -#class Composite(GenericProperty): -# def __init__(self, prop): -# super(GenericProperty, self).__init__() -# self.prop = prop - -# def evaluate_property(self, prop): -# return composite(prop.label(self.name)) - -#start = Composite(Point, lambda c: (c.x1, c.y1)) - -#mapper(Vertex, vertices, properties={ -# 'start':composite(Point, vertices.c.x1, vertices.c.y1), -# 'end':composite(Point, vertices.c.x2, vertices.c.y2) -#}) - - -has_property = PropertyStatement(GenericProperty) - diff --git a/libs/elixir/relationships.py b/libs/elixir/relationships.py deleted file mode 100644 index 6c14dbb6..00000000 --- a/libs/elixir/relationships.py +++ /dev/null @@ -1,1247 +0,0 @@ -''' -This module provides support for defining relationships between your Elixir -entities. Elixir currently supports two syntaxes to do so: the default -`Attribute-based syntax`_ which supports the following types of relationships: -ManyToOne_, OneToMany_, OneToOne_ and ManyToMany_, as well as a -`DSL-based syntax`_ which provides the following statements: belongs_to_, -has_many_, has_one_ and has_and_belongs_to_many_. - -====================== -Attribute-based syntax -====================== - -The first argument to all these "normal" relationship classes is the name of -the class (entity) you are relating to. - -Following that first mandatory argument, any number of additional keyword -arguments can be specified for advanced behavior. See each relationship type -for a list of their specific keyword arguments. At this point, we'll just note -that all the arguments that are not specifically processed by Elixir, as -mentioned in the documentation below are passed on to the SQLAlchemy -``relation`` function. So, please refer to the `SQLAlchemy relation function's -documentation `_ for further detail about which -keyword arguments are supported. - -You should keep in mind that the following -keyword arguments are automatically generated by Elixir and should not be used -unless you want to override the value provided by Elixir: ``uselist``, -``remote_side``, ``secondary``, ``primaryjoin`` and ``secondaryjoin``. - -Additionally, if you want a bidirectionnal relationship, you should define the -inverse relationship on the other entity explicitly (as opposed to how -SQLAlchemy's backrefs are defined). In non-ambiguous situations, Elixir will -match relationships together automatically. If there are several relationships -of the same type between two entities, Elixir is not able to determine which -relationship is the inverse of which, so you have to disambiguate the -situation by giving the name of the inverse relationship in the ``inverse`` -keyword argument. - -Here is a detailed explanation of each relation type: - -`ManyToOne` ------------ - -Describes the child's side of a parent-child relationship. For example, -a `Pet` object may belong to its owner, who is a `Person`. This could be -expressed like so: - -.. sourcecode:: python - - class Pet(Entity): - owner = ManyToOne('Person') - -Behind the scene, assuming the primary key of the `Person` entity is -an integer column named `id`, the ``ManyToOne`` relationship will -automatically add an integer column named `owner_id` to the entity, with a -foreign key referencing the `id` column of the `Person` entity. - -In addition to the keyword arguments inherited from SQLAlchemy's relation -function, ``ManyToOne`` relationships accept the following optional arguments -which will be directed to the created column: - -+----------------------+------------------------------------------------------+ -| Option Name | Description | -+======================+======================================================+ -| ``colname`` | Specify a custom name for the foreign key column(s). | -| | This argument accepts either a single string or a | -| | list of strings. The number of strings passed must | -| | match the number of primary key columns of the target| -| | entity. If this argument is not used, the name of the| -| | column(s) is generated with the pattern | -| | defined in options.FKCOL_NAMEFORMAT, which is, by | -| | default: "%(relname)s_%(key)s", where relname is the | -| | name of the ManyToOne relationship, and 'key' is the | -| | name (key) of the primary column in the target | -| | entity. That's with, in the above Pet/owner example, | -| | the name of the column would be: "owner_id". | -+----------------------+------------------------------------------------------+ -| ``required`` | Specify whether or not this field can be set to None | -| | (left without a value). Defaults to ``False``, | -| | unless the field is a primary key. | -+----------------------+------------------------------------------------------+ -| ``primary_key`` | Specify whether or not the column(s) created by this | -| | relationship should act as a primary_key. | -| | Defaults to ``False``. | -+----------------------+------------------------------------------------------+ -| ``column_kwargs`` | A dictionary holding any other keyword argument you | -| | might want to pass to the Column. | -+----------------------+------------------------------------------------------+ -| ``target_column`` | Name (or list of names) of the target column(s). | -| | If this argument is not specified, the target entity | -| | primary key column(s) are used. | -+----------------------+------------------------------------------------------+ - -The following optional arguments are also supported to customize the -ForeignKeyConstraint that is created: - -+----------------------+------------------------------------------------------+ -| Option Name | Description | -+======================+======================================================+ -| ``use_alter`` | If True, SQLAlchemy will add the constraint in a | -| | second SQL statement (as opposed to within the | -| | create table statement). This permits to define | -| | tables with a circular foreign key dependency | -| | between them. | -+----------------------+------------------------------------------------------+ -| ``ondelete`` | Value for the foreign key constraint ondelete clause.| -| | May be one of: ``cascade``, ``restrict``, | -| | ``set null``, or ``set default``. | -+----------------------+------------------------------------------------------+ -| ``onupdate`` | Value for the foreign key constraint onupdate clause.| -| | May be one of: ``cascade``, ``restrict``, | -| | ``set null``, or ``set default``. | -+----------------------+------------------------------------------------------+ -| ``constraint_kwargs``| A dictionary holding any other keyword argument you | -| | might want to pass to the Constraint. | -+----------------------+------------------------------------------------------+ - -In some cases, you may want to declare the foreign key column explicitly, -instead of letting it be generated automatically. There are several reasons to -that: it could be because you want to declare it with precise arguments and -using column_kwargs makes your code ugly, or because the name of -your column conflicts with the property name (in which case an error is -thrown). In those cases, you can use the ``field`` argument to specify an -already-declared field to be used for the foreign key column. - -For example, for the Pet example above, if you want the database column -(holding the foreign key) to be called 'owner', one should use the field -parameter to specify the field manually. - -.. sourcecode:: python - - class Pet(Entity): - owner_id = Field(Integer, colname='owner') - owner = ManyToOne('Person', field=owner_id) - -+----------------------+------------------------------------------------------+ -| Option Name | Description | -+======================+======================================================+ -| ``field`` | Specify the previously-declared field to be used for | -| | the foreign key column. Use of this parameter is | -| | mutually exclusive with the colname and column_kwargs| -| | arguments. | -+----------------------+------------------------------------------------------+ - - -Additionally, Elixir supports the belongs_to_ statement as an alternative, -DSL-based, syntax to define ManyToOne_ relationships. - - -`OneToMany` ------------ - -Describes the parent's side of a parent-child relationship when there can be -several children. For example, a `Person` object has many children, each of -them being a `Person`. This could be expressed like so: - -.. sourcecode:: python - - class Person(Entity): - parent = ManyToOne('Person') - children = OneToMany('Person') - -Note that a ``OneToMany`` relationship **cannot exist** without a -corresponding ``ManyToOne`` relationship in the other way. This is because the -``OneToMany`` relationship needs the foreign key created by the ``ManyToOne`` -relationship. - -In addition to keyword arguments inherited from SQLAlchemy, ``OneToMany`` -relationships accept the following optional (keyword) arguments: - -+--------------------+--------------------------------------------------------+ -| Option Name | Description | -+====================+========================================================+ -| ``order_by`` | Specify which field(s) should be used to sort the | -| | results given by accessing the relation field. | -| | Note that this sort order is only applied when loading | -| | objects from the database. Objects appended to the | -| | collection afterwards are not re-sorted in-memory on | -| | the fly. | -| | This argument accepts either a string or a list of | -| | strings, each corresponding to the name of a field in | -| | the target entity. These field names can optionally be | -| | prefixed by a minus (for descending order). | -+--------------------+--------------------------------------------------------+ -| ``filter`` | Specify a filter criterion (as a clause element) for | -| | this relationship. This criterion will be ``and_`` ed | -| | with the normal join criterion (primaryjoin) generated | -| | by Elixir for the relationship. For example: | -| | boston_addresses = | -| | OneToMany('Address', filter=Address.city == 'Boston') | -+--------------------+--------------------------------------------------------+ - -Additionally, Elixir supports an alternate, DSL-based, syntax to define -OneToMany_ relationships, with the has_many_ statement. - - -`OneToOne` ----------- - -Describes the parent's side of a parent-child relationship when there is only -one child. For example, a `Car` object has one gear stick, which is -represented as a `GearStick` object. This could be expressed like so: - -.. sourcecode:: python - - class Car(Entity): - gear_stick = OneToOne('GearStick', inverse='car') - - class GearStick(Entity): - car = ManyToOne('Car') - -Note that a ``OneToOne`` relationship **cannot exist** without a corresponding -``ManyToOne`` relationship in the other way. This is because the ``OneToOne`` -relationship needs the foreign_key created by the ``ManyToOne`` relationship. - -Additionally, Elixir supports an alternate, DSL-based, syntax to define -OneToOne_ relationships, with the has_one_ statement. - - -`ManyToMany` ------------- - -Describes a relationship in which one kind of entity can be related to several -objects of the other kind but the objects of that other kind can be related to -several objects of the first kind. For example, an `Article` can have several -tags, but the same `Tag` can be used on several articles. - -.. sourcecode:: python - - class Article(Entity): - tags = ManyToMany('Tag') - - class Tag(Entity): - articles = ManyToMany('Article') - -Behind the scene, the ``ManyToMany`` relationship will automatically create an -intermediate table to host its data. - -Note that you don't necessarily need to define the inverse relationship. In -our example, even though we want tags to be usable on several articles, we -might not be interested in which articles correspond to a particular tag. In -that case, we could have omitted the `Tag` side of the relationship. - -If your ``ManyToMany`` relationship is self-referencial, the entity -containing it is autoloaded (and you don't intend to specify both the -primaryjoin and secondaryjoin arguments manually), you must specify at least -one of either the ``remote_colname`` or ``local_colname`` argument. - -In addition to keyword arguments inherited from SQLAlchemy, ``ManyToMany`` -relationships accept the following optional (keyword) arguments: - -+--------------------+--------------------------------------------------------+ -| Option Name | Description | -+====================+========================================================+ -| ``tablename`` | Specify a custom name for the intermediary table. This | -| | can be used both when the tables needs to be created | -| | and when the table is autoloaded/reflected from the | -| | database. If this argument is not used, a name will be | -| | automatically generated by Elixir depending on the name| -| | of the tables of the two entities of the relationship, | -| | the name of the relationship, and, if present, the name| -| | of its inverse. Even though this argument is optional, | -| | it is wise to use it if you are not sure what are the | -| | exact consequence of using a generated table name. | -+--------------------+--------------------------------------------------------+ -| ``schema`` | Specify a custom schema for the intermediate table. | -| | This can be used both when the tables needs to | -| | be created and when the table is autoloaded/reflected | -| | from the database. | -+--------------------+--------------------------------------------------------+ -| ``remote_colname`` | A string or list of strings specifying the names of | -| | the column(s) in the intermediary table which | -| | reference the "remote"/target entity's table. | -+--------------------+--------------------------------------------------------+ -| ``local_colname`` | A string or list of strings specifying the names of | -| | the column(s) in the intermediary table which | -| | reference the "local"/current entity's table. | -+--------------------+--------------------------------------------------------+ -| ``table`` | Use a manually created table. If this argument is | -| | used, Elixir will not generate a table for this | -| | relationship, and use the one given instead. This | -| | argument only accepts SQLAlchemy's Table objects. | -+--------------------+--------------------------------------------------------+ -| ``order_by`` | Specify which field(s) should be used to sort the | -| | results given by accessing the relation field. | -| | Note that this sort order is only applied when loading | -| | objects from the database. Objects appended to the | -| | collection afterwards are not re-sorted in-memory on | -| | the fly. | -| | This argument accepts either a string or a list of | -| | strings, each corresponding to the name of a field in | -| | the target entity. These field names can optionally be | -| | prefixed by a minus (for descending order). | -+----------------------+------------------------------------------------------+ -| ``ondelete`` | Value for the foreign key constraint ondelete clause. | -| | May be one of: ``cascade``, ``restrict``, | -| | ``set null``, or ``set default``. | -+--------------------+--------------------------------------------------------+ -| ``onupdate`` | Value for the foreign key constraint onupdate clause. | -| | May be one of: ``cascade``, ``restrict``, | -| | ``set null``, or ``set default``. | -+--------------------+--------------------------------------------------------+ -| ``table_kwargs`` | A dictionary holding any other keyword argument you | -| | might want to pass to the underlying Table object. | -+--------------------+--------------------------------------------------------+ - - -================ -DSL-based syntax -================ - -The following DSL statements provide an alternative way to define relationships -between your entities. The first argument to all those statements is the name -of the relationship, the second is the 'kind' of object you are relating to -(it is usually given using the ``of_kind`` keyword). - -`belongs_to` ------------- - -The ``belongs_to`` statement is the DSL syntax equivalent to the ManyToOne_ -relationship. As such, it supports all the same arguments as ManyToOne_ -relationships. - -.. sourcecode:: python - - class Pet(Entity): - belongs_to('feeder', of_kind='Person') - belongs_to('owner', of_kind='Person', colname="owner_id") - - -`has_many` ----------- - -The ``has_many`` statement is the DSL syntax equivalent to the OneToMany_ -relationship. As such, it supports all the same arguments as OneToMany_ -relationships. - -.. sourcecode:: python - - class Person(Entity): - belongs_to('parent', of_kind='Person') - has_many('children', of_kind='Person') - -There is also an alternate form of the ``has_many`` relationship that takes -only two keyword arguments: ``through`` and ``via`` in order to encourage a -richer form of many-to-many relationship that is an alternative to the -``has_and_belongs_to_many`` statement. Here is an example: - -.. sourcecode:: python - - class Person(Entity): - has_field('name', Unicode) - has_many('assignments', of_kind='Assignment') - has_many('projects', through='assignments', via='project') - - class Assignment(Entity): - has_field('start_date', DateTime) - belongs_to('person', of_kind='Person') - belongs_to('project', of_kind='Project') - - class Project(Entity): - has_field('title', Unicode) - has_many('assignments', of_kind='Assignment') - -In the above example, a `Person` has many `projects` through the `Assignment` -relationship object, via a `project` attribute. - - -`has_one` ---------- - -The ``has_one`` statement is the DSL syntax equivalent to the OneToOne_ -relationship. As such, it supports all the same arguments as OneToOne_ -relationships. - -.. sourcecode:: python - - class Car(Entity): - has_one('gear_stick', of_kind='GearStick', inverse='car') - - class GearStick(Entity): - belongs_to('car', of_kind='Car') - - -`has_and_belongs_to_many` -------------------------- - -The ``has_and_belongs_to_many`` statement is the DSL syntax equivalent to the -ManyToMany_ relationship. As such, it supports all the same arguments as -ManyToMany_ relationships. - -.. sourcecode:: python - - class Article(Entity): - has_and_belongs_to_many('tags', of_kind='Tag') - - class Tag(Entity): - has_and_belongs_to_many('articles', of_kind='Article') - -''' - -import warnings - -from sqlalchemy import ForeignKeyConstraint, Column, Table, and_ -from sqlalchemy.orm import relation, backref, class_mapper -from sqlalchemy.ext.associationproxy import association_proxy - -import options -from elixir.statements import ClassMutator -from elixir.properties import Property -from elixir.entity import EntityMeta, DEBUG - -__doc_all__ = [] - - -class Relationship(Property): - ''' - Base class for relationships. - ''' - - def __init__(self, of_kind, inverse=None, *args, **kwargs): - super(Relationship, self).__init__() - - self.of_kind = of_kind - self.inverse_name = inverse - - self._target = None - - self.property = None # sqlalchemy property - self.backref = None # sqlalchemy backref - - #TODO: unused for now - self.args = args - self.kwargs = kwargs - - def attach(self, entity, name): - super(Relationship, self).attach(entity, name) - entity._descriptor.relationships.append(self) - - def create_pk_cols(self): - self.create_keys(True) - - def create_non_pk_cols(self): - self.create_keys(False) - - def create_keys(self, pk): - ''' - Subclasses (ie. concrete relationships) may override this method to - create foreign keys. - ''' - - def create_properties(self): - if self.property or self.backref: - return - - kwargs = self.get_prop_kwargs() - if 'order_by' in kwargs: - kwargs['order_by'] = \ - self.target._descriptor.translate_order_by(kwargs['order_by']) - - # transform callable arguments - for arg in ('primaryjoin', 'secondaryjoin', 'remote_side', - 'foreign_keys'): - kwarg = kwargs.get(arg, None) - if hasattr(kwarg, '__call__'): - kwargs[arg] = kwarg() - - # viewonly relationships need to create "standalone" relations (ie - # shouldn't be a backref of another relation). - if self.inverse and not kwargs.get('viewonly', False): - # check if the inverse was already processed (and thus has already - # defined a backref we can use) - if self.inverse.backref: - # let the user override the backref argument - if 'backref' not in kwargs: - kwargs['backref'] = self.inverse.backref - else: - # SQLAlchemy doesn't like when 'secondary' is both defined on - # the relation and the backref - kwargs.pop('secondary', None) - - # define backref for use by the inverse - self.backref = backref(self.name, **kwargs) - return - - self.property = relation(self.target, **kwargs) - self.add_mapper_property(self.name, self.property) - - @property - def target(self): - if not self._target: - if isinstance(self.of_kind, basestring): - collection = self.entity._descriptor.collection - self._target = collection.resolve(self.of_kind, self.entity) - else: - self._target = self.of_kind - return self._target - - @property - def inverse(self): - if not hasattr(self, '_inverse'): - if self.inverse_name: - desc = self.target._descriptor - inverse = desc.find_relationship(self.inverse_name) - if inverse is None: - raise Exception( - "Couldn't find a relationship named '%s' in " - "entity '%s' or its parent entities." - % (self.inverse_name, self.target.__name__)) - assert self.match_type_of(inverse), \ - "Relationships '%s' in entity '%s' and '%s' in entity " \ - "'%s' cannot be inverse of each other because their " \ - "types do not form a valid combination." % \ - (self.name, self.entity.__name__, - self.inverse_name, self.target.__name__) - else: - check_reverse = not self.kwargs.get('viewonly', False) - if isinstance(self.target, EntityMeta): - inverse = self.target._descriptor.get_inverse_relation( - self, check_reverse=check_reverse) - else: - inverse = None - self._inverse = inverse - if inverse and not self.kwargs.get('viewonly', False): - inverse._inverse = self - - return self._inverse - - def match_type_of(self, other): - return False - - def is_inverse(self, other): - # viewonly relationships are not symmetrical: a viewonly relationship - # should have exactly one inverse (a ManyToOne relationship), but that - # inverse shouldn't have the viewonly relationship as its inverse. - return not other.kwargs.get('viewonly', False) and \ - other is not self and \ - self.match_type_of(other) and \ - self.entity == other.target and \ - other.entity == self.target and \ - (self.inverse_name == other.name or not self.inverse_name) and \ - (other.inverse_name == self.name or not other.inverse_name) - - -class ManyToOne(Relationship): - ''' - - ''' - - def __init__(self, of_kind, - column_kwargs=None, - colname=None, required=None, primary_key=None, - field=None, - constraint_kwargs=None, - use_alter=None, ondelete=None, onupdate=None, - target_column=None, - *args, **kwargs): - - # 1) handle column-related args - - # check that the column arguments don't conflict - assert not (field and (column_kwargs or colname)), \ - "ManyToOne can accept the 'field' argument or column " \ - "arguments ('colname' or 'column_kwargs') but not both!" - - if colname and not isinstance(colname, list): - colname = [colname] - self.colname = colname or [] - - column_kwargs = column_kwargs or {} - # kwargs go by default to the relation(), so we need to manually - # extract those targeting the Column - if required is not None: - column_kwargs['nullable'] = not required - if primary_key is not None: - column_kwargs['primary_key'] = primary_key - # by default, created columns will have an index. - column_kwargs.setdefault('index', True) - self.column_kwargs = column_kwargs - - if field and not isinstance(field, list): - field = [field] - self.field = field or [] - - # 2) handle constraint kwargs - constraint_kwargs = constraint_kwargs or {} - if use_alter is not None: - constraint_kwargs['use_alter'] = use_alter - if ondelete is not None: - constraint_kwargs['ondelete'] = ondelete - if onupdate is not None: - constraint_kwargs['onupdate'] = onupdate - self.constraint_kwargs = constraint_kwargs - - # 3) misc arguments - if target_column and not isinstance(target_column, list): - target_column = [target_column] - self.target_column = target_column - - self.foreign_key = [] - self.primaryjoin_clauses = [] - - super(ManyToOne, self).__init__(of_kind, *args, **kwargs) - - def match_type_of(self, other): - return isinstance(other, (OneToMany, OneToOne)) - - @property - def target_table(self): - if isinstance(self.target, EntityMeta): - return self.target._descriptor.table - else: - return class_mapper(self.target).local_table - - def create_keys(self, pk): - ''' - Find all primary keys on the target and create foreign keys on the - source accordingly. - ''' - - if self.foreign_key: - return - - if self.column_kwargs.get('primary_key', False) != pk: - return - - source_desc = self.entity._descriptor - if isinstance(self.target, EntityMeta): - # make sure the target has all its pk set up - #FIXME: this is not enough when specifying target_column manually, - # on unique, non-pk col, see tests/test_m2o.py:test_non_pk_forward - self.target._descriptor.create_pk_cols() - - #XXX: another option, instead of the FakeTable, would be to create an - # EntityDescriptor for the SA class. - target_table = self.target_table - - if source_desc.autoload: - #TODO: allow target_column to be used as an alternative to - # specifying primaryjoin, to be consistent with non-autoloaded - # tables - if self.colname: - if 'primaryjoin' not in self.kwargs: - self.primaryjoin_clauses = \ - _get_join_clauses(self.entity.table, - self.colname, None, - target_table)[0] - if not self.primaryjoin_clauses: - colnames = ', '.join(self.colname) - raise Exception( - "Couldn't find a foreign key constraint in table " - "'%s' using the following columns: %s." - % (self.entity.table.name, colnames)) - else: - # in this case we let SA handle everything. - # XXX: we might want to try to build join clauses anyway so - # that we know whether there is an ambiguity or not, and - # suggest using colname if there is one - pass - if self.field: - raise NotImplementedError( - "'field' argument not allowed on autoloaded table " - "relationships.") - else: - fk_refcols = [] - fk_colnames = [] - - if self.target_column is None: - target_columns = target_table.primary_key.columns - else: - target_columns = [target_table.columns[col] - for col in self.target_column] - - if not target_columns: - raise Exception("No primary key found in target table ('%s') " - "for the '%s' relationship of the '%s' entity." - % (target_table.name, self.name, - self.entity.__name__)) - if self.colname and \ - len(self.colname) != len(target_columns): - raise Exception( - "The number of column names provided in the colname " - "keyword argument of the '%s' relationship of the " - "'%s' entity is not the same as the number of columns " - "of the primary key of '%s'." - % (self.name, self.entity.__name__, - self.target.__name__)) - - for key_num, target_col in enumerate(target_columns): - if self.field: - col = self.field[key_num].column - else: - if self.colname: - colname = self.colname[key_num] - else: - colname = options.FKCOL_NAMEFORMAT % \ - {'relname': self.name, - 'key': target_col.key} - - # We can't add the column to the table directly as the - # table might not be created yet. - col = Column(colname, target_col.type, - **self.column_kwargs) - source_desc.add_column(col) - - # If the column name was specified, and it is the same as - # this property's name, there is going to be a conflict. - # Don't allow this to happen. - if col.key == self.name: - raise ValueError( - "ManyToOne named '%s' in '%s' conficts " - " with the column of the same name. " - "You should probably define the foreign key " - "field manually and use the 'field' " - "argument on the ManyToOne relationship" - % (self.name, self.entity.__name__)) - - # Build the list of local columns which will be part of - # the foreign key - self.foreign_key.append(col) - - # Store the names of those columns - fk_colnames.append(col.key) - - # Build the list of column "paths" the foreign key will - # point to - fk_refcols.append("%s.%s" % \ - (target_table.fullname, target_col.key)) - - # Build up the primary join. This is needed when you have - # several ManyToOne relationships between two objects - self.primaryjoin_clauses.append(col == target_col) - - if 'name' not in self.constraint_kwargs: - # In some databases (at least MySQL) the constraint name needs - # to be unique for the whole database, instead of per table. - fk_name = options.CONSTRAINT_NAMEFORMAT % \ - {'tablename': source_desc.tablename, - 'colnames': '_'.join(fk_colnames)} - self.constraint_kwargs['name'] = fk_name - - source_desc.add_constraint( - ForeignKeyConstraint(fk_colnames, fk_refcols, - **self.constraint_kwargs)) - - def get_prop_kwargs(self): - kwargs = {'uselist': False} - - if self.entity.table is self.target_table: - # this is needed because otherwise SA has no way to know what is - # the direction of the relationship since both columns present in - # the primaryjoin belong to the same table. In other words, it is - # necessary to know if this particular relation - # is the many-to-one side, or the one-to-xxx side. The foreignkey - # doesn't help in this case. - kwargs['remote_side'] = \ - [col for col in self.target_table.primary_key.columns] - - if self.primaryjoin_clauses: - kwargs['primaryjoin'] = and_(*self.primaryjoin_clauses) - - kwargs.update(self.kwargs) - - return kwargs - - -class OneToOne(Relationship): - uselist = False - - def __init__(self, of_kind, filter=None, *args, **kwargs): - self.filter = filter - if filter is not None: - # We set viewonly to True by default for filtered relationships, - # unless manually overridden. - # This is not strictly necessary, as SQLAlchemy allows non viewonly - # relationships with a custom join/filter. The example at: - # SADOCS/05/mappers.html#advdatamapping_relation_customjoin - # is not viewonly. Those relationships can be used as if the extra - # filter wasn't present when inserting. This can lead to a - # confusing behavior (if you insert data which doesn't match the - # extra criterion it'll get inserted anyway but you won't see it - # when you query back the attribute after a round-trip to the - # database). - if 'viewonly' not in kwargs: - kwargs['viewonly'] = True - super(OneToOne, self).__init__(of_kind, *args, **kwargs) - - def match_type_of(self, other): - return isinstance(other, ManyToOne) - - def create_keys(self, pk): - # make sure an inverse relationship exists - if self.inverse is None: - raise Exception( - "Couldn't find any relationship in '%s' which " - "match as inverse of the '%s' relationship " - "defined in the '%s' entity. If you are using " - "inheritance you " - "might need to specify inverse relationships " - "manually by using the 'inverse' argument." - % (self.target, self.name, - self.entity)) - - def get_prop_kwargs(self): - kwargs = {'uselist': self.uselist} - - #TODO: for now, we don't break any test if we remove those 2 lines. - # So, we should either complete the selfref test to prove that they - # are indeed useful, or remove them. It might be they are indeed - # useless because the remote_side is already setup in the other way - # (ManyToOne). - if self.entity.table is self.target.table: - # When using a manual/autoloaded table, it will be assigned - # an empty list, which doesn't seem to upset SQLAlchemy - kwargs['remote_side'] = self.inverse.foreign_key - - # Contrary to ManyToMany relationships, we need to specify the join - # clauses even if this relationship is not self-referencial because - # there could be several ManyToOne from the target class to us. - joinclauses = self.inverse.primaryjoin_clauses - if self.filter: - # We need to make a copy of the joinclauses, to not add the filter - # on the backref - joinclauses = joinclauses[:] + [self.filter(self.target.table.c)] - if joinclauses: - kwargs['primaryjoin'] = and_(*joinclauses) - - kwargs.update(self.kwargs) - - return kwargs - - -class OneToMany(OneToOne): - uselist = True - - -class ManyToMany(Relationship): - uselist = True - - def __init__(self, of_kind, tablename=None, - local_colname=None, remote_colname=None, - ondelete=None, onupdate=None, - table=None, schema=None, - filter=None, - table_kwargs=None, - *args, **kwargs): - self.user_tablename = tablename - - if local_colname and not isinstance(local_colname, list): - local_colname = [local_colname] - self.local_colname = local_colname or [] - if remote_colname and not isinstance(remote_colname, list): - remote_colname = [remote_colname] - self.remote_colname = remote_colname or [] - - self.ondelete = ondelete - self.onupdate = onupdate - - self.table = table - self.schema = schema - - #TODO: this can probably be simplified/moved elsewhere since the - #argument disappeared - self.column_format = options.M2MCOL_NAMEFORMAT - if not hasattr(self.column_format, '__call__'): - # we need to store the format in a variable so that the - # closure of the lambda is correct - format = self.column_format - self.column_format = lambda data: format % data - if options.MIGRATION_TO_07_AID: - self.column_format = \ - migration_aid_m2m_column_formatter( - lambda data: options.OLD_M2MCOL_NAMEFORMAT % data, - self.column_format) - - self.filter = filter - if filter is not None: - # We set viewonly to True by default for filtered relationships, - # unless manually overridden. - if 'viewonly' not in kwargs: - kwargs['viewonly'] = True - - self.table_kwargs = table_kwargs or {} - - self.primaryjoin_clauses = [] - self.secondaryjoin_clauses = [] - - super(ManyToMany, self).__init__(of_kind, *args, **kwargs) - - def match_type_of(self, other): - return isinstance(other, ManyToMany) - - def create_tables(self): - if self.table is not None: - if 'primaryjoin' not in self.kwargs or \ - 'secondaryjoin' not in self.kwargs: - self._build_join_clauses() - assert self.inverse is None or self.inverse.table is None or \ - self.inverse.table is self.table - return - - if self.inverse: - inverse = self.inverse - if inverse.table is not None: - self.table = inverse.table - self.primaryjoin_clauses = inverse.secondaryjoin_clauses - self.secondaryjoin_clauses = inverse.primaryjoin_clauses - return - - assert not inverse.user_tablename or not self.user_tablename or \ - inverse.user_tablename == self.user_tablename - assert not inverse.remote_colname or not self.local_colname or \ - inverse.remote_colname == self.local_colname - assert not inverse.local_colname or not self.remote_colname or \ - inverse.local_colname == self.remote_colname - assert not inverse.schema or not self.schema or \ - inverse.schema == self.schema - assert not inverse.table_kwargs or not self.table_kwargs or \ - inverse.table_kwargs == self.table_kwargs - - self.user_tablename = inverse.user_tablename or self.user_tablename - self.local_colname = inverse.remote_colname or self.local_colname - self.remote_colname = inverse.local_colname or self.remote_colname - self.schema = inverse.schema or self.schema - self.local_colname = inverse.remote_colname or self.local_colname - - # compute table_kwargs - complete_kwargs = options.options_defaults['table_options'].copy() - complete_kwargs.update(self.table_kwargs) - - #needs: table_options['schema'], autoload, tablename, primary_keys, - #entity.__name__, table_fullname - e1_desc = self.entity._descriptor - e2_desc = self.target._descriptor - - e1_schema = e1_desc.table_options.get('schema', None) - e2_schema = e2_desc.table_options.get('schema', None) - schema = (self.schema is not None) and self.schema or e1_schema - - assert e1_schema == e2_schema or self.schema, \ - "Schema %r for entity %s differs from schema %r of entity %s." \ - " Consider using the schema-parameter. "\ - % (e1_schema, self.entity.__name__, - e2_schema, self.target.__name__) - - # First, we compute the name of the table. Note that some of the - # intermediary variables are reused later for the constraint - # names. - - # We use the name of the relation for the first entity - # (instead of the name of its primary key), so that we can - # have two many-to-many relations between the same objects - # without having a table name collision. - source_part = "%s_%s" % (e1_desc.tablename, self.name) - - # And we use only the name of the table of the second entity - # when there is no inverse, so that a many-to-many relation - # can be defined without an inverse. - if self.inverse: - target_part = "%s_%s" % (e2_desc.tablename, self.inverse.name) - else: - target_part = e2_desc.tablename - - if self.user_tablename: - tablename = self.user_tablename - else: - # We need to keep the table name consistent (independant of - # whether this relation or its inverse is setup first). - if self.inverse and source_part < target_part: - #XXX: use a different scheme for selfref (to not include the - # table name twice)? - tablename = "%s__%s" % (target_part, source_part) - else: - tablename = "%s__%s" % (source_part, target_part) - - if options.MIGRATION_TO_07_AID: - oldname = (self.inverse and - e1_desc.tablename < e2_desc.tablename) and \ - "%s__%s" % (target_part, source_part) or \ - "%s__%s" % (source_part, target_part) - if oldname != tablename: - warnings.warn( - "The generated table name for the '%s' relationship " - "on the '%s' entity changed from '%s' (the name " - "generated by Elixir 0.6.1 and earlier) to '%s'. " - "You should either rename the table in the database " - "to the new name or use the tablename argument on the " - "relationship to force the old name: tablename='%s'!" - % (self.name, self.entity.__name__, oldname, - tablename, oldname)) - - if e1_desc.autoload: - if not e2_desc.autoload: - raise Exception( - "Entity '%s' is autoloaded and its '%s' " - "ManyToMany relationship points to " - "the '%s' entity which is not autoloaded" - % (self.entity.__name__, self.name, - self.target.__name__)) - - self.table = Table(tablename, e1_desc.metadata, autoload=True, - **complete_kwargs) - if 'primaryjoin' not in self.kwargs or \ - 'secondaryjoin' not in self.kwargs: - self._build_join_clauses() - else: - # We pre-compute the names of the foreign key constraints - # pointing to the source (local) entity's table and to the - # target's table - - # In some databases (at least MySQL) the constraint names need - # to be unique for the whole database, instead of per table. - source_fk_name = "%s_fk" % source_part - if self.inverse: - target_fk_name = "%s_fk" % target_part - else: - target_fk_name = "%s_inverse_fk" % source_part - - columns = [] - constraints = [] - - for num, desc, fk_name, rel, inverse, colnames, join_clauses in ( - (0, e1_desc, source_fk_name, self, self.inverse, - self.local_colname, self.primaryjoin_clauses), - (1, e2_desc, target_fk_name, self.inverse, self, - self.remote_colname, self.secondaryjoin_clauses)): - - fk_colnames = [] - fk_refcols = [] - if colnames: - assert len(colnames) == len(desc.primary_keys) - else: - # The data generated here will be fed to the M2M column - # formatter to generate the name of the columns of the - # intermediate table for *one* side of the relationship, - # that is, from the intermediate table to the current - # entity, as stored in the "desc" variable. - data = {# A) relationships info - - # the name of the rel going *from* the entity - # we are currently generating a column pointing - # *to*. This is generally *not* what you want to - # use. eg in a "Post" and "Tag" example, with - # relationships named 'tags' and 'posts', when - # creating the columns from the intermediate - # table to the "Post" entity, 'relname' will - # contain 'tags'. - 'relname': rel and rel.name or 'inverse', - - # the name of the inverse relationship. In the - # above example, 'inversename' will contain - # 'posts'. - 'inversename': inverse and inverse.name - or 'inverse', - # is A == B? - 'selfref': e1_desc is e2_desc, - # provided for backward compatibility, DO NOT USE! - 'num': num, - # provided for backward compatibility, DO NOT USE! - 'numifself': e1_desc is e2_desc and str(num + 1) - or '', - # B) target information (from the perspective of - # the intermediate table) - 'target': desc.entity, - 'entity': desc.entity.__name__.lower(), - 'tablename': desc.tablename, - - # C) current (intermediate) table name - 'current_table': tablename - } - colnames = [] - for pk_col in desc.primary_keys: - data.update(key=pk_col.key) - colnames.append(self.column_format(data)) - - for pk_col, colname in zip(desc.primary_keys, colnames): - col = Column(colname, pk_col.type, primary_key=True) - columns.append(col) - - # Build the list of local columns which will be part - # of the foreign key. - fk_colnames.append(colname) - - # Build the list of column "paths" the foreign key will - # point to - target_path = "%s.%s" % (desc.table_fullname, pk_col.key) - fk_refcols.append(target_path) - - # Build join clauses (in case we have a self-ref) - if self.entity is self.target: - join_clauses.append(col == pk_col) - - onupdate = rel and rel.onupdate - ondelete = rel and rel.ondelete - - #FIXME: fk_name is misleading - constraints.append( - ForeignKeyConstraint(fk_colnames, fk_refcols, - name=fk_name, onupdate=onupdate, - ondelete=ondelete)) - - args = columns + constraints - - self.table = Table(tablename, e1_desc.metadata, - schema=schema, *args, **complete_kwargs) - if DEBUG: - print self.table.repr2() - - def _build_join_clauses(self): - # In the case we have a self-reference, we need to build join clauses - if self.entity is self.target: - if not self.local_colname and not self.remote_colname: - raise Exception( - "Self-referential ManyToMany " - "relationships in autoloaded entities need to have at " - "least one of either 'local_colname' or 'remote_colname' " - "argument specified. The '%s' relationship in the '%s' " - "entity doesn't have either." - % (self.name, self.entity.__name__)) - - self.primaryjoin_clauses, self.secondaryjoin_clauses = \ - _get_join_clauses(self.table, - self.local_colname, self.remote_colname, - self.entity.table) - - def get_prop_kwargs(self): - kwargs = {'secondary': self.table, - 'uselist': self.uselist} - - if self.filter: - # we need to make a copy of the joinclauses - secondaryjoin_clauses = self.secondaryjoin_clauses[:] + \ - [self.filter(self.target.table.c)] - else: - secondaryjoin_clauses = self.secondaryjoin_clauses - - if self.target is self.entity or self.filter: - kwargs['primaryjoin'] = and_(*self.primaryjoin_clauses) - kwargs['secondaryjoin'] = and_(*secondaryjoin_clauses) - - kwargs.update(self.kwargs) - - return kwargs - - def is_inverse(self, other): - return super(ManyToMany, self).is_inverse(other) and \ - (self.user_tablename == other.user_tablename or - (not self.user_tablename and not other.user_tablename)) - - -def migration_aid_m2m_column_formatter(oldformatter, newformatter): - def debug_formatter(data): - old_name = oldformatter(data) - new_name = newformatter(data) - if new_name != old_name: - complete_data = data.copy() - complete_data.update(old_name=old_name, - new_name=new_name, - targetname=data['target'].__name__) - # Specifying a stacklevel is useless in this case as the name - # generation is triggered by setup_all(), not by the declaration - # of the offending relationship. - warnings.warn("The '%(old_name)s' column in the " - "'%(current_table)s' table, used as the " - "intermediate table for the '%(relname)s' " - "relationship on the '%(targetname)s' entity " - "was renamed to '%(new_name)s'." - % complete_data) - return new_name - return debug_formatter - - -def _get_join_clauses(local_table, local_cols1, local_cols2, target_table): - primary_join, secondary_join = [], [] - cols1 = local_cols1[:] - cols1.sort() - cols1 = tuple(cols1) - - if local_cols2 is not None: - cols2 = local_cols2[:] - cols2.sort() - cols2 = tuple(cols2) - else: - cols2 = None - - # Build a map of fk constraints pointing to the correct table. - # The map is indexed on the local col names. - constraint_map = {} - for constraint in local_table.constraints: - if isinstance(constraint, ForeignKeyConstraint): - use_constraint = True - fk_colnames = [] - - # if all columns point to the correct table, we use the constraint - #TODO: check that it contains as many columns as the pk of the - #target entity, or even that it points to the actual pk columns - for fk in constraint.elements: - if fk.references(target_table): - # local column key - fk_colnames.append(fk.parent.key) - else: - use_constraint = False - if use_constraint: - fk_colnames.sort() - constraint_map[tuple(fk_colnames)] = constraint - - # Either the fk column names match explicitely with the columns given for - # one of the joins (primary or secondary), or we assume the current - # columns match because the columns for this join were not given and we - # know the other join is either not used (is None) or has an explicit - # match. - -#TODO: rewrite this. Even with the comment, I don't even understand it myself. - for cols, constraint in constraint_map.iteritems(): - if cols == cols1 or (cols != cols2 and - not cols1 and (cols2 in constraint_map or - cols2 is None)): - join = primary_join - elif cols == cols2 or (cols2 == () and cols1 in constraint_map): - join = secondary_join - else: - continue - for fk in constraint.elements: - join.append(fk.parent == fk.column) - return primary_join, secondary_join - - -def rel_mutator_handler(target): - def handler(entity, name, of_kind=None, through=None, via=None, - *args, **kwargs): - if through and via: - setattr(entity, name, - association_proxy(through, via, **kwargs)) - return - elif through or via: - raise Exception("'through' and 'via' relationship keyword " - "arguments should be used in combination.") - rel = target(of_kind, *args, **kwargs) - rel.attach(entity, name) - return handler - - -belongs_to = ClassMutator(rel_mutator_handler(ManyToOne)) -has_one = ClassMutator(rel_mutator_handler(OneToOne)) -has_many = ClassMutator(rel_mutator_handler(OneToMany)) -has_and_belongs_to_many = ClassMutator(rel_mutator_handler(ManyToMany)) diff --git a/libs/elixir/statements.py b/libs/elixir/statements.py deleted file mode 100644 index c21bf305..00000000 --- a/libs/elixir/statements.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys - -MUTATORS = '__elixir_mutators__' - -class ClassMutator(object): - ''' - DSL-style syntax - - A ``ClassMutator`` object represents a DSL term. - ''' - - def __init__(self, handler): - ''' - Create a new ClassMutator, using the `handler` callable to process it - when the time will come. - ''' - self.handler = handler - - # called when a mutator (eg. "has_field(...)") is parsed - def __call__(self, *args, **kwargs): - # self in this case is the "generic" mutator (eg "has_field") - - # jam this mutator into the class's mutator list - class_locals = sys._getframe(1).f_locals - mutators = class_locals.setdefault(MUTATORS, []) - mutators.append((self, args, kwargs)) - - def process(self, entity, *args, **kwargs): - ''' - Process one mutator. This version simply calls the handler callable, - but another mutator (sub)class could do more processing. - ''' - self.handler(entity, *args, **kwargs) - - -#TODO: move this to the super class (to be created here) of EntityMeta -def process_mutators(entity): - ''' - Apply all mutators of the given entity. That is, loop over all mutators - in the class's mutator list and process them. - ''' - # we don't use getattr here to not inherit from the parent mutators - # inadvertantly if the current entity hasn't defined any mutator. - mutators = entity.__dict__.get(MUTATORS, []) - for mutator, args, kwargs in mutators: - mutator.process(entity, *args, **kwargs) - -class Statement(ClassMutator): - - def process(self, entity, *args, **kwargs): - builder = self.handler(entity, *args, **kwargs) - entity._descriptor.builders.append(builder) - -class PropertyStatement(ClassMutator): - - def process(self, entity, name, *args, **kwargs): - prop = self.handler(*args, **kwargs) - prop.attach(entity, name) - diff --git a/libs/sqlalchemy/__init__.py b/libs/sqlalchemy/__init__.py index 9a21a70f..f6667f7d 100644 --- a/libs/sqlalchemy/__init__.py +++ b/libs/sqlalchemy/__init__.py @@ -1,15 +1,11 @@ # sqlalchemy/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -import inspect -import sys -import sqlalchemy.exc as exceptions - -from sqlalchemy.sql import ( +from .sql import ( alias, and_, asc, @@ -25,6 +21,7 @@ from sqlalchemy.sql import ( except_all, exists, extract, + false, func, insert, intersect, @@ -42,6 +39,7 @@ from sqlalchemy.sql import ( select, subquery, text, + true, tuple_, type_coerce, union, @@ -49,7 +47,7 @@ from sqlalchemy.sql import ( update, ) -from sqlalchemy.types import ( +from .types import ( BIGINT, BINARY, BLOB, @@ -94,12 +92,11 @@ from sqlalchemy.types import ( ) -from sqlalchemy.schema import ( +from .schema import ( CheckConstraint, Column, ColumnDefault, Constraint, - DDL, DefaultClause, FetchedValue, ForeignKey, @@ -112,17 +109,25 @@ from sqlalchemy.schema import ( Table, ThreadLocalMetaData, UniqueConstraint, - ) - -from sqlalchemy.engine import create_engine, engine_from_config + DDL, +) -__all__ = sorted(name for name, obj in locals().items() - if not (name.startswith('_') or inspect.ismodule(obj))) +from .inspection import inspect +from .engine import create_engine, engine_from_config -__version__ = '0.7.10' +__version__ = '0.9.1' -del inspect, sys +def __go(lcls): + global __all__ -from sqlalchemy import util as _sa_util -_sa_util.importlater.resolve_all() + from . import events + from . import util as _sa_util + + import inspect as _inspect + + __all__ = sorted(name for name, obj in lcls.items() + if not (name.startswith('_') or _inspect.ismodule(obj))) + + _sa_util.dependencies.resolve_all("sqlalchemy") +__go(locals()) \ No newline at end of file diff --git a/libs/sqlalchemy/cextension/processors.c b/libs/sqlalchemy/cextension/processors.c index 427db5d8..003c2e32 100644 --- a/libs/sqlalchemy/cextension/processors.c +++ b/libs/sqlalchemy/cextension/processors.c @@ -1,6 +1,7 @@ /* processors.c -Copyright (C) 2010 Gaetan de Menten gdementen@gmail.com +Copyright (C) 2010-2014 the SQLAlchemy authors and contributors +Copyright (C) 2010-2011 Gaetan de Menten gdementen@gmail.com This module is part of SQLAlchemy and is released under the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -9,13 +10,15 @@ the MIT License: http://www.opensource.org/licenses/mit-license.php #include #include +#define MODULE_NAME "cprocessors" +#define MODULE_DOC "Module containing C versions of data processing functions." + #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX #define PY_SSIZE_T_MIN INT_MIN #endif - static PyObject * int_to_boolean(PyObject *self, PyObject *arg) { @@ -25,7 +28,12 @@ int_to_boolean(PyObject *self, PyObject *arg) if (arg == Py_None) Py_RETURN_NONE; + +#if PY_MAJOR_VERSION >= 3 + l = PyLong_AsLong(arg); +#else l = PyInt_AsLong(arg); +#endif if (l == 0) { res = Py_False; } else if (l == 1) { @@ -64,23 +72,48 @@ to_float(PyObject *self, PyObject *arg) static PyObject * str_to_datetime(PyObject *self, PyObject *arg) { +#if PY_MAJOR_VERSION >= 3 + PyObject *bytes; + PyObject *err_bytes; +#endif const char *str; + int numparsed; unsigned int year, month, day, hour, minute, second, microsecond = 0; PyObject *err_repr; if (arg == Py_None) Py_RETURN_NONE; +#if PY_MAJOR_VERSION >= 3 + bytes = PyUnicode_AsASCIIString(arg); + if (bytes == NULL) + str = NULL; + else + str = PyBytes_AS_STRING(bytes); +#else str = PyString_AsString(arg); +#endif if (str == NULL) { err_repr = PyObject_Repr(arg); if (err_repr == NULL) return NULL; +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(err_repr); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_ValueError, + "Couldn't parse datetime string '%.200s' " + "- value is not a string.", + PyBytes_AS_STRING(err_bytes)); + Py_DECREF(err_bytes); +#else PyErr_Format( PyExc_ValueError, "Couldn't parse datetime string '%.200s' " "- value is not a string.", PyString_AsString(err_repr)); +#endif Py_DECREF(err_repr); return NULL; } @@ -91,15 +124,30 @@ str_to_datetime(PyObject *self, PyObject *arg) not accept "2000-01-01 00:00:00.". I don't know which is better, but they should be coherent. */ - if (sscanf(str, "%4u-%2u-%2u %2u:%2u:%2u.%6u", &year, &month, &day, - &hour, &minute, &second, µsecond) < 6) { + numparsed = sscanf(str, "%4u-%2u-%2u %2u:%2u:%2u.%6u", &year, &month, &day, + &hour, &minute, &second, µsecond); +#if PY_MAJOR_VERSION >= 3 + Py_DECREF(bytes); +#endif + if (numparsed < 6) { err_repr = PyObject_Repr(arg); if (err_repr == NULL) return NULL; +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(err_repr); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_ValueError, + "Couldn't parse datetime string: %.200s", + PyBytes_AS_STRING(err_bytes)); + Py_DECREF(err_bytes); +#else PyErr_Format( PyExc_ValueError, "Couldn't parse datetime string: %.200s", PyString_AsString(err_repr)); +#endif Py_DECREF(err_repr); return NULL; } @@ -110,22 +158,47 @@ str_to_datetime(PyObject *self, PyObject *arg) static PyObject * str_to_time(PyObject *self, PyObject *arg) { +#if PY_MAJOR_VERSION >= 3 + PyObject *bytes; + PyObject *err_bytes; +#endif const char *str; + int numparsed; unsigned int hour, minute, second, microsecond = 0; PyObject *err_repr; if (arg == Py_None) Py_RETURN_NONE; +#if PY_MAJOR_VERSION >= 3 + bytes = PyUnicode_AsASCIIString(arg); + if (bytes == NULL) + str = NULL; + else + str = PyBytes_AS_STRING(bytes); +#else str = PyString_AsString(arg); +#endif if (str == NULL) { err_repr = PyObject_Repr(arg); if (err_repr == NULL) return NULL; + +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(err_repr); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_ValueError, + "Couldn't parse time string '%.200s' - value is not a string.", + PyBytes_AS_STRING(err_bytes)); + Py_DECREF(err_bytes); +#else PyErr_Format( PyExc_ValueError, "Couldn't parse time string '%.200s' - value is not a string.", PyString_AsString(err_repr)); +#endif Py_DECREF(err_repr); return NULL; } @@ -136,15 +209,30 @@ str_to_time(PyObject *self, PyObject *arg) not accept "00:00:00.". I don't know which is better, but they should be coherent. */ - if (sscanf(str, "%2u:%2u:%2u.%6u", &hour, &minute, &second, - µsecond) < 3) { + numparsed = sscanf(str, "%2u:%2u:%2u.%6u", &hour, &minute, &second, + µsecond); +#if PY_MAJOR_VERSION >= 3 + Py_DECREF(bytes); +#endif + if (numparsed < 3) { err_repr = PyObject_Repr(arg); if (err_repr == NULL) return NULL; +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(err_repr); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_ValueError, + "Couldn't parse time string: %.200s", + PyBytes_AS_STRING(err_bytes)); + Py_DECREF(err_bytes); +#else PyErr_Format( PyExc_ValueError, "Couldn't parse time string: %.200s", PyString_AsString(err_repr)); +#endif Py_DECREF(err_repr); return NULL; } @@ -154,34 +242,73 @@ str_to_time(PyObject *self, PyObject *arg) static PyObject * str_to_date(PyObject *self, PyObject *arg) { +#if PY_MAJOR_VERSION >= 3 + PyObject *bytes; + PyObject *err_bytes; +#endif const char *str; + int numparsed; unsigned int year, month, day; PyObject *err_repr; if (arg == Py_None) Py_RETURN_NONE; +#if PY_MAJOR_VERSION >= 3 + bytes = PyUnicode_AsASCIIString(arg); + if (bytes == NULL) + str = NULL; + else + str = PyBytes_AS_STRING(bytes); +#else str = PyString_AsString(arg); +#endif if (str == NULL) { err_repr = PyObject_Repr(arg); if (err_repr == NULL) return NULL; +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(err_repr); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_ValueError, + "Couldn't parse date string '%.200s' - value is not a string.", + PyBytes_AS_STRING(err_bytes)); + Py_DECREF(err_bytes); +#else PyErr_Format( PyExc_ValueError, "Couldn't parse date string '%.200s' - value is not a string.", PyString_AsString(err_repr)); +#endif Py_DECREF(err_repr); return NULL; } - if (sscanf(str, "%4u-%2u-%2u", &year, &month, &day) != 3) { + numparsed = sscanf(str, "%4u-%2u-%2u", &year, &month, &day); +#if PY_MAJOR_VERSION >= 3 + Py_DECREF(bytes); +#endif + if (numparsed != 3) { err_repr = PyObject_Repr(arg); if (err_repr == NULL) return NULL; +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(err_repr); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_ValueError, + "Couldn't parse date string: %.200s", + PyBytes_AS_STRING(err_bytes)); + Py_DECREF(err_bytes); +#else PyErr_Format( PyExc_ValueError, "Couldn't parse date string: %.200s", PyString_AsString(err_repr)); +#endif Py_DECREF(err_repr); return NULL; } @@ -218,17 +345,35 @@ UnicodeResultProcessor_init(UnicodeResultProcessor *self, PyObject *args, PyObject *encoding, *errors = NULL; static char *kwlist[] = {"encoding", "errors", NULL}; +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|U:__init__", kwlist, + &encoding, &errors)) + return -1; +#else if (!PyArg_ParseTupleAndKeywords(args, kwds, "S|S:__init__", kwlist, &encoding, &errors)) return -1; +#endif +#if PY_MAJOR_VERSION >= 3 + encoding = PyUnicode_AsASCIIString(encoding); +#else Py_INCREF(encoding); +#endif self->encoding = encoding; if (errors) { +#if PY_MAJOR_VERSION >= 3 + errors = PyUnicode_AsASCIIString(errors); +#else Py_INCREF(errors); +#endif } else { +#if PY_MAJOR_VERSION >= 3 + errors = PyBytes_FromString("strict"); +#else errors = PyString_FromString("strict"); +#endif if (errors == NULL) return -1; } @@ -247,11 +392,19 @@ UnicodeResultProcessor_process(UnicodeResultProcessor *self, PyObject *value) if (value == Py_None) Py_RETURN_NONE; +#if PY_MAJOR_VERSION >= 3 + if (PyBytes_AsStringAndSize(value, &str, &len)) + return NULL; + + encoding = PyBytes_AS_STRING(self->encoding); + errors = PyBytes_AS_STRING(self->errors); +#else if (PyString_AsStringAndSize(value, &str, &len)) return NULL; encoding = PyString_AS_STRING(self->encoding); errors = PyString_AS_STRING(self->errors); +#endif return PyUnicode_Decode(str, len, encoding, errors); } @@ -261,7 +414,11 @@ UnicodeResultProcessor_dealloc(UnicodeResultProcessor *self) { Py_XDECREF(self->encoding); Py_XDECREF(self->errors); +#if PY_MAJOR_VERSION >= 3 + Py_TYPE(self)->tp_free((PyObject*)self); +#else self->ob_type->tp_free((PyObject*)self); +#endif } static PyMethodDef UnicodeResultProcessor_methods[] = { @@ -271,8 +428,7 @@ static PyMethodDef UnicodeResultProcessor_methods[] = { }; static PyTypeObject UnicodeResultProcessorType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "sqlalchemy.cprocessors.UnicodeResultProcessor", /* tp_name */ sizeof(UnicodeResultProcessor), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -322,7 +478,11 @@ DecimalResultProcessor_init(DecimalResultProcessor *self, PyObject *args, { PyObject *type, *format; +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "OU", &type, &format)) +#else if (!PyArg_ParseTuple(args, "OS", &type, &format)) +#endif return -1; Py_INCREF(type); @@ -342,11 +502,21 @@ DecimalResultProcessor_process(DecimalResultProcessor *self, PyObject *value) if (value == Py_None) Py_RETURN_NONE; + /* Decimal does not accept float values directly */ + /* SQLite can also give us an integer here (see [ticket:2432]) */ + /* XXX: starting with Python 3.1, we could use Decimal.from_float(f), + but the result wouldn't be the same */ + args = PyTuple_Pack(1, value); if (args == NULL) return NULL; +#if PY_MAJOR_VERSION >= 3 + str = PyUnicode_Format(self->format, args); +#else str = PyString_Format(self->format, args); +#endif + Py_DECREF(args); if (str == NULL) return NULL; @@ -361,7 +531,11 @@ DecimalResultProcessor_dealloc(DecimalResultProcessor *self) { Py_XDECREF(self->type); Py_XDECREF(self->format); +#if PY_MAJOR_VERSION >= 3 + Py_TYPE(self)->tp_free((PyObject*)self); +#else self->ob_type->tp_free((PyObject*)self); +#endif } static PyMethodDef DecimalResultProcessor_methods[] = { @@ -371,8 +545,7 @@ static PyMethodDef DecimalResultProcessor_methods[] = { }; static PyTypeObject DecimalResultProcessorType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "sqlalchemy.DecimalResultProcessor", /* tp_name */ sizeof(DecimalResultProcessor), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -412,11 +585,6 @@ static PyTypeObject DecimalResultProcessorType = { 0, /* tp_new */ }; -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif - - static PyMethodDef module_methods[] = { {"int_to_boolean", int_to_boolean, METH_O, "Convert an integer to a boolean."}, @@ -433,23 +601,53 @@ static PyMethodDef module_methods[] = { {NULL, NULL, 0, NULL} /* Sentinel */ }; +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + + +#if PY_MAJOR_VERSION >= 3 + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + MODULE_NAME, + MODULE_DOC, + -1, + module_methods +}; + +#define INITERROR return NULL + +PyObject * +PyInit_cprocessors(void) + +#else + +#define INITERROR return + PyMODINIT_FUNC initcprocessors(void) + +#endif + { PyObject *m; UnicodeResultProcessorType.tp_new = PyType_GenericNew; if (PyType_Ready(&UnicodeResultProcessorType) < 0) - return; + INITERROR; DecimalResultProcessorType.tp_new = PyType_GenericNew; if (PyType_Ready(&DecimalResultProcessorType) < 0) - return; + INITERROR; - m = Py_InitModule3("cprocessors", module_methods, - "Module containing C versions of data processing functions."); +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&module_def); +#else + m = Py_InitModule3(MODULE_NAME, module_methods, MODULE_DOC); +#endif if (m == NULL) - return; + INITERROR; PyDateTime_IMPORT; @@ -460,5 +658,8 @@ initcprocessors(void) Py_INCREF(&DecimalResultProcessorType); PyModule_AddObject(m, "DecimalResultProcessor", (PyObject *)&DecimalResultProcessorType); -} +#if PY_MAJOR_VERSION >= 3 + return m; +#endif +} diff --git a/libs/sqlalchemy/cextension/resultproxy.c b/libs/sqlalchemy/cextension/resultproxy.c index ca9d28e6..481352a4 100644 --- a/libs/sqlalchemy/cextension/resultproxy.c +++ b/libs/sqlalchemy/cextension/resultproxy.c @@ -1,6 +1,7 @@ /* resultproxy.c -Copyright (C) 2010 Gaetan de Menten gdementen@gmail.com +Copyright (C) 2010-2014 the SQLAlchemy authors and contributors +Copyright (C) 2010-2011 Gaetan de Menten gdementen@gmail.com This module is part of SQLAlchemy and is released under the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,6 +9,9 @@ the MIT License: http://www.opensource.org/licenses/mit-license.php #include +#define MODULE_NAME "cresultproxy" +#define MODULE_DOC "Module containing C versions of core ResultProxy classes." + #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX @@ -121,7 +125,7 @@ BaseRowProxy_reduce(PyObject *self) if (state == NULL) return NULL; - module = PyImport_ImportModule("sqlalchemy.engine.base"); + module = PyImport_ImportModule("sqlalchemy.engine.result"); if (module == NULL) return NULL; @@ -149,7 +153,11 @@ BaseRowProxy_dealloc(BaseRowProxy *self) Py_XDECREF(self->row); Py_XDECREF(self->processors); Py_XDECREF(self->keymap); +#if PY_MAJOR_VERSION >= 3 + Py_TYPE(self)->tp_free((PyObject *)self); +#else self->ob_type->tp_free((PyObject *)self); +#endif } static PyObject * @@ -244,14 +252,21 @@ BaseRowProxy_subscript(BaseRowProxy *self, PyObject *key) PyObject *processor, *value, *processed_value; PyObject *row, *record, *result, *indexobject; PyObject *exc_module, *exception, *cstr_obj; +#if PY_MAJOR_VERSION >= 3 + PyObject *bytes; +#endif char *cstr_key; long index; int key_fallback = 0; int tuple_check = 0; +#if PY_MAJOR_VERSION < 3 if (PyInt_CheckExact(key)) { index = PyInt_AS_LONG(key); - } else if (PyLong_CheckExact(key)) { + } +#endif + + if (PyLong_CheckExact(key)) { index = PyLong_AsLong(key); if ((index == -1) && PyErr_Occurred()) /* -1 can be either the actual value, or an error flag. */ @@ -304,7 +319,21 @@ BaseRowProxy_subscript(BaseRowProxy *self, PyObject *key) cstr_obj = PyObject_Str(key); if (cstr_obj == NULL) return NULL; + +/* + FIXME: raise encoding error exception (in both versions below) + if the key contains non-ascii chars, instead of an + InvalidRequestError without any message like in the + python version. +*/ +#if PY_MAJOR_VERSION >= 3 + bytes = PyUnicode_AsASCIIString(cstr_obj); + if (bytes == NULL) + return NULL; + cstr_key = PyBytes_AS_STRING(bytes); +#else cstr_key = PyString_AsString(cstr_obj); +#endif if (cstr_key == NULL) { Py_DECREF(cstr_obj); return NULL; @@ -317,7 +346,11 @@ BaseRowProxy_subscript(BaseRowProxy *self, PyObject *key) return NULL; } +#if PY_MAJOR_VERSION >= 3 + index = PyLong_AsLong(indexobject); +#else index = PyInt_AsLong(indexobject); +#endif if ((index == -1) && PyErr_Occurred()) /* -1 can be either the actual value, or an error flag. */ return NULL; @@ -356,13 +389,23 @@ BaseRowProxy_subscript(BaseRowProxy *self, PyObject *key) static PyObject * BaseRowProxy_getitem(PyObject *self, Py_ssize_t i) { - return BaseRowProxy_subscript((BaseRowProxy*)self, PyInt_FromSsize_t(i)); + PyObject *index; + +#if PY_MAJOR_VERSION >= 3 + index = PyLong_FromSsize_t(i); +#else + index = PyInt_FromSsize_t(i); +#endif + return BaseRowProxy_subscript((BaseRowProxy*)self, index); } static PyObject * BaseRowProxy_getattro(BaseRowProxy *self, PyObject *name) { PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + PyObject *err_bytes; +#endif if (!(tmp = PyObject_GenericGetAttr((PyObject *)self, name))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) @@ -374,11 +417,23 @@ BaseRowProxy_getattro(BaseRowProxy *self, PyObject *name) tmp = BaseRowProxy_subscript(self, name); if (tmp == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + +#if PY_MAJOR_VERSION >= 3 + err_bytes = PyUnicode_AsASCIIString(name); + if (err_bytes == NULL) + return NULL; + PyErr_Format( + PyExc_AttributeError, + "Could not locate column in row for column '%.200s'", + PyBytes_AS_STRING(err_bytes) + ); +#else PyErr_Format( PyExc_AttributeError, "Could not locate column in row for column '%.200s'", PyString_AsString(name) ); +#endif return NULL; } return tmp; @@ -406,7 +461,7 @@ BaseRowProxy_setparent(BaseRowProxy *self, PyObject *value, void *closure) return -1; } - module = PyImport_ImportModule("sqlalchemy.engine.base"); + module = PyImport_ImportModule("sqlalchemy.engine.result"); if (module == NULL) return -1; @@ -564,8 +619,7 @@ static PyMappingMethods BaseRowProxy_as_mapping = { }; static PyTypeObject BaseRowProxyType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "sqlalchemy.cresultproxy.BaseRowProxy", /* tp_name */ sizeof(BaseRowProxy), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -605,34 +659,60 @@ static PyTypeObject BaseRowProxyType = { 0 /* tp_new */ }; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif - - static PyMethodDef module_methods[] = { {"safe_rowproxy_reconstructor", safe_rowproxy_reconstructor, METH_VARARGS, "reconstruct a RowProxy instance from its pickled form."}, {NULL, NULL, 0, NULL} /* Sentinel */ }; +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + + +#if PY_MAJOR_VERSION >= 3 + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + MODULE_NAME, + MODULE_DOC, + -1, + module_methods +}; + +#define INITERROR return NULL + +PyObject * +PyInit_cresultproxy(void) + +#else + +#define INITERROR return + PyMODINIT_FUNC initcresultproxy(void) + +#endif + { PyObject *m; BaseRowProxyType.tp_new = PyType_GenericNew; if (PyType_Ready(&BaseRowProxyType) < 0) - return; + INITERROR; - m = Py_InitModule3("cresultproxy", module_methods, - "Module containing C versions of core ResultProxy classes."); +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&module_def); +#else + m = Py_InitModule3(MODULE_NAME, module_methods, MODULE_DOC); +#endif if (m == NULL) - return; + INITERROR; Py_INCREF(&BaseRowProxyType); PyModule_AddObject(m, "BaseRowProxy", (PyObject *)&BaseRowProxyType); +#if PY_MAJOR_VERSION >= 3 + return m; +#endif } - diff --git a/libs/sqlalchemy/cextension/utils.c b/libs/sqlalchemy/cextension/utils.c new file mode 100644 index 00000000..3c556e1c --- /dev/null +++ b/libs/sqlalchemy/cextension/utils.c @@ -0,0 +1,225 @@ +/* +utils.c +Copyright (C) 2012-2014 the SQLAlchemy authors and contributors + +This module is part of SQLAlchemy and is released under +the MIT License: http://www.opensource.org/licenses/mit-license.php +*/ + +#include + +#define MODULE_NAME "cutils" +#define MODULE_DOC "Module containing C versions of utility functions." + +/* + Given arguments from the calling form *multiparams, **params, + return a list of bind parameter structures, usually a list of + dictionaries. + + In the case of 'raw' execution which accepts positional parameters, + it may be a list of tuples or lists. + + */ +static PyObject * +distill_params(PyObject *self, PyObject *args) +{ + PyObject *multiparams, *params; + PyObject *enclosing_list, *double_enclosing_list; + PyObject *zero_element, *zero_element_item; + Py_ssize_t multiparam_size, zero_element_length; + + if (!PyArg_UnpackTuple(args, "_distill_params", 2, 2, &multiparams, ¶ms)) { + return NULL; + } + + if (multiparams != Py_None) { + multiparam_size = PyTuple_Size(multiparams); + if (multiparam_size < 0) { + return NULL; + } + } + else { + multiparam_size = 0; + } + + if (multiparam_size == 0) { + if (params != Py_None && PyDict_Size(params) != 0) { + enclosing_list = PyList_New(1); + if (enclosing_list == NULL) { + return NULL; + } + Py_INCREF(params); + if (PyList_SetItem(enclosing_list, 0, params) == -1) { + Py_DECREF(params); + Py_DECREF(enclosing_list); + return NULL; + } + } + else { + enclosing_list = PyList_New(0); + if (enclosing_list == NULL) { + return NULL; + } + } + return enclosing_list; + } + else if (multiparam_size == 1) { + zero_element = PyTuple_GetItem(multiparams, 0); + if (PyTuple_Check(zero_element) || PyList_Check(zero_element)) { + zero_element_length = PySequence_Length(zero_element); + + if (zero_element_length != 0) { + zero_element_item = PySequence_GetItem(zero_element, 0); + if (zero_element_item == NULL) { + return NULL; + } + } + else { + zero_element_item = NULL; + } + + if (zero_element_length == 0 || + ( + PyObject_HasAttrString(zero_element_item, "__iter__") && + !PyObject_HasAttrString(zero_element_item, "strip") + ) + ) { + /* + * execute(stmt, [{}, {}, {}, ...]) + * execute(stmt, [(), (), (), ...]) + */ + Py_XDECREF(zero_element_item); + Py_INCREF(zero_element); + return zero_element; + } + else { + /* + * execute(stmt, ("value", "value")) + */ + Py_XDECREF(zero_element_item); + enclosing_list = PyList_New(1); + if (enclosing_list == NULL) { + return NULL; + } + Py_INCREF(zero_element); + if (PyList_SetItem(enclosing_list, 0, zero_element) == -1) { + Py_DECREF(zero_element); + Py_DECREF(enclosing_list); + return NULL; + } + return enclosing_list; + } + } + else if (PyObject_HasAttrString(zero_element, "keys")) { + /* + * execute(stmt, {"key":"value"}) + */ + enclosing_list = PyList_New(1); + if (enclosing_list == NULL) { + return NULL; + } + Py_INCREF(zero_element); + if (PyList_SetItem(enclosing_list, 0, zero_element) == -1) { + Py_DECREF(zero_element); + Py_DECREF(enclosing_list); + return NULL; + } + return enclosing_list; + } else { + enclosing_list = PyList_New(1); + if (enclosing_list == NULL) { + return NULL; + } + double_enclosing_list = PyList_New(1); + if (double_enclosing_list == NULL) { + Py_DECREF(enclosing_list); + return NULL; + } + Py_INCREF(zero_element); + if (PyList_SetItem(enclosing_list, 0, zero_element) == -1) { + Py_DECREF(zero_element); + Py_DECREF(enclosing_list); + Py_DECREF(double_enclosing_list); + return NULL; + } + if (PyList_SetItem(double_enclosing_list, 0, enclosing_list) == -1) { + Py_DECREF(zero_element); + Py_DECREF(enclosing_list); + Py_DECREF(double_enclosing_list); + return NULL; + } + return double_enclosing_list; + } + } + else { + zero_element = PyTuple_GetItem(multiparams, 0); + if (PyObject_HasAttrString(zero_element, "__iter__") && + !PyObject_HasAttrString(zero_element, "strip") + ) { + Py_INCREF(multiparams); + return multiparams; + } + else { + enclosing_list = PyList_New(1); + if (enclosing_list == NULL) { + return NULL; + } + Py_INCREF(multiparams); + if (PyList_SetItem(enclosing_list, 0, multiparams) == -1) { + Py_DECREF(multiparams); + Py_DECREF(enclosing_list); + return NULL; + } + return enclosing_list; + } + } +} + +static PyMethodDef module_methods[] = { + {"_distill_params", distill_params, METH_VARARGS, + "Distill an execute() parameter structure."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + +#if PY_MAJOR_VERSION >= 3 + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + MODULE_NAME, + MODULE_DOC, + -1, + module_methods + }; +#endif + + +#if PY_MAJOR_VERSION >= 3 +PyObject * +PyInit_cutils(void) +#else +PyMODINIT_FUNC +initcutils(void) +#endif +{ + PyObject *m; + +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&module_def); +#else + m = Py_InitModule3(MODULE_NAME, module_methods, MODULE_DOC); +#endif + +#if PY_MAJOR_VERSION >= 3 + if (m == NULL) + return NULL; + return m; +#else + if (m == NULL) + return; +#endif +} + diff --git a/libs/sqlalchemy/connectors/__init__.py b/libs/sqlalchemy/connectors/__init__.py index a4e017c4..761024fe 100644 --- a/libs/sqlalchemy/connectors/__init__.py +++ b/libs/sqlalchemy/connectors/__init__.py @@ -1,5 +1,5 @@ # connectors/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,4 +7,3 @@ class Connector(object): pass - diff --git a/libs/sqlalchemy/connectors/mxodbc.py b/libs/sqlalchemy/connectors/mxodbc.py index 2848f200..e5562a25 100644 --- a/libs/sqlalchemy/connectors/mxodbc.py +++ b/libs/sqlalchemy/connectors/mxodbc.py @@ -1,5 +1,5 @@ # connectors/mxodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -22,14 +22,15 @@ import sys import re import warnings -from sqlalchemy.connectors import Connector +from . import Connector + class MxODBCConnector(Connector): - driver='mxodbc' + driver = 'mxodbc' supports_sane_multi_rowcount = False - supports_unicode_statements = False - supports_unicode_binds = False + supports_unicode_statements = True + supports_unicode_binds = True supports_native_decimal = True @@ -47,7 +48,7 @@ class MxODBCConnector(Connector): elif platform == 'darwin': from mx.ODBC import iODBC as module else: - raise ImportError, "Unrecognized platform for mxODBC import" + raise ImportError("Unrecognized platform for mxODBC import") return module @classmethod @@ -73,15 +74,15 @@ class MxODBCConnector(Connector): emit Python standard warnings. """ from mx.ODBC.Error import Warning as MxOdbcWarning - def error_handler(connection, cursor, errorclass, errorvalue): + def error_handler(connection, cursor, errorclass, errorvalue): if issubclass(errorclass, MxOdbcWarning): errorclass.__bases__ = (Warning,) warnings.warn(message=str(errorvalue), category=errorclass, stacklevel=2) else: - raise errorclass, errorvalue + raise errorclass(errorvalue) return error_handler def create_connect_args(self, url): @@ -130,21 +131,19 @@ class MxODBCConnector(Connector): version.append(n) return tuple(version) - def do_execute(self, cursor, statement, parameters, context=None): + def _get_direct(self, context): if context: native_odbc_execute = context.execution_options.\ get('native_odbc_execute', 'auto') - if native_odbc_execute is True: - # user specified native_odbc_execute=True - cursor.execute(statement, parameters) - elif native_odbc_execute is False: - # user specified native_odbc_execute=False - cursor.executedirect(statement, parameters) - elif context.is_crud: - # statement is UPDATE, DELETE, INSERT - cursor.execute(statement, parameters) - else: - # all other statements - cursor.executedirect(statement, parameters) + # default to direct=True in all cases, is more generally + # compatible especially with SQL Server + return False if native_odbc_execute is True else True else: - cursor.executedirect(statement, parameters) + return True + + def do_executemany(self, cursor, statement, parameters, context=None): + cursor.executemany( + statement, parameters, direct=self._get_direct(context)) + + def do_execute(self, cursor, statement, parameters, context=None): + cursor.execute(statement, parameters, direct=self._get_direct(context)) diff --git a/libs/sqlalchemy/connectors/mysqldb.py b/libs/sqlalchemy/connectors/mysqldb.py index be1f3530..0f250dfd 100644 --- a/libs/sqlalchemy/connectors/mysqldb.py +++ b/libs/sqlalchemy/connectors/mysqldb.py @@ -1,19 +1,26 @@ +# connectors/mysqldb.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + """Define behaviors common to MySQLdb dialects. Currently includes MySQL and Drizzle. """ -from sqlalchemy.connectors import Connector -from sqlalchemy.engine import base as engine_base, default -from sqlalchemy.sql import operators as sql_operators -from sqlalchemy import exc, log, schema, sql, types as sqltypes, util -from sqlalchemy import processors +from . import Connector +from ..engine import base as engine_base, default +from ..sql import operators as sql_operators +from .. import exc, log, schema, sql, types as sqltypes, util, processors import re + # the subclassing of Connector by all classes # here is not strictly necessary + class MySQLDBExecutionContext(Connector): @property @@ -23,19 +30,23 @@ class MySQLDBExecutionContext(Connector): else: return self.cursor.rowcount + class MySQLDBCompiler(Connector): - def visit_mod(self, binary, **kw): - return self.process(binary.left) + " %% " + self.process(binary.right) + def visit_mod_binary(self, binary, operator, **kw): + return self.process(binary.left, **kw) + " %% " + \ + self.process(binary.right, **kw) def post_process_text(self, text): return text.replace('%', '%%') + class MySQLDBIdentifierPreparer(Connector): def _escape_identifier(self, value): value = value.replace(self.escape_quote, self.escape_to_quote) return value.replace("%", "%%") + class MySQLDBConnector(Connector): driver = 'mysqldb' supports_unicode_statements = False @@ -76,7 +87,8 @@ class MySQLDBConnector(Connector): # query string. ssl = {} - for key in ['ssl_ca', 'ssl_key', 'ssl_cert', 'ssl_capath', 'ssl_cipher']: + keys = ['ssl_ca', 'ssl_key', 'ssl_cert', 'ssl_capath', 'ssl_cipher'] + for key in keys: if key in opts: ssl[key[4:]] = opts[key] util.coerce_kw_type(ssl, key[4:], str) @@ -148,4 +160,3 @@ class MySQLDBConnector(Connector): "combination of MySQL server and MySQL-python. " "MySQL-python >= 1.2.2 is recommended. Assuming latin1.") return 'latin1' - diff --git a/libs/sqlalchemy/connectors/pyodbc.py b/libs/sqlalchemy/connectors/pyodbc.py index 5be65d2d..284de288 100644 --- a/libs/sqlalchemy/connectors/pyodbc.py +++ b/libs/sqlalchemy/connectors/pyodbc.py @@ -1,23 +1,27 @@ # connectors/pyodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy.connectors import Connector -from sqlalchemy.util import asbool +from . import Connector +from .. import util + import sys import re -import urllib + class PyODBCConnector(Connector): - driver='pyodbc' + driver = 'pyodbc' supports_sane_multi_rowcount = False - # PyODBC unicode is broken on UCS-4 builds - supports_unicode = sys.maxunicode == 65535 - supports_unicode_statements = supports_unicode + + if util.py2k: + # PyODBC unicode is broken on UCS-4 builds + supports_unicode = sys.maxunicode == 65535 + supports_unicode_statements = supports_unicode + supports_native_decimal = True default_paramstyle = 'named' @@ -55,15 +59,15 @@ class PyODBCConnector(Connector): connect_args = {} for param in ('ansi', 'unicode_results', 'autocommit'): if param in keys: - connect_args[param] = asbool(keys.pop(param)) + connect_args[param] = util.asbool(keys.pop(param)) if 'odbc_connect' in keys: - connectors = [urllib.unquote_plus(keys.pop('odbc_connect'))] + connectors = [util.unquote_plus(keys.pop('odbc_connect'))] else: dsn_connection = 'dsn' in keys or \ ('host' in keys and 'database' not in keys) if dsn_connection: - connectors= ['dsn=%s' % (keys.pop('host', '') or \ + connectors = ['dsn=%s' % (keys.pop('host', '') or \ keys.pop('dsn', ''))] else: port = '' @@ -73,7 +77,7 @@ class PyODBCConnector(Connector): connectors = ["DRIVER={%s}" % keys.pop('driver', self.pyodbc_driver_name), 'Server=%s%s' % (keys.pop('host', ''), port), - 'Database=%s' % keys.pop('database', '') ] + 'Database=%s' % keys.pop('database', '')] user = keys.pop("user", None) if user: @@ -90,8 +94,8 @@ class PyODBCConnector(Connector): connectors.append("AutoTranslate=%s" % keys.pop("odbc_autotranslate")) - connectors.extend(['%s=%s' % (k,v) for k,v in keys.iteritems()]) - return [[";".join (connectors)], connect_args] + connectors.extend(['%s=%s' % (k, v) for k, v in keys.items()]) + return [[";".join(connectors)], connect_args] def is_disconnect(self, e, connection, cursor): if isinstance(e, self.dbapi.ProgrammingError): @@ -117,19 +121,22 @@ class PyODBCConnector(Connector): )) if self.freetds: - self.freetds_driver_version = dbapi_con.getinfo(pyodbc.SQL_DRIVER_VER) + self.freetds_driver_version = dbapi_con.getinfo( + pyodbc.SQL_DRIVER_VER) + + self.supports_unicode_statements = ( + not util.py2k or + (not self.freetds and not self.easysoft) + ) - # the "Py2K only" part here is theoretical. - # have not tried pyodbc + python3.1 yet. - # Py2K - self.supports_unicode_statements = not self.freetds and not self.easysoft if self._user_supports_unicode_binds is not None: self.supports_unicode_binds = self._user_supports_unicode_binds + elif util.py2k: + self.supports_unicode_binds = ( + not self.freetds or self.freetds_driver_version >= '0.91' + ) and not self.easysoft else: - self.supports_unicode_binds = (not self.freetds or - self.freetds_driver_version >= '0.91' - ) and not self.easysoft - # end Py2K + self.supports_unicode_binds = True # run other initialization which asks for user name, etc. super(PyODBCConnector, self).initialize(connection) diff --git a/libs/sqlalchemy/connectors/zxJDBC.py b/libs/sqlalchemy/connectors/zxJDBC.py index e2bfed2e..e0bbc573 100644 --- a/libs/sqlalchemy/connectors/zxJDBC.py +++ b/libs/sqlalchemy/connectors/zxJDBC.py @@ -1,11 +1,12 @@ # connectors/zxJDBC.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php import sys -from sqlalchemy.connectors import Connector +from . import Connector + class ZxJDBCConnector(Connector): driver = 'zxjdbc' diff --git a/libs/sqlalchemy/databases/__init__.py b/libs/sqlalchemy/databases/__init__.py index bb0b370e..915eefa4 100644 --- a/libs/sqlalchemy/databases/__init__.py +++ b/libs/sqlalchemy/databases/__init__.py @@ -1,5 +1,5 @@ # databases/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,26 +8,20 @@ compatibility with pre 0.6 versions. """ -from sqlalchemy.dialects.sqlite import base as sqlite -from sqlalchemy.dialects.postgresql import base as postgresql +from ..dialects.sqlite import base as sqlite +from ..dialects.postgresql import base as postgresql postgres = postgresql -from sqlalchemy.dialects.mysql import base as mysql -from sqlalchemy.dialects.drizzle import base as drizzle -from sqlalchemy.dialects.oracle import base as oracle -from sqlalchemy.dialects.firebird import base as firebird -from sqlalchemy.dialects.maxdb import base as maxdb -from sqlalchemy.dialects.informix import base as informix -from sqlalchemy.dialects.mssql import base as mssql -from sqlalchemy.dialects.access import base as access -from sqlalchemy.dialects.sybase import base as sybase +from ..dialects.mysql import base as mysql +from ..dialects.drizzle import base as drizzle +from ..dialects.oracle import base as oracle +from ..dialects.firebird import base as firebird +from ..dialects.mssql import base as mssql +from ..dialects.sybase import base as sybase __all__ = ( - 'access', 'drizzle', 'firebird', - 'informix', - 'maxdb', 'mssql', 'mysql', 'postgresql', diff --git a/libs/sqlalchemy/dialects/__init__.py b/libs/sqlalchemy/dialects/__init__.py index a427cde4..974d4f78 100644 --- a/libs/sqlalchemy/dialects/__init__.py +++ b/libs/sqlalchemy/dialects/__init__.py @@ -1,15 +1,12 @@ # dialects/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php __all__ = ( -# 'access', 'drizzle', 'firebird', -# 'informix', -# 'maxdb', 'mssql', 'mysql', 'oracle', @@ -17,3 +14,31 @@ __all__ = ( 'sqlite', 'sybase', ) + +from .. import util + +def _auto_fn(name): + """default dialect importer. + + plugs into the :class:`.PluginLoader` + as a first-hit system. + + """ + if "." in name: + dialect, driver = name.split(".") + else: + dialect = name + driver = "base" + try: + module = __import__('sqlalchemy.dialects.%s' % (dialect, )).dialects + except ImportError: + return None + + module = getattr(module, dialect) + if hasattr(module, driver): + module = getattr(module, driver) + return lambda: module.dialect + else: + return None + +registry = util.PluginLoader("sqlalchemy.dialects", auto_fn=_auto_fn) diff --git a/libs/sqlalchemy/dialects/access/__init__.py b/libs/sqlalchemy/dialects/access/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/libs/sqlalchemy/dialects/access/base.py b/libs/sqlalchemy/dialects/access/base.py deleted file mode 100644 index f107c9c8..00000000 --- a/libs/sqlalchemy/dialects/access/base.py +++ /dev/null @@ -1,451 +0,0 @@ -# access/base.py -# Copyright (C) 2007-2011 the SQLAlchemy authors and contributors -# Copyright (C) 2007 Paul Johnston, paj@pajhome.org.uk -# Portions derived from jet2sql.py by Matt Keranen, mksql@yahoo.com -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -""" -Support for the Microsoft Access database. - -.. note:: - - The Access dialect is **non-functional as of SQLAlchemy 0.6**, - pending development efforts to bring it up-to-date. - - -""" -from sqlalchemy import sql, schema, types, exc, pool -from sqlalchemy.sql import compiler, expression -from sqlalchemy.engine import default, base, reflection -from sqlalchemy import processors - -class AcNumeric(types.Numeric): - def get_col_spec(self): - return "NUMERIC" - - def bind_processor(self, dialect): - return processors.to_str - - def result_processor(self, dialect, coltype): - return None - -class AcFloat(types.Float): - def get_col_spec(self): - return "FLOAT" - - def bind_processor(self, dialect): - """By converting to string, we can use Decimal types round-trip.""" - return processors.to_str - -class AcInteger(types.Integer): - def get_col_spec(self): - return "INTEGER" - -class AcTinyInteger(types.Integer): - def get_col_spec(self): - return "TINYINT" - -class AcSmallInteger(types.SmallInteger): - def get_col_spec(self): - return "SMALLINT" - -class AcDateTime(types.DateTime): - def get_col_spec(self): - return "DATETIME" - -class AcDate(types.Date): - - def get_col_spec(self): - return "DATETIME" - -class AcText(types.Text): - def get_col_spec(self): - return "MEMO" - -class AcString(types.String): - def get_col_spec(self): - return "TEXT" + (self.length and ("(%d)" % self.length) or "") - -class AcUnicode(types.Unicode): - def get_col_spec(self): - return "TEXT" + (self.length and ("(%d)" % self.length) or "") - - def bind_processor(self, dialect): - return None - - def result_processor(self, dialect, coltype): - return None - -class AcChar(types.CHAR): - def get_col_spec(self): - return "TEXT" + (self.length and ("(%d)" % self.length) or "") - -class AcBinary(types.LargeBinary): - def get_col_spec(self): - return "BINARY" - -class AcBoolean(types.Boolean): - def get_col_spec(self): - return "YESNO" - -class AcTimeStamp(types.TIMESTAMP): - def get_col_spec(self): - return "TIMESTAMP" - -class AccessExecutionContext(default.DefaultExecutionContext): - def _has_implicit_sequence(self, column): - if column.primary_key and column.autoincrement: - if isinstance(column.type, types.Integer) and \ - not column.foreign_keys: - if column.default is None or \ - (isinstance(column.default, schema.Sequence) and \ - column.default.optional): - return True - return False - - def post_exec(self): - """If we inserted into a row with a COUNTER column, fetch the ID""" - - if self.compiled.isinsert: - tbl = self.compiled.statement.table - if not hasattr(tbl, 'has_sequence'): - tbl.has_sequence = None - for column in tbl.c: - if getattr(column, 'sequence', False) or \ - self._has_implicit_sequence(column): - tbl.has_sequence = column - break - - if bool(tbl.has_sequence): - # TBD: for some reason _last_inserted_ids doesn't exist here - # (but it does at corresponding point in mssql???) - #if not len(self._last_inserted_ids) or - # self._last_inserted_ids[0] is None: - self.cursor.execute("SELECT @@identity AS lastrowid") - row = self.cursor.fetchone() - self._last_inserted_ids = [int(row[0])] - #+ self._last_inserted_ids[1:] - # print "LAST ROW ID", self._last_inserted_ids - - super(AccessExecutionContext, self).post_exec() - - -const, daoEngine = None, None -class AccessDialect(default.DefaultDialect): - colspecs = { - types.Unicode : AcUnicode, - types.Integer : AcInteger, - types.SmallInteger: AcSmallInteger, - types.Numeric : AcNumeric, - types.Float : AcFloat, - types.DateTime : AcDateTime, - types.Date : AcDate, - types.String : AcString, - types.LargeBinary : AcBinary, - types.Boolean : AcBoolean, - types.Text : AcText, - types.CHAR: AcChar, - types.TIMESTAMP: AcTimeStamp, - } - name = 'access' - supports_sane_rowcount = False - supports_sane_multi_rowcount = False - - ported_sqla_06 = False - - def type_descriptor(self, typeobj): - newobj = types.adapt_type(typeobj, self.colspecs) - return newobj - - def __init__(self, **params): - super(AccessDialect, self).__init__(**params) - self.text_as_varchar = False - self._dtbs = None - - @classmethod - def dbapi(cls): - import win32com.client, pythoncom - - global const, daoEngine - if const is None: - const = win32com.client.constants - for suffix in (".36", ".35", ".30"): - try: - daoEngine = win32com.client.\ - gencache.\ - EnsureDispatch("DAO.DBEngine" + suffix) - break - except pythoncom.com_error: - pass - else: - raise exc.InvalidRequestError( - "Can't find a DB engine. Check " - "http://support.microsoft.com/kb/239114 for details.") - - import pyodbc as module - return module - - def create_connect_args(self, url): - opts = url.translate_connect_args() - connectors = ["Driver={Microsoft Access Driver (*.mdb)}"] - connectors.append("Dbq=%s" % opts["database"]) - user = opts.get("username", None) - if user: - connectors.append("UID=%s" % user) - connectors.append("PWD=%s" % opts.get("password", "")) - return [[";".join(connectors)], {}] - - def last_inserted_ids(self): - return self.context.last_inserted_ids - - def do_execute(self, cursor, statement, params, context=None): - if params == {}: - params = () - super(AccessDialect, self).\ - do_execute(cursor, statement, params, **kwargs) - - def _execute(self, c, statement, parameters): - try: - if parameters == {}: - parameters = () - c.execute(statement, parameters) - self.context.rowcount = c.rowcount - except Exception, e: - raise exc.DBAPIError.instance(statement, parameters, e) - - def has_table(self, connection, tablename, schema=None): - # This approach seems to be more reliable that using DAO - try: - connection.execute('select top 1 * from [%s]' % tablename) - return True - except Exception, e: - return False - - def reflecttable(self, connection, table, include_columns): - # This is defined in the function, as it relies on win32com constants, - # that aren't imported until dbapi method is called - if not hasattr(self, 'ischema_names'): - self.ischema_names = { - const.dbByte: AcBinary, - const.dbInteger: AcInteger, - const.dbLong: AcInteger, - const.dbSingle: AcFloat, - const.dbDouble: AcFloat, - const.dbDate: AcDateTime, - const.dbLongBinary: AcBinary, - const.dbMemo: AcText, - const.dbBoolean: AcBoolean, - const.dbText: AcUnicode, # All Access strings are - # unicode - const.dbCurrency: AcNumeric, - } - - # A fresh DAO connection is opened for each reflection - # This is necessary, so we get the latest updates - dtbs = daoEngine.OpenDatabase(connection.engine.url.database) - - try: - for tbl in dtbs.TableDefs: - if tbl.Name.lower() == table.name.lower(): - break - else: - raise exc.NoSuchTableError(table.name) - - for col in tbl.Fields: - coltype = self.ischema_names[col.Type] - if col.Type == const.dbText: - coltype = coltype(col.Size) - - colargs = \ - { - 'nullable': not(col.Required or - col.Attributes & const.dbAutoIncrField), - } - default = col.DefaultValue - - if col.Attributes & const.dbAutoIncrField: - colargs['default'] = schema.Sequence(col.Name + '_seq') - elif default: - if col.Type == const.dbBoolean: - default = default == 'Yes' and '1' or '0' - colargs['server_default'] = \ - schema.DefaultClause(sql.text(default)) - - table.append_column( - schema.Column(col.Name, coltype, **colargs)) - - # TBD: check constraints - - # Find primary key columns first - for idx in tbl.Indexes: - if idx.Primary: - for col in idx.Fields: - thecol = table.c[col.Name] - table.primary_key.add(thecol) - if isinstance(thecol.type, AcInteger) and \ - not (thecol.default and - isinstance( - thecol.default.arg, - schema.Sequence - )): - thecol.autoincrement = False - - # Then add other indexes - for idx in tbl.Indexes: - if not idx.Primary: - if len(idx.Fields) == 1: - col = table.c[idx.Fields[0].Name] - if not col.primary_key: - col.index = True - col.unique = idx.Unique - else: - pass # TBD: multi-column indexes - - - for fk in dtbs.Relations: - if fk.ForeignTable != table.name: - continue - scols = [c.ForeignName for c in fk.Fields] - rcols = ['%s.%s' % (fk.Table, c.Name) for c in fk.Fields] - table.append_constraint( - schema.ForeignKeyConstraint(scols, rcols,\ - link_to_name=True)) - - finally: - dtbs.Close() - - @reflection.cache - def get_table_names(self, connection, schema=None, **kw): - # A fresh DAO connection is opened for each reflection - # This is necessary, so we get the latest updates - dtbs = daoEngine.OpenDatabase(connection.engine.url.database) - - names = [t.Name for t in dtbs.TableDefs - if t.Name[:4] != "MSys" and t.Name[:4] != "~TMP"] - dtbs.Close() - return names - - -class AccessCompiler(compiler.SQLCompiler): - extract_map = compiler.SQLCompiler.extract_map.copy() - extract_map.update ({ - 'month': 'm', - 'day': 'd', - 'year': 'yyyy', - 'second': 's', - 'hour': 'h', - 'doy': 'y', - 'minute': 'n', - 'quarter': 'q', - 'dow': 'w', - 'week': 'ww' - }) - - def visit_select_precolumns(self, select): - """Access puts TOP, it's version of LIMIT here """ - s = select.distinct and "DISTINCT " or "" - if select.limit: - s += "TOP %s " % (select.limit) - if select.offset: - raise exc.InvalidRequestError( - 'Access does not support LIMIT with an offset') - return s - - def limit_clause(self, select): - """Limit in access is after the select keyword""" - return "" - - def binary_operator_string(self, binary): - """Access uses "mod" instead of "%" """ - return binary.operator == '%' and 'mod' or binary.operator - - def label_select_column(self, select, column, asfrom): - if isinstance(column, expression.Function): - return column.label() - else: - return super(AccessCompiler, self).\ - label_select_column(select, column, asfrom) - - function_rewrites = {'current_date': 'now', - 'current_timestamp': 'now', - 'length': 'len', - } - def visit_function(self, func): - """Access function names differ from the ANSI SQL names; - rewrite common ones""" - func.name = self.function_rewrites.get(func.name, func.name) - return super(AccessCompiler, self).visit_function(func) - - def for_update_clause(self, select): - """FOR UPDATE is not supported by Access; silently ignore""" - return '' - - # Strip schema - def visit_table(self, table, asfrom=False, **kwargs): - if asfrom: - return self.preparer.quote(table.name, table.quote) - else: - return "" - - def visit_join(self, join, asfrom=False, **kwargs): - return (self.process(join.left, asfrom=True) + \ - (join.isouter and " LEFT OUTER JOIN " or " INNER JOIN ") + \ - self.process(join.right, asfrom=True) + " ON " + \ - self.process(join.onclause)) - - def visit_extract(self, extract, **kw): - field = self.extract_map.get(extract.field, extract.field) - return 'DATEPART("%s", %s)' % \ - (field, self.process(extract.expr, **kw)) - -class AccessDDLCompiler(compiler.DDLCompiler): - def get_column_specification(self, column, **kwargs): - colspec = self.preparer.format_column(column) + " " + \ - column.type.dialect_impl(self.dialect).get_col_spec() - - # install a sequence if we have an implicit IDENTITY column - if (not getattr(column.table, 'has_sequence', False)) and \ - column.primary_key and \ - column.autoincrement and \ - isinstance(column.type, types.Integer) and \ - not column.foreign_keys: - if column.default is None or \ - (isinstance(column.default, schema.Sequence) and - column.default.optional): - column.sequence = schema.Sequence(column.name + '_seq') - - if not column.nullable: - colspec += " NOT NULL" - - if hasattr(column, 'sequence'): - column.table.has_sequence = column - colspec = self.preparer.format_column(column) + " counter" - else: - default = self.get_column_default_string(column) - if default is not None: - colspec += " DEFAULT " + default - - return colspec - - def visit_drop_index(self, drop): - index = drop.element - self.append("\nDROP INDEX [%s].[%s]" % \ - (index.table.name, - self._index_identifier(index.name))) - -class AccessIdentifierPreparer(compiler.IdentifierPreparer): - reserved_words = compiler.RESERVED_WORDS.copy() - reserved_words.update(['value', 'text']) - def __init__(self, dialect): - super(AccessIdentifierPreparer, self).\ - __init__(dialect, initial_quote='[', final_quote=']') - - -dialect = AccessDialect -dialect.poolclass = pool.SingletonThreadPool -dialect.statement_compiler = AccessCompiler -dialect.ddlcompiler = AccessDDLCompiler -dialect.preparer = AccessIdentifierPreparer -dialect.execution_ctx_cls = AccessExecutionContext diff --git a/libs/sqlalchemy/dialects/drizzle/base.py b/libs/sqlalchemy/dialects/drizzle/base.py index 0165a2aa..b5addb42 100644 --- a/libs/sqlalchemy/dialects/drizzle/base.py +++ b/libs/sqlalchemy/dialects/drizzle/base.py @@ -1,12 +1,15 @@ # drizzle/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # Copyright (C) 2010-2011 Monty Taylor # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the Drizzle database. +""" + +.. dialect:: drizzle + :name: Drizzle Drizzle is a variant of MySQL. Unlike MySQL, Drizzle's default storage engine is InnoDB (transactions, foreign-keys) rather than MyISAM. For more @@ -16,10 +19,6 @@ the `Drizzle Documentation `_. The SQLAlchemy Drizzle dialect leans heavily on the MySQL dialect, so much of the :doc:`SQLAlchemy MySQL ` documentation is also relevant. -Connecting ----------- - -See the individual driver sections below for details on connecting. """ @@ -183,7 +182,7 @@ class BIGINT(sqltypes.BIGINT): super(BIGINT, self).__init__(**kw) -class _DrizzleTime(mysql_dialect._MSTime): +class TIME(mysql_dialect.TIME): """Drizzle TIME type.""" @@ -317,7 +316,7 @@ class _DrizzleBoolean(sqltypes.Boolean): colspecs = { sqltypes.Numeric: NUMERIC, sqltypes.Float: FLOAT, - sqltypes.Time: _DrizzleTime, + sqltypes.Time: TIME, sqltypes.Enum: ENUM, sqltypes.Boolean: _DrizzleBoolean, } @@ -418,6 +417,7 @@ class DrizzleIdentifierPreparer(mysql_dialect.MySQLIdentifierPreparer): pass +@log.class_logger class DrizzleDialect(mysql_dialect.MySQLDialect): """Details of the Drizzle dialect. @@ -447,16 +447,6 @@ class DrizzleDialect(mysql_dialect.MySQLDialect): conn.autocommit(False) return connect - def do_commit(self, connection): - """Execute a COMMIT.""" - - connection.commit() - - def do_rollback(self, connection): - """Execute a ROLLBACK.""" - - connection.rollback() - @reflection.cache def get_table_names(self, connection, schema=None, **kw): """Return a Unicode SHOW TABLES from a given schema.""" @@ -506,4 +496,3 @@ class DrizzleDialect(mysql_dialect.MySQLDialect): self._backslash_escapes = False -log.class_logger(DrizzleDialect) diff --git a/libs/sqlalchemy/dialects/drizzle/mysqldb.py b/libs/sqlalchemy/dialects/drizzle/mysqldb.py index ce9518a8..7d91cc36 100644 --- a/libs/sqlalchemy/dialects/drizzle/mysqldb.py +++ b/libs/sqlalchemy/dialects/drizzle/mysqldb.py @@ -1,15 +1,10 @@ -"""Support for the Drizzle database via the mysql-python adapter. +""" +.. dialect:: drizzle+mysqldb + :name: MySQL-Python + :dbapi: mysqldb + :connectstring: drizzle+mysqldb://:@[:]/ + :url: http://sourceforge.net/projects/mysql-python -MySQL-Python is available at: - - http://sourceforge.net/projects/mysql-python - -Connecting ------------ - -Connect string format:: - - drizzle+mysqldb://:@[:]/ """ diff --git a/libs/sqlalchemy/dialects/firebird/__init__.py b/libs/sqlalchemy/dialects/firebird/__init__.py index 2a3b756f..094ac3e8 100644 --- a/libs/sqlalchemy/dialects/firebird/__init__.py +++ b/libs/sqlalchemy/dialects/firebird/__init__.py @@ -1,12 +1,12 @@ # firebird/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy.dialects.firebird import base, kinterbasdb +from sqlalchemy.dialects.firebird import base, kinterbasdb, fdb -base.dialect = kinterbasdb.dialect +base.dialect = fdb.dialect from sqlalchemy.dialects.firebird.base import \ SMALLINT, BIGINT, FLOAT, FLOAT, DATE, TIME, \ @@ -18,5 +18,3 @@ __all__ = ( 'TEXT', 'NUMERIC', 'FLOAT', 'TIMESTAMP', 'VARCHAR', 'CHAR', 'BLOB', 'dialect' ) - - diff --git a/libs/sqlalchemy/dialects/firebird/base.py b/libs/sqlalchemy/dialects/firebird/base.py index a0bb9c20..b9af6e58 100644 --- a/libs/sqlalchemy/dialects/firebird/base.py +++ b/libs/sqlalchemy/dialects/firebird/base.py @@ -1,16 +1,16 @@ # firebird/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -Support for the Firebird database. -Connectivity is usually supplied via the kinterbasdb_ DBAPI module. +.. dialect:: firebird + :name: Firebird -Dialects -~~~~~~~~ +Firebird Dialects +----------------- Firebird offers two distinct dialects_ (not to be confused with a SQLAlchemy ``Dialect``): @@ -27,7 +27,7 @@ support for dialect 1 is not well tested and probably has incompatibilities. Locking Behavior -~~~~~~~~~~~~~~~~ +---------------- Firebird locks tables aggressively. For this reason, a DROP TABLE may hang until other transactions are released. SQLAlchemy does its best @@ -47,7 +47,7 @@ The above use case can be alleviated by calling ``first()`` on the all remaining cursor/connection resources. RETURNING support -~~~~~~~~~~~~~~~~~ +----------------- Firebird 2.0 supports returning a result set from inserts, and 2.1 extends that to deletes and updates. This is generically exposed by @@ -69,7 +69,7 @@ the SQLAlchemy ``returning()`` method, such as:: """ -import datetime, re +import datetime from sqlalchemy import schema as sa_schema from sqlalchemy import exc, types as sqltypes, sql, util @@ -78,9 +78,8 @@ from sqlalchemy.engine import base, default, reflection from sqlalchemy.sql import compiler -from sqlalchemy.types import (BIGINT, BLOB, BOOLEAN, DATE, - FLOAT, INTEGER, NUMERIC, SMALLINT, - TEXT, TIME, TIMESTAMP) +from sqlalchemy.types import (BIGINT, BLOB, DATE, FLOAT, INTEGER, NUMERIC, + SMALLINT, TEXT, TIME, TIMESTAMP, Integer) RESERVED_WORDS = set([ @@ -126,36 +125,49 @@ RESERVED_WORDS = set([ class _StringType(sqltypes.String): """Base for Firebird string types.""" - def __init__(self, charset = None, **kw): + def __init__(self, charset=None, **kw): self.charset = charset super(_StringType, self).__init__(**kw) + class VARCHAR(_StringType, sqltypes.VARCHAR): """Firebird VARCHAR type""" __visit_name__ = 'VARCHAR' - def __init__(self, length = None, **kwargs): + def __init__(self, length=None, **kwargs): super(VARCHAR, self).__init__(length=length, **kwargs) + class CHAR(_StringType, sqltypes.CHAR): """Firebird CHAR type""" __visit_name__ = 'CHAR' - def __init__(self, length = None, **kwargs): + def __init__(self, length=None, **kwargs): super(CHAR, self).__init__(length=length, **kwargs) + +class _FBDateTime(sqltypes.DateTime): + def bind_processor(self, dialect): + def process(value): + if type(value) == datetime.date: + return datetime.datetime(value.year, value.month, value.day) + else: + return value + return process + colspecs = { + sqltypes.DateTime: _FBDateTime } ischema_names = { 'SHORT': SMALLINT, - 'LONG': BIGINT, + 'LONG': INTEGER, 'QUAD': FLOAT, 'FLOAT': FLOAT, 'DATE': DATE, 'TIME': TIME, 'TEXT': TEXT, - 'INT64': NUMERIC, + 'INT64': BIGINT, 'DOUBLE': FLOAT, 'TIMESTAMP': TIMESTAMP, 'VARYING': VARCHAR, @@ -192,20 +204,42 @@ class FBTypeCompiler(compiler.GenericTypeCompiler): return self._extend_string(type_, basic) def visit_VARCHAR(self, type_): + if not type_.length: + raise exc.CompileError( + "VARCHAR requires a length on dialect %s" % + self.dialect.name) basic = super(FBTypeCompiler, self).visit_VARCHAR(type_) return self._extend_string(type_, basic) - class FBCompiler(sql.compiler.SQLCompiler): """Firebird specific idiosyncrasies""" - def visit_mod(self, binary, **kw): - # Firebird lacks a builtin modulo operator, but there is - # an equivalent function in the ib_udf library. + ansi_bind_rules = True + + #def visit_contains_op_binary(self, binary, operator, **kw): + # cant use CONTAINING b.c. it's case insensitive. + + #def visit_notcontains_op_binary(self, binary, operator, **kw): + # cant use NOT CONTAINING b.c. it's case insensitive. + + def visit_now_func(self, fn, **kw): + return "CURRENT_TIMESTAMP" + + def visit_startswith_op_binary(self, binary, operator, **kw): + return '%s STARTING WITH %s' % ( + binary.left._compiler_dispatch(self, **kw), + binary.right._compiler_dispatch(self, **kw)) + + def visit_notstartswith_op_binary(self, binary, operator, **kw): + return '%s NOT STARTING WITH %s' % ( + binary.left._compiler_dispatch(self, **kw), + binary.right._compiler_dispatch(self, **kw)) + + def visit_mod_binary(self, binary, operator, **kw): return "mod(%s, %s)" % ( - self.process(binary.left), - self.process(binary.right)) + self.process(binary.left, **kw), + self.process(binary.right, **kw)) def visit_alias(self, alias, asfrom=False, **kwargs): if self.dialect._version_two: @@ -249,7 +283,7 @@ class FBCompiler(sql.compiler.SQLCompiler): # may require parens - see similar example in the oracle # dialect if func.clauses is not None and len(func.clauses): - return self.process(func.clause_expr) + return self.process(func.clause_expr, **kw) else: return "" @@ -267,9 +301,9 @@ class FBCompiler(sql.compiler.SQLCompiler): result = "" if select._limit: - result += "FIRST %s " % self.process(sql.literal(select._limit)) + result += "FIRST %s " % self.process(sql.literal(select._limit)) if select._offset: - result +="SKIP %s " % self.process(sql.literal(select._offset)) + result += "SKIP %s " % self.process(sql.literal(select._offset)) if select._distinct: result += "DISTINCT " return result @@ -280,15 +314,11 @@ class FBCompiler(sql.compiler.SQLCompiler): return "" def returning_clause(self, stmt, returning_cols): - columns = [ - self.process( - self.label_select_column(None, c, asfrom=False), - within_columns_clause=True, - result_map=self.result_map - ) + self._label_select_column(None, c, True, False, {}) for c in expression._select_iterables(returning_cols) ] + return 'RETURNING ' + ', '.join(columns) @@ -329,6 +359,7 @@ class FBIdentifierPreparer(sql.compiler.IdentifierPreparer): """Install Firebird specific reserved words.""" reserved_words = RESERVED_WORDS + illegal_initial_characters = compiler.ILLEGAL_INITIAL_CHARACTERS.union(['_']) def __init__(self, dialect): super(FBIdentifierPreparer, self).__init__(dialect, omit_schema=True) @@ -444,18 +475,34 @@ class FBDialect(default.DefaultDialect): @reflection.cache def get_table_names(self, connection, schema=None, **kw): + # there are two queries commonly mentioned for this. + # this one, using view_blr, is at the Firebird FAQ among other places: + # http://www.firebirdfaq.org/faq174/ s = """ - SELECT DISTINCT rdb$relation_name - FROM rdb$relation_fields - WHERE rdb$system_flag=0 AND rdb$view_context IS NULL + select rdb$relation_name + from rdb$relations + where rdb$view_blr is null + and (rdb$system_flag is null or rdb$system_flag = 0); """ + + # the other query is this one. It's not clear if there's really + # any difference between these two. This link: + # http://www.alberton.info/firebird_sql_meta_info.html#.Ur3vXfZGni8 + # states them as interchangeable. Some discussion at [ticket:2898] + # SELECT DISTINCT rdb$relation_name + # FROM rdb$relation_fields + # WHERE rdb$system_flag=0 AND rdb$view_context IS NULL + return [self.normalize_name(row[0]) for row in connection.execute(s)] @reflection.cache def get_view_names(self, connection, schema=None, **kw): + # see http://www.firebirdfaq.org/faq174/ s = """ - SELECT distinct rdb$view_name - FROM rdb$view_relations + select rdb$relation_name + from rdb$relations + where rdb$view_blr is not null + and (rdb$system_flag is null or rdb$system_flag = 0); """ return [self.normalize_name(row[0]) for row in connection.execute(s)] @@ -474,7 +521,7 @@ class FBDialect(default.DefaultDialect): return None @reflection.cache - def get_primary_keys(self, connection, table_name, schema=None, **kw): + def get_pk_constraint(self, connection, table_name, schema=None, **kw): # Query to extract the PK/FK constrained fields of the given table keyqry = """ SELECT se.rdb$field_name AS fname @@ -486,7 +533,7 @@ class FBDialect(default.DefaultDialect): # get primary key fields c = connection.execute(keyqry, ["PRIMARY KEY", tablename]) pkfields = [self.normalize_name(r['fname']) for r in c.fetchall()] - return pkfields + return {'constrained_columns': pkfields, 'name': None} @reflection.cache def get_column_sequence(self, connection, @@ -541,7 +588,8 @@ class FBDialect(default.DefaultDialect): ORDER BY r.rdb$field_position """ # get the PK, used to determine the eventual associated sequence - pkey_cols = self.get_primary_keys(connection, table_name) + pk_constraint = self.get_pk_constraint(connection, table_name) + pkey_cols = pk_constraint['constrained_columns'] tablename = self.denormalize_name(table_name) # get all of the fields for this table @@ -561,8 +609,8 @@ class FBDialect(default.DefaultDialect): util.warn("Did not recognize type '%s' of column '%s'" % (colspec, name)) coltype = sqltypes.NULLTYPE - elif colspec == 'INT64': - coltype = coltype( + elif issubclass(coltype, Integer) and row['fprec'] != 0: + coltype = NUMERIC( precision=row['fprec'], scale=row['fscale'] * -1) elif colspec in ('VARYING', 'CSTRING'): @@ -593,11 +641,11 @@ class FBDialect(default.DefaultDialect): # Redundant defvalue = None col_d = { - 'name' : name, - 'type' : coltype, - 'nullable' : not bool(row['null_flag']), - 'default' : defvalue, - 'autoincrement':defvalue is None + 'name': name, + 'type': coltype, + 'nullable': not bool(row['null_flag']), + 'default': defvalue, + 'autoincrement': defvalue is None } if orig_colname.lower() == orig_colname: @@ -605,7 +653,7 @@ class FBDialect(default.DefaultDialect): # if the PK is a single field, try to see if its linked to # a sequence thru a trigger - if len(pkey_cols)==1 and name==pkey_cols[0]: + if len(pkey_cols) == 1 and name == pkey_cols[0]: seq_d = self.get_column_sequence(connection, tablename, name) if seq_d is not None: col_d['sequence'] = seq_d @@ -635,12 +683,12 @@ class FBDialect(default.DefaultDialect): tablename = self.denormalize_name(table_name) c = connection.execute(fkqry, ["FOREIGN KEY", tablename]) - fks = util.defaultdict(lambda:{ - 'name' : None, - 'constrained_columns' : [], - 'referred_schema' : None, - 'referred_table' : None, - 'referred_columns' : [] + fks = util.defaultdict(lambda: { + 'name': None, + 'constrained_columns': [], + 'referred_schema': None, + 'referred_table': None, + 'referred_columns': [] }) for row in c: @@ -653,7 +701,7 @@ class FBDialect(default.DefaultDialect): self.normalize_name(row['fname'])) fk['referred_columns'].append( self.normalize_name(row['targetfname'])) - return fks.values() + return list(fks.values()) @reflection.cache def get_indexes(self, connection, table_name, schema=None, **kw): @@ -669,7 +717,7 @@ class FBDialect(default.DefaultDialect): ic.rdb$index_name WHERE ix.rdb$relation_name=? AND ix.rdb$foreign_key IS NULL AND rdb$relation_constraints.rdb$constraint_type IS NULL - ORDER BY index_name, field_name + ORDER BY index_name, ic.rdb$field_position """ c = connection.execute(qry, [self.denormalize_name(table_name)]) @@ -684,17 +732,5 @@ class FBDialect(default.DefaultDialect): indexrec['column_names'].append( self.normalize_name(row['field_name'])) - return indexes.values() + return list(indexes.values()) - def do_execute(self, cursor, statement, parameters, context=None): - # kinterbase does not accept a None, but wants an empty list - # when there are no arguments. - cursor.execute(statement, parameters or []) - - def do_rollback(self, connection): - # Use the retaining feature, that keeps the transaction going - connection.rollback(True) - - def do_commit(self, connection): - # Use the retaining feature, that keeps the transaction going - connection.commit(True) diff --git a/libs/sqlalchemy/dialects/firebird/fdb.py b/libs/sqlalchemy/dialects/firebird/fdb.py new file mode 100644 index 00000000..4d94ef0d --- /dev/null +++ b/libs/sqlalchemy/dialects/firebird/fdb.py @@ -0,0 +1,115 @@ +# firebird/fdb.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" +.. dialect:: firebird+fdb + :name: fdb + :dbapi: pyodbc + :connectstring: firebird+fdb://user:password@host:port/path/to/db[?key=value&key=value...] + :url: http://pypi.python.org/pypi/fdb/ + + fdb is a kinterbasdb compatible DBAPI for Firebird. + + .. versionadded:: 0.8 - Support for the fdb Firebird driver. + + .. versionchanged:: 0.9 - The fdb dialect is now the default dialect + under the ``firebird://`` URL space, as ``fdb`` is now the official + Python driver for Firebird. + +Arguments +---------- + +The ``fdb`` dialect is based on the :mod:`sqlalchemy.dialects.firebird.kinterbasdb` +dialect, however does not accept every argument that Kinterbasdb does. + +* ``enable_rowcount`` - True by default, setting this to False disables + the usage of "cursor.rowcount" with the + Kinterbasdb dialect, which SQLAlchemy ordinarily calls upon automatically + after any UPDATE or DELETE statement. When disabled, SQLAlchemy's + ResultProxy will return -1 for result.rowcount. The rationale here is + that Kinterbasdb requires a second round trip to the database when + .rowcount is called - since SQLA's resultproxy automatically closes + the cursor after a non-result-returning statement, rowcount must be + called, if at all, before the result object is returned. Additionally, + cursor.rowcount may not return correct results with older versions + of Firebird, and setting this flag to False will also cause the + SQLAlchemy ORM to ignore its usage. The behavior can also be controlled on a + per-execution basis using the ``enable_rowcount`` option with + :meth:`.Connection.execution_options`:: + + conn = engine.connect().execution_options(enable_rowcount=True) + r = conn.execute(stmt) + print r.rowcount + +* ``retaining`` - False by default. Setting this to True will pass the + ``retaining=True`` keyword argument to the ``.commit()`` and ``.rollback()`` + methods of the DBAPI connection, which can improve performance in some + situations, but apparently with significant caveats. + Please read the fdb and/or kinterbasdb DBAPI documentation in order to + understand the implications of this flag. + + .. versionadded:: 0.8.2 - ``retaining`` keyword argument specifying + transaction retaining behavior - in 0.8 it defaults to ``True`` + for backwards compatibility. + + .. versionchanged:: 0.9.0 - the ``retaining`` flag defaults to ``False``. + In 0.8 it defaulted to ``True``. + + .. seealso:: + + http://pythonhosted.org/fdb/usage-guide.html#retaining-transactions - information + on the "retaining" flag. + +""" + +from .kinterbasdb import FBDialect_kinterbasdb +from ... import util + + +class FBDialect_fdb(FBDialect_kinterbasdb): + + def __init__(self, enable_rowcount=True, + retaining=False, **kwargs): + super(FBDialect_fdb, self).__init__( + enable_rowcount=enable_rowcount, + retaining=retaining, **kwargs) + + @classmethod + def dbapi(cls): + return __import__('fdb') + + def create_connect_args(self, url): + opts = url.translate_connect_args(username='user') + if opts.get('port'): + opts['host'] = "%s/%s" % (opts['host'], opts['port']) + del opts['port'] + opts.update(url.query) + + util.coerce_kw_type(opts, 'type_conv', int) + + return ([], opts) + + def _get_server_version_info(self, connection): + """Get the version of the Firebird server used by a connection. + + Returns a tuple of (`major`, `minor`, `build`), three integers + representing the version of the attached server. + """ + + # This is the simpler approach (the other uses the services api), + # that for backward compatibility reasons returns a string like + # LI-V6.3.3.12981 Firebird 2.0 + # where the first version is a fake one resembling the old + # Interbase signature. + + isc_info_firebird_version = 103 + fbconn = connection.connection + + version = fbconn.db_info(isc_info_firebird_version) + + return self._parse_version_info(version) + +dialect = FBDialect_fdb diff --git a/libs/sqlalchemy/dialects/firebird/kinterbasdb.py b/libs/sqlalchemy/dialects/firebird/kinterbasdb.py index ddca91db..b8a83a07 100644 --- a/libs/sqlalchemy/dialects/firebird/kinterbasdb.py +++ b/libs/sqlalchemy/dialects/firebird/kinterbasdb.py @@ -1,58 +1,48 @@ # firebird/kinterbasdb.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -The most common way to connect to a Firebird engine is implemented by -kinterbasdb__, currently maintained__ directly by the Firebird people. +.. dialect:: firebird+kinterbasdb + :name: kinterbasdb + :dbapi: kinterbasdb + :connectstring: firebird+kinterbasdb://user:password@host:port/path/to/db[?key=value&key=value...] + :url: http://firebirdsql.org/index.php?op=devel&sub=python -The connection URL is of the form -``firebird[+kinterbasdb]://user:password@host:port/path/to/db[?key=value&key=value...]``. +Arguments +---------- -Kinterbasedb backend specific keyword arguments are: +The Kinterbasdb backend accepts the ``enable_rowcount`` and ``retaining`` +arguments accepted by the :mod:`sqlalchemy.dialects.firebird.fdb` dialect. In addition, it +also accepts the following: -* type_conv - select the kind of mapping done on the types: by default - SQLAlchemy uses 200 with Unicode, datetime and decimal support (see - details__). +* ``type_conv`` - select the kind of mapping done on the types: by default + SQLAlchemy uses 200 with Unicode, datetime and decimal support. See + the linked documents below for further information. -* concurrency_level - set the backend policy with regards to threading - issues: by default SQLAlchemy uses policy 1 (see details__). +* ``concurrency_level`` - set the backend policy with regards to threading + issues: by default SQLAlchemy uses policy 1. See the linked documents + below for futher information. -* enable_rowcount - True by default, setting this to False disables - the usage of "cursor.rowcount" with the - Kinterbasdb dialect, which SQLAlchemy ordinarily calls upon automatically - after any UPDATE or DELETE statement. When disabled, SQLAlchemy's - ResultProxy will return -1 for result.rowcount. The rationale here is - that Kinterbasdb requires a second round trip to the database when - .rowcount is called - since SQLA's resultproxy automatically closes - the cursor after a non-result-returning statement, rowcount must be - called, if at all, before the result object is returned. Additionally, - cursor.rowcount may not return correct results with older versions - of Firebird, and setting this flag to False will also cause the - SQLAlchemy ORM to ignore its usage. The behavior can also be controlled on a - per-execution basis using the `enable_rowcount` option with - :meth:`execution_options()`:: +.. seealso:: - conn = engine.connect().execution_options(enable_rowcount=True) - r = conn.execute(stmt) - print r.rowcount + http://sourceforge.net/projects/kinterbasdb + + http://kinterbasdb.sourceforge.net/dist_docs/usage.html#adv_param_conv_dynamic_type_translation + + http://kinterbasdb.sourceforge.net/dist_docs/usage.html#special_issue_concurrency -__ http://sourceforge.net/projects/kinterbasdb -__ http://firebirdsql.org/index.php?op=devel&sub=python -__ http://kinterbasdb.sourceforge.net/dist_docs/usage.html#adv_param_conv_dynamic_type_translation -__ http://kinterbasdb.sourceforge.net/dist_docs/usage.html#special_issue_concurrency """ -from sqlalchemy.dialects.firebird.base import FBDialect, \ - FBCompiler, FBExecutionContext -from sqlalchemy import util, types as sqltypes -from sqlalchemy.util.compat import decimal +from .base import FBDialect, FBExecutionContext +from ... import util, types as sqltypes from re import match +import decimal -class _FBNumeric_kinterbasdb(sqltypes.Numeric): +class _kinterbasdb_numeric(object): def bind_processor(self, dialect): def process(value): if isinstance(value, decimal.Decimal): @@ -61,6 +51,13 @@ class _FBNumeric_kinterbasdb(sqltypes.Numeric): return value return process +class _FBNumeric_kinterbasdb(_kinterbasdb_numeric, sqltypes.Numeric): + pass + +class _FBFloat_kinterbasdb(_kinterbasdb_numeric, sqltypes.Float): + pass + + class FBExecutionContext_kinterbasdb(FBExecutionContext): @property def rowcount(self): @@ -70,6 +67,7 @@ class FBExecutionContext_kinterbasdb(FBExecutionContext): else: return -1 + class FBDialect_kinterbasdb(FBDialect): driver = 'kinterbasdb' supports_sane_rowcount = False @@ -81,24 +79,37 @@ class FBDialect_kinterbasdb(FBDialect): colspecs = util.update_copy( FBDialect.colspecs, { - sqltypes.Numeric:_FBNumeric_kinterbasdb, + sqltypes.Numeric: _FBNumeric_kinterbasdb, + sqltypes.Float: _FBFloat_kinterbasdb, } ) def __init__(self, type_conv=200, concurrency_level=1, - enable_rowcount=True, **kwargs): + enable_rowcount=True, + retaining=False, **kwargs): super(FBDialect_kinterbasdb, self).__init__(**kwargs) self.enable_rowcount = enable_rowcount self.type_conv = type_conv self.concurrency_level = concurrency_level + self.retaining = retaining if enable_rowcount: self.supports_sane_rowcount = True @classmethod def dbapi(cls): - k = __import__('kinterbasdb') - return k + return __import__('kinterbasdb') + + def do_execute(self, cursor, statement, parameters, context=None): + # kinterbase does not accept a None, but wants an empty list + # when there are no arguments. + cursor.execute(statement, parameters or []) + + def do_rollback(self, dbapi_connection): + dbapi_connection.rollback(self.retaining) + + def do_commit(self, dbapi_connection): + dbapi_connection.commit(self.retaining) def create_connect_args(self, url): opts = url.translate_connect_args(username='user') @@ -117,7 +128,8 @@ class FBDialect_kinterbasdb(FBDialect): initialized = getattr(self.dbapi, 'initialized', None) if initialized is None: # CVS rev 1.96 changed the name of the attribute: - # http://kinterbasdb.cvs.sourceforge.net/viewvc/kinterbasdb/Kinterbasdb-3.0/__init__.py?r1=1.95&r2=1.96 + # http://kinterbasdb.cvs.sourceforge.net/viewvc/kinterbasdb/ + # Kinterbasdb-3.0/__init__.py?r1=1.95&r2=1.96 initialized = getattr(self.dbapi, '_initialized', False) if not initialized: self.dbapi.init(type_conv=type_conv, diff --git a/libs/sqlalchemy/dialects/informix/__init__.py b/libs/sqlalchemy/dialects/informix/__init__.py deleted file mode 100644 index bd633da5..00000000 --- a/libs/sqlalchemy/dialects/informix/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# informix/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -from sqlalchemy.dialects.informix import base, informixdb - -base.dialect = informixdb.dialect \ No newline at end of file diff --git a/libs/sqlalchemy/dialects/informix/base.py b/libs/sqlalchemy/dialects/informix/base.py deleted file mode 100644 index 07561f8d..00000000 --- a/libs/sqlalchemy/dialects/informix/base.py +++ /dev/null @@ -1,596 +0,0 @@ -# informix/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# coding: gbk -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -"""Support for the Informix database. - -.. note:: - - The Informix dialect functions on current SQLAlchemy versions - but is not regularly tested, and may have many issues and - caveats not currently handled. - -""" - - -import datetime - -from sqlalchemy import sql, schema, exc, pool, util -from sqlalchemy.sql import compiler, text -from sqlalchemy.engine import default, reflection -from sqlalchemy import types as sqltypes - -RESERVED_WORDS = set( - ["abs", "absolute", "access", "access_method", "acos", "active", "add", - "address", "add_months", "admin", "after", "aggregate", "alignment", - "all", "allocate", "all_rows", "alter", "and", "ansi", "any", "append", - "array", "as", "asc", "ascii", "asin", "at", "atan", "atan2", "attach", - "attributes", "audit", "authentication", "authid", "authorization", - "authorized", "auto", "autofree", "auto_reprepare", "auto_stat_mode", - "avg", "avoid_execute", "avoid_fact", "avoid_full", "avoid_hash", - "avoid_index", "avoid_index_sj", "avoid_multi_index", "avoid_nl", - "avoid_star_join", "avoid_subqf", "based", "before", "begin", - "between", "bigint", "bigserial", "binary", "bitand", "bitandnot", - "bitnot", "bitor", "bitxor", "blob", "blobdir", "boolean", "both", - "bound_impl_pdq", "buffered", "builtin", "by", "byte", "cache", "call", - "cannothash", "cardinality", "cascade", "case", "cast", "ceil", "char", - "character", "character_length", "char_length", "check", "class", - "class_origin", "client", "clob", "clobdir", "close", "cluster", - "clustersize", "cobol", "codeset", "collation", "collection", - "column", "columns", "commit", "committed", "commutator", "component", - "components", "concat", "concurrent", "connect", "connection", - "connection_name", "connect_by_iscycle", "connect_by_isleaf", - "connect_by_rootconst", "constraint", "constraints", "constructor", - "context", "continue", "copy", "cos", "costfunc", "count", "crcols", - "create", "cross", "current", "current_role", "currval", "cursor", - "cycle", "database", "datafiles", "dataskip", "date", "datetime", - "day", "dba", "dbdate", "dbinfo", "dbpassword", "dbsecadm", - "dbservername", "deallocate", "debug", "debugmode", "debug_env", "dec", - "decimal", "declare", "decode", "decrypt_binary", "decrypt_char", - "dec_t", "default", "default_role", "deferred", "deferred_prepare", - "define", "delay", "delete", "deleting", "delimited", "delimiter", - "deluxe", "desc", "describe", "descriptor", "detach", "diagnostics", - "directives", "dirty", "disable", "disabled", "disconnect", "disk", - "distinct", "distributebinary", "distributesreferences", - "distributions", "document", "domain", "donotdistribute", "dormant", - "double", "drop", "dtime_t", "each", "elif", "else", "enabled", - "encryption", "encrypt_aes", "encrypt_tdes", "end", "enum", - "environment", "error", "escape", "exception", "exclusive", "exec", - "execute", "executeanywhere", "exemption", "exists", "exit", "exp", - "explain", "explicit", "express", "expression", "extdirectives", - "extend", "extent", "external", "fact", "false", "far", "fetch", - "file", "filetoblob", "filetoclob", "fillfactor", "filtering", "first", - "first_rows", "fixchar", "fixed", "float", "floor", "flush", "for", - "force", "forced", "force_ddl_exec", "foreach", "foreign", "format", - "format_units", "fortran", "found", "fraction", "fragment", - "fragments", "free", "from", "full", "function", "general", "get", - "gethint", "global", "go", "goto", "grant", "greaterthan", - "greaterthanorequal", "group", "handlesnulls", "hash", "having", "hdr", - "hex", "high", "hint", "hold", "home", "hour", "idslbacreadarray", - "idslbacreadset", "idslbacreadtree", "idslbacrules", - "idslbacwritearray", "idslbacwriteset", "idslbacwritetree", - "idssecuritylabel", "if", "ifx_auto_reprepare", "ifx_batchedread_table", - "ifx_int8_t", "ifx_lo_create_spec_t", "ifx_lo_stat_t", "immediate", - "implicit", "implicit_pdq", "in", "inactive", "increment", "index", - "indexes", "index_all", "index_sj", "indicator", "informix", "init", - "initcap", "inline", "inner", "inout", "insert", "inserting", "instead", - "int", "int8", "integ", "integer", "internal", "internallength", - "interval", "into", "intrvl_t", "is", "iscanonical", "isolation", - "item", "iterator", "java", "join", "keep", "key", "label", "labeleq", - "labelge", "labelglb", "labelgt", "labelle", "labellt", "labellub", - "labeltostring", "language", "last", "last_day", "leading", "left", - "length", "lessthan", "lessthanorequal", "let", "level", "like", - "limit", "list", "listing", "load", "local", "locator", "lock", "locks", - "locopy", "loc_t", "log", "log10", "logn", "long", "loop", "lotofile", - "low", "lower", "lpad", "ltrim", "lvarchar", "matched", "matches", - "max", "maxerrors", "maxlen", "maxvalue", "mdy", "median", "medium", - "memory", "memory_resident", "merge", "message_length", "message_text", - "middle", "min", "minute", "minvalue", "mod", "mode", "moderate", - "modify", "module", "money", "month", "months_between", "mounting", - "multiset", "multi_index", "name", "nchar", "negator", "new", "next", - "nextval", "next_day", "no", "nocache", "nocycle", "nomaxvalue", - "nomigrate", "nominvalue", "none", "non_dim", "non_resident", "noorder", - "normal", "not", "notemplatearg", "notequal", "null", "nullif", - "numeric", "numrows", "numtodsinterval", "numtoyminterval", "nvarchar", - "nvl", "octet_length", "of", "off", "old", "on", "online", "only", - "opaque", "opclass", "open", "optcompind", "optical", "optimization", - "option", "or", "order", "ordered", "out", "outer", "output", - "override", "page", "parallelizable", "parameter", "partition", - "pascal", "passedbyvalue", "password", "pdqpriority", "percaltl_cos", - "pipe", "pli", "pload", "policy", "pow", "power", "precision", - "prepare", "previous", "primary", "prior", "private", "privileges", - "procedure", "properties", "public", "put", "raise", "range", "raw", - "read", "real", "recordend", "references", "referencing", "register", - "rejectfile", "relative", "release", "remainder", "rename", - "reoptimization", "repeatable", "replace", "replication", "reserve", - "resolution", "resource", "restart", "restrict", "resume", "retain", - "retainupdatelocks", "return", "returned_sqlstate", "returning", - "returns", "reuse", "revoke", "right", "robin", "role", "rollback", - "rollforward", "root", "round", "routine", "row", "rowid", "rowids", - "rows", "row_count", "rpad", "rtrim", "rule", "sameas", "samples", - "sampling", "save", "savepoint", "schema", "scroll", "seclabel_by_comp", - "seclabel_by_name", "seclabel_to_char", "second", "secondary", - "section", "secured", "security", "selconst", "select", "selecting", - "selfunc", "selfuncargs", "sequence", "serial", "serial8", - "serializable", "serveruuid", "server_name", "session", "set", - "setsessionauth", "share", "short", "siblings", "signed", "sin", - "sitename", "size", "skall", "skinhibit", "skip", "skshow", - "smallfloat", "smallint", "some", "specific", "sql", "sqlcode", - "sqlcontext", "sqlerror", "sqlstate", "sqlwarning", "sqrt", - "stability", "stack", "standard", "start", "star_join", "statchange", - "statement", "static", "statistics", "statlevel", "status", "stdev", - "step", "stop", "storage", "store", "strategies", "string", - "stringtolabel", "struct", "style", "subclass_origin", "substr", - "substring", "sum", "support", "sync", "synonym", "sysdate", - "sysdbclose", "sysdbopen", "system", "sys_connect_by_path", "table", - "tables", "tan", "task", "temp", "template", "test", "text", "then", - "time", "timeout", "to", "today", "to_char", "to_date", - "to_dsinterval", "to_number", "to_yminterval", "trace", "trailing", - "transaction", "transition", "tree", "trigger", "triggers", "trim", - "true", "trunc", "truncate", "trusted", "type", "typedef", "typeid", - "typename", "typeof", "uid", "uncommitted", "under", "union", - "unique", "units", "unknown", "unload", "unlock", "unsigned", - "update", "updating", "upon", "upper", "usage", "use", - "uselastcommitted", "user", "use_hash", "use_nl", "use_subqf", - "using", "value", "values", "var", "varchar", "variable", "variance", - "variant", "varying", "vercols", "view", "violations", "void", - "volatile", "wait", "warning", "weekday", "when", "whenever", "where", - "while", "with", "without", "work", "write", "writedown", "writeup", - "xadatasource", "xid", "xload", "xunload", "year" - ]) - -class InfoDateTime(sqltypes.DateTime): - def bind_processor(self, dialect): - def process(value): - if value is not None: - if value.microsecond: - value = value.replace(microsecond=0) - return value - return process - -class InfoTime(sqltypes.Time): - def bind_processor(self, dialect): - def process(value): - if value is not None: - if value.microsecond: - value = value.replace(microsecond=0) - return value - return process - - def result_processor(self, dialect, coltype): - def process(value): - if isinstance(value, datetime.datetime): - return value.time() - else: - return value - return process - -colspecs = { - sqltypes.DateTime : InfoDateTime, - sqltypes.TIMESTAMP: InfoDateTime, - sqltypes.Time: InfoTime, -} - - -ischema_names = { - 0 : sqltypes.CHAR, # CHAR - 1 : sqltypes.SMALLINT, # SMALLINT - 2 : sqltypes.INTEGER, # INT - 3 : sqltypes.FLOAT, # Float - 3 : sqltypes.Float, # SmallFloat - 5 : sqltypes.DECIMAL, # DECIMAL - 6 : sqltypes.Integer, # Serial - 7 : sqltypes.DATE, # DATE - 8 : sqltypes.Numeric, # MONEY - 10 : sqltypes.DATETIME, # DATETIME - 11 : sqltypes.LargeBinary, # BYTE - 12 : sqltypes.TEXT, # TEXT - 13 : sqltypes.VARCHAR, # VARCHAR - 15 : sqltypes.NCHAR, # NCHAR - 16 : sqltypes.NVARCHAR, # NVARCHAR - 17 : sqltypes.Integer, # INT8 - 18 : sqltypes.Integer, # Serial8 - 43 : sqltypes.String, # LVARCHAR - -1 : sqltypes.BLOB, # BLOB - -1 : sqltypes.CLOB, # CLOB -} - - -class InfoTypeCompiler(compiler.GenericTypeCompiler): - def visit_DATETIME(self, type_): - return "DATETIME YEAR TO SECOND" - - def visit_TIME(self, type_): - return "DATETIME HOUR TO SECOND" - - def visit_TIMESTAMP(self, type_): - return "DATETIME YEAR TO SECOND" - - def visit_large_binary(self, type_): - return "BYTE" - - def visit_boolean(self, type_): - return "SMALLINT" - -class InfoSQLCompiler(compiler.SQLCompiler): - def default_from(self): - return " from systables where tabname = 'systables' " - - def get_select_precolumns(self, select): - s = "" - if select._offset: - s += "SKIP %s " % select._offset - if select._limit: - s += "FIRST %s " % select._limit - s += select._distinct and "DISTINCT " or "" - return s - - def visit_select(self, select, asfrom=False, parens=True, **kw): - text = compiler.SQLCompiler.visit_select(self, select, asfrom, parens, **kw) - if asfrom and parens and self.dialect.server_version_info < (11,): - #assuming that 11 version doesn't need this, not tested - return "table(multiset" + text + ")" - else: - return text - - def limit_clause(self, select): - return "" - - def visit_function(self, func, **kw): - if func.name.lower() == 'current_date': - return "today" - elif func.name.lower() == 'current_time': - return "CURRENT HOUR TO SECOND" - elif func.name.lower() in ('current_timestamp', 'now'): - return "CURRENT YEAR TO SECOND" - else: - return compiler.SQLCompiler.visit_function(self, func, **kw) - - def visit_mod(self, binary, **kw): - return "MOD(%s, %s)" % (self.process(binary.left), self.process(binary.right)) - - -class InfoDDLCompiler(compiler.DDLCompiler): - - def visit_add_constraint(self, create): - preparer = self.preparer - return "ALTER TABLE %s ADD CONSTRAINT %s" % ( - self.preparer.format_table(create.element.table), - self.process(create.element) - ) - - def get_column_specification(self, column, **kw): - colspec = self.preparer.format_column(column) - first = None - if column.primary_key and column.autoincrement: - try: - first = [c for c in column.table.primary_key.columns - if (c.autoincrement and - isinstance(c.type, sqltypes.Integer) and - not c.foreign_keys)].pop(0) - except IndexError: - pass - - if column is first: - colspec += " SERIAL" - else: - colspec += " " + self.dialect.type_compiler.process(column.type) - default = self.get_column_default_string(column) - if default is not None: - colspec += " DEFAULT " + default - - if not column.nullable: - colspec += " NOT NULL" - - return colspec - - def get_column_default_string(self, column): - if (isinstance(column.server_default, schema.DefaultClause) and - isinstance(column.server_default.arg, basestring)): - if isinstance(column.type, (sqltypes.Integer, sqltypes.Numeric)): - return self.sql_compiler.process(text(column.server_default.arg)) - - return super(InfoDDLCompiler, self).get_column_default_string(column) - - ### Informix wants the constraint name at the end, hence this ist c&p from sql/compiler.py - def visit_primary_key_constraint(self, constraint): - if len(constraint) == 0: - return '' - text = "PRIMARY KEY " - text += "(%s)" % ', '.join(self.preparer.quote(c.name, c.quote) - for c in constraint) - text += self.define_constraint_deferrability(constraint) - - if constraint.name is not None: - text += " CONSTRAINT %s" % self.preparer.format_constraint(constraint) - return text - - def visit_foreign_key_constraint(self, constraint): - preparer = self.dialect.identifier_preparer - remote_table = list(constraint._elements.values())[0].column.table - text = "FOREIGN KEY (%s) REFERENCES %s (%s)" % ( - ', '.join(preparer.quote(f.parent.name, f.parent.quote) - for f in constraint._elements.values()), - preparer.format_table(remote_table), - ', '.join(preparer.quote(f.column.name, f.column.quote) - for f in constraint._elements.values()) - ) - text += self.define_constraint_cascades(constraint) - text += self.define_constraint_deferrability(constraint) - - if constraint.name is not None: - text += " CONSTRAINT %s " % \ - preparer.format_constraint(constraint) - return text - - def visit_unique_constraint(self, constraint): - text = "UNIQUE (%s)" % (', '.join(self.preparer.quote(c.name, c.quote) for c in constraint)) - text += self.define_constraint_deferrability(constraint) - - if constraint.name is not None: - text += "CONSTRAINT %s " % self.preparer.format_constraint(constraint) - return text - -class InformixIdentifierPreparer(compiler.IdentifierPreparer): - - reserved_words = RESERVED_WORDS - - -class InformixDialect(default.DefaultDialect): - name = 'informix' - - max_identifier_length = 128 # adjusts at runtime based on server version - - type_compiler = InfoTypeCompiler - statement_compiler = InfoSQLCompiler - ddl_compiler = InfoDDLCompiler - colspecs = colspecs - ischema_names = ischema_names - preparer = InformixIdentifierPreparer - default_paramstyle = 'qmark' - - def __init__(self, has_transactions=True, *args, **kwargs): - self.has_transactions = has_transactions - default.DefaultDialect.__init__(self, *args, **kwargs) - - def initialize(self, connection): - super(InformixDialect, self).initialize(connection) - - # http://www.querix.com/support/knowledge-base/error_number_message/error_200 - if self.server_version_info < (9, 2): - self.max_identifier_length = 18 - else: - self.max_identifier_length = 128 - - def do_begin(self, connection): - cu = connection.cursor() - cu.execute('SET LOCK MODE TO WAIT') - if self.has_transactions: - cu.execute('SET ISOLATION TO REPEATABLE READ') - - def do_commit(self, connection): - if self.has_transactions: - connection.commit() - - def do_rollback(self, connection): - if self.has_transactions: - connection.rollback() - - def _get_table_names(self, connection, schema, type, **kw): - schema = schema or self.default_schema_name - s = "select tabname, owner from systables where owner=? and tabtype=?" - return [row[0] for row in connection.execute(s, schema, type)] - - @reflection.cache - def get_table_names(self, connection, schema=None, **kw): - return self._get_table_names(connection, schema, 'T', **kw) - - @reflection.cache - def get_view_names(self, connection, schema=None, **kw): - return self._get_table_names(connection, schema, 'V', **kw) - - @reflection.cache - def get_schema_names(self, connection, **kw): - s = "select owner from systables" - return [row[0] for row in connection.execute(s)] - - def has_table(self, connection, table_name, schema=None): - schema = schema or self.default_schema_name - cursor = connection.execute( - """select tabname from systables where tabname=? and owner=?""", - table_name, schema) - return cursor.first() is not None - - @reflection.cache - def get_columns(self, connection, table_name, schema=None, **kw): - schema = schema or self.default_schema_name - c = connection.execute( - """select colname, coltype, collength, t3.default, t1.colno from - syscolumns as t1 , systables as t2 , OUTER sysdefaults as t3 - where t1.tabid = t2.tabid and t2.tabname=? and t2.owner=? - and t3.tabid = t2.tabid and t3.colno = t1.colno - order by t1.colno""", table_name, schema) - - primary_cols = self.get_primary_keys(connection, table_name, schema, **kw) - - columns = [] - rows = c.fetchall() - for name, colattr, collength, default, colno in rows: - name = name.lower() - - autoincrement = False - primary_key = False - - if name in primary_cols: - primary_key = True - - # in 7.31, coltype = 0x000 - # ^^-- column type - # ^-- 1 not null, 0 null - not_nullable, coltype = divmod(colattr, 256) - if coltype not in (0, 13) and default: - default = default.split()[-1] - - if coltype == 6: # Serial, mark as autoincrement - autoincrement = True - - if coltype == 0 or coltype == 13: # char, varchar - coltype = ischema_names[coltype](collength) - if default: - default = "'%s'" % default - elif coltype == 5: # decimal - precision, scale = (collength & 0xFF00) >> 8, collength & 0xFF - if scale == 255: - scale = 0 - coltype = sqltypes.Numeric(precision, scale) - else: - try: - coltype = ischema_names[coltype] - except KeyError: - util.warn("Did not recognize type '%s' of column '%s'" % - (coltype, name)) - coltype = sqltypes.NULLTYPE - - column_info = dict(name=name, type=coltype, nullable=not not_nullable, - default=default, autoincrement=autoincrement, - primary_key=primary_key) - columns.append(column_info) - return columns - - @reflection.cache - def get_foreign_keys(self, connection, table_name, schema=None, **kw): - schema_sel = schema or self.default_schema_name - c = connection.execute( - """select t1.constrname as cons_name, - t4.colname as local_column, t7.tabname as remote_table, - t6.colname as remote_column, t7.owner as remote_owner - from sysconstraints as t1 , systables as t2 , - sysindexes as t3 , syscolumns as t4 , - sysreferences as t5 , syscolumns as t6 , systables as t7 , - sysconstraints as t8 , sysindexes as t9 - where t1.tabid = t2.tabid and t2.tabname=? and t2.owner=? and t1.constrtype = 'R' - and t3.tabid = t2.tabid and t3.idxname = t1.idxname - and t4.tabid = t2.tabid and t4.colno in (t3.part1, t3.part2, t3.part3, - t3.part4, t3.part5, t3.part6, t3.part7, t3.part8, t3.part9, t3.part10, - t3.part11, t3.part11, t3.part12, t3.part13, t3.part4, t3.part15, t3.part16) - and t5.constrid = t1.constrid and t8.constrid = t5.primary - and t6.tabid = t5.ptabid and t6.colno in (t9.part1, t9.part2, t9.part3, - t9.part4, t9.part5, t9.part6, t9.part7, t9.part8, t9.part9, t9.part10, - t9.part11, t9.part11, t9.part12, t9.part13, t9.part4, t9.part15, t9.part16) and t9.idxname = - t8.idxname - and t7.tabid = t5.ptabid""", table_name, schema_sel) - - - def fkey_rec(): - return { - 'name' : None, - 'constrained_columns' : [], - 'referred_schema' : None, - 'referred_table' : None, - 'referred_columns' : [] - } - - fkeys = util.defaultdict(fkey_rec) - - rows = c.fetchall() - for cons_name, local_column, \ - remote_table, remote_column, remote_owner in rows: - - rec = fkeys[cons_name] - rec['name'] = cons_name - local_cols, remote_cols = \ - rec['constrained_columns'], rec['referred_columns'] - - if not rec['referred_table']: - rec['referred_table'] = remote_table - if schema is not None: - rec['referred_schema'] = remote_owner - - if local_column not in local_cols: - local_cols.append(local_column) - if remote_column not in remote_cols: - remote_cols.append(remote_column) - - return fkeys.values() - - @reflection.cache - def get_primary_keys(self, connection, table_name, schema=None, **kw): - schema = schema or self.default_schema_name - - # Select the column positions from sysindexes for sysconstraints - data = connection.execute( - """select t2.* - from systables as t1, sysindexes as t2, sysconstraints as t3 - where t1.tabid=t2.tabid and t1.tabname=? and t1.owner=? - and t2.idxname=t3.idxname and t3.constrtype='P'""", - table_name, schema - ).fetchall() - - colpositions = set() - - for row in data: - colpos = set([getattr(row, 'part%d' % x) for x in range(1,16)]) - colpositions |= colpos - - if not len(colpositions): - return [] - - # Select the column names using the columnpositions - # TODO: Maybe cache a bit of those col infos (eg select all colnames for one table) - place_holder = ','.join('?'*len(colpositions)) - c = connection.execute( - """select t1.colname - from syscolumns as t1, systables as t2 - where t2.tabname=? and t1.tabid = t2.tabid and - t1.colno in (%s)""" % place_holder, - table_name, *colpositions - ).fetchall() - - return reduce(lambda x,y: list(x)+list(y), c, []) - - @reflection.cache - def get_indexes(self, connection, table_name, schema, **kw): - # TODO: schema... - c = connection.execute( - """select t1.* - from sysindexes as t1 , systables as t2 - where t1.tabid = t2.tabid and t2.tabname=?""", - table_name) - - indexes = [] - for row in c.fetchall(): - colnames = [getattr(row, 'part%d' % x) for x in range(1,16)] - colnames = [x for x in colnames if x] - place_holder = ','.join('?'*len(colnames)) - c = connection.execute( - """select t1.colname - from syscolumns as t1, systables as t2 - where t2.tabname=? and t1.tabid = t2.tabid and - t1.colno in (%s)""" % place_holder, - table_name, *colnames - ).fetchall() - c = reduce(lambda x,y: list(x)+list(y), c, []) - indexes.append({ - 'name': row.idxname, - 'unique': row.idxtype.lower() == 'u', - 'column_names': c - }) - return indexes - - @reflection.cache - def get_view_definition(self, connection, view_name, schema=None, **kw): - schema = schema or self.default_schema_name - c = connection.execute( - """select t1.viewtext - from sysviews as t1 , systables as t2 - where t1.tabid=t2.tabid and t2.tabname=? - and t2.owner=? order by seqno""", - view_name, schema).fetchall() - - return ''.join([row[0] for row in c]) - - def _get_default_schema_name(self, connection): - return connection.execute('select CURRENT_ROLE from systables').scalar() diff --git a/libs/sqlalchemy/dialects/informix/informixdb.py b/libs/sqlalchemy/dialects/informix/informixdb.py deleted file mode 100644 index 8b543467..00000000 --- a/libs/sqlalchemy/dialects/informix/informixdb.py +++ /dev/null @@ -1,73 +0,0 @@ -# informix/informixdb.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -""" -Support for the informixdb DBAPI. - -informixdb is available at: - - http://informixdb.sourceforge.net/ - -Connecting -^^^^^^^^^^ - -Sample informix connection:: - - engine = create_engine('informix+informixdb://user:password@host/dbname') - -""" - -import re - -from sqlalchemy.dialects.informix.base import InformixDialect -from sqlalchemy.engine import default - -VERSION_RE = re.compile(r'(\d+)\.(\d+)(.+\d+)') - -class InformixExecutionContext_informixdb(default.DefaultExecutionContext): - def post_exec(self): - if self.isinsert: - self._lastrowid = self.cursor.sqlerrd[1] - - def get_lastrowid(self): - return self._lastrowid - - -class InformixDialect_informixdb(InformixDialect): - driver = 'informixdb' - execution_ctx_cls = InformixExecutionContext_informixdb - - @classmethod - def dbapi(cls): - return __import__('informixdb') - - def create_connect_args(self, url): - if url.host: - dsn = '%s@%s' % (url.database, url.host) - else: - dsn = url.database - - if url.username: - opt = {'user': url.username, 'password': url.password} - else: - opt = {} - - return ([dsn], opt) - - def _get_server_version_info(self, connection): - # http://informixdb.sourceforge.net/manual.html#inspecting-version-numbers - v = VERSION_RE.split(connection.connection.dbms_version) - return (int(v[1]), int(v[2]), v[3]) - - def is_disconnect(self, e, connection, cursor): - if isinstance(e, self.dbapi.OperationalError): - return 'closed the connection' in str(e) \ - or 'connection not open' in str(e) - else: - return False - - -dialect = InformixDialect_informixdb diff --git a/libs/sqlalchemy/dialects/maxdb/__init__.py b/libs/sqlalchemy/dialects/maxdb/__init__.py deleted file mode 100644 index 9d1d6418..00000000 --- a/libs/sqlalchemy/dialects/maxdb/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# maxdb/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -from sqlalchemy.dialects.maxdb import base, sapdb - -base.dialect = sapdb.dialect \ No newline at end of file diff --git a/libs/sqlalchemy/dialects/maxdb/base.py b/libs/sqlalchemy/dialects/maxdb/base.py deleted file mode 100644 index 68ae630e..00000000 --- a/libs/sqlalchemy/dialects/maxdb/base.py +++ /dev/null @@ -1,1117 +0,0 @@ -# maxdb/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -"""Support for the MaxDB database. - -.. note:: - - The MaxDB dialect is **non-functional as of SQLAlchemy 0.6**, - pending development efforts to bring it up-to-date. - -Overview --------- - -The ``maxdb`` dialect is **experimental** and has only been tested on 7.6.03.007 -and 7.6.00.037. Of these, **only 7.6.03.007 will work** with SQLAlchemy's ORM. -The earlier version has severe ``LEFT JOIN`` limitations and will return -incorrect results from even very simple ORM queries. - -Only the native Python DB-API is currently supported. ODBC driver support -is a future enhancement. - -Connecting ----------- - -The username is case-sensitive. If you usually connect to the -database with sqlcli and other tools in lower case, you likely need to -use upper case for DB-API. - -Implementation Notes --------------------- - -With the 7.6.00.37 driver and Python 2.5, it seems that all DB-API -generated exceptions are broken and can cause Python to crash. - -For 'somecol.in_([])' to work, the IN operator's generation must be changed -to cast 'NULL' to a numeric, i.e. NUM(NULL). The DB-API doesn't accept a -bind parameter there, so that particular generation must inline the NULL value, -which depends on [ticket:807]. - -The DB-API is very picky about where bind params may be used in queries. - -Bind params for some functions (e.g. MOD) need type information supplied. -The dialect does not yet do this automatically. - -Max will occasionally throw up 'bad sql, compile again' exceptions for -perfectly valid SQL. The dialect does not currently handle these, more -research is needed. - -MaxDB 7.5 and Sap DB <= 7.4 reportedly do not support schemas. A very -slightly different version of this dialect would be required to support -those versions, and can easily be added if there is demand. Some other -required components such as an Max-aware 'old oracle style' join compiler -(thetas with (+) outer indicators) are already done and available for -integration- email the devel list if you're interested in working on -this. - -Versions tested: 7.6.03.07 and 7.6.00.37, native Python DB-API - -* MaxDB has severe limitations on OUTER JOINs, which are essential to ORM - eager loading. And rather than raise an error if a SELECT can't be serviced, - the database simply returns incorrect results. -* Version 7.6.03.07 seems to JOIN properly, however the docs do not show the - OUTER restrictions being lifted (as of this writing), and no changelog is - available to confirm either. If you are using a different server version and - your tasks require the ORM or any semi-advanced SQL through the SQL layer, - running the SQLAlchemy test suite against your database is HIGHLY - recommended before you begin. -* Version 7.6.00.37 is LHS/RHS sensitive in `FROM lhs LEFT OUTER JOIN rhs ON - lhs.col=rhs.col` vs `rhs.col=lhs.col`! -* Version 7.6.00.37 is confused by `SELECT DISTINCT col as alias FROM t ORDER - BY col` - these aliased, DISTINCT, ordered queries need to be re-written to - order by the alias name. -* Version 7.6.x supports creating a SAVEPOINT but not its RELEASE. -* MaxDB supports autoincrement-style columns (DEFAULT SERIAL) and independent - sequences. When including a DEFAULT SERIAL column in an insert, 0 needs to - be inserted rather than NULL to generate a value. -* MaxDB supports ANSI and "old Oracle style" theta joins with (+) outer join - indicators. -* The SQLAlchemy dialect is schema-aware and probably won't function correctly - on server versions (pre-7.6?). Support for schema-less server versions could - be added if there's call. -* ORDER BY is not supported in subqueries. LIMIT is not supported in - subqueries. In 7.6.00.37, TOP does work in subqueries, but without limit not - so useful. OFFSET does not work in 7.6 despite being in the docs. Row number - tricks in WHERE via ROWNO may be possible but it only seems to allow - less-than comparison! -* Version 7.6.03.07 can't LIMIT if a derived table is in FROM: `SELECT * FROM - (SELECT * FROM a) LIMIT 2` -* MaxDB does not support sql's CAST and can only usefullly cast two types. - There isn't much implicit type conversion, so be precise when creating - `PassiveDefaults` in DDL generation: `'3'` and `3` aren't the same. - -sapdb.dbapi -^^^^^^^^^^^ - -* As of 2007-10-22 the Python 2.4 and 2.5 compatible versions of the DB-API - are no longer available. A forum posting at SAP states that the Python - driver will be available again "in the future". The last release from MySQL - AB works if you can find it. -* sequence.NEXTVAL skips every other value! -* No rowcount for executemany() -* If an INSERT into a table with a DEFAULT SERIAL column inserts the results - of a function `INSERT INTO t VALUES (LENGTH('foo'))`, the cursor won't have - the serial id. It needs to be manually yanked from tablename.CURRVAL. -* Super-duper picky about where bind params can be placed. Not smart about - converting Python types for some functions, such as `MOD(5, ?)`. -* LONG (text, binary) values in result sets are read-once. The dialect uses a - caching RowProxy when these types are present. -* Connection objects seem like they want to be either `close()`d or garbage - collected, but not both. There's a warning issued but it seems harmless. - - -""" -import datetime, itertools, re - -from sqlalchemy import exc, schema, sql, util, processors -from sqlalchemy.sql import operators as sql_operators, expression as sql_expr -from sqlalchemy.sql import compiler, visitors -from sqlalchemy.engine import base as engine_base, default, reflection -from sqlalchemy import types as sqltypes - - -class _StringType(sqltypes.String): - _type = None - - def __init__(self, length=None, encoding=None, **kw): - super(_StringType, self).__init__(length=length, **kw) - self.encoding = encoding - - def bind_processor(self, dialect): - if self.encoding == 'unicode': - return None - else: - def process(value): - if isinstance(value, unicode): - return value.encode(dialect.encoding) - else: - return value - return process - - def result_processor(self, dialect, coltype): - #XXX: this code is probably very slow and one should try (if at all - # possible) to determine the correct code path on a per-connection - # basis (ie, here in result_processor, instead of inside the processor - # function itself) and probably also use a few generic - # processors, or possibly per query (though there is no mechanism - # for that yet). - def process(value): - while True: - if value is None: - return None - elif isinstance(value, unicode): - return value - elif isinstance(value, str): - if self.convert_unicode or dialect.convert_unicode: - return value.decode(dialect.encoding) - else: - return value - elif hasattr(value, 'read'): - # some sort of LONG, snarf and retry - value = value.read(value.remainingLength()) - continue - else: - # unexpected type, return as-is - return value - return process - - -class MaxString(_StringType): - _type = 'VARCHAR' - - -class MaxUnicode(_StringType): - _type = 'VARCHAR' - - def __init__(self, length=None, **kw): - kw['encoding'] = 'unicode' - super(MaxUnicode, self).__init__(length=length, **kw) - - -class MaxChar(_StringType): - _type = 'CHAR' - - -class MaxText(_StringType): - _type = 'LONG' - - def __init__(self, length=None, **kw): - super(MaxText, self).__init__(length, **kw) - - def get_col_spec(self): - spec = 'LONG' - if self.encoding is not None: - spec = ' '.join((spec, self.encoding)) - elif self.convert_unicode: - spec = ' '.join((spec, 'UNICODE')) - - return spec - - -class MaxNumeric(sqltypes.Numeric): - """The FIXED (also NUMERIC, DECIMAL) data type.""" - - def __init__(self, precision=None, scale=None, **kw): - kw.setdefault('asdecimal', True) - super(MaxNumeric, self).__init__(scale=scale, precision=precision, - **kw) - - def bind_processor(self, dialect): - return None - - -class MaxTimestamp(sqltypes.DateTime): - def bind_processor(self, dialect): - def process(value): - if value is None: - return None - elif isinstance(value, basestring): - return value - elif dialect.datetimeformat == 'internal': - ms = getattr(value, 'microsecond', 0) - return value.strftime("%Y%m%d%H%M%S" + ("%06u" % ms)) - elif dialect.datetimeformat == 'iso': - ms = getattr(value, 'microsecond', 0) - return value.strftime("%Y-%m-%d %H:%M:%S." + ("%06u" % ms)) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % ( - dialect.datetimeformat,)) - return process - - def result_processor(self, dialect, coltype): - if dialect.datetimeformat == 'internal': - def process(value): - if value is None: - return None - else: - return datetime.datetime( - *[int(v) - for v in (value[0:4], value[4:6], value[6:8], - value[8:10], value[10:12], value[12:14], - value[14:])]) - elif dialect.datetimeformat == 'iso': - def process(value): - if value is None: - return None - else: - return datetime.datetime( - *[int(v) - for v in (value[0:4], value[5:7], value[8:10], - value[11:13], value[14:16], value[17:19], - value[20:])]) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % - dialect.datetimeformat) - return process - - -class MaxDate(sqltypes.Date): - def bind_processor(self, dialect): - def process(value): - if value is None: - return None - elif isinstance(value, basestring): - return value - elif dialect.datetimeformat == 'internal': - return value.strftime("%Y%m%d") - elif dialect.datetimeformat == 'iso': - return value.strftime("%Y-%m-%d") - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % ( - dialect.datetimeformat,)) - return process - - def result_processor(self, dialect, coltype): - if dialect.datetimeformat == 'internal': - def process(value): - if value is None: - return None - else: - return datetime.date(int(value[0:4]), int(value[4:6]), - int(value[6:8])) - elif dialect.datetimeformat == 'iso': - def process(value): - if value is None: - return None - else: - return datetime.date(int(value[0:4]), int(value[5:7]), - int(value[8:10])) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % - dialect.datetimeformat) - return process - - -class MaxTime(sqltypes.Time): - def bind_processor(self, dialect): - def process(value): - if value is None: - return None - elif isinstance(value, basestring): - return value - elif dialect.datetimeformat == 'internal': - return value.strftime("%H%M%S") - elif dialect.datetimeformat == 'iso': - return value.strftime("%H-%M-%S") - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % ( - dialect.datetimeformat,)) - return process - - def result_processor(self, dialect, coltype): - if dialect.datetimeformat == 'internal': - def process(value): - if value is None: - return None - else: - return datetime.time(int(value[0:4]), int(value[4:6]), - int(value[6:8])) - elif dialect.datetimeformat == 'iso': - def process(value): - if value is None: - return None - else: - return datetime.time(int(value[0:4]), int(value[5:7]), - int(value[8:10])) - else: - raise exc.InvalidRequestError( - "datetimeformat '%s' is not supported." % - dialect.datetimeformat) - return process - - -class MaxBlob(sqltypes.LargeBinary): - def bind_processor(self, dialect): - return processors.to_str - - def result_processor(self, dialect, coltype): - def process(value): - if value is None: - return None - else: - return value.read(value.remainingLength()) - return process - -class MaxDBTypeCompiler(compiler.GenericTypeCompiler): - def _string_spec(self, string_spec, type_): - if type_.length is None: - spec = 'LONG' - else: - spec = '%s(%s)' % (string_spec, type_.length) - - if getattr(type_, 'encoding'): - spec = ' '.join([spec, getattr(type_, 'encoding').upper()]) - return spec - - def visit_text(self, type_): - spec = 'LONG' - if getattr(type_, 'encoding', None): - spec = ' '.join((spec, type_.encoding)) - elif type_.convert_unicode: - spec = ' '.join((spec, 'UNICODE')) - - return spec - - def visit_char(self, type_): - return self._string_spec("CHAR", type_) - - def visit_string(self, type_): - return self._string_spec("VARCHAR", type_) - - def visit_large_binary(self, type_): - return "LONG BYTE" - - def visit_numeric(self, type_): - if type_.scale and type_.precision: - return 'FIXED(%s, %s)' % (type_.precision, type_.scale) - elif type_.precision: - return 'FIXED(%s)' % type_.precision - else: - return 'INTEGER' - - def visit_BOOLEAN(self, type_): - return "BOOLEAN" - -colspecs = { - sqltypes.Numeric: MaxNumeric, - sqltypes.DateTime: MaxTimestamp, - sqltypes.Date: MaxDate, - sqltypes.Time: MaxTime, - sqltypes.String: MaxString, - sqltypes.Unicode:MaxUnicode, - sqltypes.LargeBinary: MaxBlob, - sqltypes.Text: MaxText, - sqltypes.CHAR: MaxChar, - sqltypes.TIMESTAMP: MaxTimestamp, - sqltypes.BLOB: MaxBlob, - sqltypes.Unicode: MaxUnicode, - } - -ischema_names = { - 'boolean': sqltypes.BOOLEAN, - 'char': sqltypes.CHAR, - 'character': sqltypes.CHAR, - 'date': sqltypes.DATE, - 'fixed': sqltypes.Numeric, - 'float': sqltypes.FLOAT, - 'int': sqltypes.INT, - 'integer': sqltypes.INT, - 'long binary': sqltypes.BLOB, - 'long unicode': sqltypes.Text, - 'long': sqltypes.Text, - 'long': sqltypes.Text, - 'smallint': sqltypes.SmallInteger, - 'time': sqltypes.Time, - 'timestamp': sqltypes.TIMESTAMP, - 'varchar': sqltypes.VARCHAR, - } - -# TODO: migrate this to sapdb.py -class MaxDBExecutionContext(default.DefaultExecutionContext): - def post_exec(self): - # DB-API bug: if there were any functions as values, - # then do another select and pull CURRVAL from the - # autoincrement column's implicit sequence... ugh - if self.compiled.isinsert and not self.executemany: - table = self.compiled.statement.table - index, serial_col = _autoserial_column(table) - - if serial_col and (not self.compiled._safeserial or - not(self._last_inserted_ids) or - self._last_inserted_ids[index] in (None, 0)): - if table.schema: - sql = "SELECT %s.CURRVAL FROM DUAL" % ( - self.compiled.preparer.format_table(table)) - else: - sql = "SELECT CURRENT_SCHEMA.%s.CURRVAL FROM DUAL" % ( - self.compiled.preparer.format_table(table)) - - rs = self.cursor.execute(sql) - id = rs.fetchone()[0] - - if not self._last_inserted_ids: - # This shouldn't ever be > 1? Right? - self._last_inserted_ids = \ - [None] * len(table.primary_key.columns) - self._last_inserted_ids[index] = id - - super(MaxDBExecutionContext, self).post_exec() - - def get_result_proxy(self): - if self.cursor.description is not None: - for column in self.cursor.description: - if column[1] in ('Long Binary', 'Long', 'Long Unicode'): - return MaxDBResultProxy(self) - return engine_base.ResultProxy(self) - - @property - def rowcount(self): - if hasattr(self, '_rowcount'): - return self._rowcount - else: - return self.cursor.rowcount - - def fire_sequence(self, seq): - if seq.optional: - return None - return self._execute_scalar("SELECT %s.NEXTVAL FROM DUAL" % ( - self.dialect.identifier_preparer.format_sequence(seq))) - -class MaxDBCachedColumnRow(engine_base.RowProxy): - """A RowProxy that only runs result_processors once per column.""" - - def __init__(self, parent, row): - super(MaxDBCachedColumnRow, self).__init__(parent, row) - self.columns = {} - self._row = row - self._parent = parent - - def _get_col(self, key): - if key not in self.columns: - self.columns[key] = self._parent._get_col(self._row, key) - return self.columns[key] - - def __iter__(self): - for i in xrange(len(self._row)): - yield self._get_col(i) - - def __repr__(self): - return repr(list(self)) - - def __eq__(self, other): - return ((other is self) or - (other == tuple([self._get_col(key) - for key in xrange(len(self._row))]))) - def __getitem__(self, key): - if isinstance(key, slice): - indices = key.indices(len(self._row)) - return tuple([self._get_col(i) for i in xrange(*indices)]) - else: - return self._get_col(key) - - def __getattr__(self, name): - try: - return self._get_col(name) - except KeyError: - raise AttributeError(name) - - -class MaxDBResultProxy(engine_base.ResultProxy): - _process_row = MaxDBCachedColumnRow - -class MaxDBCompiler(compiler.SQLCompiler): - - function_conversion = { - 'CURRENT_DATE': 'DATE', - 'CURRENT_TIME': 'TIME', - 'CURRENT_TIMESTAMP': 'TIMESTAMP', - } - - # These functions must be written without parens when called with no - # parameters. e.g. 'SELECT DATE FROM DUAL' not 'SELECT DATE() FROM DUAL' - bare_functions = set([ - 'CURRENT_SCHEMA', 'DATE', 'FALSE', 'SYSDBA', 'TIME', 'TIMESTAMP', - 'TIMEZONE', 'TRANSACTION', 'TRUE', 'USER', 'UID', 'USERGROUP', - 'UTCDATE', 'UTCDIFF']) - - def visit_mod(self, binary, **kw): - return "mod(%s, %s)" % \ - (self.process(binary.left), self.process(binary.right)) - - def default_from(self): - return ' FROM DUAL' - - def for_update_clause(self, select): - clause = select.for_update - if clause is True: - return " WITH LOCK EXCLUSIVE" - elif clause is None: - return "" - elif clause == "read": - return " WITH LOCK" - elif clause == "ignore": - return " WITH LOCK (IGNORE) EXCLUSIVE" - elif clause == "nowait": - return " WITH LOCK (NOWAIT) EXCLUSIVE" - elif isinstance(clause, basestring): - return " WITH LOCK %s" % clause.upper() - elif not clause: - return "" - else: - return " WITH LOCK EXCLUSIVE" - - def function_argspec(self, fn, **kw): - if fn.name.upper() in self.bare_functions: - return "" - elif len(fn.clauses) > 0: - return compiler.SQLCompiler.function_argspec(self, fn, **kw) - else: - return "" - - def visit_function(self, fn, **kw): - transform = self.function_conversion.get(fn.name.upper(), None) - if transform: - fn = fn._clone() - fn.name = transform - return super(MaxDBCompiler, self).visit_function(fn, **kw) - - def visit_cast(self, cast, **kwargs): - # MaxDB only supports casts * to NUMERIC, * to VARCHAR or - # date/time to VARCHAR. Casts of LONGs will fail. - if isinstance(cast.type, (sqltypes.Integer, sqltypes.Numeric)): - return "NUM(%s)" % self.process(cast.clause) - elif isinstance(cast.type, sqltypes.String): - return "CHR(%s)" % self.process(cast.clause) - else: - return self.process(cast.clause) - - def visit_sequence(self, sequence): - if sequence.optional: - return None - else: - return ( - self.dialect.identifier_preparer.format_sequence(sequence) + - ".NEXTVAL") - - class ColumnSnagger(visitors.ClauseVisitor): - def __init__(self): - self.count = 0 - self.column = None - def visit_column(self, column): - self.column = column - self.count += 1 - - def _find_labeled_columns(self, columns, use_labels=False): - labels = {} - for column in columns: - if isinstance(column, basestring): - continue - snagger = self.ColumnSnagger() - snagger.traverse(column) - if snagger.count == 1: - if isinstance(column, sql_expr._Label): - labels[unicode(snagger.column)] = column.name - elif use_labels: - labels[unicode(snagger.column)] = column._label - - return labels - - def order_by_clause(self, select, **kw): - order_by = self.process(select._order_by_clause, **kw) - - # ORDER BY clauses in DISTINCT queries must reference aliased - # inner columns by alias name, not true column name. - if order_by and getattr(select, '_distinct', False): - labels = self._find_labeled_columns(select.inner_columns, - select.use_labels) - if labels: - for needs_alias in labels.keys(): - r = re.compile(r'(^| )(%s)(,| |$)' % - re.escape(needs_alias)) - order_by = r.sub((r'\1%s\3' % labels[needs_alias]), - order_by) - - # No ORDER BY in subqueries. - if order_by: - if self.is_subquery(): - # It's safe to simply drop the ORDER BY if there is no - # LIMIT. Right? Other dialects seem to get away with - # dropping order. - if select._limit: - raise exc.CompileError( - "MaxDB does not support ORDER BY in subqueries") - else: - return "" - return " ORDER BY " + order_by - else: - return "" - - def get_select_precolumns(self, select): - # Convert a subquery's LIMIT to TOP - sql = select._distinct and 'DISTINCT ' or '' - if self.is_subquery() and select._limit: - if select._offset: - raise exc.InvalidRequestError( - 'MaxDB does not support LIMIT with an offset.') - sql += 'TOP %s ' % select._limit - return sql - - def limit_clause(self, select): - # The docs say offsets are supported with LIMIT. But they're not. - # TODO: maybe emulate by adding a ROWNO/ROWNUM predicate? - # TODO: does MaxDB support bind params for LIMIT / TOP ? - if self.is_subquery(): - # sub queries need TOP - return '' - elif select._offset: - raise exc.InvalidRequestError( - 'MaxDB does not support LIMIT with an offset.') - else: - return ' \n LIMIT %s' % (select._limit,) - - def visit_insert(self, insert): - self.isinsert = True - self._safeserial = True - - colparams = self._get_colparams(insert) - for value in (insert.parameters or {}).itervalues(): - if isinstance(value, sql_expr.Function): - self._safeserial = False - break - - return ''.join(('INSERT INTO ', - self.preparer.format_table(insert.table), - ' (', - ', '.join([self.preparer.format_column(c[0]) - for c in colparams]), - ') VALUES (', - ', '.join([c[1] for c in colparams]), - ')')) - - -class MaxDBIdentifierPreparer(compiler.IdentifierPreparer): - reserved_words = set([ - 'abs', 'absolute', 'acos', 'adddate', 'addtime', 'all', 'alpha', - 'alter', 'any', 'ascii', 'asin', 'atan', 'atan2', 'avg', 'binary', - 'bit', 'boolean', 'byte', 'case', 'ceil', 'ceiling', 'char', - 'character', 'check', 'chr', 'column', 'concat', 'constraint', 'cos', - 'cosh', 'cot', 'count', 'cross', 'curdate', 'current', 'curtime', - 'database', 'date', 'datediff', 'day', 'dayname', 'dayofmonth', - 'dayofweek', 'dayofyear', 'dec', 'decimal', 'decode', 'default', - 'degrees', 'delete', 'digits', 'distinct', 'double', 'except', - 'exists', 'exp', 'expand', 'first', 'fixed', 'float', 'floor', 'for', - 'from', 'full', 'get_objectname', 'get_schema', 'graphic', 'greatest', - 'group', 'having', 'hex', 'hextoraw', 'hour', 'ifnull', 'ignore', - 'index', 'initcap', 'inner', 'insert', 'int', 'integer', 'internal', - 'intersect', 'into', 'join', 'key', 'last', 'lcase', 'least', 'left', - 'length', 'lfill', 'list', 'ln', 'locate', 'log', 'log10', 'long', - 'longfile', 'lower', 'lpad', 'ltrim', 'makedate', 'maketime', - 'mapchar', 'max', 'mbcs', 'microsecond', 'min', 'minute', 'mod', - 'month', 'monthname', 'natural', 'nchar', 'next', 'no', 'noround', - 'not', 'now', 'null', 'num', 'numeric', 'object', 'of', 'on', - 'order', 'packed', 'pi', 'power', 'prev', 'primary', 'radians', - 'real', 'reject', 'relative', 'replace', 'rfill', 'right', 'round', - 'rowid', 'rowno', 'rpad', 'rtrim', 'second', 'select', 'selupd', - 'serial', 'set', 'show', 'sign', 'sin', 'sinh', 'smallint', 'some', - 'soundex', 'space', 'sqrt', 'stamp', 'statistics', 'stddev', - 'subdate', 'substr', 'substring', 'subtime', 'sum', 'sysdba', - 'table', 'tan', 'tanh', 'time', 'timediff', 'timestamp', 'timezone', - 'to', 'toidentifier', 'transaction', 'translate', 'trim', 'trunc', - 'truncate', 'ucase', 'uid', 'unicode', 'union', 'update', 'upper', - 'user', 'usergroup', 'using', 'utcdate', 'utcdiff', 'value', 'values', - 'varchar', 'vargraphic', 'variance', 'week', 'weekofyear', 'when', - 'where', 'with', 'year', 'zoned' ]) - - def _normalize_name(self, name): - if name is None: - return None - if name.isupper(): - lc_name = name.lower() - if not self._requires_quotes(lc_name): - return lc_name - return name - - def _denormalize_name(self, name): - if name is None: - return None - elif (name.islower() and - not self._requires_quotes(name)): - return name.upper() - else: - return name - - def _maybe_quote_identifier(self, name): - if self._requires_quotes(name): - return self.quote_identifier(name) - else: - return name - - -class MaxDBDDLCompiler(compiler.DDLCompiler): - def get_column_specification(self, column, **kw): - colspec = [self.preparer.format_column(column), - self.dialect.type_compiler.process(column.type)] - - if not column.nullable: - colspec.append('NOT NULL') - - default = column.default - default_str = self.get_column_default_string(column) - - # No DDL default for columns specified with non-optional sequence- - # this defaulting behavior is entirely client-side. (And as a - # consequence, non-reflectable.) - if (default and isinstance(default, schema.Sequence) and - not default.optional): - pass - # Regular default - elif default_str is not None: - colspec.append('DEFAULT %s' % default_str) - # Assign DEFAULT SERIAL heuristically - elif column.primary_key and column.autoincrement: - # For SERIAL on a non-primary key member, use - # DefaultClause(text('SERIAL')) - try: - first = [c for c in column.table.primary_key.columns - if (c.autoincrement and - (isinstance(c.type, sqltypes.Integer) or - (isinstance(c.type, MaxNumeric) and - c.type.precision)) and - not c.foreign_keys)].pop(0) - if column is first: - colspec.append('DEFAULT SERIAL') - except IndexError: - pass - return ' '.join(colspec) - - def get_column_default_string(self, column): - if isinstance(column.server_default, schema.DefaultClause): - if isinstance(column.default.arg, basestring): - if isinstance(column.type, sqltypes.Integer): - return str(column.default.arg) - else: - return "'%s'" % column.default.arg - else: - return unicode(self._compile(column.default.arg, None)) - else: - return None - - def visit_create_sequence(self, create): - """Creates a SEQUENCE. - - TODO: move to module doc? - - start - With an integer value, set the START WITH option. - - increment - An integer value to increment by. Default is the database default. - - maxdb_minvalue - maxdb_maxvalue - With an integer value, sets the corresponding sequence option. - - maxdb_no_minvalue - maxdb_no_maxvalue - Defaults to False. If true, sets the corresponding sequence option. - - maxdb_cycle - Defaults to False. If true, sets the CYCLE option. - - maxdb_cache - With an integer value, sets the CACHE option. - - maxdb_no_cache - Defaults to False. If true, sets NOCACHE. - """ - sequence = create.element - - if (not sequence.optional and - (not self.checkfirst or - not self.dialect.has_sequence(self.connection, sequence.name))): - - ddl = ['CREATE SEQUENCE', - self.preparer.format_sequence(sequence)] - - sequence.increment = 1 - - if sequence.increment is not None: - ddl.extend(('INCREMENT BY', str(sequence.increment))) - - if sequence.start is not None: - ddl.extend(('START WITH', str(sequence.start))) - - opts = dict([(pair[0][6:].lower(), pair[1]) - for pair in sequence.kwargs.items() - if pair[0].startswith('maxdb_')]) - - if 'maxvalue' in opts: - ddl.extend(('MAXVALUE', str(opts['maxvalue']))) - elif opts.get('no_maxvalue', False): - ddl.append('NOMAXVALUE') - if 'minvalue' in opts: - ddl.extend(('MINVALUE', str(opts['minvalue']))) - elif opts.get('no_minvalue', False): - ddl.append('NOMINVALUE') - - if opts.get('cycle', False): - ddl.append('CYCLE') - - if 'cache' in opts: - ddl.extend(('CACHE', str(opts['cache']))) - elif opts.get('no_cache', False): - ddl.append('NOCACHE') - - return ' '.join(ddl) - - -class MaxDBDialect(default.DefaultDialect): - name = 'maxdb' - supports_alter = True - supports_unicode_statements = True - max_identifier_length = 32 - supports_sane_rowcount = True - supports_sane_multi_rowcount = False - - preparer = MaxDBIdentifierPreparer - statement_compiler = MaxDBCompiler - ddl_compiler = MaxDBDDLCompiler - execution_ctx_cls = MaxDBExecutionContext - - ported_sqla_06 = False - - colspecs = colspecs - ischema_names = ischema_names - - # MaxDB-specific - datetimeformat = 'internal' - - def __init__(self, _raise_known_sql_errors=False, **kw): - super(MaxDBDialect, self).__init__(**kw) - self._raise_known = _raise_known_sql_errors - - if self.dbapi is None: - self.dbapi_type_map = {} - else: - self.dbapi_type_map = { - 'Long Binary': MaxBlob(), - 'Long byte_t': MaxBlob(), - 'Long Unicode': MaxText(), - 'Timestamp': MaxTimestamp(), - 'Date': MaxDate(), - 'Time': MaxTime(), - datetime.datetime: MaxTimestamp(), - datetime.date: MaxDate(), - datetime.time: MaxTime(), - } - - def do_execute(self, cursor, statement, parameters, context=None): - res = cursor.execute(statement, parameters) - if isinstance(res, int) and context is not None: - context._rowcount = res - - def do_release_savepoint(self, connection, name): - # Does MaxDB truly support RELEASE SAVEPOINT ? All my attempts - # produce "SUBTRANS COMMIT/ROLLBACK not allowed without SUBTRANS - # BEGIN SQLSTATE: I7065" - # Note that ROLLBACK TO works fine. In theory, a RELEASE should - # just free up some transactional resources early, before the overall - # COMMIT/ROLLBACK so omitting it should be relatively ok. - pass - - def _get_default_schema_name(self, connection): - return self.identifier_preparer._normalize_name( - connection.execute( - 'SELECT CURRENT_SCHEMA FROM DUAL').scalar()) - - def has_table(self, connection, table_name, schema=None): - denormalize = self.identifier_preparer._denormalize_name - bind = [denormalize(table_name)] - if schema is None: - sql = ("SELECT tablename FROM TABLES " - "WHERE TABLES.TABLENAME=? AND" - " TABLES.SCHEMANAME=CURRENT_SCHEMA ") - else: - sql = ("SELECT tablename FROM TABLES " - "WHERE TABLES.TABLENAME = ? AND" - " TABLES.SCHEMANAME=? ") - bind.append(denormalize(schema)) - - rp = connection.execute(sql, bind) - return bool(rp.first()) - - @reflection.cache - def get_table_names(self, connection, schema=None, **kw): - if schema is None: - sql = (" SELECT TABLENAME FROM TABLES WHERE " - " SCHEMANAME=CURRENT_SCHEMA ") - rs = connection.execute(sql) - else: - sql = (" SELECT TABLENAME FROM TABLES WHERE " - " SCHEMANAME=? ") - matchname = self.identifier_preparer._denormalize_name(schema) - rs = connection.execute(sql, matchname) - normalize = self.identifier_preparer._normalize_name - return [normalize(row[0]) for row in rs] - - def reflecttable(self, connection, table, include_columns): - denormalize = self.identifier_preparer._denormalize_name - normalize = self.identifier_preparer._normalize_name - - st = ('SELECT COLUMNNAME, MODE, DATATYPE, CODETYPE, LEN, DEC, ' - ' NULLABLE, "DEFAULT", DEFAULTFUNCTION ' - 'FROM COLUMNS ' - 'WHERE TABLENAME=? AND SCHEMANAME=%s ' - 'ORDER BY POS') - - fk = ('SELECT COLUMNNAME, FKEYNAME, ' - ' REFSCHEMANAME, REFTABLENAME, REFCOLUMNNAME, RULE, ' - ' (CASE WHEN REFSCHEMANAME = CURRENT_SCHEMA ' - ' THEN 1 ELSE 0 END) AS in_schema ' - 'FROM FOREIGNKEYCOLUMNS ' - 'WHERE TABLENAME=? AND SCHEMANAME=%s ' - 'ORDER BY FKEYNAME ') - - params = [denormalize(table.name)] - if not table.schema: - st = st % 'CURRENT_SCHEMA' - fk = fk % 'CURRENT_SCHEMA' - else: - st = st % '?' - fk = fk % '?' - params.append(denormalize(table.schema)) - - rows = connection.execute(st, params).fetchall() - if not rows: - raise exc.NoSuchTableError(table.fullname) - - include_columns = set(include_columns or []) - - for row in rows: - (name, mode, col_type, encoding, length, scale, - nullable, constant_def, func_def) = row - - name = normalize(name) - - if include_columns and name not in include_columns: - continue - - type_args, type_kw = [], {} - if col_type == 'FIXED': - type_args = length, scale - # Convert FIXED(10) DEFAULT SERIAL to our Integer - if (scale == 0 and - func_def is not None and func_def.startswith('SERIAL')): - col_type = 'INTEGER' - type_args = length, - elif col_type in 'FLOAT': - type_args = length, - elif col_type in ('CHAR', 'VARCHAR'): - type_args = length, - type_kw['encoding'] = encoding - elif col_type == 'LONG': - type_kw['encoding'] = encoding - - try: - type_cls = ischema_names[col_type.lower()] - type_instance = type_cls(*type_args, **type_kw) - except KeyError: - util.warn("Did not recognize type '%s' of column '%s'" % - (col_type, name)) - type_instance = sqltypes.NullType - - col_kw = {'autoincrement': False} - col_kw['nullable'] = (nullable == 'YES') - col_kw['primary_key'] = (mode == 'KEY') - - if func_def is not None: - if func_def.startswith('SERIAL'): - if col_kw['primary_key']: - # No special default- let the standard autoincrement - # support handle SERIAL pk columns. - col_kw['autoincrement'] = True - else: - # strip current numbering - col_kw['server_default'] = schema.DefaultClause( - sql.text('SERIAL')) - col_kw['autoincrement'] = True - else: - col_kw['server_default'] = schema.DefaultClause( - sql.text(func_def)) - elif constant_def is not None: - col_kw['server_default'] = schema.DefaultClause(sql.text( - "'%s'" % constant_def.replace("'", "''"))) - - table.append_column(schema.Column(name, type_instance, **col_kw)) - - fk_sets = itertools.groupby(connection.execute(fk, params), - lambda row: row.FKEYNAME) - for fkeyname, fkey in fk_sets: - fkey = list(fkey) - if include_columns: - key_cols = set([r.COLUMNNAME for r in fkey]) - if key_cols != include_columns: - continue - - columns, referants = [], [] - quote = self.identifier_preparer._maybe_quote_identifier - - for row in fkey: - columns.append(normalize(row.COLUMNNAME)) - if table.schema or not row.in_schema: - referants.append('.'.join( - [quote(normalize(row[c])) - for c in ('REFSCHEMANAME', 'REFTABLENAME', - 'REFCOLUMNNAME')])) - else: - referants.append('.'.join( - [quote(normalize(row[c])) - for c in ('REFTABLENAME', 'REFCOLUMNNAME')])) - - constraint_kw = {'name': fkeyname.lower()} - if fkey[0].RULE is not None: - rule = fkey[0].RULE - if rule.startswith('DELETE '): - rule = rule[7:] - constraint_kw['ondelete'] = rule - - table_kw = {} - if table.schema or not row.in_schema: - table_kw['schema'] = normalize(fkey[0].REFSCHEMANAME) - - ref_key = schema._get_table_key(normalize(fkey[0].REFTABLENAME), - table_kw.get('schema')) - if ref_key not in table.metadata.tables: - schema.Table(normalize(fkey[0].REFTABLENAME), - table.metadata, - autoload=True, autoload_with=connection, - **table_kw) - - constraint = schema.ForeignKeyConstraint( - columns, referants, link_to_name=True, - **constraint_kw) - table.append_constraint(constraint) - - def has_sequence(self, connection, name): - # [ticket:726] makes this schema-aware. - denormalize = self.identifier_preparer._denormalize_name - sql = ("SELECT sequence_name FROM SEQUENCES " - "WHERE SEQUENCE_NAME=? ") - - rp = connection.execute(sql, denormalize(name)) - return bool(rp.first()) - - -def _autoserial_column(table): - """Finds the effective DEFAULT SERIAL column of a Table, if any.""" - - for index, col in enumerate(table.primary_key.columns): - if (isinstance(col.type, (sqltypes.Integer, sqltypes.Numeric)) and - col.autoincrement): - if isinstance(col.default, schema.Sequence): - if col.default.optional: - return index, col - elif (col.default is None or - (not isinstance(col.server_default, schema.DefaultClause))): - return index, col - - return None, None - diff --git a/libs/sqlalchemy/dialects/maxdb/sapdb.py b/libs/sqlalchemy/dialects/maxdb/sapdb.py deleted file mode 100644 index 51f272a2..00000000 --- a/libs/sqlalchemy/dialects/maxdb/sapdb.py +++ /dev/null @@ -1,23 +0,0 @@ -# maxdb/sapdb.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -from sqlalchemy.dialects.maxdb.base import MaxDBDialect - -class MaxDBDialect_sapdb(MaxDBDialect): - driver = 'sapdb' - - @classmethod - def dbapi(cls): - from sapdb import dbapi as _dbapi - return _dbapi - - def create_connect_args(self, url): - opts = url.translate_connect_args(username='user') - opts.update(url.query) - return [], opts - - -dialect = MaxDBDialect_sapdb \ No newline at end of file diff --git a/libs/sqlalchemy/dialects/mssql/__init__.py b/libs/sqlalchemy/dialects/mssql/__init__.py index b3acbf3a..7a2dfa60 100644 --- a/libs/sqlalchemy/dialects/mssql/__init__.py +++ b/libs/sqlalchemy/dialects/mssql/__init__.py @@ -1,5 +1,5 @@ # mssql/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -23,4 +23,4 @@ __all__ = ( 'DATETIME2', 'DATETIMEOFFSET', 'DATE', 'TIME', 'SMALLDATETIME', 'BINARY', 'VARBINARY', 'BIT', 'REAL', 'IMAGE', 'TIMESTAMP', 'MONEY', 'SMALLMONEY', 'UNIQUEIDENTIFIER', 'SQL_VARIANT', 'dialect' -) \ No newline at end of file +) diff --git a/libs/sqlalchemy/dialects/mssql/adodbapi.py b/libs/sqlalchemy/dialects/mssql/adodbapi.py index 05ac6d6f..95cf4242 100644 --- a/libs/sqlalchemy/dialects/mssql/adodbapi.py +++ b/libs/sqlalchemy/dialects/mssql/adodbapi.py @@ -1,11 +1,20 @@ # mssql/adodbapi.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -The adodbapi dialect is not implemented for 0.6 at this time. +.. dialect:: mssql+adodbapi + :name: adodbapi + :dbapi: adodbapi + :connectstring: mssql+adodbapi://:@ + :url: http://adodbapi.sourceforge.net/ + +.. note:: + + The adodbapi dialect is not implemented SQLAlchemy versions 0.6 and + above at this time. """ import datetime @@ -13,6 +22,7 @@ from sqlalchemy import types as sqltypes, util from sqlalchemy.dialects.mssql.base import MSDateTime, MSDialect import sys + class MSDateTime_adodbapi(MSDateTime): def result_processor(self, dialect, coltype): def process(value): @@ -40,7 +50,7 @@ class MSDialect_adodbapi(MSDialect): colspecs = util.update_copy( MSDialect.colspecs, { - sqltypes.DateTime:MSDateTime_adodbapi + sqltypes.DateTime: MSDateTime_adodbapi } ) @@ -49,18 +59,18 @@ class MSDialect_adodbapi(MSDialect): connectors = ["Provider=SQLOLEDB"] if 'port' in keys: - connectors.append ("Data Source=%s, %s" % + connectors.append("Data Source=%s, %s" % (keys.get("host"), keys.get("port"))) else: - connectors.append ("Data Source=%s" % keys.get("host")) - connectors.append ("Initial Catalog=%s" % keys.get("database")) + connectors.append("Data Source=%s" % keys.get("host")) + connectors.append("Initial Catalog=%s" % keys.get("database")) user = keys.get("user") if user: connectors.append("User Id=%s" % user) connectors.append("Password=%s" % keys.get("password", "")) else: connectors.append("Integrated Security=SSPI") - return [[";".join (connectors)], {}] + return [[";".join(connectors)], {}] def is_disconnect(self, e, connection, cursor): return isinstance(e, self.dbapi.adodbapi.DatabaseError) and \ diff --git a/libs/sqlalchemy/dialects/mssql/base.py b/libs/sqlalchemy/dialects/mssql/base.py index b6e0d881..90fd1f38 100644 --- a/libs/sqlalchemy/dialects/mssql/base.py +++ b/libs/sqlalchemy/dialects/mssql/base.py @@ -1,15 +1,13 @@ # mssql/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the Microsoft SQL Server database. +""" +.. dialect:: mssql + :name: Microsoft SQL Server -Connecting ----------- - -See the individual driver sections below for details on connecting. Auto Increment Behavior ----------------------- @@ -47,13 +45,10 @@ does in other dialects and results in an ``IDENTITY`` column. Collation Support ----------------- -MSSQL specific string types support a collation parameter that -creates a column-level specific collation for the column. The -collation parameter accepts a Windows Collation Name or a SQL -Collation Name. Supported types are MSChar, MSNChar, MSString, -MSNVarchar, MSText, and MSNText. For example:: +Character collations are supported by the base string types, +specified by the string argument "collation":: - from sqlalchemy.dialects.mssql import VARCHAR + from sqlalchemy import VARCHAR Column('login', VARCHAR(32, collation='Latin1_General_CI_AS')) When such a column is associated with a :class:`.Table`, the @@ -61,6 +56,9 @@ CREATE TABLE statement for this column will yield:: login VARCHAR(32) COLLATE Latin1_General_CI_AS NULL +.. versionadded:: 0.8 Character collations are now part of the base string + types. + LIMIT/OFFSET Support -------------------- @@ -103,6 +101,50 @@ The DATE and TIME types are not available for MSSQL 2005 and previous - if a server version below 2008 is detected, DDL for these types will be issued as DATETIME. +.. _mssql_indexes: + +MSSQL-Specific Index Options +----------------------------- + +The MSSQL dialect supports special options for :class:`.Index`. + +CLUSTERED +^^^^^^^^^^ + +The ``mssql_clustered`` option adds the CLUSTERED keyword to the index:: + + Index("my_index", table.c.x, mssql_clustered=True) + +would render the index as ``CREATE CLUSTERED INDEX my_index ON table (x)`` + +.. versionadded:: 0.8 + +INCLUDE +^^^^^^^ + +The ``mssql_include`` option renders INCLUDE(colname) for the given string names:: + + Index("my_index", table.c.x, mssql_include=['y']) + +would render the index as ``CREATE INDEX my_index ON table (x) INCLUDE (y)`` + +.. versionadded:: 0.8 + +Index ordering +^^^^^^^^^^^^^^ + +Index ordering is available via functional expressions, such as:: + + Index("my_index", table.c.x.desc()) + +would render the index as ``CREATE INDEX my_index ON table (x DESC)`` + +.. versionadded:: 0.8 + +.. seealso:: + + :ref:`schema_indexes_functional` + Compatibility Levels -------------------- MSSQL supports the notion of setting compatibility levels at the @@ -158,30 +200,6 @@ following ALTER DATABASE commands executed at the SQL prompt:: Background on SQL Server snapshot isolation is available at http://msdn.microsoft.com/en-us/library/ms175095.aspx. -Scalar Select Comparisons -------------------------- - -.. deprecated:: 0.8 - The MSSQL dialect contains a legacy behavior whereby comparing - a scalar select to a value using the ``=`` or ``!=`` operator - will resolve to IN or NOT IN, respectively. This behavior - will be removed in 0.8 - the ``s.in_()``/``~s.in_()`` operators - should be used when IN/NOT IN are desired. - -For the time being, the existing behavior prevents a comparison -between scalar select and another value that actually wants to use ``=``. -To remove this behavior in a forwards-compatible way, apply this -compilation rule by placing the following code at the module import -level:: - - from sqlalchemy.ext.compiler import compiles - from sqlalchemy.sql.expression import _BinaryExpression - from sqlalchemy.sql.compiler import SQLCompiler - - @compiles(_BinaryExpression, 'mssql') - def override_legacy_binary(element, compiler, **kw): - return SQLCompiler.visit_binary(compiler, element, **kw) - Known Issues ------------ @@ -190,20 +208,23 @@ Known Issues SQL Server 2005 """ -import datetime, operator, re +import datetime +import operator +import re -from sqlalchemy import sql, schema as sa_schema, exc, util -from sqlalchemy.sql import select, compiler, expression, \ - operators as sql_operators, \ +from ... import sql, schema as sa_schema, exc, util +from ...sql import compiler, expression, \ util as sql_util, cast -from sqlalchemy.engine import default, base, reflection -from sqlalchemy import types as sqltypes -from sqlalchemy.types import INTEGER, BIGINT, SMALLINT, DECIMAL, NUMERIC, \ +from ... import engine +from ...engine import reflection, default +from ... import types as sqltypes +from ...types import INTEGER, BIGINT, SMALLINT, DECIMAL, NUMERIC, \ FLOAT, TIMESTAMP, DATETIME, DATE, BINARY,\ - VARBINARY, BLOB + VARBINARY, TEXT, VARCHAR, NVARCHAR, CHAR, NCHAR -from sqlalchemy.dialects.mssql import information_schema as ischema +from ...util import update_wrapper +from . import information_schema as ischema MS_2008_VERSION = (10,) MS_2005_VERSION = (9,) @@ -240,6 +261,7 @@ RESERVED_WORDS = set( 'writetext', ]) + class REAL(sqltypes.REAL): __visit_name__ = 'REAL' @@ -248,6 +270,7 @@ class REAL(sqltypes.REAL): kw['precision'] = 24 super(REAL, self).__init__(**kw) + class TINYINT(sqltypes.Integer): __visit_name__ = 'TINYINT' @@ -267,11 +290,12 @@ class _MSDate(sqltypes.Date): return process _reg = re.compile(r"(\d+)-(\d+)-(\d+)") + def result_processor(self, dialect, coltype): def process(value): if isinstance(value, datetime.datetime): return value.date() - elif isinstance(value, basestring): + elif isinstance(value, util.string_types): return datetime.date(*[ int(x or 0) for x in self._reg.match(value).groups() @@ -280,6 +304,7 @@ class _MSDate(sqltypes.Date): return value return process + class TIME(sqltypes.TIME): def __init__(self, precision=None, **kwargs): self.precision = precision @@ -298,17 +323,20 @@ class TIME(sqltypes.TIME): return process _reg = re.compile(r"(\d+):(\d+):(\d+)(?:\.(\d{0,6}))?") + def result_processor(self, dialect, coltype): def process(value): if isinstance(value, datetime.datetime): return value.time() - elif isinstance(value, basestring): + elif isinstance(value, util.string_types): return datetime.time(*[ int(x or 0) for x in self._reg.match(value).groups()]) else: return value return process +_MSTime = TIME + class _DateTimeBase(object): def bind_processor(self, dialect): @@ -319,12 +347,15 @@ class _DateTimeBase(object): return value return process + class _MSDateTime(_DateTimeBase, sqltypes.DateTime): pass + class SMALLDATETIME(_DateTimeBase, sqltypes.DateTime): __visit_name__ = 'SMALLDATETIME' + class DATETIME2(_DateTimeBase, sqltypes.DateTime): __visit_name__ = 'DATETIME2' @@ -340,135 +371,28 @@ class DATETIMEOFFSET(sqltypes.TypeEngine): def __init__(self, precision=None, **kwargs): self.precision = precision + class _StringType(object): """Base for MSSQL string types.""" def __init__(self, collation=None): - self.collation = collation + super(_StringType, self).__init__(collation=collation) -class TEXT(_StringType, sqltypes.TEXT): - """MSSQL TEXT type, for variable-length text up to 2^31 characters.""" - def __init__(self, length=None, collation=None, **kw): - """Construct a TEXT. - :param collation: Optional, a column-level collation for this string - value. Accepts a Windows Collation Name or a SQL Collation Name. - """ - _StringType.__init__(self, collation) - sqltypes.Text.__init__(self, length, **kw) - -class NTEXT(_StringType, sqltypes.UnicodeText): +class NTEXT(sqltypes.UnicodeText): """MSSQL NTEXT type, for variable-length unicode text up to 2^30 characters.""" __visit_name__ = 'NTEXT' - def __init__(self, length=None, collation=None, **kw): - """Construct a NTEXT. - :param collation: Optional, a column-level collation for this string - value. Accepts a Windows Collation Name or a SQL Collation Name. - - """ - _StringType.__init__(self, collation) - sqltypes.UnicodeText.__init__(self, length, **kw) - - -class VARCHAR(_StringType, sqltypes.VARCHAR): - """MSSQL VARCHAR type, for variable-length non-Unicode data with a maximum - of 8,000 characters.""" - - def __init__(self, length=None, collation=None, **kw): - """Construct a VARCHAR. - - :param length: Optinal, maximum data length, in characters. - - :param convert_unicode: defaults to False. If True, convert - ``unicode`` data sent to the database to a ``str`` - bytestring, and convert bytestrings coming back from the - database into ``unicode``. - - Bytestrings are encoded using the dialect's - :attr:`~sqlalchemy.engine.base.Dialect.encoding`, which - defaults to `utf-8`. - - If False, may be overridden by - :attr:`sqlalchemy.engine.base.Dialect.convert_unicode`. - - :param collation: Optional, a column-level collation for this string - value. Accepts a Windows Collation Name or a SQL Collation Name. - - """ - _StringType.__init__(self, collation) - sqltypes.VARCHAR.__init__(self, length, **kw) - -class NVARCHAR(_StringType, sqltypes.NVARCHAR): - """MSSQL NVARCHAR type. - - For variable-length unicode character data up to 4,000 characters.""" - - def __init__(self, length=None, collation=None, **kw): - """Construct a NVARCHAR. - - :param length: Optional, Maximum data length, in characters. - - :param collation: Optional, a column-level collation for this string - value. Accepts a Windows Collation Name or a SQL Collation Name. - - """ - _StringType.__init__(self, collation) - sqltypes.NVARCHAR.__init__(self, length, **kw) - -class CHAR(_StringType, sqltypes.CHAR): - """MSSQL CHAR type, for fixed-length non-Unicode data with a maximum - of 8,000 characters.""" - - def __init__(self, length=None, collation=None, **kw): - """Construct a CHAR. - - :param length: Optinal, maximum data length, in characters. - - :param convert_unicode: defaults to False. If True, convert - ``unicode`` data sent to the database to a ``str`` - bytestring, and convert bytestrings coming back from the - database into ``unicode``. - - Bytestrings are encoded using the dialect's - :attr:`~sqlalchemy.engine.base.Dialect.encoding`, which - defaults to `utf-8`. - - If False, may be overridden by - :attr:`sqlalchemy.engine.base.Dialect.convert_unicode`. - - :param collation: Optional, a column-level collation for this string - value. Accepts a Windows Collation Name or a SQL Collation Name. - - """ - _StringType.__init__(self, collation) - sqltypes.CHAR.__init__(self, length, **kw) - -class NCHAR(_StringType, sqltypes.NCHAR): - """MSSQL NCHAR type. - - For fixed-length unicode character data up to 4,000 characters.""" - - def __init__(self, length=None, collation=None, **kw): - """Construct an NCHAR. - - :param length: Optional, Maximum data length, in characters. - - :param collation: Optional, a column-level collation for this string - value. Accepts a Windows Collation Name or a SQL Collation Name. - - """ - _StringType.__init__(self, collation) - sqltypes.NCHAR.__init__(self, length, **kw) class IMAGE(sqltypes.LargeBinary): __visit_name__ = 'IMAGE' + class BIT(sqltypes.TypeEngine): __visit_name__ = 'BIT' @@ -476,12 +400,15 @@ class BIT(sqltypes.TypeEngine): class MONEY(sqltypes.TypeEngine): __visit_name__ = 'MONEY' + class SMALLMONEY(sqltypes.TypeEngine): __visit_name__ = 'SMALLMONEY' + class UNIQUEIDENTIFIER(sqltypes.TypeEngine): __visit_name__ = "UNIQUEIDENTIFIER" + class SQL_VARIANT(sqltypes.TypeEngine): __visit_name__ = 'SQL_VARIANT' @@ -510,30 +437,30 @@ MSUniqueIdentifier = UNIQUEIDENTIFIER MSVariant = SQL_VARIANT ischema_names = { - 'int' : INTEGER, + 'int': INTEGER, 'bigint': BIGINT, - 'smallint' : SMALLINT, - 'tinyint' : TINYINT, - 'varchar' : VARCHAR, - 'nvarchar' : NVARCHAR, - 'char' : CHAR, - 'nchar' : NCHAR, - 'text' : TEXT, - 'ntext' : NTEXT, - 'decimal' : DECIMAL, - 'numeric' : NUMERIC, - 'float' : FLOAT, - 'datetime' : DATETIME, - 'datetime2' : DATETIME2, - 'datetimeoffset' : DATETIMEOFFSET, + 'smallint': SMALLINT, + 'tinyint': TINYINT, + 'varchar': VARCHAR, + 'nvarchar': NVARCHAR, + 'char': CHAR, + 'nchar': NCHAR, + 'text': TEXT, + 'ntext': NTEXT, + 'decimal': DECIMAL, + 'numeric': NUMERIC, + 'float': FLOAT, + 'datetime': DATETIME, + 'datetime2': DATETIME2, + 'datetimeoffset': DATETIMEOFFSET, 'date': DATE, 'time': TIME, - 'smalldatetime' : SMALLDATETIME, - 'binary' : BINARY, - 'varbinary' : VARBINARY, + 'smalldatetime': SMALLDATETIME, + 'binary': BINARY, + 'varbinary': VARBINARY, 'bit': BIT, - 'real' : REAL, - 'image' : IMAGE, + 'real': REAL, + 'image': IMAGE, 'timestamp': TIMESTAMP, 'money': MONEY, 'smallmoney': SMALLMONEY, @@ -609,8 +536,7 @@ class MSTypeCompiler(compiler.GenericTypeCompiler): return self._extend("TEXT", type_) def visit_VARCHAR(self, type_): - return self._extend("VARCHAR", type_, - length = type_.length or 'max') + return self._extend("VARCHAR", type_, length=type_.length or 'max') def visit_CHAR(self, type_): return self._extend("CHAR", type_) @@ -619,8 +545,7 @@ class MSTypeCompiler(compiler.GenericTypeCompiler): return self._extend("NCHAR", type_) def visit_NVARCHAR(self, type_): - return self._extend("NVARCHAR", type_, - length = type_.length or 'max') + return self._extend("NVARCHAR", type_, length=type_.length or 'max') def visit_date(self, type_): if self.dialect.server_version_info < MS_2008_VERSION: @@ -664,6 +589,7 @@ class MSTypeCompiler(compiler.GenericTypeCompiler): def visit_SQL_VARIANT(self, type_): return 'SQL_VARIANT' + class MSExecutionContext(default.DefaultExecutionContext): _enable_identity_insert = False _select_lastrowid = False @@ -693,7 +619,7 @@ class MSExecutionContext(default.DefaultExecutionContext): self.root_connection._cursor_execute(self.cursor, "SET IDENTITY_INSERT %s ON" % self.dialect.identifier_preparer.format_table(tbl), - ()) + (), self) def post_exec(self): """Disable IDENTITY_INSERT if enabled.""" @@ -702,25 +628,24 @@ class MSExecutionContext(default.DefaultExecutionContext): if self._select_lastrowid: if self.dialect.use_scope_identity: conn._cursor_execute(self.cursor, - "SELECT scope_identity() AS lastrowid", ()) + "SELECT scope_identity() AS lastrowid", (), self) else: conn._cursor_execute(self.cursor, - "SELECT @@identity AS lastrowid", ()) + "SELECT @@identity AS lastrowid", (), self) # fetchall() ensures the cursor is consumed without closing it row = self.cursor.fetchall()[0] self._lastrowid = int(row[0]) if (self.isinsert or self.isupdate or self.isdelete) and \ self.compiled.returning: - self._result_proxy = base.FullyBufferedResultProxy(self) + self._result_proxy = engine.FullyBufferedResultProxy(self) if self._enable_identity_insert: conn._cursor_execute(self.cursor, "SET IDENTITY_INSERT %s OFF" % self.dialect.identifier_preparer. format_table(self.compiled.statement.table), - () - ) + (), self) def get_lastrowid(self): return self._lastrowid @@ -740,7 +665,8 @@ class MSExecutionContext(default.DefaultExecutionContext): if self._result_proxy: return self._result_proxy else: - return base.ResultProxy(self) + return engine.ResultProxy(self) + class MSSQLCompiler(compiler.SQLCompiler): returning_precedes_values = True @@ -770,12 +696,18 @@ class MSSQLCompiler(compiler.SQLCompiler): def visit_char_length_func(self, fn, **kw): return "LEN%s" % self.function_argspec(fn, **kw) - def visit_concat_op(self, binary, **kw): + def visit_concat_op_binary(self, binary, operator, **kw): return "%s + %s" % \ (self.process(binary.left, **kw), self.process(binary.right, **kw)) - def visit_match_op(self, binary, **kw): + def visit_true(self, expr, **kw): + return '1' + + def visit_false(self, expr, **kw): + return '0' + + def visit_match_op_binary(self, binary, operator, **kw): return "CONTAINS (%s, %s)" % ( self.process(binary.left, **kw), self.process(binary.right, **kw)) @@ -843,8 +775,8 @@ class MSSQLCompiler(compiler.SQLCompiler): else: return None - def visit_table(self, table, mssql_aliased=False, **kwargs): - if mssql_aliased is table: + def visit_table(self, table, mssql_aliased=False, iscrud=False, **kwargs): + if mssql_aliased is table or iscrud: return super(MSSQLCompiler, self).visit_table(table, **kwargs) # alias schema-qualified tables @@ -871,7 +803,7 @@ class MSSQLCompiler(compiler.SQLCompiler): return ("ROLLBACK TRANSACTION %s" % self.preparer.format_savepoint(savepoint_stmt)) - def visit_column(self, column, result_map=None, **kwargs): + def visit_column(self, column, add_to_result_map=None, **kwargs): if column.table is not None and \ (not self.isupdate and not self.isdelete) or self.is_subquery(): # translate for schema-qualified table aliases @@ -879,20 +811,19 @@ class MSSQLCompiler(compiler.SQLCompiler): if t is not None: converted = expression._corresponding_column_or_error( t, column) - - if result_map is not None: - result_map[column.name.lower()] = \ - (column.name, (column, column.name, - column.key), - column.type) + if add_to_result_map is not None: + add_to_result_map( + column.name, + column.name, + (column, column.name, column.key), + column.type + ) return super(MSSQLCompiler, self).\ - visit_column(converted, - result_map=None, **kwargs) + visit_column(converted, **kwargs) - return super(MSSQLCompiler, self).visit_column(column, - result_map=result_map, - **kwargs) + return super(MSSQLCompiler, self).visit_column( + column, add_to_result_map=add_to_result_map, **kwargs) def visit_binary(self, binary, **kwargs): """Move bind parameters to the right-hand side of an operator, where @@ -900,40 +831,16 @@ class MSSQLCompiler(compiler.SQLCompiler): """ if ( - isinstance(binary.left, expression._BindParamClause) + isinstance(binary.left, expression.BindParameter) and binary.operator == operator.eq - and not isinstance(binary.right, expression._BindParamClause) + and not isinstance(binary.right, expression.BindParameter) ): return self.process( - expression._BinaryExpression(binary.right, + expression.BinaryExpression(binary.right, binary.left, binary.operator), **kwargs) - else: - if ( - (binary.operator is operator.eq or - binary.operator is operator.ne) - and ( - (isinstance(binary.left, expression._FromGrouping) - and isinstance(binary.left.element, - expression._ScalarSelect)) - or (isinstance(binary.right, expression._FromGrouping) - and isinstance(binary.right.element, - expression._ScalarSelect)) - or isinstance(binary.left, expression._ScalarSelect) - or isinstance(binary.right, expression._ScalarSelect) - ) - ): - op = binary.operator == operator.eq and "IN" or "NOT IN" - util.warn_deprecated("Comparing a scalar select using ``=``/``!=`` will " - "no longer produce IN/NOT IN in 0.8. To remove this " - "behavior immediately, use the recipe at " - "http://www.sqlalchemy.org/docs/07/dialects/mssql.html#scalar-select-comparisons") - return self.process( - expression._BinaryExpression(binary.left, - binary.right, op), - **kwargs) - return super(MSSQLCompiler, self).visit_binary(binary, **kwargs) + return super(MSSQLCompiler, self).visit_binary(binary, **kwargs) def returning_clause(self, stmt, returning_cols): @@ -943,21 +850,13 @@ class MSSQLCompiler(compiler.SQLCompiler): target = stmt.table.alias("deleted") adapter = sql_util.ClauseAdapter(target) - def col_label(col): - adapted = adapter.traverse(col) - if isinstance(col, expression._Label): - return adapted.label(c.key) - else: - return self.label_select_column(None, adapted, asfrom=False) columns = [ - self.process( - col_label(c), - within_columns_clause=True, - result_map=self.result_map - ) - for c in expression._select_iterables(returning_cols) - ] + self._label_select_column(None, adapter.traverse(c), + True, False, {}) + for c in expression._select_iterables(returning_cols) + ] + return 'OUTPUT ' + ', '.join(columns) def get_cte_preamble(self, recursive): @@ -1004,6 +903,7 @@ class MSSQLCompiler(compiler.SQLCompiler): fromhints=from_hints, **kw) for t in [from_table] + extra_froms) + class MSSQLStrictCompiler(MSSQLCompiler): """A subclass of MSSQLCompiler which disables the usage of bind parameters where not allowed natively by MS-SQL. @@ -1014,24 +914,20 @@ class MSSQLStrictCompiler(MSSQLCompiler): """ ansi_bind_rules = True - def visit_in_op(self, binary, **kw): + def visit_in_op_binary(self, binary, operator, **kw): kw['literal_binds'] = True return "%s IN %s" % ( self.process(binary.left, **kw), self.process(binary.right, **kw) ) - def visit_notin_op(self, binary, **kw): + def visit_notin_op_binary(self, binary, operator, **kw): kw['literal_binds'] = True return "%s NOT IN %s" % ( self.process(binary.left, **kw), self.process(binary.right, **kw) ) - def visit_function(self, func, **kw): - kw['literal_binds'] = True - return super(MSSQLStrictCompiler, self).visit_function(func, **kw) - def render_literal_value(self, value, type_): """ For date and datetime values, convert to a string @@ -1051,13 +947,15 @@ class MSSQLStrictCompiler(MSSQLCompiler): return super(MSSQLStrictCompiler, self).\ render_literal_value(value, type_) + class MSDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kwargs): colspec = (self.preparer.format_column(column) + " " + self.dialect.type_compiler.process(column.type)) if column.nullable is not None: - if not column.nullable or column.primary_key: + if not column.nullable or column.primary_key or \ + isinstance(column.default, sa_schema.Sequence): colspec += " NOT NULL" else: colspec += " NULL" @@ -1067,18 +965,16 @@ class MSDDLCompiler(compiler.DDLCompiler): "mssql requires Table-bound columns " "in order to generate DDL") - seq_col = column.table._autoincrement_column - - # install a IDENTITY Sequence if we have an implicit IDENTITY column - if seq_col is column: - sequence = isinstance(column.default, sa_schema.Sequence) and \ - column.default - if sequence: - start, increment = sequence.start or 1, \ - sequence.increment or 1 + # install an IDENTITY Sequence if we either a sequence or an implicit IDENTITY column + if isinstance(column.default, sa_schema.Sequence): + if column.default.start == 0: + start = 0 else: - start, increment = 1, 1 - colspec += " IDENTITY(%s,%s)" % (start, increment) + start = column.default.start or 1 + + colspec += " IDENTITY(%s,%s)" % (start, column.default.increment or 1) + elif column is column.table._autoincrement_column: + colspec += " IDENTITY(1,1)" else: default = self.get_column_default_string(column) if default is not None: @@ -1086,14 +982,46 @@ class MSDDLCompiler(compiler.DDLCompiler): return colspec - def visit_drop_index(self, drop): - return "\nDROP INDEX %s.%s" % ( - self.preparer.quote_identifier(drop.element.table.name), - self.preparer.quote( - self._index_identifier(drop.element.name), - drop.element.quote) - ) + def visit_create_index(self, create, include_schema=False): + index = create.element + self._verify_index_table(index) + preparer = self.preparer + text = "CREATE " + if index.unique: + text += "UNIQUE " + # handle clustering option + if index.kwargs.get("mssql_clustered"): + text += "CLUSTERED " + + text += "INDEX %s ON %s (%s)" \ + % ( + self._prepared_index_name(index, + include_schema=include_schema), + preparer.format_table(index.table), + ', '.join( + self.sql_compiler.process(expr, + include_table=False, literal_binds=True) for + expr in index.expressions) + ) + + # handle other included columns + if index.kwargs.get("mssql_include"): + inclusions = [index.table.c[col] + if isinstance(col, util.string_types) else col + for col in index.kwargs["mssql_include"]] + + text += " INCLUDE (%s)" \ + % ', '.join([preparer.quote(c.name) + for c in inclusions]) + + return text + + def visit_drop_index(self, drop): + return "\nDROP INDEX %s ON %s" % ( + self._prepared_index_name(drop.element, include_schema=False), + self.preparer.format_table(drop.element.table) + ) class MSIdentifierPreparer(compiler.IdentifierPreparer): reserved_words = RESERVED_WORDS @@ -1105,11 +1033,48 @@ class MSIdentifierPreparer(compiler.IdentifierPreparer): def _escape_identifier(self, value): return value - def quote_schema(self, schema, force=True): + def quote_schema(self, schema, force=None): """Prepare a quoted table and schema name.""" result = '.'.join([self.quote(x, force) for x in schema.split('.')]) return result + +def _db_plus_owner_listing(fn): + def wrap(dialect, connection, schema=None, **kw): + dbname, owner = _owner_plus_db(dialect, schema) + return _switch_db(dbname, connection, fn, dialect, connection, + dbname, owner, schema, **kw) + return update_wrapper(wrap, fn) + + +def _db_plus_owner(fn): + def wrap(dialect, connection, tablename, schema=None, **kw): + dbname, owner = _owner_plus_db(dialect, schema) + return _switch_db(dbname, connection, fn, dialect, connection, + tablename, dbname, owner, schema, **kw) + return update_wrapper(wrap, fn) + + +def _switch_db(dbname, connection, fn, *arg, **kw): + if dbname: + current_db = connection.scalar("select db_name()") + connection.execute("use %s" % dbname) + try: + return fn(*arg, **kw) + finally: + if dbname: + connection.execute("use %s" % current_db) + + +def _owner_plus_db(dialect, schema): + if not schema: + return None, dialect.default_schema_name + elif "." in schema: + return schema.split(".", 1) + else: + return None, schema + + class MSDialect(default.DefaultDialect): name = 'mssql' supports_default_values = True @@ -1120,9 +1085,9 @@ class MSDialect(default.DefaultDialect): schema_name = "dbo" colspecs = { - sqltypes.DateTime : _MSDateTime, - sqltypes.Date : _MSDate, - sqltypes.Time : TIME, + sqltypes.DateTime: _MSDateTime, + sqltypes.Date: _MSDate, + sqltypes.Time: TIME, } ischema_names = ischema_names @@ -1142,7 +1107,7 @@ class MSDialect(default.DefaultDialect): query_timeout=None, use_scope_identity=True, max_identifier_length=None, - schema_name=u"dbo", **opts): + schema_name="dbo", **opts): self.query_timeout = int(query_timeout or 0) self.schema_name = schema_name @@ -1162,7 +1127,7 @@ class MSDialect(default.DefaultDialect): def initialize(self, connection): super(MSDialect, self).initialize(connection) - if self.server_version_info[0] not in range(8, 17): + if self.server_version_info[0] not in list(range(8, 17)): # FreeTDS with version 4.2 seems to report here # a number like "95.10.255". Don't know what # that is. So emit warning. @@ -1171,13 +1136,13 @@ class MSDialect(default.DefaultDialect): "behaviors may not function properly. If using ODBC " "with FreeTDS, ensure server version 7.0 or 8.0, not 4.2, " "is configured in the FreeTDS configuration." % - ".".join(str(x) for x in self.server_version_info) ) + ".".join(str(x) for x in self.server_version_info)) if self.server_version_info >= MS_2005_VERSION and \ 'implicit_returning' not in self.__dict__: self.implicit_returning = True def _get_default_schema_name(self, connection): - user_name = connection.scalar("SELECT user_name() as user_name;") + user_name = connection.scalar("SELECT user_name()") if user_name is not None: # now, get the default schema query = sql.text(""" @@ -1189,25 +1154,20 @@ class MSDialect(default.DefaultDialect): try: default_schema_name = connection.scalar(query, name=user_name) if default_schema_name is not None: - return unicode(default_schema_name) + return util.text_type(default_schema_name) except: pass return self.schema_name - def _unicode_cast(self, column): - if self.server_version_info >= MS_2005_VERSION: - return cast(column, NVARCHAR(_warn_on_bytestring=False)) - else: - return column - - def has_table(self, connection, tablename, schema=None): - current_schema = schema or self.default_schema_name + @_db_plus_owner + def has_table(self, connection, tablename, dbname, owner, schema): columns = ischema.columns - whereclause = self._unicode_cast(columns.c.table_name)==tablename - if current_schema: + whereclause = columns.c.table_name == tablename + + if owner: whereclause = sql.and_(whereclause, - columns.c.table_schema==current_schema) + columns.c.table_schema == owner) s = sql.select([columns], whereclause) c = connection.execute(s) return c.first() is not None @@ -1221,13 +1181,13 @@ class MSDialect(default.DefaultDialect): return schema_names @reflection.cache - def get_table_names(self, connection, schema=None, **kw): - current_schema = schema or self.default_schema_name + @_db_plus_owner_listing + def get_table_names(self, connection, dbname, owner, schema, **kw): tables = ischema.tables s = sql.select([tables.c.table_name], sql.and_( - tables.c.table_schema == current_schema, - tables.c.table_type == u'BASE TABLE' + tables.c.table_schema == owner, + tables.c.table_type == 'BASE TABLE' ), order_by=[tables.c.table_name] ) @@ -1235,13 +1195,13 @@ class MSDialect(default.DefaultDialect): return table_names @reflection.cache - def get_view_names(self, connection, schema=None, **kw): - current_schema = schema or self.default_schema_name + @_db_plus_owner_listing + def get_view_names(self, connection, dbname, owner, schema, **kw): tables = ischema.tables s = sql.select([tables.c.table_name], sql.and_( - tables.c.table_schema == current_schema, - tables.c.table_type == u'VIEW' + tables.c.table_schema == owner, + tables.c.table_type == 'VIEW' ), order_by=[tables.c.table_name] ) @@ -1249,15 +1209,13 @@ class MSDialect(default.DefaultDialect): return view_names @reflection.cache - def get_indexes(self, connection, tablename, schema=None, **kw): + @_db_plus_owner + def get_indexes(self, connection, tablename, dbname, owner, schema, **kw): # using system catalogs, don't support index reflection # below MS 2005 if self.server_version_info < MS_2005_VERSION: return [] - current_schema = schema or self.default_schema_name - full_tname = "%s.%s" % (current_schema, tablename) - rp = connection.execute( sql.text("select ind.index_id, ind.is_unique, ind.name " "from sys.indexes as ind join sys.tables as tab on " @@ -1269,20 +1227,20 @@ class MSDialect(default.DefaultDialect): bindparams=[ sql.bindparam('tabname', tablename, sqltypes.String(convert_unicode=True)), - sql.bindparam('schname', current_schema, + sql.bindparam('schname', owner, sqltypes.String(convert_unicode=True)) ], - typemap = { - 'name':sqltypes.Unicode() + typemap={ + 'name': sqltypes.Unicode() } ) ) indexes = {} for row in rp: indexes[row['index_id']] = { - 'name':row['name'], - 'unique':row['is_unique'] == 1, - 'column_names':[] + 'name': row['name'], + 'unique': row['is_unique'] == 1, + 'column_names': [] } rp = connection.execute( sql.text( @@ -1298,24 +1256,21 @@ class MSDialect(default.DefaultDialect): bindparams=[ sql.bindparam('tabname', tablename, sqltypes.String(convert_unicode=True)), - sql.bindparam('schname', current_schema, + sql.bindparam('schname', owner, sqltypes.String(convert_unicode=True)) ], - typemap = { - 'name':sqltypes.Unicode() - } + typemap={'name': sqltypes.Unicode()} ), ) for row in rp: if row['index_id'] in indexes: indexes[row['index_id']]['column_names'].append(row['name']) - return indexes.values() + return list(indexes.values()) @reflection.cache - def get_view_definition(self, connection, viewname, schema=None, **kw): - current_schema = schema or self.default_schema_name - + @_db_plus_owner + def get_view_definition(self, connection, viewname, dbname, owner, schema, **kw): rp = connection.execute( sql.text( "select definition from sys.sql_modules as mod, " @@ -1328,7 +1283,7 @@ class MSDialect(default.DefaultDialect): bindparams=[ sql.bindparam('viewname', viewname, sqltypes.String(convert_unicode=True)), - sql.bindparam('schname', current_schema, + sql.bindparam('schname', owner, sqltypes.String(convert_unicode=True)) ] ) @@ -1339,17 +1294,18 @@ class MSDialect(default.DefaultDialect): return view_def @reflection.cache - def get_columns(self, connection, tablename, schema=None, **kw): + @_db_plus_owner + def get_columns(self, connection, tablename, dbname, owner, schema, **kw): # Get base columns - current_schema = schema or self.default_schema_name columns = ischema.columns - if current_schema: - whereclause = sql.and_(columns.c.table_name==tablename, - columns.c.table_schema==current_schema) + if owner: + whereclause = sql.and_(columns.c.table_name == tablename, + columns.c.table_schema == owner) else: - whereclause = columns.c.table_name==tablename + whereclause = columns.c.table_name == tablename s = sql.select([columns], whereclause, order_by=[columns.c.ordinal_position]) + c = connection.execute(s) cols = [] while True: @@ -1393,11 +1349,11 @@ class MSDialect(default.DefaultDialect): coltype = coltype(**kwargs) cdict = { - 'name' : name, - 'type' : coltype, - 'nullable' : nullable, - 'default' : default, - 'autoincrement':False, + 'name': name, + 'type': coltype, + 'nullable': nullable, + 'default': default, + 'autoincrement': False, } cols.append(cdict) # autoincrement and identity @@ -1407,7 +1363,7 @@ class MSDialect(default.DefaultDialect): # We also run an sp_columns to check for identity columns: cursor = connection.execute("sp_columns @table_name = '%s', " "@table_owner = '%s'" - % (tablename, current_schema)) + % (tablename, owner)) ic = None while True: row = cursor.fetchone() @@ -1423,7 +1379,7 @@ class MSDialect(default.DefaultDialect): cursor.close() if ic is not None and self.server_version_info >= MS_2005_VERSION: - table_fullname = "%s.%s" % (current_schema, tablename) + table_fullname = "%s.%s" % (owner, tablename) cursor = connection.execute( "select ident_seed('%s'), ident_incr('%s')" % (table_fullname, table_fullname) @@ -1432,53 +1388,40 @@ class MSDialect(default.DefaultDialect): row = cursor.first() if row is not None and row[0] is not None: colmap[ic]['sequence'].update({ - 'start' : int(row[0]), - 'increment' : int(row[1]) + 'start': int(row[0]), + 'increment': int(row[1]) }) return cols @reflection.cache - def get_primary_keys(self, connection, tablename, schema=None, **kw): - current_schema = schema or self.default_schema_name + @_db_plus_owner + def get_pk_constraint(self, connection, tablename, dbname, owner, schema, **kw): pkeys = [] - # information_schema.referential_constraints - RR = ischema.ref_constraints - # information_schema.table_constraints TC = ischema.constraints - # information_schema.constraint_column_usage: - # the constrained column - C = ischema.key_constraints.alias('C') - # information_schema.constraint_column_usage: - # the referenced column - R = ischema.key_constraints.alias('R') + C = ischema.key_constraints.alias('C') # Primary key constraints - s = sql.select([C.c.column_name, TC.c.constraint_type], + s = sql.select([C.c.column_name, TC.c.constraint_type, C.c.constraint_name], sql.and_(TC.c.constraint_name == C.c.constraint_name, - TC.c.table_schema == C.c.table_schema, + TC.c.table_schema == C.c.table_schema, C.c.table_name == tablename, - C.c.table_schema == current_schema) + C.c.table_schema == owner) ) c = connection.execute(s) + constraint_name = None for row in c: if 'PRIMARY' in row[TC.c.constraint_type.name]: pkeys.append(row[0]) - return pkeys + if constraint_name is None: + constraint_name = row[C.c.constraint_name.name] + return {'constrained_columns': pkeys, 'name': constraint_name} @reflection.cache - def get_foreign_keys(self, connection, tablename, schema=None, **kw): - current_schema = schema or self.default_schema_name - # Add constraints - #information_schema.referential_constraints + @_db_plus_owner + def get_foreign_keys(self, connection, tablename, dbname, owner, schema, **kw): RR = ischema.ref_constraints - # information_schema.table_constraints - TC = ischema.constraints - # information_schema.constraint_column_usage: - # the constrained column - C = ischema.key_constraints.alias('C') - # information_schema.constraint_column_usage: - # the referenced column - R = ischema.key_constraints.alias('R') + C = ischema.key_constraints.alias('C') + R = ischema.key_constraints.alias('R') # Foreign key constraints s = sql.select([C.c.column_name, @@ -1487,16 +1430,14 @@ class MSDialect(default.DefaultDialect): RR.c.update_rule, RR.c.delete_rule], sql.and_(C.c.table_name == tablename, - C.c.table_schema == current_schema, + C.c.table_schema == owner, C.c.constraint_name == RR.c.constraint_name, R.c.constraint_name == RR.c.unique_constraint_name, C.c.ordinal_position == R.c.ordinal_position ), - order_by = [ - RR.c.constraint_name, - R.c.ordinal_position]) - + order_by=[RR.c.constraint_name, R.c.ordinal_position] + ) # group rows by constraint ID, to handle multi-column FKs fkeys = [] @@ -1504,11 +1445,11 @@ class MSDialect(default.DefaultDialect): def fkey_rec(): return { - 'name' : None, - 'constrained_columns' : [], - 'referred_schema' : None, - 'referred_table' : None, - 'referred_columns' : [] + 'name': None, + 'constrained_columns': [], + 'referred_schema': None, + 'referred_table': None, + 'referred_columns': [] } fkeys = util.defaultdict(fkey_rec) @@ -1520,8 +1461,9 @@ class MSDialect(default.DefaultDialect): rec['name'] = rfknm if not rec['referred_table']: rec['referred_table'] = rtbl - - if schema is not None or current_schema != rschema: + if schema is not None or owner != rschema: + if dbname: + rschema = dbname + "." + rschema rec['referred_schema'] = rschema local_cols, remote_cols = \ @@ -1531,5 +1473,4 @@ class MSDialect(default.DefaultDialect): local_cols.append(scol) remote_cols.append(rcol) - return fkeys.values() - + return list(fkeys.values()) diff --git a/libs/sqlalchemy/dialects/mssql/information_schema.py b/libs/sqlalchemy/dialects/mssql/information_schema.py index 0dcddae9..26e70f7f 100644 --- a/libs/sqlalchemy/dialects/mssql/information_schema.py +++ b/libs/sqlalchemy/dialects/mssql/information_schema.py @@ -1,13 +1,17 @@ # mssql/information_schema.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php # TODO: should be using the sys. catalog with SQL Server, not information schema -from sqlalchemy import Table, MetaData, Column -from sqlalchemy.types import String, Unicode, Integer, TypeDecorator +from ... import Table, MetaData, Column +from ...types import String, Unicode, UnicodeText, Integer, TypeDecorator +from ... import cast +from ... import util +from ...sql import expression +from ...ext.compiler import compiles ischema = MetaData() @@ -15,12 +19,25 @@ class CoerceUnicode(TypeDecorator): impl = Unicode def process_bind_param(self, value, dialect): - # Py2K - if isinstance(value, str): + if util.py2k and isinstance(value, util.binary_type): value = value.decode(dialect.encoding) - # end Py2K return value + def bind_expression(self, bindvalue): + return _cast_on_2005(bindvalue) + +class _cast_on_2005(expression.ColumnElement): + def __init__(self, bindvalue): + self.bindvalue = bindvalue + +@compiles(_cast_on_2005) +def _compile(element, compiler, **kw): + from . import base + if compiler.dialect.server_version_info < base.MS_2005_VERSION: + return compiler.process(element.bindvalue, **kw) + else: + return compiler.process(cast(element.bindvalue, Unicode), **kw) + schemata = Table("SCHEMATA", ischema, Column("CATALOG_NAME", CoerceUnicode, key="catalog_name"), Column("SCHEMA_NAME", CoerceUnicode, key="schema_name"), @@ -95,4 +112,3 @@ views = Table("VIEWS", ischema, Column("CHECK_OPTION", String, key="check_option"), Column("IS_UPDATABLE", String, key="is_updatable"), schema="INFORMATION_SCHEMA") - diff --git a/libs/sqlalchemy/dialects/mssql/mxodbc.py b/libs/sqlalchemy/dialects/mssql/mxodbc.py index 56a72f41..5b686c47 100644 --- a/libs/sqlalchemy/dialects/mssql/mxodbc.py +++ b/libs/sqlalchemy/dialects/mssql/mxodbc.py @@ -1,28 +1,18 @@ # mssql/mxodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -Support for MS-SQL via mxODBC. - -mxODBC is available at: - - http://www.egenix.com/ - -This was tested with mxODBC 3.1.2 and the SQL Server Native -Client connected to MSSQL 2005 and 2008 Express Editions. - -Connecting -~~~~~~~~~~ - -Connection is via DSN:: - - mssql+mxodbc://:@ +.. dialect:: mssql+mxodbc + :name: mxODBC + :dbapi: mxodbc + :connectstring: mssql+mxodbc://:@ + :url: http://www.egenix.com/ Execution Modes -~~~~~~~~~~~~~~~ +--------------- mxODBC features two styles of statement execution, using the ``cursor.execute()`` and ``cursor.executedirect()`` methods (the second being @@ -52,14 +42,38 @@ of ``False`` will unconditionally use string-escaped parameters. """ -from sqlalchemy import types as sqltypes -from sqlalchemy.connectors.mxodbc import MxODBCConnector -from sqlalchemy.dialects.mssql.pyodbc import MSExecutionContext_pyodbc -from sqlalchemy.dialects.mssql.base import (MSDialect, +from ... import types as sqltypes +from ...connectors.mxodbc import MxODBCConnector +from .pyodbc import MSExecutionContext_pyodbc, _MSNumeric_pyodbc +from .base import (MSDialect, MSSQLStrictCompiler, - _MSDateTime, _MSDate, TIME) + _MSDateTime, _MSDate, _MSTime) +class _MSNumeric_mxodbc(_MSNumeric_pyodbc): + """Include pyodbc's numeric processor. + """ + + +class _MSDate_mxodbc(_MSDate): + def bind_processor(self, dialect): + def process(value): + if value is not None: + return "%s-%s-%s" % (value.year, value.month, value.day) + else: + return None + return process + + +class _MSTime_mxodbc(_MSTime): + def bind_processor(self, dialect): + def process(value): + if value is not None: + return "%s:%s:%s" % (value.hour, value.minute, value.second) + else: + return None + return process + class MSExecutionContext_mxodbc(MSExecutionContext_pyodbc): """ @@ -71,23 +85,27 @@ class MSExecutionContext_mxodbc(MSExecutionContext_pyodbc): # is really only being used in cases where OUTPUT # won't work. + class MSDialect_mxodbc(MxODBCConnector, MSDialect): - # TODO: may want to use this only if FreeTDS is not in use, - # since FreeTDS doesn't seem to use native binds. - statement_compiler = MSSQLStrictCompiler + # this is only needed if "native ODBC" mode is used, + # which is now disabled by default. + #statement_compiler = MSSQLStrictCompiler + execution_ctx_cls = MSExecutionContext_mxodbc + + # flag used by _MSNumeric_mxodbc + _need_decimal_fix = True + colspecs = { - #sqltypes.Numeric : _MSNumeric, - sqltypes.DateTime : _MSDateTime, - sqltypes.Date : _MSDate, - sqltypes.Time : TIME, + sqltypes.Numeric: _MSNumeric_mxodbc, + sqltypes.DateTime: _MSDateTime, + sqltypes.Date: _MSDate_mxodbc, + sqltypes.Time: _MSTime_mxodbc, } - - def __init__(self, description_encoding='latin-1', **params): + def __init__(self, description_encoding=None, **params): super(MSDialect_mxodbc, self).__init__(**params) self.description_encoding = description_encoding dialect = MSDialect_mxodbc - diff --git a/libs/sqlalchemy/dialects/mssql/pymssql.py b/libs/sqlalchemy/dialects/mssql/pymssql.py index 8229d6ce..021219cb 100644 --- a/libs/sqlalchemy/dialects/mssql/pymssql.py +++ b/libs/sqlalchemy/dialects/mssql/pymssql.py @@ -1,32 +1,18 @@ # mssql/pymssql.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -Support for the pymssql dialect. - -This dialect supports pymssql 1.0 and greater. - -pymssql is available at: - - http://pymssql.sourceforge.net/ - -Connecting -^^^^^^^^^^ - -Sample connect string:: - - mssql+pymssql://:@ - -Adding "?charset=utf8" or similar will cause pymssql to return -strings as Python unicode objects. This can potentially improve -performance in some scenarios as decoding of strings is -handled natively. +.. dialect:: mssql+pymssql + :name: pymssql + :dbapi: pymssql + :connectstring: mssql+pymssql://:@?charset=utf8 + :url: http://pymssql.sourceforge.net/ Limitations -^^^^^^^^^^^ +----------- pymssql inherits a lot of limitations from FreeTDS, including: @@ -38,10 +24,11 @@ pymssql inherits a lot of limitations from FreeTDS, including: Please consult the pymssql documentation for further information. """ -from sqlalchemy.dialects.mssql.base import MSDialect -from sqlalchemy import types as sqltypes, util, processors +from .base import MSDialect +from ... import types as sqltypes, util, processors import re + class _MSNumeric_pymssql(sqltypes.Numeric): def result_processor(self, dialect, type_): if not self.asdecimal: @@ -49,6 +36,7 @@ class _MSNumeric_pymssql(sqltypes.Numeric): else: return sqltypes.Numeric.result_processor(self, dialect, type_) + class MSDialect_pymssql(MSDialect): supports_sane_rowcount = False driver = 'pymssql' @@ -56,16 +44,17 @@ class MSDialect_pymssql(MSDialect): colspecs = util.update_copy( MSDialect.colspecs, { - sqltypes.Numeric:_MSNumeric_pymssql, - sqltypes.Float:sqltypes.Float, + sqltypes.Numeric: _MSNumeric_pymssql, + sqltypes.Float: sqltypes.Float, } ) + @classmethod def dbapi(cls): module = __import__('pymssql') # pymmsql doesn't have a Binary method. we use string # TODO: monkeypatching here is less than ideal - module.Binary = str + module.Binary = lambda x: x if hasattr(x, 'decode') else str(x) client_ver = tuple(int(x) for x in module.__version__.split(".")) if client_ver < (1, ): @@ -96,6 +85,9 @@ class MSDialect_pymssql(MSDialect): def is_disconnect(self, e, connection, cursor): for msg in ( + "Adaptive Server connection timed out", + "Net-Lib error during Connection reset by peer", + "message 20003", # connection timeout "Error 10054", "Not connected to any MS SQL server", "Connection is closed" diff --git a/libs/sqlalchemy/dialects/mssql/pyodbc.py b/libs/sqlalchemy/dialects/mssql/pyodbc.py index 389018c6..8c43eb8a 100644 --- a/libs/sqlalchemy/dialects/mssql/pyodbc.py +++ b/libs/sqlalchemy/dialects/mssql/pyodbc.py @@ -1,18 +1,18 @@ # mssql/pyodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -Support for MS-SQL via pyodbc. +.. dialect:: mssql+pyodbc + :name: PyODBC + :dbapi: pyodbc + :connectstring: mssql+pyodbc://:@ + :url: http://pypi.python.org/pypi/pyodbc/ -pyodbc is available at: - - http://pypi.python.org/pypi/pyodbc/ - -Connecting -^^^^^^^^^^ +Additional Connection Examples +------------------------------- Examples of pyodbc connection string URLs: @@ -81,7 +81,7 @@ the python shell. For example:: 'dsn%3Dmydsn%3BDatabase%3Ddb' Unicode Binds -^^^^^^^^^^^^^ +------------- The current state of PyODBC on a unix backend with FreeTDS and/or EasySoft is poor regarding unicode; different OS platforms and versions of UnixODBC @@ -111,23 +111,23 @@ for unix + PyODBC. """ -from sqlalchemy.dialects.mssql.base import MSExecutionContext, MSDialect -from sqlalchemy.connectors.pyodbc import PyODBCConnector -from sqlalchemy import types as sqltypes, util +from .base import MSExecutionContext, MSDialect +from ...connectors.pyodbc import PyODBCConnector +from ... import types as sqltypes, util import decimal -class _MSNumeric_pyodbc(sqltypes.Numeric): +class _ms_numeric_pyodbc(object): + """Turns Decimals with adjusted() < 0 or > 7 into strings. - This is the only method that is proven to work with Pyodbc+MSSQL - without crashing (floats can be used but seem to cause sporadic - crashes). + The routines here are needed for older pyodbc versions + as well as current mxODBC versions. """ def bind_processor(self, dialect): - super_process = super(_MSNumeric_pyodbc, self).\ + super_process = super(_ms_numeric_pyodbc, self).\ bind_processor(dialect) if not dialect._need_decimal_fix: @@ -164,7 +164,7 @@ class _MSNumeric_pyodbc(sqltypes.Numeric): result = "%s%s%s" % ( (value < 0 and '-' or ''), "".join([str(s) for s in _int]), - "0" * (value.adjusted() - (len(_int)-1))) + "0" * (value.adjusted() - (len(_int) - 1))) else: if (len(_int) - 1) > value.adjusted(): result = "%s%s.%s" % ( @@ -180,6 +180,11 @@ class _MSNumeric_pyodbc(sqltypes.Numeric): [str(s) for s in _int][0:value.adjusted() + 1])) return result +class _MSNumeric_pyodbc(_ms_numeric_pyodbc, sqltypes.Numeric): + pass + +class _MSFloat_pyodbc(_ms_numeric_pyodbc, sqltypes.Float): + pass class MSExecutionContext_pyodbc(MSExecutionContext): _embedded_scope_identity = False @@ -219,7 +224,7 @@ class MSExecutionContext_pyodbc(MSExecutionContext): # without closing it (FreeTDS particularly) row = self.cursor.fetchall()[0] break - except self.dialect.dbapi.Error, e: + except self.dialect.dbapi.Error as e: # no way around this - nextset() consumes the previous set # so we need to just keep flipping self.cursor.nextset() @@ -238,11 +243,12 @@ class MSDialect_pyodbc(PyODBCConnector, MSDialect): colspecs = util.update_copy( MSDialect.colspecs, { - sqltypes.Numeric:_MSNumeric_pyodbc + sqltypes.Numeric: _MSNumeric_pyodbc, + sqltypes.Float: _MSFloat_pyodbc } ) - def __init__(self, description_encoding='latin-1', **params): + def __init__(self, description_encoding=None, **params): super(MSDialect_pyodbc, self).__init__(**params) self.description_encoding = description_encoding self.use_scope_identity = self.use_scope_identity and \ diff --git a/libs/sqlalchemy/dialects/mssql/zxjdbc.py b/libs/sqlalchemy/dialects/mssql/zxjdbc.py index 842225da..706eef3a 100644 --- a/libs/sqlalchemy/dialects/mssql/zxjdbc.py +++ b/libs/sqlalchemy/dialects/mssql/zxjdbc.py @@ -1,32 +1,22 @@ # mssql/zxjdbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the Microsoft SQL Server database via the zxjdbc JDBC -connector. +""" +.. dialect:: mssql+zxjdbc + :name: zxJDBC for Jython + :dbapi: zxjdbc + :connectstring: mssql+zxjdbc://user:pass@host:port/dbname[?key=value&key=value...] + :driverurl: http://jtds.sourceforge.net/ -JDBC Driver ------------ - -Requires the jTDS driver, available from: http://jtds.sourceforge.net/ - -Connecting ----------- - -URLs are of the standard form of -``mssql+zxjdbc://user:pass@host:port/dbname[?key=value&key=value...]``. - -Additional arguments which may be specified either as query string -arguments on the URL, or as keyword arguments to -:func:`~sqlalchemy.create_engine()` will be passed as Connection -properties to the underlying JDBC driver. """ -from sqlalchemy.connectors.zxJDBC import ZxJDBCConnector -from sqlalchemy.dialects.mssql.base import MSDialect, MSExecutionContext -from sqlalchemy.engine import base +from ...connectors.zxJDBC import ZxJDBCConnector +from .base import MSDialect, MSExecutionContext +from ... import engine + class MSExecutionContext_zxjdbc(MSExecutionContext): @@ -46,13 +36,13 @@ class MSExecutionContext_zxjdbc(MSExecutionContext): try: row = self.cursor.fetchall()[0] break - except self.dialect.dbapi.Error, e: + except self.dialect.dbapi.Error: self.cursor.nextset() self._lastrowid = int(row[0]) if (self.isinsert or self.isupdate or self.isdelete) and \ self.compiled.returning: - self._result_proxy = base.FullyBufferedResultProxy(self) + self._result_proxy = engine.FullyBufferedResultProxy(self) if self._enable_identity_insert: table = self.dialect.identifier_preparer.format_table( diff --git a/libs/sqlalchemy/dialects/mysql/__init__.py b/libs/sqlalchemy/dialects/mysql/__init__.py index c41dd0b1..4eb8cc6d 100644 --- a/libs/sqlalchemy/dialects/mysql/__init__.py +++ b/libs/sqlalchemy/dialects/mysql/__init__.py @@ -1,17 +1,17 @@ # mysql/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy.dialects.mysql import base, mysqldb, oursql, \ +from . import base, mysqldb, oursql, \ pyodbc, zxjdbc, mysqlconnector, pymysql,\ - gaerdbms + gaerdbms, cymysql # default dialect base.dialect = mysqldb.dialect -from sqlalchemy.dialects.mysql.base import \ +from .base import \ BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, DATETIME, \ DECIMAL, DOUBLE, ENUM, DECIMAL,\ FLOAT, INTEGER, INTEGER, LONGBLOB, LONGTEXT, MEDIUMBLOB, \ diff --git a/libs/sqlalchemy/dialects/mysql/base.py b/libs/sqlalchemy/dialects/mysql/base.py index ea180eee..a3942e89 100644 --- a/libs/sqlalchemy/dialects/mysql/base.py +++ b/libs/sqlalchemy/dialects/mysql/base.py @@ -1,10 +1,13 @@ # mysql/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database. +""" + +.. dialect:: mysql + :name: MySQL Supported Versions and Features ------------------------------- @@ -17,10 +20,7 @@ example, they won't work in SQLAlchemy either. See the official MySQL documentation for detailed information about features supported in any given server release. -Connecting ----------- - -See the API documentation on individual drivers for details on connecting. +.. _mysql_connection_timeouts: Connection Timeouts ------------------- @@ -55,7 +55,9 @@ creation option can be specified in this syntax:: .. seealso:: - `The InnoDB Storage Engine `_ - on the MySQL website. + `The InnoDB Storage Engine + `_ - + on the MySQL website. Case Sensitivity and Table Reflection ------------------------------------- @@ -125,32 +127,20 @@ engines:: Column('id', Integer, primary_key=True) ) -SQL Mode --------- +Ansi Quoting Style +------------------ -MySQL SQL modes are supported. Modes that enable ``ANSI_QUOTES`` (such as -``ANSI``) require an engine option to modify SQLAlchemy's quoting style. -When using an ANSI-quoting mode, supply ``use_ansiquotes=True`` when -creating your ``Engine``:: +MySQL features two varieties of identifier "quoting style", one using +backticks and the other using quotes, e.g. ```some_identifier``` vs. +``"some_identifier"``. All MySQL dialects detect which version +is in use by checking the value of ``sql_mode`` when a connection is first +established with a particular :class:`.Engine`. This quoting style comes +into play when rendering table and column names as well as when reflecting +existing database structures. The detection is entirely automatic and +no special configuration is needed to use either quoting style. - create_engine('mysql://localhost/test', use_ansiquotes=True) - -This is an engine-wide option and is not toggleable on a per-connection basis. -SQLAlchemy does not presume to ``SET sql_mode`` for you with this option. For -the best performance, set the quoting style server-wide in ``my.cnf`` or by -supplying ``--sql-mode`` to ``mysqld``. You can also use a -:class:`sqlalchemy.pool.Pool` listener hook to issue a ``SET SESSION -sql_mode='...'`` on connect to configure each connection. - -If you do not specify ``use_ansiquotes``, the regular MySQL quoting style is -used by default. - -If you do issue a ``SET sql_mode`` through SQLAlchemy, the dialect must be -updated if the quoting style is changed. Again, this change will affect all -connections:: - - connection.execute('SET sql_mode="ansi"') - connection.dialect.use_ansiquotes = True +.. versionchanged:: 0.6 detection of ANSI quoting style is entirely automatic, + there's no longer any end-user ``create_engine()`` options in this regard. MySQL SQL Extensions -------------------- @@ -202,13 +192,13 @@ detection, instead rendering the internal expression directly. CAST may still not be desirable on an early MySQL version post-4.0.2, as it didn't add all datatype support until 4.1.1. If your application falls into this -narrow area, the behavior of CAST can be controlled using the :ref:`sqlalchemy.ext.compiler_toplevel` -system, as per the recipe below:: +narrow area, the behavior of CAST can be controlled using the +:ref:`sqlalchemy.ext.compiler_toplevel` system, as per the recipe below:: - from sqlalchemy.sql.expression import _Cast + from sqlalchemy.sql.expression import Cast from sqlalchemy.ext.compiler import compiles - @compiles(_Cast, 'mysql') + @compiles(Cast, 'mysql') def _check_mysql_version(element, compiler, **kw): if compiler.dialect.server_version_info < (4, 1, 0): return compiler.process(element.clause, **kw) @@ -239,11 +229,18 @@ become part of the index. SQLAlchemy provides this feature via the Index('my_index', my_table.c.data, mysql_length=10) + Index('a_b_idx', my_table.c.a, my_table.c.b, mysql_length={'a': 4, 'b': 9}) + Prefix lengths are given in characters for nonbinary string types and in bytes -for binary string types. The value passed to the keyword argument will be -simply passed through to the underlying CREATE INDEX command, so it *must* be -an integer. MySQL only allows a length for an index if it is for a CHAR, -VARCHAR, TEXT, BINARY, VARBINARY and BLOB. +for binary string types. The value passed to the keyword argument *must* be +either an integer (and, thus, specify the same prefix length value for all +columns of the index) or a dict in which keys are column names and values are +prefix length values for corresponding columns. MySQL only allows a length for +a column of an index if it is for a CHAR, VARCHAR, TEXT, BINARY, VARBINARY and +BLOB. + +.. versionadded:: 0.8.2 ``mysql_length`` may now be specified as a dictionary + for use with composite indexes. Index Types ~~~~~~~~~~~~~ @@ -268,26 +265,62 @@ http://dev.mysql.com/doc/refman/5.0/en/create-index.html http://dev.mysql.com/doc/refman/5.0/en/create-table.html +.. _mysql_foreign_keys: + +MySQL Foreign Key Options +------------------------- + +MySQL does not support the foreign key arguments "DEFERRABLE", "INITIALLY", +or "MATCH". Using the ``deferrable`` or ``initially`` keyword argument with +:class:`.ForeignKeyConstraint` or :class:`.ForeignKey` will have the effect of these keywords being +rendered in a DDL expression, which will then raise an error on MySQL. +In order to use these keywords on a foreign key while having them ignored +on a MySQL backend, use a custom compile rule:: + + from sqlalchemy.ext.compiler import compiles + from sqlalchemy.schema import ForeignKeyConstraint + + @compiles(ForeignKeyConstraint, "mysql") + def process(element, compiler, **kw): + element.deferrable = element.initially = None + return compiler.visit_foreign_key_constraint(element, **kw) + +.. versionchanged:: 0.9.0 - the MySQL backend no longer silently ignores + the ``deferrable`` or ``initially`` keyword arguments of :class:`.ForeignKeyConstraint` + and :class:`.ForeignKey`. + +The "MATCH" keyword is in fact more insidious, and is explicitly disallowed +by SQLAlchemy in conjunction with the MySQL backend. This argument is silently +ignored by MySQL, but in addition has the effect of ON UPDATE and ON DELETE options +also being ignored by the backend. Therefore MATCH should never be used with the +MySQL backend; as is the case with DEFERRABLE and INITIALLY, custom compilation +rules can be used to correct a MySQL ForeignKeyConstraint at DDL definition time. + +.. versionadded:: 0.9.0 - the MySQL backend will raise a :class:`.CompileError` + when the ``match`` keyword is used with :class:`.ForeignKeyConstraint` + or :class:`.ForeignKey`. + """ -import datetime, inspect, re, sys +import datetime +import inspect +import re +import sys -from sqlalchemy import schema as sa_schema -from sqlalchemy import exc, log, sql, util -from sqlalchemy.sql import operators as sql_operators -from sqlalchemy.sql import functions as sql_functions -from sqlalchemy.sql import compiler +from ... import schema as sa_schema +from ... import exc, log, sql, util +from ...sql import compiler from array import array as _array -from sqlalchemy.engine import reflection -from sqlalchemy.engine import base as engine_base, default -from sqlalchemy import types as sqltypes -from sqlalchemy.util import topological -from sqlalchemy.types import DATE, DATETIME, BOOLEAN, TIME, \ +from ...engine import reflection +from ...engine import default +from ... import types as sqltypes +from ...util import topological +from ...types import DATE, DATETIME, BOOLEAN, TIME, \ BLOB, BINARY, VARBINARY RESERVED_WORDS = set( - ['accessible', 'add', 'all', 'alter', 'analyze','and', 'as', 'asc', + ['accessible', 'add', 'all', 'alter', 'analyze', 'and', 'as', 'asc', 'asensitive', 'before', 'between', 'bigint', 'binary', 'blob', 'both', 'by', 'call', 'cascade', 'case', 'change', 'char', 'character', 'check', 'collate', 'column', 'condition', 'constraint', 'continue', 'convert', @@ -322,10 +355,20 @@ RESERVED_WORDS = set( 'union', 'unique', 'unlock', 'unsigned', 'update', 'usage', 'use', 'using', 'utc_date', 'utc_time', 'utc_timestamp', 'values', 'varbinary', 'varchar', 'varcharacter', 'varying', 'when', 'where', 'while', 'with', - 'write', 'x509', 'xor', 'year_month', 'zerofill', # 5.0 - 'columns', 'fields', 'privileges', 'soname', 'tables', # 4.1 + + 'write', 'x509', 'xor', 'year_month', 'zerofill', # 5.0 + + 'columns', 'fields', 'privileges', 'soname', 'tables', # 4.1 + 'accessible', 'linear', 'master_ssl_verify_server_cert', 'range', - 'read_only', 'read_write', # 5.1 + 'read_only', 'read_write', # 5.1 + + 'general', 'ignore_server_ids', 'master_heartbeat_period', 'maxvalue', + 'resignal', 'signal', 'slow', # 5.5 + + 'get', 'io_after_gtids', 'io_before_gtids', 'master_bind', 'one_shot', + 'partition', 'sql_after_gtids', 'sql_before_gtids', # 5.6 + ]) AUTOCOMMIT_RE = re.compile( @@ -337,13 +380,22 @@ SET_RE = re.compile( class _NumericType(object): - """Base for MySQL numeric types.""" + """Base for MySQL numeric types. + + This is the base both for NUMERIC as well as INTEGER, hence + it's a mixin. + + """ def __init__(self, unsigned=False, zerofill=False, **kw): self.unsigned = unsigned self.zerofill = zerofill super(_NumericType, self).__init__(**kw) + def __repr__(self): + return util.generic_repr(self, + to_inspect=[_NumericType, sqltypes.Numeric]) + class _FloatType(_NumericType, sqltypes.Float): def __init__(self, precision=None, scale=None, asdecimal=True, **kw): if isinstance(self, (REAL, DOUBLE)) and \ @@ -354,48 +406,42 @@ class _FloatType(_NumericType, sqltypes.Float): raise exc.ArgumentError( "You must specify both precision and scale or omit " "both altogether.") - super(_FloatType, self).__init__(precision=precision, asdecimal=asdecimal, **kw) self.scale = scale + def __repr__(self): + return util.generic_repr(self, + to_inspect=[_FloatType, _NumericType, sqltypes.Float]) + class _IntegerType(_NumericType, sqltypes.Integer): def __init__(self, display_width=None, **kw): self.display_width = display_width super(_IntegerType, self).__init__(**kw) + def __repr__(self): + return util.generic_repr(self, + to_inspect=[_IntegerType, _NumericType, sqltypes.Integer]) + class _StringType(sqltypes.String): """Base for MySQL string types.""" def __init__(self, charset=None, collation=None, - ascii=False, binary=False, + ascii=False, binary=False, unicode=False, national=False, **kw): self.charset = charset + # allow collate= or collation= - self.collation = kw.pop('collate', collation) + kw.setdefault('collation', kw.pop('collate', collation)) + self.ascii = ascii - # We have to munge the 'unicode' param strictly as a dict - # otherwise 2to3 will turn it into str. - self.__dict__['unicode'] = kw.get('unicode', False) - # sqltypes.String does not accept the 'unicode' arg at all. - if 'unicode' in kw: - del kw['unicode'] + self.unicode = unicode self.binary = binary self.national = national super(_StringType, self).__init__(**kw) def __repr__(self): - attributes = inspect.getargspec(self.__init__)[0][1:] - attributes.extend(inspect.getargspec(_StringType.__init__)[0][1:]) - - params = {} - for attr in attributes: - val = getattr(self, attr) - if val is not None and val is not False: - params[attr] = val - - return "%s(%s)" % (self.__class__.__name__, - ', '.join(['%s=%r' % (k, params[k]) for k in params])) - + return util.generic_repr(self, + to_inspect=[_StringType, sqltypes.String]) class NUMERIC(_NumericType, sqltypes.NUMERIC): """MySQL NUMERIC type.""" @@ -418,7 +464,8 @@ class NUMERIC(_NumericType, sqltypes.NUMERIC): numeric. """ - super(NUMERIC, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + super(NUMERIC, self).__init__(precision=precision, + scale=scale, asdecimal=asdecimal, **kw) class DECIMAL(_NumericType, sqltypes.DECIMAL): @@ -454,6 +501,14 @@ class DOUBLE(_FloatType): def __init__(self, precision=None, scale=None, asdecimal=True, **kw): """Construct a DOUBLE. + .. note:: + + The :class:`.DOUBLE` type by default converts from float + to Decimal, using a truncation that defaults to 10 digits. Specify + either ``scale=n`` or ``decimal_return_scale=n`` in order to change + this scale, or ``asdecimal=False`` to return values directly as + Python floating points. + :param precision: Total digits in this number. If scale and precision are both None, values are stored to limits allowed by the server. @@ -470,6 +525,7 @@ class DOUBLE(_FloatType): super(DOUBLE, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + class REAL(_FloatType, sqltypes.REAL): """MySQL REAL type.""" @@ -478,6 +534,14 @@ class REAL(_FloatType, sqltypes.REAL): def __init__(self, precision=None, scale=None, asdecimal=True, **kw): """Construct a REAL. + .. note:: + + The :class:`.REAL` type by default converts from float + to Decimal, using a truncation that defaults to 10 digits. Specify + either ``scale=n`` or ``decimal_return_scale=n`` in order to change + this scale, or ``asdecimal=False`` to return values directly as + Python floating points. + :param precision: Total digits in this number. If scale and precision are both None, values are stored to limits allowed by the server. @@ -494,6 +558,7 @@ class REAL(_FloatType, sqltypes.REAL): super(REAL, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal, **kw) + class FLOAT(_FloatType, sqltypes.FLOAT): """MySQL FLOAT type.""" @@ -521,6 +586,7 @@ class FLOAT(_FloatType, sqltypes.FLOAT): def bind_processor(self, dialect): return None + class INTEGER(_IntegerType, sqltypes.INTEGER): """MySQL INTEGER type.""" @@ -541,6 +607,7 @@ class INTEGER(_IntegerType, sqltypes.INTEGER): """ super(INTEGER, self).__init__(display_width=display_width, **kw) + class BIGINT(_IntegerType, sqltypes.BIGINT): """MySQL BIGINTEGER type.""" @@ -561,6 +628,7 @@ class BIGINT(_IntegerType, sqltypes.BIGINT): """ super(BIGINT, self).__init__(display_width=display_width, **kw) + class MEDIUMINT(_IntegerType): """MySQL MEDIUMINTEGER type.""" @@ -581,6 +649,7 @@ class MEDIUMINT(_IntegerType): """ super(MEDIUMINT, self).__init__(display_width=display_width, **kw) + class TINYINT(_IntegerType): """MySQL TINYINT type.""" @@ -589,10 +658,6 @@ class TINYINT(_IntegerType): def __init__(self, display_width=None, **kw): """Construct a TINYINT. - Note: following the usual MySQL conventions, TINYINT(1) columns - reflected during Table(..., autoload=True) are treated as - Boolean columns. - :param display_width: Optional, maximum display width for this number. :param unsigned: a boolean, optional. @@ -605,6 +670,7 @@ class TINYINT(_IntegerType): """ super(TINYINT, self).__init__(display_width=display_width, **kw) + class SMALLINT(_IntegerType, sqltypes.SMALLINT): """MySQL SMALLINTEGER type.""" @@ -625,6 +691,7 @@ class SMALLINT(_IntegerType, sqltypes.SMALLINT): """ super(SMALLINT, self).__init__(display_width=display_width, **kw) + class BIT(sqltypes.TypeEngine): """MySQL BIT type. @@ -654,34 +721,68 @@ class BIT(sqltypes.TypeEngine): def process(value): if value is not None: - v = 0L + v = 0 for i in map(ord, value): v = v << 8 | i return v return value return process -class _MSTime(sqltypes.Time): - """MySQL TIME type.""" + +class TIME(sqltypes.TIME): + """MySQL TIME type. + + Recent versions of MySQL add support for + fractional seconds precision. While the + :class:`.mysql.TIME` type now supports this, + note that many DBAPI drivers may not yet + include support. + + """ __visit_name__ = 'TIME' + def __init__(self, timezone=False, fsp=None): + """Construct a MySQL TIME type. + + :param timezone: not used by the MySQL dialect. + :param fsp: fractional seconds precision value. + MySQL 5.6 supports storage of fractional seconds; + this parameter will be used when emitting DDL + for the TIME type. Note that many DBAPI drivers + may not yet have support for fractional seconds, + however. + + .. versionadded:: 0.8 The MySQL-specific TIME + type as well as fractional seconds support. + + """ + super(TIME, self).__init__(timezone=timezone) + self.fsp = fsp + def result_processor(self, dialect, coltype): time = datetime.time + def process(value): # convert from a timedelta value if value is not None: + microseconds = value.microseconds seconds = value.seconds - minutes = seconds / 60 - return time(minutes / 60, minutes % 60, seconds - minutes * 60) + minutes = seconds // 60 + return time(minutes // 60, + minutes % 60, + seconds - minutes * 60, + microsecond=microseconds) else: return None return process + class TIMESTAMP(sqltypes.TIMESTAMP): """MySQL TIMESTAMP type.""" __visit_name__ = 'TIMESTAMP' + class YEAR(sqltypes.TypeEngine): """MySQL YEAR type, for single byte storage of years 1901-2155.""" @@ -690,6 +791,7 @@ class YEAR(sqltypes.TypeEngine): def __init__(self, display_width=None): self.display_width = display_width + class TEXT(_StringType, sqltypes.TEXT): """MySQL TEXT type, for text up to 2^16 characters.""" @@ -725,6 +827,7 @@ class TEXT(_StringType, sqltypes.TEXT): """ super(TEXT, self).__init__(length=length, **kw) + class TINYTEXT(_StringType): """MySQL TINYTEXT type, for text up to 2^8 characters.""" @@ -756,6 +859,7 @@ class TINYTEXT(_StringType): """ super(TINYTEXT, self).__init__(**kwargs) + class MEDIUMTEXT(_StringType): """MySQL MEDIUMTEXT type, for text up to 2^24 characters.""" @@ -787,6 +891,7 @@ class MEDIUMTEXT(_StringType): """ super(MEDIUMTEXT, self).__init__(**kwargs) + class LONGTEXT(_StringType): """MySQL LONGTEXT type, for text up to 2^32 characters.""" @@ -850,6 +955,7 @@ class VARCHAR(_StringType, sqltypes.VARCHAR): """ super(VARCHAR, self).__init__(length=length, **kwargs) + class CHAR(_StringType, sqltypes.CHAR): """MySQL CHAR type, for fixed-length character data.""" @@ -870,6 +976,7 @@ class CHAR(_StringType, sqltypes.CHAR): """ super(CHAR, self).__init__(length=length, **kwargs) + class NVARCHAR(_StringType, sqltypes.NVARCHAR): """MySQL NVARCHAR type. @@ -922,24 +1029,66 @@ class NCHAR(_StringType, sqltypes.NCHAR): super(NCHAR, self).__init__(length=length, **kwargs) - - class TINYBLOB(sqltypes._Binary): """MySQL TINYBLOB type, for binary data up to 2^8 bytes.""" __visit_name__ = 'TINYBLOB' + class MEDIUMBLOB(sqltypes._Binary): """MySQL MEDIUMBLOB type, for binary data up to 2^24 bytes.""" __visit_name__ = 'MEDIUMBLOB' + class LONGBLOB(sqltypes._Binary): """MySQL LONGBLOB type, for binary data up to 2^32 bytes.""" __visit_name__ = 'LONGBLOB' -class ENUM(sqltypes.Enum, _StringType): +class _EnumeratedValues(_StringType): + def _init_values(self, values, kw): + self.quoting = kw.pop('quoting', 'auto') + + if self.quoting == 'auto' and len(values): + # What quoting character are we using? + q = None + for e in values: + if len(e) == 0: + self.quoting = 'unquoted' + break + elif q is None: + q = e[0] + + if len(e) == 1 or e[0] != q or e[-1] != q: + self.quoting = 'unquoted' + break + else: + self.quoting = 'quoted' + + if self.quoting == 'quoted': + util.warn_deprecated( + 'Manually quoting %s value literals is deprecated. Supply ' + 'unquoted values and use the quoting= option in cases of ' + 'ambiguity.' % self.__class__.__name__) + + values = self._strip_values(values) + + self._enumerated_values = values + length = max([len(v) for v in values] + [0]) + return values, length + + @classmethod + def _strip_values(cls, values): + strip_values = [] + for a in values: + if a[0:1] == '"' or a[0:1] == "'": + # strip enclosing quotes and unquote interior + a = a[1:-1].replace(a[0] * 2, a[0]) + strip_values.append(a) + return strip_values + +class ENUM(sqltypes.Enum, _EnumeratedValues): """MySQL ENUM type.""" __visit_name__ = 'ENUM' @@ -947,9 +1096,9 @@ class ENUM(sqltypes.Enum, _StringType): def __init__(self, *enums, **kw): """Construct an ENUM. - Example: + E.g.:: - Column('myenum', MSEnum("foo", "bar", "baz")) + Column('myenum', ENUM("foo", "bar", "baz")) :param enums: The range of valid values for this ENUM. Values will be quoted when generating the schema according to the quoting flag (see @@ -993,53 +1142,24 @@ class ENUM(sqltypes.Enum, _StringType): literals for you. This is a transitional option. """ - self.quoting = kw.pop('quoting', 'auto') - - if self.quoting == 'auto' and len(enums): - # What quoting character are we using? - q = None - for e in enums: - if len(e) == 0: - self.quoting = 'unquoted' - break - elif q is None: - q = e[0] - - if e[0] != q or e[-1] != q: - self.quoting = 'unquoted' - break - else: - self.quoting = 'quoted' - - if self.quoting == 'quoted': - util.warn_deprecated( - 'Manually quoting ENUM value literals is deprecated. Supply ' - 'unquoted values and use the quoting= option in cases of ' - 'ambiguity.') - enums = self._strip_enums(enums) - + values, length = self._init_values(enums, kw) self.strict = kw.pop('strict', False) - length = max([len(v) for v in enums] + [0]) kw.pop('metadata', None) kw.pop('schema', None) kw.pop('name', None) kw.pop('quote', None) kw.pop('native_enum', None) + kw.pop('inherit_schema', None) _StringType.__init__(self, length=length, **kw) - sqltypes.Enum.__init__(self, *enums) + sqltypes.Enum.__init__(self, *values) - @classmethod - def _strip_enums(cls, enums): - strip_enums = [] - for a in enums: - if a[0:1] == '"' or a[0:1] == "'": - # strip enclosing quotes and unquote interior - a = a[1:-1].replace(a[0] * 2, a[0]) - strip_enums.append(a) - return strip_enums + def __repr__(self): + return util.generic_repr(self, + to_inspect=[ENUM, _StringType, sqltypes.Enum]) def bind_processor(self, dialect): super_convert = super(ENUM, self).bind_processor(dialect) + def process(value): if self.strict and value is not None and value not in self.enums: raise exc.InvalidRequestError('"%s" not a valid value for ' @@ -1054,7 +1174,8 @@ class ENUM(sqltypes.Enum, _StringType): kw['strict'] = self.strict return sqltypes.Enum.adapt(self, impltype, **kw) -class SET(_StringType): + +class SET(_EnumeratedValues): """MySQL SET type.""" __visit_name__ = 'SET' @@ -1062,15 +1183,16 @@ class SET(_StringType): def __init__(self, *values, **kw): """Construct a SET. - Example:: + E.g.:: - Column('myset', MSSet("'foo'", "'bar'", "'baz'")) + Column('myset', SET("foo", "bar", "baz")) :param values: The range of valid values for this SET. Values will be - used exactly as they appear when generating schemas. Strings must - be quoted, as in the example above. Single-quotes are suggested for - ANSI compatibility and are required for portability to servers with - ANSI_QUOTES enabled. + quoted when generating the schema according to the quoting flag (see + below). + + .. versionchanged:: 0.9.0 quoting is applied automatically to + :class:`.mysql.SET` in the same way as for :class:`.mysql.ENUM`. :param charset: Optional, a column-level character set for this string value. Takes precedence to 'ascii' or 'unicode' short-hand. @@ -1089,18 +1211,27 @@ class SET(_StringType): BINARY in schema. This does not affect the type of data stored, only the collation of character data. + :param quoting: Defaults to 'auto': automatically determine enum value + quoting. If all enum values are surrounded by the same quoting + character, then use 'quoted' mode. Otherwise, use 'unquoted' mode. + + 'quoted': values in enums are already quoted, they will be used + directly when generating the schema - this usage is deprecated. + + 'unquoted': values in enums are not quoted, they will be escaped and + surrounded by single quotes when generating the schema. + + Previous versions of this type always required manually quoted + values to be supplied; future versions will always quote the string + literals for you. This is a transitional option. + + .. versionadded:: 0.9.0 + """ - self._ddl_values = values + values, length = self._init_values(values, kw) + self.values = tuple(values) - strip_values = [] - for a in values: - if a[0:1] == '"' or a[0:1] == "'": - # strip enclosing quotes and unquote interior - a = a[1:-1].replace(a[0] * 2, a[0]) - strip_values.append(a) - - self.values = strip_values - kw.setdefault('length', max([len(v) for v in strip_values] + [0])) + kw.setdefault('length', length) super(SET, self).__init__(**kw) def result_processor(self, dialect, coltype): @@ -1109,13 +1240,10 @@ class SET(_StringType): # No ',' quoting issues- commas aren't allowed in SET values # The bad news: # Plenty of driver inconsistencies here. - if isinstance(value, util.set_types): + if isinstance(value, set): # ..some versions convert '' to an empty set if not value: value.add('') - # ..some return sets.Set, even for pythons that have __builtin__.set - if not isinstance(value, set): - value = set(value) return value # ...and some versions return strings if value is not None: @@ -1126,8 +1254,9 @@ class SET(_StringType): def bind_processor(self, dialect): super_convert = super(SET, self).bind_processor(dialect) + def process(value): - if value is None or isinstance(value, (int, long, basestring)): + if value is None or isinstance(value, util.int_types + util.string_types): pass else: if None in value: @@ -1142,7 +1271,7 @@ class SET(_StringType): return process # old names -MSTime = _MSTime +MSTime = TIME MSSet = SET MSEnum = ENUM MSLongBlob = LONGBLOB @@ -1174,9 +1303,12 @@ MSFloat = FLOAT MSInteger = INTEGER colspecs = { + _IntegerType: _IntegerType, + _NumericType: _NumericType, + _FloatType: _FloatType, sqltypes.Numeric: NUMERIC, sqltypes.Float: FLOAT, - sqltypes.Time: _MSTime, + sqltypes.Time: TIME, sqltypes.Enum: ENUM, } @@ -1218,20 +1350,20 @@ ischema_names = { 'year': YEAR, } + class MySQLExecutionContext(default.DefaultExecutionContext): def should_autocommit_text(self, statement): return AUTOCOMMIT_RE.match(statement) + class MySQLCompiler(compiler.SQLCompiler): render_table_with_column_in_update_from = True """Overridden from base SQLCompiler value""" extract_map = compiler.SQLCompiler.extract_map.copy() - extract_map.update ({ - 'milliseconds': 'millisecond', - }) + extract_map.update({'milliseconds': 'millisecond'}) def visit_random_func(self, fn, **kw): return "rand%s" % self.function_argspec(fn) @@ -1242,11 +1374,13 @@ class MySQLCompiler(compiler.SQLCompiler): def visit_sysdate_func(self, fn, **kw): return "SYSDATE()" - def visit_concat_op(self, binary, **kw): - return "concat(%s, %s)" % (self.process(binary.left), self.process(binary.right)) + def visit_concat_op_binary(self, binary, operator, **kw): + return "concat(%s, %s)" % (self.process(binary.left), + self.process(binary.right)) - def visit_match_op(self, binary, **kw): - return "MATCH (%s) AGAINST (%s IN BOOLEAN MODE)" % (self.process(binary.left), self.process(binary.right)) + def visit_match_op_binary(self, binary, operator, **kw): + return "MATCH (%s) AGAINST (%s IN BOOLEAN MODE)" % \ + (self.process(binary.left), self.process(binary.right)) def get_from_hint_text(self, table, text): return text @@ -1260,7 +1394,8 @@ class MySQLCompiler(compiler.SQLCompiler): return 'SIGNED INTEGER' elif isinstance(type_, sqltypes.TIMESTAMP): return 'DATETIME' - elif isinstance(type_, (sqltypes.DECIMAL, sqltypes.DateTime, sqltypes.Date, sqltypes.Time)): + elif isinstance(type_, (sqltypes.DECIMAL, sqltypes.DateTime, + sqltypes.Date, sqltypes.Time)): return self.dialect.type_compiler.process(type_) elif isinstance(type_, sqltypes.Text): return 'CHAR' @@ -1273,7 +1408,8 @@ class MySQLCompiler(compiler.SQLCompiler): elif isinstance(type_, sqltypes._Binary): return 'BINARY' elif isinstance(type_, sqltypes.NUMERIC): - return self.dialect.type_compiler.process(type_).replace('NUMERIC', 'DECIMAL') + return self.dialect.type_compiler.process( + type_).replace('NUMERIC', 'DECIMAL') else: return None @@ -1304,7 +1440,7 @@ class MySQLCompiler(compiler.SQLCompiler): of a SELECT. """ - if isinstance(select._distinct, basestring): + if isinstance(select._distinct, util.string_types): return select._distinct.upper() + " " elif select._distinct: return "DISTINCT " @@ -1312,11 +1448,6 @@ class MySQLCompiler(compiler.SQLCompiler): return "" def visit_join(self, join, asfrom=False, **kwargs): - # 'JOIN ... ON ...' for inner joins isn't available until 4.0. - # Apparently < 3.23.17 requires theta joins for inner joins - # (but not outer). Not generating these currently, but - # support can be added, preferably after dialects are - # refactored to be version-sensitive. return ''.join( (self.process(join.left, asfrom=True, **kwargs), (join.isouter and " LEFT OUTER JOIN " or " INNER JOIN "), @@ -1325,10 +1456,10 @@ class MySQLCompiler(compiler.SQLCompiler): self.process(join.onclause, **kwargs))) def for_update_clause(self, select): - if select.for_update == 'read': - return ' LOCK IN SHARE MODE' + if select._for_update_arg.read: + return " LOCK IN SHARE MODE" else: - return super(MySQLCompiler, self).for_update_clause(select) + return " FOR UPDATE" def limit_clause(self, select): # MySQL supports: @@ -1389,10 +1520,11 @@ class MySQLCompiler(compiler.SQLCompiler): class MySQLDDLCompiler(compiler.DDLCompiler): def create_table_constraints(self, table): """Get table constraints.""" - constraint_string = super(MySQLDDLCompiler, self).create_table_constraints(table) + constraint_string = super( + MySQLDDLCompiler, self).create_table_constraints(table) engine_key = '%s_engine' % self.dialect.name - is_innodb = table.kwargs.has_key(engine_key) and \ + is_innodb = engine_key in table.kwargs and \ table.kwargs[engine_key].lower() == 'innodb' auto_inc_column = table._autoincrement_column @@ -1404,14 +1536,13 @@ class MySQLDDLCompiler(compiler.DDLCompiler): constraint_string += ", \n\t" constraint_string += "KEY %s (%s)" % ( self.preparer.quote( - "idx_autoinc_%s" % auto_inc_column.name, None + "idx_autoinc_%s" % auto_inc_column.name ), self.preparer.format_column(auto_inc_column) ) return constraint_string - def get_column_specification(self, column, **kw): """Builds column DDL.""" @@ -1430,7 +1561,8 @@ class MySQLDDLCompiler(compiler.DDLCompiler): elif column.nullable and is_timestamp and default is None: colspec.append('NULL') - if column is column.table._autoincrement_column and column.server_default is None: + if column is column.table._autoincrement_column and \ + column.server_default is None: colspec.append('AUTO_INCREMENT') return ' '.join(colspec) @@ -1442,7 +1574,7 @@ class MySQLDDLCompiler(compiler.DDLCompiler): opts = dict( ( - k[len(self.dialect.name)+1:].upper(), + k[len(self.dialect.name) + 1:].upper(), v ) for k, v in table.kwargs.items() @@ -1471,31 +1603,47 @@ class MySQLDDLCompiler(compiler.DDLCompiler): table_opts.append(joiner.join((opt, arg))) return ' '.join(table_opts) - def visit_create_index(self, create): index = create.element + self._verify_index_table(index) preparer = self.preparer table = preparer.format_table(index.table) - columns = [preparer.quote(c.name, c.quote) for c in index.columns] - name = preparer.quote( - self._index_identifier(index.name), - index.quote) + columns = [self.sql_compiler.process(expr, include_table=False, + literal_binds=True) + for expr in index.expressions] + + name = self._prepared_index_name(index) text = "CREATE " if index.unique: text += "UNIQUE " text += "INDEX %s ON %s " % (name, table) - columns = ', '.join(columns) if 'mysql_length' in index.kwargs: length = index.kwargs['mysql_length'] - text += "(%s(%d))" % (columns, length) + + if isinstance(length, dict): + # length value can be a (column_name --> integer value) mapping + # specifying the prefix length for each column of the index + columns = ', '.join( + ('%s(%d)' % (col, length[col]) + if col in length else '%s' % col) + for col in columns + ) + else: + # or can be an integer value specifying the same + # prefix length for all columns of the index + columns = ', '.join( + '%s(%d)' % (col, length) + for col in columns + ) else: - text += "(%s)" % (columns) + columns = ', '.join(columns) + text += '(%s)' % columns if 'mysql_using' in index.kwargs: using = index.kwargs['mysql_using'] - text += " USING %s" % (preparer.quote(using, index.quote)) + text += " USING %s" % (preparer.quote(using)) return text @@ -1504,17 +1652,15 @@ class MySQLDDLCompiler(compiler.DDLCompiler): visit_primary_key_constraint(constraint) if "mysql_using" in constraint.kwargs: using = constraint.kwargs['mysql_using'] - text += " USING %s" % ( - self.preparer.quote(using, constraint.quote)) + text += " USING %s" % (self.preparer.quote(using)) return text def visit_drop_index(self, drop): index = drop.element - return "\nDROP INDEX %s ON %s" % \ - (self.preparer.quote( - self._index_identifier(index.name), index.quote - ), + return "\nDROP INDEX %s ON %s" % ( + self._prepared_index_name(index, + include_schema=False), self.preparer.format_table(index.table)) def visit_drop_constraint(self, drop): @@ -1535,6 +1681,13 @@ class MySQLDDLCompiler(compiler.DDLCompiler): (self.preparer.format_table(constraint.table), qual, const) + def define_constraint_match(self, constraint): + if constraint.match is not None: + raise exc.CompileError( + "MySQL ignores the 'MATCH' keyword while at the same time " + "causes ON UPDATE/ON DELETE clauses to be ignored.") + return "" + class MySQLTypeCompiler(compiler.GenericTypeCompiler): def _extend_numeric(self, type_, spec): "Extend a numeric-type declaration with MySQL specific extensions." @@ -1593,7 +1746,8 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "NUMERIC(%(precision)s, %(scale)s)" % - {'precision': type_.precision, 'scale' : type_.scale}) + {'precision': type_.precision, + 'scale': type_.scale}) def visit_DECIMAL(self, type_): if type_.precision is None: @@ -1605,21 +1759,24 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): else: return self._extend_numeric(type_, "DECIMAL(%(precision)s, %(scale)s)" % - {'precision': type_.precision, 'scale' : type_.scale}) + {'precision': type_.precision, + 'scale': type_.scale}) def visit_DOUBLE(self, type_): if type_.precision is not None and type_.scale is not None: - return self._extend_numeric(type_, "DOUBLE(%(precision)s, %(scale)s)" % + return self._extend_numeric(type_, + "DOUBLE(%(precision)s, %(scale)s)" % {'precision': type_.precision, - 'scale' : type_.scale}) + 'scale': type_.scale}) else: return self._extend_numeric(type_, 'DOUBLE') def visit_REAL(self, type_): if type_.precision is not None and type_.scale is not None: - return self._extend_numeric(type_, "REAL(%(precision)s, %(scale)s)" % + return self._extend_numeric(type_, + "REAL(%(precision)s, %(scale)s)" % {'precision': type_.precision, - 'scale' : type_.scale}) + 'scale': type_.scale}) else: return self._extend_numeric(type_, 'REAL') @@ -1630,7 +1787,8 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): return self._extend_numeric(type_, "FLOAT(%s, %s)" % (type_.precision, type_.scale)) elif type_.precision is not None: - return self._extend_numeric(type_, "FLOAT(%s)" % (type_.precision,)) + return self._extend_numeric(type_, + "FLOAT(%s)" % (type_.precision,)) else: return self._extend_numeric(type_, "FLOAT") @@ -1660,7 +1818,8 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): def visit_TINYINT(self, type_): if self._mysql_type(type_) and type_.display_width is not None: - return self._extend_numeric(type_, "TINYINT(%s)" % type_.display_width) + return self._extend_numeric(type_, + "TINYINT(%s)" % type_.display_width) else: return self._extend_numeric(type_, "TINYINT") @@ -1686,7 +1845,10 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): return "DATE" def visit_TIME(self, type_): - return "TIME" + if getattr(type_, 'fsp', None): + return "TIME(%d)" % type_.fsp + else: + return "TIME" def visit_TIMESTAMP(self, type_): return 'TIMESTAMP' @@ -1722,7 +1884,8 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): def visit_CHAR(self, type_): if type_.length: - return self._extend_string(type_, {}, "CHAR(%(length)s)" % {'length' : type_.length}) + return self._extend_string(type_, {}, "CHAR(%(length)s)" % + {'length': type_.length}) else: return self._extend_string(type_, {}, "CHAR") @@ -1730,18 +1893,21 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): # We'll actually generate the equiv. "NATIONAL VARCHAR" instead # of "NVARCHAR". if type_.length: - return self._extend_string(type_, {'national':True}, "VARCHAR(%(length)s)" % {'length': type_.length}) + return self._extend_string(type_, {'national': True}, + "VARCHAR(%(length)s)" % {'length': type_.length}) else: raise exc.CompileError( "NVARCHAR requires a length on dialect %s" % self.dialect.name) def visit_NCHAR(self, type_): - # We'll actually generate the equiv. "NATIONAL CHAR" instead of "NCHAR". + # We'll actually generate the equiv. + # "NATIONAL CHAR" instead of "NCHAR". if type_.length: - return self._extend_string(type_, {'national':True}, "CHAR(%(length)s)" % {'length': type_.length}) + return self._extend_string(type_, {'national': True}, + "CHAR(%(length)s)" % {'length': type_.length}) else: - return self._extend_string(type_, {'national':True}, "CHAR") + return self._extend_string(type_, {'national': True}, "CHAR") def visit_VARBINARY(self, type_): return "VARBINARY(%d)" % type_.length @@ -1753,7 +1919,7 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): if not type_.native_enum: return super(MySQLTypeCompiler, self).visit_enum(type_) else: - return self.visit_ENUM(type_) + return self._visit_enumerated_values("ENUM", type_, type_.enums) def visit_BLOB(self, type_): if type_.length: @@ -1770,14 +1936,21 @@ class MySQLTypeCompiler(compiler.GenericTypeCompiler): def visit_LONGBLOB(self, type_): return "LONGBLOB" - def visit_ENUM(self, type_): + def _visit_enumerated_values(self, name, type_, enumerated_values): quoted_enums = [] - for e in type_.enums: + for e in enumerated_values: quoted_enums.append("'%s'" % e.replace("'", "''")) - return self._extend_string(type_, {}, "ENUM(%s)" % ",".join(quoted_enums)) + return self._extend_string(type_, {}, "%s(%s)" % ( + name, ",".join(quoted_enums)) + ) + + def visit_ENUM(self, type_): + return self._visit_enumerated_values("ENUM", type_, + type_._enumerated_values) def visit_SET(self, type_): - return self._extend_string(type_, {}, "SET(%s)" % ",".join(type_._ddl_values)) + return self._visit_enumerated_values("SET", type_, + type_._enumerated_values) def visit_BOOLEAN(self, type): return "BOOL" @@ -1803,6 +1976,8 @@ class MySQLIdentifierPreparer(compiler.IdentifierPreparer): return tuple([self.quote_identifier(i) for i in ids if i is not None]) + +@log.class_logger class MySQLDialect(default.DefaultDialect): """Details of the MySQL dialect. Not used directly in application code.""" @@ -1817,6 +1992,7 @@ class MySQLDialect(default.DefaultDialect): supports_sane_rowcount = True supports_sane_multi_rowcount = False + supports_multivalues_insert = True default_paramstyle = 'format' colspecs = colspecs @@ -1833,7 +2009,8 @@ class MySQLDialect(default.DefaultDialect): _backslash_escapes = True _server_ansiquotes = False - def __init__(self, use_ansiquotes=None, isolation_level=None, **kwargs): + def __init__(self, isolation_level=None, **kwargs): + kwargs.pop('use_ansiquotes', None) # legacy default.DefaultDialect.__init__(self, **kwargs) self.isolation_level = isolation_level @@ -1866,9 +2043,11 @@ class MySQLDialect(default.DefaultDialect): cursor.execute('SELECT @@tx_isolation') val = cursor.fetchone()[0] cursor.close() + if util.py3k and isinstance(val, bytes): + val = val.decode() return val.upper().replace("-", " ") - def do_commit(self, connection): + def do_commit(self, dbapi_connection): """Execute a COMMIT.""" # COMMIT/ROLLBACK were introduced in 3.23.15. @@ -1877,7 +2056,7 @@ class MySQLDialect(default.DefaultDialect): # Ignore commit/rollback if support isn't present, otherwise even basic # operations via autocommit fail. try: - connection.commit() + dbapi_connection.commit() except: if self.server_version_info < (3, 23, 15): args = sys.exc_info()[1].args @@ -1885,11 +2064,11 @@ class MySQLDialect(default.DefaultDialect): return raise - def do_rollback(self, connection): + def do_rollback(self, dbapi_connection): """Execute a ROLLBACK.""" try: - connection.rollback() + dbapi_connection.rollback() except: if self.server_version_info < (3, 23, 15): args = sys.exc_info()[1].args @@ -1932,17 +2111,20 @@ class MySQLDialect(default.DefaultDialect): return False def _compat_fetchall(self, rp, charset=None): - """Proxy result rows to smooth over MySQL-Python driver inconsistencies.""" + """Proxy result rows to smooth over MySQL-Python driver + inconsistencies.""" return [_DecodingRowProxy(row, charset) for row in rp.fetchall()] def _compat_fetchone(self, rp, charset=None): - """Proxy a result row to smooth over MySQL-Python driver inconsistencies.""" + """Proxy a result row to smooth over MySQL-Python driver + inconsistencies.""" return _DecodingRowProxy(rp.fetchone(), charset) def _compat_first(self, rp, charset=None): - """Proxy a result row to smooth over MySQL-Python driver inconsistencies.""" + """Proxy a result row to smooth over MySQL-Python driver + inconsistencies.""" return _DecodingRowProxy(rp.first(), charset) @@ -1952,7 +2134,6 @@ class MySQLDialect(default.DefaultDialect): def _get_default_schema_name(self, connection): return connection.execute('SELECT DATABASE()').scalar() - def has_table(self, connection, table_name, schema=None): # SHOW TABLE STATUS LIKE and SHOW TABLES LIKE do not function properly # on macosx (and maybe win?) with multibyte table names. @@ -1964,7 +2145,6 @@ class MySQLDialect(default.DefaultDialect): # full_name = self.identifier_preparer.format_table(table, # use_schema=True) - full_name = '.'.join(self.identifier_preparer._quote_free_identifiers( schema, table_name)) @@ -1973,10 +2153,10 @@ class MySQLDialect(default.DefaultDialect): try: try: rs = connection.execute(st) - have = rs.rowcount > 0 + have = rs.fetchone() is not None rs.close() return have - except exc.DBAPIError, e: + except exc.DBAPIError as e: if self._extract_error_code(e.orig) == 1146: return False raise @@ -1987,14 +2167,12 @@ class MySQLDialect(default.DefaultDialect): def initialize(self, connection): default.DefaultDialect.initialize(self, connection) self._connection_charset = self._detect_charset(connection) - self._server_casing = self._detect_casing(connection) - self._server_collations = self._detect_collations(connection) self._detect_ansiquotes(connection) if self._server_ansiquotes: # if ansiquotes == True, build a new IdentifierPreparer # with the new setting self.identifier_preparer = self.preparer(self, - server_ansiquotes=self._server_ansiquotes) + server_ansiquotes=self._server_ansiquotes) @property def _supports_cast(self): @@ -2018,13 +2196,15 @@ class MySQLDialect(default.DefaultDialect): if self.server_version_info < (5, 0, 2): rp = connection.execute("SHOW TABLES FROM %s" % self.identifier_preparer.quote_identifier(current_schema)) - return [row[0] for row in self._compat_fetchall(rp, charset=charset)] + return [row[0] for + row in self._compat_fetchall(rp, charset=charset)] else: rp = connection.execute("SHOW FULL TABLES FROM %s" % self.identifier_preparer.quote_identifier(current_schema)) - return [row[0] for row in self._compat_fetchall(rp, charset=charset)\ - if row[1] == 'BASE TABLE'] + return [row[0] + for row in self._compat_fetchall(rp, charset=charset) + if row[1] == 'BASE TABLE'] @reflection.cache def get_view_names(self, connection, schema=None, **kw): @@ -2037,34 +2217,39 @@ class MySQLDialect(default.DefaultDialect): charset = self._connection_charset rp = connection.execute("SHOW FULL TABLES FROM %s" % self.identifier_preparer.quote_identifier(schema)) - return [row[0] for row in self._compat_fetchall(rp, charset=charset)\ - if row[1] in ('VIEW', 'SYSTEM VIEW')] + return [row[0] + for row in self._compat_fetchall(rp, charset=charset) + if row[1] in ('VIEW', 'SYSTEM VIEW')] @reflection.cache def get_table_options(self, connection, table_name, schema=None, **kw): - parsed_state = self._parsed_state_or_create(connection, table_name, schema, **kw) + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) return parsed_state.table_options @reflection.cache def get_columns(self, connection, table_name, schema=None, **kw): - parsed_state = self._parsed_state_or_create(connection, table_name, schema, **kw) + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) return parsed_state.columns @reflection.cache - def get_primary_keys(self, connection, table_name, schema=None, **kw): - parsed_state = self._parsed_state_or_create(connection, table_name, schema, **kw) + def get_pk_constraint(self, connection, table_name, schema=None, **kw): + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) for key in parsed_state.keys: if key['type'] == 'PRIMARY': # There can be only one. - ##raise Exception, str(key) - return [s[0] for s in key['columns']] - return [] + cols = [s[0] for s in key['columns']] + return {'constrained_columns': cols, 'name': None} + return {'constrained_columns': [], 'name': None} @reflection.cache def get_foreign_keys(self, connection, table_name, schema=None, **kw): - parsed_state = self._parsed_state_or_create(connection, table_name, schema, **kw) + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) default_schema = None fkeys = [] @@ -2085,17 +2270,17 @@ class MySQLDialect(default.DefaultDialect): ref_names = spec['foreign'] con_kw = {} - for opt in ('name', 'onupdate', 'ondelete'): + for opt in ('onupdate', 'ondelete'): if spec.get(opt, False): con_kw[opt] = spec[opt] fkey_d = { - 'name' : spec['name'], - 'constrained_columns' : loc_names, - 'referred_schema' : ref_schema, - 'referred_table' : ref_name, - 'referred_columns' : ref_names, - 'options' : con_kw + 'name': spec['name'], + 'constrained_columns': loc_names, + 'referred_schema': ref_schema, + 'referred_table': ref_name, + 'referred_columns': ref_names, + 'options': con_kw } fkeys.append(fkey_d) return fkeys @@ -2103,7 +2288,8 @@ class MySQLDialect(default.DefaultDialect): @reflection.cache def get_indexes(self, connection, table_name, schema=None, **kw): - parsed_state = self._parsed_state_or_create(connection, table_name, schema, **kw) + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) indexes = [] for spec in parsed_state.keys: @@ -2127,6 +2313,21 @@ class MySQLDialect(default.DefaultDialect): indexes.append(index_d) return indexes + @reflection.cache + def get_unique_constraints(self, connection, table_name, + schema=None, **kw): + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) + + return [ + { + 'name': key['name'], + 'column_names': [col[0] for col in key['columns']] + } + for key in parsed_state.keys + if key['type'] == 'UNIQUE' + ] + @reflection.cache def get_view_definition(self, connection, view_name, schema=None, **kw): @@ -2137,7 +2338,8 @@ class MySQLDialect(default.DefaultDialect): full_name=full_name) return sql - def _parsed_state_or_create(self, connection, table_name, schema=None, **kw): + def _parsed_state_or_create(self, connection, table_name, + schema=None, **kw): return self._setup_parser( connection, table_name, @@ -2225,7 +2427,7 @@ class MySQLDialect(default.DefaultDialect): row = self._compat_first( connection.execute("SHOW VARIABLES LIKE 'sql_mode'"), - charset=self._connection_charset) + charset=self._connection_charset) if not row: mode = '' @@ -2252,7 +2454,7 @@ class MySQLDialect(default.DefaultDialect): rp = None try: rp = connection.execute(st) - except exc.DBAPIError, e: + except exc.DBAPIError as e: if self._extract_error_code(e.orig) == 1146: raise exc.NoSuchTableError(full_name) else: @@ -2276,7 +2478,7 @@ class MySQLDialect(default.DefaultDialect): try: try: rp = connection.execute(st) - except exc.DBAPIError, e: + except exc.DBAPIError as e: if self._extract_error_code(e.orig) == 1146: raise exc.NoSuchTableError(full_name) else: @@ -2287,6 +2489,7 @@ class MySQLDialect(default.DefaultDialect): rp.close() return rows + class ReflectedState(object): """Stores raw information about a SHOW CREATE TABLE statement.""" @@ -2297,6 +2500,8 @@ class ReflectedState(object): self.keys = [] self.constraints = [] + +@log.class_logger class MySQLTableDefinitionParser(object): """Parses the results of a SHOW CREATE TABLE statement.""" @@ -2319,7 +2524,8 @@ class MySQLTableDefinitionParser(object): pass elif line.startswith('CREATE '): self._parse_table_name(line, state) - # Not present in real reflection, but may be if loading from a file. + # Not present in real reflection, but may be if + # loading from a file. elif not line: pass else: @@ -2332,7 +2538,6 @@ class MySQLTableDefinitionParser(object): state.constraints.append(spec) else: pass - return state def _parse_constraints(self, line): @@ -2461,8 +2666,8 @@ class MySQLTableDefinitionParser(object): if spec.get(kw, False): type_kw[kw] = spec[kw] - if type_ == 'enum': - type_args = ENUM._strip_enums(type_args) + if issubclass(col_type, _EnumeratedValues): + type_args = _EnumeratedValues._strip_values(type_args) type_instance = col_type(*type_args, **type_kw) @@ -2573,7 +2778,6 @@ class MySQLTableDefinitionParser(object): # 123 or 123,456 self._re_csv_int = _re_compile(r'\d+') - # `colname` [type opts] # (NOT NULL | NULL) # DEFAULT ('value' | CURRENT_TIMESTAMP...) @@ -2637,7 +2841,7 @@ class MySQLTableDefinitionParser(object): # # unique constraints come back as KEYs kw = quotes.copy() - kw['on'] = 'RESTRICT|CASCASDE|SET NULL|NOACTION' + kw['on'] = 'RESTRICT|CASCADE|SET NULL|NOACTION' self._re_constraint = _re_compile( r' ' r'CONSTRAINT +' @@ -2658,7 +2862,8 @@ class MySQLTableDefinitionParser(object): self._re_partition = _re_compile(r'(?:.*)(?:SUB)?PARTITION(?:.*)') # Table-level options (COLLATE, ENGINE, etc.) - # Do the string options first, since they have quoted strings we need to get rid of. + # Do the string options first, since they have quoted + # strings we need to get rid of. for option in _options_of_type_string: self._add_option_string(option) @@ -2681,8 +2886,8 @@ class MySQLTableDefinitionParser(object): regex = (r'(?P%s)%s' r"'(?P(?:[^']|'')*?)'(?!')" % (re.escape(directive), self._optional_equals)) - self._pr_options.append( - _pr_compile(regex, lambda v: v.replace("\\\\","\\").replace("''", "'"))) + self._pr_options.append(_pr_compile(regex, lambda v: + v.replace("\\\\", "\\").replace("''", "'"))) def _add_option_word(self, directive): regex = (r'(?P%s)%s' @@ -2699,8 +2904,6 @@ class MySQLTableDefinitionParser(object): _options_of_type_string = ('COMMENT', 'DATA DIRECTORY', 'INDEX DIRECTORY', 'PASSWORD', 'CONNECTION') -log.class_logger(MySQLTableDefinitionParser) -log.class_logger(MySQLDialect) class _DecodingRowProxy(object): @@ -2724,11 +2927,8 @@ class _DecodingRowProxy(object): item = self.rowproxy[index] if isinstance(item, _array): item = item.tostring() - # Py2K - if self.charset and isinstance(item, str): - # end Py2K - # Py3K - #if self.charset and isinstance(item, bytes): + + if self.charset and isinstance(item, util.binary_type): return item.decode(self.charset) else: return item @@ -2737,11 +2937,7 @@ class _DecodingRowProxy(object): item = getattr(self.rowproxy, attr) if isinstance(item, _array): item = item.tostring() - # Py2K - if self.charset and isinstance(item, str): - # end Py2K - # Py3K - #if self.charset and isinstance(item, bytes): + if self.charset and isinstance(item, util.binary_type): return item.decode(self.charset) else: return item @@ -2752,8 +2948,8 @@ def _pr_compile(regex, cleanup=None): return (_re_compile(regex), cleanup) + def _re_compile(regex): """Compile a string to regex, I and UNICODE.""" return re.compile(regex, re.I | re.UNICODE) - diff --git a/libs/sqlalchemy/dialects/mysql/cymysql.py b/libs/sqlalchemy/dialects/mysql/cymysql.py new file mode 100644 index 00000000..e81a79b8 --- /dev/null +++ b/libs/sqlalchemy/dialects/mysql/cymysql.py @@ -0,0 +1,69 @@ +# mysql/cymysql.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" + +.. dialect:: mysql+cymysql + :name: CyMySQL + :dbapi: cymysql + :connectstring: mysql+cymysql://:@/[?] + :url: https://github.com/nakagami/CyMySQL + +""" + +from .mysqldb import MySQLDialect_mysqldb +from .base import (BIT, MySQLDialect) +from ... import util + +class _cymysqlBIT(BIT): + def result_processor(self, dialect, coltype): + """Convert a MySQL's 64 bit, variable length binary string to a long. + """ + + def process(value): + if value is not None: + v = 0 + for i in util.iterbytes(value): + v = v << 8 | i + return v + return value + return process + + +class MySQLDialect_cymysql(MySQLDialect_mysqldb): + driver = 'cymysql' + + description_encoding = None + supports_sane_rowcount = True + supports_sane_multi_rowcount = False + supports_unicode_statements = True + + colspecs = util.update_copy( + MySQLDialect.colspecs, + { + BIT: _cymysqlBIT, + } + ) + + @classmethod + def dbapi(cls): + return __import__('cymysql') + + def _extract_error_code(self, exception): + return exception.errno + + def is_disconnect(self, e, connection, cursor): + if isinstance(e, self.dbapi.OperationalError): + return self._extract_error_code(e) in \ + (2006, 2013, 2014, 2045, 2055) + elif isinstance(e, self.dbapi.InterfaceError): + # if underlying connection is closed, + # this is the error you get + return True + else: + return False + +dialect = MySQLDialect_cymysql diff --git a/libs/sqlalchemy/dialects/mysql/gaerdbms.py b/libs/sqlalchemy/dialects/mysql/gaerdbms.py index 2203504f..13203fce 100644 --- a/libs/sqlalchemy/dialects/mysql/gaerdbms.py +++ b/libs/sqlalchemy/dialects/mysql/gaerdbms.py @@ -1,26 +1,20 @@ # mysql/gaerdbms.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for Google Cloud SQL on Google App Engine. +""" +.. dialect:: mysql+gaerdbms + :name: Google Cloud SQL + :dbapi: rdbms + :connectstring: mysql+gaerdbms:///?instance= + :url: https://developers.google.com/appengine/docs/python/cloud-sql/developers-guide -This dialect is based primarily on the :mod:`.mysql.mysqldb` dialect with minimal -changes. + This dialect is based primarily on the :mod:`.mysql.mysqldb` dialect with minimal + changes. -.. versionadded:: 0.7.8 + .. versionadded:: 0.7.8 -Connecting ----------- - -Connect string format:: - - mysql+gaerdbms:/// - -E.g.:: - - create_engine('mysql+gaerdbms:///mydb', - connect_args={"instance":"instancename"}) Pooling ------- @@ -32,11 +26,17 @@ default. """ -from sqlalchemy.dialects.mysql.mysqldb import MySQLDialect_mysqldb -from sqlalchemy.pool import NullPool +import os + +from .mysqldb import MySQLDialect_mysqldb +from ...pool import NullPool import re +def _is_dev_environment(): + return os.environ.get('SERVER_SOFTWARE', '').startswith('Development/') + + class MySQLDialect_gaerdbms(MySQLDialect_mysqldb): @classmethod @@ -49,7 +49,10 @@ class MySQLDialect_gaerdbms(MySQLDialect_mysqldb): # see also http://stackoverflow.com/q/14224679/34549 from google.appengine.api import apiproxy_stub_map - if apiproxy_stub_map.apiproxy.GetStub('rdbms'): + if _is_dev_environment(): + from google.appengine.api import rdbms_mysqldb + return rdbms_mysqldb + elif apiproxy_stub_map.apiproxy.GetStub('rdbms'): from google.storage.speckle.python.api import rdbms_apiproxy return rdbms_apiproxy else: @@ -63,21 +66,18 @@ class MySQLDialect_gaerdbms(MySQLDialect_mysqldb): def create_connect_args(self, url): opts = url.translate_connect_args() - # 'dsn' and 'instance' are because we are skipping - # the traditional google.api.rdbms wrapper - - opts['dsn'] = '' - opts['instance'] = url.query['instance'] + if not _is_dev_environment(): + # 'dsn' and 'instance' are because we are skipping + # the traditional google.api.rdbms wrapper + opts['dsn'] = '' + opts['instance'] = url.query['instance'] return [], opts def _extract_error_code(self, exception): - match = re.compile(r"^(\d+):").match(str(exception)) + match = re.compile(r"^(\d+)L?:|^\((\d+)L?,").match(str(exception)) # The rdbms api will wrap then re-raise some types of errors # making this regex return no matches. - if match: - code = match.group(1) - else: - code = None + code = match.group(1) or match.group(2) if match else None if code: return int(code) diff --git a/libs/sqlalchemy/dialects/mysql/mysqlconnector.py b/libs/sqlalchemy/dialects/mysql/mysqlconnector.py index bd8ee013..b6e7c75f 100644 --- a/libs/sqlalchemy/dialects/mysql/mysqlconnector.py +++ b/libs/sqlalchemy/dialects/mysql/mysqlconnector.py @@ -1,34 +1,25 @@ # mysql/mysqlconnector.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database via the MySQL Connector/Python adapter. +""" +.. dialect:: mysql+mysqlconnector + :name: MySQL Connector/Python + :dbapi: myconnpy + :connectstring: mysql+mysqlconnector://:@[:]/ + :url: https://launchpad.net/myconnpy -MySQL Connector/Python is available at: - - https://launchpad.net/myconnpy - -Connecting ------------ - -Connect string format:: - - mysql+mysqlconnector://:@[:]/ """ -import re - -from sqlalchemy.dialects.mysql.base import (MySQLDialect, +from .base import (MySQLDialect, MySQLExecutionContext, MySQLCompiler, MySQLIdentifierPreparer, BIT) -from sqlalchemy.engine import base as engine_base, default -from sqlalchemy.sql import operators as sql_operators -from sqlalchemy import exc, log, schema, sql, types as sqltypes, util -from sqlalchemy import processors +from ... import util + class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext): @@ -37,24 +28,28 @@ class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext): class MySQLCompiler_mysqlconnector(MySQLCompiler): - def visit_mod(self, binary, **kw): - return self.process(binary.left) + " %% " + self.process(binary.right) + def visit_mod_binary(self, binary, operator, **kw): + return self.process(binary.left, **kw) + " %% " + \ + self.process(binary.right, **kw) def post_process_text(self, text): return text.replace('%', '%%') + class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer): def _escape_identifier(self, value): value = value.replace(self.escape_quote, self.escape_to_quote) return value.replace("%", "%%") + class _myconnpyBIT(BIT): def result_processor(self, dialect, coltype): """MySQL-connector already converts mysql bits, so.""" return None + class MySQLDialect_mysqlconnector(MySQLDialect): driver = 'mysqlconnector' supports_unicode_statements = True @@ -84,12 +79,13 @@ class MySQLDialect_mysqlconnector(MySQLDialect): def create_connect_args(self, url): opts = url.translate_connect_args(username='user') + opts.update(url.query) util.coerce_kw_type(opts, 'buffered', bool) util.coerce_kw_type(opts, 'raise_on_warnings', bool) - opts['buffered'] = True - opts['raise_on_warnings'] = True + opts.setdefault('buffered', True) + opts.setdefault('raise_on_warnings', True) # FOUND_ROWS must be set in ClientFlag to enable # supports_sane_rowcount. @@ -116,9 +112,10 @@ class MySQLDialect_mysqlconnector(MySQLDialect): def is_disconnect(self, e, connection, cursor): errnos = (2006, 2013, 2014, 2045, 2055, 2048) - exceptions = (self.dbapi.OperationalError,self.dbapi.InterfaceError) + exceptions = (self.dbapi.OperationalError, self.dbapi.InterfaceError) if isinstance(e, exceptions): - return e.errno in errnos + return e.errno in errnos or \ + "MySQL Connection not available." in str(e) else: return False diff --git a/libs/sqlalchemy/dialects/mysql/mysqldb.py b/libs/sqlalchemy/dialects/mysql/mysqldb.py index c6ae5333..c6942ae2 100644 --- a/libs/sqlalchemy/dialects/mysql/mysqldb.py +++ b/libs/sqlalchemy/dialects/mysql/mysqldb.py @@ -1,23 +1,17 @@ # mysql/mysqldb.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database via the MySQL-python adapter. +""" -MySQL-Python is available at: +.. dialect:: mysql+mysqldb + :name: MySQL-Python + :dbapi: mysqldb + :connectstring: mysql+mysqldb://:@[:]/ + :url: http://sourceforge.net/projects/mysql-python - http://sourceforge.net/projects/mysql-python - -At least version 1.2.1 or 1.2.2 should be used. - -Connecting ------------ - -Connect string format:: - - mysql+mysqldb://:@[:]/ Unicode ------- @@ -54,15 +48,16 @@ It is strongly advised to use the latest version of MySQL-Python. """ -from sqlalchemy.dialects.mysql.base import (MySQLDialect, MySQLExecutionContext, +from .base import (MySQLDialect, MySQLExecutionContext, MySQLCompiler, MySQLIdentifierPreparer) -from sqlalchemy.connectors.mysqldb import ( +from ...connectors.mysqldb import ( MySQLDBExecutionContext, MySQLDBCompiler, MySQLDBIdentifierPreparer, MySQLDBConnector ) + class MySQLExecutionContext_mysqldb(MySQLDBExecutionContext, MySQLExecutionContext): pass @@ -74,6 +69,7 @@ class MySQLCompiler_mysqldb(MySQLDBCompiler, MySQLCompiler): class MySQLIdentifierPreparer_mysqldb(MySQLDBIdentifierPreparer, MySQLIdentifierPreparer): pass + class MySQLDialect_mysqldb(MySQLDBConnector, MySQLDialect): execution_ctx_cls = MySQLExecutionContext_mysqldb statement_compiler = MySQLCompiler_mysqldb diff --git a/libs/sqlalchemy/dialects/mysql/oursql.py b/libs/sqlalchemy/dialects/mysql/oursql.py index d6d8e9ff..e6b50f33 100644 --- a/libs/sqlalchemy/dialects/mysql/oursql.py +++ b/libs/sqlalchemy/dialects/mysql/oursql.py @@ -1,21 +1,16 @@ # mysql/oursql.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database via the oursql adapter. +""" -OurSQL is available at: - - http://packages.python.org/oursql/ - -Connecting ------------ - -Connect string format:: - - mysql+oursql://:@[:]/ +.. dialect:: mysql+oursql + :name: OurSQL + :dbapi: oursql + :connectstring: mysql+oursql://:@[:]/ + :url: http://packages.python.org/oursql/ Unicode ------- @@ -40,13 +35,8 @@ defaults to, there is a separate parameter:: import re -from sqlalchemy.dialects.mysql.base import (BIT, MySQLDialect, MySQLExecutionContext, - MySQLCompiler, MySQLIdentifierPreparer) -from sqlalchemy.engine import base as engine_base, default -from sqlalchemy.sql import operators as sql_operators -from sqlalchemy import exc, log, schema, sql, types as sqltypes, util -from sqlalchemy import processors - +from .base import (BIT, MySQLDialect, MySQLExecutionContext) +from ... import types as sqltypes, util class _oursqlBIT(BIT): @@ -62,12 +52,13 @@ class MySQLExecutionContext_oursql(MySQLExecutionContext): def plain_query(self): return self.execution_options.get('_oursql_plain_query', False) + class MySQLDialect_oursql(MySQLDialect): driver = 'oursql' -# Py2K - supports_unicode_binds = True - supports_unicode_statements = True -# end Py2K + + if util.py2k: + supports_unicode_binds = True + supports_unicode_statements = True supports_native_decimal = True @@ -99,12 +90,11 @@ class MySQLDialect_oursql(MySQLDialect): connection.cursor().execute('BEGIN', plain_query=True) def _xa_query(self, connection, query, xid): -# Py2K - arg = connection.connection._escape_string(xid) -# end Py2K -# Py3K -# charset = self._connection_charset -# arg = connection.connection._escape_string(xid.encode(charset)).decode(charset) + if util.py2k: + arg = connection.connection._escape_string(xid) + else: + charset = self._connection_charset + arg = connection.connection._escape_string(xid.encode(charset)).decode(charset) arg = "'%s'" % arg connection.execution_options(_oursql_plain_query=True).execute(query % arg) @@ -135,64 +125,67 @@ class MySQLDialect_oursql(MySQLDialect): # Q: why didn't we need all these "plain_query" overrides earlier ? # am i on a newer/older version of OurSQL ? def has_table(self, connection, table_name, schema=None): - return MySQLDialect.has_table(self, - connection.connect().\ - execution_options(_oursql_plain_query=True), - table_name, schema) - - def get_table_options(self, connection, table_name, schema=None, **kw): - return MySQLDialect.get_table_options(self, - connection.connect().\ - execution_options(_oursql_plain_query=True), - table_name, - schema = schema, - **kw + return MySQLDialect.has_table( + self, + connection.connect().execution_options(_oursql_plain_query=True), + table_name, + schema ) + def get_table_options(self, connection, table_name, schema=None, **kw): + return MySQLDialect.get_table_options( + self, + connection.connect().execution_options(_oursql_plain_query=True), + table_name, + schema=schema, + **kw + ) def get_columns(self, connection, table_name, schema=None, **kw): - return MySQLDialect.get_columns(self, - connection.connect().\ - execution_options(_oursql_plain_query=True), - table_name, - schema=schema, - **kw + return MySQLDialect.get_columns( + self, + connection.connect().execution_options(_oursql_plain_query=True), + table_name, + schema=schema, + **kw ) def get_view_names(self, connection, schema=None, **kw): - return MySQLDialect.get_view_names(self, - connection.connect().\ - execution_options(_oursql_plain_query=True), - schema=schema, - **kw + return MySQLDialect.get_view_names( + self, + connection.connect().execution_options(_oursql_plain_query=True), + schema=schema, + **kw ) def get_table_names(self, connection, schema=None, **kw): - return MySQLDialect.get_table_names(self, - connection.connect().\ - execution_options(_oursql_plain_query=True), - schema + return MySQLDialect.get_table_names( + self, + connection.connect().execution_options(_oursql_plain_query=True), + schema ) def get_schema_names(self, connection, **kw): - return MySQLDialect.get_schema_names(self, - connection.connect().\ - execution_options(_oursql_plain_query=True), - **kw + return MySQLDialect.get_schema_names( + self, + connection.connect().execution_options(_oursql_plain_query=True), + **kw ) def initialize(self, connection): return MySQLDialect.initialize( - self, - connection.execution_options(_oursql_plain_query=True) - ) + self, + connection.execution_options(_oursql_plain_query=True) + ) def _show_create_table(self, connection, table, charset=None, full_name=None): - return MySQLDialect._show_create_table(self, - connection.contextual_connect(close_with_result=True). - execution_options(_oursql_plain_query=True), - table, charset, full_name) + return MySQLDialect._show_create_table( + self, + connection.contextual_connect(close_with_result=True). + execution_options(_oursql_plain_query=True), + table, charset, full_name + ) def is_disconnect(self, e, connection, cursor): if isinstance(e, self.dbapi.ProgrammingError): diff --git a/libs/sqlalchemy/dialects/mysql/pymysql.py b/libs/sqlalchemy/dialects/mysql/pymysql.py index f5aaa122..74de09c4 100644 --- a/libs/sqlalchemy/dialects/mysql/pymysql.py +++ b/libs/sqlalchemy/dialects/mysql/pymysql.py @@ -1,21 +1,16 @@ # mysql/pymysql.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database via the pymysql adapter. +""" -pymysql is available at: - - http://code.google.com/p/pymysql/ - -Connecting ----------- - -Connect string:: - - mysql+pymysql://:@/[?] +.. dialect:: mysql+pymysql + :name: PyMySQL + :dbapi: pymysql + :connectstring: mysql+pymysql://:@/[?] + :url: http://code.google.com/p/pymysql/ MySQL-Python Compatibility -------------------------- @@ -26,14 +21,24 @@ the pymysql driver as well. """ -from sqlalchemy.dialects.mysql.mysqldb import MySQLDialect_mysqldb +from .mysqldb import MySQLDialect_mysqldb +from ...util import py3k class MySQLDialect_pymysql(MySQLDialect_mysqldb): driver = 'pymysql' description_encoding = None + if py3k: + supports_unicode_statements = True + @classmethod def dbapi(cls): return __import__('pymysql') -dialect = MySQLDialect_pymysql \ No newline at end of file + if py3k: + def _extract_error_code(self, exception): + if isinstance(exception.args[0], Exception): + exception = exception.args[0] + return exception.args[0] + +dialect = MySQLDialect_pymysql diff --git a/libs/sqlalchemy/dialects/mysql/pyodbc.py b/libs/sqlalchemy/dialects/mysql/pyodbc.py index 5d631afb..e60e39ce 100644 --- a/libs/sqlalchemy/dialects/mysql/pyodbc.py +++ b/libs/sqlalchemy/dialects/mysql/pyodbc.py @@ -1,21 +1,18 @@ # mysql/pyodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database via the pyodbc adapter. +""" -pyodbc is available at: - http://pypi.python.org/pypi/pyodbc/ +.. dialect:: mysql+pyodbc + :name: PyODBC + :dbapi: pyodbc + :connectstring: mysql+pyodbc://:@ + :url: http://pypi.python.org/pypi/pyodbc/ -Connecting ----------- - -Connect string:: - - mysql+pyodbc://:@ Limitations ----------- @@ -27,12 +24,12 @@ of OurSQL, MySQLdb, or MySQL-connector/Python. """ -from sqlalchemy.dialects.mysql.base import MySQLDialect, MySQLExecutionContext -from sqlalchemy.connectors.pyodbc import PyODBCConnector -from sqlalchemy.engine import base as engine_base -from sqlalchemy import util +from .base import MySQLDialect, MySQLExecutionContext +from ...connectors.pyodbc import PyODBCConnector +from ... import util import re + class MySQLExecutionContext_pyodbc(MySQLExecutionContext): def get_lastrowid(self): @@ -42,6 +39,7 @@ class MySQLExecutionContext_pyodbc(MySQLExecutionContext): cursor.close() return lastrowid + class MySQLDialect_pyodbc(PyODBCConnector, MySQLDialect): supports_unicode_statements = False execution_ctx_cls = MySQLExecutionContext_pyodbc diff --git a/libs/sqlalchemy/dialects/mysql/zxjdbc.py b/libs/sqlalchemy/dialects/mysql/zxjdbc.py index df479043..b5fcfbda 100644 --- a/libs/sqlalchemy/dialects/mysql/zxjdbc.py +++ b/libs/sqlalchemy/dialects/mysql/zxjdbc.py @@ -1,23 +1,16 @@ # mysql/zxjdbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the MySQL database via Jython's zxjdbc JDBC connector. +""" -JDBC Driver ------------ - -The official MySQL JDBC driver is at -http://dev.mysql.com/downloads/connector/j/. - -Connecting ----------- - -Connect string format: - - mysql+zxjdbc://:@[:]/ +.. dialect:: mysql+zxjdbc + :name: zxjdbc for Jython + :dbapi: zxjdbc + :connectstring: mysql+zxjdbc://:@[:]/ + :driverurl: http://dev.mysql.com/downloads/connector/j/ Character Sets -------------- @@ -31,9 +24,10 @@ overriden via a ``create_engine`` URL parameter. """ import re -from sqlalchemy import types as sqltypes, util -from sqlalchemy.connectors.zxJDBC import ZxJDBCConnector -from sqlalchemy.dialects.mysql.base import BIT, MySQLDialect, MySQLExecutionContext +from ... import types as sqltypes, util +from ...connectors.zxJDBC import ZxJDBCConnector +from .base import BIT, MySQLDialect, MySQLExecutionContext + class _ZxJDBCBit(BIT): def result_processor(self, dialect, coltype): @@ -43,7 +37,7 @@ class _ZxJDBCBit(BIT): return value if isinstance(value, bool): return int(value) - v = 0L + v = 0 for i in value: v = v << 8 | (i & 0xff) value = v @@ -103,7 +97,7 @@ class MySQLDialect_zxjdbc(ZxJDBCConnector, MySQLDialect): if c: return int(c) - def _get_server_version_info(self,connection): + def _get_server_version_info(self, connection): dbapi_con = connection.connection version = [] r = re.compile('[.\-]') diff --git a/libs/sqlalchemy/dialects/oracle/__init__.py b/libs/sqlalchemy/dialects/oracle/__init__.py index a1e2a8dd..070e387d 100644 --- a/libs/sqlalchemy/dialects/oracle/__init__.py +++ b/libs/sqlalchemy/dialects/oracle/__init__.py @@ -1,5 +1,5 @@ # oracle/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -12,7 +12,7 @@ from sqlalchemy.dialects.oracle.base import \ VARCHAR, NVARCHAR, CHAR, DATE, DATETIME, NUMBER,\ BLOB, BFILE, CLOB, NCLOB, TIMESTAMP, RAW,\ FLOAT, DOUBLE_PRECISION, LONG, dialect, INTERVAL,\ - VARCHAR2, NVARCHAR2, ROWID + VARCHAR2, NVARCHAR2, ROWID, dialect __all__ = ( diff --git a/libs/sqlalchemy/dialects/oracle/base.py b/libs/sqlalchemy/dialects/oracle/base.py index f82991bc..e5a16044 100644 --- a/libs/sqlalchemy/dialects/oracle/base.py +++ b/libs/sqlalchemy/dialects/oracle/base.py @@ -1,15 +1,14 @@ # oracle/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the Oracle database. +""" +.. dialect:: oracle + :name: Oracle -Oracle version 8 through current (11g at the time of this writing) are supported. - -For information on connecting via specific drivers, see the documentation -for that driver. + Oracle version 8 through current (11g at the time of this writing) are supported. Connect Arguments ----------------- @@ -17,12 +16,12 @@ Connect Arguments The dialect supports several :func:`~sqlalchemy.create_engine()` arguments which affect the behavior of the dialect regardless of driver in use. -* *use_ansi* - Use ANSI JOIN constructs (see the section on Oracle 8). Defaults +* ``use_ansi`` - Use ANSI JOIN constructs (see the section on Oracle 8). Defaults to ``True``. If ``False``, Oracle-8 compatible constructs are used for joins. -* *optimize_limits* - defaults to ``False``. see the section on LIMIT/OFFSET. +* ``optimize_limits`` - defaults to ``False``. see the section on LIMIT/OFFSET. -* *use_binds_for_limits* - defaults to ``True``. see the section on LIMIT/OFFSET. +* ``use_binds_for_limits`` - defaults to ``True``. see the section on LIMIT/OFFSET. Auto Increment Behavior ----------------------- @@ -100,6 +99,41 @@ http://www.sqlalchemy.org/trac/wiki/UsageRecipes/WindowFunctionsByDefault which installs a select compiler that overrides the generation of limit/offset with a window function. +RETURNING Support +----------------- + +The Oracle database supports a limited form of RETURNING, in order to retrieve result +sets of matched rows from INSERT, UPDATE and DELETE statements. Oracle's +RETURNING..INTO syntax only supports one row being returned, as it relies upon +OUT parameters in order to function. In addition, supported DBAPIs have further +limitations (see :ref:`cx_oracle_returning`). + +SQLAlchemy's "implicit returning" feature, which employs RETURNING within an INSERT +and sometimes an UPDATE statement in order to fetch newly generated primary key values +and other SQL defaults and expressions, is normally enabled on the Oracle +backend. By default, "implicit returning" typically only fetches the value of a +single ``nextval(some_seq)`` expression embedded into an INSERT in order to increment +a sequence within an INSERT statement and get the value back at the same time. +To disable this feature across the board, specify ``implicit_returning=False`` to +:func:`.create_engine`:: + + engine = create_engine("oracle://scott:tiger@dsn", implicit_returning=False) + +Implicit returning can also be disabled on a table-by-table basis as a table option:: + + # Core Table + my_table = Table("my_table", metadata, ..., implicit_returning=False) + + + # declarative + class MyClass(Base): + __tablename__ = 'my_table' + __table_args__ = {"implicit_returning": False} + +.. seealso:: + + :ref:`cx_oracle_returning` - additional cx_oracle-specific restrictions on implicit returning. + ON UPDATE CASCADE ----------------- @@ -134,16 +168,16 @@ Synonym/DBLINK Reflection ------------------------- When using reflection with Table objects, the dialect can optionally search for tables -indicated by synonyms that reference DBLINK-ed tables by passing the flag -oracle_resolve_synonyms=True as a keyword argument to the Table construct. If DBLINK -is not in use this flag should be left off. +indicated by synonyms, either in local or remote schemas or accessed over DBLINK, +by passing the flag oracle_resolve_synonyms=True as a +keyword argument to the Table construct. If synonyms are not in use +this flag should be left off. """ -import random, re +import re -from sqlalchemy import schema as sa_schema -from sqlalchemy import util, sql, log +from sqlalchemy import util, sql from sqlalchemy.engine import default, base, reflection from sqlalchemy.sql import compiler, visitors, expression from sqlalchemy.sql import operators as sql_operators, functions as sql_functions @@ -165,18 +199,22 @@ RESERVED_WORDS = \ NO_ARG_FNS = set('UID CURRENT_DATE SYSDATE USER ' 'CURRENT_TIME CURRENT_TIMESTAMP'.split()) + class RAW(sqltypes._Binary): __visit_name__ = 'RAW' OracleRaw = RAW + class NCLOB(sqltypes.Text): __visit_name__ = 'NCLOB' + class VARCHAR2(VARCHAR): __visit_name__ = 'VARCHAR2' NVARCHAR2 = NVARCHAR + class NUMBER(sqltypes.Numeric, sqltypes.Integer): __visit_name__ = 'NUMBER' @@ -202,18 +240,22 @@ class NUMBER(sqltypes.Numeric, sqltypes.Integer): class DOUBLE_PRECISION(sqltypes.Numeric): __visit_name__ = 'DOUBLE_PRECISION' + def __init__(self, precision=None, scale=None, asdecimal=None): if asdecimal is None: asdecimal = False super(DOUBLE_PRECISION, self).__init__(precision=precision, scale=scale, asdecimal=asdecimal) + class BFILE(sqltypes.LargeBinary): __visit_name__ = 'BFILE' + class LONG(sqltypes.Text): __visit_name__ = 'LONG' + class INTERVAL(sqltypes.TypeEngine): __visit_name__ = 'INTERVAL' @@ -244,6 +286,7 @@ class INTERVAL(sqltypes.TypeEngine): def _type_affinity(self): return sqltypes.Interval + class ROWID(sqltypes.TypeEngine): """Oracle ROWID type. @@ -253,33 +296,32 @@ class ROWID(sqltypes.TypeEngine): __visit_name__ = 'ROWID' - class _OracleBoolean(sqltypes.Boolean): def get_dbapi_type(self, dbapi): return dbapi.NUMBER colspecs = { - sqltypes.Boolean : _OracleBoolean, - sqltypes.Interval : INTERVAL, + sqltypes.Boolean: _OracleBoolean, + sqltypes.Interval: INTERVAL, } ischema_names = { - 'VARCHAR2' : VARCHAR, - 'NVARCHAR2' : NVARCHAR, - 'CHAR' : CHAR, - 'DATE' : DATE, - 'NUMBER' : NUMBER, - 'BLOB' : BLOB, - 'BFILE' : BFILE, - 'CLOB' : CLOB, - 'NCLOB' : NCLOB, - 'TIMESTAMP' : TIMESTAMP, - 'TIMESTAMP WITH TIME ZONE' : TIMESTAMP, - 'INTERVAL DAY TO SECOND' : INTERVAL, - 'RAW' : RAW, - 'FLOAT' : FLOAT, - 'DOUBLE PRECISION' : DOUBLE_PRECISION, - 'LONG' : LONG, + 'VARCHAR2': VARCHAR, + 'NVARCHAR2': NVARCHAR, + 'CHAR': CHAR, + 'DATE': DATE, + 'NUMBER': NUMBER, + 'BLOB': BLOB, + 'BFILE': BFILE, + 'CLOB': CLOB, + 'NCLOB': NCLOB, + 'TIMESTAMP': TIMESTAMP, + 'TIMESTAMP WITH TIME ZONE': TIMESTAMP, + 'INTERVAL DAY TO SECOND': INTERVAL, + 'RAW': RAW, + 'FLOAT': FLOAT, + 'DOUBLE PRECISION': DOUBLE_PRECISION, + 'LONG': LONG, } @@ -336,9 +378,11 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): if precision is None: return name elif scale is None: - return "%(name)s(%(precision)s)" % {'name':name,'precision': precision} + n = "%(name)s(%(precision)s)" + return n % {'name': name, 'precision': precision} else: - return "%(name)s(%(precision)s, %(scale)s)" % {'name':name,'precision': precision, 'scale' : scale} + n = "%(name)s(%(precision)s, %(scale)s)" + return n % {'name': name, 'precision': precision, 'scale': scale} def visit_string(self, type_): return self.visit_VARCHAR2(type_) @@ -354,13 +398,14 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): return self._visit_varchar(type_, '', '') def _visit_varchar(self, type_, n, num): - if not n and self.dialect._supports_char_length: - return "VARCHAR%(two)s(%(length)s CHAR)" % { - 'length' : type_.length, - 'two':num} + if not type_.length: + return "%(n)sVARCHAR%(two)s" % {'two': num, 'n': n} + elif not n and self.dialect._supports_char_length: + varchar = "VARCHAR%(two)s(%(length)s CHAR)" + return varchar % {'length': type_.length, 'two': num} else: - return "%(n)sVARCHAR%(two)s(%(length)s)" % {'length' : type_.length, - 'two':num, 'n':n} + varchar = "%(n)sVARCHAR%(two)s(%(length)s)" + return varchar % {'length': type_.length, 'two': num, 'n': n} def visit_text(self, type_): return self.visit_CLOB(type_) @@ -382,13 +427,14 @@ class OracleTypeCompiler(compiler.GenericTypeCompiler): def visit_RAW(self, type_): if type_.length: - return "RAW(%(length)s)" % {'length' : type_.length} + return "RAW(%(length)s)" % {'length': type_.length} else: return "RAW" def visit_ROWID(self, type_): return "ROWID" + class OracleCompiler(compiler.SQLCompiler): """Oracle compiler modifies the lexical structure of Select statements to work under non-ANSI configured Oracle databases, if @@ -398,7 +444,7 @@ class OracleCompiler(compiler.SQLCompiler): compound_keywords = util.update_copy( compiler.SQLCompiler.compound_keywords, { - expression.CompoundSelect.EXCEPT : 'MINUS' + expression.CompoundSelect.EXCEPT: 'MINUS' } ) @@ -407,8 +453,9 @@ class OracleCompiler(compiler.SQLCompiler): self._quoted_bind_names = {} super(OracleCompiler, self).__init__(*args, **kwargs) - def visit_mod(self, binary, **kw): - return "mod(%s, %s)" % (self.process(binary.left), self.process(binary.right)) + def visit_mod_binary(self, binary, operator, **kw): + return "mod(%s, %s)" % (self.process(binary.left, **kw), + self.process(binary.right, **kw)) def visit_now_func(self, fn, **kw): return "CURRENT_TIMESTAMP" @@ -416,8 +463,15 @@ class OracleCompiler(compiler.SQLCompiler): def visit_char_length_func(self, fn, **kw): return "LENGTH" + self.function_argspec(fn, **kw) - def visit_match_op(self, binary, **kw): - return "CONTAINS (%s, %s)" % (self.process(binary.left), self.process(binary.right)) + def visit_match_op_binary(self, binary, operator, **kw): + return "CONTAINS (%s, %s)" % (self.process(binary.left), + self.process(binary.right)) + + def visit_true(self, expr, **kw): + return '1' + + def visit_false(self, expr, **kw): + return '0' def get_select_hint_text(self, byfroms): return " ".join( @@ -444,8 +498,13 @@ class OracleCompiler(compiler.SQLCompiler): return compiler.SQLCompiler.visit_join(self, join, **kwargs) else: kwargs['asfrom'] = True + if isinstance(join.right, expression.FromGrouping): + right = join.right.element + else: + right = join.right return self.process(join.left, **kwargs) + \ - ", " + self.process(join.right, **kwargs) + ", " + self.process(right, **kwargs) + def _get_nonansi_join_whereclause(self, froms): clauses = [] @@ -454,18 +513,20 @@ class OracleCompiler(compiler.SQLCompiler): if join.isouter: def visit_binary(binary): if binary.operator == sql_operators.eq: - if binary.left.table is join.right: + if join.right.is_derived_from(binary.left.table): binary.left = _OuterJoinColumn(binary.left) - elif binary.right.table is join.right: + elif join.right.is_derived_from(binary.right.table): binary.right = _OuterJoinColumn(binary.right) clauses.append(visitors.cloned_traverse(join.onclause, {}, - {'binary':visit_binary})) + {'binary': visit_binary})) else: clauses.append(join.onclause) for j in join.left, join.right: if isinstance(j, expression.Join): visit_join(j) + elif isinstance(j, expression.FromGrouping): + visit_join(j.element) for f in froms: if isinstance(f, expression.Join): @@ -498,20 +559,25 @@ class OracleCompiler(compiler.SQLCompiler): return self.process(alias.original, **kwargs) def returning_clause(self, stmt, returning_cols): + columns = [] + binds = [] + for i, column in enumerate(expression._select_iterables(returning_cols)): + if column.type._has_column_expression: + col_expr = column.type.column_expression(column) + else: + col_expr = column + outparam = sql.outparam("ret_%d" % i, type_=column.type) + self.binds[outparam.key] = outparam + binds.append(self.bindparam_string(self._truncate_bindparam(outparam))) + columns.append(self.process(col_expr, within_columns_clause=False)) + self.result_map[outparam.key] = ( + outparam.key, + (column, getattr(column, 'name', None), + getattr(column, 'key', None)), + column.type + ) - def create_out_param(col, i): - bindparam = sql.outparam("ret_%d" % i, type_=col.type) - self.binds[bindparam.key] = bindparam - return self.bindparam_string(self._truncate_bindparam(bindparam)) - - columnlist = list(expression._select_iterables(returning_cols)) - - # within_columns_clause =False so that labels (foo AS bar) don't render - columns = [self.process(c, within_columns_clause=False, result_map=self.result_map) for c in columnlist] - - binds = [create_out_param(c, i) for i, c in enumerate(columnlist)] - - return 'RETURNING ' + ', '.join(columns) + " INTO " + ", ".join(binds) + return 'RETURNING ' + ', '.join(columns) + " INTO " + ", ".join(binds) def _TODO_visit_compound_select(self, select): """Need to determine how to get ``LIMIT``/``OFFSET`` into a ``UNION`` for Oracle.""" @@ -524,12 +590,8 @@ class OracleCompiler(compiler.SQLCompiler): if not getattr(select, '_oracle_visit', None): if not self.dialect.use_ansi: - if self.stack and 'from' in self.stack[-1]: - existingfroms = self.stack[-1]['from'] - else: - existingfroms = None - - froms = select._get_display_froms(existingfroms) + froms = self._display_froms_for_select( + select, kwargs.get('asfrom', False)) whereclause = self._get_nonansi_join_whereclause(froms) if whereclause is not None: select = select.where(whereclause) @@ -570,7 +632,7 @@ class OracleCompiler(compiler.SQLCompiler): # If needed, add the ora_rn, and wrap again with offset. if select._offset is None: - limitselect.for_update = select.for_update + limitselect._for_update_arg = select._for_update_arg select = limitselect else: limitselect = limitselect.column( @@ -579,7 +641,7 @@ class OracleCompiler(compiler.SQLCompiler): limitselect._is_wrapper = True offsetselect = sql.select( - [c for c in limitselect.c if c.key!='ora_rn']) + [c for c in limitselect.c if c.key != 'ora_rn']) offsetselect._oracle_visit = True offsetselect._is_wrapper = True @@ -587,9 +649,9 @@ class OracleCompiler(compiler.SQLCompiler): if not self.dialect.use_binds_for_limits: offset_value = sql.literal_column("%d" % offset_value) offsetselect.append_whereclause( - sql.literal_column("ora_rn")>offset_value) + sql.literal_column("ora_rn") > offset_value) - offsetselect.for_update = select.for_update + offsetselect._for_update_arg = select._for_update_arg select = offsetselect kwargs['iswrapper'] = getattr(select, '_is_wrapper', False) @@ -601,10 +663,20 @@ class OracleCompiler(compiler.SQLCompiler): def for_update_clause(self, select): if self.is_subquery(): return "" - elif select.for_update == "nowait": - return " FOR UPDATE NOWAIT" - else: - return super(OracleCompiler, self).for_update_clause(select) + + tmp = ' FOR UPDATE' + + if select._for_update_arg.of: + tmp += ' OF ' + ', '.join( + self.process(elem) for elem in + select._for_update_arg.of + ) + + if select._for_update_arg.nowait: + tmp += " NOWAIT" + + return tmp + class OracleDDLCompiler(compiler.DDLCompiler): @@ -623,17 +695,22 @@ class OracleDDLCompiler(compiler.DDLCompiler): return text + def visit_create_index(self, create, **kw): + return super(OracleDDLCompiler, self).\ + visit_create_index(create, include_schema=True) + + class OracleIdentifierPreparer(compiler.IdentifierPreparer): reserved_words = set([x.lower() for x in RESERVED_WORDS]) - illegal_initial_characters = set(xrange(0, 10)).union(["_", "$"]) + illegal_initial_characters = set(range(0, 10)).union(["_", "$"]) def _bindparam_requires_quotes(self, value): """Return True if the given identifier requires quoting.""" lc_value = value.lower() return (lc_value in self.reserved_words or value[0] in self.illegal_initial_characters - or not self.legal_characters.match(unicode(value)) + or not self.legal_characters.match(util.text_type(value)) ) def format_savepoint(self, savepoint): @@ -647,6 +724,7 @@ class OracleExecutionContext(default.DefaultExecutionContext): self.dialect.identifier_preparer.format_sequence(seq) + ".nextval FROM DUAL", type_) + class OracleDialect(default.DefaultDialect): name = 'oracle' supports_alter = True @@ -736,10 +814,9 @@ class OracleDialect(default.DefaultDialect): def normalize_name(self, name): if name is None: return None - # Py2K - if isinstance(name, str): - name = name.decode(self.encoding) - # end Py2K + if util.py2k: + if isinstance(name, str): + name = name.decode(self.encoding) if name.upper() == name and \ not self.identifier_preparer._requires_quotes(name.lower()): return name.lower() @@ -751,16 +828,15 @@ class OracleDialect(default.DefaultDialect): return None elif name.lower() == name and not self.identifier_preparer._requires_quotes(name.lower()): name = name.upper() - # Py2K - if not self.supports_unicode_binds: - name = name.encode(self.encoding) - else: - name = unicode(name) - # end Py2K + if util.py2k: + if not self.supports_unicode_binds: + name = name.encode(self.encoding) + else: + name = unicode(name) return name def _get_default_schema_name(self, connection): - return self.normalize_name(connection.execute(u'SELECT USER FROM DUAL').scalar()) + return self.normalize_name(connection.execute('SELECT USER FROM DUAL').scalar()) def _resolve_synonym(self, connection, desired_owner=None, desired_synonym=None, desired_table=None): """search for a local synonym matching the given desired owner/name. @@ -770,14 +846,15 @@ class OracleDialect(default.DefaultDialect): returns the actual name, owner, dblink name, and synonym name if found. """ - q = "SELECT owner, table_owner, table_name, db_link, synonym_name FROM all_synonyms WHERE " + q = "SELECT owner, table_owner, table_name, db_link, "\ + "synonym_name FROM all_synonyms WHERE " clauses = [] params = {} if desired_synonym: clauses.append("synonym_name = :synonym_name") params['synonym_name'] = desired_synonym if desired_owner: - clauses.append("table_owner = :desired_owner") + clauses.append("owner = :desired_owner") params['desired_owner'] = desired_owner if desired_table: clauses.append("table_name = :tname") @@ -808,19 +885,29 @@ class OracleDialect(default.DefaultDialect): if resolve_synonyms: actual_name, owner, dblink, synonym = self._resolve_synonym( - connection, - desired_owner=self.denormalize_name(schema), - desired_synonym=self.denormalize_name(table_name) - ) + connection, + desired_owner=self.denormalize_name(schema), + desired_synonym=self.denormalize_name(table_name) + ) else: actual_name, owner, dblink, synonym = None, None, None, None if not actual_name: actual_name = self.denormalize_name(table_name) - if not dblink: - dblink = '' - if not owner: + + if dblink: + # using user_db_links here since all_db_links appears + # to have more restricted permissions. + # http://docs.oracle.com/cd/B28359_01/server.111/b28310/ds_admin005.htm + # will need to hear from more users if we are doing + # the right thing here. See [ticket:2619] + owner = connection.scalar( + sql.text("SELECT username FROM user_db_links " + "WHERE db_link=:link"), link=dblink) + dblink = "@" + dblink + elif not owner: owner = self.denormalize_name(schema or self.default_schema_name) - return (actual_name, owner, dblink, synonym) + + return (actual_name, owner, dblink or '', synonym) @reflection.cache def get_schema_names(self, connection, **kw): @@ -843,7 +930,6 @@ class OracleDialect(default.DefaultDialect): cursor = connection.execute(s, owner=schema) return [self.normalize_name(row[0]) for row in cursor] - @reflection.cache def get_view_names(self, connection, schema=None, **kw): schema = self.denormalize_name(schema or self.default_schema_name) @@ -877,18 +963,24 @@ class OracleDialect(default.DefaultDialect): else: char_length_col = 'data_length' - c = connection.execute(sql.text( - "SELECT column_name, data_type, %(char_length_col)s, data_precision, data_scale, " - "nullable, data_default FROM ALL_TAB_COLUMNS%(dblink)s " - "WHERE table_name = :table_name AND owner = :owner " - "ORDER BY column_id" % {'dblink': dblink, 'char_length_col':char_length_col}), - table_name=table_name, owner=schema) + params = {"table_name": table_name} + text = "SELECT column_name, data_type, %(char_length_col)s, "\ + "data_precision, data_scale, "\ + "nullable, data_default FROM ALL_TAB_COLUMNS%(dblink)s "\ + "WHERE table_name = :table_name" + if schema is not None: + params['owner'] = schema + text += " AND owner = :owner " + text += " ORDER BY column_id" + text = text % {'dblink': dblink, 'char_length_col': char_length_col} + + c = connection.execute(sql.text(text), **params) for row in c: (colname, orig_colname, coltype, length, precision, scale, nullable, default) = \ - (self.normalize_name(row[0]), row[0], row[1], row[2], row[3], row[4], row[5]=='Y', row[6]) + (self.normalize_name(row[0]), row[0], row[1], row[2], row[3], row[4], row[5] == 'Y', row[6]) - if coltype == 'NUMBER' : + if coltype == 'NUMBER': coltype = NUMBER(precision, scale) elif coltype in ('VARCHAR2', 'NVARCHAR2', 'CHAR'): coltype = self.ischema_names.get(coltype)(length) @@ -908,7 +1000,7 @@ class OracleDialect(default.DefaultDialect): 'type': coltype, 'nullable': nullable, 'default': default, - 'autoincrement':default is None + 'autoincrement': default is None } if orig_colname.lower() == orig_colname: cdict['quote'] = True @@ -920,33 +1012,40 @@ class OracleDialect(default.DefaultDialect): def get_indexes(self, connection, table_name, schema=None, resolve_synonyms=False, dblink='', **kw): - info_cache = kw.get('info_cache') (table_name, schema, dblink, synonym) = \ self._prepare_reflection_args(connection, table_name, schema, resolve_synonyms, dblink, info_cache=info_cache) indexes = [] - q = sql.text(""" - SELECT a.index_name, a.column_name, b.uniqueness - FROM ALL_IND_COLUMNS%(dblink)s a, - ALL_INDEXES%(dblink)s b - WHERE - a.index_name = b.index_name - AND a.table_owner = b.table_owner - AND a.table_name = b.table_name - AND a.table_name = :table_name - AND a.table_owner = :schema - ORDER BY a.index_name, a.column_position""" % {'dblink': dblink}) - rp = connection.execute(q, table_name=self.denormalize_name(table_name), - schema=self.denormalize_name(schema)) + params = {'table_name': table_name} + text = \ + "SELECT a.index_name, a.column_name, b.uniqueness "\ + "\nFROM ALL_IND_COLUMNS%(dblink)s a, "\ + "\nALL_INDEXES%(dblink)s b "\ + "\nWHERE "\ + "\na.index_name = b.index_name "\ + "\nAND a.table_owner = b.table_owner "\ + "\nAND a.table_name = b.table_name "\ + "\nAND a.table_name = :table_name " + + if schema is not None: + params['schema'] = schema + text += "AND a.table_owner = :schema " + + text += "ORDER BY a.index_name, a.column_position" + + text = text % {'dblink': dblink} + + q = sql.text(text) + rp = connection.execute(q, **params) indexes = [] last_index_name = None - pkeys = self.get_primary_keys(connection, table_name, schema, - resolve_synonyms=resolve_synonyms, - dblink=dblink, - info_cache=kw.get('info_cache')) + pk_constraint = self.get_pk_constraint( + connection, table_name, schema, resolve_synonyms=resolve_synonyms, + dblink=dblink, info_cache=kw.get('info_cache')) + pkeys = pk_constraint['constrained_columns'] uniqueness = dict(NONUNIQUE=False, UNIQUE=True) oracle_sys_col = re.compile(r'SYS_NC\d+\$', re.IGNORECASE) @@ -982,46 +1081,43 @@ class OracleDialect(default.DefaultDialect): def _get_constraint_data(self, connection, table_name, schema=None, dblink='', **kw): - rp = connection.execute( - sql.text("""SELECT - ac.constraint_name, - ac.constraint_type, - loc.column_name AS local_column, - rem.table_name AS remote_table, - rem.column_name AS remote_column, - rem.owner AS remote_owner, - loc.position as loc_pos, - rem.position as rem_pos - FROM all_constraints%(dblink)s ac, - all_cons_columns%(dblink)s loc, - all_cons_columns%(dblink)s rem - WHERE ac.table_name = :table_name - AND ac.constraint_type IN ('R','P') - AND ac.owner = :owner - AND ac.owner = loc.owner - AND ac.constraint_name = loc.constraint_name - AND ac.r_owner = rem.owner(+) - AND ac.r_constraint_name = rem.constraint_name(+) - AND (rem.position IS NULL or loc.position=rem.position) - ORDER BY ac.constraint_name, loc.position""" % {'dblink': dblink}), - table_name=table_name, owner=schema) + params = {'table_name': table_name} + + text = \ + "SELECT"\ + "\nac.constraint_name,"\ + "\nac.constraint_type,"\ + "\nloc.column_name AS local_column,"\ + "\nrem.table_name AS remote_table,"\ + "\nrem.column_name AS remote_column,"\ + "\nrem.owner AS remote_owner,"\ + "\nloc.position as loc_pos,"\ + "\nrem.position as rem_pos"\ + "\nFROM all_constraints%(dblink)s ac,"\ + "\nall_cons_columns%(dblink)s loc,"\ + "\nall_cons_columns%(dblink)s rem"\ + "\nWHERE ac.table_name = :table_name"\ + "\nAND ac.constraint_type IN ('R','P')" + + if schema is not None: + params['owner'] = schema + text += "\nAND ac.owner = :owner" + + text += \ + "\nAND ac.owner = loc.owner"\ + "\nAND ac.constraint_name = loc.constraint_name"\ + "\nAND ac.r_owner = rem.owner(+)"\ + "\nAND ac.r_constraint_name = rem.constraint_name(+)"\ + "\nAND (rem.position IS NULL or loc.position=rem.position)"\ + "\nORDER BY ac.constraint_name, loc.position" + + text = text % {'dblink': dblink} + rp = connection.execute(sql.text(text), **params) constraint_data = rp.fetchall() return constraint_data - def get_primary_keys(self, connection, table_name, schema=None, **kw): - """ - - kw arguments can be: - - oracle_resolve_synonyms - - dblink - - """ - return self._get_primary_keys(connection, table_name, schema, **kw)[0] - @reflection.cache - def _get_primary_keys(self, connection, table_name, schema=None, **kw): + def get_pk_constraint(self, connection, table_name, schema=None, **kw): resolve_synonyms = kw.get('oracle_resolve_synonyms', False) dblink = kw.get('dblink', '') info_cache = kw.get('info_cache') @@ -1037,22 +1133,13 @@ class OracleDialect(default.DefaultDialect): info_cache=kw.get('info_cache')) for row in constraint_data: - #print "ROW:" , row (cons_name, cons_type, local_column, remote_table, remote_column, remote_owner) = \ row[0:2] + tuple([self.normalize_name(x) for x in row[2:6]]) if cons_type == 'P': if constraint_name is None: constraint_name = self.normalize_name(cons_name) pkeys.append(local_column) - return pkeys, constraint_name - - def get_pk_constraint(self, connection, table_name, schema=None, **kw): - cols, name = self._get_primary_keys(connection, table_name, schema=schema, **kw) - - return { - 'constrained_columns':cols, - 'name':name - } + return {'constrained_columns': pkeys, 'name': constraint_name} @reflection.cache def get_foreign_keys(self, connection, table_name, schema=None, **kw): @@ -1066,7 +1153,7 @@ class OracleDialect(default.DefaultDialect): """ - requested_schema = schema # to check later on + requested_schema = schema # to check later on resolve_synonyms = kw.get('oracle_resolve_synonyms', False) dblink = kw.get('dblink', '') info_cache = kw.get('info_cache') @@ -1082,11 +1169,11 @@ class OracleDialect(default.DefaultDialect): def fkey_rec(): return { - 'name' : None, - 'constrained_columns' : [], - 'referred_schema' : None, - 'referred_table' : None, - 'referred_columns' : [] + 'name': None, + 'constrained_columns': [], + 'referred_schema': None, + 'referred_table': None, + 'referred_columns': [] } fkeys = util.defaultdict(fkey_rec) @@ -1101,7 +1188,7 @@ class OracleDialect(default.DefaultDialect): util.warn( ("Got 'None' querying 'table_name' from " "all_cons_columns%(dblink)s - does the user have " - "proper rights to the table?") % {'dblink':dblink}) + "proper rights to the table?") % {'dblink': dblink}) continue rec = fkeys[cons_name] @@ -1128,7 +1215,7 @@ class OracleDialect(default.DefaultDialect): local_cols.append(local_column) remote_cols.append(remote_column) - return fkeys.values() + return list(fkeys.values()) @reflection.cache def get_view_definition(self, connection, view_name, schema=None, @@ -1138,25 +1225,25 @@ class OracleDialect(default.DefaultDialect): self._prepare_reflection_args(connection, view_name, schema, resolve_synonyms, dblink, info_cache=info_cache) - s = sql.text(""" - SELECT text FROM all_views - WHERE owner = :schema - AND view_name = :view_name - """) - rp = connection.execute(s, - view_name=view_name, schema=schema).scalar() + + params = {'view_name': view_name} + text = "SELECT text FROM all_views WHERE view_name=:view_name" + + if schema is not None: + text += " AND owner = :schema" + params['schema'] = schema + + rp = connection.execute(sql.text(text), **params).scalar() if rp: - return rp.decode(self.encoding) + if util.py2k: + rp = rp.decode(self.encoding) + return rp else: return None - class _OuterJoinColumn(sql.ClauseElement): __visit_name__ = 'outer_join_column' def __init__(self, column): self.column = column - - - diff --git a/libs/sqlalchemy/dialects/oracle/cx_oracle.py b/libs/sqlalchemy/dialects/oracle/cx_oracle.py index 0154180d..c427e4bc 100644 --- a/libs/sqlalchemy/dialects/oracle/cx_oracle.py +++ b/libs/sqlalchemy/dialects/oracle/cx_oracle.py @@ -1,48 +1,57 @@ # oracle/cx_oracle.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the Oracle database via the cx_oracle driver. +""" -Driver ------- +.. dialect:: oracle+cx_oracle + :name: cx-Oracle + :dbapi: cx_oracle + :connectstring: oracle+cx_oracle://user:pass@host:port/dbname[?key=value&key=value...] + :url: http://cx-oracle.sourceforge.net/ -The Oracle dialect uses the cx_oracle driver, available at -http://cx-oracle.sourceforge.net/ . The dialect has several behaviors -which are specifically tailored towards compatibility with this module. -Version 5.0 or greater is **strongly** recommended, as SQLAlchemy makes -extensive use of the cx_oracle output converters for numeric and -string conversions. +Additional Connect Arguments +---------------------------- -Connecting ----------- +When connecting with ``dbname`` present, the host, port, and dbname tokens are +converted to a TNS name using +the cx_oracle ``makedsn()`` function. Otherwise, the host token is taken +directly as a TNS name. -Connecting with create_engine() uses the standard URL approach of -``oracle://user:pass@host:port/dbname[?key=value&key=value...]``. If dbname is present, the -host, port, and dbname tokens are converted to a TNS name using the cx_oracle -:func:`makedsn()` function. Otherwise, the host token is taken directly as a TNS name. +Additional arguments which may be specified either as query string arguments +on the URL, or as keyword arguments to :func:`.create_engine()` are: -Additional arguments which may be specified either as query string arguments on the -URL, or as keyword arguments to :func:`~sqlalchemy.create_engine()` are: +* allow_twophase - enable two-phase transactions. Defaults to ``True``. -* *allow_twophase* - enable two-phase transactions. Defaults to ``True``. - -* *arraysize* - set the cx_oracle.arraysize value on cursors, in SQLAlchemy +* arraysize - set the cx_oracle.arraysize value on cursors, in SQLAlchemy it defaults to 50. See the section on "LOB Objects" below. -* *auto_convert_lobs* - defaults to True, see the section on LOB objects. +* auto_convert_lobs - defaults to True, see the section on LOB objects. -* *auto_setinputsizes* - the cx_oracle.setinputsizes() call is issued for all bind parameters. - This is required for LOB datatypes but can be disabled to reduce overhead. Defaults - to ``True``. +* auto_setinputsizes - the cx_oracle.setinputsizes() call is issued for + all bind parameters. This is required for LOB datatypes but can be + disabled to reduce overhead. Defaults to ``True``. Specific types + can be excluded from this process using the ``exclude_setinputsizes`` + parameter. -* *mode* - This is given the string value of SYSDBA or SYSOPER, or alternatively an - integer value. This value is only available as a URL query string argument. +* exclude_setinputsizes - a tuple or list of string DBAPI type names to + be excluded from the "auto setinputsizes" feature. The type names here + must match DBAPI types that are found in the "cx_Oracle" module namespace, + such as cx_Oracle.UNICODE, cx_Oracle.NCLOB, etc. Defaults to + ``(STRING, UNICODE)``. -* *threaded* - enable multithreaded access to cx_oracle connections. Defaults - to ``True``. Note that this is the opposite default of cx_oracle itself. + .. versionadded:: 0.8 specific DBAPI types can be excluded from the + auto_setinputsizes feature via the exclude_setinputsizes attribute. + +* mode - This is given the string value of SYSDBA or SYSOPER, or alternatively + an integer value. This value is only available as a URL query string + argument. + +* threaded - enable multithreaded access to cx_oracle connections. Defaults + to ``True``. Note that this is the opposite default of the cx_Oracle DBAPI + itself. Unicode ------- @@ -56,6 +65,27 @@ of the encoding to be used. Note that this behavior is disabled when Oracle 8 is detected, as it has been observed that issues remain when passing Python unicodes to cx_oracle with Oracle 8. +.. _cx_oracle_returning: + +RETURNING Support +----------------- + +cx_oracle supports a limited subset of Oracle's already limited RETURNING support. +Typically, results can only be guaranteed for at most one column being returned; +this is the typical case when SQLAlchemy uses RETURNING to get just the value of a +primary-key-associated sequence value. Additional column expressions will +cause problems in a non-determinative way, due to cx_oracle's lack of support for +the OCI_DATA_AT_EXEC API which is required for more complex RETURNING scenarios. + +.. seealso:: + + http://docs.oracle.com/cd/B10501_01/appdev.920/a96584/oci05bnd.htm#420693 - OCI documentation for RETURNING + + http://sourceforge.net/mailarchive/message.php?msg_id=31338136 - cx_oracle developer commentary + + + + LOB Objects ----------- @@ -66,7 +96,7 @@ like result.fetchmany() and result.fetchall(). This means that by default, LOB objects are fully fetched unconditionally by SQLAlchemy, and the linkage to a live cursor is broken. -To disable this processing, pass ``auto_convert_lobs=False`` to :func:`create_engine()`. +To disable this processing, pass ``auto_convert_lobs=False`` to :func:`.create_engine()`. Two Phase Transaction Support ----------------------------- @@ -99,7 +129,7 @@ the application can make one of several choices: * For ad-hoc two-phase operations without disabling pooling, the DBAPI connection in use can be evicted from the connection pool using the - :class:`.Connection.detach` method. + :meth:`.Connection.detach` method. .. versionchanged:: 0.8.0b2,0.7.10 Support for cx_oracle prepared transactions has been implemented @@ -173,16 +203,18 @@ a period "." as the decimal character. """ -from sqlalchemy.dialects.oracle.base import OracleCompiler, OracleDialect, \ - OracleExecutionContext -from sqlalchemy.dialects.oracle import base as oracle -from sqlalchemy.engine import base +from __future__ import absolute_import + +from .base import OracleCompiler, OracleDialect, OracleExecutionContext +from . import base as oracle +from ...engine import result as _result from sqlalchemy import types as sqltypes, util, exc, processors import random import collections -from sqlalchemy.util.compat import decimal +import decimal import re + class _OracleNumeric(sqltypes.Numeric): def bind_processor(self, dialect): # cx_oracle accepts Decimal objects and floats @@ -200,10 +232,8 @@ class _OracleNumeric(sqltypes.Numeric): if dialect.supports_native_decimal: if self.asdecimal: - if self.scale is None: - fstring = "%.10f" - else: - fstring = "%%.%df" % self.scale + fstring = "%%.%df" % self._effective_decimal_return_scale + def to_decimal(value): if value is None: return None @@ -211,6 +241,7 @@ class _OracleNumeric(sqltypes.Numeric): return value else: return decimal.Decimal(fstring % value) + return to_decimal else: if self.precision is None and self.scale is None: @@ -226,6 +257,7 @@ class _OracleNumeric(sqltypes.Numeric): return super(_OracleNumeric, self).\ result_processor(dialect, coltype) + class _OracleDate(sqltypes.Date): def bind_processor(self, dialect): return None @@ -238,6 +270,7 @@ class _OracleDate(sqltypes.Date): return value return process + class _LOBMixin(object): def result_processor(self, dialect, coltype): if not dialect.auto_convert_lobs: @@ -251,38 +284,40 @@ class _LOBMixin(object): return value return process + class _NativeUnicodeMixin(object): - # Py3K - #pass - # Py2K - def bind_processor(self, dialect): - if dialect._cx_oracle_with_unicode: - def process(value): - if value is None: - return value - else: - return unicode(value) - return process - else: - return super(_NativeUnicodeMixin, self).bind_processor(dialect) - # end Py2K + if util.py2k: + def bind_processor(self, dialect): + if dialect._cx_oracle_with_unicode: + def process(value): + if value is None: + return value + else: + return unicode(value) + return process + else: + return super(_NativeUnicodeMixin, self).bind_processor(dialect) # we apply a connection output handler that returns # unicode in all cases, so the "native_unicode" flag # will be set for the default String.result_processor. + class _OracleChar(_NativeUnicodeMixin, sqltypes.CHAR): def get_dbapi_type(self, dbapi): return dbapi.FIXED_CHAR + class _OracleNVarChar(_NativeUnicodeMixin, sqltypes.NVARCHAR): def get_dbapi_type(self, dbapi): return getattr(dbapi, 'UNICODE', dbapi.STRING) + class _OracleText(_LOBMixin, sqltypes.Text): def get_dbapi_type(self, dbapi): return dbapi.CLOB + class _OracleLong(oracle.LONG): # a raw LONG is a text type, but does *not* # get the LobMixin with cx_oracle. @@ -293,6 +328,7 @@ class _OracleLong(oracle.LONG): class _OracleString(_NativeUnicodeMixin, sqltypes.String): pass + class _OracleUnicodeText(_LOBMixin, _NativeUnicodeMixin, sqltypes.UnicodeText): def get_dbapi_type(self, dbapi): return dbapi.NCLOB @@ -311,6 +347,7 @@ class _OracleUnicodeText(_LOBMixin, _NativeUnicodeMixin, sqltypes.UnicodeText): return string_processor(lob_processor(value)) return process + class _OracleInteger(sqltypes.Integer): def result_processor(self, dialect, coltype): def to_int(val): @@ -319,6 +356,7 @@ class _OracleInteger(sqltypes.Integer): return val return to_int + class _OracleBinary(_LOBMixin, sqltypes.LargeBinary): def get_dbapi_type(self, dbapi): return dbapi.BLOB @@ -326,20 +364,26 @@ class _OracleBinary(_LOBMixin, sqltypes.LargeBinary): def bind_processor(self, dialect): return None + class _OracleInterval(oracle.INTERVAL): def get_dbapi_type(self, dbapi): return dbapi.INTERVAL + class _OracleRaw(oracle.RAW): pass + class _OracleRowid(oracle.ROWID): def get_dbapi_type(self, dbapi): return dbapi.ROWID + class OracleCompiler_cx_oracle(OracleCompiler): def bindparam_string(self, name, **kw): - if self.preparer._bindparam_requires_quotes(name): + quote = getattr(name, 'quote', None) + if quote is True or quote is not False and \ + self.preparer._bindparam_requires_quotes(name): quoted_name = '"%s"' % name self._quoted_bind_names[name] = quoted_name return OracleCompiler.bindparam_string(self, quoted_name, **kw) @@ -375,7 +419,7 @@ class OracleExecutionContext_cx_oracle(OracleExecutionContext): # on String, including that outparams/RETURNING # breaks for varchars self.set_input_sizes(quoted_bind_names, - exclude_types=self.dialect._cx_oracle_exclude_setinputsizes + exclude_types=self.dialect.exclude_setinputsizes ) # if a single execute, check for outparams @@ -387,11 +431,12 @@ class OracleExecutionContext_cx_oracle(OracleExecutionContext): if not hasattr(self, 'out_parameters'): self.out_parameters = {} if dbtype is None: - raise exc.InvalidRequestError("Cannot create out parameter for parameter " - "%r - it's type %r is not supported by" - " cx_oracle" % - (name, bindparam.type) - ) + raise exc.InvalidRequestError( + "Cannot create out parameter for parameter " + "%r - it's type %r is not supported by" + " cx_oracle" % + (bindparam.key, bindparam.type) + ) name = self.compiled.bind_names[bindparam] self.out_parameters[name] = self.cursor.var(dbtype) self.parameters[0][quoted_bind_names.get(name, name)] = \ @@ -417,10 +462,10 @@ class OracleExecutionContext_cx_oracle(OracleExecutionContext): for column in self.cursor.description: type_code = column[1] if type_code in self.dialect._cx_oracle_binary_types: - result = base.BufferedColumnResultProxy(self) + result = _result.BufferedColumnResultProxy(self) if result is None: - result = base.ResultProxy(self) + result = _result.ResultProxy(self) if hasattr(self, 'out_parameters'): if self.compiled_parameters is not None and \ @@ -448,6 +493,7 @@ class OracleExecutionContext_cx_oracle(OracleExecutionContext): return result + class OracleExecutionContext_cx_oracle_with_unicode(OracleExecutionContext_cx_oracle): """Support WITH_UNICODE in Python 2.xx. @@ -463,13 +509,14 @@ class OracleExecutionContext_cx_oracle_with_unicode(OracleExecutionContext_cx_or """ def __init__(self, *arg, **kw): OracleExecutionContext_cx_oracle.__init__(self, *arg, **kw) - self.statement = unicode(self.statement) + self.statement = util.text_type(self.statement) def _execute_scalar(self, stmt): return super(OracleExecutionContext_cx_oracle_with_unicode, self).\ - _execute_scalar(unicode(stmt)) + _execute_scalar(util.text_type(stmt)) -class ReturningResultProxy(base.FullyBufferedResultProxy): + +class ReturningResultProxy(_result.FullyBufferedResultProxy): """Result proxy which stuffs the _returning clause + outparams into the fetch.""" def __init__(self, context, returning_params): @@ -478,19 +525,16 @@ class ReturningResultProxy(base.FullyBufferedResultProxy): def _cursor_description(self): returning = self.context.compiled.returning - - ret = [] - for c in returning: - if hasattr(c, 'name'): - ret.append((c.name, c.type)) - else: - ret.append((c.anon_label, c.type)) - return ret + return [ + ("ret_%d" % i, None) + for i, col in enumerate(returning) + ] def _buffer_rows(self): return collections.deque([tuple(self._returning_params["ret_%d" % i] for i, c in enumerate(self._returning_params))]) + class OracleDialect_cx_oracle(OracleDialect): execution_ctx_cls = OracleExecutionContext_cx_oracle statement_compiler = OracleCompiler_cx_oracle @@ -499,34 +543,36 @@ class OracleDialect_cx_oracle(OracleDialect): colspecs = colspecs = { sqltypes.Numeric: _OracleNumeric, - sqltypes.Date : _OracleDate, # generic type, assume datetime.date is desired + sqltypes.Date: _OracleDate, # generic type, assume datetime.date is desired oracle.DATE: oracle.DATE, # non generic type - passthru - sqltypes.LargeBinary : _OracleBinary, - sqltypes.Boolean : oracle._OracleBoolean, - sqltypes.Interval : _OracleInterval, - oracle.INTERVAL : _OracleInterval, - sqltypes.Text : _OracleText, - sqltypes.String : _OracleString, - sqltypes.UnicodeText : _OracleUnicodeText, - sqltypes.CHAR : _OracleChar, + sqltypes.LargeBinary: _OracleBinary, + sqltypes.Boolean: oracle._OracleBoolean, + sqltypes.Interval: _OracleInterval, + oracle.INTERVAL: _OracleInterval, + sqltypes.Text: _OracleText, + sqltypes.String: _OracleString, + sqltypes.UnicodeText: _OracleUnicodeText, + sqltypes.CHAR: _OracleChar, # a raw LONG is a text type, but does *not* # get the LobMixin with cx_oracle. oracle.LONG: _OracleLong, - sqltypes.Integer : _OracleInteger, # this is only needed for OUT parameters. - # it would be nice if we could not use it otherwise. + # this is only needed for OUT parameters. + # it would be nice if we could not use it otherwise. + sqltypes.Integer: _OracleInteger, + oracle.RAW: _OracleRaw, sqltypes.Unicode: _OracleNVarChar, - sqltypes.NVARCHAR : _OracleNVarChar, + sqltypes.NVARCHAR: _OracleNVarChar, oracle.ROWID: _OracleRowid, } - execute_sequence_format = list def __init__(self, auto_setinputsizes=True, + exclude_setinputsizes=("STRING", "UNICODE"), auto_convert_lobs=True, threaded=True, allow_twophase=True, @@ -536,22 +582,25 @@ class OracleDialect_cx_oracle(OracleDialect): self.threaded = threaded self.arraysize = arraysize self.allow_twophase = allow_twophase - self.supports_timestamp = self.dbapi is None or hasattr(self.dbapi, 'TIMESTAMP' ) + self.supports_timestamp = self.dbapi is None or \ + hasattr(self.dbapi, 'TIMESTAMP') self.auto_setinputsizes = auto_setinputsizes self.auto_convert_lobs = auto_convert_lobs if hasattr(self.dbapi, 'version'): - self.cx_oracle_ver = tuple([int(x) for x in self.dbapi.version.split('.')]) + self.cx_oracle_ver = tuple([int(x) for x in + self.dbapi.version.split('.')]) else: self.cx_oracle_ver = (0, 0, 0) def types(*names): - return set([ - getattr(self.dbapi, name, None) for name in names - ]).difference([None]) + return set( + getattr(self.dbapi, name, None) for name in names + ).difference([None]) - self._cx_oracle_exclude_setinputsizes = types("STRING", "UNICODE") - self._cx_oracle_string_types = types("STRING", "UNICODE", "NCLOB", "CLOB") + self.exclude_setinputsizes = types(*(exclude_setinputsizes or ())) + self._cx_oracle_string_types = types("STRING", "UNICODE", + "NCLOB", "CLOB") self._cx_oracle_unicode_types = types("UNICODE", "NCLOB") self._cx_oracle_binary_types = types("BFILE", "CLOB", "NCLOB", "BLOB") self.supports_unicode_binds = self.cx_oracle_ver >= (5, 0) @@ -573,19 +622,23 @@ class OracleDialect_cx_oracle(OracleDialect): self.supports_unicode_statements = True self.supports_unicode_binds = True self._cx_oracle_with_unicode = True - # Py2K - # There's really no reason to run with WITH_UNICODE under Python 2.x. - # Give the user a hint. - util.warn("cx_Oracle is compiled under Python 2.xx using the " - "WITH_UNICODE flag. Consider recompiling cx_Oracle without " - "this flag, which is in no way necessary for full support of Unicode. " - "Otherwise, all string-holding bind parameters must " - "be explicitly typed using SQLAlchemy's String type or one of its subtypes," - "or otherwise be passed as Python unicode. Plain Python strings " - "passed as bind parameters will be silently corrupted by cx_Oracle." - ) - self.execution_ctx_cls = OracleExecutionContext_cx_oracle_with_unicode - # end Py2K + + if util.py2k: + # There's really no reason to run with WITH_UNICODE under Python 2.x. + # Give the user a hint. + util.warn( + "cx_Oracle is compiled under Python 2.xx using the " + "WITH_UNICODE flag. Consider recompiling cx_Oracle " + "without this flag, which is in no way necessary for full " + "support of Unicode. Otherwise, all string-holding bind " + "parameters must be explicitly typed using SQLAlchemy's " + "String type or one of its subtypes," + "or otherwise be passed as Python unicode. " + "Plain Python strings passed as bind parameters will be " + "silently corrupted by cx_Oracle." + ) + self.execution_ctx_cls = \ + OracleExecutionContext_cx_oracle_with_unicode else: self._cx_oracle_with_unicode = False @@ -603,9 +656,10 @@ class OracleDialect_cx_oracle(OracleDialect): self.dbapi.BLOB: oracle.BLOB(), self.dbapi.BINARY: oracle.RAW(), } + @classmethod def dbapi(cls): - cx_Oracle = __import__('cx_Oracle') + import cx_Oracle return cx_Oracle def initialize(self, connection): @@ -668,6 +722,7 @@ class OracleDialect_cx_oracle(OracleDialect): return cx_Oracle = self.dbapi + def output_type_handler(cursor, name, defaultType, size, precision, scale): # convert all NUMBER with precision + positive scale to Decimal @@ -695,7 +750,7 @@ class OracleDialect_cx_oracle(OracleDialect): arraysize=cursor.arraysize) # allow all strings to come back natively as Unicode elif defaultType in (cx_Oracle.STRING, cx_Oracle.FIXED_CHAR): - return cursor.var(unicode, size, cursor.arraysize) + return cursor.var(util.text_type, size, cursor.arraysize) def on_connect(conn): conn.outputtypehandler = output_type_handler @@ -730,20 +785,19 @@ class OracleDialect_cx_oracle(OracleDialect): twophase=self.allow_twophase, ) - # Py2K - if self._cx_oracle_with_unicode: - for k, v in opts.items(): - if isinstance(v, str): - opts[k] = unicode(v) - else: - for k, v in opts.items(): - if isinstance(v, unicode): - opts[k] = str(v) - # end Py2K + if util.py2k: + if self._cx_oracle_with_unicode: + for k, v in opts.items(): + if isinstance(v, str): + opts[k] = unicode(v) + else: + for k, v in opts.items(): + if isinstance(v, unicode): + opts[k] = str(v) if 'mode' in url.query: opts['mode'] = url.query['mode'] - if isinstance(opts['mode'], basestring): + if isinstance(opts['mode'], util.string_types): mode = opts['mode'].upper() if mode == 'SYSDBA': opts['mode'] = self.dbapi.SYSDBA @@ -769,8 +823,9 @@ class OracleDialect_cx_oracle(OracleDialect): # ORA-03113: end-of-file on communication channel # ORA-03135: connection lost contact # ORA-01033: ORACLE initialization or shutdown in progress + # ORA-02396: exceeded maximum idle time, please connect again # TODO: Others ? - return error.code in (28, 3114, 3113, 3135, 1033) + return error.code in (28, 3114, 3113, 3135, 1033, 2396) else: return False @@ -783,6 +838,11 @@ class OracleDialect_cx_oracle(OracleDialect): id = random.randint(0, 2 ** 128) return (0x1234, "%032x" % id, "%032x" % 9) + def do_executemany(self, cursor, statement, parameters, context=None): + if isinstance(parameters, tuple): + parameters = list(parameters) + cursor.executemany(statement, parameters) + def do_begin_twophase(self, connection, xid): connection.connection.begin(*xid) diff --git a/libs/sqlalchemy/dialects/oracle/zxjdbc.py b/libs/sqlalchemy/dialects/oracle/zxjdbc.py index e4a12ce0..710645b2 100644 --- a/libs/sqlalchemy/dialects/oracle/zxjdbc.py +++ b/libs/sqlalchemy/dialects/oracle/zxjdbc.py @@ -1,16 +1,15 @@ # oracle/zxjdbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the Oracle database via the zxjdbc JDBC connector. - -JDBC Driver ------------ - -The official Oracle JDBC driver is at -http://www.oracle.com/technology/software/tech/java/sqlj_jdbc/index.html. +""" +.. dialect:: oracle+zxjdbc + :name: zxJDBC for Jython + :dbapi: zxjdbc + :connectstring: oracle+zxjdbc://user:pass@host/dbname + :driverurl: http://www.oracle.com/technology/software/tech/java/sqlj_jdbc/index.html. """ import decimal @@ -19,12 +18,13 @@ import re from sqlalchemy import sql, types as sqltypes, util from sqlalchemy.connectors.zxJDBC import ZxJDBCConnector from sqlalchemy.dialects.oracle.base import OracleCompiler, OracleDialect, OracleExecutionContext -from sqlalchemy.engine import base, default +from sqlalchemy.engine import result as _result from sqlalchemy.sql import expression import collections SQLException = zxJDBC = None + class _ZxJDBCDate(sqltypes.Date): def result_processor(self, dialect, coltype): @@ -78,7 +78,7 @@ class OracleCompiler_zxjdbc(OracleCompiler): self.binds[bindparam.key] = bindparam binds.append(self.bindparam_string(self._truncate_bindparam(bindparam))) - return 'RETURNING ' + ', '.join(columns) + " INTO " + ", ".join(binds) + return 'RETURNING ' + ', '.join(columns) + " INTO " + ", ".join(binds) class OracleExecutionContext_zxjdbc(OracleExecutionContext): @@ -95,8 +95,8 @@ class OracleExecutionContext_zxjdbc(OracleExecutionContext): try: try: rrs = self.statement.__statement__.getReturnResultSet() - rrs.next() - except SQLException, sqle: + next(rrs) + except SQLException as sqle: msg = '%s [SQLCode: %d]' % (sqle.getMessage(), sqle.getErrorCode()) if sqle.getSQLState() is not None: msg += ' [SQLState: %s]' % sqle.getSQLState() @@ -113,7 +113,7 @@ class OracleExecutionContext_zxjdbc(OracleExecutionContext): pass self.statement.close() - return base.ResultProxy(self) + return _result.ResultProxy(self) def create_cursor(self): cursor = self._dbapi_connection.cursor() @@ -121,7 +121,7 @@ class OracleExecutionContext_zxjdbc(OracleExecutionContext): return cursor -class ReturningResultProxy(base.FullyBufferedResultProxy): +class ReturningResultProxy(_result.FullyBufferedResultProxy): """ResultProxy backed by the RETURNING ResultSet results.""" @@ -178,7 +178,7 @@ class OracleDialect_zxjdbc(ZxJDBCConnector, OracleDialect): colspecs = util.update_copy( OracleDialect.colspecs, { - sqltypes.Date : _ZxJDBCDate, + sqltypes.Date: _ZxJDBCDate, sqltypes.Numeric: _ZxJDBCNumeric } ) @@ -189,17 +189,19 @@ class OracleDialect_zxjdbc(ZxJDBCConnector, OracleDialect): from java.sql import SQLException from com.ziclix.python.sql import zxJDBC from com.ziclix.python.sql.handler import OracleDataHandler - class OracleReturningDataHandler(OracleDataHandler): + class OracleReturningDataHandler(OracleDataHandler): """zxJDBC DataHandler that specially handles ReturningParam.""" def setJDBCObject(self, statement, index, object, dbtype=None): if type(object) is ReturningParam: statement.registerReturnParameter(index, object.type) elif dbtype is None: - OracleDataHandler.setJDBCObject(self, statement, index, object) + OracleDataHandler.setJDBCObject( + self, statement, index, object) else: - OracleDataHandler.setJDBCObject(self, statement, index, object, dbtype) + OracleDataHandler.setJDBCObject( + self, statement, index, object, dbtype) self.DataHandler = OracleReturningDataHandler def initialize(self, connection): diff --git a/libs/sqlalchemy/dialects/postgres.py b/libs/sqlalchemy/dialects/postgres.py index 82d1a39c..6ed7e18b 100644 --- a/libs/sqlalchemy/dialects/postgres.py +++ b/libs/sqlalchemy/dialects/postgres.py @@ -1,5 +1,5 @@ # dialects/postgres.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/libs/sqlalchemy/dialects/postgresql/__init__.py b/libs/sqlalchemy/dialects/postgresql/__init__.py index 04ae413c..180e9fc7 100644 --- a/libs/sqlalchemy/dialects/postgresql/__init__.py +++ b/libs/sqlalchemy/dialects/postgresql/__init__.py @@ -1,20 +1,29 @@ # postgresql/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy.dialects.postgresql import base, psycopg2, pg8000, pypostgresql, zxjdbc +from . import base, psycopg2, pg8000, pypostgresql, zxjdbc base.dialect = psycopg2.dialect -from sqlalchemy.dialects.postgresql.base import \ - INTEGER, BIGINT, SMALLINT, VARCHAR, CHAR, TEXT, NUMERIC, FLOAT, REAL, INET, \ - CIDR, UUID, BIT, MACADDR, DOUBLE_PRECISION, TIMESTAMP, TIME,\ - DATE, BYTEA, BOOLEAN, INTERVAL, ARRAY, ENUM, dialect +from .base import \ + INTEGER, BIGINT, SMALLINT, VARCHAR, CHAR, TEXT, NUMERIC, FLOAT, REAL, \ + INET, CIDR, UUID, BIT, MACADDR, DOUBLE_PRECISION, TIMESTAMP, TIME, \ + DATE, BYTEA, BOOLEAN, INTERVAL, ARRAY, ENUM, dialect, array, Any, All, \ + TSVECTOR +from .constraints import ExcludeConstraint +from .hstore import HSTORE, hstore +from .json import JSON, JSONElement +from .ranges import INT4RANGE, INT8RANGE, NUMRANGE, DATERANGE, TSRANGE, \ + TSTZRANGE __all__ = ( -'INTEGER', 'BIGINT', 'SMALLINT', 'VARCHAR', 'CHAR', 'TEXT', 'NUMERIC', 'FLOAT', 'REAL', 'INET', -'CIDR', 'UUID', 'BIT', 'MACADDR', 'DOUBLE_PRECISION', 'TIMESTAMP', 'TIME', -'DATE', 'BYTEA', 'BOOLEAN', 'INTERVAL', 'ARRAY', 'ENUM', 'dialect' + 'INTEGER', 'BIGINT', 'SMALLINT', 'VARCHAR', 'CHAR', 'TEXT', 'NUMERIC', + 'FLOAT', 'REAL', 'INET', 'CIDR', 'UUID', 'BIT', 'MACADDR', + 'DOUBLE_PRECISION', 'TIMESTAMP', 'TIME', 'DATE', 'BYTEA', 'BOOLEAN', + 'INTERVAL', 'ARRAY', 'ENUM', 'dialect', 'Any', 'All', 'array', 'HSTORE', + 'hstore', 'INT4RANGE', 'INT8RANGE', 'NUMRANGE', 'DATERANGE', + 'TSRANGE', 'TSTZRANGE', 'json', 'JSON', 'JSONElement' ) diff --git a/libs/sqlalchemy/dialects/postgresql/base.py b/libs/sqlalchemy/dialects/postgresql/base.py index 384b7616..b7979a3e 100644 --- a/libs/sqlalchemy/dialects/postgresql/base.py +++ b/libs/sqlalchemy/dialects/postgresql/base.py @@ -1,13 +1,13 @@ # postgresql/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the PostgreSQL database. +""" +.. dialect:: postgresql + :name: PostgreSQL -For information on connecting using specific drivers, see the documentation -section regarding that driver. Sequences/SERIAL ---------------- @@ -41,23 +41,40 @@ case. To force the usage of RETURNING by default off, specify the flag ``implicit_returning=False`` to :func:`.create_engine`. +.. _postgresql_isolation_level: + Transaction Isolation Level --------------------------- -:func:`.create_engine` accepts an ``isolation_level`` parameter which results -in the command ``SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL -`` being invoked for every new connection. Valid values for this -parameter are ``READ COMMITTED``, ``READ UNCOMMITTED``, ``REPEATABLE READ``, -and ``SERIALIZABLE``:: +All Postgresql dialects support setting of transaction isolation level +both via a dialect-specific parameter ``isolation_level`` +accepted by :func:`.create_engine`, +as well as the ``isolation_level`` argument as passed to :meth:`.Connection.execution_options`. +When using a non-psycopg2 dialect, this feature works by issuing the +command ``SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL +`` for each new connection. + +To set isolation level using :func:`.create_engine`:: engine = create_engine( "postgresql+pg8000://scott:tiger@localhost/test", isolation_level="READ UNCOMMITTED" ) -When using the psycopg2 dialect, a psycopg2-specific method of setting -transaction isolation level is used, but the API of ``isolation_level`` -remains the same - see :ref:`psycopg2_isolation`. +To set using per-connection execution options:: + + connection = engine.connect() + connection = connection.execution_options(isolation_level="READ COMMITTED") + +Valid values for ``isolation_level`` include: + +* ``READ COMMITTED`` +* ``READ UNCOMMITTED`` +* ``REPEATABLE READ`` +* ``SERIALIZABLE`` + +The :mod:`~sqlalchemy.dialects.postgresql.psycopg2` dialect also offers the special level ``AUTOCOMMIT``. See +:ref:`psycopg2_isolation_level` for details. Remote / Cross-Schema Table Introspection @@ -114,6 +131,62 @@ use the :meth:`._UpdateBase.returning` method on a per-statement basis:: where(table.c.name=='foo') print result.fetchall() +.. _postgresql_match: + +Full Text Search +---------------- + +SQLAlchemy makes available the Postgresql ``@@`` operator via the +:meth:`.ColumnElement.match` method on any textual column expression. +On a Postgresql dialect, an expression like the following:: + + select([sometable.c.text.match("search string")]) + +will emit to the database:: + + SELECT text @@ to_tsquery('search string') FROM table + +The Postgresql text search functions such as ``to_tsquery()`` +and ``to_tsvector()`` are available +explicitly using the standard :attr:`.func` construct. For example:: + + select([ + func.to_tsvector('fat cats ate rats').match('cat & rat') + ]) + +Emits the equivalent of:: + + SELECT to_tsvector('fat cats ate rats') @@ to_tsquery('cat & rat') + +The :class:`.postgresql.TSVECTOR` type can provide for explicit CAST:: + + from sqlalchemy.dialects.postgresql import TSVECTOR + from sqlalchemy import select, cast + select([cast("some text", TSVECTOR)]) + +produces a statement equivalent to:: + + SELECT CAST('some text' AS TSVECTOR) AS anon_1 + + +FROM ONLY ... +------------------------ + +The dialect supports PostgreSQL's ONLY keyword for targeting only a particular +table in an inheritance hierarchy. This can be used to produce the +``SELECT ... FROM ONLY``, ``UPDATE ONLY ...``, and ``DELETE FROM ONLY ...`` +syntaxes. It uses SQLAlchemy's hints mechanism:: + + # SELECT ... FROM ONLY ... + result = table.select().with_hint(table, 'ONLY', 'postgresql') + print result.fetchall() + + # UPDATE ONLY ... + table.update(values=dict(foo='bar')).with_hint('ONLY', + dialect_name='postgresql') + + # DELETE FROM ONLY ... + table.delete().with_hint('ONLY', dialect_name='postgresql') .. _postgresql_indexes: @@ -136,9 +209,10 @@ Operator Classes ^^^^^^^^^^^^^^^^^ PostgreSQL allows the specification of an *operator class* for each column of -an index (see http://www.postgresql.org/docs/8.3/interactive/indexes-opclass.html). -The :class:`.Index` construct allows these to be specified via the ``postgresql_ops`` -keyword argument:: +an index (see +http://www.postgresql.org/docs/8.3/interactive/indexes-opclass.html). +The :class:`.Index` construct allows these to be specified via the +``postgresql_ops`` keyword argument:: Index('my_index', my_table.c.id, my_table.c.data, postgresql_ops={ @@ -150,15 +224,15 @@ keyword argument:: ``postgresql_ops`` keyword argument to :class:`.Index` construct. Note that the keys in the ``postgresql_ops`` dictionary are the "key" name of -the :class:`.Column`, i.e. the name used to access it from the ``.c`` collection -of :class:`.Table`, which can be configured to be different than the actual -name of the column as expressed in the database. +the :class:`.Column`, i.e. the name used to access it from the ``.c`` +collection of :class:`.Table`, which can be configured to be different than +the actual name of the column as expressed in the database. Index Types ^^^^^^^^^^^^ -PostgreSQL provides several index types: B-Tree, Hash, GiST, and GIN, as well as -the ability for users to create their own (see +PostgreSQL provides several index types: B-Tree, Hash, GiST, and GIN, as well +as the ability for users to create their own (see http://www.postgresql.org/docs/8.3/static/indexes-types.html). These can be specified on :class:`.Index` using the ``postgresql_using`` keyword argument:: @@ -169,13 +243,13 @@ underlying CREATE INDEX command, so it *must* be a valid index type for your version of PostgreSQL. """ - +from collections import defaultdict import re -from sqlalchemy import sql, schema, exc, util -from sqlalchemy.engine import default, reflection -from sqlalchemy.sql import compiler, expression, util as sql_util -from sqlalchemy import types as sqltypes +from ... import sql, schema, exc, util +from ...engine import default, reflection +from ...sql import compiler, expression, operators +from ... import types as sqltypes try: from uuid import UUID as _python_UUID @@ -194,7 +268,7 @@ RESERVED_WORDS = set( "default", "deferrable", "desc", "distinct", "do", "else", "end", "except", "false", "fetch", "for", "foreign", "from", "grant", "group", "having", "in", "initially", "intersect", "into", "leading", "limit", - "localtime", "localtimestamp", "new", "not", "null", "off", "offset", + "localtime", "localtimestamp", "new", "not", "null", "of", "off", "offset", "old", "on", "only", "or", "order", "placing", "primary", "references", "returning", "select", "session_user", "some", "symmetric", "table", "then", "to", "trailing", "true", "union", "unique", "user", "using", @@ -208,24 +282,30 @@ _DECIMAL_TYPES = (1231, 1700) _FLOAT_TYPES = (700, 701, 1021, 1022) _INT_TYPES = (20, 21, 23, 26, 1005, 1007, 1016) + class BYTEA(sqltypes.LargeBinary): __visit_name__ = 'BYTEA' + class DOUBLE_PRECISION(sqltypes.Float): __visit_name__ = 'DOUBLE_PRECISION' + class INET(sqltypes.TypeEngine): __visit_name__ = "INET" PGInet = INET + class CIDR(sqltypes.TypeEngine): __visit_name__ = "CIDR" PGCidr = CIDR + class MACADDR(sqltypes.TypeEngine): __visit_name__ = "MACADDR" PGMacAddr = MACADDR + class TIMESTAMP(sqltypes.TIMESTAMP): def __init__(self, timezone=False, precision=None): super(TIMESTAMP, self).__init__(timezone=timezone) @@ -237,6 +317,7 @@ class TIME(sqltypes.TIME): super(TIME, self).__init__(timezone=timezone) self.precision = precision + class INTERVAL(sqltypes.TypeEngine): """Postgresql INTERVAL type. @@ -245,6 +326,7 @@ class INTERVAL(sqltypes.TypeEngine): """ __visit_name__ = 'INTERVAL' + def __init__(self, precision=None): self.precision = precision @@ -258,8 +340,10 @@ class INTERVAL(sqltypes.TypeEngine): PGInterval = INTERVAL + class BIT(sqltypes.TypeEngine): __visit_name__ = 'BIT' + def __init__(self, length=None, varying=False): if not varying: # BIT without VARYING defaults to length 1 @@ -271,6 +355,7 @@ class BIT(sqltypes.TypeEngine): PGBit = BIT + class UUID(sqltypes.TypeEngine): """Postgresql UUID type. @@ -295,15 +380,15 @@ class UUID(sqltypes.TypeEngine): """ if as_uuid and _python_UUID is None: raise NotImplementedError( - "This version of Python does not support the native UUID type." - ) + "This version of Python does not support the native UUID type." + ) self.as_uuid = as_uuid def bind_processor(self, dialect): if self.as_uuid: def process(value): if value is not None: - value = str(value) + value = util.text_type(value) return value return process else: @@ -321,19 +406,302 @@ class UUID(sqltypes.TypeEngine): PGUuid = UUID -class ARRAY(sqltypes.MutableType, sqltypes.Concatenable, sqltypes.TypeEngine): +class TSVECTOR(sqltypes.TypeEngine): + """The :class:`.postgresql.TSVECTOR` type implements the Postgresql + text search type TSVECTOR. + + It can be used to do full text queries on natural language + documents. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :ref:`postgresql_match` + + """ + __visit_name__ = 'TSVECTOR' + + + +class _Slice(expression.ColumnElement): + __visit_name__ = 'slice' + type = sqltypes.NULLTYPE + + def __init__(self, slice_, source_comparator): + self.start = source_comparator._check_literal( + source_comparator.expr, + operators.getitem, slice_.start) + self.stop = source_comparator._check_literal( + source_comparator.expr, + operators.getitem, slice_.stop) + + +class Any(expression.ColumnElement): + """Represent the clause ``left operator ANY (right)``. ``right`` must be + an array expression. + + .. seealso:: + + :class:`.postgresql.ARRAY` + + :meth:`.postgresql.ARRAY.Comparator.any` - ARRAY-bound method + + """ + __visit_name__ = 'any' + + def __init__(self, left, right, operator=operators.eq): + self.type = sqltypes.Boolean() + self.left = expression._literal_as_binds(left) + self.right = right + self.operator = operator + + +class All(expression.ColumnElement): + """Represent the clause ``left operator ALL (right)``. ``right`` must be + an array expression. + + .. seealso:: + + :class:`.postgresql.ARRAY` + + :meth:`.postgresql.ARRAY.Comparator.all` - ARRAY-bound method + + """ + __visit_name__ = 'all' + + def __init__(self, left, right, operator=operators.eq): + self.type = sqltypes.Boolean() + self.left = expression._literal_as_binds(left) + self.right = right + self.operator = operator + + +class array(expression.Tuple): + """A Postgresql ARRAY literal. + + This is used to produce ARRAY literals in SQL expressions, e.g.:: + + from sqlalchemy.dialects.postgresql import array + from sqlalchemy.dialects import postgresql + from sqlalchemy import select, func + + stmt = select([ + array([1,2]) + array([3,4,5]) + ]) + + print stmt.compile(dialect=postgresql.dialect()) + + Produces the SQL:: + + SELECT ARRAY[%(param_1)s, %(param_2)s] || + ARRAY[%(param_3)s, %(param_4)s, %(param_5)s]) AS anon_1 + + An instance of :class:`.array` will always have the datatype + :class:`.ARRAY`. The "inner" type of the array is inferred from + the values present, unless the ``type_`` keyword argument is passed:: + + array(['foo', 'bar'], type_=CHAR) + + .. versionadded:: 0.8 Added the :class:`~.postgresql.array` literal type. + + See also: + + :class:`.postgresql.ARRAY` + + """ + __visit_name__ = 'array' + + def __init__(self, clauses, **kw): + super(array, self).__init__(*clauses, **kw) + self.type = ARRAY(self.type) + + def _bind_param(self, operator, obj): + return array(*[ + expression.BindParameter(None, o, _compared_to_operator=operator, + _compared_to_type=self.type, unique=True) + for o in obj + ]) + + def self_group(self, against=None): + return self + + +class ARRAY(sqltypes.Concatenable, sqltypes.TypeEngine): """Postgresql ARRAY type. Represents values as Python lists. - The ARRAY type may not be supported on all DBAPIs. + An :class:`.ARRAY` type is constructed given the "type" + of element:: + + mytable = Table("mytable", metadata, + Column("data", ARRAY(Integer)) + ) + + The above type represents an N-dimensional array, + meaning Postgresql will interpret values with any number + of dimensions automatically. To produce an INSERT + construct that passes in a 1-dimensional array of integers:: + + connection.execute( + mytable.insert(), + data=[1,2,3] + ) + + The :class:`.ARRAY` type can be constructed given a fixed number + of dimensions:: + + mytable = Table("mytable", metadata, + Column("data", ARRAY(Integer, dimensions=2)) + ) + + This has the effect of the :class:`.ARRAY` type + specifying that number of bracketed blocks when a :class:`.Table` + is used in a CREATE TABLE statement, or when the type is used + within a :func:`.expression.cast` construct; it also causes + the bind parameter and result set processing of the type + to optimize itself to expect exactly that number of dimensions. + Note that Postgresql itself still allows N dimensions with such a type. + + SQL expressions of type :class:`.ARRAY` have support for "index" and + "slice" behavior. The Python ``[]`` operator works normally here, given + integer indexes or slices. Note that Postgresql arrays default + to 1-based indexing. The operator produces binary expression + constructs which will produce the appropriate SQL, both for + SELECT statements:: + + select([mytable.c.data[5], mytable.c.data[2:7]]) + + as well as UPDATE statements when the :meth:`.Update.values` method + is used:: + + mytable.update().values({ + mytable.c.data[5]: 7, + mytable.c.data[2:7]: [1, 2, 3] + }) + + :class:`.ARRAY` provides special methods for containment operations, + e.g.:: + + mytable.c.data.contains([1, 2]) + + For a full list of special methods see :class:`.ARRAY.Comparator`. + + .. versionadded:: 0.8 Added support for index and slice operations + to the :class:`.ARRAY` type, including support for UPDATE + statements, and special array containment operations. + + The :class:`.ARRAY` type may not be supported on all DBAPIs. It is known to work on psycopg2 and not pg8000. + See also: + + :class:`.postgresql.array` - produce a literal array value. """ __visit_name__ = 'ARRAY' - def __init__(self, item_type, mutable=False, as_tuple=False): + class Comparator(sqltypes.Concatenable.Comparator): + """Define comparison operations for :class:`.ARRAY`.""" + + def __getitem__(self, index): + if isinstance(index, slice): + index = _Slice(index, self) + return_type = self.type + else: + return_type = self.type.item_type + return self._binary_operate(self.expr, operators.getitem, index, + result_type=return_type) + + def any(self, other, operator=operators.eq): + """Return ``other operator ANY (array)`` clause. + + Argument places are switched, because ANY requires array + expression to be on the right hand-side. + + E.g.:: + + from sqlalchemy.sql import operators + + conn.execute( + select([table.c.data]).where( + table.c.data.any(7, operator=operators.lt) + ) + ) + + :param other: expression to be compared + :param operator: an operator object from the + :mod:`sqlalchemy.sql.operators` + package, defaults to :func:`.operators.eq`. + + .. seealso:: + + :class:`.postgresql.Any` + + :meth:`.postgresql.ARRAY.Comparator.all` + + """ + return Any(other, self.expr, operator=operator) + + def all(self, other, operator=operators.eq): + """Return ``other operator ALL (array)`` clause. + + Argument places are switched, because ALL requires array + expression to be on the right hand-side. + + E.g.:: + + from sqlalchemy.sql import operators + + conn.execute( + select([table.c.data]).where( + table.c.data.all(7, operator=operators.lt) + ) + ) + + :param other: expression to be compared + :param operator: an operator object from the + :mod:`sqlalchemy.sql.operators` + package, defaults to :func:`.operators.eq`. + + .. seealso:: + + :class:`.postgresql.All` + + :meth:`.postgresql.ARRAY.Comparator.any` + + """ + return All(other, self.expr, operator=operator) + + def contains(self, other, **kwargs): + """Boolean expression. Test if elements are a superset of the + elements of the argument array expression. + """ + return self.expr.op('@>')(other) + + def contained_by(self, other): + """Boolean expression. Test if elements are a proper subset of the + elements of the argument array expression. + """ + return self.expr.op('<@')(other) + + def overlap(self, other): + """Boolean expression. Test if array has elements in common with + an argument array expression. + """ + return self.expr.op('&&')(other) + + def _adapt_expression(self, op, other_comparator): + if isinstance(op, operators.custom_op): + if op.opstring in ['@>', '<@', '&&']: + return op, sqltypes.Boolean + return sqltypes.Concatenable.Comparator.\ + _adapt_expression(self, op, other_comparator) + + comparator_factory = Comparator + + def __init__(self, item_type, as_tuple=False, dimensions=None): """Construct an ARRAY. E.g.:: @@ -345,31 +713,20 @@ class ARRAY(sqltypes.MutableType, sqltypes.Concatenable, sqltypes.TypeEngine): :param item_type: The data type of items of this array. Note that dimensionality is irrelevant here, so multi-dimensional arrays like ``INTEGER[][]``, are constructed as ``ARRAY(Integer)``, not as - ``ARRAY(ARRAY(Integer))`` or such. The type mapping figures out on - the fly - - :param mutable=False: Specify whether lists passed to this - class should be considered mutable - this enables - "mutable types" mode in the ORM. Be sure to read the - notes for :class:`.MutableType` regarding ORM - performance implications. - - .. versionchanged:: 0.7.0 - Default changed from ``True``\ . - - .. versionchanged:: 0.7 - This functionality is now superseded by the - ``sqlalchemy.ext.mutable`` extension described in - :ref:`mutable_toplevel`. + ``ARRAY(ARRAY(Integer))`` or such. :param as_tuple=False: Specify whether return results should be converted to tuples from lists. DBAPIs such as psycopg2 return lists by default. When tuples are - returned, the results are hashable. This flag can only - be set to ``True`` when ``mutable`` is set to - ``False``. + returned, the results are hashable. - .. versionadded:: 0.6.5 + :param dimensions: if non-None, the ARRAY will assume a fixed + number of dimensions. This will cause the DDL emitted for this + ARRAY to include the exact number of bracket clauses ``[]``, + and will also optimize the performance of the type overall. + Note that PG arrays are always implicitly "non-dimensioned", + meaning they can store any number of dimensions no matter how + they were declared. """ if isinstance(item_type, ARRAY): @@ -378,77 +735,68 @@ class ARRAY(sqltypes.MutableType, sqltypes.Concatenable, sqltypes.TypeEngine): if isinstance(item_type, type): item_type = item_type() self.item_type = item_type - self.mutable = mutable - if mutable and as_tuple: - raise exc.ArgumentError( - "mutable must be set to False if as_tuple is True." - ) self.as_tuple = as_tuple - - def copy_value(self, value): - if value is None: - return None - elif self.mutable: - return list(value) - else: - return value + self.dimensions = dimensions def compare_values(self, x, y): return x == y - def is_mutable(self): - return self.mutable + def _proc_array(self, arr, itemproc, dim, collection): + if dim is None: + arr = list(arr) + if dim == 1 or dim is None and ( + # this has to be (list, tuple), or at least + # not hasattr('__iter__'), since Py3K strings + # etc. have __iter__ + not arr or not isinstance(arr[0], (list, tuple))): + if itemproc: + return collection(itemproc(x) for x in arr) + else: + return collection(arr) + else: + return collection( + self._proc_array( + x, itemproc, + dim - 1 if dim is not None else None, + collection) + for x in arr + ) def bind_processor(self, dialect): - item_proc = self.item_type.dialect_impl(dialect).bind_processor(dialect) - if item_proc: - def convert_item(item): - if isinstance(item, (list, tuple)): - return [convert_item(child) for child in item] - else: - return item_proc(item) - else: - def convert_item(item): - if isinstance(item, (list, tuple)): - return [convert_item(child) for child in item] - else: - return item + item_proc = self.item_type.\ + dialect_impl(dialect).\ + bind_processor(dialect) + def process(value): if value is None: return value - return [convert_item(item) for item in value] + else: + return self._proc_array( + value, + item_proc, + self.dimensions, + list) return process def result_processor(self, dialect, coltype): - item_proc = self.item_type.dialect_impl(dialect).result_processor(dialect, coltype) - if item_proc: - def convert_item(item): - if isinstance(item, list): - r = [convert_item(child) for child in item] - if self.as_tuple: - r = tuple(r) - return r - else: - return item_proc(item) - else: - def convert_item(item): - if isinstance(item, list): - r = [convert_item(child) for child in item] - if self.as_tuple: - r = tuple(r) - return r - else: - return item + item_proc = self.item_type.\ + dialect_impl(dialect).\ + result_processor(dialect, coltype) + def process(value): if value is None: return value - r = [convert_item(item) for item in value] - if self.as_tuple: - r = tuple(r) - return r + else: + return self._proc_array( + value, + item_proc, + self.dimensions, + tuple if self.as_tuple else list) return process + PGArray = ARRAY + class ENUM(sqltypes.Enum): """Postgresql ENUM type. @@ -585,71 +933,107 @@ class ENUM(sqltypes.Enum): self.drop(bind=bind, checkfirst=checkfirst) colspecs = { - sqltypes.Interval:INTERVAL, - sqltypes.Enum:ENUM, + sqltypes.Interval: INTERVAL, + sqltypes.Enum: ENUM, } ischema_names = { - 'integer' : INTEGER, - 'bigint' : BIGINT, - 'smallint' : SMALLINT, - 'character varying' : VARCHAR, - 'character' : CHAR, - '"char"' : sqltypes.String, - 'name' : sqltypes.String, - 'text' : TEXT, - 'numeric' : NUMERIC, - 'float' : FLOAT, - 'real' : REAL, + 'integer': INTEGER, + 'bigint': BIGINT, + 'smallint': SMALLINT, + 'character varying': VARCHAR, + 'character': CHAR, + '"char"': sqltypes.String, + 'name': sqltypes.String, + 'text': TEXT, + 'numeric': NUMERIC, + 'float': FLOAT, + 'real': REAL, 'inet': INET, 'cidr': CIDR, 'uuid': UUID, 'bit': BIT, 'bit varying': BIT, 'macaddr': MACADDR, - 'double precision' : DOUBLE_PRECISION, - 'timestamp' : TIMESTAMP, - 'timestamp with time zone' : TIMESTAMP, - 'timestamp without time zone' : TIMESTAMP, - 'time with time zone' : TIME, - 'time without time zone' : TIME, - 'date' : DATE, + 'double precision': DOUBLE_PRECISION, + 'timestamp': TIMESTAMP, + 'timestamp with time zone': TIMESTAMP, + 'timestamp without time zone': TIMESTAMP, + 'time with time zone': TIME, + 'time without time zone': TIME, + 'date': DATE, 'time': TIME, - 'bytea' : BYTEA, - 'boolean' : BOOLEAN, - 'interval':INTERVAL, - 'interval year to month':INTERVAL, - 'interval day to second':INTERVAL, + 'bytea': BYTEA, + 'boolean': BOOLEAN, + 'interval': INTERVAL, + 'interval year to month': INTERVAL, + 'interval day to second': INTERVAL, + 'tsvector' : TSVECTOR } - class PGCompiler(compiler.SQLCompiler): - def visit_match_op(self, binary, **kw): + def visit_array(self, element, **kw): + return "ARRAY[%s]" % self.visit_clauselist(element, **kw) + + def visit_slice(self, element, **kw): + return "%s:%s" % ( + self.process(element.start, **kw), + self.process(element.stop, **kw), + ) + + def visit_any(self, element, **kw): + return "%s%sANY (%s)" % ( + self.process(element.left, **kw), + compiler.OPERATORS[element.operator], + self.process(element.right, **kw) + ) + + def visit_all(self, element, **kw): + return "%s%sALL (%s)" % ( + self.process(element.left, **kw), + compiler.OPERATORS[element.operator], + self.process(element.right, **kw) + ) + + def visit_getitem_binary(self, binary, operator, **kw): + return "%s[%s]" % ( + self.process(binary.left, **kw), + self.process(binary.right, **kw) + ) + + def visit_match_op_binary(self, binary, operator, **kw): return "%s @@ to_tsquery(%s)" % ( - self.process(binary.left), - self.process(binary.right)) + self.process(binary.left, **kw), + self.process(binary.right, **kw)) - def visit_ilike_op(self, binary, **kw): + def visit_ilike_op_binary(self, binary, operator, **kw): escape = binary.modifiers.get("escape", None) - return '%s ILIKE %s' % \ - (self.process(binary.left), self.process(binary.right)) \ - + (escape and - (' ESCAPE ' + self.render_literal_value(escape, None)) - or '') - def visit_notilike_op(self, binary, **kw): + return '%s ILIKE %s' % \ + (self.process(binary.left, **kw), + self.process(binary.right, **kw)) \ + + ( + ' ESCAPE ' + + self.render_literal_value(escape, sqltypes.STRINGTYPE) + if escape else '' + ) + + def visit_notilike_op_binary(self, binary, operator, **kw): escape = binary.modifiers.get("escape", None) return '%s NOT ILIKE %s' % \ - (self.process(binary.left), self.process(binary.right)) \ - + (escape and - (' ESCAPE ' + self.render_literal_value(escape, None)) - or '') + (self.process(binary.left, **kw), + self.process(binary.right, **kw)) \ + + ( + ' ESCAPE ' + + self.render_literal_value(escape, sqltypes.STRINGTYPE) + if escape else '' + ) def render_literal_value(self, value, type_): value = super(PGCompiler, self).render_literal_value(value, type_) - # TODO: need to inspect "standard_conforming_strings" + if self.dialect._backslash_escapes: value = value.replace('\\', '\\\\') return value @@ -660,13 +1044,18 @@ class PGCompiler(compiler.SQLCompiler): def limit_clause(self, select): text = "" if select._limit is not None: - text += " \n LIMIT " + self.process(sql.literal(select._limit)) + text += " \n LIMIT " + self.process(sql.literal(select._limit)) if select._offset is not None: if select._limit is None: text += " \n LIMIT ALL" text += " OFFSET " + self.process(sql.literal(select._offset)) return text + def format_from_hint_text(self, sqltext, table, hint, iscrud): + if hint.upper() != 'ONLY': + raise exc.CompileError("Unrecognized hint: %r" % hint) + return "ONLY " + sqltext + def get_select_precolumns(self, select): if select._distinct is not False: if select._distinct is True: @@ -674,70 +1063,72 @@ class PGCompiler(compiler.SQLCompiler): elif isinstance(select._distinct, (list, tuple)): return "DISTINCT ON (" + ', '.join( [self.process(col) for col in select._distinct] - )+ ") " + ) + ") " else: return "DISTINCT ON (" + self.process(select._distinct) + ") " else: return "" def for_update_clause(self, select): - if select.for_update == 'nowait': - return " FOR UPDATE NOWAIT" - elif select.for_update == 'read': - return " FOR SHARE" - elif select.for_update == 'read_nowait': - return " FOR SHARE NOWAIT" + + if select._for_update_arg.read: + tmp = " FOR SHARE" else: - return super(PGCompiler, self).for_update_clause(select) + tmp = " FOR UPDATE" + + if select._for_update_arg.of: + tables = util.OrderedSet( + c.table if isinstance(c, expression.ColumnClause) + else c for c in select._for_update_arg.of) + tmp += " OF " + ", ".join( + self.process(table, ashint=True) + for table in tables + ) + + if select._for_update_arg.nowait: + tmp += " NOWAIT" + + return tmp def returning_clause(self, stmt, returning_cols): columns = [ - self.process( - self.label_select_column(None, c, asfrom=False), - within_columns_clause=True, - result_map=self.result_map) + self._label_select_column(None, c, True, False, {}) for c in expression._select_iterables(returning_cols) ] return 'RETURNING ' + ', '.join(columns) - def visit_extract(self, extract, **kwargs): - field = self.extract_map.get(extract.field, extract.field) - if extract.expr.type: - affinity = extract.expr.type._type_affinity - else: - affinity = None - casts = { - sqltypes.Date:'date', - sqltypes.DateTime:'timestamp', - sqltypes.Interval:'interval', sqltypes.Time:'time' - } - cast = casts.get(affinity, None) - if isinstance(extract.expr, sql.ColumnElement) and cast is not None: - expr = extract.expr.op('::')(sql.literal_column(cast)) + def visit_substring_func(self, func, **kw): + s = self.process(func.clauses.clauses[0], **kw) + start = self.process(func.clauses.clauses[1], **kw) + if len(func.clauses.clauses) > 2: + length = self.process(func.clauses.clauses[2], **kw) + return "SUBSTRING(%s FROM %s FOR %s)" % (s, start, length) else: - expr = extract.expr - return "EXTRACT(%s FROM %s)" % ( - field, self.process(expr)) + return "SUBSTRING(%s FROM %s)" % (s, start) class PGDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kwargs): + colspec = self.preparer.format_column(column) impl_type = column.type.dialect_impl(self.dialect) if column.primary_key and \ column is column.table._autoincrement_column and \ - not isinstance(impl_type, sqltypes.SmallInteger) and \ ( + self.dialect.supports_smallserial or + not isinstance(impl_type, sqltypes.SmallInteger) + ) and ( column.default is None or ( isinstance(column.default, schema.Sequence) and column.default.optional - ) - ): + )): if isinstance(impl_type, sqltypes.BigInteger): colspec += " BIGSERIAL" + elif isinstance(impl_type, sqltypes.SmallInteger): + colspec += " SMALLSERIAL" else: colspec += " SERIAL" else: @@ -755,7 +1146,9 @@ class PGDDLCompiler(compiler.DDLCompiler): return "CREATE TYPE %s AS ENUM (%s)" % ( self.preparer.format_type(type_), - ",".join("'%s'" % e for e in type_.enums) + ", ".join( + self.sql_compiler.process(sql.literal(e), literal_binds=True) + for e in type_.enums) ) def visit_drop_enum_type(self, drop): @@ -768,46 +1161,67 @@ class PGDDLCompiler(compiler.DDLCompiler): def visit_create_index(self, create): preparer = self.preparer index = create.element + self._verify_index_table(index) text = "CREATE " if index.unique: text += "UNIQUE " - ops = index.kwargs.get('postgresql_ops', {}) text += "INDEX %s ON %s " % ( - preparer.quote( - self._index_identifier(index.name), index.quote), + self._prepared_index_name(index, + include_schema=False), preparer.format_table(index.table) ) if 'postgresql_using' in index.kwargs: using = index.kwargs['postgresql_using'] - text += "USING %s " % preparer.quote(using, index.quote) + text += "USING %s " % preparer.quote(using) + ops = index.kwargs.get('postgresql_ops', {}) text += "(%s)" \ % ( ', '.join([ - preparer.format_column(c) + + self.sql_compiler.process( + expr.self_group() + if not isinstance(expr, expression.ColumnClause) + else expr, + include_table=False, literal_binds=True) + (c.key in ops and (' ' + ops[c.key]) or '') - for c in index.columns]) + for expr, c in zip(index.expressions, index.columns)]) ) - if "postgres_where" in index.kwargs: - whereclause = index.kwargs['postgres_where'] - util.warn_deprecated( - "The 'postgres_where' argument has been renamed " - "to 'postgresql_where'.") - elif 'postgresql_where' in index.kwargs: + if 'postgresql_where' in index.kwargs: whereclause = index.kwargs['postgresql_where'] else: whereclause = None if whereclause is not None: - whereclause = sql_util.expression_as_ddl(whereclause) - where_compiled = self.sql_compiler.process(whereclause) + where_compiled = self.sql_compiler.process( + whereclause, include_table=False, + literal_binds=True) text += " WHERE " + where_compiled return text + def visit_exclude_constraint(self, constraint): + text = "" + if constraint.name is not None: + text += "CONSTRAINT %s " % \ + self.preparer.format_constraint(constraint) + elements = [] + for c in constraint.columns: + op = constraint.operators[c.name] + elements.append(self.preparer.quote(c.name) + ' WITH '+op) + text += "EXCLUDE USING %s (%s)" % (constraint.using, ', '.join(elements)) + if constraint.where is not None: + text += ' WHERE (%s)' % self.sql_compiler.process( + constraint.where, + literal_binds=True) + text += self.define_constraint_deferrability(constraint) + return text + class PGTypeCompiler(compiler.GenericTypeCompiler): + def visit_TSVECTOR(self, type): + return "TSVECTOR" + def visit_INET(self, type_): return "INET" @@ -829,6 +1243,30 @@ class PGTypeCompiler(compiler.GenericTypeCompiler): def visit_BIGINT(self, type_): return "BIGINT" + def visit_HSTORE(self, type_): + return "HSTORE" + + def visit_JSON(self, type_): + return "JSON" + + def visit_INT4RANGE(self, type_): + return "INT4RANGE" + + def visit_INT8RANGE(self, type_): + return "INT8RANGE" + + def visit_NUMRANGE(self, type_): + return "NUMRANGE" + + def visit_DATERANGE(self, type_): + return "DATERANGE" + + def visit_TSRANGE(self, type_): + return "TSRANGE" + + def visit_TSTZRANGE(self, type_): + return "TSTZRANGE" + def visit_datetime(self, type_): return self.visit_TIMESTAMP(type_) @@ -880,7 +1318,9 @@ class PGTypeCompiler(compiler.GenericTypeCompiler): return "BYTEA" def visit_ARRAY(self, type_): - return self.process(type_.item_type) + '[]' + return self.process(type_.item_type) + ('[]' * (type_.dimensions + if type_.dimensions + is not None else 1)) class PGIdentifierPreparer(compiler.IdentifierPreparer): @@ -897,11 +1337,12 @@ class PGIdentifierPreparer(compiler.IdentifierPreparer): if not type_.name: raise exc.CompileError("Postgresql ENUM type requires a name.") - name = self.quote(type_.name, type_.quote) + name = self.quote(type_.name) if not self.omit_schema and use_schema and type_.schema is not None: - name = self.quote_schema(type_.schema, type_.quote) + "." + name + name = self.quote_schema(type_.schema) + "." + name return name + class PGInspector(reflection.Inspector): def __init__(self, conn): @@ -913,11 +1354,14 @@ class PGInspector(reflection.Inspector): return self.dialect.get_table_oid(self.bind, table_name, schema, info_cache=self.info_cache) + class CreateEnumType(schema._CreateDropBase): - __visit_name__ = "create_enum_type" + __visit_name__ = "create_enum_type" + class DropEnumType(schema._CreateDropBase): - __visit_name__ = "drop_enum_type" + __visit_name__ = "drop_enum_type" + class PGExecutionContext(default.DefaultExecutionContext): def fire_sequence(self, seq, type_): @@ -947,7 +1391,8 @@ class PGExecutionContext(default.DefaultExecutionContext): col = column.name tab = tab[0:29 + max(0, (29 - len(col)))] col = col[0:29 + max(0, (29 - len(tab)))] - column._postgresql_seq_name = seq_name = "%s_%s_seq" % (tab, col) + name = "%s_%s_seq" % (tab, col) + column._postgresql_seq_name = seq_name = name sch = column.table.schema if sch is not None: @@ -961,6 +1406,7 @@ class PGExecutionContext(default.DefaultExecutionContext): return super(PGExecutionContext, self).get_insert_default(column) + class PGDialect(default.DefaultDialect): name = 'postgresql' supports_alter = True @@ -969,6 +1415,7 @@ class PGDialect(default.DefaultDialect): supports_native_enum = True supports_native_boolean = True + supports_smallserial = True supports_sequences = True sequences_optional = True @@ -977,6 +1424,7 @@ class PGDialect(default.DefaultDialect): supports_default_values = True supports_empty_insert = False + supports_multivalues_insert = True default_paramstyle = 'pyformat' ischema_names = ischema_names colspecs = colspecs @@ -989,12 +1437,14 @@ class PGDialect(default.DefaultDialect): inspector = PGInspector isolation_level = None - # TODO: need to inspect "standard_conforming_strings" _backslash_escapes = True - def __init__(self, isolation_level=None, **kwargs): + def __init__(self, isolation_level=None, json_serializer=None, + json_deserializer=None, **kwargs): default.DefaultDialect.__init__(self, **kwargs) self.isolation_level = isolation_level + self._json_deserializer = json_deserializer + self._json_serializer = json_serializer def initialize(self, connection): super(PGDialect, self).initialize(connection) @@ -1008,6 +1458,13 @@ class PGDialect(default.DefaultDialect): # psycopg2, others may have placed ENUM here as well self.colspecs.pop(ENUM, None) + # http://www.postgresql.org/docs/9.3/static/release-9-2.html#AEN116689 + self.supports_smallserial = self.server_version_info >= (9, 2) + + self._backslash_escapes = connection.scalar( + "show standard_conforming_strings" + ) == 'off' + def on_connect(self): if self.isolation_level is not None: def connect(conn): @@ -1082,12 +1539,13 @@ class PGDialect(default.DefaultDialect): return connection.scalar("select current_schema()") def has_schema(self, connection, schema): + query = "select nspname from pg_namespace where lower(nspname)=:schema" cursor = connection.execute( sql.text( - "select nspname from pg_namespace where lower(nspname)=:schema", + query, bindparams=[ sql.bindparam( - 'schema', unicode(schema.lower()), + 'schema', util.text_type(schema.lower()), type_=sqltypes.Unicode)] ) ) @@ -1103,7 +1561,7 @@ class PGDialect(default.DefaultDialect): "n.oid=c.relnamespace where n.nspname=current_schema() and " "relname=:name", bindparams=[ - sql.bindparam('name', unicode(table_name), + sql.bindparam('name', util.text_type(table_name), type_=sqltypes.Unicode)] ) ) @@ -1115,9 +1573,9 @@ class PGDialect(default.DefaultDialect): "relname=:name", bindparams=[ sql.bindparam('name', - unicode(table_name), type_=sqltypes.Unicode), + util.text_type(table_name), type_=sqltypes.Unicode), sql.bindparam('schema', - unicode(schema), type_=sqltypes.Unicode)] + util.text_type(schema), type_=sqltypes.Unicode)] ) ) return bool(cursor.first()) @@ -1131,7 +1589,7 @@ class PGDialect(default.DefaultDialect): "n.nspname=current_schema() " "and relname=:name", bindparams=[ - sql.bindparam('name', unicode(sequence_name), + sql.bindparam('name', util.text_type(sequence_name), type_=sqltypes.Unicode) ] ) @@ -1143,10 +1601,10 @@ class PGDialect(default.DefaultDialect): "n.oid=c.relnamespace where relkind='S' and " "n.nspname=:schema and relname=:name", bindparams=[ - sql.bindparam('name', unicode(sequence_name), + sql.bindparam('name', util.text_type(sequence_name), type_=sqltypes.Unicode), sql.bindparam('schema', - unicode(schema), type_=sqltypes.Unicode) + util.text_type(schema), type_=sqltypes.Unicode) ] ) ) @@ -1154,12 +1612,6 @@ class PGDialect(default.DefaultDialect): return bool(cursor.first()) def has_type(self, connection, type_name, schema=None): - bindparams = [ - sql.bindparam('typname', - unicode(type_name), type_=sqltypes.Unicode), - sql.bindparam('nspname', - unicode(schema), type_=sqltypes.Unicode), - ] if schema is not None: query = """ SELECT EXISTS ( @@ -1169,6 +1621,7 @@ class PGDialect(default.DefaultDialect): AND n.nspname = :nspname ) """ + query = sql.text(query) else: query = """ SELECT EXISTS ( @@ -1177,12 +1630,25 @@ class PGDialect(default.DefaultDialect): AND pg_type_is_visible(t.oid) ) """ - cursor = connection.execute(sql.text(query, bindparams=bindparams)) + query = sql.text(query) + query = query.bindparams( + sql.bindparam('typname', + util.text_type(type_name), type_=sqltypes.Unicode), + ) + if schema is not None: + query = query.bindparams( + sql.bindparam('nspname', + util.text_type(schema), type_=sqltypes.Unicode), + ) + cursor = connection.execute(query) return bool(cursor.scalar()) def _get_server_version_info(self, connection): v = connection.execute("select version()").scalar() - m = re.match('PostgreSQL (\d+)\.(\d+)(?:\.(\d+))?(?:devel)?', v) + m = re.match( + '.*(?:PostgreSQL|EnterpriseDB) ' + '(\d+)\.(\d+)(?:\.(\d+))?(?:\.\d+)?(?:devel)?', + v) if not m: raise AssertionError( "Could not determine version from string '%s'" % v) @@ -1211,15 +1677,13 @@ class PGDialect(default.DefaultDialect): """ % schema_where_clause # Since we're binding to unicode, table_name and schema_name must be # unicode. - table_name = unicode(table_name) + table_name = util.text_type(table_name) if schema is not None: - schema = unicode(schema) - s = sql.text(query, bindparams=[ - sql.bindparam('table_name', type_=sqltypes.Unicode), - sql.bindparam('schema', type_=sqltypes.Unicode) - ], - typemap={'oid':sqltypes.Integer} - ) + schema = util.text_type(schema) + s = sql.text(query).bindparams(table_name=sqltypes.Unicode) + s = s.columns(oid=sqltypes.Integer) + if schema: + s = s.bindparams(sql.bindparam('schema', type_=sqltypes.Unicode)) c = connection.execute(s, table_name=table_name, schema=schema) table_oid = c.scalar() if table_oid is None: @@ -1235,13 +1699,13 @@ class PGDialect(default.DefaultDialect): """ rp = connection.execute(s) # what about system tables? - # Py3K - #schema_names = [row[0] for row in rp \ - # if not row[0].startswith('pg_')] - # Py2K - schema_names = [row[0].decode(self.encoding) for row in rp \ + + if util.py2k: + schema_names = [row[0].decode(self.encoding) for row in rp \ + if not row[0].startswith('pg_')] + else: + schema_names = [row[0] for row in rp \ if not row[0].startswith('pg_')] - # end Py2K return schema_names @reflection.cache @@ -1252,17 +1716,16 @@ class PGDialect(default.DefaultDialect): current_schema = self.default_schema_name result = connection.execute( - sql.text(u"SELECT relname FROM pg_class c " + sql.text("SELECT relname FROM pg_class c " "WHERE relkind = 'r' " "AND '%s' = (select nspname from pg_namespace n " "where n.oid = c.relnamespace) " % current_schema, - typemap = {'relname':sqltypes.Unicode} + typemap={'relname': sqltypes.Unicode} ) ) return [row[0] for row in result] - @reflection.cache def get_view_names(self, connection, schema=None, **kw): if schema is not None: @@ -1276,12 +1739,12 @@ class PGDialect(default.DefaultDialect): AND '%(schema)s' = (select nspname from pg_namespace n where n.oid = c.relnamespace) """ % dict(schema=current_schema) - # Py3K - #view_names = [row[0] for row in connection.execute(s)] - # Py2K - view_names = [row[0].decode(self.encoding) + + if util.py2k: + view_names = [row[0].decode(self.encoding) for row in connection.execute(s)] - # end Py2K + else: + view_names = [row[0] for row in connection.execute(s)] return view_names @reflection.cache @@ -1298,11 +1761,10 @@ class PGDialect(default.DefaultDialect): rp = connection.execute(sql.text(s), view_name=view_name, schema=current_schema) if rp: - # Py3K - #view_def = rp.scalar() - # Py2K - view_def = rp.scalar().decode(self.encoding) - # end Py2K + if util.py2k: + view_def = rp.scalar().decode(self.encoding) + else: + view_def = rp.scalar() return view_def @reflection.cache @@ -1313,8 +1775,7 @@ class PGDialect(default.DefaultDialect): SQL_COLS = """ SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), - (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) - for 128) + (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid) FROM pg_catalog.pg_attrdef d WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) @@ -1327,7 +1788,7 @@ class PGDialect(default.DefaultDialect): """ s = sql.text(SQL_COLS, bindparams=[sql.bindparam('table_oid', type_=sqltypes.Integer)], - typemap={'attname':sqltypes.Unicode, 'default':sqltypes.Unicode} + typemap={'attname': sqltypes.Unicode, 'default': sqltypes.Unicode} ) c = connection.execute(s, table_oid=table_oid) rows = c.fetchall() @@ -1337,117 +1798,125 @@ class PGDialect(default.DefaultDialect): # format columns columns = [] for name, format_type, default, notnull, attnum, table_oid in rows: - ## strip (5) from character varying(5), timestamp(5) - # with time zone, etc - attype = re.sub(r'\([\d,]+\)', '', format_type) - - # strip '[]' from integer[], etc. - attype = re.sub(r'\[\]', '', attype) - - nullable = not notnull - is_array = format_type.endswith('[]') - charlen = re.search('\(([\d,]+)\)', format_type) - if charlen: - charlen = charlen.group(1) - kwargs = {} - args = None - - if attype == 'numeric': - if charlen: - prec, scale = charlen.split(',') - args = (int(prec), int(scale)) - else: - args = () - elif attype == 'double precision': - args = (53, ) - elif attype == 'integer': - args = () - elif attype in ('timestamp with time zone', - 'time with time zone'): - kwargs['timezone'] = True - if charlen: - kwargs['precision'] = int(charlen) - args = () - elif attype in ('timestamp without time zone', - 'time without time zone', 'time'): - kwargs['timezone'] = False - if charlen: - kwargs['precision'] = int(charlen) - args = () - elif attype == 'bit varying': - kwargs['varying'] = True - if charlen: - args = (int(charlen),) - else: - args = () - elif attype in ('interval','interval year to month', - 'interval day to second'): - if charlen: - kwargs['precision'] = int(charlen) - args = () - elif charlen: - args = (int(charlen),) - else: - args = () - - while True: - if attype in self.ischema_names: - coltype = self.ischema_names[attype] - break - elif attype in enums: - enum = enums[attype] - coltype = ENUM - if "." in attype: - kwargs['schema'], kwargs['name'] = attype.split('.') - else: - kwargs['name'] = attype - args = tuple(enum['labels']) - break - elif attype in domains: - domain = domains[attype] - attype = domain['attype'] - # A table can't override whether the domain is nullable. - nullable = domain['nullable'] - if domain['default'] and not default: - # It can, however, override the default - # value, but can't set it to null. - default = domain['default'] - continue - else: - coltype = None - break - - if coltype: - coltype = coltype(*args, **kwargs) - if is_array: - coltype = ARRAY(coltype) - else: - util.warn("Did not recognize type '%s' of column '%s'" % - (attype, name)) - coltype = sqltypes.NULLTYPE - # adjust the default value - autoincrement = False - if default is not None: - match = re.search(r"""(nextval\(')([^']+)('.*$)""", default) - if match is not None: - autoincrement = True - # the default is related to a Sequence - sch = schema - if '.' not in match.group(2) and sch is not None: - # unconditionally quote the schema name. this could - # later be enhanced to obey quoting rules / - # "quote schema" - default = match.group(1) + \ - ('"%s"' % sch) + '.' + \ - match.group(2) + match.group(3) - - column_info = dict(name=name, type=coltype, nullable=nullable, - default=default, autoincrement=autoincrement) + column_info = self._get_column_info( + name, format_type, default, notnull, domains, enums, schema) columns.append(column_info) return columns + def _get_column_info(self, name, format_type, default, + notnull, domains, enums, schema): + ## strip (*) from character varying(5), timestamp(5) + # with time zone, geometry(POLYGON), etc. + attype = re.sub(r'\(.*\)', '', format_type) + + # strip '[]' from integer[], etc. + attype = re.sub(r'\[\]', '', attype) + + nullable = not notnull + is_array = format_type.endswith('[]') + charlen = re.search('\(([\d,]+)\)', format_type) + if charlen: + charlen = charlen.group(1) + args = re.search('\((.*)\)', format_type) + if args and args.group(1): + args = tuple(re.split('\s*,\s*', args.group(1))) + else: + args = () + kwargs = {} + + if attype == 'numeric': + if charlen: + prec, scale = charlen.split(',') + args = (int(prec), int(scale)) + else: + args = () + elif attype == 'double precision': + args = (53, ) + elif attype == 'integer': + args = () + elif attype in ('timestamp with time zone', + 'time with time zone'): + kwargs['timezone'] = True + if charlen: + kwargs['precision'] = int(charlen) + args = () + elif attype in ('timestamp without time zone', + 'time without time zone', 'time'): + kwargs['timezone'] = False + if charlen: + kwargs['precision'] = int(charlen) + args = () + elif attype == 'bit varying': + kwargs['varying'] = True + if charlen: + args = (int(charlen),) + else: + args = () + elif attype in ('interval', 'interval year to month', + 'interval day to second'): + if charlen: + kwargs['precision'] = int(charlen) + args = () + elif charlen: + args = (int(charlen),) + + while True: + if attype in self.ischema_names: + coltype = self.ischema_names[attype] + break + elif attype in enums: + enum = enums[attype] + coltype = ENUM + if "." in attype: + kwargs['schema'], kwargs['name'] = attype.split('.') + else: + kwargs['name'] = attype + args = tuple(enum['labels']) + break + elif attype in domains: + domain = domains[attype] + attype = domain['attype'] + # A table can't override whether the domain is nullable. + nullable = domain['nullable'] + if domain['default'] and not default: + # It can, however, override the default + # value, but can't set it to null. + default = domain['default'] + continue + else: + coltype = None + break + + if coltype: + coltype = coltype(*args, **kwargs) + if is_array: + coltype = ARRAY(coltype) + else: + util.warn("Did not recognize type '%s' of column '%s'" % + (attype, name)) + coltype = sqltypes.NULLTYPE + # adjust the default value + autoincrement = False + if default is not None: + match = re.search(r"""(nextval\(')([^']+)('.*$)""", default) + if match is not None: + autoincrement = True + # the default is related to a Sequence + sch = schema + if '.' not in match.group(2) and sch is not None: + # unconditionally quote the schema name. this could + # later be enhanced to obey quoting rules / + # "quote schema" + default = match.group(1) + \ + ('"%s"' % sch) + '.' + \ + match.group(2) + match.group(3) + + column_info = dict(name=name, type=coltype, nullable=nullable, + default=default, autoincrement=autoincrement) + return column_info + @reflection.cache - def get_primary_keys(self, connection, table_name, schema=None, **kw): + def get_pk_constraint(self, connection, table_name, schema=None, **kw): table_oid = self.get_table_oid(connection, table_name, schema, info_cache=kw.get('info_cache')) @@ -1479,16 +1948,7 @@ class PGDialect(default.DefaultDialect): """ t = sql.text(PK_SQL, typemap={'attname': sqltypes.Unicode}) c = connection.execute(t, table_oid=table_oid) - primary_keys = [r[0] for r in c.fetchall()] - return primary_keys - - @reflection.cache - def get_pk_constraint(self, connection, table_name, schema=None, **kw): - cols = self.get_primary_keys(connection, table_name, - schema=schema, **kw) - - table_oid = self.get_table_oid(connection, table_name, schema, - info_cache=kw.get('info_cache')) + cols = [r[0] for r in c.fetchall()] PK_CONS_SQL = """ SELECT conname @@ -1496,13 +1956,11 @@ class PGDialect(default.DefaultDialect): WHERE r.conrelid = :table_oid AND r.contype = 'p' ORDER BY 1 """ - t = sql.text(PK_CONS_SQL, typemap={'conname':sqltypes.Unicode}) + t = sql.text(PK_CONS_SQL, typemap={'conname': sqltypes.Unicode}) c = connection.execute(t, table_oid=table_oid) name = c.scalar() - return { - 'constrained_columns':cols, - 'name':name - } + + return {'constrained_columns': cols, 'name': name} @reflection.cache def get_foreign_keys(self, connection, table_name, schema=None, **kw): @@ -1524,22 +1982,34 @@ class PGDialect(default.DefaultDialect): n.oid = c.relnamespace ORDER BY 1 """ + # http://www.postgresql.org/docs/9.0/static/sql-createtable.html + FK_REGEX = re.compile( + r'FOREIGN KEY \((.*?)\) REFERENCES (?:(.*?)\.)?(.*?)\((.*?)\)' + r'[\s]?(MATCH (FULL|PARTIAL|SIMPLE)+)?' + r'[\s]?(ON UPDATE (CASCADE|RESTRICT|NO ACTION|SET NULL|SET DEFAULT)+)?' + r'[\s]?(ON DELETE (CASCADE|RESTRICT|NO ACTION|SET NULL|SET DEFAULT)+)?' + r'[\s]?(DEFERRABLE|NOT DEFERRABLE)?' + r'[\s]?(INITIALLY (DEFERRED|IMMEDIATE)+)?' + ) t = sql.text(FK_SQL, typemap={ - 'conname':sqltypes.Unicode, - 'condef':sqltypes.Unicode}) + 'conname': sqltypes.Unicode, + 'condef': sqltypes.Unicode}) c = connection.execute(t, table=table_oid) fkeys = [] for conname, condef, conschema in c.fetchall(): - m = re.search('FOREIGN KEY \((.*?)\) REFERENCES ' - '(?:(.*?)\.)?(.*?)\((.*?)\)', condef).groups() + m = re.search(FK_REGEX, condef).groups() constrained_columns, referred_schema, \ - referred_table, referred_columns = m + referred_table, referred_columns, \ + _, match, _, onupdate, _, ondelete, \ + deferrable, _, initially = m + if deferrable is not None: + deferrable = True if deferrable == 'DEFERRABLE' else False constrained_columns = [preparer._unquote_identifier(x) for x in re.split(r'\s*,\s*', constrained_columns)] if referred_schema: - referred_schema =\ + referred_schema = \ preparer._unquote_identifier(referred_schema) elif schema is not None and schema == conschema: # no schema was returned by pg_get_constraintdef(). This @@ -1553,11 +2023,18 @@ class PGDialect(default.DefaultDialect): referred_columns = [preparer._unquote_identifier(x) for x in re.split(r'\s*,\s', referred_columns)] fkey_d = { - 'name' : conname, - 'constrained_columns' : constrained_columns, - 'referred_schema' : referred_schema, - 'referred_table' : referred_table, - 'referred_columns' : referred_columns + 'name': conname, + 'constrained_columns': constrained_columns, + 'referred_schema': referred_schema, + 'referred_table': referred_table, + 'referred_columns': referred_columns, + 'options': { + 'onupdate': onupdate, + 'ondelete': ondelete, + 'deferrable': deferrable, + 'initially': initially, + 'match': match + } } fkeys.append(fkey_d) return fkeys @@ -1567,11 +2044,14 @@ class PGDialect(default.DefaultDialect): table_oid = self.get_table_oid(connection, table_name, schema, info_cache=kw.get('info_cache')) + # cast indkey as varchar since it's an int2vector, + # returned as a list by some drivers such as pypostgresql + IDX_SQL = """ SELECT i.relname as relname, ix.indisunique, ix.indexprs, ix.indpred, - a.attname + a.attname, a.attnum, ix.indkey::varchar FROM pg_class t join pg_index ix on t.oid = ix.indrelid @@ -1588,14 +2068,15 @@ class PGDialect(default.DefaultDialect): i.relname """ - t = sql.text(IDX_SQL, typemap={'attname':sqltypes.Unicode}) + t = sql.text(IDX_SQL, typemap={'attname': sqltypes.Unicode}) c = connection.execute(t, table_oid=table_oid) - index_names = {} - indexes = [] + indexes = defaultdict(lambda: defaultdict(dict)) + sv_idx_name = None for row in c.fetchall(): - idx_name, unique, expr, prd, col = row + idx_name, unique, expr, prd, col, col_num, idx_key = row + if expr: if idx_name != sv_idx_name: util.warn( @@ -1604,22 +2085,61 @@ class PGDialect(default.DefaultDialect): % idx_name) sv_idx_name = idx_name continue + if prd and not idx_name == sv_idx_name: util.warn( "Predicate of partial index %s ignored during reflection" % idx_name) sv_idx_name = idx_name - if idx_name in index_names: - index_d = index_names[idx_name] - else: - index_d = {'column_names':[]} - indexes.append(index_d) - index_names[idx_name] = index_d - index_d['name'] = idx_name + + index = indexes[idx_name] if col is not None: - index_d['column_names'].append(col) - index_d['unique'] = unique - return indexes + index['cols'][col_num] = col + index['key'] = [int(k.strip()) for k in idx_key.split()] + index['unique'] = unique + + return [ + {'name': name, + 'unique': idx['unique'], + 'column_names': [idx['cols'][i] for i in idx['key']]} + for name, idx in indexes.items() + ] + + @reflection.cache + def get_unique_constraints(self, connection, table_name, + schema=None, **kw): + table_oid = self.get_table_oid(connection, table_name, schema, + info_cache=kw.get('info_cache')) + + UNIQUE_SQL = """ + SELECT + cons.conname as name, + cons.conkey as key, + a.attnum as col_num, + a.attname as col_name + FROM + pg_catalog.pg_constraint cons + join pg_attribute a + on cons.conrelid = a.attrelid AND a.attnum = ANY(cons.conkey) + WHERE + cons.conrelid = :table_oid AND + cons.contype = 'u' + """ + + t = sql.text(UNIQUE_SQL, typemap={'col_name': sqltypes.Unicode}) + c = connection.execute(t, table_oid=table_oid) + + uniques = defaultdict(lambda: defaultdict(dict)) + for row in c.fetchall(): + uc = uniques[row.name] + uc["key"] = row.key + uc["cols"][row.col_num] = row.col_name + + return [ + {'name': name, + 'column_names': [uc["cols"][i] for i in uc["key"]]} + for name, uc in uniques.items() + ] def _load_enums(self, connection): if not self.supports_native_enum: @@ -1641,8 +2161,8 @@ class PGDialect(default.DefaultDialect): """ s = sql.text(SQL_ENUMS, typemap={ - 'attname':sqltypes.Unicode, - 'label':sqltypes.Unicode}) + 'attname': sqltypes.Unicode, + 'label': sqltypes.Unicode}) c = connection.execute(s) enums = {} @@ -1679,7 +2199,7 @@ class PGDialect(default.DefaultDialect): WHERE t.typtype = 'd' """ - s = sql.text(SQL_DOMAINS, typemap={'attname':sqltypes.Unicode}) + s = sql.text(SQL_DOMAINS, typemap={'attname': sqltypes.Unicode}) c = connection.execute(s) domains = {} @@ -1696,10 +2216,9 @@ class PGDialect(default.DefaultDialect): name = "%s.%s" % (domain['schema'], domain['name']) domains[name] = { - 'attype':attype, + 'attype': attype, 'nullable': domain['nullable'], 'default': domain['default'] } return domains - diff --git a/libs/sqlalchemy/dialects/postgresql/constraints.py b/libs/sqlalchemy/dialects/postgresql/constraints.py new file mode 100644 index 00000000..f45cef1a --- /dev/null +++ b/libs/sqlalchemy/dialects/postgresql/constraints.py @@ -0,0 +1,73 @@ +# Copyright (C) 2013-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +from sqlalchemy.schema import ColumnCollectionConstraint +from sqlalchemy.sql import expression + +class ExcludeConstraint(ColumnCollectionConstraint): + """A table-level EXCLUDE constraint. + + Defines an EXCLUDE constraint as described in the `postgres + documentation`__. + + __ http://www.postgresql.org/docs/9.0/static/sql-createtable.html#SQL-CREATETABLE-EXCLUDE + """ + + __visit_name__ = 'exclude_constraint' + + where = None + + def __init__(self, *elements, **kw): + """ + :param \*elements: + A sequence of two tuples of the form ``(column, operator)`` where + column must be a column name or Column object and operator must + be a string containing the operator to use. + + :param name: + Optional, the in-database name of this constraint. + + :param deferrable: + Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when + issuing DDL for this constraint. + + :param initially: + Optional string. If set, emit INITIALLY when issuing DDL + for this constraint. + + :param using: + Optional string. If set, emit USING when issuing DDL + for this constraint. Defaults to 'gist'. + + :param where: + Optional string. If set, emit WHERE when issuing DDL + for this constraint. + + """ + ColumnCollectionConstraint.__init__( + self, + *[col for col, op in elements], + name=kw.get('name'), + deferrable=kw.get('deferrable'), + initially=kw.get('initially') + ) + self.operators = {} + for col_or_string, op in elements: + name = getattr(col_or_string, 'name', col_or_string) + self.operators[name] = op + self.using = kw.get('using', 'gist') + where = kw.get('where') + if where: + self.where = expression._literal_as_text(where) + + def copy(self, **kw): + elements = [(col, self.operators[col]) + for col in self.columns.keys()] + c = self.__class__(*elements, + name=self.name, + deferrable=self.deferrable, + initially=self.initially) + c.dispatch._update(self.dispatch) + return c + diff --git a/libs/sqlalchemy/dialects/postgresql/hstore.py b/libs/sqlalchemy/dialects/postgresql/hstore.py new file mode 100644 index 00000000..76562088 --- /dev/null +++ b/libs/sqlalchemy/dialects/postgresql/hstore.py @@ -0,0 +1,369 @@ +# postgresql/hstore.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import re + +from .base import ARRAY, ischema_names +from ... import types as sqltypes +from ...sql import functions as sqlfunc +from ...sql.operators import custom_op +from ... import util + +__all__ = ('HSTORE', 'hstore') + +# My best guess at the parsing rules of hstore literals, since no formal +# grammar is given. This is mostly reverse engineered from PG's input parser +# behavior. +HSTORE_PAIR_RE = re.compile(r""" +( + "(?P (\\ . | [^"])* )" # Quoted key +) +[ ]* => [ ]* # Pair operator, optional adjoining whitespace +( + (?P NULL ) # NULL value + | "(?P (\\ . | [^"])* )" # Quoted value +) +""", re.VERBOSE) + +HSTORE_DELIMITER_RE = re.compile(r""" +[ ]* , [ ]* +""", re.VERBOSE) + + +def _parse_error(hstore_str, pos): + """format an unmarshalling error.""" + + ctx = 20 + hslen = len(hstore_str) + + parsed_tail = hstore_str[max(pos - ctx - 1, 0):min(pos, hslen)] + residual = hstore_str[min(pos, hslen):min(pos + ctx + 1, hslen)] + + if len(parsed_tail) > ctx: + parsed_tail = '[...]' + parsed_tail[1:] + if len(residual) > ctx: + residual = residual[:-1] + '[...]' + + return "After %r, could not parse residual at position %d: %r" % ( + parsed_tail, pos, residual) + + +def _parse_hstore(hstore_str): + """Parse an hstore from it's literal string representation. + + Attempts to approximate PG's hstore input parsing rules as closely as + possible. Although currently this is not strictly necessary, since the + current implementation of hstore's output syntax is stricter than what it + accepts as input, the documentation makes no guarantees that will always + be the case. + + + + """ + result = {} + pos = 0 + pair_match = HSTORE_PAIR_RE.match(hstore_str) + + while pair_match is not None: + key = pair_match.group('key').replace(r'\"', '"').replace("\\\\", "\\") + if pair_match.group('value_null'): + value = None + else: + value = pair_match.group('value').replace(r'\"', '"').replace("\\\\", "\\") + result[key] = value + + pos += pair_match.end() + + delim_match = HSTORE_DELIMITER_RE.match(hstore_str[pos:]) + if delim_match is not None: + pos += delim_match.end() + + pair_match = HSTORE_PAIR_RE.match(hstore_str[pos:]) + + if pos != len(hstore_str): + raise ValueError(_parse_error(hstore_str, pos)) + + return result + + +def _serialize_hstore(val): + """Serialize a dictionary into an hstore literal. Keys and values must + both be strings (except None for values). + + """ + def esc(s, position): + if position == 'value' and s is None: + return 'NULL' + elif isinstance(s, util.string_types): + return '"%s"' % s.replace("\\", "\\\\").replace('"', r'\"') + else: + raise ValueError("%r in %s position is not a string." % + (s, position)) + + return ', '.join('%s=>%s' % (esc(k, 'key'), esc(v, 'value')) + for k, v in val.items()) + + +class HSTORE(sqltypes.Concatenable, sqltypes.TypeEngine): + """Represent the Postgresql HSTORE type. + + The :class:`.HSTORE` type stores dictionaries containing strings, e.g.:: + + data_table = Table('data_table', metadata, + Column('id', Integer, primary_key=True), + Column('data', HSTORE) + ) + + with engine.connect() as conn: + conn.execute( + data_table.insert(), + data = {"key1": "value1", "key2": "value2"} + ) + + :class:`.HSTORE` provides for a wide range of operations, including: + + * Index operations:: + + data_table.c.data['some key'] == 'some value' + + * Containment operations:: + + data_table.c.data.has_key('some key') + + data_table.c.data.has_all(['one', 'two', 'three']) + + * Concatenation:: + + data_table.c.data + {"k1": "v1"} + + For a full list of special methods see :class:`.HSTORE.comparator_factory`. + + For usage with the SQLAlchemy ORM, it may be desirable to combine + the usage of :class:`.HSTORE` with :class:`.MutableDict` dictionary + now part of the :mod:`sqlalchemy.ext.mutable` + extension. This extension will allow "in-place" changes to the + dictionary, e.g. addition of new keys or replacement/removal of existing + keys to/from the current dictionary, to produce events which will be detected + by the unit of work:: + + from sqlalchemy.ext.mutable import MutableDict + + class MyClass(Base): + __tablename__ = 'data_table' + + id = Column(Integer, primary_key=True) + data = Column(MutableDict.as_mutable(HSTORE)) + + my_object = session.query(MyClass).one() + + # in-place mutation, requires Mutable extension + # in order for the ORM to detect + my_object.data['some_key'] = 'some value' + + session.commit() + + When the :mod:`sqlalchemy.ext.mutable` extension is not used, the ORM + will not be alerted to any changes to the contents of an existing dictionary, + unless that dictionary value is re-assigned to the HSTORE-attribute itself, + thus generating a change event. + + .. versionadded:: 0.8 + + .. seealso:: + + :class:`.hstore` - render the Postgresql ``hstore()`` function. + + + """ + + __visit_name__ = 'HSTORE' + + class comparator_factory(sqltypes.Concatenable.Comparator): + """Define comparison operations for :class:`.HSTORE`.""" + + def has_key(self, other): + """Boolean expression. Test for presence of a key. Note that the + key may be a SQLA expression. + """ + return self.expr.op('?')(other) + + def has_all(self, other): + """Boolean expression. Test for presence of all keys in the PG + array. + """ + return self.expr.op('?&')(other) + + def has_any(self, other): + """Boolean expression. Test for presence of any key in the PG + array. + """ + return self.expr.op('?|')(other) + + def defined(self, key): + """Boolean expression. Test for presence of a non-NULL value for + the key. Note that the key may be a SQLA expression. + """ + return _HStoreDefinedFunction(self.expr, key) + + def contains(self, other, **kwargs): + """Boolean expression. Test if keys are a superset of the keys of + the argument hstore expression. + """ + return self.expr.op('@>')(other) + + def contained_by(self, other): + """Boolean expression. Test if keys are a proper subset of the + keys of the argument hstore expression. + """ + return self.expr.op('<@')(other) + + def __getitem__(self, other): + """Text expression. Get the value at a given key. Note that the + key may be a SQLA expression. + """ + return self.expr.op('->', precedence=5)(other) + + def delete(self, key): + """HStore expression. Returns the contents of this hstore with the + given key deleted. Note that the key may be a SQLA expression. + """ + if isinstance(key, dict): + key = _serialize_hstore(key) + return _HStoreDeleteFunction(self.expr, key) + + def slice(self, array): + """HStore expression. Returns a subset of an hstore defined by + array of keys. + """ + return _HStoreSliceFunction(self.expr, array) + + def keys(self): + """Text array expression. Returns array of keys.""" + return _HStoreKeysFunction(self.expr) + + def vals(self): + """Text array expression. Returns array of values.""" + return _HStoreValsFunction(self.expr) + + def array(self): + """Text array expression. Returns array of alternating keys and + values. + """ + return _HStoreArrayFunction(self.expr) + + def matrix(self): + """Text array expression. Returns array of [key, value] pairs.""" + return _HStoreMatrixFunction(self.expr) + + def _adapt_expression(self, op, other_comparator): + if isinstance(op, custom_op): + if op.opstring in ['?', '?&', '?|', '@>', '<@']: + return op, sqltypes.Boolean + elif op.opstring == '->': + return op, sqltypes.Text + return sqltypes.Concatenable.Comparator.\ + _adapt_expression(self, op, other_comparator) + + def bind_processor(self, dialect): + if util.py2k: + encoding = dialect.encoding + def process(value): + if isinstance(value, dict): + return _serialize_hstore(value).encode(encoding) + else: + return value + else: + def process(value): + if isinstance(value, dict): + return _serialize_hstore(value) + else: + return value + return process + + def result_processor(self, dialect, coltype): + if util.py2k: + encoding = dialect.encoding + def process(value): + if value is not None: + return _parse_hstore(value.decode(encoding)) + else: + return value + else: + def process(value): + if value is not None: + return _parse_hstore(value) + else: + return value + return process + + +ischema_names['hstore'] = HSTORE + + +class hstore(sqlfunc.GenericFunction): + """Construct an hstore value within a SQL expression using the + Postgresql ``hstore()`` function. + + The :class:`.hstore` function accepts one or two arguments as described + in the Postgresql documentation. + + E.g.:: + + from sqlalchemy.dialects.postgresql import array, hstore + + select([hstore('key1', 'value1')]) + + select([ + hstore( + array(['key1', 'key2', 'key3']), + array(['value1', 'value2', 'value3']) + ) + ]) + + .. versionadded:: 0.8 + + .. seealso:: + + :class:`.HSTORE` - the Postgresql ``HSTORE`` datatype. + + """ + type = HSTORE + name = 'hstore' + + +class _HStoreDefinedFunction(sqlfunc.GenericFunction): + type = sqltypes.Boolean + name = 'defined' + + +class _HStoreDeleteFunction(sqlfunc.GenericFunction): + type = HSTORE + name = 'delete' + + +class _HStoreSliceFunction(sqlfunc.GenericFunction): + type = HSTORE + name = 'slice' + + +class _HStoreKeysFunction(sqlfunc.GenericFunction): + type = ARRAY(sqltypes.Text) + name = 'akeys' + + +class _HStoreValsFunction(sqlfunc.GenericFunction): + type = ARRAY(sqltypes.Text) + name = 'avals' + + +class _HStoreArrayFunction(sqlfunc.GenericFunction): + type = ARRAY(sqltypes.Text) + name = 'hstore_to_array' + + +class _HStoreMatrixFunction(sqlfunc.GenericFunction): + type = ARRAY(sqltypes.Text) + name = 'hstore_to_matrix' diff --git a/libs/sqlalchemy/dialects/postgresql/json.py b/libs/sqlalchemy/dialects/postgresql/json.py new file mode 100644 index 00000000..2e29185e --- /dev/null +++ b/libs/sqlalchemy/dialects/postgresql/json.py @@ -0,0 +1,199 @@ +# postgresql/json.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +from __future__ import absolute_import + +import json + +from .base import ischema_names +from ... import types as sqltypes +from ...sql.operators import custom_op +from ... import sql +from ...sql import elements +from ... import util + +__all__ = ('JSON', 'JSONElement') + + +class JSONElement(elements.BinaryExpression): + """Represents accessing an element of a :class:`.JSON` value. + + The :class:`.JSONElement` is produced whenever using the Python index + operator on an expression that has the type :class:`.JSON`:: + + expr = mytable.c.json_data['some_key'] + + The expression typically compiles to a JSON access such as ``col -> key``. + Modifiers are then available for typing behavior, including :meth:`.JSONElement.cast` + and :attr:`.JSONElement.astext`. + + """ + def __init__(self, left, right, astext=False, opstring=None, result_type=None): + self._astext = astext + if opstring is None: + if hasattr(right, '__iter__') and \ + not isinstance(right, util.string_types): + opstring = "#>" + right = "{%s}" % (", ".join(util.text_type(elem) for elem in right)) + else: + opstring = "->" + + self._json_opstring = opstring + operator = custom_op(opstring, precedence=5) + right = left._check_literal(left, operator, right) + super(JSONElement, self).__init__(left, right, operator, type_=result_type) + + @property + def astext(self): + """Convert this :class:`.JSONElement` to use the 'astext' operator + when evaluated. + + E.g.:: + + select([data_table.c.data['some key'].astext]) + + .. seealso:: + + :meth:`.JSONElement.cast` + + """ + if self._astext: + return self + else: + return JSONElement( + self.left, + self.right, + astext=True, + opstring=self._json_opstring + ">", + result_type=sqltypes.String(convert_unicode=True) + ) + + def cast(self, type_): + """Convert this :class:`.JSONElement` to apply both the 'astext' operator + as well as an explicit type cast when evaulated. + + E.g.:: + + select([data_table.c.data['some key'].cast(Integer)]) + + .. seealso:: + + :attr:`.JSONElement.astext` + + """ + if not self._astext: + return self.astext.cast(type_) + else: + return sql.cast(self, type_) + + +class JSON(sqltypes.TypeEngine): + """Represent the Postgresql JSON type. + + The :class:`.JSON` type stores arbitrary JSON format data, e.g.:: + + data_table = Table('data_table', metadata, + Column('id', Integer, primary_key=True), + Column('data', JSON) + ) + + with engine.connect() as conn: + conn.execute( + data_table.insert(), + data = {"key1": "value1", "key2": "value2"} + ) + + :class:`.JSON` provides several operations: + + * Index operations:: + + data_table.c.data['some key'] + + * Index operations returning text (required for text comparison):: + + data_table.c.data['some key'].astext == 'some value' + + * Index operations with a built-in CAST call:: + + data_table.c.data['some key'].cast(Integer) == 5 + + * Path index operations:: + + data_table.c.data[('key_1', 'key_2', ..., 'key_n')] + + * Path index operations returning text (required for text comparison):: + + data_table.c.data[('key_1', 'key_2', ..., 'key_n')].astext == 'some value' + + Index operations return an instance of :class:`.JSONElement`, which represents + an expression such as ``column -> index``. This element then defines + methods such as :attr:`.JSONElement.astext` and :meth:`.JSONElement.cast` + for setting up type behavior. + + The :class:`.JSON` type, when used with the SQLAlchemy ORM, does not detect + in-place mutations to the structure. In order to detect these, the + :mod:`sqlalchemy.ext.mutable` extension must be used. This extension will + allow "in-place" changes to the datastructure to produce events which + will be detected by the unit of work. See the example at :class:`.HSTORE` + for a simple example involving a dictionary. + + Custom serializers and deserializers are specified at the dialect level, + that is using :func:`.create_engine`. The reason for this is that when + using psycopg2, the DBAPI only allows serializers at the per-cursor + or per-connection level. E.g.:: + + engine = create_engine("postgresql://scott:tiger@localhost/test", + json_serializer=my_serialize_fn, + json_deserializer=my_deserialize_fn + ) + + When using the psycopg2 dialect, the json_deserializer is registered + against the database using ``psycopg2.extras.register_default_json``. + + .. versionadded:: 0.9 + + """ + + __visit_name__ = 'JSON' + + class comparator_factory(sqltypes.Concatenable.Comparator): + """Define comparison operations for :class:`.JSON`.""" + + def __getitem__(self, other): + """Get the value at a given key.""" + + return JSONElement(self.expr, other) + + def _adapt_expression(self, op, other_comparator): + if isinstance(op, custom_op): + if op.opstring == '->': + return op, sqltypes.Text + return sqltypes.Concatenable.Comparator.\ + _adapt_expression(self, op, other_comparator) + + def bind_processor(self, dialect): + json_serializer = dialect._json_serializer or json.dumps + if util.py2k: + encoding = dialect.encoding + def process(value): + return json_serializer(value).encode(encoding) + else: + def process(value): + return json_serializer(value) + return process + + def result_processor(self, dialect, coltype): + json_deserializer = dialect._json_deserializer or json.loads + if util.py2k: + encoding = dialect.encoding + def process(value): + return json_deserializer(value.decode(encoding)) + else: + def process(value): + return json_deserializer(value) + return process + + +ischema_names['json'] = JSON diff --git a/libs/sqlalchemy/dialects/postgresql/pg8000.py b/libs/sqlalchemy/dialects/postgresql/pg8000.py index dc72555e..bc73f975 100644 --- a/libs/sqlalchemy/dialects/postgresql/pg8000.py +++ b/libs/sqlalchemy/dialects/postgresql/pg8000.py @@ -1,16 +1,15 @@ # postgresql/pg8000.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the PostgreSQL database via the pg8000 driver. - -Connecting ----------- - -URLs are of the form -``postgresql+pg8000://user:password@host:port/dbname[?key=value&key=value...]``. +""" +.. dialect:: postgresql+pg8000 + :name: pg8000 + :dbapi: pg8000 + :connectstring: postgresql+pg8000://user:password@host:port/dbname[?key=value&key=value...] + :url: http://pybrary.net/pg8000/ Unicode ------- @@ -27,19 +26,22 @@ Passing data from/to the Interval type is not supported as of yet. """ -from sqlalchemy import util, exc -from sqlalchemy.util.compat import decimal -from sqlalchemy import processors -from sqlalchemy import types as sqltypes -from sqlalchemy.dialects.postgresql.base import PGDialect, \ +from ... import util, exc +import decimal +from ... import processors +from ... import types as sqltypes +from .base import PGDialect, \ PGCompiler, PGIdentifierPreparer, PGExecutionContext,\ _DECIMAL_TYPES, _FLOAT_TYPES, _INT_TYPES + class _PGNumeric(sqltypes.Numeric): def result_processor(self, dialect, coltype): if self.asdecimal: if coltype in _FLOAT_TYPES: - return processors.to_decimal_processor_factory(decimal.Decimal) + return processors.to_decimal_processor_factory( + decimal.Decimal, + self._effective_decimal_return_scale) elif coltype in _DECIMAL_TYPES or coltype in _INT_TYPES: # pg8000 returns Decimal natively for 1700 return None @@ -61,17 +63,20 @@ class _PGNumericNoBind(_PGNumeric): def bind_processor(self, dialect): return None + class PGExecutionContext_pg8000(PGExecutionContext): pass class PGCompiler_pg8000(PGCompiler): - def visit_mod(self, binary, **kw): - return self.process(binary.left) + " %% " + self.process(binary.right) + def visit_mod_binary(self, binary, operator, **kw): + return self.process(binary.left, **kw) + " %% " + \ + self.process(binary.right, **kw) def post_process_text(self, text): if '%%' in text: - util.warn("The SQLAlchemy postgresql dialect now automatically escapes '%' in text() " + util.warn("The SQLAlchemy postgresql dialect " + "now automatically escapes '%' in text() " "expressions to '%%'.") return text.replace('%', '%%') @@ -99,8 +104,8 @@ class PGDialect_pg8000(PGDialect): colspecs = util.update_copy( PGDialect.colspecs, { - sqltypes.Numeric : _PGNumericNoBind, - sqltypes.Float : _PGNumeric + sqltypes.Numeric: _PGNumericNoBind, + sqltypes.Float: _PGNumeric } ) diff --git a/libs/sqlalchemy/dialects/postgresql/psycopg2.py b/libs/sqlalchemy/dialects/postgresql/psycopg2.py index ecc8d331..e9f64f82 100644 --- a/libs/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/libs/sqlalchemy/dialects/postgresql/psycopg2.py @@ -1,42 +1,39 @@ # postgresql/psycopg2.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the PostgreSQL database via the psycopg2 driver. +""" +.. dialect:: postgresql+psycopg2 + :name: psycopg2 + :dbapi: psycopg2 + :connectstring: postgresql+psycopg2://user:password@host:port/dbname[?key=value&key=value...] + :url: http://pypi.python.org/pypi/psycopg2/ -Driver ------- - -The psycopg2 driver is available at http://pypi.python.org/pypi/psycopg2/ . -The dialect has several behaviors which are specifically tailored towards compatibility -with this module. - -Note that psycopg1 is **not** supported. - -Connecting ----------- - -URLs are of the form -``postgresql+psycopg2://user:password@host:port/dbname[?key=value&key=value...]``. +psycopg2 Connect Arguments +----------------------------------- psycopg2-specific keyword arguments which are accepted by :func:`.create_engine()` are: -* *server_side_cursors* - Enable the usage of "server side cursors" for SQL +* ``server_side_cursors``: Enable the usage of "server side cursors" for SQL statements which support this feature. What this essentially means from a psycopg2 point of view is that the cursor is created using a name, e.g. ``connection.cursor('some name')``, which has the effect that result rows are not immediately pre-fetched and buffered after statement execution, but are instead left on the server and only retrieved as needed. SQLAlchemy's - :class:`~sqlalchemy.engine.base.ResultProxy` uses special row-buffering + :class:`~sqlalchemy.engine.ResultProxy` uses special row-buffering behavior when this feature is enabled, such that groups of 100 rows at a time are fetched over the wire to reduce conversational overhead. Note that the ``stream_results=True`` execution option is a more targeted way of enabling this mode on a per-execution basis. -* *use_native_unicode* - Enable the usage of Psycopg2 "native unicode" mode - per connection. True by default. +* ``use_native_unicode``: Enable the usage of Psycopg2 "native unicode" mode + per connection. True by default. +* ``isolation_level``: This option, available for all Posgtresql dialects, + includes the ``AUTOCOMMIT`` isolation level when using the psycopg2 + dialect. See :ref:`psycopg2_isolation_level`. + Unix Domain Connections ------------------------ @@ -66,11 +63,14 @@ The following DBAPI-specific options are respected when used with :meth:`.Query.execution_options`, in addition to those not specific to DBAPIs: * isolation_level - Set the transaction isolation level for the lifespan of a - :class:`.Connection` (can only be set on a connection, not a statement or query). - This includes the options ``SERIALIZABLE``, ``READ COMMITTED``, - ``READ UNCOMMITTED`` and ``REPEATABLE READ``. -* stream_results - Enable or disable usage of server side cursors. - If ``None`` or not set, the ``server_side_cursors`` option of the :class:`.Engine` is used. + :class:`.Connection` (can only be set on a connection, not a statement + or query). See :ref:`psycopg2_isolation_level`. + +* stream_results - Enable or disable usage of psycopg2 server side cursors - + this feature makes use of "named" cursors in combination with special + result handling methods so that result rows are not fully buffered. + If ``None`` or not set, the ``server_side_cursors`` option of the + :class:`.Engine` is used. Unicode ------- @@ -98,13 +98,14 @@ on all new connections based on the value passed to This overrides the encoding specified in the Postgresql client configuration. .. versionadded:: 0.7.3 - The psycopg2-specific ``client_encoding`` parameter to :func:`.create_engine`. + The psycopg2-specific ``client_encoding`` parameter to + :func:`.create_engine`. SQLAlchemy can also be instructed to skip the usage of the psycopg2 ``UNICODE`` extension and to instead utilize it's own unicode encode/decode services, which are normally reserved only for those DBAPIs that don't -fully support unicode directly. Passing ``use_native_unicode=False`` -to :func:`.create_engine` will disable usage of ``psycopg2.extensions.UNICODE``. +fully support unicode directly. Passing ``use_native_unicode=False`` to +:func:`.create_engine` will disable usage of ``psycopg2.extensions.UNICODE``. SQLAlchemy will instead encode data itself into Python bytestrings on the way in and coerce from bytes on the way back, using the value of the :func:`.create_engine` ``encoding`` parameter, which @@ -118,16 +119,31 @@ Transactions The psycopg2 dialect fully supports SAVEPOINT and two-phase commit operations. -.. _psycopg2_isolation: +.. _psycopg2_isolation_level: -Transaction Isolation Level ---------------------------- +Psycopg2 Transaction Isolation Level +------------------------------------- -The ``isolation_level`` parameter of :func:`.create_engine` here makes use +As discussed in :ref:`postgresql_isolation_level`, +all Postgresql dialects support setting of transaction isolation level +both via the ``isolation_level`` parameter passed to :func:`.create_engine`, +as well as the ``isolation_level`` argument used by :meth:`.Connection.execution_options`. +When using the psycopg2 dialect, these options make use of psycopg2's ``set_isolation_level()`` connection method, rather than -issuing a ``SET SESSION CHARACTERISTICS`` command. This because psycopg2 -resets the isolation level on each new transaction, and needs to know -at the API level what level should be used. +emitting a Postgresql directive; this is because psycopg2's API-level +setting is always emitted at the start of each transaction in any case. + +The psycopg2 dialect supports these constants for isolation level: + +* ``READ COMMITTED`` +* ``READ UNCOMMITTED`` +* ``REPEATABLE READ`` +* ``SERIALIZABLE`` +* ``AUTOCOMMIT`` + +.. versionadded:: 0.8.2 support for AUTOCOMMIT isolation level when using + psycopg2. + NOTICE logging --------------- @@ -138,22 +154,32 @@ The psycopg2 dialect will log Postgresql NOTICE messages via the import logging logging.getLogger('sqlalchemy.dialects.postgresql').setLevel(logging.INFO) +HSTORE type +------------ + +The psycopg2 dialect will make use of the +``psycopg2.extensions.register_hstore()`` extension when using the HSTORE +type. This replaces SQLAlchemy's pure-Python HSTORE coercion which takes +effect for other DBAPIs. """ +from __future__ import absolute_import import re import logging -from sqlalchemy import util, exc -from sqlalchemy.util.compat import decimal -from sqlalchemy import processors -from sqlalchemy.engine import base -from sqlalchemy.sql import expression -from sqlalchemy import types as sqltypes -from sqlalchemy.dialects.postgresql.base import PGDialect, PGCompiler, \ +from ... import util, exc +import decimal +from ... import processors +from ...engine import result as _result +from ...sql import expression +from ... import types as sqltypes +from .base import PGDialect, PGCompiler, \ PGIdentifierPreparer, PGExecutionContext, \ ENUM, ARRAY, _DECIMAL_TYPES, _FLOAT_TYPES,\ _INT_TYPES +from .hstore import HSTORE +from .json import JSON logger = logging.getLogger('sqlalchemy.dialects.postgresql') @@ -166,7 +192,9 @@ class _PGNumeric(sqltypes.Numeric): def result_processor(self, dialect, coltype): if self.asdecimal: if coltype in _FLOAT_TYPES: - return processors.to_decimal_processor_factory(decimal.Decimal) + return processors.to_decimal_processor_factory( + decimal.Decimal, + self._effective_decimal_return_scale) elif coltype in _DECIMAL_TYPES or coltype in _INT_TYPES: # pg8000 returns Decimal natively for 1700 return None @@ -183,24 +211,37 @@ class _PGNumeric(sqltypes.Numeric): raise exc.InvalidRequestError( "Unknown PG numeric type: %d" % coltype) -class _PGEnum(ENUM): - def __init__(self, *arg, **kw): - super(_PGEnum, self).__init__(*arg, **kw) - # Py2K - if self.convert_unicode: - self.convert_unicode = "force" - # end Py2K -class _PGArray(ARRAY): - def __init__(self, *arg, **kw): - super(_PGArray, self).__init__(*arg, **kw) - # Py2K - # FIXME: this check won't work for setups that - # have convert_unicode only on their create_engine(). - if isinstance(self.item_type, sqltypes.String) and \ - self.item_type.convert_unicode: - self.item_type.convert_unicode = "force" - # end Py2K +class _PGEnum(ENUM): + def result_processor(self, dialect, coltype): + if util.py2k and self.convert_unicode is True: + # we can't easily use PG's extensions here because + # the OID is on the fly, and we need to give it a python + # function anyway - not really worth it. + self.convert_unicode = "force_nocheck" + return super(_PGEnum, self).result_processor(dialect, coltype) + +class _PGHStore(HSTORE): + def bind_processor(self, dialect): + if dialect._has_native_hstore: + return None + else: + return super(_PGHStore, self).bind_processor(dialect) + + def result_processor(self, dialect, coltype): + if dialect._has_native_hstore: + return None + else: + return super(_PGHStore, self).result_processor(dialect, coltype) + + +class _PGJSON(JSON): + + def result_processor(self, dialect, coltype): + if dialect._has_native_json: + return None + else: + return super(_PGJSON, self).result_processor(dialect, coltype) # When we're handed literal SQL, ensure it's a SELECT-query. Since # 8.3, combining cursors and "FOR UPDATE" has been fine. @@ -210,6 +251,7 @@ SERVER_SIDE_CURSOR_RE = re.compile( _server_side_id = util.counter() + class PGExecutionContext_psycopg2(PGExecutionContext): def create_cursor(self): # TODO: coverage for server side cursors + select.for_update() @@ -221,12 +263,13 @@ class PGExecutionContext_psycopg2(PGExecutionContext): or \ ( (not self.compiled or - isinstance(self.compiled.statement, expression._TextClause)) + isinstance(self.compiled.statement, expression.TextClause)) and self.statement and SERVER_SIDE_CURSOR_RE.match(self.statement)) ) ) else: - is_server_side = self.execution_options.get('stream_results', False) + is_server_side = \ + self.execution_options.get('stream_results', False) self.__is_server_side = is_server_side if is_server_side: @@ -243,9 +286,9 @@ class PGExecutionContext_psycopg2(PGExecutionContext): self._log_notices(self.cursor) if self.__is_server_side: - return base.BufferedRowResultProxy(self) + return _result.BufferedRowResultProxy(self) else: - return base.ResultProxy(self) + return _result.ResultProxy(self) def _log_notices(self, cursor): for notice in cursor.connection.notices: @@ -257,8 +300,9 @@ class PGExecutionContext_psycopg2(PGExecutionContext): class PGCompiler_psycopg2(PGCompiler): - def visit_mod(self, binary, **kw): - return self.process(binary.left) + " %% " + self.process(binary.right) + def visit_mod_binary(self, binary, operator, **kw): + return self.process(binary.left, **kw) + " %% " + \ + self.process(binary.right, **kw) def post_process_text(self, text): return text.replace('%', '%%') @@ -269,11 +313,12 @@ class PGIdentifierPreparer_psycopg2(PGIdentifierPreparer): value = value.replace(self.escape_quote, self.escape_to_quote) return value.replace('%', '%%') + class PGDialect_psycopg2(PGDialect): driver = 'psycopg2' - # Py2K - supports_unicode_statements = False - # end Py2K + if util.py2k: + supports_unicode_statements = False + default_paramstyle = 'pyformat' supports_sane_multi_rowcount = False execution_ctx_cls = PGExecutionContext_psycopg2 @@ -281,21 +326,28 @@ class PGDialect_psycopg2(PGDialect): preparer = PGIdentifierPreparer_psycopg2 psycopg2_version = (0, 0) + _has_native_hstore = False + _has_native_json = False + colspecs = util.update_copy( PGDialect.colspecs, { - sqltypes.Numeric : _PGNumeric, - ENUM : _PGEnum, # needs force_unicode - sqltypes.Enum : _PGEnum, # needs force_unicode - ARRAY : _PGArray, # needs force_unicode + sqltypes.Numeric: _PGNumeric, + ENUM: _PGEnum, # needs force_unicode + sqltypes.Enum: _PGEnum, # needs force_unicode + HSTORE: _PGHStore, + JSON: _PGJSON } ) def __init__(self, server_side_cursors=False, use_native_unicode=True, - client_encoding=None, **kwargs): + client_encoding=None, + use_native_hstore=True, + **kwargs): PGDialect.__init__(self, **kwargs) self.server_side_cursors = server_side_cursors self.use_native_unicode = use_native_unicode + self.use_native_hstore = use_native_hstore self.supports_unicode_binds = use_native_unicode self.client_encoding = client_encoding if self.dbapi and hasattr(self.dbapi, '__version__'): @@ -307,19 +359,27 @@ class PGDialect_psycopg2(PGDialect): for x in m.group(1, 2, 3) if x is not None) + def initialize(self, connection): + super(PGDialect_psycopg2, self).initialize(connection) + self._has_native_hstore = self.use_native_hstore and \ + self._hstore_oids(connection.connection) \ + is not None + self._has_native_json = self.psycopg2_version >= (2, 5) + @classmethod def dbapi(cls): - psycopg = __import__('psycopg2') - return psycopg + import psycopg2 + return psycopg2 @util.memoized_property def _isolation_lookup(self): - extensions = __import__('psycopg2.extensions').extensions + from psycopg2 import extensions return { - 'READ COMMITTED':extensions.ISOLATION_LEVEL_READ_COMMITTED, - 'READ UNCOMMITTED':extensions.ISOLATION_LEVEL_READ_UNCOMMITTED, - 'REPEATABLE READ':extensions.ISOLATION_LEVEL_REPEATABLE_READ, - 'SERIALIZABLE':extensions.ISOLATION_LEVEL_SERIALIZABLE + 'AUTOCOMMIT': extensions.ISOLATION_LEVEL_AUTOCOMMIT, + 'READ COMMITTED': extensions.ISOLATION_LEVEL_READ_COMMITTED, + 'READ UNCOMMITTED': extensions.ISOLATION_LEVEL_READ_UNCOMMITTED, + 'REPEATABLE READ': extensions.ISOLATION_LEVEL_REPEATABLE_READ, + 'SERIALIZABLE': extensions.ISOLATION_LEVEL_SERIALIZABLE } def set_isolation_level(self, connection, level): @@ -335,6 +395,8 @@ class PGDialect_psycopg2(PGDialect): connection.set_isolation_level(level) def on_connect(self): + from psycopg2 import extras, extensions + fns = [] if self.client_encoding is not None: def on_connect(conn): @@ -347,9 +409,28 @@ class PGDialect_psycopg2(PGDialect): fns.append(on_connect) if self.dbapi and self.use_native_unicode: - extensions = __import__('psycopg2.extensions').extensions def on_connect(conn): extensions.register_type(extensions.UNICODE, conn) + extensions.register_type(extensions.UNICODEARRAY, conn) + fns.append(on_connect) + + if self.dbapi and self.use_native_hstore: + def on_connect(conn): + hstore_oids = self._hstore_oids(conn) + if hstore_oids is not None: + oid, array_oid = hstore_oids + if util.py2k: + extras.register_hstore(conn, oid=oid, + array_oid=array_oid, + unicode=True) + else: + extras.register_hstore(conn, oid=oid, + array_oid=array_oid) + fns.append(on_connect) + + if self.dbapi and self._json_deserializer: + def on_connect(conn): + extras.register_default_json(conn, loads=self._json_deserializer) fns.append(on_connect) if fns: @@ -360,6 +441,15 @@ class PGDialect_psycopg2(PGDialect): else: return None + @util.memoized_instancemethod + def _hstore_oids(self, conn): + if self.psycopg2_version >= (2, 4): + from psycopg2 import extras + oids = extras.HstoreAdapter.get_oids(conn) + if oids is not None and oids[0]: + return oids[0:2] + return None + def create_connect_args(self, url): opts = url.translate_connect_args(username='user') if 'port' in opts: @@ -368,24 +458,27 @@ class PGDialect_psycopg2(PGDialect): return ([], opts) def is_disconnect(self, e, connection, cursor): - if isinstance(e, self.dbapi.OperationalError): - # these error messages from libpq: interfaces/libpq/fe-misc.c. - # TODO: these are sent through gettext in libpq and we can't - # check within other locales - consider using connection.closed - return 'terminating connection' in str(e) or \ - 'closed the connection' in str(e) or \ - 'connection not open' in str(e) or \ - 'could not receive data from server' in str(e) - elif isinstance(e, self.dbapi.InterfaceError): - # psycopg2 client errors, psycopg2/conenction.h, psycopg2/cursor.h - return 'connection already closed' in str(e) or \ - 'cursor already closed' in str(e) - elif isinstance(e, self.dbapi.ProgrammingError): - # not sure where this path is originally from, it may - # be obsolete. It really says "losed", not "closed". - return "losed the connection unexpectedly" in str(e) - else: - return False + if isinstance(e, self.dbapi.Error): + str_e = str(e).partition("\n")[0] + for msg in [ + # these error messages from libpq: interfaces/libpq/fe-misc.c + # and interfaces/libpq/fe-secure.c. + # TODO: these are sent through gettext in libpq and we can't + # check within other locales - consider using connection.closed + 'terminating connection', + 'closed the connection', + 'connection not open', + 'could not receive data from server', + # psycopg2 client errors, psycopg2/conenction.h, psycopg2/cursor.h + 'connection already closed', + 'cursor already closed', + # not sure where this path is originally from, it may + # be obsolete. It really says "losed", not "closed". + 'losed the connection unexpectedly' + ]: + idx = str_e.find(msg) + if idx >= 0 and '"' not in str_e[:idx]: + return True + return False dialect = PGDialect_psycopg2 - diff --git a/libs/sqlalchemy/dialects/postgresql/pypostgresql.py b/libs/sqlalchemy/dialects/postgresql/pypostgresql.py index 5303d047..f030d2c1 100644 --- a/libs/sqlalchemy/dialects/postgresql/pypostgresql.py +++ b/libs/sqlalchemy/dialects/postgresql/pypostgresql.py @@ -1,22 +1,23 @@ # postgresql/pypostgresql.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the PostgreSQL database via py-postgresql. - -Connecting ----------- - -URLs are of the form ``postgresql+pypostgresql://user:password@host:port/dbname[?key=value&key=value...]``. +""" +.. dialect:: postgresql+pypostgresql + :name: py-postgresql + :dbapi: pypostgresql + :connectstring: postgresql+pypostgresql://user:password@host:port/dbname[?key=value&key=value...] + :url: http://python.projects.pgfoundry.org/ """ -from sqlalchemy import util -from sqlalchemy import types as sqltypes -from sqlalchemy.dialects.postgresql.base import PGDialect, PGExecutionContext -from sqlalchemy import processors +from ... import util +from ... import types as sqltypes +from .base import PGDialect, PGExecutionContext +from ... import processors + class PGNumeric(sqltypes.Numeric): def bind_processor(self, dialect): @@ -28,9 +29,11 @@ class PGNumeric(sqltypes.Numeric): else: return processors.to_float + class PGExecutionContext_pypostgresql(PGExecutionContext): pass + class PGDialect_pypostgresql(PGDialect): driver = 'pypostgresql' @@ -48,8 +51,10 @@ class PGDialect_pypostgresql(PGDialect): colspecs = util.update_copy( PGDialect.colspecs, { - sqltypes.Numeric : PGNumeric, - sqltypes.Float: sqltypes.Float, # prevents PGNumeric from being used + sqltypes.Numeric: PGNumeric, + + # prevents PGNumeric from being used + sqltypes.Float: sqltypes.Float, } ) diff --git a/libs/sqlalchemy/dialects/postgresql/ranges.py b/libs/sqlalchemy/dialects/postgresql/ranges.py new file mode 100644 index 00000000..57b0c4c3 --- /dev/null +++ b/libs/sqlalchemy/dialects/postgresql/ranges.py @@ -0,0 +1,160 @@ +# Copyright (C) 2013-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from .base import ischema_names +from ... import types as sqltypes + +__all__ = ('INT4RANGE', 'INT8RANGE', 'NUMRANGE') + +class RangeOperators(object): + """ + This mixin provides functionality for the Range Operators + listed in Table 9-44 of the `postgres documentation`__ for Range + Functions and Operators. It is used by all the range types + provided in the ``postgres`` dialect and can likely be used for + any range types you create yourself. + + __ http://www.postgresql.org/docs/devel/static/functions-range.html + + No extra support is provided for the Range Functions listed in + Table 9-45 of the postgres documentation. For these, the normal + :func:`~sqlalchemy.sql.expression.func` object should be used. + + .. versionadded:: 0.8.2 Support for Postgresql RANGE operations. + + """ + + class comparator_factory(sqltypes.Concatenable.Comparator): + """Define comparison operations for range types.""" + + def __ne__(self, other): + "Boolean expression. Returns true if two ranges are not equal" + return self.expr.op('<>')(other) + + def contains(self, other, **kw): + """Boolean expression. Returns true if the right hand operand, + which can be an element or a range, is contained within the + column. + """ + return self.expr.op('@>')(other) + + def contained_by(self, other): + """Boolean expression. Returns true if the column is contained + within the right hand operand. + """ + return self.expr.op('<@')(other) + + def overlaps(self, other): + """Boolean expression. Returns true if the column overlaps + (has points in common with) the right hand operand. + """ + return self.expr.op('&&')(other) + + def strictly_left_of(self, other): + """Boolean expression. Returns true if the column is strictly + left of the right hand operand. + """ + return self.expr.op('<<')(other) + + __lshift__ = strictly_left_of + + def strictly_right_of(self, other): + """Boolean expression. Returns true if the column is strictly + right of the right hand operand. + """ + return self.expr.op('>>')(other) + + __rshift__ = strictly_right_of + + def not_extend_right_of(self, other): + """Boolean expression. Returns true if the range in the column + does not extend right of the range in the operand. + """ + return self.expr.op('&<')(other) + + def not_extend_left_of(self, other): + """Boolean expression. Returns true if the range in the column + does not extend left of the range in the operand. + """ + return self.expr.op('&>')(other) + + def adjacent_to(self, other): + """Boolean expression. Returns true if the range in the column + is adjacent to the range in the operand. + """ + return self.expr.op('-|-')(other) + + def __add__(self, other): + """Range expression. Returns the union of the two ranges. + Will raise an exception if the resulting range is not + contigous. + """ + return self.expr.op('+')(other) + +class INT4RANGE(RangeOperators, sqltypes.TypeEngine): + """Represent the Postgresql INT4RANGE type. + + .. versionadded:: 0.8.2 + + """ + + __visit_name__ = 'INT4RANGE' + +ischema_names['int4range'] = INT4RANGE + +class INT8RANGE(RangeOperators, sqltypes.TypeEngine): + """Represent the Postgresql INT8RANGE type. + + .. versionadded:: 0.8.2 + + """ + + __visit_name__ = 'INT8RANGE' + +ischema_names['int8range'] = INT8RANGE + +class NUMRANGE(RangeOperators, sqltypes.TypeEngine): + """Represent the Postgresql NUMRANGE type. + + .. versionadded:: 0.8.2 + + """ + + __visit_name__ = 'NUMRANGE' + +ischema_names['numrange'] = NUMRANGE + +class DATERANGE(RangeOperators, sqltypes.TypeEngine): + """Represent the Postgresql DATERANGE type. + + .. versionadded:: 0.8.2 + + """ + + __visit_name__ = 'DATERANGE' + +ischema_names['daterange'] = DATERANGE + +class TSRANGE(RangeOperators, sqltypes.TypeEngine): + """Represent the Postgresql TSRANGE type. + + .. versionadded:: 0.8.2 + + """ + + __visit_name__ = 'TSRANGE' + +ischema_names['tsrange'] = TSRANGE + +class TSTZRANGE(RangeOperators, sqltypes.TypeEngine): + """Represent the Postgresql TSTZRANGE type. + + .. versionadded:: 0.8.2 + + """ + + __visit_name__ = 'TSTZRANGE' + +ischema_names['tstzrange'] = TSTZRANGE diff --git a/libs/sqlalchemy/dialects/postgresql/zxjdbc.py b/libs/sqlalchemy/dialects/postgresql/zxjdbc.py index 4aea9c9b..67e7d53e 100644 --- a/libs/sqlalchemy/dialects/postgresql/zxjdbc.py +++ b/libs/sqlalchemy/dialects/postgresql/zxjdbc.py @@ -1,19 +1,21 @@ # postgresql/zxjdbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the PostgreSQL database via the zxjdbc JDBC connector. +""" +.. dialect:: postgresql+zxjdbc + :name: zxJDBC for Jython + :dbapi: zxjdbc + :connectstring: postgresql+zxjdbc://scott:tiger@localhost/db + :driverurl: http://jdbc.postgresql.org/ -JDBC Driver ------------ - -The official Postgresql JDBC driver is at http://jdbc.postgresql.org/. """ -from sqlalchemy.connectors.zxJDBC import ZxJDBCConnector -from sqlalchemy.dialects.postgresql.base import PGDialect, PGExecutionContext +from ...connectors.zxJDBC import ZxJDBCConnector +from .base import PGDialect, PGExecutionContext + class PGExecutionContext_zxjdbc(PGExecutionContext): @@ -37,6 +39,7 @@ class PGDialect_zxjdbc(ZxJDBCConnector, PGDialect): self.DataHandler = PostgresqlDataHandler def _get_server_version_info(self, connection): - return tuple(int(x) for x in connection.connection.dbversion.split('.')) + parts = connection.connection.dbversion.split('.') + return tuple(int(x) for x in parts) dialect = PGDialect_zxjdbc diff --git a/libs/sqlalchemy/dialects/sqlite/__init__.py b/libs/sqlalchemy/dialects/sqlite/__init__.py index c1157b63..a9b23575 100644 --- a/libs/sqlalchemy/dialects/sqlite/__init__.py +++ b/libs/sqlalchemy/dialects/sqlite/__init__.py @@ -1,5 +1,5 @@ # sqlite/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -15,6 +15,7 @@ from sqlalchemy.dialects.sqlite.base import \ NUMERIC, SMALLINT, TEXT, TIME, TIMESTAMP, VARCHAR, dialect __all__ = ( - 'BLOB', 'BOOLEAN', 'CHAR', 'DATE', 'DATETIME', 'DECIMAL', 'FLOAT', 'INTEGER', - 'NUMERIC', 'SMALLINT', 'TEXT', 'TIME', 'TIMESTAMP', 'VARCHAR', 'dialect', 'REAL' -) \ No newline at end of file + 'BLOB', 'BOOLEAN', 'CHAR', 'DATE', 'DATETIME', 'DECIMAL', 'FLOAT', + 'INTEGER', 'NUMERIC', 'SMALLINT', 'TEXT', 'TIME', 'TIMESTAMP', 'VARCHAR', + 'REAL', 'dialect' +) diff --git a/libs/sqlalchemy/dialects/sqlite/base.py b/libs/sqlalchemy/dialects/sqlite/base.py index 9118ace2..ac644f8d 100644 --- a/libs/sqlalchemy/dialects/sqlite/base.py +++ b/libs/sqlalchemy/dialects/sqlite/base.py @@ -1,25 +1,26 @@ # sqlite/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the SQLite database. +""" +.. dialect:: sqlite + :name: SQLite -For information on connecting using a specific driver, see the documentation -section regarding that driver. Date and Time Types ------------------- -SQLite does not have built-in DATE, TIME, or DATETIME types, and pysqlite does not provide -out of the box functionality for translating values between Python `datetime` objects -and a SQLite-supported format. SQLAlchemy's own :class:`~sqlalchemy.types.DateTime` -and related types provide date formatting and parsing functionality when SQlite is used. -The implementation classes are :class:`~.sqlite.DATETIME`, :class:`~.sqlite.DATE` and :class:`~.sqlite.TIME`. -These types represent dates and times as ISO formatted strings, which also nicely -support ordering. There's no reliance on typical "libc" internals for these functions -so historical dates are fully supported. +SQLite does not have built-in DATE, TIME, or DATETIME types, and pysqlite +does not provide out of the box functionality for translating values between +Python `datetime` objects and a SQLite-supported format. SQLAlchemy's own +:class:`~sqlalchemy.types.DateTime` and related types provide date formatting +and parsing functionality when SQlite is used. The implementation classes are +:class:`~.sqlite.DATETIME`, :class:`~.sqlite.DATE` and :class:`~.sqlite.TIME`. +These types represent dates and times as ISO formatted strings, which also +nicely support ordering. There's no reliance on typical "libc" internals +for these functions so historical dates are fully supported. Auto Incrementing Behavior -------------------------- @@ -46,44 +47,47 @@ to the Table construct:: Transaction Isolation Level --------------------------- -:func:`.create_engine` accepts an ``isolation_level`` parameter which results in -the command ``PRAGMA read_uncommitted `` being invoked for every new -connection. Valid values for this parameter are ``SERIALIZABLE`` and -``READ UNCOMMITTED`` corresponding to a value of 0 and 1, respectively. +:func:`.create_engine` accepts an ``isolation_level`` parameter which +results in the command ``PRAGMA read_uncommitted `` being invoked for +every new connection. Valid values for this parameter are ``SERIALIZABLE`` +and ``READ UNCOMMITTED`` corresponding to a value of 0 and 1, respectively. See the section :ref:`pysqlite_serializable` for an important workaround when using serializable isolation with Pysqlite. Database Locking Behavior / Concurrency --------------------------------------- -Note that SQLite is not designed for a high level of concurrency. The database -itself, being a file, is locked completely during write operations and within -transactions, meaning exactly one connection has exclusive access to the database -during this period - all other connections will be blocked during this time. +Note that SQLite is not designed for a high level of concurrency. The +database itself, being a file, is locked completely during write operations +and within transactions, meaning exactly one connection has exclusive access +to the database during this period - all other connections will be blocked +during this time. The Python DBAPI specification also calls for a connection model that is always -in a transaction; there is no BEGIN method, only commit and rollback. This implies -that a SQLite DBAPI driver would technically allow only serialized access to a -particular database file at all times. The pysqlite driver attempts to ameliorate this by -deferring the actual BEGIN statement until the first DML (INSERT, UPDATE, or -DELETE) is received within a transaction. While this breaks serializable isolation, -it at least delays the exclusive locking inherent in SQLite's design. +in a transaction; there is no BEGIN method, only commit and rollback. This +implies that a SQLite DBAPI driver would technically allow only serialized +access to a particular database file at all times. The pysqlite driver +attempts to ameliorate this by deferring the actual BEGIN statement until +the first DML (INSERT, UPDATE, or DELETE) is received within a +transaction. While this breaks serializable isolation, it at least delays +the exclusive locking inherent in SQLite's design. SQLAlchemy's default mode of usage with the ORM is known -as "autocommit=False", which means the moment the :class:`.Session` begins to be -used, a transaction is begun. As the :class:`.Session` is used, the autoflush -feature, also on by default, will flush out pending changes to the database -before each query. The effect of this is that a :class:`.Session` used in its -default mode will often emit DML early on, long before the transaction is actually -committed. This again will have the effect of serializing access to the SQLite -database. If highly concurrent reads are desired against the SQLite database, -it is advised that the autoflush feature be disabled, and potentially even -that autocommit be re-enabled, which has the effect of each SQL statement and -flush committing changes immediately. +as "autocommit=False", which means the moment the :class:`.Session` begins to +be used, a transaction is begun. As the :class:`.Session` is used, the +autoflush feature, also on by default, will flush out pending changes to the +database before each query. The effect of this is that a :class:`.Session` +used in its default mode will often emit DML early on, long before the +transaction is actually committed. This again will have the effect of +serializing access to the SQLite database. If highly concurrent reads are +desired against the SQLite database, it is advised that the autoflush feature +be disabled, and potentially even that autocommit be re-enabled, which has +the effect of each SQL statement and flush committing changes immediately. For more information on SQLite's lack of concurrency by design, please -see `Situations Where Another RDBMS May Work Better - High Concurrency `_ -near the bottom of the page. +see `Situations Where Another RDBMS May Work Better - High +Concurrency `_ near the bottom of +the page. .. _sqlite_foreign_keys: @@ -123,7 +127,8 @@ for new connections through the usage of events:: """ -import datetime, re +import datetime +import re from sqlalchemy import sql, exc from sqlalchemy.engine import default, base, reflection @@ -132,8 +137,10 @@ from sqlalchemy import util from sqlalchemy.sql import compiler from sqlalchemy import processors -from sqlalchemy.types import BLOB, BOOLEAN, CHAR, DATE, DATETIME, DECIMAL,\ - FLOAT, REAL, INTEGER, NUMERIC, SMALLINT, TEXT, TIME, TIMESTAMP, VARCHAR +from sqlalchemy.types import BIGINT, BLOB, BOOLEAN, CHAR,\ + DECIMAL, FLOAT, REAL, INTEGER, NUMERIC, SMALLINT, TEXT,\ + TIMESTAMP, VARCHAR + class _DateTimeMixin(object): _reg = None @@ -146,15 +153,26 @@ class _DateTimeMixin(object): if storage_format is not None: self._storage_format = storage_format + def adapt(self, cls, **kw): + if self._storage_format: + kw["storage_format"] = self._storage_format + if self._reg: + kw["regexp"] = self._reg + return util.constructor_copy(self, cls, **kw) + + def literal_processor(self, dialect): + bp = self.bind_processor(dialect) + def process(value): + return "'%s'" % bp(value) + return process + + class DATETIME(_DateTimeMixin, sqltypes.DateTime): """Represent a Python datetime object in SQLite using a string. The default string storage format is:: - "%04d-%02d-%02d %02d:%02d:%02d.%06d" % (value.year, - value.month, value.day, - value.hour, value.minute, - value.second, value.microsecond) + "%(year)04d-%(month)02d-%(day)02d %(hour)02d:%(min)02d:%(second)02d.%(microsecond)06d" e.g.:: @@ -167,37 +185,68 @@ class DATETIME(_DateTimeMixin, sqltypes.DateTime): from sqlalchemy.dialects.sqlite import DATETIME dt = DATETIME( - storage_format="%04d/%02d/%02d %02d-%02d-%02d-%06d", - regexp=re.compile("(\d+)/(\d+)/(\d+) (\d+)-(\d+)-(\d+)(?:-(\d+))?") - ) + storage_format="%(year)04d/%(month)02d/%(day)02d %(hour)02d:%(min)02d:%(second)02d", + regexp=r"(\d+)/(\d+)/(\d+) (\d+)-(\d+)-(\d+)" + ) :param storage_format: format string which will be applied to the - tuple ``(value.year, value.month, value.day, value.hour, - value.minute, value.second, value.microsecond)``, given a - Python datetime.datetime() object. + dict with keys year, month, day, hour, minute, second, and microsecond. :param regexp: regular expression which will be applied to - incoming result rows. The resulting match object is applied to - the Python datetime() constructor via ``*map(int, - match_obj.groups(0))``. + incoming result rows. If the regexp contains named groups, the + resulting match dict is applied to the Python datetime() constructor + as keyword arguments. Otherwise, if positional groups are used, the + the datetime() constructor is called with positional arguments via + ``*map(int, match_obj.groups(0))``. """ - _storage_format = "%04d-%02d-%02d %02d:%02d:%02d.%06d" + _storage_format = ( + "%(year)04d-%(month)02d-%(day)02d " + "%(hour)02d:%(minute)02d:%(second)02d.%(microsecond)06d" + ) + + def __init__(self, *args, **kwargs): + truncate_microseconds = kwargs.pop('truncate_microseconds', False) + super(DATETIME, self).__init__(*args, **kwargs) + if truncate_microseconds: + assert 'storage_format' not in kwargs, "You can specify only "\ + "one of truncate_microseconds or storage_format." + assert 'regexp' not in kwargs, "You can specify only one of "\ + "truncate_microseconds or regexp." + self._storage_format = ( + "%(year)04d-%(month)02d-%(day)02d " + "%(hour)02d:%(minute)02d:%(second)02d" + ) + def bind_processor(self, dialect): datetime_datetime = datetime.datetime datetime_date = datetime.date format = self._storage_format + def process(value): if value is None: return None elif isinstance(value, datetime_datetime): - return format % (value.year, value.month, value.day, - value.hour, value.minute, value.second, - value.microsecond) + return format % { + 'year': value.year, + 'month': value.month, + 'day': value.day, + 'hour': value.hour, + 'minute': value.minute, + 'second': value.second, + 'microsecond': value.microsecond, + } elif isinstance(value, datetime_date): - return format % (value.year, value.month, value.day, - 0, 0, 0, 0) + return format % { + 'year': value.year, + 'month': value.month, + 'day': value.day, + 'hour': 0, + 'minute': 0, + 'second': 0, + 'microsecond': 0, + } else: raise TypeError("SQLite DateTime type only accepts Python " "datetime and date objects as input.") @@ -210,12 +259,13 @@ class DATETIME(_DateTimeMixin, sqltypes.DateTime): else: return processors.str_to_datetime + class DATE(_DateTimeMixin, sqltypes.Date): """Represent a Python date object in SQLite using a string. The default string storage format is:: - "%04d-%02d-%02d" % (value.year, value.month, value.day) + "%(year)04d-%(month)02d-%(day)02d" e.g.:: @@ -228,31 +278,36 @@ class DATE(_DateTimeMixin, sqltypes.Date): from sqlalchemy.dialects.sqlite import DATE d = DATE( - storage_format="%02d/%02d/%02d", - regexp=re.compile("(\d+)/(\d+)/(\d+)") + storage_format="%(month)02d/%(day)02d/%(year)04d", + regexp=re.compile("(?P\d+)/(?P\d+)/(?P\d+)") ) :param storage_format: format string which will be applied to the - tuple ``(value.year, value.month, value.day)``, - given a Python datetime.date() object. + dict with keys year, month, and day. :param regexp: regular expression which will be applied to - incoming result rows. The resulting match object is applied to - the Python date() constructor via ``*map(int, - match_obj.groups(0))``. - + incoming result rows. If the regexp contains named groups, the + resulting match dict is applied to the Python date() constructor + as keyword arguments. Otherwise, if positional groups are used, the + the date() constructor is called with positional arguments via + ``*map(int, match_obj.groups(0))``. """ - _storage_format = "%04d-%02d-%02d" + _storage_format = "%(year)04d-%(month)02d-%(day)02d" def bind_processor(self, dialect): datetime_date = datetime.date format = self._storage_format + def process(value): if value is None: return None elif isinstance(value, datetime_date): - return format % (value.year, value.month, value.day) + return format % { + 'year': value.year, + 'month': value.month, + 'day': value.day, + } else: raise TypeError("SQLite Date type only accepts Python " "date objects as input.") @@ -265,14 +320,13 @@ class DATE(_DateTimeMixin, sqltypes.Date): else: return processors.str_to_date + class TIME(_DateTimeMixin, sqltypes.Time): """Represent a Python time object in SQLite using a string. The default string storage format is:: - "%02d:%02d:%02d.%06d" % (value.hour, value.minute, - value.second, - value.microsecond) + "%(hour)02d:%(minute)02d:%(second)02d.%(microsecond)06d" e.g.:: @@ -285,32 +339,47 @@ class TIME(_DateTimeMixin, sqltypes.Time): from sqlalchemy.dialects.sqlite import TIME t = TIME( - storage_format="%02d-%02d-%02d-%06d", - regexp=re.compile("(\d+)-(\d+)-(\d+)-(?:-(\d+))?") - ) + storage_format="%(hour)02d-%(minute)02d-%(second)02d-%(microsecond)06d", + regexp=re.compile("(\d+)-(\d+)-(\d+)-(?:-(\d+))?") + ) - :param storage_format: format string which will be applied - to the tuple ``(value.hour, value.minute, value.second, - value.microsecond)``, given a Python datetime.time() object. + :param storage_format: format string which will be applied to the + dict with keys hour, minute, second, and microsecond. :param regexp: regular expression which will be applied to - incoming result rows. The resulting match object is applied to - the Python time() constructor via ``*map(int, - match_obj.groups(0))``. - + incoming result rows. If the regexp contains named groups, the + resulting match dict is applied to the Python time() constructor + as keyword arguments. Otherwise, if positional groups are used, the + the time() constructor is called with positional arguments via + ``*map(int, match_obj.groups(0))``. """ - _storage_format = "%02d:%02d:%02d.%06d" + _storage_format = "%(hour)02d:%(minute)02d:%(second)02d.%(microsecond)06d" + + def __init__(self, *args, **kwargs): + truncate_microseconds = kwargs.pop('truncate_microseconds', False) + super(TIME, self).__init__(*args, **kwargs) + if truncate_microseconds: + assert 'storage_format' not in kwargs, "You can specify only "\ + "one of truncate_microseconds or storage_format." + assert 'regexp' not in kwargs, "You can specify only one of "\ + "truncate_microseconds or regexp." + self._storage_format = "%(hour)02d:%(minute)02d:%(second)02d" def bind_processor(self, dialect): datetime_time = datetime.time format = self._storage_format + def process(value): if value is None: return None elif isinstance(value, datetime_time): - return format % (value.hour, value.minute, value.second, - value.microsecond) + return format % { + 'hour': value.hour, + 'minute': value.minute, + 'second': value.second, + 'microsecond': value.microsecond, + } else: raise TypeError("SQLite Time type only accepts Python " "time objects as input.") @@ -330,6 +399,7 @@ colspecs = { } ischema_names = { + 'BIGINT': sqltypes.BIGINT, 'BLOB': sqltypes.BLOB, 'BOOL': sqltypes.BOOLEAN, 'BOOLEAN': sqltypes.BOOLEAN, @@ -347,10 +417,11 @@ ischema_names = { 'TIME': sqltypes.TIME, 'TIMESTAMP': sqltypes.TIMESTAMP, 'VARCHAR': sqltypes.VARCHAR, + 'NVARCHAR': sqltypes.NVARCHAR, + 'NCHAR': sqltypes.NCHAR, } - class SQLiteCompiler(compiler.SQLCompiler): extract_map = util.update_copy( compiler.SQLCompiler.extract_map, @@ -391,7 +462,9 @@ class SQLiteCompiler(compiler.SQLCompiler): def visit_extract(self, extract, **kw): try: return "CAST(STRFTIME('%s', %s) AS INTEGER)" % ( - self.extract_map[extract.field], self.process(extract.expr, **kw)) + self.extract_map[extract.field], + self.process(extract.expr, **kw) + ) except KeyError: raise exc.CompileError( "%s is not a valid extract argument." % extract.field) @@ -399,7 +472,7 @@ class SQLiteCompiler(compiler.SQLCompiler): def limit_clause(self, select): text = "" if select._limit is not None: - text += "\n LIMIT " + self.process(sql.literal(select._limit)) + text += "\n LIMIT " + self.process(sql.literal(select._limit)) if select._offset is not None: if select._limit is None: text += "\n LIMIT " + self.process(sql.literal(-1)) @@ -416,7 +489,8 @@ class SQLiteCompiler(compiler.SQLCompiler): class SQLiteDDLCompiler(compiler.DDLCompiler): def get_column_specification(self, column, **kwargs): - colspec = self.preparer.format_column(column) + " " + self.dialect.type_compiler.process(column.type) + coltype = self.dialect.type_compiler.process(column.type) + colspec = self.preparer.format_column(column) + " " + coltype default = self.get_column_default_string(column) if default is not None: colspec += " DEFAULT " + default @@ -424,12 +498,12 @@ class SQLiteDDLCompiler(compiler.DDLCompiler): if not column.nullable: colspec += " NOT NULL" - if column.primary_key and \ - column.table.kwargs.get('sqlite_autoincrement', False) and \ - len(column.table.primary_key.columns) == 1 and \ - issubclass(column.type._type_affinity, sqltypes.Integer) and \ - not column.foreign_keys: - colspec += " PRIMARY KEY AUTOINCREMENT" + if (column.primary_key and + column.table.kwargs.get('sqlite_autoincrement', False) and + len(column.table.primary_key.columns) == 1 and + issubclass(column.type._type_affinity, sqltypes.Integer) and + not column.foreign_keys): + colspec += " PRIMARY KEY AUTOINCREMENT" return colspec @@ -450,7 +524,7 @@ class SQLiteDDLCompiler(compiler.DDLCompiler): def visit_foreign_key_constraint(self, constraint): - local_table = constraint._elements.values()[0].parent.table + local_table = list(constraint._elements.values())[0].parent.table remote_table = list(constraint._elements.values())[0].column.table if local_table.schema != remote_table.schema: @@ -464,23 +538,15 @@ class SQLiteDDLCompiler(compiler.DDLCompiler): return preparer.format_table(table, use_schema=False) def visit_create_index(self, create): - index = create.element - preparer = self.preparer - text = "CREATE " - if index.unique: - text += "UNIQUE " - text += "INDEX %s ON %s (%s)" \ - % (preparer.format_index(index, - name=self._index_identifier(index.name)), - preparer.format_table(index.table, use_schema=False), - ', '.join(preparer.quote(c.name, c.quote) - for c in index.columns)) - return text + return super(SQLiteDDLCompiler, self).\ + visit_create_index(create, include_table_schema=False) + class SQLiteTypeCompiler(compiler.GenericTypeCompiler): def visit_large_binary(self, type_): return self.visit_BLOB(type_) + class SQLiteIdentifierPreparer(compiler.IdentifierPreparer): reserved_words = set([ 'add', 'after', 'all', 'alter', 'analyze', 'and', 'as', 'asc', @@ -492,14 +558,15 @@ class SQLiteIdentifierPreparer(compiler.IdentifierPreparer): 'drop', 'each', 'else', 'end', 'escape', 'except', 'exclusive', 'explain', 'false', 'fail', 'for', 'foreign', 'from', 'full', 'glob', 'group', 'having', 'if', 'ignore', 'immediate', 'in', 'index', - 'indexed', 'initially', 'inner', 'insert', 'instead', 'intersect', 'into', 'is', - 'isnull', 'join', 'key', 'left', 'like', 'limit', 'match', 'natural', - 'not', 'notnull', 'null', 'of', 'offset', 'on', 'or', 'order', 'outer', - 'plan', 'pragma', 'primary', 'query', 'raise', 'references', - 'reindex', 'rename', 'replace', 'restrict', 'right', 'rollback', - 'row', 'select', 'set', 'table', 'temp', 'temporary', 'then', 'to', - 'transaction', 'trigger', 'true', 'union', 'unique', 'update', 'using', - 'vacuum', 'values', 'view', 'virtual', 'when', 'where', + 'indexed', 'initially', 'inner', 'insert', 'instead', 'intersect', + 'into', 'is', 'isnull', 'join', 'key', 'left', 'like', 'limit', + 'match', 'natural', 'not', 'notnull', 'null', 'of', 'offset', 'on', + 'or', 'order', 'outer', 'plan', 'pragma', 'primary', 'query', + 'raise', 'references', 'reindex', 'rename', 'replace', 'restrict', + 'right', 'rollback', 'row', 'select', 'set', 'table', 'temp', + 'temporary', 'then', 'to', 'transaction', 'trigger', 'true', 'union', + 'unique', 'update', 'using', 'vacuum', 'values', 'view', 'virtual', + 'when', 'where', ]) def format_index(self, index, use_schema=True, name=None): @@ -508,10 +575,14 @@ class SQLiteIdentifierPreparer(compiler.IdentifierPreparer): if name is None: name = index.name result = self.quote(name, index.quote) - if not self.omit_schema and use_schema and getattr(index.table, "schema", None): - result = self.quote_schema(index.table.schema, index.table.quote_schema) + "." + result + if (not self.omit_schema and + use_schema and + getattr(index.table, "schema", None)): + result = self.quote_schema( + index.table.schema, index.table.quote_schema) + "." + result return result + class SQLiteExecutionContext(default.DefaultExecutionContext): @util.memoized_property def _preserve_raw_colnames(self): @@ -536,6 +607,8 @@ class SQLiteDialect(default.DefaultDialect): supports_default_values = True supports_empty_insert = False supports_cast = True + supports_multivalues_insert = True + supports_right_nested_joins = False default_paramstyle = 'qmark' execution_ctx_cls = SQLiteExecutionContext @@ -567,6 +640,9 @@ class SQLiteDialect(default.DefaultDialect): self.dbapi.sqlite_version_info >= (3, 3, 8) self.supports_cast = \ self.dbapi.sqlite_version_info >= (3, 2, 3) + self.supports_multivalues_insert = \ + self.dbapi.sqlite_version_info >= (3, 7, 11) + # http://www.sqlite.org/releaselog/3_7_11.html # see http://www.sqlalchemy.org/trac/ticket/2568 # as well as http://www.sqlite.org/src/info/600482d161 @@ -575,9 +651,10 @@ class SQLiteDialect(default.DefaultDialect): _isolation_lookup = { - 'READ UNCOMMITTED':1, - 'SERIALIZABLE':0 + 'READ UNCOMMITTED': 1, + 'SERIALIZABLE': 0 } + def set_isolation_level(self, connection, level): try: isolation_level = self._isolation_lookup[level.replace('_', ' ')] @@ -650,7 +727,8 @@ class SQLiteDialect(default.DefaultDialect): else: pragma = "PRAGMA " qtable = quote(table_name) - cursor = _pragma_cursor(connection.execute("%stable_info(%s)" % (pragma, qtable))) + statement = "%stable_info(%s)" % (pragma, qtable) + cursor = _pragma_cursor(connection.execute(statement)) row = cursor.fetchone() # consume remaining rows, to work around @@ -716,9 +794,8 @@ class SQLiteDialect(default.DefaultDialect): else: pragma = "PRAGMA " qtable = quote(table_name) - c = _pragma_cursor( - connection.execute("%stable_info(%s)" % - (pragma, qtable))) + statement = "%stable_info(%s)" % (pragma, qtable) + c = _pragma_cursor(connection.execute(statement)) rows = c.fetchall() columns = [] @@ -752,7 +829,7 @@ class SQLiteDialect(default.DefaultDialect): coltype = sqltypes.NullType() if default is not None: - default = unicode(default) + default = util.text_type(default) return { 'name': name, @@ -764,13 +841,13 @@ class SQLiteDialect(default.DefaultDialect): } @reflection.cache - def get_primary_keys(self, connection, table_name, schema=None, **kw): + def get_pk_constraint(self, connection, table_name, schema=None, **kw): cols = self.get_columns(connection, table_name, schema, **kw) pkeys = [] for col in cols: if col['primary_key']: pkeys.append(col['name']) - return pkeys + return {'constrained_columns': pkeys, 'name': None} @reflection.cache def get_foreign_keys(self, connection, table_name, schema=None, **kw): @@ -830,7 +907,8 @@ class SQLiteDialect(default.DefaultDialect): pragma = "PRAGMA " include_auto_indexes = kw.pop('include_auto_indexes', False) qtable = quote(table_name) - c = _pragma_cursor(connection.execute("%sindex_list(%s)" % (pragma, qtable))) + statement = "%sindex_list(%s)" % (pragma, qtable) + c = _pragma_cursor(connection.execute(statement)) indexes = [] while True: row = c.fetchone() @@ -838,13 +916,15 @@ class SQLiteDialect(default.DefaultDialect): break # ignore implicit primary key index. # http://www.mail-archive.com/sqlite-users@sqlite.org/msg30517.html - elif not include_auto_indexes and row[1].startswith('sqlite_autoindex'): + elif (not include_auto_indexes and + row[1].startswith('sqlite_autoindex')): continue indexes.append(dict(name=row[1], column_names=[], unique=row[2])) # loop thru unique indexes to get the column names. for idx in indexes: - c = connection.execute("%sindex_info(%s)" % (pragma, quote(idx['name']))) + statement = "%sindex_info(%s)" % (pragma, quote(idx['name'])) + c = connection.execute(statement) cols = idx['column_names'] while True: row = c.fetchone() @@ -853,6 +933,27 @@ class SQLiteDialect(default.DefaultDialect): cols.append(row[2]) return indexes + @reflection.cache + def get_unique_constraints(self, connection, table_name, + schema=None, **kw): + UNIQUE_SQL = """ + SELECT sql + FROM + sqlite_master + WHERE + type='table' AND + name=:table_name + """ + c = connection.execute(UNIQUE_SQL, table_name=table_name) + table_data = c.fetchone()[0] + + UNIQUE_PATTERN = 'CONSTRAINT (\w+) UNIQUE \(([^\)]+)\)' + return [ + {'name': name, + 'column_names': [col.strip(' "') for col in cols.split(',')]} + for name, cols in re.findall(UNIQUE_PATTERN, table_data) + ] + def _pragma_cursor(cursor): """work around SQLite issue whereby cursor.description diff --git a/libs/sqlalchemy/dialects/sqlite/pysqlite.py b/libs/sqlalchemy/dialects/sqlite/pysqlite.py index 826eefd8..b53f4d4a 100644 --- a/libs/sqlalchemy/dialects/sqlite/pysqlite.py +++ b/libs/sqlalchemy/dialects/sqlite/pysqlite.py @@ -1,13 +1,18 @@ # sqlite/pysqlite.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for the SQLite database via pysqlite. +""" +.. dialect:: sqlite+pysqlite + :name: pysqlite + :dbapi: sqlite3 + :connectstring: sqlite+pysqlite:///file_path + :url: http://docs.python.org/library/sqlite3.html -Note that pysqlite is the same driver as the ``sqlite3`` -module included with the Python distribution. + Note that ``pysqlite`` is the same driver as the ``sqlite3`` + module included with the Python distribution. Driver ------ @@ -26,37 +31,36 @@ this explicitly:: from sqlite3 import dbapi2 as sqlite e = create_engine('sqlite+pysqlite:///file.db', module=sqlite) -Full documentation on pysqlite is available at: -``_ Connect Strings --------------- -The file specification for the SQLite database is taken as the "database" portion of -the URL. Note that the format of a url is:: +The file specification for the SQLite database is taken as the "database" +portion of the URL. Note that the format of a SQLAlchemy url is:: driver://user:pass@host/database -This means that the actual filename to be used starts with the characters to the -**right** of the third slash. So connecting to a relative filepath looks like:: +This means that the actual filename to be used starts with the characters to +the **right** of the third slash. So connecting to a relative filepath +looks like:: # relative path e = create_engine('sqlite:///path/to/database.db') -An absolute path, which is denoted by starting with a slash, means you need **four** -slashes:: +An absolute path, which is denoted by starting with a slash, means you +need **four** slashes:: # absolute path e = create_engine('sqlite:////path/to/database.db') -To use a Windows path, regular drive specifications and backslashes can be used. -Double backslashes are probably needed:: +To use a Windows path, regular drive specifications and backslashes can be +used. Double backslashes are probably needed:: # absolute path on Windows e = create_engine('sqlite:///C:\\\\path\\\\to\\\\database.db') -The sqlite ``:memory:`` identifier is the default if no filepath is present. Specify -``sqlite://`` and nothing else:: +The sqlite ``:memory:`` identifier is the default if no filepath is +present. Specify ``sqlite://`` and nothing else:: # in-memory database e = create_engine('sqlite://') @@ -83,22 +87,24 @@ nor should be necessary, for use with SQLAlchemy, usage of PARSE_DECLTYPES can be forced if one configures "native_datetime=True" on create_engine():: engine = create_engine('sqlite://', - connect_args={'detect_types': sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES}, - native_datetime=True - ) + connect_args={'detect_types': sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES}, + native_datetime=True + ) -With this flag enabled, the DATE and TIMESTAMP types (but note - not the DATETIME -or TIME types...confused yet ?) will not perform any bind parameter or result -processing. Execution of "func.current_date()" will return a string. +With this flag enabled, the DATE and TIMESTAMP types (but note - not the +DATETIME or TIME types...confused yet ?) will not perform any bind parameter +or result processing. Execution of "func.current_date()" will return a string. "func.current_timestamp()" is registered as returning a DATETIME type in SQLAlchemy, so this function still receives SQLAlchemy-level result processing. +.. _pysqlite_threading_pooling: + Threading/Pooling Behavior --------------------------- Pysqlite's default behavior is to prohibit the usage of a single connection -in more than one thread. This is originally intended to work with older versions -of SQLite that did not support multithreaded operation under +in more than one thread. This is originally intended to work with older +versions of SQLite that did not support multithreaded operation under various circumstances. In particular, older SQLite versions did not allow a ``:memory:`` database to be used in multiple threads under any circumstances. @@ -114,17 +120,17 @@ thread-safety to make this usage worth it. SQLAlchemy sets up pooling to work with Pysqlite's default behavior: -* When a ``:memory:`` SQLite database is specified, the dialect by default will use - :class:`.SingletonThreadPool`. This pool maintains a single connection per - thread, so that all access to the engine within the current thread use the - same ``:memory:`` database - other threads would access a different - ``:memory:`` database. -* When a file-based database is specified, the dialect will use :class:`.NullPool` - as the source of connections. This pool closes and discards connections - which are returned to the pool immediately. SQLite file-based connections - have extremely low overhead, so pooling is not necessary. The scheme also - prevents a connection from being used again in a different thread and works - best with SQLite's coarse-grained file locking. +* When a ``:memory:`` SQLite database is specified, the dialect by default + will use :class:`.SingletonThreadPool`. This pool maintains a single + connection per thread, so that all access to the engine within the current + thread use the same ``:memory:`` database - other threads would access a + different ``:memory:`` database. +* When a file-based database is specified, the dialect will use + :class:`.NullPool` as the source of connections. This pool closes and + discards connections which are returned to the pool immediately. SQLite + file-based connections have extremely low overhead, so pooling is not + necessary. The scheme also prevents a connection from being used again in + a different thread and works best with SQLite's coarse-grained file locking. .. versionchanged:: 0.7 Default selection of :class:`.NullPool` for SQLite file-based databases. @@ -137,9 +143,10 @@ Using a Memory Database in Multiple Threads To use a ``:memory:`` database in a multithreaded scenario, the same connection object must be shared among threads, since the database exists -only within the scope of that connection. The :class:`.StaticPool` implementation -will maintain a single connection globally, and the ``check_same_thread`` flag -can be passed to Pysqlite as ``False``:: +only within the scope of that connection. The +:class:`.StaticPool` implementation will maintain a single connection +globally, and the ``check_same_thread`` flag can be passed to Pysqlite +as ``False``:: from sqlalchemy.pool import StaticPool engine = create_engine('sqlite://', @@ -152,13 +159,14 @@ version of SQLite. Using Temporary Tables with SQLite ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Due to the way SQLite deals with temporary tables, if you wish to use a temporary table -in a file-based SQLite database across multiple checkouts from the connection pool, such -as when using an ORM :class:`.Session` where the temporary table should continue to remain -after :meth:`.commit` or :meth:`.rollback` is called, -a pool which maintains a single connection must be used. Use :class:`.SingletonThreadPool` -if the scope is only needed within the current thread, or :class:`.StaticPool` is scope is -needed within multiple threads for this case:: +Due to the way SQLite deals with temporary tables, if you wish to use a +temporary table in a file-based SQLite database across multiple checkouts +from the connection pool, such as when using an ORM :class:`.Session` where +the temporary table should continue to remain after :meth:`.Session.commit` or +:meth:`.Session.rollback` is called, a pool which maintains a single connection must +be used. Use :class:`.SingletonThreadPool` if the scope is only needed +within the current thread, or :class:`.StaticPool` is scope is needed within +multiple threads for this case:: # maintain the same connection per thread from sqlalchemy.pool import SingletonThreadPool @@ -171,17 +179,17 @@ needed within multiple threads for this case:: engine = create_engine('sqlite:///mydb.db', poolclass=StaticPool) -Note that :class:`.SingletonThreadPool` should be configured for the number of threads -that are to be used; beyond that number, connections will be closed out in a non deterministic -way. +Note that :class:`.SingletonThreadPool` should be configured for the number +of threads that are to be used; beyond that number, connections will be +closed out in a non deterministic way. Unicode ------- -The pysqlite driver only returns Python ``unicode`` objects in result sets, never -plain strings, and accommodates ``unicode`` objects within bound parameter -values in all cases. Regardless of the SQLAlchemy string type in use, -string-based result values will by Python ``unicode`` in Python 2. +The pysqlite driver only returns Python ``unicode`` objects in result sets, +never plain strings, and accommodates ``unicode`` objects within bound +parameter values in all cases. Regardless of the SQLAlchemy string type in +use, string-based result values will by Python ``unicode`` in Python 2. The :class:`.Unicode` type should still be used to indicate those columns that require unicode, however, so that non-``unicode`` values passed inadvertently will emit a warning. Pysqlite will emit an error if a non-``unicode`` string @@ -221,6 +229,7 @@ from sqlalchemy import util import os + class _SQLite_pysqliteTimeStamp(DATETIME): def bind_processor(self, dialect): if dialect.native_datetime: @@ -234,6 +243,7 @@ class _SQLite_pysqliteTimeStamp(DATETIME): else: return DATETIME.result_processor(self, dialect, coltype) + class _SQLite_pysqliteDate(DATE): def bind_processor(self, dialect): if dialect.native_datetime: @@ -247,19 +257,20 @@ class _SQLite_pysqliteDate(DATE): else: return DATE.result_processor(self, dialect, coltype) + class SQLiteDialect_pysqlite(SQLiteDialect): default_paramstyle = 'qmark' colspecs = util.update_copy( SQLiteDialect.colspecs, { - sqltypes.Date:_SQLite_pysqliteDate, - sqltypes.TIMESTAMP:_SQLite_pysqliteTimeStamp, + sqltypes.Date: _SQLite_pysqliteDate, + sqltypes.TIMESTAMP: _SQLite_pysqliteTimeStamp, } ) - # Py3K - #description_encoding = None + if not util.py2k: + description_encoding = None driver = 'pysqlite' @@ -279,9 +290,9 @@ class SQLiteDialect_pysqlite(SQLiteDialect): def dbapi(cls): try: from pysqlite2 import dbapi2 as sqlite - except ImportError, e: + except ImportError as e: try: - from sqlite3 import dbapi2 as sqlite #try the 2.5+ stdlib name. + from sqlite3 import dbapi2 as sqlite # try 2.5+ stdlib name. except ImportError: raise e return sqlite diff --git a/libs/sqlalchemy/dialects/sybase/__init__.py b/libs/sqlalchemy/dialects/sybase/__init__.py index 528ebf23..85f9dd9c 100644 --- a/libs/sqlalchemy/dialects/sybase/__init__.py +++ b/libs/sqlalchemy/dialects/sybase/__init__.py @@ -1,26 +1,27 @@ # sybase/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php from sqlalchemy.dialects.sybase import base, pysybase, pyodbc - -from base import CHAR, VARCHAR, TIME, NCHAR, NVARCHAR,\ - TEXT,DATE,DATETIME, FLOAT, NUMERIC,\ - BIGINT,INT, INTEGER, SMALLINT, BINARY,\ - VARBINARY,UNITEXT,UNICHAR,UNIVARCHAR,\ - IMAGE,BIT,MONEY,SMALLMONEY,TINYINT - # default dialect base.dialect = pyodbc.dialect +from .base import CHAR, VARCHAR, TIME, NCHAR, NVARCHAR,\ + TEXT, DATE, DATETIME, FLOAT, NUMERIC,\ + BIGINT, INT, INTEGER, SMALLINT, BINARY,\ + VARBINARY, UNITEXT, UNICHAR, UNIVARCHAR,\ + IMAGE, BIT, MONEY, SMALLMONEY, TINYINT,\ + dialect + + __all__ = ( - 'CHAR', 'VARCHAR', 'TIME', 'NCHAR', 'NVARCHAR', - 'TEXT','DATE','DATETIME', 'FLOAT', 'NUMERIC', - 'BIGINT','INT', 'INTEGER', 'SMALLINT', 'BINARY', - 'VARBINARY','UNITEXT','UNICHAR','UNIVARCHAR', - 'IMAGE','BIT','MONEY','SMALLMONEY','TINYINT', - 'dialect' + 'CHAR', 'VARCHAR', 'TIME', 'NCHAR', 'NVARCHAR', + 'TEXT', 'DATE', 'DATETIME', 'FLOAT', 'NUMERIC', + 'BIGINT', 'INT', 'INTEGER', 'SMALLINT', 'BINARY', + 'VARBINARY', 'UNITEXT', 'UNICHAR', 'UNIVARCHAR', + 'IMAGE', 'BIT', 'MONEY', 'SMALLMONEY', 'TINYINT', + 'dialect' ) diff --git a/libs/sqlalchemy/dialects/sybase/base.py b/libs/sqlalchemy/dialects/sybase/base.py index f551bff9..2f58aed9 100644 --- a/libs/sqlalchemy/dialects/sybase/base.py +++ b/libs/sqlalchemy/dialects/sybase/base.py @@ -1,5 +1,5 @@ # sybase/base.py -# Copyright (C) 2010-2011 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2014 the SQLAlchemy authors and contributors # get_select_precolumns(), limit_clause() implementation # copyright (C) 2007 Fisch Asset Management # AG http://www.fam.ch, with coding by Alexander Houben @@ -8,18 +8,21 @@ # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Support for Sybase Adaptive Server Enterprise (ASE). +""" + +.. dialect:: sybase + :name: Sybase .. note:: The Sybase dialect functions on current SQLAlchemy versions but is not regularly tested, and may have many issues and - caveats not currently handled. In particular, the table - and database reflection features are not implemented. + caveats not currently handled. """ - import operator +import re + from sqlalchemy.sql import compiler, expression, text, bindparam from sqlalchemy.engine import default, base, reflection from sqlalchemy import types as sqltypes @@ -28,10 +31,10 @@ from sqlalchemy import schema as sa_schema from sqlalchemy import util, sql, exc from sqlalchemy.types import CHAR, VARCHAR, TIME, NCHAR, NVARCHAR,\ - TEXT,DATE,DATETIME, FLOAT, NUMERIC,\ - BIGINT,INT, INTEGER, SMALLINT, BINARY,\ + TEXT, DATE, DATETIME, FLOAT, NUMERIC,\ + BIGINT, INT, INTEGER, SMALLINT, BINARY,\ VARBINARY, DECIMAL, TIMESTAMP, Unicode,\ - UnicodeText + UnicodeText, REAL RESERVED_WORDS = set([ "add", "all", "alter", "and", @@ -99,35 +102,44 @@ class _SybaseUnitypeMixin(object): def result_processor(self, dialect, coltype): def process(value): if value is not None: - return str(value) #.decode("ucs-2") + return str(value) # decode("ucs-2") else: return None return process + class UNICHAR(_SybaseUnitypeMixin, sqltypes.Unicode): __visit_name__ = 'UNICHAR' + class UNIVARCHAR(_SybaseUnitypeMixin, sqltypes.Unicode): __visit_name__ = 'UNIVARCHAR' + class UNITEXT(_SybaseUnitypeMixin, sqltypes.UnicodeText): __visit_name__ = 'UNITEXT' + class TINYINT(sqltypes.Integer): __visit_name__ = 'TINYINT' + class BIT(sqltypes.TypeEngine): __visit_name__ = 'BIT' + class MONEY(sqltypes.TypeEngine): __visit_name__ = "MONEY" + class SMALLMONEY(sqltypes.TypeEngine): __visit_name__ = "SMALLMONEY" + class UNIQUEIDENTIFIER(sqltypes.TypeEngine): __visit_name__ = "UNIQUEIDENTIFIER" + class IMAGE(sqltypes.LargeBinary): __visit_name__ = 'IMAGE' @@ -170,32 +182,68 @@ class SybaseTypeCompiler(compiler.GenericTypeCompiler): return "UNIQUEIDENTIFIER" ischema_names = { - 'integer' : INTEGER, - 'unsigned int' : INTEGER, # TODO: unsigned flags - 'unsigned smallint' : SMALLINT, # TODO: unsigned flags - 'unsigned bigint' : BIGINT, # TODO: unsigned flags 'bigint': BIGINT, - 'smallint' : SMALLINT, - 'tinyint' : TINYINT, - 'varchar' : VARCHAR, - 'long varchar' : TEXT, # TODO - 'char' : CHAR, - 'decimal' : DECIMAL, - 'numeric' : NUMERIC, - 'float' : FLOAT, - 'double' : NUMERIC, # TODO - 'binary' : BINARY, - 'varbinary' : VARBINARY, - 'bit': BIT, - 'image' : IMAGE, - 'timestamp': TIMESTAMP, + 'int': INTEGER, + 'integer': INTEGER, + 'smallint': SMALLINT, + 'tinyint': TINYINT, + 'unsigned bigint': BIGINT, # TODO: unsigned flags + 'unsigned int': INTEGER, # TODO: unsigned flags + 'unsigned smallint': SMALLINT, # TODO: unsigned flags + 'numeric': NUMERIC, + 'decimal': DECIMAL, + 'dec': DECIMAL, + 'float': FLOAT, + 'double': NUMERIC, # TODO + 'double precision': NUMERIC, # TODO + 'real': REAL, + 'smallmoney': SMALLMONEY, 'money': MONEY, - 'smallmoney': MONEY, + 'smalldatetime': DATETIME, + 'datetime': DATETIME, + 'date': DATE, + 'time': TIME, + 'char': CHAR, + 'character': CHAR, + 'varchar': VARCHAR, + 'character varying': VARCHAR, + 'char varying': VARCHAR, + 'unichar': UNICHAR, + 'unicode character': UNIVARCHAR, + 'nchar': NCHAR, + 'national char': NCHAR, + 'national character': NCHAR, + 'nvarchar': NVARCHAR, + 'nchar varying': NVARCHAR, + 'national char varying': NVARCHAR, + 'national character varying': NVARCHAR, + 'text': TEXT, + 'unitext': UNITEXT, + 'binary': BINARY, + 'varbinary': VARBINARY, + 'image': IMAGE, + 'bit': BIT, + +# not in documentation for ASE 15.7 + 'long varchar': TEXT, # TODO + 'timestamp': TIMESTAMP, 'uniqueidentifier': UNIQUEIDENTIFIER, } +class SybaseInspector(reflection.Inspector): + + def __init__(self, conn): + reflection.Inspector.__init__(self, conn) + + def get_table_id(self, table_name, schema=None): + """Return the table id from `table_name` and `schema`.""" + + return self.dialect.get_table_id(self.bind, table_name, schema, + info_cache=self.info_cache) + + class SybaseExecutionContext(default.DefaultExecutionContext): _enable_identity_insert = False @@ -243,12 +291,11 @@ class SybaseExecutionContext(default.DefaultExecutionContext): self.root_connection.connection.connection, True) - def post_exec(self): - if self.isddl: + if self.isddl: self.set_ddl_autocommit(self.root_connection, False) - if self._enable_identity_insert: + if self._enable_identity_insert: self.cursor.execute( "SET IDENTITY_INSERT %s OFF" % self.dialect.identifier_preparer. @@ -262,6 +309,7 @@ class SybaseExecutionContext(default.DefaultExecutionContext): cursor.close() return lastrowid + class SybaseSQLCompiler(compiler.SQLCompiler): ansi_bind_rules = True @@ -288,7 +336,7 @@ class SybaseSQLCompiler(compiler.SQLCompiler): # FIXME: sybase doesn't allow an offset without a limit # so use a huge value for TOP here s += "TOP 1000000 " - s += "START AT %s " % (select._offset+1,) + s += "START AT %s " % (select._offset + 1,) return s def get_from_hint_text(self, table, text): @@ -303,6 +351,9 @@ class SybaseSQLCompiler(compiler.SQLCompiler): return 'DATEPART("%s", %s)' % ( field, self.process(extract.expr, **kw)) + def visit_now_func(self, fn, **kw): + return "GETDATE()" + def for_update_clause(self, select): # "FOR UPDATE" is only allowed on "DECLARE CURSOR" # which SQLAlchemy doesn't use @@ -345,29 +396,31 @@ class SybaseDDLCompiler(compiler.DDLCompiler): # TODO: need correct syntax for this colspec += " IDENTITY(%s,%s)" % (start, increment) else: + default = self.get_column_default_string(column) + if default is not None: + colspec += " DEFAULT " + default + if column.nullable is not None: if not column.nullable or column.primary_key: colspec += " NOT NULL" else: colspec += " NULL" - default = self.get_column_default_string(column) - if default is not None: - colspec += " DEFAULT " + default - return colspec def visit_drop_index(self, drop): index = drop.element return "\nDROP INDEX %s.%s" % ( self.preparer.quote_identifier(index.table.name), - self.preparer.quote( - self._index_identifier(index.name), index.quote) + self._prepared_index_name(drop.element, + include_schema=False) ) + class SybaseIdentifierPreparer(compiler.IdentifierPreparer): reserved_words = RESERVED_WORDS + class SybaseDialect(default.DefaultDialect): name = 'sybase' supports_unicode_statements = False @@ -385,11 +438,12 @@ class SybaseDialect(default.DefaultDialect): statement_compiler = SybaseSQLCompiler ddl_compiler = SybaseDDLCompiler preparer = SybaseIdentifierPreparer + inspector = SybaseInspector def _get_default_schema_name(self, connection): return connection.scalar( text("SELECT user_name() as user_name", - typemap={'user_name':Unicode}) + typemap={'user_name': Unicode}) ) def initialize(self, connection): @@ -400,39 +454,361 @@ class SybaseDialect(default.DefaultDialect): else: self.max_identifier_length = 255 + def get_table_id(self, connection, table_name, schema=None, **kw): + """Fetch the id for schema.table_name. + + Several reflection methods require the table id. The idea for using + this method is that it can be fetched one time and cached for + subsequent calls. + + """ + + table_id = None + if schema is None: + schema = self.default_schema_name + + TABLEID_SQL = text(""" + SELECT o.id AS id + FROM sysobjects o JOIN sysusers u ON o.uid=u.uid + WHERE u.name = :schema_name + AND o.name = :table_name + AND o.type in ('U', 'V') + """) + + if util.py2k: + if isinstance(schema, unicode): + schema = schema.encode("ascii") + if isinstance(table_name, unicode): + table_name = table_name.encode("ascii") + result = connection.execute(TABLEID_SQL, + schema_name=schema, + table_name=table_name) + table_id = result.scalar() + if table_id is None: + raise exc.NoSuchTableError(table_name) + return table_id + + @reflection.cache + def get_columns(self, connection, table_name, schema=None, **kw): + table_id = self.get_table_id(connection, table_name, schema, + info_cache=kw.get("info_cache")) + + COLUMN_SQL = text(""" + SELECT col.name AS name, + t.name AS type, + (col.status & 8) AS nullable, + (col.status & 128) AS autoincrement, + com.text AS 'default', + col.prec AS precision, + col.scale AS scale, + col.length AS length + FROM systypes t, syscolumns col LEFT OUTER JOIN syscomments com ON + col.cdefault = com.id + WHERE col.usertype = t.usertype + AND col.id = :table_id + ORDER BY col.colid + """) + + results = connection.execute(COLUMN_SQL, table_id=table_id) + + columns = [] + for (name, type_, nullable, autoincrement, default, precision, scale, + length) in results: + col_info = self._get_column_info(name, type_, bool(nullable), + bool(autoincrement), default, precision, scale, + length) + columns.append(col_info) + + return columns + + def _get_column_info(self, name, type_, nullable, autoincrement, default, + precision, scale, length): + + coltype = self.ischema_names.get(type_, None) + + kwargs = {} + + if coltype in (NUMERIC, DECIMAL): + args = (precision, scale) + elif coltype == FLOAT: + args = (precision,) + elif coltype in (CHAR, VARCHAR, UNICHAR, UNIVARCHAR, NCHAR, NVARCHAR): + args = (length,) + else: + args = () + + if coltype: + coltype = coltype(*args, **kwargs) + #is this necessary + #if is_array: + # coltype = ARRAY(coltype) + else: + util.warn("Did not recognize type '%s' of column '%s'" % + (type_, name)) + coltype = sqltypes.NULLTYPE + + if default: + default = re.sub("DEFAULT", "", default).strip() + default = re.sub("^'(.*)'$", lambda m: m.group(1), default) + else: + default = None + + column_info = dict(name=name, type=coltype, nullable=nullable, + default=default, autoincrement=autoincrement) + return column_info + + @reflection.cache + def get_foreign_keys(self, connection, table_name, schema=None, **kw): + + table_id = self.get_table_id(connection, table_name, schema, + info_cache=kw.get("info_cache")) + + table_cache = {} + column_cache = {} + foreign_keys = [] + + table_cache[table_id] = {"name": table_name, "schema": schema} + + COLUMN_SQL = text(""" + SELECT c.colid AS id, c.name AS name + FROM syscolumns c + WHERE c.id = :table_id + """) + + results = connection.execute(COLUMN_SQL, table_id=table_id) + columns = {} + for col in results: + columns[col["id"]] = col["name"] + column_cache[table_id] = columns + + REFCONSTRAINT_SQL = text(""" + SELECT o.name AS name, r.reftabid AS reftable_id, + r.keycnt AS 'count', + r.fokey1 AS fokey1, r.fokey2 AS fokey2, r.fokey3 AS fokey3, + r.fokey4 AS fokey4, r.fokey5 AS fokey5, r.fokey6 AS fokey6, + r.fokey7 AS fokey7, r.fokey1 AS fokey8, r.fokey9 AS fokey9, + r.fokey10 AS fokey10, r.fokey11 AS fokey11, r.fokey12 AS fokey12, + r.fokey13 AS fokey13, r.fokey14 AS fokey14, r.fokey15 AS fokey15, + r.fokey16 AS fokey16, + r.refkey1 AS refkey1, r.refkey2 AS refkey2, r.refkey3 AS refkey3, + r.refkey4 AS refkey4, r.refkey5 AS refkey5, r.refkey6 AS refkey6, + r.refkey7 AS refkey7, r.refkey1 AS refkey8, r.refkey9 AS refkey9, + r.refkey10 AS refkey10, r.refkey11 AS refkey11, + r.refkey12 AS refkey12, r.refkey13 AS refkey13, + r.refkey14 AS refkey14, r.refkey15 AS refkey15, + r.refkey16 AS refkey16 + FROM sysreferences r JOIN sysobjects o on r.tableid = o.id + WHERE r.tableid = :table_id + """) + referential_constraints = connection.execute(REFCONSTRAINT_SQL, + table_id=table_id) + + REFTABLE_SQL = text(""" + SELECT o.name AS name, u.name AS 'schema' + FROM sysobjects o JOIN sysusers u ON o.uid = u.uid + WHERE o.id = :table_id + """) + + for r in referential_constraints: + reftable_id = r["reftable_id"] + + if reftable_id not in table_cache: + c = connection.execute(REFTABLE_SQL, table_id=reftable_id) + reftable = c.fetchone() + c.close() + table_info = {"name": reftable["name"], "schema": None} + if (schema is not None or + reftable["schema"] != self.default_schema_name): + table_info["schema"] = reftable["schema"] + + table_cache[reftable_id] = table_info + results = connection.execute(COLUMN_SQL, table_id=reftable_id) + reftable_columns = {} + for col in results: + reftable_columns[col["id"]] = col["name"] + column_cache[reftable_id] = reftable_columns + + reftable = table_cache[reftable_id] + reftable_columns = column_cache[reftable_id] + + constrained_columns = [] + referred_columns = [] + for i in range(1, r["count"] + 1): + constrained_columns.append(columns[r["fokey%i" % i]]) + referred_columns.append(reftable_columns[r["refkey%i" % i]]) + + fk_info = { + "constrained_columns": constrained_columns, + "referred_schema": reftable["schema"], + "referred_table": reftable["name"], + "referred_columns": referred_columns, + "name": r["name"] + } + + foreign_keys.append(fk_info) + + return foreign_keys + + @reflection.cache + def get_indexes(self, connection, table_name, schema=None, **kw): + table_id = self.get_table_id(connection, table_name, schema, + info_cache=kw.get("info_cache")) + + INDEX_SQL = text(""" + SELECT object_name(i.id) AS table_name, + i.keycnt AS 'count', + i.name AS name, + (i.status & 0x2) AS 'unique', + index_col(object_name(i.id), i.indid, 1) AS col_1, + index_col(object_name(i.id), i.indid, 2) AS col_2, + index_col(object_name(i.id), i.indid, 3) AS col_3, + index_col(object_name(i.id), i.indid, 4) AS col_4, + index_col(object_name(i.id), i.indid, 5) AS col_5, + index_col(object_name(i.id), i.indid, 6) AS col_6, + index_col(object_name(i.id), i.indid, 7) AS col_7, + index_col(object_name(i.id), i.indid, 8) AS col_8, + index_col(object_name(i.id), i.indid, 9) AS col_9, + index_col(object_name(i.id), i.indid, 10) AS col_10, + index_col(object_name(i.id), i.indid, 11) AS col_11, + index_col(object_name(i.id), i.indid, 12) AS col_12, + index_col(object_name(i.id), i.indid, 13) AS col_13, + index_col(object_name(i.id), i.indid, 14) AS col_14, + index_col(object_name(i.id), i.indid, 15) AS col_15, + index_col(object_name(i.id), i.indid, 16) AS col_16 + FROM sysindexes i, sysobjects o + WHERE o.id = i.id + AND o.id = :table_id + AND (i.status & 2048) = 0 + AND i.indid BETWEEN 1 AND 254 + """) + + results = connection.execute(INDEX_SQL, table_id=table_id) + indexes = [] + for r in results: + column_names = [] + for i in range(1, r["count"]): + column_names.append(r["col_%i" % (i,)]) + index_info = {"name": r["name"], + "unique": bool(r["unique"]), + "column_names": column_names} + indexes.append(index_info) + + return indexes + + @reflection.cache + def get_pk_constraint(self, connection, table_name, schema=None, **kw): + table_id = self.get_table_id(connection, table_name, schema, + info_cache=kw.get("info_cache")) + + PK_SQL = text(""" + SELECT object_name(i.id) AS table_name, + i.keycnt AS 'count', + i.name AS name, + index_col(object_name(i.id), i.indid, 1) AS pk_1, + index_col(object_name(i.id), i.indid, 2) AS pk_2, + index_col(object_name(i.id), i.indid, 3) AS pk_3, + index_col(object_name(i.id), i.indid, 4) AS pk_4, + index_col(object_name(i.id), i.indid, 5) AS pk_5, + index_col(object_name(i.id), i.indid, 6) AS pk_6, + index_col(object_name(i.id), i.indid, 7) AS pk_7, + index_col(object_name(i.id), i.indid, 8) AS pk_8, + index_col(object_name(i.id), i.indid, 9) AS pk_9, + index_col(object_name(i.id), i.indid, 10) AS pk_10, + index_col(object_name(i.id), i.indid, 11) AS pk_11, + index_col(object_name(i.id), i.indid, 12) AS pk_12, + index_col(object_name(i.id), i.indid, 13) AS pk_13, + index_col(object_name(i.id), i.indid, 14) AS pk_14, + index_col(object_name(i.id), i.indid, 15) AS pk_15, + index_col(object_name(i.id), i.indid, 16) AS pk_16 + FROM sysindexes i, sysobjects o + WHERE o.id = i.id + AND o.id = :table_id + AND (i.status & 2048) = 2048 + AND i.indid BETWEEN 1 AND 254 + """) + + results = connection.execute(PK_SQL, table_id=table_id) + pks = results.fetchone() + results.close() + + constrained_columns = [] + for i in range(1, pks["count"] + 1): + constrained_columns.append(pks["pk_%i" % (i,)]) + return {"constrained_columns": constrained_columns, + "name": pks["name"]} + + @reflection.cache + def get_schema_names(self, connection, **kw): + + SCHEMA_SQL = text("SELECT u.name AS name FROM sysusers u") + + schemas = connection.execute(SCHEMA_SQL) + + return [s["name"] for s in schemas] + @reflection.cache def get_table_names(self, connection, schema=None, **kw): if schema is None: schema = self.default_schema_name - result = connection.execute( - text("select sysobjects.name from sysobjects, sysusers " - "where sysobjects.uid=sysusers.uid and " - "sysusers.name=:schemaname and " - "sysobjects.type='U'", - bindparams=[ - bindparam('schemaname', schema) - ]) - ) - return [r[0] for r in result] + TABLE_SQL = text(""" + SELECT o.name AS name + FROM sysobjects o JOIN sysusers u ON o.uid = u.uid + WHERE u.name = :schema_name + AND o.type = 'U' + """) - def has_table(self, connection, tablename, schema=None): + if util.py2k: + if isinstance(schema, unicode): + schema = schema.encode("ascii") + + tables = connection.execute(TABLE_SQL, schema_name=schema) + + return [t["name"] for t in tables] + + @reflection.cache + def get_view_definition(self, connection, view_name, schema=None, **kw): if schema is None: schema = self.default_schema_name - result = connection.execute( - text("select sysobjects.name from sysobjects, sysusers " - "where sysobjects.uid=sysusers.uid and " - "sysobjects.name=:tablename and " - "sysusers.name=:schemaname and " - "sysobjects.type='U'", - bindparams=[ - bindparam('tablename', tablename), - bindparam('schemaname', schema) - ]) - ) - return result.scalar() is not None + VIEW_DEF_SQL = text(""" + SELECT c.text + FROM syscomments c JOIN sysobjects o ON c.id = o.id + WHERE o.name = :view_name + AND o.type = 'V' + """) - def reflecttable(self, connection, table, include_columns): - raise NotImplementedError() + if util.py2k: + if isinstance(view_name, unicode): + view_name = view_name.encode("ascii") + view = connection.execute(VIEW_DEF_SQL, view_name=view_name) + + return view.scalar() + + @reflection.cache + def get_view_names(self, connection, schema=None, **kw): + if schema is None: + schema = self.default_schema_name + + VIEW_SQL = text(""" + SELECT o.name AS name + FROM sysobjects o JOIN sysusers u ON o.uid = u.uid + WHERE u.name = :schema_name + AND o.type = 'V' + """) + + if util.py2k: + if isinstance(schema, unicode): + schema = schema.encode("ascii") + views = connection.execute(VIEW_SQL, schema_name=schema) + + return [v["name"] for v in views] + + def has_table(self, connection, table_name, schema=None): + try: + self.get_table_id(connection, table_name, schema) + except exc.NoSuchTableError: + return False + else: + return True diff --git a/libs/sqlalchemy/dialects/sybase/mxodbc.py b/libs/sqlalchemy/dialects/sybase/mxodbc.py index db60b9b2..f14d1c42 100644 --- a/libs/sqlalchemy/dialects/sybase/mxodbc.py +++ b/libs/sqlalchemy/dialects/sybase/mxodbc.py @@ -1,22 +1,31 @@ # sybase/mxodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php - """ -Support for Sybase via mxodbc. -This dialect is a stub only and is likely non functional at this time. +.. dialect:: sybase+mxodbc + :name: mxODBC + :dbapi: mxodbc + :connectstring: sybase+mxodbc://:@ + :url: http://www.egenix.com/ + +.. note:: + + This dialect is a stub only and is likely non functional at this time. """ -from sqlalchemy.dialects.sybase.base import SybaseDialect, SybaseExecutionContext +from sqlalchemy.dialects.sybase.base import SybaseDialect +from sqlalchemy.dialects.sybase.base import SybaseExecutionContext from sqlalchemy.connectors.mxodbc import MxODBCConnector + class SybaseExecutionContext_mxodbc(SybaseExecutionContext): pass + class SybaseDialect_mxodbc(MxODBCConnector, SybaseDialect): execution_ctx_cls = SybaseExecutionContext_mxodbc diff --git a/libs/sqlalchemy/dialects/sybase/pyodbc.py b/libs/sqlalchemy/dialects/sybase/pyodbc.py index 8e3729b3..f773e5a6 100644 --- a/libs/sqlalchemy/dialects/sybase/pyodbc.py +++ b/libs/sqlalchemy/dialects/sybase/pyodbc.py @@ -1,18 +1,16 @@ # sybase/pyodbc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -Support for Sybase via pyodbc. +.. dialect:: sybase+pyodbc + :name: PyODBC + :dbapi: pyodbc + :connectstring: sybase+pyodbc://:@[/] + :url: http://pypi.python.org/pypi/pyodbc/ -http://pypi.python.org/pypi/pyodbc/ - -Connect strings are of the form:: - - sybase+pyodbc://:@/ - sybase+pyodbc://:@/ Unicode Support --------------- @@ -37,8 +35,9 @@ Currently *not* supported are:: from sqlalchemy.dialects.sybase.base import SybaseDialect,\ SybaseExecutionContext from sqlalchemy.connectors.pyodbc import PyODBCConnector -from sqlalchemy import types as sqltypes, util, processors -from sqlalchemy.util.compat import decimal +from sqlalchemy import types as sqltypes, processors +import decimal + class _SybNumeric_pyodbc(sqltypes.Numeric): """Turns Decimals with adjusted() < -6 into floats. @@ -50,7 +49,7 @@ class _SybNumeric_pyodbc(sqltypes.Numeric): """ def bind_processor(self, dialect): - super_process = super(_SybNumeric_pyodbc,self).\ + super_process = super(_SybNumeric_pyodbc, self).\ bind_processor(dialect) def process(value): @@ -66,6 +65,7 @@ class _SybNumeric_pyodbc(sqltypes.Numeric): return value return process + class SybaseExecutionContext_pyodbc(SybaseExecutionContext): def set_ddl_autocommit(self, connection, value): if value: @@ -73,11 +73,12 @@ class SybaseExecutionContext_pyodbc(SybaseExecutionContext): else: connection.autocommit = False + class SybaseDialect_pyodbc(PyODBCConnector, SybaseDialect): execution_ctx_cls = SybaseExecutionContext_pyodbc colspecs = { - sqltypes.Numeric:_SybNumeric_pyodbc, + sqltypes.Numeric: _SybNumeric_pyodbc, } dialect = SybaseDialect_pyodbc diff --git a/libs/sqlalchemy/dialects/sybase/pysybase.py b/libs/sqlalchemy/dialects/sybase/pysybase.py index bf8c2096..664bd9ac 100644 --- a/libs/sqlalchemy/dialects/sybase/pysybase.py +++ b/libs/sqlalchemy/dialects/sybase/pysybase.py @@ -1,17 +1,15 @@ # sybase/pysybase.py -# Copyright (C) 2010-2011 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """ -Support for Sybase via the python-sybase driver. - -http://python-sybase.sourceforge.net/ - -Connect strings are of the form:: - - sybase+pysybase://:@/[database name] +.. dialect:: sybase+pysybase + :name: Python-Sybase + :dbapi: Sybase + :connectstring: sybase+pysybase://:@/[database name] + :url: http://python-sybase.sourceforge.net/ Unicode Support --------------- @@ -33,6 +31,7 @@ class _SybNumeric(sqltypes.Numeric): else: return sqltypes.Numeric.result_processor(self, dialect, type_) + class SybaseExecutionContext_pysybase(SybaseExecutionContext): def set_ddl_autocommit(self, dbapi_connection, value): @@ -55,14 +54,15 @@ class SybaseSQLCompiler_pysybase(SybaseSQLCompiler): def bindparam_string(self, name, **kw): return "@" + name + class SybaseDialect_pysybase(SybaseDialect): driver = 'pysybase' execution_ctx_cls = SybaseExecutionContext_pysybase statement_compiler = SybaseSQLCompiler_pysybase - colspecs={ - sqltypes.Numeric:_SybNumeric, - sqltypes.Float:sqltypes.Float + colspecs = { + sqltypes.Numeric: _SybNumeric, + sqltypes.Float: sqltypes.Float } @classmethod diff --git a/libs/sqlalchemy/engine/__init__.py b/libs/sqlalchemy/engine/__init__.py index 6ff8ba15..890c7664 100644 --- a/libs/sqlalchemy/engine/__init__.py +++ b/libs/sqlalchemy/engine/__init__.py @@ -1,5 +1,5 @@ # engine/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -50,54 +50,47 @@ url.py within a URL. """ -# not sure what this was used for -#import sqlalchemy.databases +from .interfaces import ( + Connectable, + Dialect, + ExecutionContext, -from sqlalchemy.engine.base import ( + # backwards compat + Compiled, + TypeCompiler +) + +from .base import ( + Connection, + Engine, + NestedTransaction, + RootTransaction, + Transaction, + TwoPhaseTransaction, + ) + +from .result import ( BufferedColumnResultProxy, BufferedColumnRow, BufferedRowResultProxy, - Compiled, - Connectable, - Connection, - Dialect, - Engine, - ExecutionContext, - NestedTransaction, + FullyBufferedResultProxy, ResultProxy, - RootTransaction, RowProxy, - Transaction, - TwoPhaseTransaction, - TypeCompiler - ) -from sqlalchemy.engine import strategies -from sqlalchemy import util - - -__all__ = ( - 'BufferedColumnResultProxy', - 'BufferedColumnRow', - 'BufferedRowResultProxy', - 'Compiled', - 'Connectable', - 'Connection', - 'Dialect', - 'Engine', - 'ExecutionContext', - 'NestedTransaction', - 'ResultProxy', - 'RootTransaction', - 'RowProxy', - 'Transaction', - 'TwoPhaseTransaction', - 'TypeCompiler', - 'create_engine', - 'engine_from_config', ) +from .util import ( + connection_memoize + ) + + +from . import util, strategies + +# backwards compat +from ..sql import ddl default_strategy = 'plain' + + def create_engine(*args, **kwargs): """Create a new :class:`.Engine` instance. @@ -117,11 +110,11 @@ def create_engine(*args, **kwargs): the URL can be an instance of :class:`~sqlalchemy.engine.url.URL`. ``**kwargs`` takes a wide variety of options which are routed - towards their appropriate components. Arguments may be - specific to the :class:`.Engine`, the underlying :class:`.Dialect`, as well as the - :class:`.Pool`. Specific dialects also accept keyword arguments that - are unique to that dialect. Here, we describe the parameters - that are common to most :func:`.create_engine()` usage. + towards their appropriate components. Arguments may be specific + to the :class:`.Engine`, the underlying :class:`.Dialect`, as well as + the :class:`.Pool`. Specific dialects also accept keyword + arguments that are unique to that dialect. Here, we describe the + parameters that are common to most :func:`.create_engine()` usage. Once established, the newly resulting :class:`.Engine` will request a connection from the underlying :class:`.Pool` once @@ -133,15 +126,17 @@ def create_engine(*args, **kwargs): See also: - :ref:`engines_toplevel` + :doc:`/core/engines` :ref:`connections_toplevel` - :param assert_unicode: Deprecated. This flag - sets an engine-wide default value for - the ``assert_unicode`` flag on the - :class:`.String` type - see that - type for further details. + :param case_sensitive=True: if False, result column names + will match in a case-insensitive fashion, that is, + ``row['SomeColumn']``. + + .. versionchanged:: 0.8 + By default, result row names match case-sensitively. + In version 0.7 and prior, all matches were case-insensitive. :param connect_args: a dictionary of options which will be passed directly to the DBAPI's ``connect()`` method as @@ -231,7 +226,7 @@ def create_engine(*args, **kwargs): :param execution_options: Dictionary execution options which will be applied to all connections. See - :meth:`~sqlalchemy.engine.base.Connection.execution_options` + :meth:`~sqlalchemy.engine.Connection.execution_options` :param implicit_returning=True: When ``True``, a RETURNING- compatible construct, if available, will be used to @@ -262,13 +257,13 @@ def create_engine(*args, **kwargs): opened above and beyond the pool_size setting, which defaults to five. this is only used with :class:`~sqlalchemy.pool.QueuePool`. - :param module=None: reference to a Python module object (the module itself, not - its string name). Specifies an alternate DBAPI module to be used - by the engine's dialect. Each sub-dialect references a specific DBAPI which - will be imported before first connect. This parameter causes the - import to be bypassed, and the given module to be used instead. - Can be used for testing of DBAPIs as well as to inject "mock" - DBAPI implementations into the :class:`.Engine`. + :param module=None: reference to a Python module object (the module + itself, not its string name). Specifies an alternate DBAPI module to + be used by the engine's dialect. Each sub-dialect references a + specific DBAPI which will be imported before first connect. This + parameter causes the import to be bypassed, and the given module to + be used instead. Can be used for testing of DBAPIs as well as to + inject "mock" DBAPI implementations into the :class:`.Engine`. :param pool=None: an already-constructed instance of :class:`~sqlalchemy.pool.Pool`, such as a @@ -291,7 +286,8 @@ def create_engine(*args, **kwargs): id. :param pool_size=5: the number of connections to keep open - inside the connection pool. This used with :class:`~sqlalchemy.pool.QueuePool` as + inside the connection pool. This used with + :class:`~sqlalchemy.pool.QueuePool` as well as :class:`~sqlalchemy.pool.SingletonThreadPool`. With :class:`~sqlalchemy.pool.QueuePool`, a ``pool_size`` setting of 0 indicates no limit; to disable pooling, set ``poolclass`` to @@ -325,7 +321,8 @@ def create_engine(*args, **kwargs): :ref:`threadlocal_strategy`; * the ``mock`` strategy, which dispatches all statement execution to a function passed as the argument ``executor``. - See `example in the FAQ `_. + See `example in the FAQ + `_. :param executor=None: a function taking arguments ``(sql, *multiparams, **params)``, to which the ``mock`` strategy will @@ -337,6 +334,7 @@ def create_engine(*args, **kwargs): strategy = strategies.strategies[strategy] return strategy.create(*args, **kwargs) + def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): """Create a new Engine instance using a configuration dictionary. @@ -350,27 +348,16 @@ def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): arguments. """ - opts = _coerce_config(configuration, prefix) - opts.update(kwargs) - url = opts.pop('url') - return create_engine(url, **opts) - -def _coerce_config(configuration, prefix): - """Convert configuration values to expected types.""" - options = dict((key[len(prefix):], configuration[key]) for key in configuration if key.startswith(prefix)) - for option, type_ in ( - ('convert_unicode', util.bool_or_str('force')), - ('pool_timeout', int), - ('echo', util.bool_or_str('debug')), - ('echo_pool', util.bool_or_str('debug')), - ('pool_recycle', int), - ('pool_size', int), - ('max_overflow', int), - ('pool_threadlocal', bool), - ('use_native_unicode', bool), - ): - util.coerce_kw_type(options, option, type_) - return options + options['_coerce_config'] = True + options.update(kwargs) + url = options.pop('url') + return create_engine(url, **options) + + +__all__ = ( + 'create_engine', + 'engine_from_config', + ) diff --git a/libs/sqlalchemy/engine/base.py b/libs/sqlalchemy/engine/base.py index 302fb779..0e888ca4 100644 --- a/libs/sqlalchemy/engine/base.py +++ b/libs/sqlalchemy/engine/base.py @@ -1,842 +1,21 @@ # engine/base.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php +from __future__ import with_statement +"""Defines :class:`.Connection` and :class:`.Engine`. -"""Basic components for SQL execution and interfacing with DB-API. - -Defines the basic components used to interface DB-API modules with -higher-level statement-construction, connection-management, execution -and result contexts. """ -__all__ = [ - 'BufferedColumnResultProxy', 'BufferedColumnRow', - 'BufferedRowResultProxy','Compiled', 'Connectable', 'Connection', - 'Dialect', 'Engine','ExecutionContext', 'NestedTransaction', - 'ResultProxy', 'RootTransaction','RowProxy', 'SchemaIterator', - 'StringIO', 'Transaction', 'TwoPhaseTransaction', - 'connection_memoize'] -import inspect, StringIO, sys, operator -from itertools import izip -from sqlalchemy import exc, schema, util, types, log, interfaces, \ - event, events -from sqlalchemy.sql import expression, util as sql_util -from sqlalchemy import processors -import collections - -class Dialect(object): - """Define the behavior of a specific database and DB-API combination. - - Any aspect of metadata definition, SQL query generation, - execution, result-set handling, or anything else which varies - between databases is defined under the general category of the - Dialect. The Dialect acts as a factory for other - database-specific object implementations including - ExecutionContext, Compiled, DefaultGenerator, and TypeEngine. - - All Dialects implement the following attributes: - - name - identifying name for the dialect from a DBAPI-neutral point of view - (i.e. 'sqlite') - - driver - identifying name for the dialect's DBAPI - - positional - True if the paramstyle for this Dialect is positional. - - paramstyle - the paramstyle to be used (some DB-APIs support multiple - paramstyles). - - convert_unicode - True if Unicode conversion should be applied to all ``str`` - types. - - encoding - type of encoding to use for unicode, usually defaults to - 'utf-8'. - - statement_compiler - a :class:`~Compiled` class used to compile SQL statements - - ddl_compiler - a :class:`~Compiled` class used to compile DDL statements - - server_version_info - a tuple containing a version number for the DB backend in use. - This value is only available for supporting dialects, and is - typically populated during the initial connection to the database. - - default_schema_name - the name of the default schema. This value is only available for - supporting dialects, and is typically populated during the - initial connection to the database. - - execution_ctx_cls - a :class:`.ExecutionContext` class used to handle statement execution - - execute_sequence_format - either the 'tuple' or 'list' type, depending on what cursor.execute() - accepts for the second argument (they vary). - - preparer - a :class:`~sqlalchemy.sql.compiler.IdentifierPreparer` class used to - quote identifiers. - - supports_alter - ``True`` if the database supports ``ALTER TABLE``. - - max_identifier_length - The maximum length of identifier names. - - supports_unicode_statements - Indicate whether the DB-API can receive SQL statements as Python - unicode strings - - supports_unicode_binds - Indicate whether the DB-API can receive string bind parameters - as Python unicode strings - - supports_sane_rowcount - Indicate whether the dialect properly implements rowcount for - ``UPDATE`` and ``DELETE`` statements. - - supports_sane_multi_rowcount - Indicate whether the dialect properly implements rowcount for - ``UPDATE`` and ``DELETE`` statements when executed via - executemany. - - preexecute_autoincrement_sequences - True if 'implicit' primary key functions must be executed separately - in order to get their value. This is currently oriented towards - Postgresql. - - implicit_returning - use RETURNING or equivalent during INSERT execution in order to load - newly generated primary keys and other column defaults in one execution, - which are then available via inserted_primary_key. - If an insert statement has returning() specified explicitly, - the "implicit" functionality is not used and inserted_primary_key - will not be available. - - dbapi_type_map - A mapping of DB-API type objects present in this Dialect's - DB-API implementation mapped to TypeEngine implementations used - by the dialect. - - This is used to apply types to result sets based on the DB-API - types present in cursor.description; it only takes effect for - result sets against textual statements where no explicit - typemap was present. - - colspecs - A dictionary of TypeEngine classes from sqlalchemy.types mapped - to subclasses that are specific to the dialect class. This - dictionary is class-level only and is not accessed from the - dialect instance itself. - - supports_default_values - Indicates if the construct ``INSERT INTO tablename DEFAULT - VALUES`` is supported - - supports_sequences - Indicates if the dialect supports CREATE SEQUENCE or similar. - - sequences_optional - If True, indicates if the "optional" flag on the Sequence() construct - should signal to not generate a CREATE SEQUENCE. Applies only to - dialects that support sequences. Currently used only to allow Postgresql - SERIAL to be used on a column that specifies Sequence() for usage on - other backends. - - supports_native_enum - Indicates if the dialect supports a native ENUM construct. - This will prevent types.Enum from generating a CHECK - constraint when that type is used. - - supports_native_boolean - Indicates if the dialect supports a native boolean construct. - This will prevent types.Boolean from generating a CHECK - constraint when that type is used. - - """ - - def create_connect_args(self, url): - """Build DB-API compatible connection arguments. - - Given a :class:`~sqlalchemy.engine.url.URL` object, returns a tuple - consisting of a `*args`/`**kwargs` suitable to send directly - to the dbapi's connect function. - - """ - - raise NotImplementedError() - - @classmethod - def type_descriptor(cls, typeobj): - """Transform a generic type to a dialect-specific type. - - Dialect classes will usually use the - :func:`~sqlalchemy.types.adapt_type` function in the types module to - make this job easy. - - The returned result is cached *per dialect class* so can - contain no dialect-instance state. - - """ - - raise NotImplementedError() - - def initialize(self, connection): - """Called during strategized creation of the dialect with a - connection. - - Allows dialects to configure options based on server version info or - other properties. - - The connection passed here is a SQLAlchemy Connection object, - with full capabilities. - - The initalize() method of the base dialect should be called via - super(). - - """ - - pass - - def reflecttable(self, connection, table, include_columns=None): - """Load table description from the database. - - Given a :class:`.Connection` and a - :class:`~sqlalchemy.schema.Table` object, reflect its columns and - properties from the database. If include_columns (a list or - set) is specified, limit the autoload to the given column - names. - - The default implementation uses the - :class:`~sqlalchemy.engine.reflection.Inspector` interface to - provide the output, building upon the granular table/column/ - constraint etc. methods of :class:`.Dialect`. - - """ - - raise NotImplementedError() - - def get_columns(self, connection, table_name, schema=None, **kw): - """Return information about columns in `table_name`. - - Given a :class:`.Connection`, a string - `table_name`, and an optional string `schema`, return column - information as a list of dictionaries with these keys: - - name - the column's name - - type - [sqlalchemy.types#TypeEngine] - - nullable - boolean - - default - the column's default value - - autoincrement - boolean - - sequence - a dictionary of the form - {'name' : str, 'start' :int, 'increment': int} - - Additional column attributes may be present. - """ - - raise NotImplementedError() - - def get_primary_keys(self, connection, table_name, schema=None, **kw): - """Return information about primary keys in `table_name`. - - Given a :class:`.Connection`, a string - `table_name`, and an optional string `schema`, return primary - key information as a list of column names. - - """ - raise NotImplementedError() - - def get_pk_constraint(self, table_name, schema=None, **kw): - """Return information about the primary key constraint on - table_name`. - - Given a string `table_name`, and an optional string `schema`, return - primary key information as a dictionary with these keys: - - constrained_columns - a list of column names that make up the primary key - - name - optional name of the primary key constraint. - - """ - raise NotImplementedError() - - def get_foreign_keys(self, connection, table_name, schema=None, **kw): - """Return information about foreign_keys in `table_name`. - - Given a :class:`.Connection`, a string - `table_name`, and an optional string `schema`, return foreign - key information as a list of dicts with these keys: - - name - the constraint's name - - constrained_columns - a list of column names that make up the foreign key - - referred_schema - the name of the referred schema - - referred_table - the name of the referred table - - referred_columns - a list of column names in the referred table that correspond to - constrained_columns - """ - - raise NotImplementedError() - - def get_table_names(self, connection, schema=None, **kw): - """Return a list of table names for `schema`.""" - - raise NotImplementedError - - def get_view_names(self, connection, schema=None, **kw): - """Return a list of all view names available in the database. - - schema: - Optional, retrieve names from a non-default schema. - """ - - raise NotImplementedError() - - def get_view_definition(self, connection, view_name, schema=None, **kw): - """Return view definition. - - Given a :class:`.Connection`, a string - `view_name`, and an optional string `schema`, return the view - definition. - """ - - raise NotImplementedError() - - def get_indexes(self, connection, table_name, schema=None, **kw): - """Return information about indexes in `table_name`. - - Given a :class:`.Connection`, a string - `table_name` and an optional string `schema`, return index - information as a list of dictionaries with these keys: - - name - the index's name - - column_names - list of column names in order - - unique - boolean - """ - - raise NotImplementedError() - - def normalize_name(self, name): - """convert the given name to lowercase if it is detected as - case insensitive. - - this method is only used if the dialect defines - requires_name_normalize=True. - - """ - raise NotImplementedError() - - def denormalize_name(self, name): - """convert the given name to a case insensitive identifier - for the backend if it is an all-lowercase name. - - this method is only used if the dialect defines - requires_name_normalize=True. - - """ - raise NotImplementedError() - - def has_table(self, connection, table_name, schema=None): - """Check the existence of a particular table in the database. - - Given a :class:`.Connection` object and a string - `table_name`, return True if the given table (possibly within - the specified `schema`) exists in the database, False - otherwise. - """ - - raise NotImplementedError() - - def has_sequence(self, connection, sequence_name, schema=None): - """Check the existence of a particular sequence in the database. - - Given a :class:`.Connection` object and a string - `sequence_name`, return True if the given sequence exists in - the database, False otherwise. - """ - - raise NotImplementedError() - - def _get_server_version_info(self, connection): - """Retrieve the server version info from the given connection. - - This is used by the default implementation to populate the - "server_version_info" attribute and is called exactly - once upon first connect. - - """ - - raise NotImplementedError() - - def _get_default_schema_name(self, connection): - """Return the string name of the currently selected schema from - the given connection. - - This is used by the default implementation to populate the - "default_schema_name" attribute and is called exactly - once upon first connect. - - """ - - raise NotImplementedError() - - def do_begin(self, connection): - """Provide an implementation of *connection.begin()*, given a - DB-API connection.""" - - raise NotImplementedError() - - def do_rollback(self, connection): - """Provide an implementation of *connection.rollback()*, given - a DB-API connection.""" - - raise NotImplementedError() - - def create_xid(self): - """Create a two-phase transaction ID. - - This id will be passed to do_begin_twophase(), - do_rollback_twophase(), do_commit_twophase(). Its format is - unspecified. - """ - - raise NotImplementedError() - - def do_commit(self, connection): - """Provide an implementation of *connection.commit()*, given a - DB-API connection.""" - - raise NotImplementedError() - - def do_savepoint(self, connection, name): - """Create a savepoint with the given name on a SQLAlchemy - connection.""" - - raise NotImplementedError() - - def do_rollback_to_savepoint(self, connection, name): - """Rollback a SQL Alchemy connection to the named savepoint.""" - - raise NotImplementedError() - - def do_release_savepoint(self, connection, name): - """Release the named savepoint on a SQL Alchemy connection.""" - - raise NotImplementedError() - - def do_begin_twophase(self, connection, xid): - """Begin a two phase transaction on the given connection.""" - - raise NotImplementedError() - - def do_prepare_twophase(self, connection, xid): - """Prepare a two phase transaction on the given connection.""" - - raise NotImplementedError() - - def do_rollback_twophase(self, connection, xid, is_prepared=True, - recover=False): - """Rollback a two phase transaction on the given connection.""" - - raise NotImplementedError() - - def do_commit_twophase(self, connection, xid, is_prepared=True, - recover=False): - """Commit a two phase transaction on the given connection.""" - - raise NotImplementedError() - - def do_recover_twophase(self, connection): - """Recover list of uncommited prepared two phase transaction - identifiers on the given connection.""" - - raise NotImplementedError() - - def do_executemany(self, cursor, statement, parameters, context=None): - """Provide an implementation of ``cursor.executemany(statement, - parameters)``.""" - - raise NotImplementedError() - - def do_execute(self, cursor, statement, parameters, context=None): - """Provide an implementation of ``cursor.execute(statement, - parameters)``.""" - - raise NotImplementedError() - - def do_execute_no_params(self, cursor, statement, parameters, context=None): - """Provide an implementation of ``cursor.execute(statement)``. - - The parameter collection should not be sent. - - """ - - raise NotImplementedError() - - def is_disconnect(self, e, connection, cursor): - """Return True if the given DB-API error indicates an invalid - connection""" - - raise NotImplementedError() - - def connect(self): - """return a callable which sets up a newly created DBAPI connection. - - The callable accepts a single argument "conn" which is the - DBAPI connection itself. It has no return value. - - This is used to set dialect-wide per-connection options such as - isolation modes, unicode modes, etc. - - If a callable is returned, it will be assembled into a pool listener - that receives the direct DBAPI connection, with all wrappers removed. - - If None is returned, no listener will be generated. - - """ - return None - - def reset_isolation_level(self, dbapi_conn): - """Given a DBAPI connection, revert its isolation to the default.""" - - raise NotImplementedError() - - def set_isolation_level(self, dbapi_conn, level): - """Given a DBAPI connection, set its isolation level.""" - - raise NotImplementedError() - - def get_isolation_level(self, dbapi_conn): - """Given a DBAPI connection, return its isolation level.""" - - raise NotImplementedError() - - -class ExecutionContext(object): - """A messenger object for a Dialect that corresponds to a single - execution. - - ExecutionContext should have these data members: - - connection - Connection object which can be freely used by default value - generators to execute SQL. This Connection should reference the - same underlying connection/transactional resources of - root_connection. - - root_connection - Connection object which is the source of this ExecutionContext. This - Connection may have close_with_result=True set, in which case it can - only be used once. - - dialect - dialect which created this ExecutionContext. - - cursor - DB-API cursor procured from the connection, - - compiled - if passed to constructor, sqlalchemy.engine.base.Compiled object - being executed, - - statement - string version of the statement to be executed. Is either - passed to the constructor, or must be created from the - sql.Compiled object by the time pre_exec() has completed. - - parameters - bind parameters passed to the execute() method. For compiled - statements, this is a dictionary or list of dictionaries. For - textual statements, it should be in a format suitable for the - dialect's paramstyle (i.e. dict or list of dicts for non - positional, list or list of lists/tuples for positional). - - isinsert - True if the statement is an INSERT. - - isupdate - True if the statement is an UPDATE. - - should_autocommit - True if the statement is a "committable" statement. - - postfetch_cols - a list of Column objects for which a server-side default or - inline SQL expression value was fired off. Applies to inserts - and updates. - """ - - def create_cursor(self): - """Return a new cursor generated from this ExecutionContext's - connection. - - Some dialects may wish to change the behavior of - connection.cursor(), such as postgresql which may return a PG - "server side" cursor. - """ - - raise NotImplementedError() - - def pre_exec(self): - """Called before an execution of a compiled statement. - - If a compiled statement was passed to this ExecutionContext, - the `statement` and `parameters` datamembers must be - initialized after this statement is complete. - """ - - raise NotImplementedError() - - def post_exec(self): - """Called after the execution of a compiled statement. - - If a compiled statement was passed to this ExecutionContext, - the `last_insert_ids`, `last_inserted_params`, etc. - datamembers should be available after this method completes. - """ - - raise NotImplementedError() - - def result(self): - """Return a result object corresponding to this ExecutionContext. - - Returns a ResultProxy. - """ - - raise NotImplementedError() - - def handle_dbapi_exception(self, e): - """Receive a DBAPI exception which occurred upon execute, result - fetch, etc.""" - - raise NotImplementedError() - - def should_autocommit_text(self, statement): - """Parse the given textual statement and return True if it refers to - a "committable" statement""" - - raise NotImplementedError() - - def lastrow_has_defaults(self): - """Return True if the last INSERT or UPDATE row contained - inlined or database-side defaults. - """ - - raise NotImplementedError() - - def get_rowcount(self): - """Return the DBAPI ``cursor.rowcount`` value, or in some - cases an interpreted value. - - See :attr:`.ResultProxy.rowcount` for details on this. - - """ - - raise NotImplementedError() - - -class Compiled(object): - """Represent a compiled SQL or DDL expression. - - The ``__str__`` method of the ``Compiled`` object should produce - the actual text of the statement. ``Compiled`` objects are - specific to their underlying database dialect, and also may - or may not be specific to the columns referenced within a - particular set of bind parameters. In no case should the - ``Compiled`` object be dependent on the actual values of those - bind parameters, even though it may reference those values as - defaults. - """ - - def __init__(self, dialect, statement, bind=None): - """Construct a new ``Compiled`` object. - - :param dialect: ``Dialect`` to compile against. - - :param statement: ``ClauseElement`` to be compiled. - - :param bind: Optional Engine or Connection to compile this - statement against. - """ - - self.dialect = dialect - self.bind = bind - if statement is not None: - self.statement = statement - self.can_execute = statement.supports_execution - self.string = self.process(self.statement) - - @util.deprecated("0.7", ":class:`.Compiled` objects now compile " - "within the constructor.") - def compile(self): - """Produce the internal string representation of this element.""" - pass - - @property - def sql_compiler(self): - """Return a Compiled that is capable of processing SQL expressions. - - If this compiler is one, it would likely just return 'self'. - - """ - - raise NotImplementedError() - - def process(self, obj, **kwargs): - return obj._compiler_dispatch(self, **kwargs) - - def __str__(self): - """Return the string text of the generated SQL or DDL.""" - - return self.string or '' - - def construct_params(self, params=None): - """Return the bind params for this compiled object. - - :param params: a dict of string/object pairs whose values will - override bind values compiled in to the - statement. - """ - - raise NotImplementedError() - - @property - def params(self): - """Return the bind params for this compiled object.""" - return self.construct_params() - - def execute(self, *multiparams, **params): - """Execute this compiled object.""" - - e = self.bind - if e is None: - raise exc.UnboundExecutionError( - "This Compiled object is not bound to any Engine " - "or Connection.") - return e._execute_compiled(self, multiparams, params) - - def scalar(self, *multiparams, **params): - """Execute this compiled object and return the result's - scalar value.""" - - return self.execute(*multiparams, **params).scalar() - - -class TypeCompiler(object): - """Produces DDL specification for TypeEngine objects.""" - - def __init__(self, dialect): - self.dialect = dialect - - def process(self, type_): - return type_._compiler_dispatch(self) - - -class Connectable(object): - """Interface for an object which supports execution of SQL constructs. - - The two implementations of :class:`.Connectable` are :class:`.Connection` and - :class:`.Engine`. - - Connectable must also implement the 'dialect' member which references a - :class:`.Dialect` instance. - - """ - - def connect(self, **kwargs): - """Return a :class:`.Connection` object. - - Depending on context, this may be ``self`` if this object - is already an instance of :class:`.Connection`, or a newly - procured :class:`.Connection` if this object is an instance - of :class:`.Engine`. - - """ - - def contextual_connect(self): - """Return a :class:`.Connection` object which may be part of an ongoing - context. - - Depending on context, this may be ``self`` if this object - is already an instance of :class:`.Connection`, or a newly - procured :class:`.Connection` if this object is an instance - of :class:`.Engine`. - - """ - - raise NotImplementedError() - - @util.deprecated("0.7", "Use the create() method on the given schema " - "object directly, i.e. :meth:`.Table.create`, " - ":meth:`.Index.create`, :meth:`.MetaData.create_all`") - def create(self, entity, **kwargs): - """Emit CREATE statements for the given schema entity.""" - - raise NotImplementedError() - - @util.deprecated("0.7", "Use the drop() method on the given schema " - "object directly, i.e. :meth:`.Table.drop`, " - ":meth:`.Index.drop`, :meth:`.MetaData.drop_all`") - def drop(self, entity, **kwargs): - """Emit DROP statements for the given schema entity.""" - - raise NotImplementedError() - - def execute(self, object, *multiparams, **params): - """Executes the given construct and returns a :class:`.ResultProxy`.""" - raise NotImplementedError() - - def scalar(self, object, *multiparams, **params): - """Executes and returns the first column of the first row. - - The underlying cursor is closed after execution. - """ - raise NotImplementedError() - - def _run_visitor(self, visitorcallable, element, - **kwargs): - raise NotImplementedError() - - def _execute_clauseelement(self, elem, multiparams=None, params=None): - raise NotImplementedError() +import sys +from .. import exc, util, log, interfaces +from ..sql import expression, util as sql_util, schema, ddl +from .interfaces import Connectable, Compiled +from .util import _distill_params +import contextlib class Connection(Connectable): @@ -865,7 +44,9 @@ class Connection(Connectable): """ def __init__(self, engine, connection=None, close_with_result=False, - _branch=False, _execution_options=None): + _branch=False, _execution_options=None, + _dispatch=None, + _has_events=None): """Construct a new Connection. The constructor here is not public and is only called only by an @@ -881,7 +62,14 @@ class Connection(Connectable): self.__savepoint_seq = 0 self.__branch = _branch self.__invalid = False - self._has_events = engine._has_events + self.__can_reconnect = True + if _dispatch: + self.dispatch = _dispatch + elif engine._has_events: + self.dispatch = self.dispatch._join(engine.dispatch) + self._has_events = _has_events or ( + _has_events is None and engine._has_events) + self._echo = self.engine._should_log_info() if _execution_options: self._execution_options =\ @@ -889,6 +77,9 @@ class Connection(Connectable): else: self._execution_options = engine._execution_options + if self._has_events: + self.dispatch.engine_connect(self, _branch) + def _branch(self): """Return a new Connection which references this Connection's engine and connection; but does not have close_with_result enabled, @@ -900,7 +91,10 @@ class Connection(Connectable): return self.engine._connection_cls( self.engine, - self.__connection, _branch=True) + self.__connection, + _branch=True, + _has_events=self._has_events, + _dispatch=self.dispatch) def _clone(self): """Create a shallow copy of this Connection. @@ -924,16 +118,21 @@ class Connection(Connectable): the same underlying DBAPI connection, but also defines the given execution options which will take effect for a call to :meth:`execute`. As the new :class:`.Connection` references the same - underlying resource, it is probably best to ensure that the copies + underlying resource, it's usually a good idea to ensure that the copies would be discarded immediately, which is implicit if used as in:: result = connection.execution_options(stream_results=True).\\ execute(stmt) - :meth:`.Connection.execution_options` accepts all options as those - accepted by :meth:`.Executable.execution_options`. Additionally, - it includes options that are applicable only to - :class:`.Connection`. + Note that any key/value can be passed to + :meth:`.Connection.execution_options`, and it will be stored in the + ``_execution_options`` dictionary of the :class:`.Connection`. It + is suitable for usage by end-user schemes to communicate with + event listeners, for example. + + The keywords that are currently recognized by SQLAlchemy itself + include all those listed under :meth:`.Executable.execution_options`, + as well as others that are specific to :class:`.Connection`. :param autocommit: Available on: Connection, statement. When True, a COMMIT will be invoked after execution @@ -1005,22 +204,17 @@ class Connection(Connectable): """ c = self._clone() c._execution_options = c._execution_options.union(opt) - if 'isolation_level' in opt: - c._set_isolation_level() + if self._has_events: + self.dispatch.set_connection_execution_options(c, opt) + self.dialect.set_connection_execution_options(c, opt) return c - def _set_isolation_level(self): - self.dialect.set_isolation_level(self.connection, - self._execution_options['isolation_level']) - self.connection._connection_record.finalize_callback = \ - self.dialect.reset_isolation_level - @property def closed(self): """Return True if this connection is closed.""" - return not self.__invalid and '_Connection__connection' \ - not in self.__dict__ + return '_Connection__connection' not in self.__dict__ \ + and not self.__can_reconnect @property def invalidated(self): @@ -1038,7 +232,7 @@ class Connection(Connectable): return self._revalidate_connection() def _revalidate_connection(self): - if self.__invalid: + if self.__can_reconnect and self.__invalid: if self.__transaction is not None: raise exc.InvalidRequestError( "Can't reconnect until invalid " @@ -1065,29 +259,47 @@ class Connection(Connectable): @property def info(self): - """A collection of per-DB-API connection instance properties.""" + """Info dictionary associated with the underlying DBAPI connection + referred to by this :class:`.Connection`, allowing user-defined + data to be associated with the connection. + + The data here will follow along with the DBAPI connection including + after it is returned to the connection pool and used again + in subsequent instances of :class:`.Connection`. + + """ return self.connection.info def connect(self): - """Returns self. + """Returns a branched version of this :class:`.Connection`. + + The :meth:`.Connection.close` method on the returned + :class:`.Connection` can be called and this + :class:`.Connection` will remain open. + + This method provides usage symmetry with + :meth:`.Engine.connect`, including for usage + with context managers. - This ``Connectable`` interface method returns self, allowing - Connections to be used interchangeably with Engines in most - situations that require a bind. """ - return self + return self._branch() def contextual_connect(self, **kwargs): - """Returns self. + """Returns a branched version of this :class:`.Connection`. + + The :meth:`.Connection.close` method on the returned + :class:`.Connection` can be called and this + :class:`.Connection` will remain open. + + This method provides usage symmetry with + :meth:`.Engine.contextual_connect`, including for usage + with context managers. - This ``Connectable`` interface method returns self, allowing - Connections to be used interchangeably with Engines in most - situations that require a bind. """ - return self + return self._branch() def invalidate(self, exception=None): """Invalidate the underlying DBAPI connection associated with @@ -1118,7 +330,6 @@ class Connection(Connectable): del self.__connection self.__invalid = True - def detach(self): """Detach the underlying DB-API connection from its connection pool. @@ -1167,7 +378,8 @@ class Connection(Connectable): :meth:`.Connection.begin_twophase` - use a two phase /XID transaction - :meth:`.Engine.begin` - context manager available from :class:`.Engine`. + :meth:`.Engine.begin` - context manager available from + :class:`.Engine`. """ @@ -1204,8 +416,8 @@ class Connection(Connectable): The returned object is an instance of :class:`.TwoPhaseTransaction`, which in addition to the methods provided by - :class:`.Transaction`, also provides a :meth:`~.TwoPhaseTransaction.prepare` - method. + :class:`.Transaction`, also provides a + :meth:`~.TwoPhaseTransaction.prepare` method. :param xid: the two phase transaction id. If not supplied, a random id will be generated. @@ -1220,7 +432,7 @@ class Connection(Connectable): "Cannot start a two phase transaction when a transaction " "is already in progress.") if xid is None: - xid = self.engine.dialect.create_xid(); + xid = self.engine.dialect.create_xid() self.__transaction = TwoPhaseTransaction(self, xid) return self.__transaction @@ -1243,17 +455,16 @@ class Connection(Connectable): self.engine.logger.info("BEGIN (implicit)") if self._has_events: - self.engine.dispatch.begin(self) + self.dispatch.begin(self) try: self.engine.dialect.do_begin(self.connection) - except Exception, e: + except Exception as e: self._handle_dbapi_exception(e, None, None, None, None) - raise def _rollback_impl(self): if self._has_events: - self.engine.dispatch.rollback(self) + self.dispatch.rollback(self) if self._still_open_and_connection_is_valid: if self._echo: @@ -1261,28 +472,26 @@ class Connection(Connectable): try: self.engine.dialect.do_rollback(self.connection) self.__transaction = None - except Exception, e: + except Exception as e: self._handle_dbapi_exception(e, None, None, None, None) - raise else: self.__transaction = None - def _commit_impl(self): + def _commit_impl(self, autocommit=False): if self._has_events: - self.engine.dispatch.commit(self) + self.dispatch.commit(self) if self._echo: self.engine.logger.info("COMMIT") try: self.engine.dialect.do_commit(self.connection) self.__transaction = None - except Exception, e: + except Exception as e: self._handle_dbapi_exception(e, None, None, None, None) - raise def _savepoint_impl(self, name=None): if self._has_events: - self.engine.dispatch.savepoint(self, name) + self.dispatch.savepoint(self, name) if name is None: self.__savepoint_seq += 1 @@ -1293,7 +502,7 @@ class Connection(Connectable): def _rollback_to_savepoint_impl(self, name, context): if self._has_events: - self.engine.dispatch.rollback_savepoint(self, name, context) + self.dispatch.rollback_savepoint(self, name, context) if self._still_open_and_connection_is_valid: self.engine.dialect.do_rollback_to_savepoint(self, name) @@ -1301,22 +510,24 @@ class Connection(Connectable): def _release_savepoint_impl(self, name, context): if self._has_events: - self.engine.dispatch.release_savepoint(self, name, context) + self.dispatch.release_savepoint(self, name, context) if self._still_open_and_connection_is_valid: self.engine.dialect.do_release_savepoint(self, name) self.__transaction = context def _begin_twophase_impl(self, xid): + if self._echo: + self.engine.logger.info("BEGIN TWOPHASE (implicit)") if self._has_events: - self.engine.dispatch.begin_twophase(self, xid) + self.dispatch.begin_twophase(self, xid) if self._still_open_and_connection_is_valid: self.engine.dialect.do_begin_twophase(self, xid) def _prepare_twophase_impl(self, xid): if self._has_events: - self.engine.dispatch.prepare_twophase(self, xid) + self.dispatch.prepare_twophase(self, xid) if self._still_open_and_connection_is_valid: assert isinstance(self.__transaction, TwoPhaseTransaction) @@ -1324,7 +535,7 @@ class Connection(Connectable): def _rollback_twophase_impl(self, xid, is_prepared): if self._has_events: - self.engine.dispatch.rollback_twophase(self, xid, is_prepared) + self.dispatch.rollback_twophase(self, xid, is_prepared) if self._still_open_and_connection_is_valid: assert isinstance(self.__transaction, TwoPhaseTransaction) @@ -1333,7 +544,7 @@ class Connection(Connectable): def _commit_twophase_impl(self, xid, is_prepared): if self._has_events: - self.engine.dispatch.commit_twophase(self, xid, is_prepared) + self.dispatch.commit_twophase(self, xid, is_prepared) if self._still_open_and_connection_is_valid: assert isinstance(self.__transaction, TwoPhaseTransaction) @@ -1363,15 +574,15 @@ class Connection(Connectable): and will allow no further operations. """ - try: conn = self.__connection except AttributeError: - return - if not self.__branch: - conn.close() - self.__invalid = False - del self.__connection + pass + else: + if not self.__branch: + conn.close() + del self.__connection + self.__can_reconnect = False self.__transaction = None def scalar(self, object, *multiparams, **params): @@ -1383,7 +594,8 @@ class Connection(Connectable): return self.execute(object, *multiparams, **params).scalar() def execute(self, object, *multiparams, **params): - """Executes the a SQL statement construct and returns a :class:`.ResultProxy`. + """Executes the a SQL statement construct and returns a + :class:`.ResultProxy`. :param object: The statement to be executed. May be one of: @@ -1440,51 +652,16 @@ class Connection(Connectable): DBAPI-agnostic way, use the :func:`~.expression.text` construct. """ - for c in type(object).__mro__: - if c in Connection.executors: - return Connection.executors[c]( - self, - object, - multiparams, - params) - else: + if isinstance(object, util.string_types[0]): + return self._execute_text(object, multiparams, params) + try: + meth = object._execute_on_connection + except AttributeError: raise exc.InvalidRequestError( "Unexecutable object type: %s" % type(object)) - - def __distill_params(self, multiparams, params): - """Given arguments from the calling form *multiparams, **params, - return a list of bind parameter structures, usually a list of - dictionaries. - - In the case of 'raw' execution which accepts positional parameters, - it may be a list of tuples or lists. - - """ - - if not multiparams: - if params: - return [params] - else: - return [] - elif len(multiparams) == 1: - zero = multiparams[0] - if isinstance(zero, (list, tuple)): - if not zero or hasattr(zero[0], '__iter__') and \ - not hasattr(zero[0], 'strip'): - return zero - else: - return [zero] - elif hasattr(zero, 'keys'): - return [zero] - else: - return [[zero]] else: - if hasattr(multiparams[0], '__iter__') and \ - not hasattr(multiparams[0], 'strip'): - return multiparams - else: - return [multiparams] + return meth(self, multiparams, params) def _execute_function(self, func, multiparams, params): """Execute a sql.FunctionElement object.""" @@ -1496,7 +673,7 @@ class Connection(Connectable): """Execute a schema.ColumnDefault object.""" if self._has_events: - for fn in self.engine.dispatch.before_execute: + for fn in self.dispatch.before_execute: default, multiparams, params = \ fn(self, default, multiparams, params) @@ -1509,16 +686,15 @@ class Connection(Connectable): dialect = self.dialect ctx = dialect.execution_ctx_cls._init_default( dialect, self, conn) - except Exception, e: + except Exception as e: self._handle_dbapi_exception(e, None, None, None, None) - raise ret = ctx._exec_default(default, None) if self.should_close_with_result: self.close() if self._has_events: - self.engine.dispatch.after_execute(self, + self.dispatch.after_execute(self, default, multiparams, params, ret) return ret @@ -1527,7 +703,7 @@ class Connection(Connectable): """Execute a schema.DDL object.""" if self._has_events: - for fn in self.engine.dispatch.before_execute: + for fn in self.dispatch.before_execute: ddl, multiparams, params = \ fn(self, ddl, multiparams, params) @@ -1542,7 +718,7 @@ class Connection(Connectable): compiled ) if self._has_events: - self.engine.dispatch.after_execute(self, + self.dispatch.after_execute(self, ddl, multiparams, params, ret) return ret @@ -1550,12 +726,14 @@ class Connection(Connectable): """Execute a sql.ClauseElement object.""" if self._has_events: - for fn in self.engine.dispatch.before_execute: + for fn in self.dispatch.before_execute: elem, multiparams, params = \ fn(self, elem, multiparams, params) - distilled_params = self.__distill_params(multiparams, params) + distilled_params = _distill_params(multiparams, params) if distilled_params: + # note this is usually dict but we support RowProxy + # as well; but dict.keys() as an iterator is OK keys = distilled_params[0].keys() else: keys = [] @@ -1575,7 +753,6 @@ class Connection(Connectable): dialect=dialect, column_keys=keys, inline=len(distilled_params) > 1) - ret = self._execute_context( dialect, dialect.execution_ctx_cls._init_compiled, @@ -1584,7 +761,7 @@ class Connection(Connectable): compiled_sql, distilled_params ) if self._has_events: - self.engine.dispatch.after_execute(self, + self.dispatch.after_execute(self, elem, multiparams, params, ret) return ret @@ -1592,12 +769,12 @@ class Connection(Connectable): """Execute a sql.Compiled object.""" if self._has_events: - for fn in self.engine.dispatch.before_execute: + for fn in self.dispatch.before_execute: compiled, multiparams, params = \ fn(self, compiled, multiparams, params) dialect = self.dialect - parameters=self.__distill_params(multiparams, params) + parameters = _distill_params(multiparams, params) ret = self._execute_context( dialect, dialect.execution_ctx_cls._init_compiled, @@ -1606,7 +783,7 @@ class Connection(Connectable): compiled, parameters ) if self._has_events: - self.engine.dispatch.after_execute(self, + self.dispatch.after_execute(self, compiled, multiparams, params, ret) return ret @@ -1614,12 +791,12 @@ class Connection(Connectable): """Execute a string SQL statement.""" if self._has_events: - for fn in self.engine.dispatch.before_execute: + for fn in self.dispatch.before_execute: statement, multiparams, params = \ fn(self, statement, multiparams, params) dialect = self.dialect - parameters = self.__distill_params(multiparams, params) + parameters = _distill_params(multiparams, params) ret = self._execute_context( dialect, dialect.execution_ctx_cls._init_statement, @@ -1628,7 +805,7 @@ class Connection(Connectable): statement, parameters ) if self._has_events: - self.engine.dispatch.after_execute(self, + self.dispatch.after_execute(self, statement, multiparams, params, ret) return ret @@ -1645,11 +822,10 @@ class Connection(Connectable): conn = self._revalidate_connection() context = constructor(dialect, self, conn, *args) - except Exception, e: + except Exception as e: self._handle_dbapi_exception(e, - str(statement), parameters, + util.text_type(statement), parameters, None, None) - raise if context.compiled: context.pre_exec() @@ -1662,7 +838,7 @@ class Connection(Connectable): parameters = parameters[0] if self._has_events: - for fn in self.engine.dispatch.before_cursor_execute: + for fn in self.dispatch.before_cursor_execute: statement, parameters = \ fn(self, cursor, statement, parameters, context, context.executemany) @@ -1689,18 +865,16 @@ class Connection(Connectable): statement, parameters, context) - except Exception, e: + except Exception as e: self._handle_dbapi_exception( e, statement, parameters, cursor, context) - raise - if self._has_events: - self.engine.dispatch.after_cursor_execute(self, cursor, + self.dispatch.after_cursor_execute(self, cursor, statement, parameters, context, @@ -1719,8 +893,15 @@ class Connection(Connectable): if context._is_implicit_returning: context._fetch_implicit_returning(result) result.close(_autoclose_connection=False) + result._metadata = None elif not context._is_explicit_returning: result.close(_autoclose_connection=False) + result._metadata = None + elif context.isupdate and context._is_implicit_returning: + context._fetch_implicit_update_returning(result) + result.close(_autoclose_connection=False) + result._metadata = None + elif result._metadata is None: # no results, get rowcount # (which requires open cursor on some drivers @@ -1729,14 +910,14 @@ class Connection(Connectable): result.close(_autoclose_connection=False) if self.__transaction is None and context.should_autocommit: - self._commit_impl() + self._commit_impl(autocommit=True) if result.closed and self.should_close_with_result: self.close() return result - def _cursor_execute(self, cursor, statement, parameters): + def _cursor_execute(self, cursor, statement, parameters, context=None): """Execute a statement + params on the given cursor. Adds appropriate logging and exception handling. @@ -1747,6 +928,14 @@ class Connection(Connectable): terminates at _execute_context(). """ + if self._has_events: + for fn in self.dispatch.before_cursor_execute: + statement, parameters = \ + fn(self, cursor, statement, parameters, + context, + context.executemany + if context is not None else False) + if self._echo: self.engine.logger.info(statement) self.engine.logger.info("%r", parameters) @@ -1755,14 +944,13 @@ class Connection(Connectable): cursor, statement, parameters) - except Exception, e: + except Exception as e: self._handle_dbapi_exception( e, statement, parameters, cursor, None) - raise def _safe_close_cursor(self, cursor): """Close the given cursor, catching exceptions @@ -1771,15 +959,14 @@ class Connection(Connectable): """ try: cursor.close() - except Exception, e: - try: - ex_text = str(e) - except TypeError: - ex_text = repr(e) - self.connection._logger.warn("Error closing cursor: %s", ex_text) + except (SystemExit, KeyboardInterrupt): + raise + except Exception: + self.connection._logger.error( + "Error closing cursor", exc_info=True) - if isinstance(e, (SystemExit, KeyboardInterrupt)): - raise + _reentrant_error = False + _is_disconnect = False def _handle_dbapi_exception(self, e, @@ -1787,17 +974,22 @@ class Connection(Connectable): parameters, cursor, context): - if getattr(self, '_reentrant_error', False): - # Py3K - #raise exc.DBAPIError.instance(statement, parameters, e, - # self.dialect.dbapi.Error) from e - # Py2K - raise exc.DBAPIError.instance(statement, + + exc_info = sys.exc_info() + + if not self._is_disconnect: + self._is_disconnect = isinstance(e, self.dialect.dbapi.Error) and \ + not self.closed and \ + self.dialect.is_disconnect(e, self.__connection, cursor) + + if self._reentrant_error: + util.raise_from_cause( + exc.DBAPIError.instance(statement, parameters, e, - self.dialect.dbapi.Error), \ - None, sys.exc_info()[2] - # end Py2K + self.dialect.dbapi.Error), + exc_info + ) self._reentrant_error = True try: # non-DBAPI error - if we already got a context, @@ -1807,7 +999,7 @@ class Connection(Connectable): if should_wrap and context: if self._has_events: - self.engine.dispatch.dbapi_error(self, + self.dispatch.dbapi_error(self, cursor, statement, parameters, @@ -1815,88 +1007,35 @@ class Connection(Connectable): e) context.handle_dbapi_exception(e) - is_disconnect = isinstance(e, self.dialect.dbapi.Error) and \ - self.dialect.is_disconnect(e, self.__connection, cursor) - - - if is_disconnect: - self.invalidate(e) - self.engine.dispose() - else: + if not self._is_disconnect: if cursor: self._safe_close_cursor(cursor) self._autorollback() - if self.should_close_with_result: - self.close() - if not should_wrap: - return + if should_wrap: + util.raise_from_cause( + exc.DBAPIError.instance( + statement, + parameters, + e, + self.dialect.dbapi.Error, + connection_invalidated=self._is_disconnect), + exc_info + ) - # Py3K - #raise exc.DBAPIError.instance( - # statement, - # parameters, - # e, - # self.dialect.dbapi.Error, - # connection_invalidated=is_disconnect) \ - # from e - # Py2K - raise exc.DBAPIError.instance( - statement, - parameters, - e, - self.dialect.dbapi.Error, - connection_invalidated=is_disconnect), \ - None, sys.exc_info()[2] - # end Py2K + util.reraise(*exc_info) finally: del self._reentrant_error - - # poor man's multimethod/generic function thingy - executors = { - expression.FunctionElement: _execute_function, - expression.ClauseElement: _execute_clauseelement, - Compiled: _execute_compiled, - schema.SchemaItem: _execute_default, - schema.DDLElement: _execute_ddl, - basestring: _execute_text - } - - @util.deprecated("0.7", "Use the create() method on the given schema " - "object directly, i.e. :meth:`.Table.create`, " - ":meth:`.Index.create`, :meth:`.MetaData.create_all`") - def create(self, entity, **kwargs): - """Emit CREATE statements for the given schema entity.""" - - return self.engine.create(entity, connection=self, **kwargs) - - @util.deprecated("0.7", "Use the drop() method on the given schema " - "object directly, i.e. :meth:`.Table.drop`, " - ":meth:`.Index.drop`, :meth:`.MetaData.drop_all`") - def drop(self, entity, **kwargs): - """Emit DROP statements for the given schema entity.""" - - return self.engine.drop(entity, connection=self, **kwargs) - - @util.deprecated("0.7", "Use autoload=True with :class:`.Table`, " - "or use the :class:`.Inspector` object.") - def reflecttable(self, table, include_columns=None): - """Load table description from the database. - - Given a :class:`.Table` object, reflect its columns and - properties from the database, populating the given :class:`.Table` - object with attributes.. If include_columns (a list or - set) is specified, limit the autoload to the given column - names. - - The default implementation uses the - :class:`.Inspector` interface to - provide the output, building upon the granular table/column/ - constraint etc. methods of :class:`.Dialect`. - - """ - return self.engine.reflecttable(table, self, include_columns) + if self._is_disconnect: + del self._is_disconnect + dbapi_conn_wrapper = self.connection + self.invalidate(e) + if not hasattr(dbapi_conn_wrapper, '_pool') or \ + dbapi_conn_wrapper._pool is self.engine.pool: + self.engine.dispose() + if self.should_close_with_result: + self.close() def default_schema_name(self): return self.engine.dialect.get_default_schema_name(self) @@ -1949,8 +1088,8 @@ class Connection(Connectable): trans.commit() return ret except: - trans.rollback() - raise + with util.safe_reraise(): + trans.rollback() def run_callable(self, callable_, *args, **kwargs): """Given a callable object or function, execute it, passing @@ -2056,11 +1195,12 @@ class Transaction(object): try: self.commit() except: - self.rollback() - raise + with util.safe_reraise(): + self.rollback() else: self.rollback() + class RootTransaction(Transaction): def __init__(self, connection): super(RootTransaction, self).__init__(connection, None) @@ -2136,15 +1276,15 @@ class TwoPhaseTransaction(Transaction): class Engine(Connectable, log.Identified): """ Connects a :class:`~sqlalchemy.pool.Pool` and - :class:`~sqlalchemy.engine.base.Dialect` together to provide a source - of database connectivity and behavior. + :class:`~sqlalchemy.engine.interfaces.Dialect` together to provide a + source of database connectivity and behavior. An :class:`.Engine` object is instantiated publicly using the :func:`~sqlalchemy.create_engine` function. See also: - :ref:`engines_toplevel` + :doc:`/core/engines` :ref:`connections_toplevel` @@ -2161,6 +1301,7 @@ class Engine(Connectable, log.Identified): self.pool = pool self.url = url self.dialect = dialect + self.pool._dialect = dialect if logging_name: self.logging_name = logging_name self.echo = echo @@ -2169,17 +1310,8 @@ class Engine(Connectable, log.Identified): if proxy: interfaces.ConnectionProxy._adapt_listener(self, proxy) if execution_options: - if 'isolation_level' in execution_options: - raise exc.ArgumentError( - "'isolation_level' execution option may " - "only be specified on Connection.execution_options(). " - "To set engine-wide isolation level, " - "use the isolation_level argument to create_engine()." - ) self.update_execution_options(**execution_options) - dispatch = event.dispatcher(events.ConnectionEvents) - def update_execution_options(self, **opt): """Update the default execution_options dictionary of this :class:`.Engine`. @@ -2190,31 +1322,102 @@ class Engine(Connectable, log.Identified): can be sent via the ``execution_options`` parameter to :func:`.create_engine`. - See :meth:`.Connection.execution_options` for more - details on execution options. + .. seealso:: + + :meth:`.Connection.execution_options` + + :meth:`.Engine.execution_options` """ self._execution_options = \ self._execution_options.union(opt) + self.dispatch.set_engine_execution_options(self, opt) + self.dialect.set_engine_execution_options(self, opt) + + def execution_options(self, **opt): + """Return a new :class:`.Engine` that will provide + :class:`.Connection` objects with the given execution options. + + The returned :class:`.Engine` remains related to the original + :class:`.Engine` in that it shares the same connection pool and + other state: + + * The :class:`.Pool` used by the new :class:`.Engine` is the + same instance. The :meth:`.Engine.dispose` method will replace + the connection pool instance for the parent engine as well + as this one. + * Event listeners are "cascaded" - meaning, the new :class:`.Engine` + inherits the events of the parent, and new events can be associated + with the new :class:`.Engine` individually. + * The logging configuration and logging_name is copied from the parent + :class:`.Engine`. + + The intent of the :meth:`.Engine.execution_options` method is + to implement "sharding" schemes where multiple :class:`.Engine` + objects refer to the same connection pool, but are differentiated + by options that would be consumed by a custom event:: + + primary_engine = create_engine("mysql://") + shard1 = primary_engine.execution_options(shard_id="shard1") + shard2 = primary_engine.execution_options(shard_id="shard2") + + Above, the ``shard1`` engine serves as a factory for + :class:`.Connection` objects that will contain the execution option + ``shard_id=shard1``, and ``shard2`` will produce :class:`.Connection` + objects that contain the execution option ``shard_id=shard2``. + + An event handler can consume the above execution option to perform + a schema switch or other operation, given a connection. Below + we emit a MySQL ``use`` statement to switch databases, at the same + time keeping track of which database we've established using the + :attr:`.Connection.info` dictionary, which gives us a persistent + storage space that follows the DBAPI connection:: + + from sqlalchemy import event + from sqlalchemy.engine import Engine + + shards = {"default": "base", shard_1: "db1", "shard_2": "db2"} + + @event.listens_for(Engine, "before_cursor_execute") + def _switch_shard(conn, cursor, stmt, params, context, executemany): + shard_id = conn._execution_options.get('shard_id', "default") + current_shard = conn.info.get("current_shard", None) + + if current_shard != shard_id: + cursor.execute("use %s" % shards[shard_id]) + conn.info["current_shard"] = shard_id + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`.Connection.execution_options` - update execution options + on a :class:`.Connection` object. + + :meth:`.Engine.update_execution_options` - update the execution + options for a given :class:`.Engine` in place. + + """ + return OptionEngine(self, opt) @property def name(self): - """String name of the :class:`~sqlalchemy.engine.Dialect` in use by - this ``Engine``.""" + """String name of the :class:`~sqlalchemy.engine.interfaces.Dialect` + in use by this :class:`Engine`.""" return self.dialect.name @property def driver(self): - """Driver name of the :class:`~sqlalchemy.engine.Dialect` in use by - this ``Engine``.""" + """Driver name of the :class:`~sqlalchemy.engine.interfaces.Dialect` + in use by this :class:`Engine`.""" return self.dialect.driver echo = log.echo_property() def __repr__(self): - return 'Engine(%s)' % str(self.url) + return 'Engine(%r)' % self.url def dispose(self): """Dispose of the connection pool used by this :class:`.Engine`. @@ -2240,69 +1443,24 @@ class Engine(Connectable, log.Identified): the engine are not affected. """ - self.pool.dispose() self.pool = self.pool._replace() - @util.deprecated("0.7", "Use the create() method on the given schema " - "object directly, i.e. :meth:`.Table.create`, " - ":meth:`.Index.create`, :meth:`.MetaData.create_all`") - def create(self, entity, connection=None, **kwargs): - """Emit CREATE statements for the given schema entity.""" - - from sqlalchemy.engine import ddl - - self._run_visitor(ddl.SchemaGenerator, entity, - connection=connection, **kwargs) - - @util.deprecated("0.7", "Use the drop() method on the given schema " - "object directly, i.e. :meth:`.Table.drop`, " - ":meth:`.Index.drop`, :meth:`.MetaData.drop_all`") - def drop(self, entity, connection=None, **kwargs): - """Emit DROP statements for the given schema entity.""" - - from sqlalchemy.engine import ddl - - self._run_visitor(ddl.SchemaDropper, entity, - connection=connection, **kwargs) - def _execute_default(self, default): - connection = self.contextual_connect() - try: - return connection._execute_default(default, (), {}) - finally: - connection.close() + with self.contextual_connect() as conn: + return conn._execute_default(default, (), {}) - @property - @util.deprecated("0.7", - "Use :attr:`~sqlalchemy.sql.expression.func` to create function constructs.") - def func(self): - return expression._FunctionGenerator(bind=self) - - @util.deprecated("0.7", - "Use :func:`.expression.text` to create text constructs.") - def text(self, text, *args, **kwargs): - """Return a :func:`~sqlalchemy.sql.expression.text` construct, - bound to this engine. - - This is equivalent to:: - - text("SELECT * FROM table", bind=engine) - - """ - - return expression.text(text, bind=self, *args, **kwargs) + @contextlib.contextmanager + def _optional_conn_ctx_manager(self, connection=None): + if connection is None: + with self.contextual_connect() as conn: + yield conn + else: + yield connection def _run_visitor(self, visitorcallable, element, connection=None, **kwargs): - if connection is None: - conn = self.contextual_connect(close_with_result=False) - else: - conn = connection - try: + with self._optional_conn_ctx_manager(connection) as conn: conn._run_visitor(visitorcallable, element, **kwargs) - finally: - if connection is None: - conn.close() class _trans_ctx(object): def __init__(self, conn, transaction, close_with_result): @@ -2337,11 +1495,11 @@ class Engine(Connectable, log.Identified): The ``close_with_result`` flag is normally ``False``, and indicates that the :class:`.Connection` will be closed when the operation - is complete. When set to ``True``, it indicates the :class:`.Connection` - is in "single use" mode, where the :class:`.ResultProxy` - returned by the first call to :meth:`.Connection.execute` will - close the :class:`.Connection` when that :class:`.ResultProxy` - has exhausted all result rows. + is complete. When set to ``True``, it indicates the + :class:`.Connection` is in "single use" mode, where the + :class:`.ResultProxy` returned by the first call to + :meth:`.Connection.execute` will close the :class:`.Connection` when + that :class:`.ResultProxy` has exhausted all result rows. .. versionadded:: 0.7.6 @@ -2358,8 +1516,8 @@ class Engine(Connectable, log.Identified): try: trans = conn.begin() except: - conn.close() - raise + with util.safe_reraise(): + conn.close() return Engine._trans_ctx(conn, trans, close_with_result) def transaction(self, callable_, *args, **kwargs): @@ -2401,11 +1559,8 @@ class Engine(Connectable, log.Identified): """ - conn = self.contextual_connect() - try: + with self.contextual_connect() as conn: return conn.transaction(callable_, *args, **kwargs) - finally: - conn.close() def run_callable(self, callable_, *args, **kwargs): """Given a callable object or function, execute it, passing @@ -2420,11 +1575,8 @@ class Engine(Connectable, log.Identified): which one is being dealt with. """ - conn = self.contextual_connect() - try: + with self.contextual_connect() as conn: return conn.run_callable(callable_, *args, **kwargs) - finally: - conn.close() def execute(self, statement, *multiparams, **params): """Executes the given construct and returns a :class:`.ResultProxy`. @@ -2459,29 +1611,33 @@ class Engine(Connectable, log.Identified): def connect(self, **kwargs): """Return a new :class:`.Connection` object. - The :class:`.Connection` object is a facade that uses a DBAPI connection internally - in order to communicate with the database. This connection is procured - from the connection-holding :class:`.Pool` referenced by this :class:`.Engine`. - When the :meth:`~.Connection.close` method of the :class:`.Connection` object is called, - the underlying DBAPI connection is then returned to the connection pool, - where it may be used again in a subsequent call to :meth:`~.Engine.connect`. + The :class:`.Connection` object is a facade that uses a DBAPI + connection internally in order to communicate with the database. This + connection is procured from the connection-holding :class:`.Pool` + referenced by this :class:`.Engine`. When the + :meth:`~.Connection.close` method of the :class:`.Connection` object + is called, the underlying DBAPI connection is then returned to the + connection pool, where it may be used again in a subsequent call to + :meth:`~.Engine.connect`. """ return self._connection_cls(self, **kwargs) def contextual_connect(self, close_with_result=False, **kwargs): - """Return a :class:`.Connection` object which may be part of some ongoing context. + """Return a :class:`.Connection` object which may be part of some + ongoing context. By default, this method does the same thing as :meth:`.Engine.connect`. Subclasses of :class:`.Engine` may override this method to provide contextual behavior. - :param close_with_result: When True, the first :class:`.ResultProxy` created - by the :class:`.Connection` will call the :meth:`.Connection.close` method - of that connection as soon as any pending result rows are exhausted. - This is used to supply the "connectionless execution" behavior provided - by the :meth:`.Engine.execute` method. + :param close_with_result: When True, the first :class:`.ResultProxy` + created by the :class:`.Connection` will call the + :meth:`.Connection.close` method of that connection as soon as any + pending result rows are exhausted. This is used to supply the + "connectionless execution" behavior provided by the + :meth:`.Engine.execute` method. """ @@ -2499,42 +1655,23 @@ class Engine(Connectable, log.Identified): the ``contextual_connect`` for this ``Engine``. """ - if connection is None: - conn = self.contextual_connect() - else: - conn = connection - if not schema: - schema = self.dialect.default_schema_name - try: + with self._optional_conn_ctx_manager(connection) as conn: + if not schema: + schema = self.dialect.default_schema_name return self.dialect.get_table_names(conn, schema) - finally: - if connection is None: - conn.close() - - @util.deprecated("0.7", "Use autoload=True with :class:`.Table`, " - "or use the :class:`.Inspector` object.") - def reflecttable(self, table, connection=None, include_columns=None): - """Load table description from the database. - - Uses the given :class:`.Connection`, or if None produces - its own :class:`.Connection`, and passes the ``table`` - and ``include_columns`` arguments onto that - :class:`.Connection` object's :meth:`.Connection.reflecttable` - method. The :class:`.Table` object is then populated - with new attributes. - - """ - if connection is None: - conn = self.contextual_connect() - else: - conn = connection - try: - self.dialect.reflecttable(conn, table, include_columns) - finally: - if connection is None: - conn.close() def has_table(self, table_name, schema=None): + """Return True if the given backend has a table of the given name. + + .. seealso:: + + :ref:`metadata_reflection_inspector` - detailed schema inspection using + the :class:`.Inspector` interface. + + :class:`.quoted_name` - used to pass quoting information along + with a schema identifier. + + """ return self.run_callable(self.dialect.has_table, table_name, schema) def raw_connection(self): @@ -2557,917 +1694,31 @@ class Engine(Connectable, log.Identified): return self.pool.unique_connection() -# This reconstructor is necessary so that pickles with the C extension or -# without use the same Binary format. -try: - # We need a different reconstructor on the C extension so that we can - # add extra checks that fields have correctly been initialized by - # __setstate__. - from sqlalchemy.cresultproxy import safe_rowproxy_reconstructor - - # The extra function embedding is needed so that the - # reconstructor function has the same signature whether or not - # the extension is present. - def rowproxy_reconstructor(cls, state): - return safe_rowproxy_reconstructor(cls, state) -except ImportError: - def rowproxy_reconstructor(cls, state): - obj = cls.__new__(cls) - obj.__setstate__(state) - return obj - -try: - from sqlalchemy.cresultproxy import BaseRowProxy -except ImportError: - class BaseRowProxy(object): - __slots__ = ('_parent', '_row', '_processors', '_keymap') - - def __init__(self, parent, row, processors, keymap): - """RowProxy objects are constructed by ResultProxy objects.""" - - self._parent = parent - self._row = row - self._processors = processors - self._keymap = keymap - - def __reduce__(self): - return (rowproxy_reconstructor, - (self.__class__, self.__getstate__())) - - def values(self): - """Return the values represented by this RowProxy as a list.""" - return list(self) - - def __iter__(self): - for processor, value in izip(self._processors, self._row): - if processor is None: - yield value - else: - yield processor(value) - - def __len__(self): - return len(self._row) - - def __getitem__(self, key): - try: - processor, obj, index = self._keymap[key] - except KeyError: - processor, obj, index = self._parent._key_fallback(key) - except TypeError: - if isinstance(key, slice): - l = [] - for processor, value in izip(self._processors[key], - self._row[key]): - if processor is None: - l.append(value) - else: - l.append(processor(value)) - return tuple(l) - else: - raise - if index is None: - raise exc.InvalidRequestError( - "Ambiguous column name '%s' in result set! " - "try 'use_labels' option on select statement." % key) - if processor is not None: - return processor(self._row[index]) - else: - return self._row[index] - - def __getattr__(self, name): - try: - return self[name] - except KeyError, e: - raise AttributeError(e.args[0]) - - -class RowProxy(BaseRowProxy): - """Proxy values from a single cursor row. - - Mostly follows "ordered dictionary" behavior, mapping result - values to the string-based column name, the integer position of - the result in the row, as well as Column instances which can be - mapped to the original Columns that produced this result set (for - results that correspond to constructed SQL expressions). - """ - __slots__ = () - - def __contains__(self, key): - return self._parent._has_key(self._row, key) - - def __getstate__(self): - return { - '_parent': self._parent, - '_row': tuple(self) - } - - def __setstate__(self, state): - self._parent = parent = state['_parent'] - self._row = state['_row'] - self._processors = parent._processors - self._keymap = parent._keymap - - __hash__ = None - - def __eq__(self, other): - return other is self or other == tuple(self) - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return repr(tuple(self)) - - def has_key(self, key): - """Return True if this RowProxy contains the given key.""" - - return self._parent._has_key(self._row, key) - - def items(self): - """Return a list of tuples, each tuple containing a key/value pair.""" - # TODO: no coverage here - return [(key, self[key]) for key in self.iterkeys()] - - def keys(self): - """Return the list of keys as strings represented by this RowProxy.""" - - return self._parent.keys - - def iterkeys(self): - return iter(self._parent.keys) - - def itervalues(self): - return iter(self) - -try: - # Register RowProxy with Sequence, - # so sequence protocol is implemented - from collections import Sequence - Sequence.register(RowProxy) -except ImportError: - pass - - -class ResultMetaData(object): - """Handle cursor.description, applying additional info from an execution - context.""" - - def __init__(self, parent, metadata): - self._processors = processors = [] - - # We do not strictly need to store the processor in the key mapping, - # though it is faster in the Python version (probably because of the - # saved attribute lookup self._processors) - self._keymap = keymap = {} - self.keys = [] - context = parent.context - dialect = context.dialect - typemap = dialect.dbapi_type_map - translate_colname = context._translate_colname - - # high precedence key values. - primary_keymap = {} - - for i, rec in enumerate(metadata): - colname = rec[0] - coltype = rec[1] - - if dialect.description_encoding: - colname = dialect._description_decoder(colname) - - if translate_colname: - colname, untranslated = translate_colname(colname) - - if context.result_map: - try: - name, obj, type_ = context.result_map[colname.lower()] - except KeyError: - name, obj, type_ = \ - colname, None, typemap.get(coltype, types.NULLTYPE) - else: - name, obj, type_ = \ - colname, None, typemap.get(coltype, types.NULLTYPE) - - processor = type_._cached_result_processor(dialect, coltype) - - processors.append(processor) - rec = (processor, obj, i) - - # indexes as keys. This is only needed for the Python version of - # RowProxy (the C version uses a faster path for integer indexes). - primary_keymap[i] = rec - - # populate primary keymap, looking for conflicts. - if primary_keymap.setdefault(name.lower(), rec) is not rec: - # place a record that doesn't have the "index" - this - # is interpreted later as an AmbiguousColumnError, - # but only when actually accessed. Columns - # colliding by name is not a problem if those names - # aren't used; integer and ColumnElement access is always - # unambiguous. - primary_keymap[name.lower()] = (processor, obj, None) - - if dialect.requires_name_normalize: - colname = dialect.normalize_name(colname) - - self.keys.append(colname) - if obj: - for o in obj: - keymap[o] = rec - - if translate_colname and \ - untranslated: - keymap[untranslated] = rec - - # overwrite keymap values with those of the - # high precedence keymap. - keymap.update(primary_keymap) - - if parent._echo: - context.engine.logger.debug( - "Col %r", tuple(x[0] for x in metadata)) - - @util.pending_deprecation("0.8", "sqlite dialect uses " - "_translate_colname() now") - def _set_keymap_synonym(self, name, origname): - """Set a synonym for the given name. - - Some dialects (SQLite at the moment) may use this to - adjust the column names that are significant within a - row. - - """ - rec = (processor, obj, i) = self._keymap[origname.lower()] - if self._keymap.setdefault(name, rec) is not rec: - self._keymap[name] = (processor, obj, None) - - def _key_fallback(self, key, raiseerr=True): - map = self._keymap - result = None - if isinstance(key, basestring): - result = map.get(key.lower()) - # fallback for targeting a ColumnElement to a textual expression - # this is a rare use case which only occurs when matching text() - # or colummn('name') constructs to ColumnElements, or after a - # pickle/unpickle roundtrip - elif isinstance(key, expression.ColumnElement): - if key._label and key._label.lower() in map: - result = map[key._label.lower()] - elif hasattr(key, 'name') and key.name.lower() in map: - # match is only on name. - result = map[key.name.lower()] - # search extra hard to make sure this - # isn't a column/label name overlap. - # this check isn't currently available if the row - # was unpickled. - if result is not None and \ - result[1] is not None: - for obj in result[1]: - if key._compare_name_for_result(obj): - break - else: - result = None - if result is None: - if raiseerr: - raise exc.NoSuchColumnError( - "Could not locate column in row for column '%s'" % - expression._string_or_unprintable(key)) - else: - return None - else: - map[key] = result - return result - - def _has_key(self, row, key): - if key in self._keymap: - return True - else: - return self._key_fallback(key, False) is not None - - def __getstate__(self): - return { - '_pickled_keymap': dict( - (key, index) - for key, (processor, obj, index) in self._keymap.iteritems() - if isinstance(key, (basestring, int)) - ), - 'keys': self.keys - } - - def __setstate__(self, state): - # the row has been processed at pickling time so we don't need any - # processor anymore - self._processors = [None for _ in xrange(len(state['keys']))] - self._keymap = keymap = {} - for key, index in state['_pickled_keymap'].iteritems(): - # not preserving "obj" here, unfortunately our - # proxy comparison fails with the unpickle - keymap[key] = (None, None, index) - self.keys = state['keys'] - self._echo = False - - -class ResultProxy(object): - """Wraps a DB-API cursor object to provide easier access to row columns. - - Individual columns may be accessed by their integer position, - case-insensitive column name, or by ``schema.Column`` - object. e.g.:: - - row = fetchone() - - col1 = row[0] # access via integer position - - col2 = row['col2'] # access via name - - col3 = row[mytable.c.mycol] # access via Column object. - - ``ResultProxy`` also handles post-processing of result column - data using ``TypeEngine`` objects, which are referenced from - the originating SQL statement that produced this result set. - - """ - - _process_row = RowProxy - out_parameters = None - _can_close_connection = False - - def __init__(self, context): - self.context = context - self.dialect = context.dialect - self.closed = False - self.cursor = self._saved_cursor = context.cursor - self.connection = context.root_connection - self._echo = self.connection._echo and \ - context.engine._should_log_debug() - self._init_metadata() - - def _init_metadata(self): - metadata = self._cursor_description() - if metadata is None: - self._metadata = None - else: - self._metadata = ResultMetaData(self, metadata) - - def keys(self): - """Return the current set of string keys for rows.""" - if self._metadata: - return self._metadata.keys - else: - return [] - - @util.memoized_property - def rowcount(self): - """Return the 'rowcount' for this result. - - The 'rowcount' reports the number of rows *matched* - by the WHERE criterion of an UPDATE or DELETE statement. - - .. note:: - - Notes regarding :attr:`.ResultProxy.rowcount`: - - - * This attribute returns the number of rows *matched*, - which is not necessarily the same as the number of rows - that were actually *modified* - an UPDATE statement, for example, - may have no net change on a given row if the SET values - given are the same as those present in the row already. - Such a row would be matched but not modified. - On backends that feature both styles, such as MySQL, - rowcount is configured by default to return the match - count in all cases. - - * :attr:`.ResultProxy.rowcount` is *only* useful in conjunction - with an UPDATE or DELETE statement. Contrary to what the Python - DBAPI says, it does *not* return the - number of rows available from the results of a SELECT statement - as DBAPIs cannot support this functionality when rows are - unbuffered. - - * :attr:`.ResultProxy.rowcount` may not be fully implemented by - all dialects. In particular, most DBAPIs do not support an - aggregate rowcount result from an executemany call. - The :meth:`.ResultProxy.supports_sane_rowcount` and - :meth:`.ResultProxy.supports_sane_multi_rowcount` methods - will report from the dialect if each usage is known to be - supported. - - * Statements that use RETURNING may not return a correct - rowcount. - - """ - try: - return self.context.rowcount - except Exception, e: - self.connection._handle_dbapi_exception( - e, None, None, self.cursor, self.context) - raise - - @property - def lastrowid(self): - """return the 'lastrowid' accessor on the DBAPI cursor. - - This is a DBAPI specific method and is only functional - for those backends which support it, for statements - where it is appropriate. It's behavior is not - consistent across backends. - - Usage of this method is normally unnecessary; the - :attr:`~ResultProxy.inserted_primary_key` attribute provides a - tuple of primary key values for a newly inserted row, - regardless of database backend. - - """ - try: - return self._saved_cursor.lastrowid - except Exception, e: - self.connection._handle_dbapi_exception( - e, None, None, - self._saved_cursor, self.context) - raise - - @property - def returns_rows(self): - """True if this :class:`.ResultProxy` returns rows. - - I.e. if it is legal to call the methods - :meth:`~.ResultProxy.fetchone`, - :meth:`~.ResultProxy.fetchmany` - :meth:`~.ResultProxy.fetchall`. - - """ - return self._metadata is not None - - @property - def is_insert(self): - """True if this :class:`.ResultProxy` is the result - of a executing an expression language compiled - :func:`.expression.insert` construct. - - When True, this implies that the - :attr:`inserted_primary_key` attribute is accessible, - assuming the statement did not include - a user defined "returning" construct. - - """ - return self.context.isinsert - - def _cursor_description(self): - """May be overridden by subclasses.""" - - return self._saved_cursor.description - - def close(self, _autoclose_connection=True): - """Close this ResultProxy. - - Closes the underlying DBAPI cursor corresponding to the execution. - - Note that any data cached within this ResultProxy is still available. - For some types of results, this may include buffered rows. - - If this ResultProxy was generated from an implicit execution, - the underlying Connection will also be closed (returns the - underlying DBAPI connection to the connection pool.) - - This method is called automatically when: - - * all result rows are exhausted using the fetchXXX() methods. - * cursor.description is None. - - """ - - if not self.closed: - self.closed = True - self.connection._safe_close_cursor(self.cursor) - if _autoclose_connection and \ - self.connection.should_close_with_result: - self.connection.close() - # allow consistent errors - self.cursor = None - - def __iter__(self): - while True: - row = self.fetchone() - if row is None: - raise StopIteration - else: - yield row - - @util.memoized_property - def inserted_primary_key(self): - """Return the primary key for the row just inserted. - - The return value is a list of scalar values - corresponding to the list of primary key columns - in the target table. - - This only applies to single row :func:`.insert` - constructs which did not explicitly specify - :meth:`.Insert.returning`. - - Note that primary key columns which specify a - server_default clause, - or otherwise do not qualify as "autoincrement" - columns (see the notes at :class:`.Column`), and were - generated using the database-side default, will - appear in this list as ``None`` unless the backend - supports "returning" and the insert statement executed - with the "implicit returning" enabled. - - """ - - if not self.context.isinsert: - raise exc.InvalidRequestError( - "Statement is not an insert() expression construct.") - elif self.context._is_explicit_returning: - raise exc.InvalidRequestError( - "Can't call inserted_primary_key when returning() " - "is used.") - - return self.context.inserted_primary_key - - @util.deprecated("0.6", "Use :attr:`.ResultProxy.inserted_primary_key`") - def last_inserted_ids(self): - """Return the primary key for the row just inserted.""" - - return self.inserted_primary_key - - def last_updated_params(self): - """Return the collection of updated parameters from this - execution. - - """ - if self.context.executemany: - return self.context.compiled_parameters - else: - return self.context.compiled_parameters[0] - - def last_inserted_params(self): - """Return the collection of inserted parameters from this - execution. - - """ - if self.context.executemany: - return self.context.compiled_parameters - else: - return self.context.compiled_parameters[0] - - def lastrow_has_defaults(self): - """Return ``lastrow_has_defaults()`` from the underlying - ExecutionContext. - - See ExecutionContext for details. - """ - - return self.context.lastrow_has_defaults() - - def postfetch_cols(self): - """Return ``postfetch_cols()`` from the underlying ExecutionContext. - - See ExecutionContext for details. - """ - - return self.context.postfetch_cols - - def prefetch_cols(self): - return self.context.prefetch_cols - - def supports_sane_rowcount(self): - """Return ``supports_sane_rowcount`` from the dialect. - - See :attr:`.ResultProxy.rowcount` for background. - - """ - - return self.dialect.supports_sane_rowcount - - def supports_sane_multi_rowcount(self): - """Return ``supports_sane_multi_rowcount`` from the dialect. - - See :attr:`.ResultProxy.rowcount` for background. - - """ - - return self.dialect.supports_sane_multi_rowcount - - def _fetchone_impl(self): - try: - return self.cursor.fetchone() - except AttributeError: - self._non_result() - - def _fetchmany_impl(self, size=None): - try: - if size is None: - return self.cursor.fetchmany() - else: - return self.cursor.fetchmany(size) - except AttributeError: - self._non_result() - - def _fetchall_impl(self): - try: - return self.cursor.fetchall() - except AttributeError: - self._non_result() - - def _non_result(self): - if self._metadata is None: - raise exc.ResourceClosedError( - "This result object does not return rows. " - "It has been closed automatically.", - ) - else: - raise exc.ResourceClosedError("This result object is closed.") - - def process_rows(self, rows): - process_row = self._process_row - metadata = self._metadata - keymap = metadata._keymap - processors = metadata._processors - if self._echo: - log = self.context.engine.logger.debug - l = [] - for row in rows: - log("Row %r", row) - l.append(process_row(metadata, row, processors, keymap)) - return l - else: - return [process_row(metadata, row, processors, keymap) - for row in rows] - - def fetchall(self): - """Fetch all rows, just like DB-API ``cursor.fetchall()``.""" - - try: - l = self.process_rows(self._fetchall_impl()) - self.close() - return l - except Exception, e: - self.connection._handle_dbapi_exception( - e, None, None, - self.cursor, self.context) - raise - - def fetchmany(self, size=None): - """Fetch many rows, just like DB-API - ``cursor.fetchmany(size=cursor.arraysize)``. - - If rows are present, the cursor remains open after this is called. - Else the cursor is automatically closed and an empty list is returned. - - """ - - try: - l = self.process_rows(self._fetchmany_impl(size)) - if len(l) == 0: - self.close() - return l - except Exception, e: - self.connection._handle_dbapi_exception( - e, None, None, - self.cursor, self.context) - raise - - def fetchone(self): - """Fetch one row, just like DB-API ``cursor.fetchone()``. - - If a row is present, the cursor remains open after this is called. - Else the cursor is automatically closed and None is returned. - - """ - try: - row = self._fetchone_impl() - if row is not None: - return self.process_rows([row])[0] - else: - self.close() - return None - except Exception, e: - self.connection._handle_dbapi_exception( - e, None, None, - self.cursor, self.context) - raise - - def first(self): - """Fetch the first row and then close the result set unconditionally. - - Returns None if no row is present. - - """ - if self._metadata is None: - self._non_result() - - try: - row = self._fetchone_impl() - except Exception, e: - self.connection._handle_dbapi_exception( - e, None, None, - self.cursor, self.context) - raise - - try: - if row is not None: - return self.process_rows([row])[0] - else: - return None - finally: - self.close() - - def scalar(self): - """Fetch the first column of the first row, and close the result set. - - Returns None if no row is present. - - """ - row = self.first() - if row is not None: - return row[0] - else: - return None - -class BufferedRowResultProxy(ResultProxy): - """A ResultProxy with row buffering behavior. - - ``ResultProxy`` that buffers the contents of a selection of rows - before ``fetchone()`` is called. This is to allow the results of - ``cursor.description`` to be available immediately, when - interfacing with a DB-API that requires rows to be consumed before - this information is available (currently psycopg2, when used with - server-side cursors). - - The pre-fetching behavior fetches only one row initially, and then - grows its buffer size by a fixed amount with each successive need - for additional rows up to a size of 100. - """ - - def _init_metadata(self): - self.__buffer_rows() - super(BufferedRowResultProxy, self)._init_metadata() - - # this is a "growth chart" for the buffering of rows. - # each successive __buffer_rows call will use the next - # value in the list for the buffer size until the max - # is reached - size_growth = { - 1 : 5, - 5 : 10, - 10 : 20, - 20 : 50, - 50 : 100, - 100 : 250, - 250 : 500, - 500 : 1000 - } - - def __buffer_rows(self): - size = getattr(self, '_bufsize', 1) - self.__rowbuffer = collections.deque(self.cursor.fetchmany(size)) - self._bufsize = self.size_growth.get(size, size) - - def _fetchone_impl(self): - if self.closed: - return None - if not self.__rowbuffer: - self.__buffer_rows() - if not self.__rowbuffer: - return None - return self.__rowbuffer.popleft() - - def _fetchmany_impl(self, size=None): - if size is None: - return self._fetchall_impl() - result = [] - for x in range(0, size): - row = self._fetchone_impl() - if row is None: - break - result.append(row) - return result - - def _fetchall_impl(self): - self.__rowbuffer.extend(self.cursor.fetchall()) - ret = self.__rowbuffer - self.__rowbuffer = collections.deque() - return ret - -class FullyBufferedResultProxy(ResultProxy): - """A result proxy that buffers rows fully upon creation. - - Used for operations where a result is to be delivered - after the database conversation can not be continued, - such as MSSQL INSERT...OUTPUT after an autocommit. - - """ - def _init_metadata(self): - super(FullyBufferedResultProxy, self)._init_metadata() - self.__rowbuffer = self._buffer_rows() - - def _buffer_rows(self): - return collections.deque(self.cursor.fetchall()) - - def _fetchone_impl(self): - if self.__rowbuffer: - return self.__rowbuffer.popleft() - else: - return None - - def _fetchmany_impl(self, size=None): - if size is None: - return self._fetchall_impl() - result = [] - for x in range(0, size): - row = self._fetchone_impl() - if row is None: - break - result.append(row) - return result - - def _fetchall_impl(self): - ret = self.__rowbuffer - self.__rowbuffer = collections.deque() - return ret - -class BufferedColumnRow(RowProxy): - def __init__(self, parent, row, processors, keymap): - # preprocess row - row = list(row) - # this is a tad faster than using enumerate - index = 0 - for processor in parent._orig_processors: - if processor is not None: - row[index] = processor(row[index]) - index += 1 - row = tuple(row) - super(BufferedColumnRow, self).__init__(parent, row, - processors, keymap) - -class BufferedColumnResultProxy(ResultProxy): - """A ResultProxy with column buffering behavior. - - ``ResultProxy`` that loads all columns into memory each time - fetchone() is called. If fetchmany() or fetchall() are called, - the full grid of results is fetched. This is to operate with - databases where result rows contain "live" results that fall out - of scope unless explicitly fetched. Currently this includes - cx_Oracle LOB objects. - - """ - - _process_row = BufferedColumnRow - - def _init_metadata(self): - super(BufferedColumnResultProxy, self)._init_metadata() - metadata = self._metadata - # orig_processors will be used to preprocess each row when they are - # constructed. - metadata._orig_processors = metadata._processors - # replace the all type processors by None processors. - metadata._processors = [None for _ in xrange(len(metadata.keys))] - keymap = {} - for k, (func, obj, index) in metadata._keymap.iteritems(): - keymap[k] = (None, obj, index) - self._metadata._keymap = keymap - - def fetchall(self): - # can't call cursor.fetchall(), since rows must be - # fully processed before requesting more from the DBAPI. - l = [] - while True: - row = self.fetchone() - if row is None: - break - l.append(row) - return l - - def fetchmany(self, size=None): - # can't call cursor.fetchmany(), since rows must be - # fully processed before requesting more from the DBAPI. - if size is None: - return self.fetchall() - l = [] - for i in xrange(size): - row = self.fetchone() - if row is None: - break - l.append(row) - return l - -def connection_memoize(key): - """Decorator, memoize a function in a connection.info stash. - - Only applicable to functions which take no arguments other than a - connection. The memo will be stored in ``connection.info[key]``. - """ - - @util.decorator - def decorated(fn, self, connection): - connection = connection.connect() - try: - return connection.info[key] - except KeyError: - connection.info[key] = val = fn(self, connection) - return val - - return decorated +class OptionEngine(Engine): + def __init__(self, proxied, execution_options): + self._proxied = proxied + self.url = proxied.url + self.dialect = proxied.dialect + self.logging_name = proxied.logging_name + self.echo = proxied.echo + log.instance_logger(self, echoflag=self.echo) + self.dispatch = self.dispatch._join(proxied.dispatch) + self._execution_options = proxied._execution_options + self.update_execution_options(**execution_options) + + def _get_pool(self): + return self._proxied.pool + + def _set_pool(self, pool): + self._proxied.pool = pool + + pool = property(_get_pool, _set_pool) + + def _get_has_events(self): + return self._proxied._has_events or \ + self.__dict__.get('_has_events', False) + + def _set_has_events(self, value): + self.__dict__['_has_events'] = value + + _has_events = property(_get_has_events, _set_has_events) diff --git a/libs/sqlalchemy/engine/ddl.py b/libs/sqlalchemy/engine/ddl.py deleted file mode 100644 index c3b32505..00000000 --- a/libs/sqlalchemy/engine/ddl.py +++ /dev/null @@ -1,182 +0,0 @@ -# engine/ddl.py -# Copyright (C) 2009-2011 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -"""Routines to handle CREATE/DROP workflow.""" - -from sqlalchemy import engine, schema -from sqlalchemy.sql import util as sql_util - - -class DDLBase(schema.SchemaVisitor): - def __init__(self, connection): - self.connection = connection - -class SchemaGenerator(DDLBase): - def __init__(self, dialect, connection, checkfirst=False, tables=None, **kwargs): - super(SchemaGenerator, self).__init__(connection, **kwargs) - self.checkfirst = checkfirst - self.tables = tables and set(tables) or None - self.preparer = dialect.identifier_preparer - self.dialect = dialect - self.memo = {} - - def _can_create_table(self, table): - self.dialect.validate_identifier(table.name) - if table.schema: - self.dialect.validate_identifier(table.schema) - return not self.checkfirst or \ - not self.dialect.has_table(self.connection, - table.name, schema=table.schema) - - def _can_create_sequence(self, sequence): - return self.dialect.supports_sequences and \ - ( - (not self.dialect.sequences_optional or - not sequence.optional) and - ( - not self.checkfirst or - not self.dialect.has_sequence( - self.connection, - sequence.name, - schema=sequence.schema) - ) - ) - - def visit_metadata(self, metadata): - if self.tables: - tables = self.tables - else: - tables = metadata.tables.values() - collection = [t for t in sql_util.sort_tables(tables) - if self._can_create_table(t)] - seq_coll = [s for s in metadata._sequences.values() - if s.column is None and self._can_create_sequence(s)] - - metadata.dispatch.before_create(metadata, self.connection, - tables=collection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - for seq in seq_coll: - self.traverse_single(seq, create_ok=True) - - for table in collection: - self.traverse_single(table, create_ok=True) - - metadata.dispatch.after_create(metadata, self.connection, - tables=collection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - def visit_table(self, table, create_ok=False): - if not create_ok and not self._can_create_table(table): - return - - table.dispatch.before_create(table, self.connection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - for column in table.columns: - if column.default is not None: - self.traverse_single(column.default) - - self.connection.execute(schema.CreateTable(table)) - - if hasattr(table, 'indexes'): - for index in table.indexes: - self.traverse_single(index) - - table.dispatch.after_create(table, self.connection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - def visit_sequence(self, sequence, create_ok=False): - if not create_ok and not self._can_create_sequence(sequence): - return - self.connection.execute(schema.CreateSequence(sequence)) - - def visit_index(self, index): - self.connection.execute(schema.CreateIndex(index)) - - -class SchemaDropper(DDLBase): - def __init__(self, dialect, connection, checkfirst=False, tables=None, **kwargs): - super(SchemaDropper, self).__init__(connection, **kwargs) - self.checkfirst = checkfirst - self.tables = tables - self.preparer = dialect.identifier_preparer - self.dialect = dialect - self.memo = {} - - def visit_metadata(self, metadata): - if self.tables: - tables = self.tables - else: - tables = metadata.tables.values() - collection = [t for t in reversed(sql_util.sort_tables(tables)) - if self._can_drop_table(t)] - seq_coll = [s for s in metadata._sequences.values() - if s.column is None and self._can_drop_sequence(s)] - - metadata.dispatch.before_drop(metadata, self.connection, - tables=collection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - for table in collection: - self.traverse_single(table, drop_ok=True) - - for seq in seq_coll: - self.traverse_single(seq, drop_ok=True) - - metadata.dispatch.after_drop(metadata, self.connection, - tables=collection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - def _can_drop_table(self, table): - self.dialect.validate_identifier(table.name) - if table.schema: - self.dialect.validate_identifier(table.schema) - return not self.checkfirst or self.dialect.has_table(self.connection, - table.name, schema=table.schema) - - def _can_drop_sequence(self, sequence): - return self.dialect.supports_sequences and \ - ((not self.dialect.sequences_optional or - not sequence.optional) and - (not self.checkfirst or - self.dialect.has_sequence( - self.connection, - sequence.name, - schema=sequence.schema)) - ) - - def visit_index(self, index): - self.connection.execute(schema.DropIndex(index)) - - def visit_table(self, table, drop_ok=False): - if not drop_ok and not self._can_drop_table(table): - return - - table.dispatch.before_drop(table, self.connection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - for column in table.columns: - if column.default is not None: - self.traverse_single(column.default) - - self.connection.execute(schema.DropTable(table)) - - table.dispatch.after_drop(table, self.connection, - checkfirst=self.checkfirst, - _ddl_runner=self) - - def visit_sequence(self, sequence, drop_ok=False): - if not drop_ok and not self._can_drop_sequence(sequence): - return - self.connection.execute(schema.DropSequence(sequence)) diff --git a/libs/sqlalchemy/engine/default.py b/libs/sqlalchemy/engine/default.py index f3dfd95e..509d772a 100644 --- a/libs/sqlalchemy/engine/default.py +++ b/libs/sqlalchemy/engine/default.py @@ -1,5 +1,5 @@ # engine/default.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -12,19 +12,23 @@ as the base class for their own corresponding classes. """ -import re, random -from sqlalchemy.engine import base, reflection -from sqlalchemy.sql import compiler, expression -from sqlalchemy import exc, types as sqltypes, util, pool, processors +import re +import random +from . import reflection, interfaces, result +from ..sql import compiler, expression +from .. import types as sqltypes +from .. import exc, util, pool, processors import codecs import weakref +from .. import event AUTOCOMMIT_REGEXP = re.compile( r'\s*(?:UPDATE|INSERT|CREATE|DELETE|DROP|ALTER)', re.I | re.UNICODE) -class DefaultDialect(base.Dialect): + +class DefaultDialect(interfaces.Dialect): """Default implementation of Dialect""" statement_compiler = compiler.SQLCompiler @@ -33,6 +37,10 @@ class DefaultDialect(base.Dialect): preparer = compiler.IdentifierPreparer supports_alter = True + # the first value we'd get for an autoincrement + # column. + default_sequence_base = 1 + # most DBAPIs happy with this for execute(). # not cx_oracle. execute_sequence_format = tuple @@ -44,27 +52,40 @@ class DefaultDialect(base.Dialect): postfetch_lastrowid = True implicit_returning = False + supports_right_nested_joins = True supports_native_enum = False supports_native_boolean = False + supports_simple_order_by_label = True + + engine_config_types = util.immutabledict([ + ('convert_unicode', util.bool_or_str('force')), + ('pool_timeout', int), + ('echo', util.bool_or_str('debug')), + ('echo_pool', util.bool_or_str('debug')), + ('pool_recycle', int), + ('pool_size', int), + ('max_overflow', int), + ('pool_threadlocal', bool), + ('use_native_unicode', bool), + ]) + # if the NUMERIC type # returns decimal.Decimal. # *not* the FLOAT type however. supports_native_decimal = False - # Py3K - #supports_unicode_statements = True - #supports_unicode_binds = True - #returns_unicode_strings = True - #description_encoding = None - # Py2K - supports_unicode_statements = False - supports_unicode_binds = False - returns_unicode_strings = False - description_encoding = 'use_encoding' - # end Py2K - + if util.py3k: + supports_unicode_statements = True + supports_unicode_binds = True + returns_unicode_strings = True + description_encoding = None + else: + supports_unicode_statements = False + supports_unicode_binds = False + returns_unicode_strings = False + description_encoding = 'use_encoding' name = 'default' @@ -86,6 +107,7 @@ class DefaultDialect(base.Dialect): default_paramstyle = 'named' supports_default_values = False supports_empty_insert = True + supports_multivalues_insert = False server_version_info = None @@ -98,27 +120,20 @@ class DefaultDialect(base.Dialect): reflection_options = () - def __init__(self, convert_unicode=False, assert_unicode=False, + def __init__(self, convert_unicode=False, encoding='utf-8', paramstyle=None, dbapi=None, implicit_returning=None, + supports_right_nested_joins=None, + case_sensitive=True, + supports_native_boolean=None, label_length=None, **kwargs): if not getattr(self, 'ported_sqla_06', True): util.warn( - "The %s dialect is not yet ported to SQLAlchemy 0.6/0.7" % + "The %s dialect is not yet ported to the 0.6 format" % self.name) self.convert_unicode = convert_unicode - if assert_unicode: - util.warn_deprecated( - "assert_unicode is deprecated. " - "SQLAlchemy emits a warning in all cases where it " - "would otherwise like to encode a Python unicode object " - "into a specific encoding but a plain bytestring is " - "received. " - "This does *not* apply to DBAPIs that coerce Unicode " - "natively.") - self.encoding = encoding self.positional = False self._ischema = None @@ -134,6 +149,11 @@ class DefaultDialect(base.Dialect): self.positional = self.paramstyle in ('qmark', 'format', 'numeric') self.identifier_preparer = self.preparer(self) self.type_compiler = self.type_compiler(self) + if supports_right_nested_joins is not None: + self.supports_right_nested_joins = supports_right_nested_joins + if supports_native_boolean is not None: + self.supports_native_boolean = supports_native_boolean + self.case_sensitive = case_sensitive if label_length and label_length > self.max_identifier_length: raise exc.ArgumentError( @@ -143,16 +163,19 @@ class DefaultDialect(base.Dialect): self.label_length = label_length if self.description_encoding == 'use_encoding': - self._description_decoder = processors.to_unicode_processor_factory( + self._description_decoder = \ + processors.to_unicode_processor_factory( encoding ) elif self.description_encoding is not None: - self._description_decoder = processors.to_unicode_processor_factory( + self._description_decoder = \ + processors.to_unicode_processor_factory( self.description_encoding ) self._encoder = codecs.getencoder(self.encoding) self._decoder = processors.to_unicode_processor_factory(self.encoding) + @util.memoized_property def _type_memos(self): return weakref.WeakKeyDictionary() @@ -185,6 +208,10 @@ class DefaultDialect(base.Dialect): self.returns_unicode_strings = self._check_unicode_returns(connection) + if self.description_encoding is not None and \ + self._check_unicode_description(connection): + self._description_decoder = self.description_encoding = None + self.do_rollback(connection.connection) def on_connect(self): @@ -202,14 +229,11 @@ class DefaultDialect(base.Dialect): return None def _check_unicode_returns(self, connection): - # Py2K - if self.supports_unicode_statements: - cast_to = unicode + if util.py2k and not self.supports_unicode_statements: + cast_to = util.binary_type else: - cast_to = str - # end Py2K - # Py3K - #cast_to = str + cast_to = util.text_type + def check_unicode(formatstr, type_): cursor = connection.connection.cursor() try: @@ -217,16 +241,17 @@ class DefaultDialect(base.Dialect): cursor.execute( cast_to( expression.select( - [expression.cast( - expression.literal_column( - "'test %s returns'" % formatstr), type_) + [expression.cast( + expression.literal_column( + "'test %s returns'" % formatstr), + type_) ]).compile(dialect=self) ) ) row = cursor.fetchone() - return isinstance(row[0], unicode) - except self.dbapi.Error, de: + return isinstance(row[0], util.text_type) + except self.dbapi.Error as de: util.warn("Exception attempting to " "detect unicode returns: %r" % de) return False @@ -244,18 +269,41 @@ class DefaultDialect(base.Dialect): else: return unicode_for_varchar + def _check_unicode_description(self, connection): + # all DBAPIs on Py2K return cursor.description as encoded, + # until pypy2.1beta2 with sqlite, so let's just check it - + # it's likely others will start doing this too in Py2k. + + if util.py2k and not self.supports_unicode_statements: + cast_to = util.binary_type + else: + cast_to = util.text_type + + cursor = connection.connection.cursor() + try: + cursor.execute( + cast_to( + expression.select([ + expression.literal_column("'x'").label("some_label") + ]).compile(dialect=self) + ) + ) + return isinstance(cursor.description[0][0], util.text_type) + finally: + cursor.close() + def type_descriptor(self, typeobj): - """Provide a database-specific ``TypeEngine`` object, given + """Provide a database-specific :class:`.TypeEngine` object, given the generic object which comes from the types module. This method looks for a dictionary called ``colspecs`` as a class or instance-level variable, - and passes on to ``types.adapt_type()``. + and passes on to :func:`.types.adapt_type`. """ return sqltypes.adapt_type(typeobj, self.colspecs) - def reflecttable(self, connection, table, include_columns, exclude_columns=None): + def reflecttable(self, connection, table, include_columns, exclude_columns): insp = reflection.Inspector.from_engine(connection) return insp.reflecttable(table, include_columns, exclude_columns) @@ -285,26 +333,35 @@ class DefaultDialect(base.Dialect): opts.update(url.query) return [[], opts] - def do_begin(self, connection): - """Implementations might want to put logic here for turning - autocommit on/off, etc. - """ + def set_engine_execution_options(self, engine, opts): + if 'isolation_level' in opts: + isolation_level = opts['isolation_level'] + @event.listens_for(engine, "engine_connect") + def set_isolation(connection, branch): + if not branch: + self._set_connection_isolation(connection, isolation_level) + def set_connection_execution_options(self, connection, opts): + if 'isolation_level' in opts: + self._set_connection_isolation(connection, opts['isolation_level']) + + def _set_connection_isolation(self, connection, level): + self.set_isolation_level(connection.connection, level) + connection.connection._connection_record.\ + finalize_callback.append(self.reset_isolation_level) + + + def do_begin(self, dbapi_connection): pass - def do_rollback(self, connection): - """Implementations might want to put logic here for turning - autocommit on/off, etc. - """ + def do_rollback(self, dbapi_connection): + dbapi_connection.rollback() - connection.rollback() + def do_commit(self, dbapi_connection): + dbapi_connection.commit() - def do_commit(self, connection): - """Implementations might want to put logic here for turning - autocommit on/off, etc. - """ - - connection.commit() + def do_close(self, dbapi_connection): + dbapi_connection.close() def create_xid(self): """Create a random two-phase transaction ID. @@ -342,7 +399,8 @@ class DefaultDialect(base.Dialect): # the configured default of this dialect. self.set_isolation_level(dbapi_conn, self.default_isolation_level) -class DefaultExecutionContext(base.ExecutionContext): + +class DefaultExecutionContext(interfaces.ExecutionContext): isinsert = False isupdate = False isdelete = False @@ -353,6 +411,7 @@ class DefaultExecutionContext(base.ExecutionContext): statement = None postfetch_cols = None prefetch_cols = None + returning_cols = None _is_implicit_returning = False _is_explicit_returning = False @@ -379,10 +438,10 @@ class DefaultExecutionContext(base.ExecutionContext): self.execution_options.update(connection._execution_options) if not dialect.supports_unicode_statements: - self.unicode_statement = unicode(compiled) + self.unicode_statement = util.text_type(compiled) self.statement = dialect._encoder(self.unicode_statement)[0] else: - self.statement = self.unicode_statement = unicode(compiled) + self.statement = self.unicode_statement = util.text_type(compiled) self.cursor = self.create_cursor() self.compiled_parameters = [] @@ -395,7 +454,8 @@ class DefaultExecutionContext(base.ExecutionContext): return self @classmethod - def _init_compiled(cls, dialect, connection, dbapi_connection, compiled, parameters): + def _init_compiled(cls, dialect, connection, dbapi_connection, + compiled, parameters): """Initialize execution context for a Compiled construct.""" self = cls.__new__(cls) @@ -419,9 +479,10 @@ class DefaultExecutionContext(base.ExecutionContext): self.result_map = compiled.result_map - self.unicode_statement = unicode(compiled) + self.unicode_statement = util.text_type(compiled) if not dialect.supports_unicode_statements: - self.statement = self.unicode_statement.encode(self.dialect.encoding) + self.statement = self.unicode_statement.encode( + self.dialect.encoding) else: self.statement = self.unicode_statement @@ -430,16 +491,16 @@ class DefaultExecutionContext(base.ExecutionContext): self.isdelete = compiled.isdelete if self.isinsert or self.isupdate or self.isdelete: - self._is_explicit_returning = compiled.statement._returning - self._is_implicit_returning = compiled.returning and \ - not compiled.statement._returning + self._is_explicit_returning = bool(compiled.statement._returning) + self._is_implicit_returning = bool(compiled.returning and \ + not compiled.statement._returning) if not parameters: self.compiled_parameters = [compiled.construct_params()] else: self.compiled_parameters = \ [compiled.construct_params(m, _group_number=grp) for - grp,m in enumerate(parameters)] + grp, m in enumerate(parameters)] self.executemany = len(parameters) > 1 @@ -447,6 +508,7 @@ class DefaultExecutionContext(base.ExecutionContext): if self.isinsert or self.isupdate: self.postfetch_cols = self.compiled.postfetch self.prefetch_cols = self.compiled.prefetch + self.returning_cols = self.compiled.returning self.__process_defaults() processors = compiled._bind_processors @@ -474,7 +536,8 @@ class DefaultExecutionContext(base.ExecutionContext): param[dialect._encoder(key)[0]] = \ processors[key](compiled_params[key]) else: - param[dialect._encoder(key)[0]] = compiled_params[key] + param[dialect._encoder(key)[0]] = \ + compiled_params[key] else: for key in compiled_params: if key in processors: @@ -487,7 +550,8 @@ class DefaultExecutionContext(base.ExecutionContext): return self @classmethod - def _init_statement(cls, dialect, connection, dbapi_connection, statement, parameters): + def _init_statement(cls, dialect, connection, dbapi_connection, + statement, parameters): """Initialize execution context for a string SQL statement.""" self = cls.__new__(cls) @@ -510,7 +574,7 @@ class DefaultExecutionContext(base.ExecutionContext): if dialect.supports_unicode_statements: self.parameters = parameters else: - self.parameters= [ + self.parameters = [ dict((dialect._encoder(k)[0], d[k]) for k in d) for d in parameters ] or [{}] @@ -520,7 +584,8 @@ class DefaultExecutionContext(base.ExecutionContext): self.executemany = len(parameters) > 1 - if not dialect.supports_unicode_statements and isinstance(statement, unicode): + if not dialect.supports_unicode_statements and \ + isinstance(statement, util.text_type): self.unicode_statement = statement self.statement = dialect._encoder(statement)[0] else: @@ -574,8 +639,8 @@ class DefaultExecutionContext(base.ExecutionContext): """ conn = self.root_connection - if isinstance(stmt, unicode) and \ - not self.dialect.supports_unicode_statements: + if isinstance(stmt, util.text_type) and \ + not self.dialect.supports_unicode_statements: stmt = self.dialect._encoder(stmt)[0] if self.dialect.positional: @@ -583,7 +648,7 @@ class DefaultExecutionContext(base.ExecutionContext): else: default_params = {} - conn._cursor_execute(self.cursor, stmt, default_params) + conn._cursor_execute(self.cursor, stmt, default_params, context=self) r = self.cursor.fetchone()[0] if type_ is not None: # apply type post processors to the result @@ -611,6 +676,16 @@ class DefaultExecutionContext(base.ExecutionContext): def post_exec(self): pass + def get_result_processor(self, type_, colname, coltype): + """Return a 'result processor' for a given type as present in + cursor.description. + + This has a default implementation that dialects can override + for context-sensitive result type handling. + + """ + return type_._cached_result_processor(self.dialect, coltype) + def get_lastrowid(self): """return self.cursor.lastrowid, or equivalent, after an INSERT. @@ -643,7 +718,7 @@ class DefaultExecutionContext(base.ExecutionContext): pass def get_result_proxy(self): - return base.ResultProxy(self) + return result.ResultProxy(self) @property def rowcount(self): @@ -657,22 +732,24 @@ class DefaultExecutionContext(base.ExecutionContext): def post_insert(self): if not self._is_implicit_returning and \ + not self._is_explicit_returning and \ + not self.compiled.inline and \ self.dialect.postfetch_lastrowid and \ (not self.inserted_primary_key or \ None in self.inserted_primary_key): table = self.compiled.statement.table lastrowid = self.get_lastrowid() - autoinc_col = table._autoincrement_column if autoinc_col is not None: # apply type post processors to the lastrowid - proc = autoinc_col.type._cached_result_processor(self.dialect, None) + proc = autoinc_col.type._cached_result_processor( + self.dialect, None) if proc is not None: lastrowid = proc(lastrowid) self.inserted_primary_key = [ - c is autoinc_col and lastrowid or v + lastrowid if c is autoinc_col else v for c, v in zip( table.primary_key, self.inserted_primary_key) @@ -690,6 +767,11 @@ class DefaultExecutionContext(base.ExecutionContext): ipk.append(row[c]) self.inserted_primary_key = ipk + self.returned_defaults = row + + def _fetch_implicit_update_returning(self, resultproxy): + row = resultproxy.fetchone() + self.returned_defaults = row def lastrow_has_defaults(self): return (self.isinsert or self.isupdate) and \ @@ -716,28 +798,34 @@ class DefaultExecutionContext(base.ExecutionContext): inputsizes = [] for key in self.compiled.positiontup: typeengine = types[key] - dbtype = typeengine.dialect_impl(self.dialect).get_dbapi_type(self.dialect.dbapi) - if dbtype is not None and (not exclude_types or dbtype not in exclude_types): + dbtype = typeengine.dialect_impl(self.dialect).\ + get_dbapi_type(self.dialect.dbapi) + if dbtype is not None and \ + (not exclude_types or dbtype not in exclude_types): inputsizes.append(dbtype) try: self.cursor.setinputsizes(*inputsizes) - except Exception, e: - self.root_connection._handle_dbapi_exception(e, None, None, None, self) - raise + except Exception as e: + self.root_connection._handle_dbapi_exception( + e, None, None, None, self) else: inputsizes = {} for key in self.compiled.bind_names.values(): typeengine = types[key] - dbtype = typeengine.dialect_impl(self.dialect).get_dbapi_type(self.dialect.dbapi) - if dbtype is not None and (not exclude_types or dbtype not in exclude_types): + dbtype = typeengine.dialect_impl(self.dialect).\ + get_dbapi_type(self.dialect.dbapi) + if dbtype is not None and \ + (not exclude_types or dbtype not in exclude_types): if translate: key = translate.get(key, key) - inputsizes[self.dialect._encoder(key)[0]] = dbtype + if not self.dialect.supports_unicode_binds: + key = self.dialect._encoder(key)[0] + inputsizes[key] = dbtype try: self.cursor.setinputsizes(**inputsizes) - except Exception, e: - self.root_connection._handle_dbapi_exception(e, None, None, None, self) - raise + except Exception as e: + self.root_connection._handle_dbapi_exception( + e, None, None, None, self) def _exec_default(self, default, type_): if default.is_sequence: diff --git a/libs/sqlalchemy/engine/interfaces.py b/libs/sqlalchemy/engine/interfaces.py new file mode 100644 index 00000000..c26681cc --- /dev/null +++ b/libs/sqlalchemy/engine/interfaces.py @@ -0,0 +1,846 @@ +# engine/interfaces.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Define core interfaces used by the engine system.""" + +from .. import util, event + +# backwards compat +from ..sql.compiler import Compiled, TypeCompiler + +class Dialect(object): + """Define the behavior of a specific database and DB-API combination. + + Any aspect of metadata definition, SQL query generation, + execution, result-set handling, or anything else which varies + between databases is defined under the general category of the + Dialect. The Dialect acts as a factory for other + database-specific object implementations including + ExecutionContext, Compiled, DefaultGenerator, and TypeEngine. + + All Dialects implement the following attributes: + + name + identifying name for the dialect from a DBAPI-neutral point of view + (i.e. 'sqlite') + + driver + identifying name for the dialect's DBAPI + + positional + True if the paramstyle for this Dialect is positional. + + paramstyle + the paramstyle to be used (some DB-APIs support multiple + paramstyles). + + convert_unicode + True if Unicode conversion should be applied to all ``str`` + types. + + encoding + type of encoding to use for unicode, usually defaults to + 'utf-8'. + + statement_compiler + a :class:`.Compiled` class used to compile SQL statements + + ddl_compiler + a :class:`.Compiled` class used to compile DDL statements + + server_version_info + a tuple containing a version number for the DB backend in use. + This value is only available for supporting dialects, and is + typically populated during the initial connection to the database. + + default_schema_name + the name of the default schema. This value is only available for + supporting dialects, and is typically populated during the + initial connection to the database. + + execution_ctx_cls + a :class:`.ExecutionContext` class used to handle statement execution + + execute_sequence_format + either the 'tuple' or 'list' type, depending on what cursor.execute() + accepts for the second argument (they vary). + + preparer + a :class:`~sqlalchemy.sql.compiler.IdentifierPreparer` class used to + quote identifiers. + + supports_alter + ``True`` if the database supports ``ALTER TABLE``. + + max_identifier_length + The maximum length of identifier names. + + supports_unicode_statements + Indicate whether the DB-API can receive SQL statements as Python + unicode strings + + supports_unicode_binds + Indicate whether the DB-API can receive string bind parameters + as Python unicode strings + + supports_sane_rowcount + Indicate whether the dialect properly implements rowcount for + ``UPDATE`` and ``DELETE`` statements. + + supports_sane_multi_rowcount + Indicate whether the dialect properly implements rowcount for + ``UPDATE`` and ``DELETE`` statements when executed via + executemany. + + preexecute_autoincrement_sequences + True if 'implicit' primary key functions must be executed separately + in order to get their value. This is currently oriented towards + Postgresql. + + implicit_returning + use RETURNING or equivalent during INSERT execution in order to load + newly generated primary keys and other column defaults in one execution, + which are then available via inserted_primary_key. + If an insert statement has returning() specified explicitly, + the "implicit" functionality is not used and inserted_primary_key + will not be available. + + dbapi_type_map + A mapping of DB-API type objects present in this Dialect's + DB-API implementation mapped to TypeEngine implementations used + by the dialect. + + This is used to apply types to result sets based on the DB-API + types present in cursor.description; it only takes effect for + result sets against textual statements where no explicit + typemap was present. + + colspecs + A dictionary of TypeEngine classes from sqlalchemy.types mapped + to subclasses that are specific to the dialect class. This + dictionary is class-level only and is not accessed from the + dialect instance itself. + + supports_default_values + Indicates if the construct ``INSERT INTO tablename DEFAULT + VALUES`` is supported + + supports_sequences + Indicates if the dialect supports CREATE SEQUENCE or similar. + + sequences_optional + If True, indicates if the "optional" flag on the Sequence() construct + should signal to not generate a CREATE SEQUENCE. Applies only to + dialects that support sequences. Currently used only to allow Postgresql + SERIAL to be used on a column that specifies Sequence() for usage on + other backends. + + supports_native_enum + Indicates if the dialect supports a native ENUM construct. + This will prevent types.Enum from generating a CHECK + constraint when that type is used. + + supports_native_boolean + Indicates if the dialect supports a native boolean construct. + This will prevent types.Boolean from generating a CHECK + constraint when that type is used. + + """ + + def create_connect_args(self, url): + """Build DB-API compatible connection arguments. + + Given a :class:`~sqlalchemy.engine.url.URL` object, returns a tuple + consisting of a `*args`/`**kwargs` suitable to send directly + to the dbapi's connect function. + + """ + + raise NotImplementedError() + + @classmethod + def type_descriptor(cls, typeobj): + """Transform a generic type to a dialect-specific type. + + Dialect classes will usually use the + :func:`.types.adapt_type` function in the types module to + accomplish this. + + The returned result is cached *per dialect class* so can + contain no dialect-instance state. + + """ + + raise NotImplementedError() + + def initialize(self, connection): + """Called during strategized creation of the dialect with a + connection. + + Allows dialects to configure options based on server version info or + other properties. + + The connection passed here is a SQLAlchemy Connection object, + with full capabilities. + + The initalize() method of the base dialect should be called via + super(). + + """ + + pass + + def reflecttable(self, connection, table, include_columns, exclude_columns): + """Load table description from the database. + + Given a :class:`.Connection` and a + :class:`~sqlalchemy.schema.Table` object, reflect its columns and + properties from the database. + + The implementation of this method is provided by + :meth:`.DefaultDialect.reflecttable`, which makes use of + :class:`.Inspector` to retrieve column information. + + Dialects should **not** seek to implement this method, and should + instead implement individual schema inspection operations such as + :meth:`.Dialect.get_columns`, :meth:`.Dialect.get_pk_constraint`, + etc. + + """ + + raise NotImplementedError() + + def get_columns(self, connection, table_name, schema=None, **kw): + """Return information about columns in `table_name`. + + Given a :class:`.Connection`, a string + `table_name`, and an optional string `schema`, return column + information as a list of dictionaries with these keys: + + name + the column's name + + type + [sqlalchemy.types#TypeEngine] + + nullable + boolean + + default + the column's default value + + autoincrement + boolean + + sequence + a dictionary of the form + {'name' : str, 'start' :int, 'increment': int} + + Additional column attributes may be present. + """ + + raise NotImplementedError() + + def get_primary_keys(self, connection, table_name, schema=None, **kw): + """Return information about primary keys in `table_name`. + + + Deprecated. This method is only called by the default + implementation of :meth:`.Dialect.get_pk_constraint`. Dialects should + instead implement the :meth:`.Dialect.get_pk_constraint` method directly. + + """ + + raise NotImplementedError() + + def get_pk_constraint(self, connection, table_name, schema=None, **kw): + """Return information about the primary key constraint on + table_name`. + + Given a :class:`.Connection`, a string + `table_name`, and an optional string `schema`, return primary + key information as a dictionary with these keys: + + constrained_columns + a list of column names that make up the primary key + + name + optional name of the primary key constraint. + + """ + raise NotImplementedError() + + def get_foreign_keys(self, connection, table_name, schema=None, **kw): + """Return information about foreign_keys in `table_name`. + + Given a :class:`.Connection`, a string + `table_name`, and an optional string `schema`, return foreign + key information as a list of dicts with these keys: + + name + the constraint's name + + constrained_columns + a list of column names that make up the foreign key + + referred_schema + the name of the referred schema + + referred_table + the name of the referred table + + referred_columns + a list of column names in the referred table that correspond to + constrained_columns + """ + + raise NotImplementedError() + + def get_table_names(self, connection, schema=None, **kw): + """Return a list of table names for `schema`.""" + + raise NotImplementedError + + def get_view_names(self, connection, schema=None, **kw): + """Return a list of all view names available in the database. + + schema: + Optional, retrieve names from a non-default schema. + """ + + raise NotImplementedError() + + def get_view_definition(self, connection, view_name, schema=None, **kw): + """Return view definition. + + Given a :class:`.Connection`, a string + `view_name`, and an optional string `schema`, return the view + definition. + """ + + raise NotImplementedError() + + def get_indexes(self, connection, table_name, schema=None, **kw): + """Return information about indexes in `table_name`. + + Given a :class:`.Connection`, a string + `table_name` and an optional string `schema`, return index + information as a list of dictionaries with these keys: + + name + the index's name + + column_names + list of column names in order + + unique + boolean + """ + + raise NotImplementedError() + + def get_unique_constraints(self, table_name, schema=None, **kw): + """Return information about unique constraints in `table_name`. + + Given a string `table_name` and an optional string `schema`, return + unique constraint information as a list of dicts with these keys: + + name + the unique constraint's name + + column_names + list of column names in order + + \**kw + other options passed to the dialect's get_unique_constraints() method. + + .. versionadded:: 0.9.0 + + """ + + raise NotImplementedError() + + def normalize_name(self, name): + """convert the given name to lowercase if it is detected as + case insensitive. + + this method is only used if the dialect defines + requires_name_normalize=True. + + """ + raise NotImplementedError() + + def denormalize_name(self, name): + """convert the given name to a case insensitive identifier + for the backend if it is an all-lowercase name. + + this method is only used if the dialect defines + requires_name_normalize=True. + + """ + raise NotImplementedError() + + def has_table(self, connection, table_name, schema=None): + """Check the existence of a particular table in the database. + + Given a :class:`.Connection` object and a string + `table_name`, return True if the given table (possibly within + the specified `schema`) exists in the database, False + otherwise. + """ + + raise NotImplementedError() + + def has_sequence(self, connection, sequence_name, schema=None): + """Check the existence of a particular sequence in the database. + + Given a :class:`.Connection` object and a string + `sequence_name`, return True if the given sequence exists in + the database, False otherwise. + """ + + raise NotImplementedError() + + def _get_server_version_info(self, connection): + """Retrieve the server version info from the given connection. + + This is used by the default implementation to populate the + "server_version_info" attribute and is called exactly + once upon first connect. + + """ + + raise NotImplementedError() + + def _get_default_schema_name(self, connection): + """Return the string name of the currently selected schema from + the given connection. + + This is used by the default implementation to populate the + "default_schema_name" attribute and is called exactly + once upon first connect. + + """ + + raise NotImplementedError() + + def do_begin(self, dbapi_connection): + """Provide an implementation of ``connection.begin()``, given a + DB-API connection. + + The DBAPI has no dedicated "begin" method and it is expected + that transactions are implicit. This hook is provided for those + DBAPIs that might need additional help in this area. + + Note that :meth:`.Dialect.do_begin` is not called unless a + :class:`.Transaction` object is in use. The + :meth:`.Dialect.do_autocommit` + hook is provided for DBAPIs that need some extra commands emitted + after a commit in order to enter the next transaction, when the + SQLAlchemy :class:`.Connection` is used in it's default "autocommit" + mode. + + :param dbapi_connection: a DBAPI connection, typically + proxied within a :class:`.ConnectionFairy`. + + """ + + raise NotImplementedError() + + def do_rollback(self, dbapi_connection): + """Provide an implementation of ``connection.rollback()``, given + a DB-API connection. + + :param dbapi_connection: a DBAPI connection, typically + proxied within a :class:`.ConnectionFairy`. + + """ + + raise NotImplementedError() + + + def do_commit(self, dbapi_connection): + """Provide an implementation of ``connection.commit()``, given a + DB-API connection. + + :param dbapi_connection: a DBAPI connection, typically + proxied within a :class:`.ConnectionFairy`. + + """ + + raise NotImplementedError() + + def do_close(self, dbapi_connection): + """Provide an implementation of ``connection.close()``, given a DBAPI + connection. + + This hook is called by the :class:`.Pool` when a connection has been + detached from the pool, or is being returned beyond the normal + capacity of the pool. + + .. versionadded:: 0.8 + + """ + + raise NotImplementedError() + + def create_xid(self): + """Create a two-phase transaction ID. + + This id will be passed to do_begin_twophase(), + do_rollback_twophase(), do_commit_twophase(). Its format is + unspecified. + """ + + raise NotImplementedError() + + def do_savepoint(self, connection, name): + """Create a savepoint with the given name. + + :param connection: a :class:`.Connection`. + :param name: savepoint name. + + """ + + raise NotImplementedError() + + def do_rollback_to_savepoint(self, connection, name): + """Rollback a connection to the named savepoint. + + :param connection: a :class:`.Connection`. + :param name: savepoint name. + + """ + + raise NotImplementedError() + + def do_release_savepoint(self, connection, name): + """Release the named savepoint on a connection. + + :param connection: a :class:`.Connection`. + :param name: savepoint name. + """ + + raise NotImplementedError() + + def do_begin_twophase(self, connection, xid): + """Begin a two phase transaction on the given connection. + + :param connection: a :class:`.Connection`. + :param xid: xid + + """ + + raise NotImplementedError() + + def do_prepare_twophase(self, connection, xid): + """Prepare a two phase transaction on the given connection. + + :param connection: a :class:`.Connection`. + :param xid: xid + + """ + + raise NotImplementedError() + + def do_rollback_twophase(self, connection, xid, is_prepared=True, + recover=False): + """Rollback a two phase transaction on the given connection. + + :param connection: a :class:`.Connection`. + :param xid: xid + :param is_prepared: whether or not + :meth:`.TwoPhaseTransaction.prepare` was called. + :param recover: if the recover flag was passed. + + """ + + raise NotImplementedError() + + def do_commit_twophase(self, connection, xid, is_prepared=True, + recover=False): + """Commit a two phase transaction on the given connection. + + + :param connection: a :class:`.Connection`. + :param xid: xid + :param is_prepared: whether or not + :meth:`.TwoPhaseTransaction.prepare` was called. + :param recover: if the recover flag was passed. + + """ + + raise NotImplementedError() + + def do_recover_twophase(self, connection): + """Recover list of uncommited prepared two phase transaction + identifiers on the given connection. + + :param connection: a :class:`.Connection`. + + """ + + raise NotImplementedError() + + def do_executemany(self, cursor, statement, parameters, context=None): + """Provide an implementation of ``cursor.executemany(statement, + parameters)``.""" + + raise NotImplementedError() + + def do_execute(self, cursor, statement, parameters, context=None): + """Provide an implementation of ``cursor.execute(statement, + parameters)``.""" + + raise NotImplementedError() + + def do_execute_no_params(self, cursor, statement, parameters, + context=None): + """Provide an implementation of ``cursor.execute(statement)``. + + The parameter collection should not be sent. + + """ + + raise NotImplementedError() + + def is_disconnect(self, e, connection, cursor): + """Return True if the given DB-API error indicates an invalid + connection""" + + raise NotImplementedError() + + def connect(self): + """return a callable which sets up a newly created DBAPI connection. + + The callable accepts a single argument "conn" which is the + DBAPI connection itself. It has no return value. + + This is used to set dialect-wide per-connection options such as + isolation modes, unicode modes, etc. + + If a callable is returned, it will be assembled into a pool listener + that receives the direct DBAPI connection, with all wrappers removed. + + If None is returned, no listener will be generated. + + """ + return None + + def reset_isolation_level(self, dbapi_conn): + """Given a DBAPI connection, revert its isolation to the default.""" + + raise NotImplementedError() + + def set_isolation_level(self, dbapi_conn, level): + """Given a DBAPI connection, set its isolation level.""" + + raise NotImplementedError() + + def get_isolation_level(self, dbapi_conn): + """Given a DBAPI connection, return its isolation level.""" + + raise NotImplementedError() + + +class ExecutionContext(object): + """A messenger object for a Dialect that corresponds to a single + execution. + + ExecutionContext should have these data members: + + connection + Connection object which can be freely used by default value + generators to execute SQL. This Connection should reference the + same underlying connection/transactional resources of + root_connection. + + root_connection + Connection object which is the source of this ExecutionContext. This + Connection may have close_with_result=True set, in which case it can + only be used once. + + dialect + dialect which created this ExecutionContext. + + cursor + DB-API cursor procured from the connection, + + compiled + if passed to constructor, sqlalchemy.engine.base.Compiled object + being executed, + + statement + string version of the statement to be executed. Is either + passed to the constructor, or must be created from the + sql.Compiled object by the time pre_exec() has completed. + + parameters + bind parameters passed to the execute() method. For compiled + statements, this is a dictionary or list of dictionaries. For + textual statements, it should be in a format suitable for the + dialect's paramstyle (i.e. dict or list of dicts for non + positional, list or list of lists/tuples for positional). + + isinsert + True if the statement is an INSERT. + + isupdate + True if the statement is an UPDATE. + + should_autocommit + True if the statement is a "committable" statement. + + prefetch_cols + a list of Column objects for which a client-side default + was fired off. Applies to inserts and updates. + + postfetch_cols + a list of Column objects for which a server-side default or + inline SQL expression value was fired off. Applies to inserts + and updates. + """ + + def create_cursor(self): + """Return a new cursor generated from this ExecutionContext's + connection. + + Some dialects may wish to change the behavior of + connection.cursor(), such as postgresql which may return a PG + "server side" cursor. + """ + + raise NotImplementedError() + + def pre_exec(self): + """Called before an execution of a compiled statement. + + If a compiled statement was passed to this ExecutionContext, + the `statement` and `parameters` datamembers must be + initialized after this statement is complete. + """ + + raise NotImplementedError() + + def post_exec(self): + """Called after the execution of a compiled statement. + + If a compiled statement was passed to this ExecutionContext, + the `last_insert_ids`, `last_inserted_params`, etc. + datamembers should be available after this method completes. + """ + + raise NotImplementedError() + + def result(self): + """Return a result object corresponding to this ExecutionContext. + + Returns a ResultProxy. + """ + + raise NotImplementedError() + + def handle_dbapi_exception(self, e): + """Receive a DBAPI exception which occurred upon execute, result + fetch, etc.""" + + raise NotImplementedError() + + def should_autocommit_text(self, statement): + """Parse the given textual statement and return True if it refers to + a "committable" statement""" + + raise NotImplementedError() + + def lastrow_has_defaults(self): + """Return True if the last INSERT or UPDATE row contained + inlined or database-side defaults. + """ + + raise NotImplementedError() + + def get_rowcount(self): + """Return the DBAPI ``cursor.rowcount`` value, or in some + cases an interpreted value. + + See :attr:`.ResultProxy.rowcount` for details on this. + + """ + + raise NotImplementedError() + + +class Connectable(object): + """Interface for an object which supports execution of SQL constructs. + + The two implementations of :class:`.Connectable` are + :class:`.Connection` and :class:`.Engine`. + + Connectable must also implement the 'dialect' member which references a + :class:`.Dialect` instance. + + """ + + def connect(self, **kwargs): + """Return a :class:`.Connection` object. + + Depending on context, this may be ``self`` if this object + is already an instance of :class:`.Connection`, or a newly + procured :class:`.Connection` if this object is an instance + of :class:`.Engine`. + + """ + + def contextual_connect(self): + """Return a :class:`.Connection` object which may be part of an ongoing + context. + + Depending on context, this may be ``self`` if this object + is already an instance of :class:`.Connection`, or a newly + procured :class:`.Connection` if this object is an instance + of :class:`.Engine`. + + """ + + raise NotImplementedError() + + @util.deprecated("0.7", + "Use the create() method on the given schema " + "object directly, i.e. :meth:`.Table.create`, " + ":meth:`.Index.create`, :meth:`.MetaData.create_all`") + def create(self, entity, **kwargs): + """Emit CREATE statements for the given schema entity. + """ + + raise NotImplementedError() + + @util.deprecated("0.7", + "Use the drop() method on the given schema " + "object directly, i.e. :meth:`.Table.drop`, " + ":meth:`.Index.drop`, :meth:`.MetaData.drop_all`") + def drop(self, entity, **kwargs): + """Emit DROP statements for the given schema entity. + """ + + raise NotImplementedError() + + def execute(self, object, *multiparams, **params): + """Executes the given construct and returns a :class:`.ResultProxy`.""" + raise NotImplementedError() + + def scalar(self, object, *multiparams, **params): + """Executes and returns the first column of the first row. + + The underlying cursor is closed after execution. + """ + raise NotImplementedError() + + def _run_visitor(self, visitorcallable, element, + **kwargs): + raise NotImplementedError() + + def _execute_clauseelement(self, elem, multiparams=None, params=None): + raise NotImplementedError() diff --git a/libs/sqlalchemy/engine/reflection.py b/libs/sqlalchemy/engine/reflection.py index 6d34a279..badec84e 100644 --- a/libs/sqlalchemy/engine/reflection.py +++ b/libs/sqlalchemy/engine/reflection.py @@ -1,5 +1,5 @@ # engine/reflection.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -24,12 +24,14 @@ methods such as get_table_names, get_columns, etc. 'name' attribute.. """ -import sqlalchemy -from sqlalchemy import exc, sql -from sqlalchemy import util -from sqlalchemy.util import topological -from sqlalchemy.types import TypeEngine -from sqlalchemy import schema as sa_schema +from .. import exc, sql +from ..sql import schema as sa_schema +from .. import util +from ..sql.type_api import TypeEngine +from ..util import deprecated +from ..util import topological +from .. import inspection +from .base import Connectable @util.decorator @@ -39,8 +41,12 @@ def cache(fn, self, con, *args, **kw): return fn(self, con, *args, **kw) key = ( fn.__name__, - tuple(a for a in args if isinstance(a, basestring)), - tuple((k, v) for k, v in kw.iteritems() if isinstance(v, (basestring, int, float))) + tuple(a for a in args if isinstance(a, util.string_types)), + tuple((k, v) for k, v in kw.items() if + isinstance(v, + util.string_types + util.int_types + (float, ) + ) + ) ) ret = info_cache.get(key) if ret is None: @@ -53,17 +59,24 @@ class Inspector(object): """Performs database schema inspection. The Inspector acts as a proxy to the reflection methods of the - :class:`~sqlalchemy.engine.base.Dialect`, providing a + :class:`~sqlalchemy.engine.interfaces.Dialect`, providing a consistent interface as well as caching support for previously fetched metadata. - The preferred method to construct an :class:`.Inspector` is via the - :meth:`Inspector.from_engine` method. I.e.:: + A :class:`.Inspector` object is usually created via the + :func:`.inspect` function:: + + from sqlalchemy import inspect, create_engine + engine = create_engine('...') + insp = inspect(engine) + + The inspection method above is equivalent to using the + :meth:`.Inspector.from_engine` method, i.e.:: engine = create_engine('...') insp = Inspector.from_engine(engine) - Where above, the :class:`~sqlalchemy.engine.base.Dialect` may opt + Where above, the :class:`~sqlalchemy.engine.interfaces.Dialect` may opt to return an :class:`.Inspector` subclass that provides additional methods specific to the dialect's target database. @@ -72,13 +85,13 @@ class Inspector(object): def __init__(self, bind): """Initialize a new :class:`.Inspector`. - :param bind: a :class:`~sqlalchemy.engine.base.Connectable`, + :param bind: a :class:`~sqlalchemy.engine.Connectable`, which is typically an instance of - :class:`~sqlalchemy.engine.base.Engine` or - :class:`~sqlalchemy.engine.base.Connection`. + :class:`~sqlalchemy.engine.Engine` or + :class:`~sqlalchemy.engine.Connection`. For a dialect-specific instance of :class:`.Inspector`, see - :meth:`Inspector.from_engine` + :meth:`.Inspector.from_engine` """ # this might not be a connection, it could be an engine. @@ -99,17 +112,19 @@ class Inspector(object): @classmethod def from_engine(cls, bind): - """Construct a new dialect-specific Inspector object from the given engine or connection. + """Construct a new dialect-specific Inspector object from the given + engine or connection. - :param bind: a :class:`~sqlalchemy.engine.base.Connectable`, + :param bind: a :class:`~sqlalchemy.engine.Connectable`, which is typically an instance of - :class:`~sqlalchemy.engine.base.Engine` or - :class:`~sqlalchemy.engine.base.Connection`. + :class:`~sqlalchemy.engine.Engine` or + :class:`~sqlalchemy.engine.Connection`. - This method differs from direct a direct constructor call of :class:`.Inspector` - in that the :class:`~sqlalchemy.engine.base.Dialect` is given a chance to provide - a dialect-specific :class:`.Inspector` instance, which may provide additional - methods. + This method differs from direct a direct constructor call of + :class:`.Inspector` in that the + :class:`~sqlalchemy.engine.interfaces.Dialect` is given a chance to + provide a dialect-specific :class:`.Inspector` instance, which may + provide additional methods. See the example at :class:`.Inspector`. @@ -118,6 +133,10 @@ class Inspector(object): return bind.dialect.inspector(bind) return Inspector(bind) + @inspection._inspects(Connectable) + def _insp(bind): + return Inspector.from_engine(bind) + @property def default_schema_name(self): """Return the default schema name presented by the dialect @@ -139,14 +158,32 @@ class Inspector(object): return [] def get_table_names(self, schema=None, order_by=None): - """Return all table names in `schema`. + """Return all table names in referred to within a particular schema. + + The names are expected to be real tables only, not views. + Views are instead returned using the :meth:`.Inspector.get_view_names` + method. + + + :param schema: Schema name. If ``schema`` is left at ``None``, the + database's default schema is + used, else the named schema is searched. If the database does not + support named schemas, behavior is undefined if ``schema`` is not + passed as ``None``. For special quoting, use :class:`.quoted_name`. - :param schema: Optional, retrieve names from a non-default schema. :param order_by: Optional, may be the string "foreign_key" to sort - the result on foreign key dependencies. + the result on foreign key dependencies. + + .. versionchanged:: 0.8 the "foreign_key" sorting sorts tables + in order of dependee to dependent; that is, in creation + order, rather than in drop order. This is to maintain + consistency with similar features such as + :attr:`.MetaData.sorted_tables` and :func:`.util.sort_tables`. + + .. seealso:: + + :attr:`.MetaData.sorted_tables` - This should probably not return view names or maybe it should return - them with an indicator t or v. """ if hasattr(self.dialect, 'get_table_names'): @@ -155,33 +192,40 @@ class Inspector(object): else: tnames = self.engine.table_names(schema) if order_by == 'foreign_key': - import random - random.shuffle(tnames) - tuples = [] for tname in tnames: for fkey in self.get_foreign_keys(tname, schema): if tname != fkey['referred_table']: - tuples.append((tname, fkey['referred_table'])) + tuples.append((fkey['referred_table'], tname)) tnames = list(topological.sort(tuples, tnames)) return tnames def get_table_options(self, table_name, schema=None, **kw): - """Return a dictionary of options specified when the table of the given name was created. + """Return a dictionary of options specified when the table of the + given name was created. This currently includes some options that apply to MySQL tables. + :param table_name: string name of the table. For special quoting, + use :class:`.quoted_name`. + + :param schema: string schema name; if omitted, uses the default schema + of the database connection. For special quoting, + use :class:`.quoted_name`. + """ if hasattr(self.dialect, 'get_table_options'): - return self.dialect.get_table_options(self.bind, table_name, schema, - info_cache=self.info_cache, - **kw) + return self.dialect.get_table_options( + self.bind, table_name, schema, + info_cache=self.info_cache, **kw) return {} def get_view_names(self, schema=None): """Return all view names in `schema`. :param schema: Optional, retrieve names from a non-default schema. + For special quoting, use :class:`.quoted_name`. + """ return self.dialect.get_view_names(self.bind, schema, @@ -191,6 +235,8 @@ class Inspector(object): """Return definition for `view_name`. :param schema: Optional, retrieve names from a non-default schema. + For special quoting, use :class:`.quoted_name`. + """ return self.dialect.get_view_definition( @@ -216,6 +262,14 @@ class Inspector(object): attrs dict containing optional column attributes + + :param table_name: string name of the table. For special quoting, + use :class:`.quoted_name`. + + :param schema: string schema name; if omitted, uses the default schema + of the database connection. For special quoting, + use :class:`.quoted_name`. + """ col_defs = self.dialect.get_columns(self.bind, table_name, schema, @@ -228,6 +282,8 @@ class Inspector(object): col_def['type'] = coltype() return col_defs + @deprecated('0.7', 'Call to deprecated method get_primary_keys.' + ' Use get_pk_constraint instead.') def get_primary_keys(self, table_name, schema=None, **kw): """Return information about primary keys in `table_name`. @@ -235,11 +291,9 @@ class Inspector(object): primary key information as a list of column names. """ - pkeys = self.dialect.get_primary_keys(self.bind, table_name, schema, - info_cache=self.info_cache, - **kw) - - return pkeys + return self.dialect.get_pk_constraint(self.bind, table_name, schema, + info_cache=self.info_cache, + **kw)['constrained_columns'] def get_pk_constraint(self, table_name, schema=None, **kw): """Return information about primary key constraint on `table_name`. @@ -253,14 +307,18 @@ class Inspector(object): name optional name of the primary key constraint. + :param table_name: string name of the table. For special quoting, + use :class:`.quoted_name`. + + :param schema: string schema name; if omitted, uses the default schema + of the database connection. For special quoting, + use :class:`.quoted_name`. + """ - pkeys = self.dialect.get_pk_constraint(self.bind, table_name, schema, + return self.dialect.get_pk_constraint(self.bind, table_name, schema, info_cache=self.info_cache, **kw) - return pkeys - - def get_foreign_keys(self, table_name, schema=None, **kw): """Return information about foreign_keys in `table_name`. @@ -283,15 +341,18 @@ class Inspector(object): name optional name of the foreign key constraint. - \**kw - other options passed to the dialect's get_foreign_keys() method. + :param table_name: string name of the table. For special quoting, + use :class:`.quoted_name`. + + :param schema: string schema name; if omitted, uses the default schema + of the database connection. For special quoting, + use :class:`.quoted_name`. """ - fk_defs = self.dialect.get_foreign_keys(self.bind, table_name, schema, + return self.dialect.get_foreign_keys(self.bind, table_name, schema, info_cache=self.info_cache, **kw) - return fk_defs def get_indexes(self, table_name, schema=None, **kw): """Return information about indexes in `table_name`. @@ -308,17 +369,48 @@ class Inspector(object): unique boolean - \**kw - other options passed to the dialect's get_indexes() method. + :param table_name: string name of the table. For special quoting, + use :class:`.quoted_name`. + + :param schema: string schema name; if omitted, uses the default schema + of the database connection. For special quoting, + use :class:`.quoted_name`. + """ - indexes = self.dialect.get_indexes(self.bind, table_name, + return self.dialect.get_indexes(self.bind, table_name, schema, info_cache=self.info_cache, **kw) - return indexes + + def get_unique_constraints(self, table_name, schema=None, **kw): + """Return information about unique constraints in `table_name`. + + Given a string `table_name` and an optional string `schema`, return + unique constraint information as a list of dicts with these keys: + + name + the unique constraint's name + + column_names + list of column names in order + + :param table_name: string name of the table. For special quoting, + use :class:`.quoted_name`. + + :param schema: string schema name; if omitted, uses the default schema + of the database connection. For special quoting, + use :class:`.quoted_name`. + + .. versionadded:: 0.8.4 + + """ + + return self.dialect.get_unique_constraints( + self.bind, table_name, schema, info_cache=self.info_cache, **kw) def reflecttable(self, table, include_columns, exclude_columns=()): - """Given a Table object, load its internal constructs based on introspection. + """Given a Table object, load its internal constructs based on + introspection. This is the underlying method used by most dialects to produce table reflection. Direct usage is like:: @@ -341,7 +433,8 @@ class Inspector(object): # table attributes we might need. reflection_options = dict( - (k, table.kwargs.get(k)) for k in dialect.reflection_options if k in table.kwargs) + (k, table.kwargs.get(k)) + for k in dialect.reflection_options if k in table.kwargs) schema = table.schema table_name = table.name @@ -354,22 +447,25 @@ class Inspector(object): # table.kwargs will need to be passed to each reflection method. Make # sure keywords are strings. tblkw = table.kwargs.copy() - for (k, v) in tblkw.items(): + for (k, v) in list(tblkw.items()): del tblkw[k] tblkw[str(k)] = v - # Py2K - if isinstance(schema, str): - schema = schema.decode(dialect.encoding) - if isinstance(table_name, str): - table_name = table_name.decode(dialect.encoding) - # end Py2K + if util.py2k: + if isinstance(schema, str): + schema = schema.decode(dialect.encoding) + if isinstance(table_name, str): + table_name = table_name.decode(dialect.encoding) # columns found_table = False + cols_by_orig_name = {} + for col_d in self.get_columns(table_name, schema, **tblkw): found_table = True - table.dispatch.column_reflect(table, col_d) + orig_name = col_d['name'] + + table.dispatch.column_reflect(self, table, col_d) name = col_d['name'] if include_columns and name not in include_columns: @@ -387,8 +483,9 @@ class Inspector(object): colargs = [] if col_d.get('default') is not None: - # the "default" value is assumed to be a literal SQL expression, - # so is wrapped in text() so that no quoting occurs on re-issuance. + # the "default" value is assumed to be a literal SQL + # expression, so is wrapped in text() so that no quoting + # occurs on re-issuance. colargs.append( sa_schema.DefaultClause( sql.text(col_d['default']), _reflected=True @@ -396,7 +493,7 @@ class Inspector(object): ) if 'sequence' in col_d: - # TODO: mssql, maxdb and sybase are using this. + # TODO: mssql and sybase are using this. seq = col_d['sequence'] sequence = sa_schema.Sequence(seq['name'], 1, 1) if 'start' in seq: @@ -405,7 +502,9 @@ class Inspector(object): sequence.increment = seq['increment'] colargs.append(sequence) - col = sa_schema.Column(name, coltype, *colargs, **col_kw) + cols_by_orig_name[orig_name] = col = \ + sa_schema.Column(name, coltype, *colargs, **col_kw) + table.append_column(col) if not found_table: @@ -414,11 +513,18 @@ class Inspector(object): # Primary keys pk_cons = self.get_pk_constraint(table_name, schema, **tblkw) if pk_cons: - pk_cols = [table.c[pk] - for pk in pk_cons['constrained_columns'] - if pk in table.c and pk not in exclude_columns - ] + [pk for pk in table.primary_key if pk.key in exclude_columns] - primary_key_constraint = sa_schema.PrimaryKeyConstraint(name=pk_cons.get('name'), + pk_cols = [ + cols_by_orig_name[pk] + for pk in pk_cons['constrained_columns'] + if pk in cols_by_orig_name and pk not in exclude_columns + ] + pk_cols += [ + pk + for pk in table.primary_key + if pk.key in exclude_columns + ] + primary_key_constraint = sa_schema.PrimaryKeyConstraint( + name=pk_cons.get('name'), *pk_cols ) @@ -428,7 +534,16 @@ class Inspector(object): fkeys = self.get_foreign_keys(table_name, schema, **tblkw) for fkey_d in fkeys: conname = fkey_d['name'] - constrained_columns = fkey_d['constrained_columns'] + # look for columns by orig name in cols_by_orig_name, + # but support columns that are in-Python only as fallback + constrained_columns = [ + cols_by_orig_name[c].key + if c in cols_by_orig_name else c + for c in fkey_d['constrained_columns'] + ] + if exclude_columns and set(constrained_columns).intersection( + exclude_columns): + continue referred_schema = fkey_d['referred_schema'] referred_table = fkey_d['referred_table'] referred_columns = fkey_d['referred_columns'] @@ -449,9 +564,14 @@ class Inspector(object): ) for column in referred_columns: refspec.append(".".join([referred_table, column])) + if 'options' in fkey_d: + options = fkey_d['options'] + else: + options = {} table.append_constraint( sa_schema.ForeignKeyConstraint(constrained_columns, refspec, - conname, link_to_name=True)) + conname, link_to_name=True, + **options)) # Indexes indexes = self.get_indexes(table_name, schema) for index_d in indexes: @@ -465,5 +585,11 @@ class Inspector(object): "Omitting %s KEY for (%s), key covers omitted columns." % (flavor, ', '.join(columns))) continue - sa_schema.Index(name, *[table.columns[c] for c in columns], + # look for columns by orig name in cols_by_orig_name, + # but support columns that are in-Python only as fallback + sa_schema.Index(name, *[ + cols_by_orig_name[c] if c in cols_by_orig_name + else table.c[c] + for c in columns + ], **dict(unique=unique)) diff --git a/libs/sqlalchemy/engine/result.py b/libs/sqlalchemy/engine/result.py new file mode 100644 index 00000000..f9e0ca0d --- /dev/null +++ b/libs/sqlalchemy/engine/result.py @@ -0,0 +1,1018 @@ +# engine/result.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Define result set constructs including :class:`.ResultProxy` +and :class:`.RowProxy.""" + + + +from .. import exc, util +from ..sql import expression, sqltypes +import collections + +# This reconstructor is necessary so that pickles with the C extension or +# without use the same Binary format. +try: + # We need a different reconstructor on the C extension so that we can + # add extra checks that fields have correctly been initialized by + # __setstate__. + from sqlalchemy.cresultproxy import safe_rowproxy_reconstructor + + # The extra function embedding is needed so that the + # reconstructor function has the same signature whether or not + # the extension is present. + def rowproxy_reconstructor(cls, state): + return safe_rowproxy_reconstructor(cls, state) +except ImportError: + def rowproxy_reconstructor(cls, state): + obj = cls.__new__(cls) + obj.__setstate__(state) + return obj + +try: + from sqlalchemy.cresultproxy import BaseRowProxy +except ImportError: + class BaseRowProxy(object): + __slots__ = ('_parent', '_row', '_processors', '_keymap') + + def __init__(self, parent, row, processors, keymap): + """RowProxy objects are constructed by ResultProxy objects.""" + + self._parent = parent + self._row = row + self._processors = processors + self._keymap = keymap + + def __reduce__(self): + return (rowproxy_reconstructor, + (self.__class__, self.__getstate__())) + + def values(self): + """Return the values represented by this RowProxy as a list.""" + return list(self) + + def __iter__(self): + for processor, value in zip(self._processors, self._row): + if processor is None: + yield value + else: + yield processor(value) + + def __len__(self): + return len(self._row) + + def __getitem__(self, key): + try: + processor, obj, index = self._keymap[key] + except KeyError: + processor, obj, index = self._parent._key_fallback(key) + except TypeError: + if isinstance(key, slice): + l = [] + for processor, value in zip(self._processors[key], + self._row[key]): + if processor is None: + l.append(value) + else: + l.append(processor(value)) + return tuple(l) + else: + raise + if index is None: + raise exc.InvalidRequestError( + "Ambiguous column name '%s' in result set! " + "try 'use_labels' option on select statement." % key) + if processor is not None: + return processor(self._row[index]) + else: + return self._row[index] + + def __getattr__(self, name): + try: + return self[name] + except KeyError as e: + raise AttributeError(e.args[0]) + + +class RowProxy(BaseRowProxy): + """Proxy values from a single cursor row. + + Mostly follows "ordered dictionary" behavior, mapping result + values to the string-based column name, the integer position of + the result in the row, as well as Column instances which can be + mapped to the original Columns that produced this result set (for + results that correspond to constructed SQL expressions). + """ + __slots__ = () + + def __contains__(self, key): + return self._parent._has_key(self._row, key) + + def __getstate__(self): + return { + '_parent': self._parent, + '_row': tuple(self) + } + + def __setstate__(self, state): + self._parent = parent = state['_parent'] + self._row = state['_row'] + self._processors = parent._processors + self._keymap = parent._keymap + + __hash__ = None + + def __lt__(self, other): + return tuple(self) < tuple(other) + + def __eq__(self, other): + return other is self or tuple(other) == tuple(self) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return repr(tuple(self)) + + def has_key(self, key): + """Return True if this RowProxy contains the given key.""" + + return self._parent._has_key(self._row, key) + + def items(self): + """Return a list of tuples, each tuple containing a key/value pair.""" + # TODO: no coverage here + return [(key, self[key]) for key in self.keys()] + + def keys(self): + """Return the list of keys as strings represented by this RowProxy.""" + + return self._parent.keys + + def iterkeys(self): + return iter(self._parent.keys) + + def itervalues(self): + return iter(self) + +try: + # Register RowProxy with Sequence, + # so sequence protocol is implemented + from collections import Sequence + Sequence.register(RowProxy) +except ImportError: + pass + + +class ResultMetaData(object): + """Handle cursor.description, applying additional info from an execution + context.""" + + def __init__(self, parent, metadata): + self._processors = processors = [] + + # We do not strictly need to store the processor in the key mapping, + # though it is faster in the Python version (probably because of the + # saved attribute lookup self._processors) + self._keymap = keymap = {} + self.keys = [] + context = parent.context + dialect = context.dialect + typemap = dialect.dbapi_type_map + translate_colname = context._translate_colname + self.case_sensitive = dialect.case_sensitive + + # high precedence key values. + primary_keymap = {} + + for i, rec in enumerate(metadata): + colname = rec[0] + coltype = rec[1] + + if dialect.description_encoding: + colname = dialect._description_decoder(colname) + + if translate_colname: + colname, untranslated = translate_colname(colname) + + if dialect.requires_name_normalize: + colname = dialect.normalize_name(colname) + + if context.result_map: + try: + name, obj, type_ = context.result_map[colname + if self.case_sensitive + else colname.lower()] + except KeyError: + name, obj, type_ = \ + colname, None, typemap.get(coltype, sqltypes.NULLTYPE) + else: + name, obj, type_ = \ + colname, None, typemap.get(coltype, sqltypes.NULLTYPE) + + processor = context.get_result_processor(type_, colname, coltype) + + processors.append(processor) + rec = (processor, obj, i) + + # indexes as keys. This is only needed for the Python version of + # RowProxy (the C version uses a faster path for integer indexes). + primary_keymap[i] = rec + + # populate primary keymap, looking for conflicts. + if primary_keymap.setdefault( + name if self.case_sensitive + else name.lower(), + rec) is not rec: + # place a record that doesn't have the "index" - this + # is interpreted later as an AmbiguousColumnError, + # but only when actually accessed. Columns + # colliding by name is not a problem if those names + # aren't used; integer access is always + # unambiguous. + primary_keymap[name + if self.case_sensitive + else name.lower()] = rec = (None, obj, None) + + self.keys.append(colname) + if obj: + for o in obj: + keymap[o] = rec + # technically we should be doing this but we + # are saving on callcounts by not doing so. + # if keymap.setdefault(o, rec) is not rec: + # keymap[o] = (None, obj, None) + + if translate_colname and \ + untranslated: + keymap[untranslated] = rec + + # overwrite keymap values with those of the + # high precedence keymap. + keymap.update(primary_keymap) + + if parent._echo: + context.engine.logger.debug( + "Col %r", tuple(x[0] for x in metadata)) + + @util.pending_deprecation("0.8", "sqlite dialect uses " + "_translate_colname() now") + def _set_keymap_synonym(self, name, origname): + """Set a synonym for the given name. + + Some dialects (SQLite at the moment) may use this to + adjust the column names that are significant within a + row. + + """ + rec = (processor, obj, i) = self._keymap[origname if + self.case_sensitive + else origname.lower()] + if self._keymap.setdefault(name, rec) is not rec: + self._keymap[name] = (processor, obj, None) + + def _key_fallback(self, key, raiseerr=True): + map = self._keymap + result = None + if isinstance(key, util.string_types): + result = map.get(key if self.case_sensitive else key.lower()) + # fallback for targeting a ColumnElement to a textual expression + # this is a rare use case which only occurs when matching text() + # or colummn('name') constructs to ColumnElements, or after a + # pickle/unpickle roundtrip + elif isinstance(key, expression.ColumnElement): + if key._label and ( + key._label + if self.case_sensitive + else key._label.lower()) in map: + result = map[key._label + if self.case_sensitive + else key._label.lower()] + elif hasattr(key, 'name') and ( + key.name + if self.case_sensitive + else key.name.lower()) in map: + # match is only on name. + result = map[key.name + if self.case_sensitive + else key.name.lower()] + # search extra hard to make sure this + # isn't a column/label name overlap. + # this check isn't currently available if the row + # was unpickled. + if result is not None and \ + result[1] is not None: + for obj in result[1]: + if key._compare_name_for_result(obj): + break + else: + result = None + if result is None: + if raiseerr: + raise exc.NoSuchColumnError( + "Could not locate column in row for column '%s'" % + expression._string_or_unprintable(key)) + else: + return None + else: + map[key] = result + return result + + def _has_key(self, row, key): + if key in self._keymap: + return True + else: + return self._key_fallback(key, False) is not None + + def __getstate__(self): + return { + '_pickled_keymap': dict( + (key, index) + for key, (processor, obj, index) in self._keymap.items() + if isinstance(key, util.string_types + util.int_types) + ), + 'keys': self.keys, + "case_sensitive": self.case_sensitive, + } + + def __setstate__(self, state): + # the row has been processed at pickling time so we don't need any + # processor anymore + self._processors = [None for _ in range(len(state['keys']))] + self._keymap = keymap = {} + for key, index in state['_pickled_keymap'].items(): + # not preserving "obj" here, unfortunately our + # proxy comparison fails with the unpickle + keymap[key] = (None, None, index) + self.keys = state['keys'] + self.case_sensitive = state['case_sensitive'] + self._echo = False + + +class ResultProxy(object): + """Wraps a DB-API cursor object to provide easier access to row columns. + + Individual columns may be accessed by their integer position, + case-insensitive column name, or by ``schema.Column`` + object. e.g.:: + + row = fetchone() + + col1 = row[0] # access via integer position + + col2 = row['col2'] # access via name + + col3 = row[mytable.c.mycol] # access via Column object. + + ``ResultProxy`` also handles post-processing of result column + data using ``TypeEngine`` objects, which are referenced from + the originating SQL statement that produced this result set. + + """ + + _process_row = RowProxy + out_parameters = None + _can_close_connection = False + _metadata = None + + def __init__(self, context): + self.context = context + self.dialect = context.dialect + self.closed = False + self.cursor = self._saved_cursor = context.cursor + self.connection = context.root_connection + self._echo = self.connection._echo and \ + context.engine._should_log_debug() + self._init_metadata() + + def _init_metadata(self): + metadata = self._cursor_description() + if metadata is not None: + self._metadata = ResultMetaData(self, metadata) + + def keys(self): + """Return the current set of string keys for rows.""" + if self._metadata: + return self._metadata.keys + else: + return [] + + @util.memoized_property + def rowcount(self): + """Return the 'rowcount' for this result. + + The 'rowcount' reports the number of rows *matched* + by the WHERE criterion of an UPDATE or DELETE statement. + + .. note:: + + Notes regarding :attr:`.ResultProxy.rowcount`: + + + * This attribute returns the number of rows *matched*, + which is not necessarily the same as the number of rows + that were actually *modified* - an UPDATE statement, for example, + may have no net change on a given row if the SET values + given are the same as those present in the row already. + Such a row would be matched but not modified. + On backends that feature both styles, such as MySQL, + rowcount is configured by default to return the match + count in all cases. + + * :attr:`.ResultProxy.rowcount` is *only* useful in conjunction + with an UPDATE or DELETE statement. Contrary to what the Python + DBAPI says, it does *not* return the + number of rows available from the results of a SELECT statement + as DBAPIs cannot support this functionality when rows are + unbuffered. + + * :attr:`.ResultProxy.rowcount` may not be fully implemented by + all dialects. In particular, most DBAPIs do not support an + aggregate rowcount result from an executemany call. + The :meth:`.ResultProxy.supports_sane_rowcount` and + :meth:`.ResultProxy.supports_sane_multi_rowcount` methods + will report from the dialect if each usage is known to be + supported. + + * Statements that use RETURNING may not return a correct + rowcount. + + """ + try: + return self.context.rowcount + except Exception as e: + self.connection._handle_dbapi_exception( + e, None, None, self.cursor, self.context) + + @property + def lastrowid(self): + """return the 'lastrowid' accessor on the DBAPI cursor. + + This is a DBAPI specific method and is only functional + for those backends which support it, for statements + where it is appropriate. It's behavior is not + consistent across backends. + + Usage of this method is normally unnecessary when + using insert() expression constructs; the + :attr:`~ResultProxy.inserted_primary_key` attribute provides a + tuple of primary key values for a newly inserted row, + regardless of database backend. + + """ + try: + return self._saved_cursor.lastrowid + except Exception as e: + self.connection._handle_dbapi_exception( + e, None, None, + self._saved_cursor, self.context) + + @property + def returns_rows(self): + """True if this :class:`.ResultProxy` returns rows. + + I.e. if it is legal to call the methods + :meth:`~.ResultProxy.fetchone`, + :meth:`~.ResultProxy.fetchmany` + :meth:`~.ResultProxy.fetchall`. + + """ + return self._metadata is not None + + @property + def is_insert(self): + """True if this :class:`.ResultProxy` is the result + of a executing an expression language compiled + :func:`.expression.insert` construct. + + When True, this implies that the + :attr:`inserted_primary_key` attribute is accessible, + assuming the statement did not include + a user defined "returning" construct. + + """ + return self.context.isinsert + + def _cursor_description(self): + """May be overridden by subclasses.""" + + return self._saved_cursor.description + + def close(self, _autoclose_connection=True): + """Close this ResultProxy. + + Closes the underlying DBAPI cursor corresponding to the execution. + + Note that any data cached within this ResultProxy is still available. + For some types of results, this may include buffered rows. + + If this ResultProxy was generated from an implicit execution, + the underlying Connection will also be closed (returns the + underlying DBAPI connection to the connection pool.) + + This method is called automatically when: + + * all result rows are exhausted using the fetchXXX() methods. + * cursor.description is None. + + """ + + if not self.closed: + self.closed = True + self.connection._safe_close_cursor(self.cursor) + if _autoclose_connection and \ + self.connection.should_close_with_result: + self.connection.close() + # allow consistent errors + self.cursor = None + + def __iter__(self): + while True: + row = self.fetchone() + if row is None: + raise StopIteration + else: + yield row + + @util.memoized_property + def inserted_primary_key(self): + """Return the primary key for the row just inserted. + + The return value is a list of scalar values + corresponding to the list of primary key columns + in the target table. + + This only applies to single row :func:`.insert` + constructs which did not explicitly specify + :meth:`.Insert.returning`. + + Note that primary key columns which specify a + server_default clause, + or otherwise do not qualify as "autoincrement" + columns (see the notes at :class:`.Column`), and were + generated using the database-side default, will + appear in this list as ``None`` unless the backend + supports "returning" and the insert statement executed + with the "implicit returning" enabled. + + Raises :class:`~sqlalchemy.exc.InvalidRequestError` if the executed + statement is not a compiled expression construct + or is not an insert() construct. + + """ + + if not self.context.compiled: + raise exc.InvalidRequestError( + "Statement is not a compiled " + "expression construct.") + elif not self.context.isinsert: + raise exc.InvalidRequestError( + "Statement is not an insert() " + "expression construct.") + elif self.context._is_explicit_returning: + raise exc.InvalidRequestError( + "Can't call inserted_primary_key " + "when returning() " + "is used.") + + return self.context.inserted_primary_key + + def last_updated_params(self): + """Return the collection of updated parameters from this + execution. + + Raises :class:`~sqlalchemy.exc.InvalidRequestError` if the executed + statement is not a compiled expression construct + or is not an update() construct. + + """ + if not self.context.compiled: + raise exc.InvalidRequestError( + "Statement is not a compiled " + "expression construct.") + elif not self.context.isupdate: + raise exc.InvalidRequestError( + "Statement is not an update() " + "expression construct.") + elif self.context.executemany: + return self.context.compiled_parameters + else: + return self.context.compiled_parameters[0] + + def last_inserted_params(self): + """Return the collection of inserted parameters from this + execution. + + Raises :class:`~sqlalchemy.exc.InvalidRequestError` if the executed + statement is not a compiled expression construct + or is not an insert() construct. + + """ + if not self.context.compiled: + raise exc.InvalidRequestError( + "Statement is not a compiled " + "expression construct.") + elif not self.context.isinsert: + raise exc.InvalidRequestError( + "Statement is not an insert() " + "expression construct.") + elif self.context.executemany: + return self.context.compiled_parameters + else: + return self.context.compiled_parameters[0] + + @property + def returned_defaults(self): + """Return the values of default columns that were fetched using + the :meth:`.ValuesBase.return_defaults` feature. + + The value is an instance of :class:`.RowProxy`, or ``None`` + if :meth:`.ValuesBase.return_defaults` was not used or if the + backend does not support RETURNING. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :meth:`.ValuesBase.return_defaults` + + """ + return self.context.returned_defaults + + def lastrow_has_defaults(self): + """Return ``lastrow_has_defaults()`` from the underlying + :class:`.ExecutionContext`. + + See :class:`.ExecutionContext` for details. + + """ + + return self.context.lastrow_has_defaults() + + def postfetch_cols(self): + """Return ``postfetch_cols()`` from the underlying + :class:`.ExecutionContext`. + + See :class:`.ExecutionContext` for details. + + Raises :class:`~sqlalchemy.exc.InvalidRequestError` if the executed + statement is not a compiled expression construct + or is not an insert() or update() construct. + + """ + + if not self.context.compiled: + raise exc.InvalidRequestError( + "Statement is not a compiled " + "expression construct.") + elif not self.context.isinsert and not self.context.isupdate: + raise exc.InvalidRequestError( + "Statement is not an insert() or update() " + "expression construct.") + return self.context.postfetch_cols + + def prefetch_cols(self): + """Return ``prefetch_cols()`` from the underlying + :class:`.ExecutionContext`. + + See :class:`.ExecutionContext` for details. + + Raises :class:`~sqlalchemy.exc.InvalidRequestError` if the executed + statement is not a compiled expression construct + or is not an insert() or update() construct. + + """ + + if not self.context.compiled: + raise exc.InvalidRequestError( + "Statement is not a compiled " + "expression construct.") + elif not self.context.isinsert and not self.context.isupdate: + raise exc.InvalidRequestError( + "Statement is not an insert() or update() " + "expression construct.") + return self.context.prefetch_cols + + def supports_sane_rowcount(self): + """Return ``supports_sane_rowcount`` from the dialect. + + See :attr:`.ResultProxy.rowcount` for background. + + """ + + return self.dialect.supports_sane_rowcount + + def supports_sane_multi_rowcount(self): + """Return ``supports_sane_multi_rowcount`` from the dialect. + + See :attr:`.ResultProxy.rowcount` for background. + + """ + + return self.dialect.supports_sane_multi_rowcount + + def _fetchone_impl(self): + try: + return self.cursor.fetchone() + except AttributeError: + self._non_result() + + def _fetchmany_impl(self, size=None): + try: + if size is None: + return self.cursor.fetchmany() + else: + return self.cursor.fetchmany(size) + except AttributeError: + self._non_result() + + def _fetchall_impl(self): + try: + return self.cursor.fetchall() + except AttributeError: + self._non_result() + + def _non_result(self): + if self._metadata is None: + raise exc.ResourceClosedError( + "This result object does not return rows. " + "It has been closed automatically.", + ) + else: + raise exc.ResourceClosedError("This result object is closed.") + + def process_rows(self, rows): + process_row = self._process_row + metadata = self._metadata + keymap = metadata._keymap + processors = metadata._processors + if self._echo: + log = self.context.engine.logger.debug + l = [] + for row in rows: + log("Row %r", row) + l.append(process_row(metadata, row, processors, keymap)) + return l + else: + return [process_row(metadata, row, processors, keymap) + for row in rows] + + def fetchall(self): + """Fetch all rows, just like DB-API ``cursor.fetchall()``.""" + + try: + l = self.process_rows(self._fetchall_impl()) + self.close() + return l + except Exception as e: + self.connection._handle_dbapi_exception( + e, None, None, + self.cursor, self.context) + + def fetchmany(self, size=None): + """Fetch many rows, just like DB-API + ``cursor.fetchmany(size=cursor.arraysize)``. + + If rows are present, the cursor remains open after this is called. + Else the cursor is automatically closed and an empty list is returned. + + """ + + try: + l = self.process_rows(self._fetchmany_impl(size)) + if len(l) == 0: + self.close() + return l + except Exception as e: + self.connection._handle_dbapi_exception( + e, None, None, + self.cursor, self.context) + + def fetchone(self): + """Fetch one row, just like DB-API ``cursor.fetchone()``. + + If a row is present, the cursor remains open after this is called. + Else the cursor is automatically closed and None is returned. + + """ + try: + row = self._fetchone_impl() + if row is not None: + return self.process_rows([row])[0] + else: + self.close() + return None + except Exception as e: + self.connection._handle_dbapi_exception( + e, None, None, + self.cursor, self.context) + + def first(self): + """Fetch the first row and then close the result set unconditionally. + + Returns None if no row is present. + + """ + if self._metadata is None: + self._non_result() + + try: + row = self._fetchone_impl() + except Exception as e: + self.connection._handle_dbapi_exception( + e, None, None, + self.cursor, self.context) + + try: + if row is not None: + return self.process_rows([row])[0] + else: + return None + finally: + self.close() + + def scalar(self): + """Fetch the first column of the first row, and close the result set. + + Returns None if no row is present. + + """ + row = self.first() + if row is not None: + return row[0] + else: + return None + + +class BufferedRowResultProxy(ResultProxy): + """A ResultProxy with row buffering behavior. + + ``ResultProxy`` that buffers the contents of a selection of rows + before ``fetchone()`` is called. This is to allow the results of + ``cursor.description`` to be available immediately, when + interfacing with a DB-API that requires rows to be consumed before + this information is available (currently psycopg2, when used with + server-side cursors). + + The pre-fetching behavior fetches only one row initially, and then + grows its buffer size by a fixed amount with each successive need + for additional rows up to a size of 100. + """ + + def _init_metadata(self): + self.__buffer_rows() + super(BufferedRowResultProxy, self)._init_metadata() + + # this is a "growth chart" for the buffering of rows. + # each successive __buffer_rows call will use the next + # value in the list for the buffer size until the max + # is reached + size_growth = { + 1: 5, + 5: 10, + 10: 20, + 20: 50, + 50: 100, + 100: 250, + 250: 500, + 500: 1000 + } + + def __buffer_rows(self): + size = getattr(self, '_bufsize', 1) + self.__rowbuffer = collections.deque(self.cursor.fetchmany(size)) + self._bufsize = self.size_growth.get(size, size) + + def _fetchone_impl(self): + if self.closed: + return None + if not self.__rowbuffer: + self.__buffer_rows() + if not self.__rowbuffer: + return None + return self.__rowbuffer.popleft() + + def _fetchmany_impl(self, size=None): + if size is None: + return self._fetchall_impl() + result = [] + for x in range(0, size): + row = self._fetchone_impl() + if row is None: + break + result.append(row) + return result + + def _fetchall_impl(self): + self.__rowbuffer.extend(self.cursor.fetchall()) + ret = self.__rowbuffer + self.__rowbuffer = collections.deque() + return ret + + +class FullyBufferedResultProxy(ResultProxy): + """A result proxy that buffers rows fully upon creation. + + Used for operations where a result is to be delivered + after the database conversation can not be continued, + such as MSSQL INSERT...OUTPUT after an autocommit. + + """ + def _init_metadata(self): + super(FullyBufferedResultProxy, self)._init_metadata() + self.__rowbuffer = self._buffer_rows() + + def _buffer_rows(self): + return collections.deque(self.cursor.fetchall()) + + def _fetchone_impl(self): + if self.__rowbuffer: + return self.__rowbuffer.popleft() + else: + return None + + def _fetchmany_impl(self, size=None): + if size is None: + return self._fetchall_impl() + result = [] + for x in range(0, size): + row = self._fetchone_impl() + if row is None: + break + result.append(row) + return result + + def _fetchall_impl(self): + ret = self.__rowbuffer + self.__rowbuffer = collections.deque() + return ret + + +class BufferedColumnRow(RowProxy): + def __init__(self, parent, row, processors, keymap): + # preprocess row + row = list(row) + # this is a tad faster than using enumerate + index = 0 + for processor in parent._orig_processors: + if processor is not None: + row[index] = processor(row[index]) + index += 1 + row = tuple(row) + super(BufferedColumnRow, self).__init__(parent, row, + processors, keymap) + + +class BufferedColumnResultProxy(ResultProxy): + """A ResultProxy with column buffering behavior. + + ``ResultProxy`` that loads all columns into memory each time + fetchone() is called. If fetchmany() or fetchall() are called, + the full grid of results is fetched. This is to operate with + databases where result rows contain "live" results that fall out + of scope unless explicitly fetched. Currently this includes + cx_Oracle LOB objects. + + """ + + _process_row = BufferedColumnRow + + def _init_metadata(self): + super(BufferedColumnResultProxy, self)._init_metadata() + metadata = self._metadata + # orig_processors will be used to preprocess each row when they are + # constructed. + metadata._orig_processors = metadata._processors + # replace the all type processors by None processors. + metadata._processors = [None for _ in range(len(metadata.keys))] + keymap = {} + for k, (func, obj, index) in metadata._keymap.items(): + keymap[k] = (None, obj, index) + self._metadata._keymap = keymap + + def fetchall(self): + # can't call cursor.fetchall(), since rows must be + # fully processed before requesting more from the DBAPI. + l = [] + while True: + row = self.fetchone() + if row is None: + break + l.append(row) + return l + + def fetchmany(self, size=None): + # can't call cursor.fetchmany(), since rows must be + # fully processed before requesting more from the DBAPI. + if size is None: + return self.fetchall() + l = [] + for i in range(size): + row = self.fetchone() + if row is None: + break + l.append(row) + return l diff --git a/libs/sqlalchemy/engine/strategies.py b/libs/sqlalchemy/engine/strategies.py index fab97975..f6c06403 100644 --- a/libs/sqlalchemy/engine/strategies.py +++ b/libs/sqlalchemy/engine/strategies.py @@ -1,5 +1,5 @@ # engine/strategies.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -49,18 +49,27 @@ class DefaultEngineStrategy(EngineStrategy): dialect_cls = u.get_dialect() + if kwargs.pop('_coerce_config', False): + def pop_kwarg(key, default=None): + value = kwargs.pop(key, default) + if key in dialect_cls.engine_config_types: + value = dialect_cls.engine_config_types[key](value) + return value + else: + pop_kwarg = kwargs.pop + dialect_args = {} # consume dialect arguments from kwargs for k in util.get_cls_kwargs(dialect_cls): if k in kwargs: - dialect_args[k] = kwargs.pop(k) + dialect_args[k] = pop_kwarg(k) dbapi = kwargs.pop('module', None) if dbapi is None: dbapi_args = {} for k in util.get_func_kwargs(dialect_cls.dbapi): if k in kwargs: - dbapi_args[k] = kwargs.pop(k) + dbapi_args[k] = pop_kwarg(k) dbapi = dialect_cls.dbapi(**dbapi_args) dialect_args['dbapi'] = dbapi @@ -70,33 +79,26 @@ class DefaultEngineStrategy(EngineStrategy): # assemble connection arguments (cargs, cparams) = dialect.create_connect_args(u) - cparams.update(kwargs.pop('connect_args', {})) + cparams.update(pop_kwarg('connect_args', {})) # look for existing pool or create - pool = kwargs.pop('pool', None) + pool = pop_kwarg('pool', None) if pool is None: def connect(): try: return dialect.connect(*cargs, **cparams) - except Exception, e: - # Py3K - #raise exc.DBAPIError.instance(None, None, - # e, dialect.dbapi.Error, - # connection_invalidated= - # dialect.is_disconnect(e, None, None) - # ) from e - # Py2K - import sys - raise exc.DBAPIError.instance( - None, None, e, dialect.dbapi.Error, - connection_invalidated= - dialect.is_disconnect(e, None, None)), \ - None, sys.exc_info()[2] - # end Py2K + except dialect.dbapi.Error as e: + invalidated = dialect.is_disconnect(e, None, None) + util.raise_from_cause( + exc.DBAPIError.instance(None, None, + e, dialect.dbapi.Error, + connection_invalidated=invalidated + ) + ) - creator = kwargs.pop('creator', connect) + creator = pop_kwarg('creator', connect) - poolclass = kwargs.pop('poolclass', None) + poolclass = pop_kwarg('poolclass', None) if poolclass is None: poolclass = dialect_cls.get_pool_class(u) pool_args = {} @@ -107,13 +109,13 @@ class DefaultEngineStrategy(EngineStrategy): 'echo': 'echo_pool', 'timeout': 'pool_timeout', 'recycle': 'pool_recycle', - 'events':'pool_events', - 'use_threadlocal':'pool_threadlocal', - 'reset_on_return':'pool_reset_on_return'} + 'events': 'pool_events', + 'use_threadlocal': 'pool_threadlocal', + 'reset_on_return': 'pool_reset_on_return'} for k in util.get_cls_kwargs(poolclass): tk = translate.get(k, k) if tk in kwargs: - pool_args[k] = kwargs.pop(tk) + pool_args[k] = pop_kwarg(tk) pool = poolclass(creator, **pool_args) else: if isinstance(pool, poollib._DBProxy): @@ -126,7 +128,7 @@ class DefaultEngineStrategy(EngineStrategy): engine_args = {} for k in util.get_cls_kwargs(engineclass): if k in kwargs: - engine_args[k] = kwargs.pop(k) + engine_args[k] = pop_kwarg(k) _initialize = kwargs.pop('_initialize', True) @@ -147,7 +149,8 @@ class DefaultEngineStrategy(EngineStrategy): do_on_connect = dialect.on_connect() if do_on_connect: def on_connect(dbapi_connection, connection_record): - conn = getattr(dbapi_connection, '_sqla_unwrap', dbapi_connection) + conn = getattr( + dbapi_connection, '_sqla_unwrap', dbapi_connection) if conn is None: return do_on_connect(conn) @@ -155,14 +158,10 @@ class DefaultEngineStrategy(EngineStrategy): event.listen(pool, 'first_connect', on_connect) event.listen(pool, 'connect', on_connect) + @util.only_once def first_connect(dbapi_connection, connection_record): - c = base.Connection(engine, connection=dbapi_connection) - - # TODO: removing this allows the on connect activities - # to generate events. tests currently assume these aren't - # sent. do we want users to get all the initial connect - # activities as events ? - c._has_events = False + c = base.Connection(engine, connection=dbapi_connection, + _has_events=False) dialect.initialize(c) event.listen(pool, 'first_connect', first_connect) @@ -238,12 +237,14 @@ class MockEngineStrategy(EngineStrategy): kwargs['checkfirst'] = False from sqlalchemy.engine import ddl - ddl.SchemaGenerator(self.dialect, self, **kwargs).traverse_single(entity) + ddl.SchemaGenerator( + self.dialect, self, **kwargs).traverse_single(entity) def drop(self, entity, **kwargs): kwargs['checkfirst'] = False from sqlalchemy.engine import ddl - ddl.SchemaDropper(self.dialect, self, **kwargs).traverse_single(entity) + ddl.SchemaDropper( + self.dialect, self, **kwargs).traverse_single(entity) def _run_visitor(self, visitorcallable, element, connection=None, diff --git a/libs/sqlalchemy/engine/threadlocal.py b/libs/sqlalchemy/engine/threadlocal.py index c8a16272..ae647a78 100644 --- a/libs/sqlalchemy/engine/threadlocal.py +++ b/libs/sqlalchemy/engine/threadlocal.py @@ -1,21 +1,24 @@ # engine/threadlocal.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """Provides a thread-local transactional wrapper around the root Engine class. -The ``threadlocal`` module is invoked when using the ``strategy="threadlocal"`` flag -with :func:`~sqlalchemy.engine.create_engine`. This module is semi-private and is -invoked automatically when the threadlocal engine strategy is used. +The ``threadlocal`` module is invoked when using the +``strategy="threadlocal"`` flag with :func:`~sqlalchemy.engine.create_engine`. +This module is semi-private and is invoked automatically when the threadlocal +engine strategy is used. """ -from sqlalchemy import util, event -from sqlalchemy.engine import base +from .. import util +from . import base import weakref + class TLConnection(base.Connection): + def __init__(self, *arg, **kw): super(TLConnection, self).__init__(*arg, **kw) self.__opencount = 0 @@ -33,16 +36,18 @@ class TLConnection(base.Connection): self.__opencount = 0 base.Connection.close(self) -class TLEngine(base.Engine): - """An Engine that includes support for thread-local managed transactions.""" +class TLEngine(base.Engine): + """An Engine that includes support for thread-local managed + transactions. + + """ _tl_connection_cls = TLConnection def __init__(self, *args, **kwargs): super(TLEngine, self).__init__(*args, **kwargs) self._connections = util.threading.local() - def contextual_connect(self, **kw): if not hasattr(self._connections, 'conn'): connection = None @@ -52,21 +57,24 @@ class TLEngine(base.Engine): if connection is None or connection.closed: # guards against pool-level reapers, if desired. # or not connection.connection.is_valid: - connection = self._tl_connection_cls(self, self.pool.connect(), **kw) - self._connections.conn = conn = weakref.ref(connection) + connection = self._tl_connection_cls( + self, self.pool.connect(), **kw) + self._connections.conn = weakref.ref(connection) return connection._increment_connect() def begin_twophase(self, xid=None): if not hasattr(self._connections, 'trans'): self._connections.trans = [] - self._connections.trans.append(self.contextual_connect().begin_twophase(xid=xid)) + self._connections.trans.append( + self.contextual_connect().begin_twophase(xid=xid)) return self def begin_nested(self): if not hasattr(self._connections, 'trans'): self._connections.trans = [] - self._connections.trans.append(self.contextual_connect().begin_nested()) + self._connections.trans.append( + self.contextual_connect().begin_nested()) return self def begin(self): diff --git a/libs/sqlalchemy/engine/url.py b/libs/sqlalchemy/engine/url.py index 9cabc8dc..78ac0618 100644 --- a/libs/sqlalchemy/engine/url.py +++ b/libs/sqlalchemy/engine/url.py @@ -1,5 +1,5 @@ # engine/url.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,13 +7,16 @@ """Provides the :class:`~sqlalchemy.engine.url.URL` class which encapsulates information about a database connection specification. -The URL object is created automatically when :func:`~sqlalchemy.engine.create_engine` is called -with a string argument; alternatively, the URL is a public-facing construct which can +The URL object is created automatically when +:func:`~sqlalchemy.engine.create_engine` is called with a string +argument; alternatively, the URL is a public-facing construct which can be used directly and is also accepted directly by ``create_engine()``. """ -import re, urllib -from sqlalchemy import exc, util +import re +from .. import exc, util +from . import Dialect +from ..dialects import registry class URL(object): @@ -21,8 +24,8 @@ class URL(object): Represent the components of a URL used to connect to a database. This object is suitable to be passed directly to a - ``create_engine()`` call. The fields of the URL are parsed from a - string by the ``module-level make_url()`` function. the string + :func:`~sqlalchemy.create_engine` call. The fields of the URL are parsed from a + string by the :func:`.make_url` function. the string format of the URL is an RFC-1738-style string. All initialization parameters are available as public attributes. @@ -59,25 +62,35 @@ class URL(object): self.database = database self.query = query or {} - def __str__(self): + def __to_string__(self, hide_password=True): s = self.drivername + "://" if self.username is not None: - s += self.username + s += _rfc_1738_quote(self.username) if self.password is not None: - s += ':' + urllib.quote_plus(self.password) + s += ':' + ('***' if hide_password + else _rfc_1738_quote(self.password)) s += "@" if self.host is not None: - s += self.host + if ':' in self.host: + s += "[%s]" % self.host + else: + s += self.host if self.port is not None: s += ':' + str(self.port) if self.database is not None: s += '/' + self.database if self.query: - keys = self.query.keys() + keys = list(self.query) keys.sort() s += '?' + "&".join("%s=%s" % (k, self.query[k]) for k in keys) return s + def __str__(self): + return self.__to_string__(hide_password=False) + + def __repr__(self): + return self.__to_string__() + def __hash__(self): return hash(str(self)) @@ -96,49 +109,20 @@ class URL(object): to this URL's driver name. """ - try: - if '+' in self.drivername: - dialect, driver = self.drivername.split('+') - else: - dialect, driver = self.drivername, 'base' - - module = __import__('sqlalchemy.dialects.%s' % (dialect, )).dialects - module = getattr(module, dialect) - if hasattr(module, driver): - module = getattr(module, driver) - else: - module = self._load_entry_point() - if module is None: - raise exc.ArgumentError( - "Could not determine dialect for '%s'." % - self.drivername) - - return module.dialect - except ImportError: - module = self._load_entry_point() - if module is not None: - return module - else: - raise exc.ArgumentError( - "Could not determine dialect for '%s'." % self.drivername) - - def _load_entry_point(self): - """attempt to load this url's dialect from entry points, or return None - if pkg_resources is not installed or there is no matching entry point. - - Raise ImportError if the actual load fails. - - """ - try: - import pkg_resources - except ImportError: - return None - - for res in pkg_resources.iter_entry_points('sqlalchemy.dialects'): - if res.name == self.drivername.replace("+", "."): - return res.load() + if '+' not in self.drivername: + name = self.drivername else: - return None + name = self.drivername.replace('+', '.') + cls = registry.load(name) + # check for legacy dialects that + # would return a module with 'dialect' as the + # actual class + if hasattr(cls, 'dialect') and \ + isinstance(cls.dialect, type) and \ + issubclass(cls.dialect, Dialect): + return cls.dialect + else: + return cls def translate_connect_args(self, names=[], **kw): """Translate url attributes into a dictionary of connection arguments. @@ -150,8 +134,8 @@ class URL(object): :param \**kw: Optional, alternate key names for url attributes. - :param names: Deprecated. Same purpose as the keyword-based alternate names, - but correlates the name to the original positionally. + :param names: Deprecated. Same purpose as the keyword-based alternate + names, but correlates the name to the original positionally. """ translated = {} @@ -167,6 +151,7 @@ class URL(object): translated[name] = getattr(self, sname) return translated + def make_url(name_or_url): """Given a string or unicode instance, produce a new URL instance. @@ -174,25 +159,28 @@ def make_url(name_or_url): existing URL object is passed, just returns the object. """ - if isinstance(name_or_url, basestring): + if isinstance(name_or_url, util.string_types): return _parse_rfc1738_args(name_or_url) else: return name_or_url + def _parse_rfc1738_args(name): pattern = re.compile(r''' (?P[\w\+]+):// (?: (?P[^:/]*) - (?::(?P[^/]*))? + (?::(?P.*))? @)? (?: - (?P[^/:]*) + (?: + \[(?P[^/]+)\] | + (?P[^/:]+) + )? (?::(?P[^/]*))? )? (?:/(?P.*))? - ''' - , re.X) + ''', re.X) m = pattern.match(name) if m is not None: @@ -201,28 +189,39 @@ def _parse_rfc1738_args(name): tokens = components['database'].split('?', 2) components['database'] = tokens[0] query = (len(tokens) > 1 and dict(util.parse_qsl(tokens[1]))) or None - # Py2K - if query is not None: + if util.py2k and query is not None: query = dict((k.encode('ascii'), query[k]) for k in query) - # end Py2K else: query = None components['query'] = query - if components['password'] is not None: - components['password'] = urllib.unquote_plus(components['password']) + if components['username'] is not None: + components['username'] = _rfc_1738_unquote(components['username']) + if components['password'] is not None: + components['password'] = _rfc_1738_unquote(components['password']) + + ipv4host = components.pop('ipv4host') + ipv6host = components.pop('ipv6host') + components['host'] = ipv4host or ipv6host name = components.pop('name') return URL(name, **components) else: raise exc.ArgumentError( "Could not parse rfc1738 URL from string '%s'" % name) + +def _rfc_1738_quote(text): + return re.sub(r'[:@/]', lambda m: "%%%X" % ord(m.group(0)), text) + +def _rfc_1738_unquote(text): + return util.unquote(text) + def _parse_keyvalue_args(name): - m = re.match( r'(\w+)://(.*)', name) + m = re.match(r'(\w+)://(.*)', name) if m is not None: (name, args) = m.group(1, 2) - opts = dict( util.parse_qsl( args ) ) + opts = dict(util.parse_qsl(args)) return URL(name, *opts) else: return None diff --git a/libs/sqlalchemy/engine/util.py b/libs/sqlalchemy/engine/util.py new file mode 100644 index 00000000..6c0644be --- /dev/null +++ b/libs/sqlalchemy/engine/util.py @@ -0,0 +1,72 @@ +# engine/util.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from .. import util + +def connection_memoize(key): + """Decorator, memoize a function in a connection.info stash. + + Only applicable to functions which take no arguments other than a + connection. The memo will be stored in ``connection.info[key]``. + """ + + @util.decorator + def decorated(fn, self, connection): + connection = connection.connect() + try: + return connection.info[key] + except KeyError: + connection.info[key] = val = fn(self, connection) + return val + + return decorated + + +def py_fallback(): + def _distill_params(multiparams, params): + """Given arguments from the calling form *multiparams, **params, + return a list of bind parameter structures, usually a list of + dictionaries. + + In the case of 'raw' execution which accepts positional parameters, + it may be a list of tuples or lists. + + """ + + if not multiparams: + if params: + return [params] + else: + return [] + elif len(multiparams) == 1: + zero = multiparams[0] + if isinstance(zero, (list, tuple)): + if not zero or hasattr(zero[0], '__iter__') and \ + not hasattr(zero[0], 'strip'): + # execute(stmt, [{}, {}, {}, ...]) + # execute(stmt, [(), (), (), ...]) + return zero + else: + # execute(stmt, ("value", "value")) + return [zero] + elif hasattr(zero, 'keys'): + # execute(stmt, {"key":"value"}) + return [zero] + else: + # execute(stmt, "value") + return [[zero]] + else: + if hasattr(multiparams[0], '__iter__') and \ + not hasattr(multiparams[0], 'strip'): + return multiparams + else: + return [multiparams] + + return locals() +try: + from sqlalchemy.cutils import _distill_params +except ImportError: + globals().update(py_fallback()) diff --git a/libs/sqlalchemy/event.py b/libs/sqlalchemy/event.py deleted file mode 100644 index dabebb81..00000000 --- a/libs/sqlalchemy/event.py +++ /dev/null @@ -1,460 +0,0 @@ -# sqlalchemy/event.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -"""Base event API.""" - -from sqlalchemy import util, exc -import weakref - -CANCEL = util.symbol('CANCEL') -NO_RETVAL = util.symbol('NO_RETVAL') - -def listen(target, identifier, fn, *args, **kw): - """Register a listener function for the given target. - - e.g.:: - - from sqlalchemy import event - from sqlalchemy.schema import UniqueConstraint - - def unique_constraint_name(const, table): - const.name = "uq_%s_%s" % ( - table.name, - list(const.columns)[0].name - ) - event.listen( - UniqueConstraint, - "after_parent_attach", - unique_constraint_name) - - """ - - for evt_cls in _registrars[identifier]: - tgt = evt_cls._accept_with(target) - if tgt is not None: - tgt.dispatch._listen(tgt, identifier, fn, *args, **kw) - return - raise exc.InvalidRequestError("No such event '%s' for target '%s'" % - (identifier, target)) - -def listens_for(target, identifier, *args, **kw): - """Decorate a function as a listener for the given target + identifier. - - e.g.:: - - from sqlalchemy import event - from sqlalchemy.schema import UniqueConstraint - - @event.listens_for(UniqueConstraint, "after_parent_attach") - def unique_constraint_name(const, table): - const.name = "uq_%s_%s" % ( - table.name, - list(const.columns)[0].name - ) - """ - def decorate(fn): - listen(target, identifier, fn, *args, **kw) - return fn - return decorate - -def remove(target, identifier, fn): - """Remove an event listener. - - Note that some event removals, particularly for those event dispatchers - which create wrapper functions and secondary even listeners, may not yet - be supported. - - """ - for evt_cls in _registrars[identifier]: - for tgt in evt_cls._accept_with(target): - tgt.dispatch._remove(identifier, tgt, fn) - return - -_registrars = util.defaultdict(list) - -def _is_event_name(name): - return not name.startswith('_') and name != 'dispatch' - -class _UnpickleDispatch(object): - """Serializable callable that re-generates an instance of :class:`_Dispatch` - given a particular :class:`.Events` subclass. - - """ - def __call__(self, _parent_cls): - for cls in _parent_cls.__mro__: - if 'dispatch' in cls.__dict__: - return cls.__dict__['dispatch'].dispatch_cls(_parent_cls) - else: - raise AttributeError("No class with a 'dispatch' member present.") - -class _Dispatch(object): - """Mirror the event listening definitions of an Events class with - listener collections. - - Classes which define a "dispatch" member will return a - non-instantiated :class:`._Dispatch` subclass when the member - is accessed at the class level. When the "dispatch" member is - accessed at the instance level of its owner, an instance - of the :class:`._Dispatch` class is returned. - - A :class:`._Dispatch` class is generated for each :class:`.Events` - class defined, by the :func:`._create_dispatcher_class` function. - The original :class:`.Events` classes remain untouched. - This decouples the construction of :class:`.Events` subclasses from - the implementation used by the event internals, and allows - inspecting tools like Sphinx to work in an unsurprising - way against the public API. - - """ - - def __init__(self, _parent_cls): - self._parent_cls = _parent_cls - - def __reduce__(self): - return _UnpickleDispatch(), (self._parent_cls, ) - - def _update(self, other, only_propagate=True): - """Populate from the listeners in another :class:`_Dispatch` - object.""" - - for ls in _event_descriptors(other): - getattr(self, ls.name).\ - for_modify(self)._update(ls, only_propagate=only_propagate) - -def _event_descriptors(target): - return [getattr(target, k) for k in dir(target) if _is_event_name(k)] - -class _EventMeta(type): - """Intercept new Event subclasses and create - associated _Dispatch classes.""" - - def __init__(cls, classname, bases, dict_): - _create_dispatcher_class(cls, classname, bases, dict_) - return type.__init__(cls, classname, bases, dict_) - -def _create_dispatcher_class(cls, classname, bases, dict_): - """Create a :class:`._Dispatch` class corresponding to an - :class:`.Events` class.""" - - # there's all kinds of ways to do this, - # i.e. make a Dispatch class that shares the '_listen' method - # of the Event class, this is the straight monkeypatch. - dispatch_base = getattr(cls, 'dispatch', _Dispatch) - cls.dispatch = dispatch_cls = type("%sDispatch" % classname, - (dispatch_base, ), {}) - dispatch_cls._listen = cls._listen - dispatch_cls._clear = cls._clear - - for k in dict_: - if _is_event_name(k): - setattr(dispatch_cls, k, _DispatchDescriptor(dict_[k])) - _registrars[k].append(cls) - -def _remove_dispatcher(cls): - for k in dir(cls): - if _is_event_name(k): - _registrars[k].remove(cls) - if not _registrars[k]: - del _registrars[k] - -class Events(object): - """Define event listening functions for a particular target type.""" - - - __metaclass__ = _EventMeta - - @classmethod - def _accept_with(cls, target): - # Mapper, ClassManager, Session override this to - # also accept classes, scoped_sessions, sessionmakers, etc. - if hasattr(target, 'dispatch') and ( - isinstance(target.dispatch, cls.dispatch) or \ - isinstance(target.dispatch, type) and \ - issubclass(target.dispatch, cls.dispatch) - ): - return target - else: - return None - - @classmethod - def _listen(cls, target, identifier, fn, propagate=False, insert=False): - if insert: - getattr(target.dispatch, identifier).\ - for_modify(target.dispatch).insert(fn, target, propagate) - else: - getattr(target.dispatch, identifier).\ - for_modify(target.dispatch).append(fn, target, propagate) - - @classmethod - def _remove(cls, target, identifier, fn): - getattr(target.dispatch, identifier).remove(fn, target) - - @classmethod - def _clear(cls): - for attr in dir(cls.dispatch): - if _is_event_name(attr): - getattr(cls.dispatch, attr).clear() - -class _DispatchDescriptor(object): - """Class-level attributes on :class:`._Dispatch` classes.""" - - def __init__(self, fn): - self.__name__ = fn.__name__ - self.__doc__ = fn.__doc__ - self._clslevel = weakref.WeakKeyDictionary() - self._empty_listeners = weakref.WeakKeyDictionary() - - def _contains(self, cls, evt): - return cls in self._clslevel and \ - evt in self._clslevel[cls] - - def insert(self, obj, target, propagate): - assert isinstance(target, type), \ - "Class-level Event targets must be classes." - stack = [target] - while stack: - cls = stack.pop(0) - stack.extend(cls.__subclasses__()) - if cls is not target and cls not in self._clslevel: - self.update_subclass(cls) - else: - if cls not in self._clslevel: - self._clslevel[cls] = [] - self._clslevel[cls].insert(0, obj) - - def append(self, obj, target, propagate): - assert isinstance(target, type), \ - "Class-level Event targets must be classes." - - stack = [target] - while stack: - cls = stack.pop(0) - stack.extend(cls.__subclasses__()) - if cls is not target and cls not in self._clslevel: - self.update_subclass(cls) - else: - if cls not in self._clslevel: - self._clslevel[cls] = [] - self._clslevel[cls].append(obj) - - def update_subclass(self, target): - if target not in self._clslevel: - self._clslevel[target] = [] - clslevel = self._clslevel[target] - for cls in target.__mro__[1:]: - if cls in self._clslevel: - clslevel.extend([ - fn for fn - in self._clslevel[cls] - if fn not in clslevel - ]) - - def remove(self, obj, target): - stack = [target] - while stack: - cls = stack.pop(0) - stack.extend(cls.__subclasses__()) - if cls in self._clslevel: - self._clslevel[cls].remove(obj) - - def clear(self): - """Clear all class level listeners""" - - for dispatcher in self._clslevel.values(): - dispatcher[:] = [] - - def for_modify(self, obj): - """Return an event collection which can be modified. - - For _DispatchDescriptor at the class level of - a dispatcher, this returns self. - - """ - return self - - def __get__(self, obj, cls): - if obj is None: - return self - elif obj._parent_cls in self._empty_listeners: - ret = self._empty_listeners[obj._parent_cls] - else: - self._empty_listeners[obj._parent_cls] = ret = \ - _EmptyListener(self, obj._parent_cls) - # assigning it to __dict__ means - # memoized for fast re-access. but more memory. - obj.__dict__[self.__name__] = ret - return ret - -class _EmptyListener(object): - """Serves as a class-level interface to the events - served by a _DispatchDescriptor, when there are no - instance-level events present. - - Is replaced by _ListenerCollection when instance-level - events are added. - - """ - def __init__(self, parent, target_cls): - if target_cls not in parent._clslevel: - parent.update_subclass(target_cls) - self.parent = parent - self.parent_listeners = parent._clslevel[target_cls] - self.name = parent.__name__ - self.propagate = frozenset() - self.listeners = () - - def for_modify(self, obj): - """Return an event collection which can be modified. - - For _EmptyListener at the instance level of - a dispatcher, this generates a new - _ListenerCollection, applies it to the instance, - and returns it. - - """ - obj.__dict__[self.name] = result = _ListenerCollection( - self.parent, obj._parent_cls) - return result - - def _needs_modify(self, *args, **kw): - raise NotImplementedError("need to call for_modify()") - - exec_once = insert = append = remove = clear = _needs_modify - - def __call__(self, *args, **kw): - """Execute this event.""" - - for fn in self.parent_listeners: - fn(*args, **kw) - - def __len__(self): - return len(self.parent_listeners) - - def __iter__(self): - return iter(self.parent_listeners) - - def __getitem__(self, index): - return (self.parent_listeners)[index] - - def __nonzero__(self): - return bool(self.parent_listeners) - - -class _ListenerCollection(object): - """Instance-level attributes on instances of :class:`._Dispatch`. - - Represents a collection of listeners. - - As of 0.7.9, _ListenerCollection is only first - created via the _EmptyListener.for_modify() method. - - """ - - _exec_once = False - - def __init__(self, parent, target_cls): - if target_cls not in parent._clslevel: - parent.update_subclass(target_cls) - self.parent_listeners = parent._clslevel[target_cls] - self.name = parent.__name__ - self.listeners = [] - self.propagate = set() - - def for_modify(self, obj): - """Return an event collection which can be modified. - - For _ListenerCollection at the instance level of - a dispatcher, this returns self. - - """ - return self - - def exec_once(self, *args, **kw): - """Execute this event, but only if it has not been - executed already for this collection.""" - - if not self._exec_once: - self(*args, **kw) - self._exec_once = True - - def __call__(self, *args, **kw): - """Execute this event.""" - - for fn in self.parent_listeners: - fn(*args, **kw) - for fn in self.listeners: - fn(*args, **kw) - - # I'm not entirely thrilled about the overhead here, - # but this allows class-level listeners to be added - # at any point. - # - # In the absense of instance-level listeners, - # we stay with the _EmptyListener object when called - # at the instance level. - - def __len__(self): - return len(self.parent_listeners + self.listeners) - - def __iter__(self): - return iter(self.parent_listeners + self.listeners) - - def __getitem__(self, index): - return (self.parent_listeners + self.listeners)[index] - - def __nonzero__(self): - return bool(self.listeners or self.parent_listeners) - - def _update(self, other, only_propagate=True): - """Populate from the listeners in another :class:`_Dispatch` - object.""" - - existing_listeners = self.listeners - existing_listener_set = set(existing_listeners) - self.propagate.update(other.propagate) - existing_listeners.extend([l for l - in other.listeners - if l not in existing_listener_set - and not only_propagate or l in self.propagate - ]) - - def insert(self, obj, target, propagate): - if obj not in self.listeners: - self.listeners.insert(0, obj) - if propagate: - self.propagate.add(obj) - - def append(self, obj, target, propagate): - if obj not in self.listeners: - self.listeners.append(obj) - if propagate: - self.propagate.add(obj) - - def remove(self, obj, target): - if obj in self.listeners: - self.listeners.remove(obj) - self.propagate.discard(obj) - - def clear(self): - self.listeners[:] = [] - self.propagate.clear() - -class dispatcher(object): - """Descriptor used by target classes to - deliver the _Dispatch class at the class level - and produce new _Dispatch instances for target - instances. - - """ - def __init__(self, events): - self.dispatch_cls = events.dispatch - self.events = events - - def __get__(self, obj, cls): - if obj is None: - return self.dispatch_cls - obj.__dict__['dispatch'] = disp = self.dispatch_cls(cls) - return disp diff --git a/libs/sqlalchemy/event/__init__.py b/libs/sqlalchemy/event/__init__.py new file mode 100644 index 00000000..b43bf9bf --- /dev/null +++ b/libs/sqlalchemy/event/__init__.py @@ -0,0 +1,10 @@ +# event/__init__.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from .api import CANCEL, NO_RETVAL, listen, listens_for, remove, contains +from .base import Events, dispatcher +from .attr import RefCollection +from .legacy import _legacy_signature diff --git a/libs/sqlalchemy/event/api.py b/libs/sqlalchemy/event/api.py new file mode 100644 index 00000000..20e74d90 --- /dev/null +++ b/libs/sqlalchemy/event/api.py @@ -0,0 +1,107 @@ +# event/api.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Public API functions for the event system. + +""" +from __future__ import absolute_import + +from .. import util, exc +from .base import _registrars +from .registry import _EventKey + +CANCEL = util.symbol('CANCEL') +NO_RETVAL = util.symbol('NO_RETVAL') + + +def _event_key(target, identifier, fn): + for evt_cls in _registrars[identifier]: + tgt = evt_cls._accept_with(target) + if tgt is not None: + return _EventKey(target, identifier, fn, tgt) + else: + raise exc.InvalidRequestError("No such event '%s' for target '%s'" % + (identifier, target)) + +def listen(target, identifier, fn, *args, **kw): + """Register a listener function for the given target. + + e.g.:: + + from sqlalchemy import event + from sqlalchemy.schema import UniqueConstraint + + def unique_constraint_name(const, table): + const.name = "uq_%s_%s" % ( + table.name, + list(const.columns)[0].name + ) + event.listen( + UniqueConstraint, + "after_parent_attach", + unique_constraint_name) + + """ + + _event_key(target, identifier, fn).listen(*args, **kw) + + +def listens_for(target, identifier, *args, **kw): + """Decorate a function as a listener for the given target + identifier. + + e.g.:: + + from sqlalchemy import event + from sqlalchemy.schema import UniqueConstraint + + @event.listens_for(UniqueConstraint, "after_parent_attach") + def unique_constraint_name(const, table): + const.name = "uq_%s_%s" % ( + table.name, + list(const.columns)[0].name + ) + """ + def decorate(fn): + listen(target, identifier, fn, *args, **kw) + return fn + return decorate + + +def remove(target, identifier, fn): + """Remove an event listener. + + The arguments here should match exactly those which were sent to + :func:`.listen`; all the event registration which proceeded as a result + of this call will be reverted by calling :func:`.remove` with the same + arguments. + + e.g.:: + + # if a function was registered like this... + @event.listens_for(SomeMappedClass, "before_insert", propagate=True) + def my_listener_function(*arg): + pass + + # ... it's removed like this + event.remove(SomeMappedClass, "before_insert", my_listener_function) + + Above, the listener function associated with ``SomeMappedClass`` was also + propagated to subclasses of ``SomeMappedClass``; the :func:`.remove` function + will revert all of these operations. + + .. versionadded:: 0.9.0 + + """ + _event_key(target, identifier, fn).remove() + +def contains(target, identifier, fn): + """Return True if the given target/ident/fn is set up to listen. + + .. versionadded:: 0.9.0 + + """ + + return _event_key(target, identifier, fn).contains() diff --git a/libs/sqlalchemy/event/attr.py b/libs/sqlalchemy/event/attr.py new file mode 100644 index 00000000..3f894754 --- /dev/null +++ b/libs/sqlalchemy/event/attr.py @@ -0,0 +1,376 @@ +# event/attr.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Attribute implementation for _Dispatch classes. + +The various listener targets for a particular event class are represented +as attributes, which refer to collections of listeners to be fired off. +These collections can exist at the class level as well as at the instance +level. An event is fired off using code like this:: + + some_object.dispatch.first_connect(arg1, arg2) + +Above, ``some_object.dispatch`` would be an instance of ``_Dispatch`` and +``first_connect`` is typically an instance of ``_ListenerCollection`` +if event listeners are present, or ``_EmptyListener`` if none are present. + +The attribute mechanics here spend effort trying to ensure listener functions +are available with a minimum of function call overhead, that unnecessary +objects aren't created (i.e. many empty per-instance listener collections), +as well as that everything is garbage collectable when owning references are +lost. Other features such as "propagation" of listener functions across +many ``_Dispatch`` instances, "joining" of multiple ``_Dispatch`` instances, +as well as support for subclass propagation (e.g. events assigned to +``Pool`` vs. ``QueuePool``) are all implemented here. + +""" + +from __future__ import absolute_import + +from .. import util +from . import registry +from . import legacy +from itertools import chain +import weakref + +class RefCollection(object): + @util.memoized_property + def ref(self): + return weakref.ref(self, registry._collection_gced) + +class _DispatchDescriptor(RefCollection): + """Class-level attributes on :class:`._Dispatch` classes.""" + + def __init__(self, parent_dispatch_cls, fn): + self.__name__ = fn.__name__ + argspec = util.inspect_getargspec(fn) + self.arg_names = argspec.args[1:] + self.has_kw = bool(argspec.keywords) + self.legacy_signatures = list(reversed( + sorted( + getattr(fn, '_legacy_signatures', []), + key=lambda s: s[0] + ) + )) + self.__doc__ = fn.__doc__ = legacy._augment_fn_docs( + self, parent_dispatch_cls, fn) + + self._clslevel = weakref.WeakKeyDictionary() + self._empty_listeners = weakref.WeakKeyDictionary() + + def _adjust_fn_spec(self, fn, named): + if named: + fn = self._wrap_fn_for_kw(fn) + if self.legacy_signatures: + try: + argspec = util.get_callable_argspec(fn, no_self=True) + except ValueError: + pass + else: + fn = legacy._wrap_fn_for_legacy(self, fn, argspec) + return fn + + def _wrap_fn_for_kw(self, fn): + def wrap_kw(*args, **kw): + argdict = dict(zip(self.arg_names, args)) + argdict.update(kw) + return fn(**argdict) + return wrap_kw + + + def insert(self, event_key, propagate): + target = event_key.dispatch_target + assert isinstance(target, type), \ + "Class-level Event targets must be classes." + stack = [target] + while stack: + cls = stack.pop(0) + stack.extend(cls.__subclasses__()) + if cls is not target and cls not in self._clslevel: + self.update_subclass(cls) + else: + if cls not in self._clslevel: + self._clslevel[cls] = [] + self._clslevel[cls].insert(0, event_key._listen_fn) + registry._stored_in_collection(event_key, self) + + def append(self, event_key, propagate): + target = event_key.dispatch_target + assert isinstance(target, type), \ + "Class-level Event targets must be classes." + + stack = [target] + while stack: + cls = stack.pop(0) + stack.extend(cls.__subclasses__()) + if cls is not target and cls not in self._clslevel: + self.update_subclass(cls) + else: + if cls not in self._clslevel: + self._clslevel[cls] = [] + self._clslevel[cls].append(event_key._listen_fn) + registry._stored_in_collection(event_key, self) + + def update_subclass(self, target): + if target not in self._clslevel: + self._clslevel[target] = [] + clslevel = self._clslevel[target] + for cls in target.__mro__[1:]: + if cls in self._clslevel: + clslevel.extend([ + fn for fn + in self._clslevel[cls] + if fn not in clslevel + ]) + + def remove(self, event_key): + target = event_key.dispatch_target + stack = [target] + while stack: + cls = stack.pop(0) + stack.extend(cls.__subclasses__()) + if cls in self._clslevel: + self._clslevel[cls].remove(event_key.fn) + registry._removed_from_collection(event_key, self) + + def clear(self): + """Clear all class level listeners""" + + to_clear = set() + for dispatcher in self._clslevel.values(): + to_clear.update(dispatcher) + dispatcher[:] = [] + registry._clear(self, to_clear) + + def for_modify(self, obj): + """Return an event collection which can be modified. + + For _DispatchDescriptor at the class level of + a dispatcher, this returns self. + + """ + return self + + def __get__(self, obj, cls): + if obj is None: + return self + elif obj._parent_cls in self._empty_listeners: + ret = self._empty_listeners[obj._parent_cls] + else: + self._empty_listeners[obj._parent_cls] = ret = \ + _EmptyListener(self, obj._parent_cls) + # assigning it to __dict__ means + # memoized for fast re-access. but more memory. + obj.__dict__[self.__name__] = ret + return ret + +class _HasParentDispatchDescriptor(object): + def _adjust_fn_spec(self, fn, named): + return self.parent._adjust_fn_spec(fn, named) + +class _EmptyListener(_HasParentDispatchDescriptor): + """Serves as a class-level interface to the events + served by a _DispatchDescriptor, when there are no + instance-level events present. + + Is replaced by _ListenerCollection when instance-level + events are added. + + """ + def __init__(self, parent, target_cls): + if target_cls not in parent._clslevel: + parent.update_subclass(target_cls) + self.parent = parent # _DispatchDescriptor + self.parent_listeners = parent._clslevel[target_cls] + self.name = parent.__name__ + self.propagate = frozenset() + self.listeners = () + + + def for_modify(self, obj): + """Return an event collection which can be modified. + + For _EmptyListener at the instance level of + a dispatcher, this generates a new + _ListenerCollection, applies it to the instance, + and returns it. + + """ + result = _ListenerCollection(self.parent, obj._parent_cls) + if obj.__dict__[self.name] is self: + obj.__dict__[self.name] = result + return result + + def _needs_modify(self, *args, **kw): + raise NotImplementedError("need to call for_modify()") + + exec_once = insert = append = remove = clear = _needs_modify + + def __call__(self, *args, **kw): + """Execute this event.""" + + for fn in self.parent_listeners: + fn(*args, **kw) + + def __len__(self): + return len(self.parent_listeners) + + def __iter__(self): + return iter(self.parent_listeners) + + def __bool__(self): + return bool(self.parent_listeners) + + __nonzero__ = __bool__ + + +class _CompoundListener(_HasParentDispatchDescriptor): + _exec_once = False + + def exec_once(self, *args, **kw): + """Execute this event, but only if it has not been + executed already for this collection.""" + + if not self._exec_once: + self(*args, **kw) + self._exec_once = True + + def __call__(self, *args, **kw): + """Execute this event.""" + + for fn in self.parent_listeners: + fn(*args, **kw) + for fn in self.listeners: + fn(*args, **kw) + + def __len__(self): + return len(self.parent_listeners) + len(self.listeners) + + def __iter__(self): + return chain(self.parent_listeners, self.listeners) + + def __bool__(self): + return bool(self.listeners or self.parent_listeners) + + __nonzero__ = __bool__ + +class _ListenerCollection(RefCollection, _CompoundListener): + """Instance-level attributes on instances of :class:`._Dispatch`. + + Represents a collection of listeners. + + As of 0.7.9, _ListenerCollection is only first + created via the _EmptyListener.for_modify() method. + + """ + + def __init__(self, parent, target_cls): + if target_cls not in parent._clslevel: + parent.update_subclass(target_cls) + self.parent_listeners = parent._clslevel[target_cls] + self.parent = parent + self.name = parent.__name__ + self.listeners = [] + self.propagate = set() + + def for_modify(self, obj): + """Return an event collection which can be modified. + + For _ListenerCollection at the instance level of + a dispatcher, this returns self. + + """ + return self + + def _update(self, other, only_propagate=True): + """Populate from the listeners in another :class:`_Dispatch` + object.""" + + existing_listeners = self.listeners + existing_listener_set = set(existing_listeners) + self.propagate.update(other.propagate) + other_listeners = [l for l + in other.listeners + if l not in existing_listener_set + and not only_propagate or l in self.propagate + ] + + existing_listeners.extend(other_listeners) + + to_associate = other.propagate.union(other_listeners) + registry._stored_in_collection_multi(self, other, to_associate) + + def insert(self, event_key, propagate): + if event_key._listen_fn not in self.listeners: + event_key.prepend_to_list(self, self.listeners) + if propagate: + self.propagate.add(event_key._listen_fn) + + def append(self, event_key, propagate): + if event_key._listen_fn not in self.listeners: + event_key.append_to_list(self, self.listeners) + if propagate: + self.propagate.add(event_key._listen_fn) + + def remove(self, event_key): + self.listeners.remove(event_key._listen_fn) + self.propagate.discard(event_key._listen_fn) + registry._removed_from_collection(event_key, self) + + def clear(self): + registry._clear(self, self.listeners) + self.propagate.clear() + self.listeners[:] = [] + + +class _JoinedDispatchDescriptor(object): + def __init__(self, name): + self.name = name + + def __get__(self, obj, cls): + if obj is None: + return self + else: + obj.__dict__[self.name] = ret = _JoinedListener( + obj.parent, self.name, + getattr(obj.local, self.name) + ) + return ret + + +class _JoinedListener(_CompoundListener): + _exec_once = False + + def __init__(self, parent, name, local): + self.parent = parent + self.name = name + self.local = local + self.parent_listeners = self.local + + @property + def listeners(self): + return getattr(self.parent, self.name) + + def _adjust_fn_spec(self, fn, named): + return self.local._adjust_fn_spec(fn, named) + + def for_modify(self, obj): + self.local = self.parent_listeners = self.local.for_modify(obj) + return self + + def insert(self, event_key, propagate): + self.local.insert(event_key, propagate) + + def append(self, event_key, propagate): + self.local.append(event_key, propagate) + + def remove(self, event_key): + self.local.remove(event_key) + + def clear(self): + raise NotImplementedError() + + diff --git a/libs/sqlalchemy/event/base.py b/libs/sqlalchemy/event/base.py new file mode 100644 index 00000000..5c8d92cb --- /dev/null +++ b/libs/sqlalchemy/event/base.py @@ -0,0 +1,217 @@ +# event/base.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Base implementation classes. + +The public-facing ``Events`` serves as the base class for an event interface; +it's public attributes represent different kinds of events. These attributes +are mirrored onto a ``_Dispatch`` class, which serves as a container for +collections of listener functions. These collections are represented both +at the class level of a particular ``_Dispatch`` class as well as within +instances of ``_Dispatch``. + +""" +from __future__ import absolute_import + +from .. import util +from .attr import _JoinedDispatchDescriptor, _EmptyListener, _DispatchDescriptor + +_registrars = util.defaultdict(list) + + +def _is_event_name(name): + return not name.startswith('_') and name != 'dispatch' + + +class _UnpickleDispatch(object): + """Serializable callable that re-generates an instance of + :class:`_Dispatch` given a particular :class:`.Events` subclass. + + """ + def __call__(self, _parent_cls): + for cls in _parent_cls.__mro__: + if 'dispatch' in cls.__dict__: + return cls.__dict__['dispatch'].dispatch_cls(_parent_cls) + else: + raise AttributeError("No class with a 'dispatch' member present.") + + +class _Dispatch(object): + """Mirror the event listening definitions of an Events class with + listener collections. + + Classes which define a "dispatch" member will return a + non-instantiated :class:`._Dispatch` subclass when the member + is accessed at the class level. When the "dispatch" member is + accessed at the instance level of its owner, an instance + of the :class:`._Dispatch` class is returned. + + A :class:`._Dispatch` class is generated for each :class:`.Events` + class defined, by the :func:`._create_dispatcher_class` function. + The original :class:`.Events` classes remain untouched. + This decouples the construction of :class:`.Events` subclasses from + the implementation used by the event internals, and allows + inspecting tools like Sphinx to work in an unsurprising + way against the public API. + + """ + + _events = None + """reference the :class:`.Events` class which this + :class:`._Dispatch` is created for.""" + + def __init__(self, _parent_cls): + self._parent_cls = _parent_cls + + @util.classproperty + def _listen(cls): + return cls._events._listen + + def _join(self, other): + """Create a 'join' of this :class:`._Dispatch` and another. + + This new dispatcher will dispatch events to both + :class:`._Dispatch` objects. + + """ + if '_joined_dispatch_cls' not in self.__class__.__dict__: + cls = type( + "Joined%s" % self.__class__.__name__, + (_JoinedDispatcher, self.__class__), {} + ) + for ls in _event_descriptors(self): + setattr(cls, ls.name, _JoinedDispatchDescriptor(ls.name)) + + self.__class__._joined_dispatch_cls = cls + return self._joined_dispatch_cls(self, other) + + def __reduce__(self): + return _UnpickleDispatch(), (self._parent_cls, ) + + def _update(self, other, only_propagate=True): + """Populate from the listeners in another :class:`_Dispatch` + object.""" + + for ls in _event_descriptors(other): + if isinstance(ls, _EmptyListener): + continue + getattr(self, ls.name).\ + for_modify(self)._update(ls, only_propagate=only_propagate) + + @util.hybridmethod + def _clear(self): + for attr in dir(self): + if _is_event_name(attr): + getattr(self, attr).for_modify(self).clear() + + +def _event_descriptors(target): + return [getattr(target, k) for k in dir(target) if _is_event_name(k)] + + +class _EventMeta(type): + """Intercept new Event subclasses and create + associated _Dispatch classes.""" + + def __init__(cls, classname, bases, dict_): + _create_dispatcher_class(cls, classname, bases, dict_) + return type.__init__(cls, classname, bases, dict_) + + +def _create_dispatcher_class(cls, classname, bases, dict_): + """Create a :class:`._Dispatch` class corresponding to an + :class:`.Events` class.""" + + # there's all kinds of ways to do this, + # i.e. make a Dispatch class that shares the '_listen' method + # of the Event class, this is the straight monkeypatch. + dispatch_base = getattr(cls, 'dispatch', _Dispatch) + dispatch_cls = type("%sDispatch" % classname, + (dispatch_base, ), {}) + cls._set_dispatch(cls, dispatch_cls) + + for k in dict_: + if _is_event_name(k): + setattr(dispatch_cls, k, _DispatchDescriptor(cls, dict_[k])) + _registrars[k].append(cls) + + if getattr(cls, '_dispatch_target', None): + cls._dispatch_target.dispatch = dispatcher(cls) + + +def _remove_dispatcher(cls): + for k in dir(cls): + if _is_event_name(k): + _registrars[k].remove(cls) + if not _registrars[k]: + del _registrars[k] + +class Events(util.with_metaclass(_EventMeta, object)): + """Define event listening functions for a particular target type.""" + + @staticmethod + def _set_dispatch(cls, dispatch_cls): + # this allows an Events subclass to define additional utility + # methods made available to the target via + # "self.dispatch._events." + # @staticemethod to allow easy "super" calls while in a metaclass + # constructor. + cls.dispatch = dispatch_cls + dispatch_cls._events = cls + + + @classmethod + def _accept_with(cls, target): + # Mapper, ClassManager, Session override this to + # also accept classes, scoped_sessions, sessionmakers, etc. + if hasattr(target, 'dispatch') and ( + isinstance(target.dispatch, cls.dispatch) or \ + isinstance(target.dispatch, type) and \ + issubclass(target.dispatch, cls.dispatch) + ): + return target + else: + return None + + @classmethod + def _listen(cls, event_key, propagate=False, insert=False, named=False): + event_key.base_listen(propagate=propagate, insert=insert, named=named) + + @classmethod + def _remove(cls, event_key): + event_key.remove() + + @classmethod + def _clear(cls): + cls.dispatch._clear() + + +class _JoinedDispatcher(object): + """Represent a connection between two _Dispatch objects.""" + + def __init__(self, local, parent): + self.local = local + self.parent = parent + self._parent_cls = local._parent_cls + + +class dispatcher(object): + """Descriptor used by target classes to + deliver the _Dispatch class at the class level + and produce new _Dispatch instances for target + instances. + + """ + def __init__(self, events): + self.dispatch_cls = events.dispatch + self.events = events + + def __get__(self, obj, cls): + if obj is None: + return self.dispatch_cls + obj.__dict__['dispatch'] = disp = self.dispatch_cls(cls) + return disp + diff --git a/libs/sqlalchemy/event/legacy.py b/libs/sqlalchemy/event/legacy.py new file mode 100644 index 00000000..0b0d290f --- /dev/null +++ b/libs/sqlalchemy/event/legacy.py @@ -0,0 +1,156 @@ +# event/legacy.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Routines to handle adaption of legacy call signatures, +generation of deprecation notes and docstrings. + +""" + +from .. import util + +def _legacy_signature(since, argnames, converter=None): + def leg(fn): + if not hasattr(fn, '_legacy_signatures'): + fn._legacy_signatures = [] + fn._legacy_signatures.append((since, argnames, converter)) + return fn + return leg + +def _wrap_fn_for_legacy(dispatch_descriptor, fn, argspec): + for since, argnames, conv in dispatch_descriptor.legacy_signatures: + if argnames[-1] == "**kw": + has_kw = True + argnames = argnames[0:-1] + else: + has_kw = False + + if len(argnames) == len(argspec.args) \ + and has_kw is bool(argspec.keywords): + + if conv: + assert not has_kw + def wrap_leg(*args): + return fn(*conv(*args)) + else: + def wrap_leg(*args, **kw): + argdict = dict(zip(dispatch_descriptor.arg_names, args)) + args = [argdict[name] for name in argnames] + if has_kw: + return fn(*args, **kw) + else: + return fn(*args) + return wrap_leg + else: + return fn + +def _indent(text, indent): + return "\n".join( + indent + line + for line in text.split("\n") + ) + +def _standard_listen_example(dispatch_descriptor, sample_target, fn): + example_kw_arg = _indent( + "\n".join( + "%(arg)s = kw['%(arg)s']" % {"arg": arg} + for arg in dispatch_descriptor.arg_names[0:2] + ), + " ") + if dispatch_descriptor.legacy_signatures: + current_since = max(since for since, args, conv + in dispatch_descriptor.legacy_signatures) + else: + current_since = None + text = ( + "from sqlalchemy import event\n\n" + "# standard decorator style%(current_since)s\n" + "@event.listens_for(%(sample_target)s, '%(event_name)s')\n" + "def receive_%(event_name)s(%(named_event_arguments)s%(has_kw_arguments)s):\n" + " \"listen for the '%(event_name)s' event\"\n" + "\n # ... (event handling logic) ...\n" + ) + + if len(dispatch_descriptor.arg_names) > 2: + text += ( + + "\n# named argument style (new in 0.9)\n" + "@event.listens_for(%(sample_target)s, '%(event_name)s', named=True)\n" + "def receive_%(event_name)s(**kw):\n" + " \"listen for the '%(event_name)s' event\"\n" + "%(example_kw_arg)s\n" + "\n # ... (event handling logic) ...\n" + ) + + text %= { + "current_since": " (arguments as of %s)" % + current_since if current_since else "", + "event_name": fn.__name__, + "has_kw_arguments": ", **kw" if dispatch_descriptor.has_kw else "", + "named_event_arguments": ", ".join(dispatch_descriptor.arg_names), + "example_kw_arg": example_kw_arg, + "sample_target": sample_target + } + return text + +def _legacy_listen_examples(dispatch_descriptor, sample_target, fn): + text = "" + for since, args, conv in dispatch_descriptor.legacy_signatures: + text += ( + "\n# legacy calling style (pre-%(since)s)\n" + "@event.listens_for(%(sample_target)s, '%(event_name)s')\n" + "def receive_%(event_name)s(%(named_event_arguments)s%(has_kw_arguments)s):\n" + " \"listen for the '%(event_name)s' event\"\n" + "\n # ... (event handling logic) ...\n" % { + "since": since, + "event_name": fn.__name__, + "has_kw_arguments": " **kw" if dispatch_descriptor.has_kw else "", + "named_event_arguments": ", ".join(args), + "sample_target": sample_target + } + ) + return text + +def _version_signature_changes(dispatch_descriptor): + since, args, conv = dispatch_descriptor.legacy_signatures[0] + return ( + "\n.. versionchanged:: %(since)s\n" + " The ``%(event_name)s`` event now accepts the \n" + " arguments ``%(named_event_arguments)s%(has_kw_arguments)s``.\n" + " Listener functions which accept the previous argument \n" + " signature(s) listed above will be automatically \n" + " adapted to the new signature." % { + "since": since, + "event_name": dispatch_descriptor.__name__, + "named_event_arguments": ", ".join(dispatch_descriptor.arg_names), + "has_kw_arguments": ", **kw" if dispatch_descriptor.has_kw else "" + } + ) + +def _augment_fn_docs(dispatch_descriptor, parent_dispatch_cls, fn): + header = ".. container:: event_signatures\n\n"\ + " Example argument forms::\n"\ + "\n" + + sample_target = getattr(parent_dispatch_cls, "_target_class_doc", "obj") + text = ( + header + + _indent( + _standard_listen_example( + dispatch_descriptor, sample_target, fn), + " " * 8) + ) + if dispatch_descriptor.legacy_signatures: + text += _indent( + _legacy_listen_examples( + dispatch_descriptor, sample_target, fn), + " " * 8) + + text += _version_signature_changes(dispatch_descriptor) + + return util.inject_docstring_text(fn.__doc__, + text, + 1 + ) diff --git a/libs/sqlalchemy/event/registry.py b/libs/sqlalchemy/event/registry.py new file mode 100644 index 00000000..7710ff2d --- /dev/null +++ b/libs/sqlalchemy/event/registry.py @@ -0,0 +1,236 @@ +# event/registry.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Provides managed registration services on behalf of :func:`.listen` +arguments. + +By "managed registration", we mean that event listening functions and +other objects can be added to various collections in such a way that their +membership in all those collections can be revoked at once, based on +an equivalent :class:`._EventKey`. + +""" + +from __future__ import absolute_import + +import weakref +import collections +import types +from .. import exc + + +_key_to_collection = collections.defaultdict(dict) +""" +Given an original listen() argument, can locate all +listener collections and the listener fn contained + +(target, identifier, fn) -> { + ref(listenercollection) -> ref(listener_fn) + ref(listenercollection) -> ref(listener_fn) + ref(listenercollection) -> ref(listener_fn) + } +""" + +_collection_to_key = collections.defaultdict(dict) +""" +Given a _ListenerCollection or _DispatchDescriptor, can locate +all the original listen() arguments and the listener fn contained + +ref(listenercollection) -> { + ref(listener_fn) -> (target, identifier, fn), + ref(listener_fn) -> (target, identifier, fn), + ref(listener_fn) -> (target, identifier, fn), + } +""" + +def _collection_gced(ref): + # defaultdict, so can't get a KeyError + if not _collection_to_key or ref not in _collection_to_key: + return + listener_to_key = _collection_to_key.pop(ref) + for key in listener_to_key.values(): + if key in _key_to_collection: + # defaultdict, so can't get a KeyError + dispatch_reg = _key_to_collection[key] + dispatch_reg.pop(ref) + if not dispatch_reg: + _key_to_collection.pop(key) + +def _stored_in_collection(event_key, owner): + key = event_key._key + + dispatch_reg = _key_to_collection[key] + + owner_ref = owner.ref + listen_ref = weakref.ref(event_key._listen_fn) + + if owner_ref in dispatch_reg: + assert dispatch_reg[owner_ref] == listen_ref + else: + dispatch_reg[owner_ref] = listen_ref + + listener_to_key = _collection_to_key[owner_ref] + listener_to_key[listen_ref] = key + +def _removed_from_collection(event_key, owner): + key = event_key._key + + dispatch_reg = _key_to_collection[key] + + listen_ref = weakref.ref(event_key._listen_fn) + + owner_ref = owner.ref + dispatch_reg.pop(owner_ref, None) + if not dispatch_reg: + del _key_to_collection[key] + + if owner_ref in _collection_to_key: + listener_to_key = _collection_to_key[owner_ref] + listener_to_key.pop(listen_ref) + +def _stored_in_collection_multi(newowner, oldowner, elements): + if not elements: + return + + oldowner = oldowner.ref + newowner = newowner.ref + + old_listener_to_key = _collection_to_key[oldowner] + new_listener_to_key = _collection_to_key[newowner] + + for listen_fn in elements: + listen_ref = weakref.ref(listen_fn) + key = old_listener_to_key[listen_ref] + dispatch_reg = _key_to_collection[key] + if newowner in dispatch_reg: + assert dispatch_reg[newowner] == listen_ref + else: + dispatch_reg[newowner] = listen_ref + + new_listener_to_key[listen_ref] = key + +def _clear(owner, elements): + if not elements: + return + + owner = owner.ref + listener_to_key = _collection_to_key[owner] + for listen_fn in elements: + listen_ref = weakref.ref(listen_fn) + key = listener_to_key[listen_ref] + dispatch_reg = _key_to_collection[key] + dispatch_reg.pop(owner, None) + + if not dispatch_reg: + del _key_to_collection[key] + + +class _EventKey(object): + """Represent :func:`.listen` arguments. + """ + + + def __init__(self, target, identifier, fn, dispatch_target, _fn_wrap=None): + self.target = target + self.identifier = identifier + self.fn = fn + if isinstance(fn, types.MethodType): + self.fn_key = id(fn.__func__), id(fn.__self__) + else: + self.fn_key = id(fn) + self.fn_wrap = _fn_wrap + self.dispatch_target = dispatch_target + + @property + def _key(self): + return (id(self.target), self.identifier, self.fn_key) + + def with_wrapper(self, fn_wrap): + if fn_wrap is self._listen_fn: + return self + else: + return _EventKey( + self.target, + self.identifier, + self.fn, + self.dispatch_target, + _fn_wrap=fn_wrap + ) + + def with_dispatch_target(self, dispatch_target): + if dispatch_target is self.dispatch_target: + return self + else: + return _EventKey( + self.target, + self.identifier, + self.fn, + dispatch_target, + _fn_wrap=self.fn_wrap + ) + + def listen(self, *args, **kw): + self.dispatch_target.dispatch._listen(self, *args, **kw) + + def remove(self): + key = self._key + + if key not in _key_to_collection: + raise exc.InvalidRequestError( + "No listeners found for event %s / %r / %s " % + (self.target, self.identifier, self.fn) + ) + dispatch_reg = _key_to_collection.pop(key) + + for collection_ref, listener_ref in dispatch_reg.items(): + collection = collection_ref() + listener_fn = listener_ref() + if collection is not None and listener_fn is not None: + collection.remove(self.with_wrapper(listener_fn)) + + def contains(self): + """Return True if this event key is registered to listen. + """ + return self._key in _key_to_collection + + def base_listen(self, propagate=False, insert=False, + named=False): + + target, identifier, fn = \ + self.dispatch_target, self.identifier, self._listen_fn + + dispatch_descriptor = getattr(target.dispatch, identifier) + + fn = dispatch_descriptor._adjust_fn_spec(fn, named) + self = self.with_wrapper(fn) + + if insert: + dispatch_descriptor.\ + for_modify(target.dispatch).insert(self, propagate) + else: + dispatch_descriptor.\ + for_modify(target.dispatch).append(self, propagate) + + @property + def _listen_fn(self): + return self.fn_wrap or self.fn + + def append_value_to_list(self, owner, list_, value): + _stored_in_collection(self, owner) + list_.append(value) + + def append_to_list(self, owner, list_): + _stored_in_collection(self, owner) + list_.append(self._listen_fn) + + def remove_from_list(self, owner, list_): + _removed_from_collection(self, owner) + list_.remove(self._listen_fn) + + def prepend_to_list(self, owner, list_): + _stored_in_collection(self, owner) + list_.insert(0, self._listen_fn) + diff --git a/libs/sqlalchemy/events.py b/libs/sqlalchemy/events.py index e7aa34f2..cf77bbb7 100644 --- a/libs/sqlalchemy/events.py +++ b/libs/sqlalchemy/events.py @@ -1,20 +1,20 @@ # sqlalchemy/events.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """Core event interfaces.""" -from sqlalchemy import event, exc, util -engine = util.importlater('sqlalchemy', 'engine') -pool = util.importlater('sqlalchemy', 'pool') - +from . import event, exc +from .pool import Pool +from .engine import Connectable, Engine +from .sql.base import SchemaEventTarget class DDLEvents(event.Events): """ Define event listeners for schema objects, - that is, :class:`.SchemaItem` and :class:`.SchemaEvent` + that is, :class:`.SchemaItem` and other :class:`.SchemaEventTarget` subclasses, including :class:`.MetaData`, :class:`.Table`, :class:`.Column`. @@ -70,6 +70,9 @@ class DDLEvents(event.Events): """ + _target_class_doc = "SomeSchemaClassOrObject" + _dispatch_target = SchemaEventTarget + def before_create(self, target, connection, **kw): """Called before CREATE statments are emitted. @@ -166,7 +169,7 @@ class DDLEvents(event.Events): """ - def column_reflect(self, table, column_info): + def column_reflect(self, inspector, table, column_info): """Called for each unit of 'column info' retrieved when a :class:`.Table` is being reflected. @@ -188,7 +191,7 @@ class DDLEvents(event.Events): from sqlalchemy.schema import Table from sqlalchemy import event - def listen_for_reflect(table, column_info): + def listen_for_reflect(inspector, table, column_info): "receive a column_reflect event" # ... @@ -200,7 +203,7 @@ class DDLEvents(event.Events): ...or with a specific :class:`.Table` instance using the ``listeners`` argument:: - def listen_for_reflect(table, column_info): + def listen_for_reflect(inspector, table, column_info): "receive a column_reflect event" # ... @@ -216,23 +219,7 @@ class DDLEvents(event.Events): """ -class SchemaEventTarget(object): - """Base class for elements that are the targets of :class:`.DDLEvents` events. - This includes :class:`.SchemaItem` as well as :class:`.SchemaType`. - - """ - dispatch = event.dispatcher(DDLEvents) - - def _set_parent(self, parent): - """Associate with this SchemaEvent's parent object.""" - - raise NotImplementedError() - - def _set_parent_with_dispatch(self, parent): - self.dispatch.before_parent_attach(self, parent) - self._set_parent(parent) - self.dispatch.after_parent_attach(self, parent) class PoolEvents(event.Events): """Available events for :class:`.Pool`. @@ -250,11 +237,11 @@ class PoolEvents(event.Events): event.listen(Pool, 'checkout', my_on_checkout) - In addition to accepting the :class:`.Pool` class and :class:`.Pool` instances, - :class:`.PoolEvents` also accepts :class:`.Engine` objects and - the :class:`.Engine` class as targets, which will be resolved - to the ``.pool`` attribute of the given engine or the :class:`.Pool` - class:: + In addition to accepting the :class:`.Pool` class and + :class:`.Pool` instances, :class:`.PoolEvents` also accepts + :class:`.Engine` objects and the :class:`.Engine` class as + targets, which will be resolved to the ``.pool`` attribute of the + given engine or the :class:`.Pool` class:: engine = create_engine("postgresql://scott:tiger@localhost/test") @@ -263,14 +250,17 @@ class PoolEvents(event.Events): """ + _target_class_doc = "SomeEngineOrPool" + _dispatch_target = Pool + @classmethod def _accept_with(cls, target): if isinstance(target, type): - if issubclass(target, engine.Engine): - return pool.Pool - elif issubclass(target, pool.Pool): + if issubclass(target, Engine): + return Pool + elif issubclass(target, Pool): return target - elif isinstance(target, engine.Engine): + elif isinstance(target, Engine): return target.pool else: return target @@ -316,6 +306,10 @@ class PoolEvents(event.Events): connection will be disposed and a fresh connection retrieved. Processing of all checkout listeners will abort and restart using the new connection. + + .. seealso:: :meth:`.ConnectionEvents.engine_connect` - a similar event + which occurs upon creation of a new :class:`.Connection`. + """ def checkin(self, dbapi_connection, connection_record): @@ -333,22 +327,95 @@ class PoolEvents(event.Events): """ + def reset(self, dbapi_con, con_record): + """Called before the "reset" action occurs for a pooled connection. + + This event represents + when the ``rollback()`` method is called on the DBAPI connection + before it is returned to the pool. The behavior of "reset" can + be controlled, including disabled, using the ``reset_on_return`` + pool argument. + + + The :meth:`.PoolEvents.reset` event is usually followed by the + the :meth:`.PoolEvents.checkin` event is called, except in those + cases where the connection is discarded immediately after reset. + + :param dbapi_con: + A raw DB-API connection + + :param con_record: + The ``_ConnectionRecord`` that persistently manages the connection + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`.ConnectionEvents.rollback` + + :meth:`.ConnectionEvents.commit` + + """ + + + class ConnectionEvents(event.Events): - """Available events for :class:`.Connection`. + """Available events for :class:`.Connectable`, which includes + :class:`.Connection` and :class:`.Engine`. - The methods here define the name of an event as well as the names of members that are passed to listener functions. + The methods here define the name of an event as well as the names of + members that are passed to listener functions. - e.g.:: + An event listener can be associated with any :class:`.Connectable` + class or instance, such as an :class:`.Engine`, e.g.:: from sqlalchemy import event, create_engine - def before_execute(conn, clauseelement, multiparams, params): - log.info("Received statement: %s" % clauseelement) + def before_cursor_execute(conn, cursor, statement, parameters, context, + executemany): + log.info("Received statement: %s" % statement) engine = create_engine('postgresql://scott:tiger@localhost/test') - event.listen(engine, "before_execute", before_execute) + event.listen(engine, "before_cursor_execute", before_cursor_execute) - Some events allow modifiers to the listen() function. + or with a specific :class:`.Connection`:: + + with engine.begin() as conn: + @event.listens_for(conn, 'before_cursor_execute') + def before_cursor_execute(conn, cursor, statement, parameters, + context, executemany): + log.info("Received statement: %s" % statement) + + The :meth:`.before_execute` and :meth:`.before_cursor_execute` + events can also be established with the ``retval=True`` flag, which + allows modification of the statement and parameters to be sent + to the database. The :meth:`.before_cursor_execute` event is + particularly useful here to add ad-hoc string transformations, such + as comments, to all executions:: + + from sqlalchemy.engine import Engine + from sqlalchemy import event + + @event.listens_for(Engine, "before_cursor_execute", retval=True) + def comment_sql_calls(conn, cursor, statement, parameters, + context, executemany): + statement = statement + " -- some comment" + return statement, parameters + + .. note:: :class:`.ConnectionEvents` can be established on any + combination of :class:`.Engine`, :class:`.Connection`, as well + as instances of each of those classes. Events across all + four scopes will fire off for a given instance of + :class:`.Connection`. However, for performance reasons, the + :class:`.Connection` object determines at instantiation time + whether or not its parent :class:`.Engine` has event listeners + established. Event listeners added to the :class:`.Engine` + class or to an instance of :class:`.Engine` *after* the instantiation + of a dependent :class:`.Connection` instance will usually + *not* be available on that :class:`.Connection` instance. The newly + added listeners will instead take effect for :class:`.Connection` + instances created subsequent to those event listeners being + established on the parent :class:`.Engine` class or instance. :param retval=False: Applies to the :meth:`.before_execute` and :meth:`.before_cursor_execute` events only. When True, the @@ -357,49 +424,153 @@ class ConnectionEvents(event.Events): and parameters. See those methods for a description of specific return arguments. + .. versionchanged:: 0.8 :class:`.ConnectionEvents` can now be associated + with any :class:`.Connectable` including :class:`.Connection`, + in addition to the existing support for :class:`.Engine`. + """ + _target_class_doc = "SomeEngine" + _dispatch_target = Connectable + + @classmethod - def _listen(cls, target, identifier, fn, retval=False): + def _listen(cls, event_key, retval=False): + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn + target._has_events = True if not retval: if identifier == 'before_execute': orig_fn = fn - def wrap(conn, clauseelement, multiparams, params): + + def wrap_before_execute(conn, clauseelement, + multiparams, params): orig_fn(conn, clauseelement, multiparams, params) return clauseelement, multiparams, params - fn = wrap + fn = wrap_before_execute elif identifier == 'before_cursor_execute': orig_fn = fn - def wrap(conn, cursor, statement, + + def wrap_before_cursor_execute(conn, cursor, statement, parameters, context, executemany): orig_fn(conn, cursor, statement, parameters, context, executemany) return statement, parameters - fn = wrap + fn = wrap_before_cursor_execute - elif retval and identifier not in ('before_execute', 'before_cursor_execute'): + elif retval and \ + identifier not in ('before_execute', 'before_cursor_execute'): raise exc.ArgumentError( "Only the 'before_execute' and " "'before_cursor_execute' engine " "event listeners accept the 'retval=True' " "argument.") - event.Events._listen(target, identifier, fn) + event_key.with_wrapper(fn).base_listen() def before_execute(self, conn, clauseelement, multiparams, params): - """Intercept high level execute() events.""" + """Intercept high level execute() events, receiving uncompiled + SQL constructs and other objects prior to rendering into SQL. + + This event is good for debugging SQL compilation issues as well + as early manipulation of the parameters being sent to the database, + as the parameter lists will be in a consistent format here. + + This event can be optionally established with the ``retval=True`` + flag. The ``clauseelement``, ``multiparams``, and ``params`` + arguments should be returned as a three-tuple in this case:: + + @event.listens_for(Engine, "before_execute", retval=True) + def before_execute(conn, conn, clauseelement, multiparams, params): + # do something with clauseelement, multiparams, params + return clauseelement, multiparams, params + + :param conn: :class:`.Connection` object + :param clauseelement: SQL expression construct, :class:`.Compiled` + instance, or string statement passed to :meth:`.Connection.execute`. + :param multiparams: Multiple parameter sets, a list of dictionaries. + :param params: Single parameter set, a single dictionary. + + See also: + + :meth:`.before_cursor_execute` + + """ def after_execute(self, conn, clauseelement, multiparams, params, result): - """Intercept high level execute() events.""" + """Intercept high level execute() events after execute. + + + :param conn: :class:`.Connection` object + :param clauseelement: SQL expression construct, :class:`.Compiled` + instance, or string statement passed to :meth:`.Connection.execute`. + :param multiparams: Multiple parameter sets, a list of dictionaries. + :param params: Single parameter set, a single dictionary. + :param result: :class:`.ResultProxy` generated by the execution. + + """ def before_cursor_execute(self, conn, cursor, statement, parameters, context, executemany): - """Intercept low-level cursor execute() events.""" + """Intercept low-level cursor execute() events before execution, + receiving the string + SQL statement and DBAPI-specific parameter list to be invoked + against a cursor. + + This event is a good choice for logging as well as late modifications + to the SQL string. It's less ideal for parameter modifications except + for those which are specific to a target backend. + + This event can be optionally established with the ``retval=True`` + flag. The ``statement`` and ``parameters`` arguments should be + returned as a two-tuple in this case:: + + @event.listens_for(Engine, "before_cursor_execute", retval=True) + def before_cursor_execute(conn, cursor, statement, + parameters, context, executemany): + # do something with statement, parameters + return statement, parameters + + See the example at :class:`.ConnectionEvents`. + + :param conn: :class:`.Connection` object + :param cursor: DBAPI cursor object + :param statement: string SQL statement + :param parameters: Dictionary, tuple, or list of parameters being + passed to the ``execute()`` or ``executemany()`` method of the + DBAPI ``cursor``. In some cases may be ``None``. + :param context: :class:`.ExecutionContext` object in use. May + be ``None``. + :param executemany: boolean, if ``True``, this is an ``executemany()`` + call, if ``False``, this is an ``execute()`` call. + + See also: + + :meth:`.before_execute` + + :meth:`.after_cursor_execute` + + """ def after_cursor_execute(self, conn, cursor, statement, parameters, context, executemany): - """Intercept low-level cursor execute() events.""" + """Intercept low-level cursor execute() events after execution. + + :param conn: :class:`.Connection` object + :param cursor: DBAPI cursor object. Will have results pending + if the statement was a SELECT, but these should not be consumed + as they will be needed by the :class:`.ResultProxy`. + :param statement: string SQL statement + :param parameters: Dictionary, tuple, or list of parameters being + passed to the ``execute()`` or ``executemany()`` method of the + DBAPI ``cursor``. In some cases may be ``None``. + :param context: :class:`.ExecutionContext` object in use. May + be ``None``. + :param executemany: boolean, if ``True``, this is an ``executemany()`` + call, if ``False``, this is an ``execute()`` call. + + """ def dbapi_error(self, conn, cursor, statement, parameters, context, exception): @@ -427,37 +598,212 @@ class ConnectionEvents(event.Events): exception is then wrapped in a SQLAlchemy DBAPI exception wrapper and re-thrown. + :param conn: :class:`.Connection` object + :param cursor: DBAPI cursor object + :param statement: string SQL statement + :param parameters: Dictionary, tuple, or list of parameters being + passed to the ``execute()`` or ``executemany()`` method of the + DBAPI ``cursor``. In some cases may be ``None``. + :param context: :class:`.ExecutionContext` object in use. May + be ``None``. + :param exception: The **unwrapped** exception emitted directly from the + DBAPI. The class here is specific to the DBAPI module in use. + .. versionadded:: 0.7.7 """ + def engine_connect(self, conn, branch): + """Intercept the creation of a new :class:`.Connection`. + + This event is called typically as the direct result of calling + the :meth:`.Engine.connect` method. + + It differs from the :meth:`.PoolEvents.connect` method, which + refers to the actual connection to a database at the DBAPI level; + a DBAPI connection may be pooled and reused for many operations. + In contrast, this event refers only to the production of a higher level + :class:`.Connection` wrapper around such a DBAPI connection. + + It also differs from the :meth:`.PoolEvents.checkout` event + in that it is specific to the :class:`.Connection` object, not the + DBAPI connection that :meth:`.PoolEvents.checkout` deals with, although + this DBAPI connection is available here via the :attr:`.Connection.connection` + attribute. But note there can in fact + be multiple :meth:`.PoolEvents.checkout` events within the lifespan + of a single :class:`.Connection` object, if that :class:`.Connection` + is invalidated and re-established. There can also be multiple + :class:`.Connection` objects generated for the same already-checked-out + DBAPI connection, in the case that a "branch" of a :class:`.Connection` + is produced. + + :param conn: :class:`.Connection` object. + :param branch: if True, this is a "branch" of an existing + :class:`.Connection`. A branch is generated within the course + of a statement execution to invoke supplemental statements, most + typically to pre-execute a SELECT of a default value for the purposes + of an INSERT statement. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :meth:`.PoolEvents.checkout` the lower-level pool checkout event + for an individual DBAPI connection + + :meth:`.ConnectionEvents.set_connection_execution_options` - a copy of a + :class:`.Connection` is also made when the + :meth:`.Connection.execution_options` method is called. + + """ + + def set_connection_execution_options(self, conn, opts): + """Intercept when the :meth:`.Connection.execution_options` + method is called. + + This method is called after the new :class:`.Connection` has been + produced, with the newly updated execution options collection, but + before the :class:`.Dialect` has acted upon any of those new options. + + Note that this method is not called when a new :class:`.Connection` + is produced which is inheriting execution options from its parent + :class:`.Engine`; to intercept this condition, use the + :meth:`.ConnectionEvents.engine_connect` event. + + :param conn: The newly copied :class:`.Connection` object + + :param opts: dictionary of options that were passed to the + :meth:`.Connection.execution_options` method. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :meth:`.ConnectionEvents.set_engine_execution_options` - event + which is called when :meth:`.Engine.execution_options` is called. + + + """ + + def set_engine_execution_options(self, engine, opts): + """Intercept when the :meth:`.Engine.execution_options` + method is called. + + The :meth:`.Engine.execution_options` method produces a shallow + copy of the :class:`.Engine` which stores the new options. That new + :class:`.Engine` is passed here. A particular application of this + method is to add a :meth:`.ConnectionEvents.engine_connect` event + handler to the given :class:`.Engine` which will perform some per- + :class:`.Connection` task specific to these execution options. + + :param conn: The newly copied :class:`.Engine` object + + :param opts: dictionary of options that were passed to the + :meth:`.Connection.execution_options` method. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :meth:`.ConnectionEvents.set_connection_execution_options` - event + which is called when :meth:`.Connection.execution_options` is called. + + """ + def begin(self, conn): - """Intercept begin() events.""" + """Intercept begin() events. + + :param conn: :class:`.Connection` object + + """ def rollback(self, conn): - """Intercept rollback() events.""" + """Intercept rollback() events, as initiated by a + :class:`.Transaction`. + + Note that the :class:`.Pool` also "auto-rolls back" + a DBAPI connection upon checkin, if the ``reset_on_return`` + flag is set to its default value of ``'rollback'``. + To intercept this + rollback, use the :meth:`.PoolEvents.reset` hook. + + :param conn: :class:`.Connection` object + + .. seealso:: + + :meth:`.PoolEvents.reset` + + """ def commit(self, conn): - """Intercept commit() events.""" + """Intercept commit() events, as initiated by a + :class:`.Transaction`. - def savepoint(self, conn, name=None): - """Intercept savepoint() events.""" + Note that the :class:`.Pool` may also "auto-commit" + a DBAPI connection upon checkin, if the ``reset_on_return`` + flag is set to the value ``'commit'``. To intercept this + commit, use the :meth:`.PoolEvents.reset` hook. + + :param conn: :class:`.Connection` object + """ + + def savepoint(self, conn, name): + """Intercept savepoint() events. + + :param conn: :class:`.Connection` object + :param name: specified name used for the savepoint. + + """ def rollback_savepoint(self, conn, name, context): - """Intercept rollback_savepoint() events.""" + """Intercept rollback_savepoint() events. + + :param conn: :class:`.Connection` object + :param name: specified name used for the savepoint. + :param context: :class:`.ExecutionContext` in use. May be ``None``. + + """ def release_savepoint(self, conn, name, context): - """Intercept release_savepoint() events.""" + """Intercept release_savepoint() events. + + :param conn: :class:`.Connection` object + :param name: specified name used for the savepoint. + :param context: :class:`.ExecutionContext` in use. May be ``None``. + + """ def begin_twophase(self, conn, xid): - """Intercept begin_twophase() events.""" + """Intercept begin_twophase() events. + + :param conn: :class:`.Connection` object + :param xid: two-phase XID identifier + + """ def prepare_twophase(self, conn, xid): - """Intercept prepare_twophase() events.""" + """Intercept prepare_twophase() events. + + :param conn: :class:`.Connection` object + :param xid: two-phase XID identifier + """ def rollback_twophase(self, conn, xid, is_prepared): - """Intercept rollback_twophase() events.""" + """Intercept rollback_twophase() events. + + :param conn: :class:`.Connection` object + :param xid: two-phase XID identifier + :param is_prepared: boolean, indicates if + :meth:`.TwoPhaseTransaction.prepare` was called. + + """ def commit_twophase(self, conn, xid, is_prepared): - """Intercept commit_twophase() events.""" + """Intercept commit_twophase() events. + :param conn: :class:`.Connection` object + :param xid: two-phase XID identifier + :param is_prepared: boolean, indicates if + :meth:`.TwoPhaseTransaction.prepare` was called. + + """ diff --git a/libs/sqlalchemy/exc.py b/libs/sqlalchemy/exc.py index febee3fe..fe6879b1 100644 --- a/libs/sqlalchemy/exc.py +++ b/libs/sqlalchemy/exc.py @@ -1,19 +1,20 @@ # sqlalchemy/exc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """Exceptions used with SQLAlchemy. -The base exception class is :class:`.SQLAlchemyError`. Exceptions which are raised as a -result of DBAPI exceptions are all subclasses of -:class:`.DBAPIError`. +The base exception class is :exc:`.SQLAlchemyError`. Exceptions which are +raised as a result of DBAPI exceptions are all subclasses of +:exc:`.DBAPIError`. """ import traceback + class SQLAlchemyError(Exception): """Generic error class.""" @@ -26,6 +27,16 @@ class ArgumentError(SQLAlchemyError): """ +class NoForeignKeysError(ArgumentError): + """Raised when no foreign keys can be located between two selectables + during a join.""" + + +class AmbiguousForeignKeysError(ArgumentError): + """Raised when more than one foreign key matching can be located + between two selectables during a join.""" + + class CircularDependencyError(SQLAlchemyError): """Raised by topological sorts when a circular dependency is detected. @@ -57,30 +68,39 @@ class CircularDependencyError(SQLAlchemyError): return self.__class__, (None, self.cycles, self.edges, self.args[0]) + class CompileError(SQLAlchemyError): """Raised when an error occurs during SQL compilation""" +class UnsupportedCompilationError(CompileError): + """Raised when an operation is not supported by the given compiler. + + + .. versionadded:: 0.8.3 + + """ + + def __init__(self, compiler, element_type): + super(UnsupportedCompilationError, self).__init__( + "Compiler %r can't render element of type %s" % + (compiler, element_type)) + class IdentifierError(SQLAlchemyError): """Raised when a schema name is beyond the max character limit""" -# Moved to orm.exc; compatibility definition installed by orm import until 0.6 -ConcurrentModificationError = None class DisconnectionError(SQLAlchemyError): """A disconnect is detected on a raw DB-API connection. This error is raised and consumed internally by a connection pool. It can - be raised by the :meth:`.PoolEvents.checkout` event - so that the host pool forces a retry; the exception will be caught - three times in a row before the pool gives up and raises - :class:`~sqlalchemy.exc.InvalidRequestError` regarding the connection attempt. + be raised by the :meth:`.PoolEvents.checkout` event so that the host pool + forces a retry; the exception will be caught three times in a row before + the pool gives up and raises :class:`~sqlalchemy.exc.InvalidRequestError` + regarding the connection attempt. """ -# Moved to orm.exc; compatibility definition installed by orm import until 0.6 -FlushError = None - class TimeoutError(SQLAlchemyError): """Raised when a connection pool times out on getting a connection.""" @@ -92,19 +112,30 @@ class InvalidRequestError(SQLAlchemyError): """ + +class NoInspectionAvailable(InvalidRequestError): + """A subject passed to :func:`sqlalchemy.inspection.inspect` produced + no context for inspection.""" + + class ResourceClosedError(InvalidRequestError): """An operation was requested from a connection, cursor, or other object that's in a closed state.""" + class NoSuchColumnError(KeyError, InvalidRequestError): """A nonexistent column is requested from a ``RowProxy``.""" + class NoReferenceError(InvalidRequestError): """Raised by ``ForeignKey`` to indicate a reference cannot be resolved.""" -class NoReferencedTableError(NoReferenceError): - """Raised by ``ForeignKey`` when the referred ``Table`` cannot be located.""" +class NoReferencedTableError(NoReferenceError): + """Raised by ``ForeignKey`` when the referred ``Table`` cannot be + located. + + """ def __init__(self, message, tname): NoReferenceError.__init__(self, message) self.table_name = tname @@ -112,9 +143,12 @@ class NoReferencedTableError(NoReferenceError): def __reduce__(self): return self.__class__, (self.args[0], self.table_name) -class NoReferencedColumnError(NoReferenceError): - """Raised by ``ForeignKey`` when the referred ``Column`` cannot be located.""" +class NoReferencedColumnError(NoReferenceError): + """Raised by ``ForeignKey`` when the referred ``Column`` cannot be + located. + + """ def __init__(self, message, tname, cname): NoReferenceError.__init__(self, message) self.table_name = tname @@ -124,6 +158,7 @@ class NoReferencedColumnError(NoReferenceError): return self.__class__, (self.args[0], self.table_name, self.column_name) + class NoSuchTableError(InvalidRequestError): """Table does not exist or is not visible to a connection.""" @@ -134,10 +169,11 @@ class UnboundExecutionError(InvalidRequestError): class DontWrapMixin(object): """A mixin class which, when applied to a user-defined Exception class, - will not be wrapped inside of :class:`.StatementError` if the error is + will not be wrapped inside of :exc:`.StatementError` if the error is emitted within the process of executing a statement. E.g.:: + from sqlalchemy.exc import DontWrapMixin class MyCustomException(Exception, DontWrapMixin): @@ -151,14 +187,11 @@ class DontWrapMixin(object): raise MyCustomException("invalid!") """ -import sys -if sys.version_info < (2, 5): - class DontWrapMixin: - pass # Moved to orm.exc; compatibility definition installed by orm import until 0.6 UnmappedColumnError = None + class StatementError(SQLAlchemyError): """An error occurred during execution of a SQL statement. @@ -186,6 +219,10 @@ class StatementError(SQLAlchemyError): self.statement = statement self.params = params self.orig = orig + self.detail = [] + + def add_detail(self, msg): + self.detail.append(msg) def __reduce__(self): return self.__class__, (self.args[0], self.statement, @@ -194,8 +231,16 @@ class StatementError(SQLAlchemyError): def __str__(self): from sqlalchemy.sql import util params_repr = util._repr_params(self.params, 10) - return ' '.join((SQLAlchemyError.__str__(self), - repr(self.statement), repr(params_repr))) + + return ' '.join([ + "(%s)" % det for det in self.detail + ] + [ + SQLAlchemyError.__str__(self), + repr(self.statement), repr(params_repr) + ]) + + def __unicode__(self): + return self.__str__() class DBAPIError(StatementError): @@ -210,13 +255,14 @@ class DBAPIError(StatementError): raise the same exception type for any given error condition. :class:`DBAPIError` features :attr:`~.StatementError.statement` - and :attr:`~.StatementError.params` attributes which supply context regarding - the specifics of the statement which had an issue, for the + and :attr:`~.StatementError.params` attributes which supply context + regarding the specifics of the statement which had an issue, for the typical case when the error was raised within the context of emitting a SQL statement. - The wrapped exception object is available in the :attr:`~.StatementError.orig` attribute. - Its type and properties are DB-API implementation specific. + The wrapped exception object is available in the + :attr:`~.StatementError.orig` attribute. Its type and properties are + DB-API implementation specific. """ @@ -234,11 +280,12 @@ class DBAPIError(StatementError): # not a DBAPI error, statement is present. # raise a StatementError if not isinstance(orig, dbapi_base_err) and statement: + msg = traceback.format_exception_only( + orig.__class__, orig)[-1].strip() return StatementError( - "%s (original cause: %s)" % ( - str(orig), - traceback.format_exception_only(orig.__class__, orig)[-1].strip() - ), statement, params, orig) + "%s (original cause: %s)" % (str(orig), msg), + statement, params, orig + ) name, glob = orig.__class__.__name__, globals() if name in glob and issubclass(glob[name], DBAPIError): @@ -255,7 +302,7 @@ class DBAPIError(StatementError): text = str(orig) except (KeyboardInterrupt, SystemExit): raise - except Exception, e: + except Exception as e: text = 'Error in str() of DB-API-generated exception: ' + str(e) StatementError.__init__( self, diff --git a/libs/sqlalchemy/ext/__init__.py b/libs/sqlalchemy/ext/__init__.py index 4a6e1952..1d77acaa 100644 --- a/libs/sqlalchemy/ext/__init__.py +++ b/libs/sqlalchemy/ext/__init__.py @@ -1,6 +1,5 @@ # ext/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php - diff --git a/libs/sqlalchemy/ext/associationproxy.py b/libs/sqlalchemy/ext/associationproxy.py index d5b0ab69..e62958b4 100644 --- a/libs/sqlalchemy/ext/associationproxy.py +++ b/libs/sqlalchemy/ext/associationproxy.py @@ -1,5 +1,5 @@ # ext/associationproxy.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -15,11 +15,9 @@ See the example ``examples/association/proxied_association.py``. import itertools import operator import weakref -from sqlalchemy import exceptions -from sqlalchemy import orm -from sqlalchemy import util -from sqlalchemy.orm import collections, ColumnProperty -from sqlalchemy.sql import not_ +from .. import exc, orm, util +from ..orm import collections, interfaces +from ..sql import not_, or_ def association_proxy(target_collection, attr, **kw): @@ -29,24 +27,25 @@ def association_proxy(target_collection, attr, **kw): The returned value is an instance of :class:`.AssociationProxy`. - Implements a Python property representing a relationship as a collection of - simpler values, or a scalar value. The proxied property will mimic the collection type of - the target (list, dict or set), or, in the case of a one to one relationship, - a simple scalar value. + Implements a Python property representing a relationship as a collection + of simpler values, or a scalar value. The proxied property will mimic + the collection type of the target (list, dict or set), or, in the case of + a one to one relationship, a simple scalar value. :param target_collection: Name of the attribute we'll proxy to. This attribute is typically mapped by :func:`~sqlalchemy.orm.relationship` to link to a target collection, but can also be a many-to-one or non-scalar relationship. - :param attr: Attribute on the associated instance or instances we'll proxy for. + :param attr: Attribute on the associated instance or instances we'll + proxy for. For example, given a target collection of [obj1, obj2], a list created by this proxy property would look like [getattr(obj1, *attr*), getattr(obj2, *attr*)] - If the relationship is one-to-one or otherwise uselist=False, then simply: - getattr(obj, *attr*) + If the relationship is one-to-one or otherwise uselist=False, then + simply: getattr(obj, *attr*) :param creator: optional. @@ -76,9 +75,22 @@ def association_proxy(target_collection, attr, **kw): return AssociationProxy(target_collection, attr, **kw) -class AssociationProxy(object): +ASSOCIATION_PROXY = util.symbol('ASSOCIATION_PROXY') +"""Symbol indicating an :class:`_InspectionAttr` that's + of type :class:`.AssociationProxy`. + + Is assigned to the :attr:`._InspectionAttr.extension_type` + attibute. + +""" + +class AssociationProxy(interfaces._InspectionAttr): """A descriptor that presents a read/write view of an object attribute.""" + is_attribute = False + extension_type = ASSOCIATION_PROXY + + def __init__(self, target_collection, attr, creator=None, getset_factory=None, proxy_factory=None, proxy_bulk_set=None): @@ -91,34 +103,36 @@ class AssociationProxy(object): :param target_collection: Name of the collection we'll proxy to, usually created with :func:`.relationship`. - :param attr: Attribute on the collected instances we'll proxy for. For example, - given a target collection of [obj1, obj2], a list created by this - proxy property would look like [getattr(obj1, attr), getattr(obj2, - attr)] + :param attr: Attribute on the collected instances we'll proxy + for. For example, given a target collection of [obj1, obj2], a + list created by this proxy property would look like + [getattr(obj1, attr), getattr(obj2, attr)] - :param creator: Optional. When new items are added to this proxied collection, new - instances of the class collected by the target collection will be - created. For list and set collections, the target class constructor - will be called with the 'value' for the new instance. For dict - types, two arguments are passed: key and value. + :param creator: Optional. When new items are added to this proxied + collection, new instances of the class collected by the target + collection will be created. For list and set collections, the + target class constructor will be called with the 'value' for the + new instance. For dict types, two arguments are passed: + key and value. If you want to construct instances differently, supply a 'creator' function that takes arguments as above and returns instances. - :param getset_factory: Optional. Proxied attribute access is automatically handled by - routines that get and set values based on the `attr` argument for - this proxy. + :param getset_factory: Optional. Proxied attribute access is + automatically handled by routines that get and set values based on + the `attr` argument for this proxy. If you would like to customize this behavior, you may supply a `getset_factory` callable that produces a tuple of `getter` and `setter` functions. The factory is called with two arguments, the abstract type of the underlying collection and this proxy instance. - :param proxy_factory: Optional. The type of collection to emulate is determined by - sniffing the target collection. If your collection type can't be - determined by duck typing or you'd like to use a different - collection implementation, you may supply a factory function to - produce those collections. Only applicable to non-scalar relationships. + :param proxy_factory: Optional. The type of collection to emulate is + determined by sniffing the target collection. If your collection + type can't be determined by duck typing or you'd like to use a + different collection implementation, you may supply a factory + function to produce those collections. Only applicable to + non-scalar relationships. :param proxy_bulk_set: Optional, use with proxy_factory. See the _set() method for details. @@ -217,6 +231,10 @@ class AssociationProxy(object): return not self._get_property().\ mapper.get_property(self.value_attr).uselist + @util.memoized_property + def _target_is_object(self): + return getattr(self.target_class, self.value_attr).impl.uses_objects + def __get__(self, obj, class_): if self.owning_class is None: self.owning_class = class_ and class_ or type(obj) @@ -224,7 +242,11 @@ class AssociationProxy(object): return self if self.scalar: - return self._scalar_get(getattr(obj, self.target_collection)) + target = getattr(obj, self.target_collection) + if target is not None: + return self._scalar_get(target) + else: + return None else: try: # If the owning instance is reborn (orm session resurrect, @@ -281,7 +303,8 @@ class AssociationProxy(object): self.collection_class = util.duck_type_collection(lazy_collection()) if self.proxy_factory: - return self.proxy_factory(lazy_collection, creator, self.value_attr, self) + return self.proxy_factory( + lazy_collection, creator, self.value_attr, self) if self.getset_factory: getter, setter = self.getset_factory(self.collection_class, self) @@ -289,13 +312,16 @@ class AssociationProxy(object): getter, setter = self._default_getset(self.collection_class) if self.collection_class is list: - return _AssociationList(lazy_collection, creator, getter, setter, self) + return _AssociationList( + lazy_collection, creator, getter, setter, self) elif self.collection_class is dict: - return _AssociationDict(lazy_collection, creator, getter, setter, self) + return _AssociationDict( + lazy_collection, creator, getter, setter, self) elif self.collection_class is set: - return _AssociationSet(lazy_collection, creator, getter, setter, self) + return _AssociationSet( + lazy_collection, creator, getter, setter, self) else: - raise exceptions.ArgumentError( + raise exc.ArgumentError( 'could not guess which interface to use for ' 'collection_class "%s" backing "%s"; specify a ' 'proxy_factory and proxy_bulk_set manually' % @@ -323,7 +349,7 @@ class AssociationProxy(object): elif self.collection_class is set: proxy.update(values) else: - raise exceptions.ArgumentError( + raise exc.ArgumentError( 'no proxy_bulk_set supplied for custom ' 'collection_class implementation') @@ -342,9 +368,11 @@ class AssociationProxy(object): """ if self._value_is_scalar: - value_expr = getattr(self.target_class, self.value_attr).has(criterion, **kwargs) + value_expr = getattr( + self.target_class, self.value_attr).has(criterion, **kwargs) else: - value_expr = getattr(self.target_class, self.value_attr).any(criterion, **kwargs) + value_expr = getattr( + self.target_class, self.value_attr).any(criterion, **kwargs) # check _value_is_scalar here, otherwise # we're scalar->scalar - call .any() so that @@ -368,10 +396,17 @@ class AssociationProxy(object): """ - return self._comparator.has( + if self._target_is_object: + return self._comparator.has( getattr(self.target_class, self.value_attr).\ has(criterion, **kwargs) ) + else: + if criterion is not None or kwargs: + raise exc.ArgumentError( + "Non-empty has() not allowed for " + "column-targeted association proxy; use ==") + return self._comparator.has() def contains(self, obj): """Produce a proxied 'contains' expression using EXISTS. @@ -391,10 +426,21 @@ class AssociationProxy(object): return self._comparator.any(**{self.value_attr: obj}) def __eq__(self, obj): - return self._comparator.has(**{self.value_attr: obj}) + # note the has() here will fail for collections; eq_() + # is only allowed with a scalar. + if obj is None: + return or_( + self._comparator.has(**{self.value_attr: obj}), + self._comparator == None + ) + else: + return self._comparator.has(**{self.value_attr: obj}) def __ne__(self, obj): - return not_(self.__eq__(obj)) + # note the has() here will fail for collections; eq_() + # is only allowed with a scalar. + return self._comparator.has( + getattr(self.target_class, self.value_attr) != obj) class _lazy_collection(object): @@ -405,18 +451,19 @@ class _lazy_collection(object): def __call__(self): obj = self.ref() if obj is None: - raise exceptions.InvalidRequestError( + raise exc.InvalidRequestError( "stale association proxy, parent object has gone out of " "scope") return getattr(obj, self.target) def __getstate__(self): - return {'obj':self.ref(), 'target':self.target} + return {'obj': self.ref(), 'target': self.target} def __setstate__(self, state): self.ref = weakref.ref(state['obj']) self.target = state['target'] + class _AssociationCollection(object): def __init__(self, lazy_collection, creator, getter, setter, parent): """Constructs an _AssociationCollection. @@ -454,17 +501,20 @@ class _AssociationCollection(object): def __len__(self): return len(self.col) - def __nonzero__(self): + def __bool__(self): return bool(self.col) + __nonzero__ = __bool__ + def __getstate__(self): - return {'parent':self.parent, 'lazy_collection':self.lazy_collection} + return {'parent': self.parent, 'lazy_collection': self.lazy_collection} def __setstate__(self, state): self.parent = state['parent'] self.lazy_collection = state['lazy_collection'] self.parent._inflate(self) + class _AssociationList(_AssociationCollection): """Generic, converting, list-to-list proxy.""" @@ -492,7 +542,7 @@ class _AssociationList(_AssociationCollection): stop = index.stop step = index.step or 1 - rng = range(index.start or 0, stop, step) + rng = list(range(index.start or 0, stop, step)) if step == 1: for i in rng: del self[index.start] @@ -547,7 +597,7 @@ class _AssociationList(_AssociationCollection): def count(self, value): return sum([1 for _ in - itertools.ifilter(lambda v: v == value, iter(self))]) + util.itertools_filter(lambda v: v == value, iter(self))]) def extend(self, values): for v in values: @@ -646,14 +696,16 @@ class _AssociationList(_AssociationCollection): def __hash__(self): raise TypeError("%s objects are unhashable" % type(self).__name__) - for func_name, func in locals().items(): - if (util.callable(func) and func.func_name == func_name and + for func_name, func in list(locals().items()): + if (util.callable(func) and func.__name__ == func_name and not func.__doc__ and hasattr(list, func_name)): func.__doc__ = getattr(list, func_name).__doc__ del func_name, func _NotProvided = util.symbol('_NotProvided') + + class _AssociationDict(_AssociationCollection): """Generic, converting, dict-to-dict proxy.""" @@ -687,7 +739,7 @@ class _AssociationDict(_AssociationCollection): return key in self.col def __iter__(self): - return self.col.iterkeys() + return iter(self.col.keys()) def clear(self): self.col.clear() @@ -732,24 +784,27 @@ class _AssociationDict(_AssociationCollection): def keys(self): return self.col.keys() - def iterkeys(self): - return self.col.iterkeys() + if util.py2k: + def iteritems(self): + return ((key, self._get(self.col[key])) for key in self.col) - def values(self): - return [ self._get(member) for member in self.col.values() ] + def itervalues(self): + return (self._get(self.col[key]) for key in self.col) - def itervalues(self): - for key in self.col: - yield self._get(self.col[key]) - raise StopIteration + def iterkeys(self): + return self.col.iterkeys() - def items(self): - return [(k, self._get(self.col[k])) for k in self] + def values(self): + return [self._get(member) for member in self.col.values()] - def iteritems(self): - for key in self.col: - yield (key, self._get(self.col[key])) - raise StopIteration + def items(self): + return [(k, self._get(self.col[k])) for k in self] + else: + def items(self): + return ((key, self._get(self.col[key])) for key in self.col) + + def values(self): + return (self._get(self.col[key]) for key in self.col) def pop(self, key, default=_NotProvided): if default is _NotProvided: @@ -768,8 +823,8 @@ class _AssociationDict(_AssociationCollection): len(a)) elif len(a) == 1: seq_or_map = a[0] - # discern dict from sequence - took the advice - # from http://www.voidspace.org.uk/python/articles/duck_typing.shtml + # discern dict from sequence - took the advice from + # http://www.voidspace.org.uk/python/articles/duck_typing.shtml # still not perfect :( if hasattr(seq_or_map, 'keys'): for item in seq_or_map: @@ -792,8 +847,8 @@ class _AssociationDict(_AssociationCollection): def __hash__(self): raise TypeError("%s objects are unhashable" % type(self).__name__) - for func_name, func in locals().items(): - if (util.callable(func) and func.func_name == func_name and + for func_name, func in list(locals().items()): + if (util.callable(func) and func.__name__ == func_name and not func.__doc__ and hasattr(dict, func_name)): func.__doc__ = getattr(dict, func_name).__doc__ del func_name, func @@ -814,12 +869,14 @@ class _AssociationSet(_AssociationCollection): def __len__(self): return len(self.col) - def __nonzero__(self): + def __bool__(self): if self.col: return True else: return False + __nonzero__ = __bool__ + def __contains__(self, value): for member in self.col: # testlib.pragma exempt:__eq__ @@ -990,8 +1047,8 @@ class _AssociationSet(_AssociationCollection): def __hash__(self): raise TypeError("%s objects are unhashable" % type(self).__name__) - for func_name, func in locals().items(): - if (util.callable(func) and func.func_name == func_name and + for func_name, func in list(locals().items()): + if (util.callable(func) and func.__name__ == func_name and not func.__doc__ and hasattr(set, func_name)): func.__doc__ = getattr(set, func_name).__doc__ del func_name, func diff --git a/libs/sqlalchemy/ext/automap.py b/libs/sqlalchemy/ext/automap.py new file mode 100644 index 00000000..7a1512f6 --- /dev/null +++ b/libs/sqlalchemy/ext/automap.py @@ -0,0 +1,840 @@ +# ext/automap.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Define an extension to the :mod:`sqlalchemy.ext.declarative` system +which automatically generates mapped classes and relationships from a database +schema, typically though not necessarily one which is reflected. + +.. versionadded:: 0.9.1 Added :mod:`sqlalchemy.ext.automap`. + +.. note:: + + The :mod:`sqlalchemy.ext.automap` extension should be considered + **experimental** as of 0.9.1. Featureset and API stability is + not guaranteed at this time. + +It is hoped that the :class:`.AutomapBase` system provides a quick +and modernized solution to the problem that the very famous +`SQLSoup `_ +also tries to solve, that of generating a quick and rudimentary object +model from an existing database on the fly. By addressing the issue strictly +at the mapper configuration level, and integrating fully with existing +Declarative class techniques, :class:`.AutomapBase` seeks to provide +a well-integrated approach to the issue of expediently auto-generating ad-hoc +mappings. + + +Basic Use +========= + +The simplest usage is to reflect an existing database into a new model. +We create a new :class:`.AutomapBase` class in a similar manner as to how +we create a declarative base class, using :func:`.automap_base`. +We then call :meth:`.AutomapBase.prepare` on the resulting base class, +asking it to reflect the schema and produce mappings:: + + from sqlalchemy.ext.automap import automap_base + from sqlalchemy.orm import Session + from sqlalchemy import create_engine + + Base = automap_base() + + # engine, suppose it has two tables 'user' and 'address' set up + engine = create_engine("sqlite:///mydatabase.db") + + # reflect the tables + Base.prepare(engine, reflect=True) + + # mapped classes are now created with names by default + # matching that of the table name. + User = Base.classes.user + Address = Base.classes.address + + session = Session(engine) + + # rudimentary relationships are produced + session.add(Address(email_address="foo@bar.com", user=User(name="foo"))) + session.commit() + + # collection-based relationships are by default named "_collection" + print (u1.address_collection) + +Above, calling :meth:`.AutomapBase.prepare` while passing along the +:paramref:`.AutomapBase.prepare.reflect` parameter indicates that the +:meth:`.MetaData.reflect` method will be called on this declarative base +classes' :class:`.MetaData` collection; then, each viable +:class:`.Table` within the :class:`.MetaData` will get a new mapped class +generated automatically. The :class:`.ForeignKeyConstraint` objects which +link the various tables together will be used to produce new, bidirectional +:func:`.relationship` objects between classes. The classes and relationships +follow along a default naming scheme that we can customize. At this point, +our basic mapping consisting of related ``User`` and ``Address`` classes is ready +to use in the traditional way. + +Generating Mappings from an Existing MetaData +============================================= + +We can pass a pre-declared :class:`.MetaData` object to :func:`.automap_base`. +This object can be constructed in any way, including programmatically, from +a serialized file, or from itself being reflected using :meth:`.MetaData.reflect`. +Below we illustrate a combination of reflection and explicit table declaration:: + + from sqlalchemy import create_engine, MetaData, Table, Column, ForeignKey + engine = create_engine("sqlite:///mydatabase.db") + + # produce our own MetaData object + metadata = MetaData() + + # we can reflect it ourselves from a database, using options + # such as 'only' to limit what tables we look at... + metadata.reflect(engine, only=['user', 'address']) + + # ... or just define our own Table objects with it (or combine both) + Table('user_order', metadata, + Column('id', Integer, primary_key=True), + Column('user_id', ForeignKey('user.id')) + ) + + # we can then produce a set of mappings from this MetaData. + Base = automap_base(metadata=metadata) + + # calling prepare() just sets up mapped classes and relationships. + Base.prepare() + + # mapped classes are ready + User, Address, Order = Base.classes.user, Base.classes.address, Base.classes.user_order + +Specifying Classes Explcitly +============================ + +The :mod:`.sqlalchemy.ext.automap` extension allows classes to be defined +explicitly, in a way similar to that of the :class:`.DeferredReflection` class. +Classes that extend from :class:`.AutomapBase` act like regular declarative +classes, but are not immediately mapped after their construction, and are instead +mapped when we call :meth:`.AutomapBase.prepare`. The :meth:`.AutomapBase.prepare` +method will make use of the classes we've established based on the table name +we use. If our schema contains tables ``user`` and ``address``, we can define +one or both of the classes to be used:: + + from sqlalchemy.ext.automap import automap_base + from sqlalchemy import create_engine + + # automap base + Base = automap_base() + + # pre-declare User for the 'user' table + class User(Base): + __tablename__ = 'user' + + # override schema elements like Columns + user_name = Column('name', String) + + # override relationships too, if desired. + # we must use the same name that automap would use for the relationship, + # and also must refer to the class name that automap will generate + # for "address" + address_collection = relationship("address", collection_class=set) + + # reflect + engine = create_engine("sqlite:///mydatabase.db") + Base.prepare(engine, reflect=True) + + # we still have Address generated from the tablename "address", + # but User is the same as Base.classes.User now + + Address = Base.classes.address + + u1 = session.query(User).first() + print (u1.address_collection) + + # the backref is still there: + a1 = session.query(Address).first() + print (a1.user) + +Above, one of the more intricate details is that we illustrated overriding +one of the :func:`.relationship` objects that automap would have created. +To do this, we needed to make sure the names match up with what automap +would normally generate, in that the relationship name would be ``User.address_collection`` +and the name of the class referred to, from automap's perspective, is called +``address``, even though we are referring to it as ``Address`` within our usage +of this class. + +Overriding Naming Schemes +========================= + +:mod:`.sqlalchemy.ext.automap` is tasked with producing mapped classes and +relationship names based on a schema, which means it has decision points in how +these names are determined. These three decision points are provided using +functions which can be passed to the :meth:`.AutomapBase.prepare` method, and +are known as :func:`.classname_for_table`, +:func:`.name_for_scalar_relationship`, +and :func:`.name_for_collection_relationship`. Any or all of these +functions are provided as in the example below, where we use a "camel case" +scheme for class names and a "pluralizer" for collection names using the +`Inflect `_ package:: + + import re + import inflect + + def camelize_classname(base, tablename, table): + "Produce a 'camelized' class name, e.g. " + "'words_and_underscores' -> 'WordsAndUnderscores'" + + return str(tablename[0].upper() + \\ + re.sub(r'_(\w)', lambda m: m.group(1).upper(), tablename[1:])) + + _pluralizer = inflect.engine() + def pluralize_collection(base, local_cls, referred_cls, constraint): + "Produce an 'uncamelized', 'pluralized' class name, e.g. " + "'SomeTerm' -> 'some_terms'" + + referred_name = referred_cls.__name__ + uncamelized = referred_name[0].lower() + \\ + re.sub(r'\W', + lambda m: "_%s" % m.group(0).lower(), + referred_name[1:]) + pluralized = _pluralizer.plural(uncamelized) + return pluralized + + from sqlalchemy.ext.automap import automap_base + + Base = automap_base() + + engine = create_engine("sqlite:///mydatabase.db") + + Base.prepare(engine, reflect=True, + classname_for_table=camelize_classname, + name_for_collection_relationship=pluralize_collection + ) + +From the above mapping, we would now have classes ``User`` and ``Address``, +where the collection from ``User`` to ``Address`` is called ``User.addresses``:: + + User, Address = Base.classes.User, Base.classes.Address + + u1 = User(addresses=[Address(email="foo@bar.com")]) + +Relationship Detection +====================== + +The vast majority of what automap accomplishes is the generation of +:func:`.relationship` structures based on foreign keys. The mechanism +by which this works for many-to-one and one-to-many relationships is as follows: + +1. A given :class:`.Table`, known to be mapped to a particular class, + is examined for :class:`.ForeignKeyConstraint` objects. + +2. From each :class:`.ForeignKeyConstraint`, the remote :class:`.Table` + object present is matched up to the class to which it is to be mapped, + if any, else it is skipped. + +3. As the :class:`.ForeignKeyConstraint` we are examining correponds to a reference + from the immediate mapped class, + the relationship will be set up as a many-to-one referring to the referred class; + a corresponding one-to-many backref will be created on the referred class referring + to this class. + +4. The names of the relationships are determined using the + :paramref:`.AutomapBase.prepare.name_for_scalar_relationship` and + :paramref:`.AutomapBase.prepare.name_for_collection_relationship` + callable functions. It is important to note that the default relationship + naming derives the name from the **the actual class name**. If you've + given a particular class an explicit name by declaring it, or specified an + alternate class naming scheme, that's the name from which the relationship + name will be derived. + +5. The classes are inspected for an existing mapped property matching these + names. If one is detected on one side, but none on the other side, :class:`.AutomapBase` + attempts to create a relationship on the missing side, then uses the + :paramref:`.relationship.back_populates` parameter in order to point + the new relationship to the other side. + +6. In the usual case where no relationship is on either side, + :meth:`.AutomapBase.prepare` produces a :func:`.relationship` on the "many-to-one" + side and matches it to the other using the :paramref:`.relationship.backref` + parameter. + +7. Production of the :func:`.relationship` and optionally the :func:`.backref` + is handed off to the :paramref:`.AutomapBase.prepare.generate_relationship` + function, which can be supplied by the end-user in order to augment + the arguments passed to :func:`.relationship` or :func:`.backref` or to + make use of custom implementations of these functions. + +Custom Relationship Arguments +----------------------------- + +The :paramref:`.AutomapBase.prepare.generate_relationship` hook can be used +to add parameters to relationships. For most cases, we can make use of the +existing :func:`.automap.generate_relationship` function to return +the object, after augmenting the given keyword dictionary with our own +arguments. + +Below is an illustration of how to send +:paramref:`.relationship.cascade` and +:paramref:`.relationship.passive_deletes` +options along to all one-to-many relationships:: + + from sqlalchemy.ext.automap import generate_relationship + + def _gen_relationship(base, direction, return_fn, + attrname, local_cls, referred_cls, **kw): + if direction is interfaces.ONETOMANY: + kw['cascade'] = 'all, delete-orphan' + kw['passive_deletes'] = True + # make use of the built-in function to actually return + # the result. + return generate_relationship(base, direction, return_fn, + attrname, local_cls, referred_cls, **kw) + + from sqlalchemy.ext.automap import automap_base + from sqlalchemy import create_engine + + # automap base + Base = automap_base() + + engine = create_engine("sqlite:///mydatabase.db") + Base.prepare(engine, reflect=True, + generate_relationship=_gen_relationship) + +Many-to-Many relationships +-------------------------- + +:mod:`.sqlalchemy.ext.automap` will generate many-to-many relationships, e.g. +those which contain a ``secondary`` argument. The process for producing these +is as follows: + +1. A given :class:`.Table` is examined for :class:`.ForeignKeyConstraint` objects, + before any mapped class has been assigned to it. + +2. If the table contains two and exactly two :class:`.ForeignKeyConstraint` + objects, and all columns within this table are members of these two + :class:`.ForeignKeyConstraint` objects, the table is assumed to be a + "secondary" table, and will **not be mapped directly**. + +3. The two (or one, for self-referential) external tables to which the :class:`.Table` + refers to are matched to the classes to which they will be mapped, if any. + +4. If mapped classes for both sides are located, a many-to-many bi-directional + :func:`.relationship` / :func:`.backref` pair is created between the two + classes. + +5. The override logic for many-to-many works the same as that of one-to-many/ + many-to-one; the :func:`.generate_relationship` function is called upon + to generate the strucures and existing attributes will be maintained. + +Using Automap with Explicit Declarations +======================================== + +As noted previously, automap has no dependency on reflection, and can make +use of any collection of :class:`.Table` objects within a :class:`.MetaData` +collection. From this, it follows that automap can also be used +generate missing relationships given an otherwise complete model that fully defines +table metadata:: + + from sqlalchemy.ext.automap import automap_base + from sqlalchemy import Column, Integer, String, ForeignKey + + Base = automap_base() + + class User(Base): + __tablename__ = 'user' + + id = Column(Integer, primary_key=True) + name = Column(String) + + class Address(Base): + __tablename__ = 'address' + + id = Column(Integer, primary_key=True) + email = Column(String) + user_id = Column(ForeignKey('user.id')) + + # produce relationships + Base.prepare() + + # mapping is complete, with "address_collection" and + # "user" relationships + a1 = Address(email='u1') + a2 = Address(email='u2') + u1 = User(address_collection=[a1, a2]) + assert a1.user is u1 + +Above, given mostly complete ``User`` and ``Address`` mappings, the +:class:`.ForeignKey` which we defined on ``Address.user_id`` allowed a +bidirectional relationship pair ``Address.user`` and ``User.address_collection`` +to be generated on the mapped classes. + +Note that when subclassing :class:`.AutomapBase`, the :meth:`.AutomapBase.prepare` +method is required; if not called, the classes we've declared are in an +un-mapped state. + + +""" +from .declarative import declarative_base as _declarative_base +from .declarative.base import _DeferredMapperConfig +from ..sql import and_ +from ..schema import ForeignKeyConstraint +from ..orm import relationship, backref, interfaces +from .. import util + + +def classname_for_table(base, tablename, table): + """Return the class name that should be used, given the name + of a table. + + The default implementation is:: + + return str(tablename) + + Alternate implementations can be specified using the + :paramref:`.AutomapBase.prepare.classname_for_table` + parameter. + + :param base: the :class:`.AutomapBase` class doing the prepare. + + :param tablename: string name of the :class:`.Table`. + + :param table: the :class:`.Table` object itself. + + :return: a string class name. + + .. note:: + + In Python 2, the string used for the class name **must** be a non-Unicode + object, e.g. a ``str()`` object. The ``.name`` attribute of + :class:`.Table` is typically a Python unicode subclass, so the ``str()`` + function should be applied to this name, after accounting for any non-ASCII + characters. + + """ + return str(tablename) + +def name_for_scalar_relationship(base, local_cls, referred_cls, constraint): + """Return the attribute name that should be used to refer from one + class to another, for a scalar object reference. + + The default implementation is:: + + return referred_cls.__name__.lower() + + Alternate implementations can be specified using the + :paramref:`.AutomapBase.prepare.name_for_scalar_relationship` + parameter. + + :param base: the :class:`.AutomapBase` class doing the prepare. + + :param local_cls: the class to be mapped on the local side. + + :param referred_cls: the class to be mapped on the referring side. + + :param constraint: the :class:`.ForeignKeyConstraint` that is being + inspected to produce this relationship. + + """ + return referred_cls.__name__.lower() + +def name_for_collection_relationship(base, local_cls, referred_cls, constraint): + """Return the attribute name that should be used to refer from one + class to another, for a collection reference. + + The default implementation is:: + + return referred_cls.__name__.lower() + "_collection" + + Alternate implementations + can be specified using the :paramref:`.AutomapBase.prepare.name_for_collection_relationship` + parameter. + + :param base: the :class:`.AutomapBase` class doing the prepare. + + :param local_cls: the class to be mapped on the local side. + + :param referred_cls: the class to be mapped on the referring side. + + :param constraint: the :class:`.ForeignKeyConstraint` that is being + inspected to produce this relationship. + + """ + return referred_cls.__name__.lower() + "_collection" + +def generate_relationship(base, direction, return_fn, attrname, local_cls, referred_cls, **kw): + """Generate a :func:`.relationship` or :func:`.backref` on behalf of two + mapped classes. + + An alternate implementation of this function can be specified using the + :paramref:`.AutomapBase.prepare.generate_relationship` parameter. + + The default implementation of this function is as follows:: + + if return_fn is backref: + return return_fn(attrname, **kw) + elif return_fn is relationship: + return return_fn(referred_cls, **kw) + else: + raise TypeError("Unknown relationship function: %s" % return_fn) + + :param base: the :class:`.AutomapBase` class doing the prepare. + + :param direction: indicate the "direction" of the relationship; this will + be one of :data:`.ONETOMANY`, :data:`.MANYTOONE`, :data:`.MANYTOONE`. + + :param return_fn: the function that is used by default to create the + relationship. This will be either :func:`.relationship` or :func:`.backref`. + The :func:`.backref` function's result will be used to produce a new + :func:`.relationship` in a second step, so it is critical that user-defined + implementations correctly differentiate between the two functions, if + a custom relationship function is being used. + + :attrname: the attribute name to which this relationship is being assigned. + If the value of :paramref:`.generate_relationship.return_fn` is the + :func:`.backref` function, then this name is the name that is being + assigned to the backref. + + :param local_cls: the "local" class to which this relationship or backref + will be locally present. + + :param referred_cls: the "referred" class to which the relationship or backref + refers to. + + :param \**kw: all additional keyword arguments are passed along to the + function. + + :return: a :func:`.relationship` or :func:`.backref` construct, as dictated + by the :paramref:`.generate_relationship.return_fn` parameter. + + """ + if return_fn is backref: + return return_fn(attrname, **kw) + elif return_fn is relationship: + return return_fn(referred_cls, **kw) + else: + raise TypeError("Unknown relationship function: %s" % return_fn) + +class AutomapBase(object): + """Base class for an "automap" schema. + + The :class:`.AutomapBase` class can be compared to the "declarative base" + class that is produced by the :func:`.declarative.declarative_base` + function. In practice, the :class:`.AutomapBase` class is always used + as a mixin along with an actual declarative base. + + A new subclassable :class:`.AutomapBase` is typically instantated + using the :func:`.automap_base` function. + + .. seealso:: + + :ref:`automap_toplevel` + + """ + __abstract__ = True + + classes = None + """An instance of :class:`.util.Properties` containing classes. + + This object behaves much like the ``.c`` collection on a table. Classes + are present under the name they were given, e.g.:: + + Base = automap_base() + Base.prepare(engine=some_engine, reflect=True) + + User, Address = Base.classes.User, Base.classes.Address + + """ + + @classmethod + def prepare(cls, + engine=None, + reflect=False, + classname_for_table=classname_for_table, + collection_class=list, + name_for_scalar_relationship=name_for_scalar_relationship, + name_for_collection_relationship=name_for_collection_relationship, + generate_relationship=generate_relationship): + + """Extract mapped classes and relationships from the :class:`.MetaData` and + perform mappings. + + :param engine: an :class:`.Engine` or :class:`.Connection` with which + to perform schema reflection, if specified. + If the :paramref:`.AutomapBase.prepare.reflect` argument is False, this + object is not used. + + :param reflect: if True, the :meth:`.MetaData.reflect` method is called + on the :class:`.MetaData` associated with this :class:`.AutomapBase`. + The :class:`.Engine` passed via :paramref:`.AutomapBase.prepare.engine` will + be used to perform the reflection if present; else, the :class:`.MetaData` + should already be bound to some engine else the operation will fail. + + :param classname_for_table: callable function which will be used to + produce new class names, given a table name. Defaults to + :func:`.classname_for_table`. + + :param name_for_scalar_relationship: callable function which will be used + to produce relationship names for scalar relationships. Defaults to + :func:`.name_for_scalar_relationship`. + + :param name_for_collection_relationship: callable function which will be used + to produce relationship names for collection-oriented relationships. Defaults to + :func:`.name_for_collection_relationship`. + + :param generate_relationship: callable function which will be used to + actually generate :func:`.relationship` and :func:`.backref` constructs. + Defaults to :func:`.generate_relationship`. + + :param collection_class: the Python collection class that will be used + when a new :func:`.relationship` object is created that represents a + collection. Defaults to ``list``. + + """ + if reflect: + cls.metadata.reflect( + engine, + extend_existing=True, + autoload_replace=False + ) + + table_to_map_config = dict( + (m.local_table, m) + for m in _DeferredMapperConfig.classes_for_base(cls) + ) + + many_to_many = [] + + for table in cls.metadata.tables.values(): + lcl_m2m, rem_m2m, m2m_const = _is_many_to_many(cls, table) + if lcl_m2m is not None: + many_to_many.append((lcl_m2m, rem_m2m, m2m_const, table)) + elif not table.primary_key: + continue + elif table not in table_to_map_config: + mapped_cls = type( + classname_for_table(cls, table.name, table), + (cls, ), + {"__table__": table} + ) + map_config = _DeferredMapperConfig.config_for_cls(mapped_cls) + cls.classes[map_config.cls.__name__] = mapped_cls + table_to_map_config[table] = map_config + + for map_config in table_to_map_config.values(): + _relationships_for_fks(cls, + map_config, + table_to_map_config, + collection_class, + name_for_scalar_relationship, + name_for_collection_relationship, + generate_relationship) + + for lcl_m2m, rem_m2m, m2m_const, table in many_to_many: + _m2m_relationship(cls, lcl_m2m, rem_m2m, m2m_const, table, + table_to_map_config, + collection_class, + name_for_scalar_relationship, + name_for_collection_relationship, + generate_relationship) + for map_config in table_to_map_config.values(): + map_config.map() + + + _sa_decl_prepare = True + """Indicate that the mapping of classes should be deferred. + + The presence of this attribute name indicates to declarative + that the call to mapper() should not occur immediately; instead, + information about the table and attributes to be mapped are gathered + into an internal structure called _DeferredMapperConfig. These + objects can be collected later using classes_for_base(), additional + mapping decisions can be made, and then the map() method will actually + apply the mapping. + + The only real reason this deferral of the whole + thing is needed is to support primary key columns that aren't reflected + yet when the class is declared; everything else can theoretically be + added to the mapper later. However, the _DeferredMapperConfig is a + nice interface in any case which exists at that not usually exposed point + at which declarative has the class and the Table but hasn't called + mapper() yet. + + """ + +def automap_base(declarative_base=None, **kw): + """Produce a declarative automap base. + + This function produces a new base class that is a product of the + :class:`.AutomapBase` class as well a declarative base produced by + :func:`.declarative.declarative_base`. + + All parameters other than ``declarative_base`` are keyword arguments + that are passed directly to the :func:`.declarative.declarative_base` + function. + + :param declarative_base: an existing class produced by + :func:`.declarative.declarative_base`. When this is passed, the function + no longer invokes :func:`.declarative.declarative_base` itself, and all other + keyword arguments are ignored. + + :param \**kw: keyword arguments are passed along to + :func:`.declarative.declarative_base`. + + """ + if declarative_base is None: + Base = _declarative_base(**kw) + else: + Base = declarative_base + + return type( + Base.__name__, + (AutomapBase, Base,), + {"__abstract__": True, "classes": util.Properties({})} + ) + +def _is_many_to_many(automap_base, table): + fk_constraints = [const for const in table.constraints + if isinstance(const, ForeignKeyConstraint)] + if len(fk_constraints) != 2: + return None, None, None + + cols = sum( + [[fk.parent for fk in fk_constraint.elements] + for fk_constraint in fk_constraints], []) + + if set(cols) != set(table.c): + return None, None, None + + return ( + fk_constraints[0].elements[0].column.table, + fk_constraints[1].elements[0].column.table, + fk_constraints + ) + +def _relationships_for_fks(automap_base, map_config, table_to_map_config, + collection_class, + name_for_scalar_relationship, + name_for_collection_relationship, + generate_relationship): + local_table = map_config.local_table + local_cls = map_config.cls + + for constraint in local_table.constraints: + if isinstance(constraint, ForeignKeyConstraint): + fks = constraint.elements + referred_table = fks[0].column.table + referred_cfg = table_to_map_config.get(referred_table, None) + if referred_cfg is None: + continue + referred_cls = referred_cfg.cls + + relationship_name = name_for_scalar_relationship( + automap_base, + local_cls, + referred_cls, constraint) + backref_name = name_for_collection_relationship( + automap_base, + referred_cls, + local_cls, + constraint + ) + + create_backref = backref_name not in referred_cfg.properties + + if relationship_name not in map_config.properties: + if create_backref: + backref_obj = generate_relationship(automap_base, + interfaces.ONETOMANY, backref, + backref_name, referred_cls, local_cls, + collection_class=collection_class) + else: + backref_obj = None + map_config.properties[relationship_name] = \ + generate_relationship(automap_base, + interfaces.MANYTOONE, + relationship, + relationship_name, + local_cls, referred_cls, + foreign_keys=[fk.parent for fk in constraint.elements], + backref=backref_obj, + remote_side=[fk.column for fk in constraint.elements] + ) + if not create_backref: + referred_cfg.properties[backref_name].back_populates = relationship_name + elif create_backref: + referred_cfg.properties[backref_name] = \ + generate_relationship(automap_base, + interfaces.ONETOMANY, + relationship, + backref_name, + referred_cls, local_cls, + foreign_keys=[fk.parent for fk in constraint.elements], + back_populates=relationship_name, + collection_class=collection_class) + map_config.properties[relationship_name].back_populates = backref_name + +def _m2m_relationship(automap_base, lcl_m2m, rem_m2m, m2m_const, table, + table_to_map_config, + collection_class, + name_for_scalar_relationship, + name_for_collection_relationship, + generate_relationship): + + map_config = table_to_map_config.get(lcl_m2m, None) + referred_cfg = table_to_map_config.get(rem_m2m, None) + if map_config is None or referred_cfg is None: + return + + local_cls = map_config.cls + referred_cls = referred_cfg.cls + + relationship_name = name_for_collection_relationship( + automap_base, + local_cls, + referred_cls, m2m_const[0]) + backref_name = name_for_collection_relationship( + automap_base, + referred_cls, + local_cls, + m2m_const[1] + ) + + create_backref = backref_name not in referred_cfg.properties + + if relationship_name not in map_config.properties: + if create_backref: + backref_obj = generate_relationship(automap_base, + interfaces.MANYTOMANY, + backref, + backref_name, + referred_cls, local_cls, + collection_class=collection_class + ) + else: + backref_obj = None + map_config.properties[relationship_name] = \ + generate_relationship(automap_base, + interfaces.MANYTOMANY, + relationship, + relationship_name, + local_cls, referred_cls, + secondary=table, + primaryjoin=and_(fk.column == fk.parent for fk in m2m_const[0].elements), + secondaryjoin=and_(fk.column == fk.parent for fk in m2m_const[1].elements), + backref=backref_obj, + collection_class=collection_class + ) + if not create_backref: + referred_cfg.properties[backref_name].back_populates = relationship_name + elif create_backref: + referred_cfg.properties[backref_name] = \ + generate_relationship(automap_base, + interfaces.MANYTOMANY, + relationship, + backref_name, + referred_cls, local_cls, + secondary=table, + primaryjoin=and_(fk.column == fk.parent for fk in m2m_const[1].elements), + secondaryjoin=and_(fk.column == fk.parent for fk in m2m_const[0].elements), + back_populates=relationship_name, + collection_class=collection_class) + map_config.properties[relationship_name].back_populates = backref_name diff --git a/libs/sqlalchemy/ext/compiler.py b/libs/sqlalchemy/ext/compiler.py index 9bd9b42e..5dde74e0 100644 --- a/libs/sqlalchemy/ext/compiler.py +++ b/libs/sqlalchemy/ext/compiler.py @@ -1,5 +1,5 @@ # ext/compiler.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -9,8 +9,9 @@ Synopsis ======== -Usage involves the creation of one or more :class:`~sqlalchemy.sql.expression.ClauseElement` -subclasses and one or more callables defining its compilation:: +Usage involves the creation of one or more +:class:`~sqlalchemy.sql.expression.ClauseElement` subclasses and one or +more callables defining its compilation:: from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.expression import ColumnClause @@ -58,16 +59,18 @@ invoked for the dialect in use:: def visit_alter_column(element, compiler, **kw): return "ALTER TABLE %s ALTER COLUMN %s ..." % (element.table.name, element.column.name) -The second ``visit_alter_table`` will be invoked when any ``postgresql`` dialect is used. +The second ``visit_alter_table`` will be invoked when any ``postgresql`` +dialect is used. Compiling sub-elements of a custom expression construct ======================================================= -The ``compiler`` argument is the :class:`~sqlalchemy.engine.base.Compiled` -object in use. This object can be inspected for any information about the -in-progress compilation, including ``compiler.dialect``, -``compiler.statement`` etc. The :class:`~sqlalchemy.sql.compiler.SQLCompiler` -and :class:`~sqlalchemy.sql.compiler.DDLCompiler` both include a ``process()`` +The ``compiler`` argument is the +:class:`~sqlalchemy.engine.interfaces.Compiled` object in use. This object +can be inspected for any information about the in-progress compilation, +including ``compiler.dialect``, ``compiler.statement`` etc. The +:class:`~sqlalchemy.sql.compiler.SQLCompiler` and +:class:`~sqlalchemy.sql.compiler.DDLCompiler` both include a ``process()`` method which can be used for compilation of embedded attributes:: from sqlalchemy.sql.expression import Executable, ClauseElement @@ -91,6 +94,12 @@ Produces:: "INSERT INTO mytable (SELECT mytable.x, mytable.y, mytable.z FROM mytable WHERE mytable.x > :x_1)" +.. note:: + + The above ``InsertFromSelect`` construct is only an example, this actual + functionality is already available using the + :meth:`.Insert.from_select` method. + .. note:: The above ``InsertFromSelect`` construct probably wants to have "autocommit" @@ -99,10 +108,11 @@ Produces:: Cross Compiling between SQL and DDL compilers --------------------------------------------- -SQL and DDL constructs are each compiled using different base compilers - ``SQLCompiler`` -and ``DDLCompiler``. A common need is to access the compilation rules of SQL expressions -from within a DDL expression. The ``DDLCompiler`` includes an accessor ``sql_compiler`` for this reason, such as below where we generate a CHECK -constraint that embeds a SQL expression:: +SQL and DDL constructs are each compiled using different base compilers - +``SQLCompiler`` and ``DDLCompiler``. A common need is to access the +compilation rules of SQL expressions from within a DDL expression. The +``DDLCompiler`` includes an accessor ``sql_compiler`` for this reason, such as +below where we generate a CHECK constraint that embeds a SQL expression:: @compiles(MyConstraint) def compile_my_constraint(constraint, ddlcompiler, **kw): @@ -116,20 +126,22 @@ constraint that embeds a SQL expression:: Enabling Autocommit on a Construct ================================== -Recall from the section :ref:`autocommit` that the :class:`.Engine`, when asked to execute -a construct in the absence of a user-defined transaction, detects if the given -construct represents DML or DDL, that is, a data modification or data definition statement, which -requires (or may require, in the case of DDL) that the transaction generated by the DBAPI be committed -(recall that DBAPI always has a transaction going on regardless of what SQLAlchemy does). Checking -for this is actually accomplished -by checking for the "autocommit" execution option on the construct. When building a construct like -an INSERT derivation, a new DDL type, or perhaps a stored procedure that alters data, the "autocommit" -option needs to be set in order for the statement to function with "connectionless" execution +Recall from the section :ref:`autocommit` that the :class:`.Engine`, when +asked to execute a construct in the absence of a user-defined transaction, +detects if the given construct represents DML or DDL, that is, a data +modification or data definition statement, which requires (or may require, +in the case of DDL) that the transaction generated by the DBAPI be committed +(recall that DBAPI always has a transaction going on regardless of what +SQLAlchemy does). Checking for this is actually accomplished by checking for +the "autocommit" execution option on the construct. When building a +construct like an INSERT derivation, a new DDL type, or perhaps a stored +procedure that alters data, the "autocommit" option needs to be set in order +for the statement to function with "connectionless" execution (as described in :ref:`dbengine_implicit`). -Currently a quick way to do this is to subclass :class:`.Executable`, then add the "autocommit" flag -to the ``_execution_options`` dictionary (note this is a "frozen" dictionary which supplies a generative -``union()`` method):: +Currently a quick way to do this is to subclass :class:`.Executable`, then +add the "autocommit" flag to the ``_execution_options`` dictionary (note this +is a "frozen" dictionary which supplies a generative ``union()`` method):: from sqlalchemy.sql.expression import Executable, ClauseElement @@ -137,8 +149,9 @@ to the ``_execution_options`` dictionary (note this is a "frozen" dictionary whi _execution_options = \\ Executable._execution_options.union({'autocommit': True}) -More succinctly, if the construct is truly similar to an INSERT, UPDATE, or DELETE, :class:`.UpdateBase` -can be used, which already is a subclass of :class:`.Executable`, :class:`.ClauseElement` and includes the +More succinctly, if the construct is truly similar to an INSERT, UPDATE, or +DELETE, :class:`.UpdateBase` can be used, which already is a subclass +of :class:`.Executable`, :class:`.ClauseElement` and includes the ``autocommit`` flag:: from sqlalchemy.sql.expression import UpdateBase @@ -150,7 +163,8 @@ can be used, which already is a subclass of :class:`.Executable`, :class:`.Claus -DDL elements that subclass :class:`.DDLElement` already have the "autocommit" flag turned on. +DDL elements that subclass :class:`.DDLElement` already have the +"autocommit" flag turned on. @@ -158,13 +172,16 @@ DDL elements that subclass :class:`.DDLElement` already have the "autocommit" fl Changing the default compilation of existing constructs ======================================================= -The compiler extension applies just as well to the existing constructs. When overriding -the compilation of a built in SQL construct, the @compiles decorator is invoked upon -the appropriate class (be sure to use the class, i.e. ``Insert`` or ``Select``, instead of the creation function such as ``insert()`` or ``select()``). +The compiler extension applies just as well to the existing constructs. When +overriding the compilation of a built in SQL construct, the @compiles +decorator is invoked upon the appropriate class (be sure to use the class, +i.e. ``Insert`` or ``Select``, instead of the creation function such +as ``insert()`` or ``select()``). -Within the new compilation function, to get at the "original" compilation routine, -use the appropriate visit_XXX method - this because compiler.process() will call upon the -overriding routine and cause an endless loop. Such as, to add "prefix" to all insert statements:: +Within the new compilation function, to get at the "original" compilation +routine, use the appropriate visit_XXX method - this +because compiler.process() will call upon the overriding routine and cause +an endless loop. Such as, to add "prefix" to all insert statements:: from sqlalchemy.sql.expression import Insert @@ -172,14 +189,16 @@ overriding routine and cause an endless loop. Such as, to add "prefix" to all def prefix_inserts(insert, compiler, **kw): return compiler.visit_insert(insert.prefix_with("some prefix"), **kw) -The above compiler will prefix all INSERT statements with "some prefix" when compiled. +The above compiler will prefix all INSERT statements with "some prefix" when +compiled. .. _type_compilation_extension: Changing Compilation of Types ============================= -``compiler`` works for types, too, such as below where we implement the MS-SQL specific 'max' keyword for ``String``/``VARCHAR``:: +``compiler`` works for types, too, such as below where we implement the +MS-SQL specific 'max' keyword for ``String``/``VARCHAR``:: @compiles(String, 'mssql') @compiles(VARCHAR, 'mssql') @@ -219,7 +238,7 @@ A synopsis is as follows: class timestamp(ColumnElement): type = TIMESTAMP() -* :class:`~sqlalchemy.sql.expression.FunctionElement` - This is a hybrid of a +* :class:`~sqlalchemy.sql.functions.FunctionElement` - This is a hybrid of a ``ColumnElement`` and a "from clause" like object, and represents a SQL function or stored procedure type of call. Since most databases support statements along the line of "SELECT FROM " @@ -248,10 +267,10 @@ A synopsis is as follows: ``execute_at()`` method, allowing the construct to be invoked during CREATE TABLE and DROP TABLE sequences. -* :class:`~sqlalchemy.sql.expression.Executable` - This is a mixin which should be - used with any expression class that represents a "standalone" SQL statement that - can be passed directly to an ``execute()`` method. It is already implicit - within ``DDLElement`` and ``FunctionElement``. +* :class:`~sqlalchemy.sql.expression.Executable` - This is a mixin which + should be used with any expression class that represents a "standalone" + SQL statement that can be passed directly to an ``execute()`` method. It + is already implicit within ``DDLElement`` and ``FunctionElement``. Further Examples ================ @@ -259,12 +278,13 @@ Further Examples "UTC timestamp" function ------------------------- -A function that works like "CURRENT_TIMESTAMP" except applies the appropriate conversions -so that the time is in UTC time. Timestamps are best stored in relational databases -as UTC, without time zones. UTC so that your database doesn't think time has gone -backwards in the hour when daylight savings ends, without timezones because timezones -are like character encodings - they're best applied only at the endpoints of an -application (i.e. convert to UTC upon user input, re-apply desired timezone upon display). +A function that works like "CURRENT_TIMESTAMP" except applies the +appropriate conversions so that the time is in UTC time. Timestamps are best +stored in relational databases as UTC, without time zones. UTC so that your +database doesn't think time has gone backwards in the hour when daylight +savings ends, without timezones because timezones are like character +encodings - they're best applied only at the endpoints of an application +(i.e. convert to UTC upon user input, re-apply desired timezone upon display). For Postgresql and Microsoft SQL Server:: @@ -298,10 +318,10 @@ Example usage:: "GREATEST" function ------------------- -The "GREATEST" function is given any number of arguments and returns the one that is -of the highest value - it's equivalent to Python's ``max`` function. A SQL -standard version versus a CASE based version which only accommodates two -arguments:: +The "GREATEST" function is given any number of arguments and returns the one +that is of the highest value - it's equivalent to Python's ``max`` +function. A SQL standard version versus a CASE based version which only +accommodates two arguments:: from sqlalchemy.sql import expression from sqlalchemy.ext.compiler import compiles @@ -339,7 +359,8 @@ Example usage:: "false" expression ------------------ -Render a "false" constant expression, rendering as "0" on platforms that don't have a "false" constant:: +Render a "false" constant expression, rendering as "0" on platforms that +don't have a "false" constant:: from sqlalchemy.sql import expression from sqlalchemy.ext.compiler import compiles @@ -367,9 +388,14 @@ Example usage:: ) """ -from sqlalchemy import exc +from .. import exc +from ..sql import visitors + def compiles(class_, *specs): + """Register a function as a compiler for a + given :class:`.ClauseElement` type.""" + def decorate(fn): existing = class_.__dict__.get('_compiler_dispatcher', None) existing_dispatch = class_.__dict__.get('_compiler_dispatch') @@ -380,7 +406,8 @@ def compiles(class_, *specs): existing.specs['default'] = existing_dispatch # TODO: why is the lambda needed ? - setattr(class_, '_compiler_dispatch', lambda *arg, **kw: existing(*arg, **kw)) + setattr(class_, '_compiler_dispatch', + lambda *arg, **kw: existing(*arg, **kw)) setattr(class_, '_compiler_dispatcher', existing) if specs: @@ -392,6 +419,18 @@ def compiles(class_, *specs): return fn return decorate + +def deregister(class_): + """Remove all custom compilers associated with a given + :class:`.ClauseElement` type.""" + + if hasattr(class_, '_compiler_dispatcher'): + # regenerate default _compiler_dispatch + visitors._generate_dispatch(class_) + # remove custom directive + del class_._compiler_dispatcher + + class _dispatcher(object): def __init__(self): self.specs = {} @@ -407,4 +446,3 @@ class _dispatcher(object): "%s construct has no default " "compilation handler." % type(element)) return fn(element, compiler, **kw) - diff --git a/libs/sqlalchemy/ext/declarative.py b/libs/sqlalchemy/ext/declarative/__init__.py old mode 100755 new mode 100644 similarity index 51% rename from libs/sqlalchemy/ext/declarative.py rename to libs/sqlalchemy/ext/declarative/__init__.py index b0876f0d..0ee4e33f --- a/libs/sqlalchemy/ext/declarative.py +++ b/libs/sqlalchemy/ext/declarative/__init__.py @@ -1,5 +1,5 @@ -# ext/declarative.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# ext/declarative/__init__.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -30,8 +30,7 @@ As a simple example:: Above, the :func:`declarative_base` callable returns a new base class from which all mapped classes should inherit. When the class definition is -completed, a new :class:`.Table` and -:func:`.mapper` will have been generated. +completed, a new :class:`.Table` and :func:`.mapper` will have been generated. The resulting table and mapper are accessible via ``__table__`` and ``__mapper__`` attributes on the @@ -52,7 +51,8 @@ assigned. To name columns explicitly with a name distinct from their mapped attribute, just give the column a name. Below, column "some_table_id" is mapped to the -"id" attribute of `SomeClass`, but in SQL will be represented as "some_table_id":: +"id" attribute of `SomeClass`, but in SQL will be represented as +"some_table_id":: class SomeClass(Base): __tablename__ = 'some_table' @@ -60,13 +60,13 @@ just give the column a name. Below, column "some_table_id" is mapped to the Attributes may be added to the class after its construction, and they will be added to the underlying :class:`.Table` and -:func:`.mapper()` definitions as appropriate:: +:func:`.mapper` definitions as appropriate:: SomeClass.data = Column('data', Unicode) SomeClass.related = relationship(RelatedInfo) Classes which are constructed using declarative can interact freely -with classes that are mapped explicitly with :func:`mapper`. +with classes that are mapped explicitly with :func:`.mapper`. It is recommended, though not required, that all tables share the same underlying :class:`~sqlalchemy.schema.MetaData` object, @@ -146,13 +146,52 @@ expression functions like :func:`~sqlalchemy.sql.expression.desc` and order_by="desc(Address.email)", primaryjoin="Address.user_id==User.id") -As an alternative to string-based attributes, attributes may also be -defined after all classes have been created. Just add them to the target -class after the fact:: +For the case where more than one module contains a class of the same name, +string class names can also be specified as module-qualified paths +within any of these string expressions:: + + class User(Base): + # .... + addresses = relationship("myapp.model.address.Address", + order_by="desc(myapp.model.address.Address.email)", + primaryjoin="myapp.model.address.Address.user_id==" + "myapp.model.user.User.id") + +The qualified path can be any partial path that removes ambiguity between +the names. For example, to disambiguate between +``myapp.model.address.Address`` and ``myapp.model.lookup.Address``, +we can specify ``address.Address`` or ``lookup.Address``:: + + class User(Base): + # .... + addresses = relationship("address.Address", + order_by="desc(address.Address.email)", + primaryjoin="address.Address.user_id==" + "User.id") + +.. versionadded:: 0.8 + module-qualified paths can be used when specifying string arguments + with Declarative, in order to specify specific modules. + +Two alternatives also exist to using string-based attributes. A lambda +can also be used, which will be evaluated after all mappers have been +configured:: + + class User(Base): + # ... + addresses = relationship(lambda: Address, + order_by=lambda: desc(Address.email), + primaryjoin=lambda: Address.user_id==User.id) + +Or, the relationship can be added to the class explicitly after the classes +are available:: User.addresses = relationship(Address, primaryjoin=Address.user_id==User.id) + + + Configuring Many-to-Many Relationships ====================================== @@ -175,9 +214,9 @@ the :class:`.MetaData` object used by the declarative base:: id = Column(Integer, primary_key=True) keywords = relationship("Keyword", secondary=keywords) -Like other :func:`.relationship` arguments, a string is accepted as well, -passing the string name of the table as defined in the ``Base.metadata.tables`` -collection:: +Like other :func:`~sqlalchemy.orm.relationship` arguments, a string is accepted +as well, passing the string name of the table as defined in the +``Base.metadata.tables`` collection:: class Author(Base): __tablename__ = 'authors' @@ -186,7 +225,7 @@ collection:: As with traditional mapping, its generally not a good idea to use a :class:`.Table` as the "secondary" argument which is also mapped to -a class, unless the :class:`.relationship` is declared with ``viewonly=True``. +a class, unless the :func:`.relationship` is declared with ``viewonly=True``. Otherwise, the unit-of-work system may attempt duplicate INSERT and DELETE statements against the underlying table. @@ -274,7 +313,8 @@ such as those which already take advantage of the data-driven nature of Note that when the ``__table__`` approach is used, the object is immediately usable as a plain :class:`.Table` within the class declaration body itself, as a Python class is only another syntactical block. Below this is illustrated -by using the ``id`` column in the ``primaryjoin`` condition of a :func:`.relationship`:: +by using the ``id`` column in the ``primaryjoin`` condition of a +:func:`.relationship`:: class MyClass(Base): __table__ = Table('my_table', Base.metadata, @@ -286,8 +326,8 @@ by using the ``id`` column in the ``primaryjoin`` condition of a :func:`.relatio primaryjoin=Widget.myclass_id==__table__.c.id) Similarly, mapped attributes which refer to ``__table__`` can be placed inline, -as below where we assign the ``name`` column to the attribute ``_name``, generating -a synonym for ``name``:: +as below where we assign the ``name`` column to the attribute ``_name``, +generating a synonym for ``name``:: from sqlalchemy.ext.declarative import synonym_for @@ -315,12 +355,13 @@ in conjunction with a mapped class:: However, one improvement that can be made here is to not require the :class:`.Engine` to be available when classes are -being first declared. To achieve this, use the example -described at :ref:`examples_declarative_reflection` to build a -declarative base that sets up mappings only after a special -``prepare(engine)`` step is called:: +being first declared. To achieve this, use the +:class:`.DeferredReflection` mixin, which sets up mappings +only after a special ``prepare(engine)`` step is called:: - Base = declarative_base(cls=DeclarativeReflectedBase) + from sqlalchemy.ext.declarative import declarative_base, DeferredReflection + + Base = declarative_base(cls=DeferredReflection) class Foo(Base): __tablename__ = 'foo' @@ -336,15 +377,17 @@ declarative base that sets up mappings only after a special Base.prepare(e) +.. versionadded:: 0.8 + Added :class:`.DeferredReflection`. Mapper Configuration ==================== Declarative makes use of the :func:`~.orm.mapper` function internally when it creates the mapping to the declared table. The options -for :func:`~.orm.mapper` are passed directly through via the ``__mapper_args__`` -class attribute. As always, arguments which reference locally -mapped columns can reference them directly from within the +for :func:`~.orm.mapper` are passed directly through via the +``__mapper_args__`` class attribute. As always, arguments which reference +locally mapped columns can reference them directly from within the class declaration:: from datetime import datetime @@ -407,6 +450,8 @@ only the ``engineers.id`` column, give it a different attribute name:: column over that of the superclass, such as querying above for ``Engineer.id``. Prior to 0.7 this was the reverse. +.. _declarative_single_table: + Single Table Inheritance ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -445,6 +490,115 @@ The attribute exclusion logic is provided by the behavior can be disabled by passing an explicit ``exclude_properties`` collection (empty or otherwise) to the ``__mapper_args__``. +Resolving Column Conflicts +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note above that the ``primary_language`` and ``golf_swing`` columns +are "moved up" to be applied to ``Person.__table__``, as a result of their +declaration on a subclass that has no table of its own. A tricky case +comes up when two subclasses want to specify *the same* column, as below:: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + start_date = Column(DateTime) + + class Manager(Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + start_date = Column(DateTime) + +Above, the ``start_date`` column declared on both ``Engineer`` and ``Manager`` +will result in an error:: + + sqlalchemy.exc.ArgumentError: Column 'start_date' on class + conflicts with existing + column 'people.start_date' + +In a situation like this, Declarative can't be sure +of the intent, especially if the ``start_date`` columns had, for example, +different types. A situation like this can be resolved by using +:class:`.declared_attr` to define the :class:`.Column` conditionally, taking +care to return the **existing column** via the parent ``__table__`` if it +already exists:: + + from sqlalchemy.ext.declarative import declared_attr + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class Engineer(Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + + @declared_attr + def start_date(cls): + "Start date column, if not present already." + return Person.__table__.c.get('start_date', Column(DateTime)) + + class Manager(Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + + @declared_attr + def start_date(cls): + "Start date column, if not present already." + return Person.__table__.c.get('start_date', Column(DateTime)) + +Above, when ``Manager`` is mapped, the ``start_date`` column is +already present on the ``Person`` class. Declarative lets us return +that :class:`.Column` as a result in this case, where it knows to skip +re-assigning the same column. If the mapping is mis-configured such +that the ``start_date`` column is accidentally re-assigned to a +different table (such as, if we changed ``Manager`` to be joined +inheritance without fixing ``start_date``), an error is raised which +indicates an existing :class:`.Column` is trying to be re-assigned to +a different owning :class:`.Table`. + +.. versionadded:: 0.8 :class:`.declared_attr` can be used on a non-mixin + class, and the returned :class:`.Column` or other mapped attribute + will be applied to the mapping as any other attribute. Previously, + the resulting attribute would be ignored, and also result in a warning + being emitted when a subclass was created. + +.. versionadded:: 0.8 :class:`.declared_attr`, when used either with a + mixin or non-mixin declarative class, can return an existing + :class:`.Column` already assigned to the parent :class:`.Table`, + to indicate that the re-assignment of the :class:`.Column` should be + skipped, however should still be mapped on the target class, + in order to resolve duplicate column conflicts. + +The same concept can be used with mixin classes (see +:ref:`declarative_mixins`):: + + class Person(Base): + __tablename__ = 'people' + id = Column(Integer, primary_key=True) + discriminator = Column('type', String(50)) + __mapper_args__ = {'polymorphic_on': discriminator} + + class HasStartDate(object): + @declared_attr + def start_date(cls): + return cls.__table__.c.get('start_date', Column(DateTime)) + + class Engineer(HasStartDate, Person): + __mapper_args__ = {'polymorphic_identity': 'engineer'} + + class Manager(HasStartDate, Person): + __mapper_args__ = {'polymorphic_identity': 'manager'} + +The above mixin checks the local ``__table__`` attribute for the column. +Because we're using single table inheritance, we're sure that in this case, +``cls.__table__`` refers to ``People.__table__``. If we were mixing joined- +and single-table inheritance, we might want our mixin to check more carefully +if ``cls.__table__`` is really the :class:`.Table` we're looking for. + Concrete Table Inheritance ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -502,12 +656,13 @@ Using the Concrete Helpers ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Helper classes provides a simpler pattern for concrete inheritance. -With these objects, the ``__declare_last__`` helper is used to configure the "polymorphic" -loader for the mapper after all subclasses have been declared. +With these objects, the ``__declare_last__`` helper is used to configure the +"polymorphic" loader for the mapper after all subclasses have been declared. .. versionadded:: 0.7.3 -An abstract base can be declared using the :class:`.AbstractConcreteBase` class:: +An abstract base can be declared using the +:class:`.AbstractConcreteBase` class:: from sqlalchemy.ext.declarative import AbstractConcreteBase @@ -605,8 +760,8 @@ Augmenting the Base In addition to using a pure mixin, most of the techniques in this section can also be applied to the base class itself, for patterns that -should apply to all classes derived from a particular base. This -is achieved using the ``cls`` argument of the :func:`.declarative_base` function:: +should apply to all classes derived from a particular base. This is achieved +using the ``cls`` argument of the :func:`.declarative_base` function:: from sqlalchemy.ext.declarative import declared_attr @@ -626,9 +781,9 @@ is achieved using the ``cls`` argument of the :func:`.declarative_base` function class MyModel(Base): name = Column(String(1000)) -Where above, ``MyModel`` and all other classes that derive from ``Base`` will have -a table name derived from the class name, an ``id`` primary key column, as well as -the "InnoDB" engine for MySQL. +Where above, ``MyModel`` and all other classes that derive from ``Base`` will +have a table name derived from the class name, an ``id`` primary key column, +as well as the "InnoDB" engine for MySQL. Mixing in Columns ~~~~~~~~~~~~~~~~~ @@ -668,7 +823,7 @@ keys, as a :class:`.ForeignKey` itself contains references to columns which can't be properly recreated at this level. For columns that have foreign keys, as well as for the variety of mapper-level constructs that require destination-explicit context, the -:func:`~.declared_attr` decorator is provided so that +:class:`~.declared_attr` decorator is provided so that patterns common to many classes can be defined as callables:: from sqlalchemy.ext.declarative import declared_attr @@ -688,9 +843,10 @@ extension can use the resulting :class:`.Column` object as returned by the method without the need to copy it. .. versionchanged:: > 0.6.5 - Rename 0.6.5 ``sqlalchemy.util.classproperty`` into :func:`~.declared_attr`. + Rename 0.6.5 ``sqlalchemy.util.classproperty`` + into :class:`~.declared_attr`. -Columns generated by :func:`~.declared_attr` can also be +Columns generated by :class:`~.declared_attr` can also be referenced by ``__mapper_args__`` to a limited degree, currently by ``polymorphic_on`` and ``version_id_col``, by specifying the classdecorator itself into the dictionary - the declarative extension @@ -707,6 +863,8 @@ will resolve them at class construction time:: __tablename__='test' id = Column(Integer, primary_key=True) + + Mixing in Relationships ~~~~~~~~~~~~~~~~~~~~~~~ @@ -739,11 +897,57 @@ reference a common target class via many-to-one:: __tablename__ = 'target' id = Column(Integer, primary_key=True) +Using Advanced Relationship Arguments (e.g. ``primaryjoin``, etc.) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + :func:`~sqlalchemy.orm.relationship` definitions which require explicit -primaryjoin, order_by etc. expressions should use the string forms -for these arguments, so that they are evaluated as late as possible. -To reference the mixin class in these expressions, use the given ``cls`` -to get its name:: +primaryjoin, order_by etc. expressions should in all but the most +simplistic cases use **late bound** forms +for these arguments, meaning, using either the string form or a lambda. +The reason for this is that the related :class:`.Column` objects which are to +be configured using ``@declared_attr`` are not available to another +``@declared_attr`` attribute; while the methods will work and return new +:class:`.Column` objects, those are not the :class:`.Column` objects that +Declarative will be using as it calls the methods on its own, thus using +*different* :class:`.Column` objects. + +The canonical example is the primaryjoin condition that depends upon +another mixed-in column:: + + class RefTargetMixin(object): + @declared_attr + def target_id(cls): + return Column('target_id', ForeignKey('target.id')) + + @declared_attr + def target(cls): + return relationship(Target, + primaryjoin=Target.id==cls.target_id # this is *incorrect* + ) + +Mapping a class using the above mixin, we will get an error like:: + + sqlalchemy.exc.InvalidRequestError: this ForeignKey's parent column is not + yet associated with a Table. + +This is because the ``target_id`` :class:`.Column` we've called upon in our ``target()`` +method is not the same :class:`.Column` that declarative is actually going to map +to our table. + +The condition above is resolved using a lambda:: + + class RefTargetMixin(object): + @declared_attr + def target_id(cls): + return Column('target_id', ForeignKey('target.id')) + + @declared_attr + def target(cls): + return relationship(Target, + primaryjoin=lambda: Target.id==cls.target_id + ) + +or alternatively, the string form (which ultmately generates a lambda):: class RefTargetMixin(object): @declared_attr @@ -756,8 +960,8 @@ to get its name:: primaryjoin="Target.id==%s.target_id" % cls.__name__ ) -Mixing in deferred(), column_property(), etc. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Mixing in deferred(), column_property(), and other MapperProperty classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Like :func:`~sqlalchemy.orm.relationship`, all :class:`~sqlalchemy.orm.interfaces.MapperProperty` subclasses such as @@ -775,6 +979,85 @@ requirement so that no reliance on copying is needed:: class Something(SomethingMixin, Base): __tablename__ = "something" +Mixing in Association Proxy and Other Attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Mixins can specify user-defined attributes as well as other extension +units such as :func:`.association_proxy`. The usage of +:class:`.declared_attr` is required in those cases where the attribute must +be tailored specifically to the target subclass. An example is when +constructing multiple :func:`.association_proxy` attributes which each +target a different type of child object. Below is an +:func:`.association_proxy` / mixin example which provides a scalar list of +string values to an implementing class:: + + from sqlalchemy import Column, Integer, ForeignKey, String + from sqlalchemy.orm import relationship + from sqlalchemy.ext.associationproxy import association_proxy + from sqlalchemy.ext.declarative import declarative_base, declared_attr + + Base = declarative_base() + + class HasStringCollection(object): + @declared_attr + def _strings(cls): + class StringAttribute(Base): + __tablename__ = cls.string_table_name + id = Column(Integer, primary_key=True) + value = Column(String(50), nullable=False) + parent_id = Column(Integer, + ForeignKey('%s.id' % cls.__tablename__), + nullable=False) + def __init__(self, value): + self.value = value + + return relationship(StringAttribute) + + @declared_attr + def strings(cls): + return association_proxy('_strings', 'value') + + class TypeA(HasStringCollection, Base): + __tablename__ = 'type_a' + string_table_name = 'type_a_strings' + id = Column(Integer(), primary_key=True) + + class TypeB(HasStringCollection, Base): + __tablename__ = 'type_b' + string_table_name = 'type_b_strings' + id = Column(Integer(), primary_key=True) + +Above, the ``HasStringCollection`` mixin produces a :func:`.relationship` +which refers to a newly generated class called ``StringAttribute``. The +``StringAttribute`` class is generated with it's own :class:`.Table` +definition which is local to the parent class making usage of the +``HasStringCollection`` mixin. It also produces an :func:`.association_proxy` +object which proxies references to the ``strings`` attribute onto the ``value`` +attribute of each ``StringAttribute`` instance. + +``TypeA`` or ``TypeB`` can be instantiated given the constructor +argument ``strings``, a list of strings:: + + ta = TypeA(strings=['foo', 'bar']) + tb = TypeA(strings=['bat', 'bar']) + +This list will generate a collection +of ``StringAttribute`` objects, which are persisted into a table that's +local to either the ``type_a_strings`` or ``type_b_strings`` table:: + + >>> print ta._strings + [<__main__.StringAttribute object at 0x10151cd90>, + <__main__.StringAttribute object at 0x10151ce10>] + +When constructing the :func:`.association_proxy`, the +:class:`.declared_attr` decorator must be used so that a distinct +:func:`.association_proxy` object is created for each of the ``TypeA`` +and ``TypeB`` classes. + +.. versionadded:: 0.8 :class:`.declared_attr` is usable with non-mapped + attributes, including user-defined attributes as well as + :func:`.association_proxy`. + Controlling table inheritance with mixins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -906,8 +1189,8 @@ Creating Indexes with Mixins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To define a named, potentially multicolumn :class:`.Index` that applies to all -tables derived from a mixin, use the "inline" form of :class:`.Index` and establish -it as part of ``__table_args__``:: +tables derived from a mixin, use the "inline" form of :class:`.Index` and +establish it as part of ``__table_args__``:: class MyMixin(object): a = Column(Integer) @@ -928,9 +1211,9 @@ Special Directives ~~~~~~~~~~~~~~~~~~~~~~ The ``__declare_last__()`` hook allows definition of -a class level function that is automatically called by the :meth:`.MapperEvents.after_configured` -event, which occurs after mappings are assumed to be completed and the 'configure' step -has finished:: +a class level function that is automatically called by the +:meth:`.MapperEvents.after_configured` event, which occurs after mappings are +assumed to be completed and the 'configure' step has finished:: class MyClass(Base): @classmethod @@ -946,9 +1229,9 @@ has finished:: ~~~~~~~~~~~~~~~~~~~ ``__abstract__`` causes declarative to skip the production -of a table or mapper for the class entirely. A class can be added within a hierarchy -in the same way as mixin (see :ref:`declarative_mixins`), allowing subclasses to extend -just from the special class:: +of a table or mapper for the class entirely. A class can be added within a +hierarchy in the same way as mixin (see :ref:`declarative_mixins`), allowing +subclasses to extend just from the special class:: class SomeAbstractBase(Base): __abstract__ = True @@ -963,8 +1246,8 @@ just from the special class:: class MyMappedClass(SomeAbstractBase): "" -One possible use of ``__abstract__`` is to use a distinct :class:`.MetaData` for different -bases:: +One possible use of ``__abstract__`` is to use a distinct +:class:`.MetaData` for different bases:: Base = declarative_base() @@ -976,9 +1259,10 @@ bases:: __abstract__ = True metadata = MetaData() -Above, classes which inherit from ``DefaultBase`` will use one :class:`.MetaData` as the -registry of tables, and those which inherit from ``OtherBase`` will use a different one. -The tables themselves can then be created perhaps within distinct databases:: +Above, classes which inherit from ``DefaultBase`` will use one +:class:`.MetaData` as the registry of tables, and those which inherit from +``OtherBase`` will use a different one. The tables themselves can then be +created perhaps within distinct databases:: DefaultBase.metadata.create_all(some_engine) OtherBase.metadata_create_all(some_other_engine) @@ -1000,7 +1284,7 @@ Sessions Note that ``declarative`` does nothing special with sessions, and is only intended as an easier way to configure mappers and :class:`~sqlalchemy.schema.Table` objects. A typical application -setup using :func:`~sqlalchemy.orm.scoped_session` might look like:: +setup using :class:`~sqlalchemy.orm.scoping.scoped_session` might look like:: engine = create_engine('postgresql://scott:tiger@localhost/test') Session = scoped_session(sessionmaker(autocommit=False, @@ -1013,749 +1297,13 @@ Mapped instances then make usage of """ -from sqlalchemy.schema import Table, Column, MetaData, _get_table_key -from sqlalchemy.orm import synonym as _orm_synonym, mapper,\ - comparable_property, class_mapper -from sqlalchemy.orm.interfaces import MapperProperty -from sqlalchemy.orm.properties import RelationshipProperty, ColumnProperty, CompositeProperty -from sqlalchemy.orm.util import _is_mapped_class -from sqlalchemy import util, exc -from sqlalchemy.sql import util as sql_util, expression -from sqlalchemy import event -from sqlalchemy.orm.util import polymorphic_union, _mapper_or_none +from .api import declarative_base, synonym_for, comparable_using, \ + instrument_declarative, ConcreteBase, AbstractConcreteBase, \ + DeclarativeMeta, DeferredReflection, has_inherited_table,\ + declared_attr, as_declarative -__all__ = 'declarative_base', 'synonym_for', \ - 'comparable_using', 'instrument_declarative' - -def instrument_declarative(cls, registry, metadata): - """Given a class, configure the class declaratively, - using the given registry, which can be any dictionary, and - MetaData object. - - """ - if '_decl_class_registry' in cls.__dict__: - raise exc.InvalidRequestError( - "Class %r already has been " - "instrumented declaratively" % cls) - cls._decl_class_registry = registry - cls.metadata = metadata - _as_declarative(cls, cls.__name__, cls.__dict__) - -def has_inherited_table(cls): - """Given a class, return True if any of the classes it inherits from has a - mapped table, otherwise return False. - """ - for class_ in cls.__mro__: - if getattr(class_,'__table__',None) is not None: - return True - return False - -def _as_declarative(cls, classname, dict_): - - # dict_ will be a dictproxy, which we can't write to, and we need to! - dict_ = dict(dict_) - - column_copies = {} - potential_columns = {} - - mapper_args = {} - table_args = inherited_table_args = None - tablename = None - parent_columns = () - - declarative_props = (declared_attr, util.classproperty) - - for base in cls.__mro__: - _is_declarative_inherits = hasattr(base, '_decl_class_registry') - - if '__declare_last__' in base.__dict__: - @event.listens_for(mapper, "after_configured") - def go(): - cls.__declare_last__() - if '__abstract__' in base.__dict__: - if (base is cls or - (base in cls.__bases__ and not _is_declarative_inherits) - ): - return - - class_mapped = _is_mapped_class(base) - if class_mapped: - parent_columns = base.__table__.c.keys() - - for name,obj in vars(base).items(): - if name == '__mapper_args__': - if not mapper_args and ( - not class_mapped or - isinstance(obj, declarative_props) - ): - mapper_args = cls.__mapper_args__ - elif name == '__tablename__': - if not tablename and ( - not class_mapped or - isinstance(obj, declarative_props) - ): - tablename = cls.__tablename__ - elif name == '__table_args__': - if not table_args and ( - not class_mapped or - isinstance(obj, declarative_props) - ): - table_args = cls.__table_args__ - if not isinstance(table_args, (tuple, dict, type(None))): - raise exc.ArgumentError( - "__table_args__ value must be a tuple, " - "dict, or None") - if base is not cls: - inherited_table_args = True - elif class_mapped: - if isinstance(obj, declarative_props): - util.warn("Regular (i.e. not __special__) " - "attribute '%s.%s' uses @declared_attr, " - "but owning class %s is mapped - " - "not applying to subclass %s." - % (base.__name__, name, base, cls)) - continue - elif base is not cls: - # we're a mixin. - if isinstance(obj, Column): - if obj.foreign_keys: - raise exc.InvalidRequestError( - "Columns with foreign keys to other columns " - "must be declared as @declared_attr callables " - "on declarative mixin classes. ") - if name not in dict_ and not ( - '__table__' in dict_ and - (obj.name or name) in dict_['__table__'].c - ) and name not in potential_columns: - potential_columns[name] = \ - column_copies[obj] = \ - obj.copy() - column_copies[obj]._creation_order = \ - obj._creation_order - elif isinstance(obj, MapperProperty): - raise exc.InvalidRequestError( - "Mapper properties (i.e. deferred," - "column_property(), relationship(), etc.) must " - "be declared as @declared_attr callables " - "on declarative mixin classes.") - elif isinstance(obj, declarative_props): - dict_[name] = ret = \ - column_copies[obj] = getattr(cls, name) - if isinstance(ret, (Column, MapperProperty)) and \ - ret.doc is None: - ret.doc = obj.__doc__ - - # apply inherited columns as we should - for k, v in potential_columns.items(): - if tablename or (v.name or k) not in parent_columns: - dict_[k] = v - - if inherited_table_args and not tablename: - table_args = None - - # make sure that column copies are used rather - # than the original columns from any mixins - for k in ('version_id_col', 'polymorphic_on',): - if k in mapper_args: - v = mapper_args[k] - mapper_args[k] = column_copies.get(v,v) - - if classname in cls._decl_class_registry: - util.warn("The classname %r is already in the registry of this" - " declarative base, mapped to %r" % ( - classname, - cls._decl_class_registry[classname] - )) - cls._decl_class_registry[classname] = cls - our_stuff = util.OrderedDict() - - for k in dict_: - value = dict_[k] - if isinstance(value, declarative_props): - value = getattr(cls, k) - - if (isinstance(value, tuple) and len(value) == 1 and - isinstance(value[0], (Column, MapperProperty))): - util.warn("Ignoring declarative-like tuple value of attribute " - "%s: possibly a copy-and-paste error with a comma " - "left at the end of the line?" % k) - continue - if not isinstance(value, (Column, MapperProperty)): - continue - if k == 'metadata': - raise exc.InvalidRequestError( - "Attribute name 'metadata' is reserved " - "for the MetaData instance when using a " - "declarative base class." - ) - prop = _deferred_relationship(cls, value) - our_stuff[k] = prop - - # set up attributes in the order they were created - our_stuff.sort(key=lambda key: our_stuff[key]._creation_order) - - # extract columns from the class dict - cols = set() - for key, c in our_stuff.iteritems(): - if isinstance(c, (ColumnProperty, CompositeProperty)): - for col in c.columns: - if isinstance(col, Column) and \ - col.table is None: - _undefer_column_name(key, col) - cols.add(col) - elif isinstance(c, Column): - _undefer_column_name(key, c) - cols.add(c) - # if the column is the same name as the key, - # remove it from the explicit properties dict. - # the normal rules for assigning column-based properties - # will take over, including precedence of columns - # in multi-column ColumnProperties. - if key == c.key: - del our_stuff[key] - cols = sorted(cols, key=lambda c:c._creation_order) - table = None - - if hasattr(cls, '__table_cls__'): - table_cls = util.unbound_method_to_callable(cls.__table_cls__) - else: - table_cls = Table - - if '__table__' not in dict_: - if tablename is not None: - - args, table_kw = (), {} - if table_args: - if isinstance(table_args, dict): - table_kw = table_args - elif isinstance(table_args, tuple): - if isinstance(table_args[-1], dict): - args, table_kw = table_args[0:-1], table_args[-1] - else: - args = table_args - - autoload = dict_.get('__autoload__') - if autoload: - table_kw['autoload'] = True - - cls.__table__ = table = table_cls(tablename, cls.metadata, - *(tuple(cols) + tuple(args)), - **table_kw) - else: - table = cls.__table__ - if cols: - for c in cols: - if not table.c.contains_column(c): - raise exc.ArgumentError( - "Can't add additional column %r when " - "specifying __table__" % c.key - ) - - if 'inherits' not in mapper_args: - for c in cls.__bases__: - if _is_mapped_class(c): - mapper_args['inherits'] = c - break - - if hasattr(cls, '__mapper_cls__'): - mapper_cls = util.unbound_method_to_callable(cls.__mapper_cls__) - else: - mapper_cls = mapper - - if table is None and 'inherits' not in mapper_args: - raise exc.InvalidRequestError( - "Class %r does not have a __table__ or __tablename__ " - "specified and does not inherit from an existing " - "table-mapped class." % cls - ) - - elif 'inherits' in mapper_args and not mapper_args.get('concrete', False): - inherited_mapper = class_mapper(mapper_args['inherits'], - compile=False) - inherited_table = inherited_mapper.local_table - - if table is None: - # single table inheritance. - # ensure no table args - if table_args: - raise exc.ArgumentError( - "Can't place __table_args__ on an inherited class " - "with no table." - ) - - # add any columns declared here to the inherited table. - for c in cols: - if c.primary_key: - raise exc.ArgumentError( - "Can't place primary key columns on an inherited " - "class with no table." - ) - if c.name in inherited_table.c: - raise exc.ArgumentError( - "Column '%s' on class %s conflicts with " - "existing column '%s'" % - (c, cls, inherited_table.c[c.name]) - ) - inherited_table.append_column(c) - - # single or joined inheritance - # exclude any cols on the inherited table which are not mapped on the - # parent class, to avoid - # mapping columns specific to sibling/nephew classes - inherited_mapper = class_mapper(mapper_args['inherits'], - compile=False) - inherited_table = inherited_mapper.local_table - - if 'exclude_properties' not in mapper_args: - mapper_args['exclude_properties'] = exclude_properties = \ - set([c.key for c in inherited_table.c - if c not in inherited_mapper._columntoproperty]) - exclude_properties.difference_update([c.key for c in cols]) - - # look through columns in the current mapper that - # are keyed to a propname different than the colname - # (if names were the same, we'd have popped it out above, - # in which case the mapper makes this combination). - # See if the superclass has a similar column property. - # If so, join them together. - for k, col in our_stuff.items(): - if not isinstance(col, expression.ColumnElement): - continue - if k in inherited_mapper._props: - p = inherited_mapper._props[k] - if isinstance(p, ColumnProperty): - # note here we place the subclass column - # first. See [ticket:1892] for background. - our_stuff[k] = [col] + p.columns - - - cls.__mapper__ = mapper_cls(cls, - table, - properties=our_stuff, - **mapper_args) - -class DeclarativeMeta(type): - def __init__(cls, classname, bases, dict_): - if '_decl_class_registry' not in cls.__dict__: - _as_declarative(cls, classname, cls.__dict__) - type.__init__(cls, classname, bases, dict_) - - def __setattr__(cls, key, value): - if '__mapper__' in cls.__dict__: - if isinstance(value, Column): - _undefer_column_name(key, value) - cls.__table__.append_column(value) - cls.__mapper__.add_property(key, value) - elif isinstance(value, ColumnProperty): - for col in value.columns: - if isinstance(col, Column) and col.table is None: - _undefer_column_name(key, col) - cls.__table__.append_column(col) - cls.__mapper__.add_property(key, value) - elif isinstance(value, MapperProperty): - cls.__mapper__.add_property( - key, - _deferred_relationship(cls, value) - ) - else: - type.__setattr__(cls, key, value) - else: - type.__setattr__(cls, key, value) - - -class _GetColumns(object): - def __init__(self, cls): - self.cls = cls - - def __getattr__(self, key): - mapper = class_mapper(self.cls, compile=False) - if mapper: - if not mapper.has_property(key): - raise exc.InvalidRequestError( - "Class %r does not have a mapped column named %r" - % (self.cls, key)) - - prop = mapper.get_property(key) - if not isinstance(prop, ColumnProperty): - raise exc.InvalidRequestError( - "Property %r is not an instance of" - " ColumnProperty (i.e. does not correspond" - " directly to a Column)." % key) - return getattr(self.cls, key) - -class _GetTable(object): - def __init__(self, key, metadata): - self.key = key - self.metadata = metadata - - def __getattr__(self, key): - return self.metadata.tables[ - _get_table_key(key, self.key) - ] - -def _deferred_relationship(cls, prop): - def resolve_arg(arg): - import sqlalchemy - - def access_cls(key): - if key in cls._decl_class_registry: - return _GetColumns(cls._decl_class_registry[key]) - elif key in cls.metadata.tables: - return cls.metadata.tables[key] - elif key in cls.metadata._schemas: - return _GetTable(key, cls.metadata) - else: - return sqlalchemy.__dict__[key] - - d = util.PopulateDict(access_cls) - def return_cls(): - try: - x = eval(arg, globals(), d) - - if isinstance(x, _GetColumns): - return x.cls - else: - return x - except NameError, n: - raise exc.InvalidRequestError( - "When initializing mapper %s, expression %r failed to " - "locate a name (%r). If this is a class name, consider " - "adding this relationship() to the %r class after " - "both dependent classes have been defined." % - (prop.parent, arg, n.args[0], cls) - ) - return return_cls - - if isinstance(prop, RelationshipProperty): - for attr in ('argument', 'order_by', 'primaryjoin', 'secondaryjoin', - 'secondary', '_user_defined_foreign_keys', 'remote_side'): - v = getattr(prop, attr) - if isinstance(v, basestring): - setattr(prop, attr, resolve_arg(v)) - - if prop.backref and isinstance(prop.backref, tuple): - key, kwargs = prop.backref - for attr in ('primaryjoin', 'secondaryjoin', 'secondary', - 'foreign_keys', 'remote_side', 'order_by'): - if attr in kwargs and isinstance(kwargs[attr], basestring): - kwargs[attr] = resolve_arg(kwargs[attr]) - - - return prop - -def synonym_for(name, map_column=False): - """Decorator, make a Python @property a query synonym for a column. - - A decorator version of :func:`~sqlalchemy.orm.synonym`. The function being - decorated is the 'descriptor', otherwise passes its arguments through to - synonym():: - - @synonym_for('col') - @property - def prop(self): - return 'special sauce' - - The regular ``synonym()`` is also usable directly in a declarative setting - and may be convenient for read/write properties:: - - prop = synonym('col', descriptor=property(_read_prop, _write_prop)) - - """ - def decorate(fn): - return _orm_synonym(name, map_column=map_column, descriptor=fn) - return decorate - -def comparable_using(comparator_factory): - """Decorator, allow a Python @property to be used in query criteria. - - This is a decorator front end to - :func:`~sqlalchemy.orm.comparable_property` that passes - through the comparator_factory and the function being decorated:: - - @comparable_using(MyComparatorType) - @property - def prop(self): - return 'special sauce' - - The regular ``comparable_property()`` is also usable directly in a - declarative setting and may be convenient for read/write properties:: - - prop = comparable_property(MyComparatorType) - - """ - def decorate(fn): - return comparable_property(comparator_factory, fn) - return decorate - -class declared_attr(property): - """Mark a class-level method as representing the definition of - a mapped property or special declarative member name. - - .. versionchanged:: 0.6.{2,3,4} - ``@declared_attr`` is available as - ``sqlalchemy.util.classproperty`` for SQLAlchemy versions - 0.6.2, 0.6.3, 0.6.4. - - @declared_attr turns the attribute into a scalar-like - property that can be invoked from the uninstantiated class. - Declarative treats attributes specifically marked with - @declared_attr as returning a construct that is specific - to mapping or declarative table configuration. The name - of the attribute is that of what the non-dynamic version - of the attribute would be. - - @declared_attr is more often than not applicable to mixins, - to define relationships that are to be applied to different - implementors of the class:: - - class ProvidesUser(object): - "A mixin that adds a 'user' relationship to classes." - - @declared_attr - def user(self): - return relationship("User") - - It also can be applied to mapped classes, such as to provide - a "polymorphic" scheme for inheritance:: - - class Employee(Base): - id = Column(Integer, primary_key=True) - type = Column(String(50), nullable=False) - - @declared_attr - def __tablename__(cls): - return cls.__name__.lower() - - @declared_attr - def __mapper_args__(cls): - if cls.__name__ == 'Employee': - return { - "polymorphic_on":cls.type, - "polymorphic_identity":"Employee" - } - else: - return {"polymorphic_identity":cls.__name__} - - """ - - def __init__(self, fget, *arg, **kw): - super(declared_attr, self).__init__(fget, *arg, **kw) - self.__doc__ = fget.__doc__ - - def __get__(desc, self, cls): - return desc.fget(cls) - -def _declarative_constructor(self, **kwargs): - """A simple constructor that allows initialization from kwargs. - - Sets attributes on the constructed instance using the names and - values in ``kwargs``. - - Only keys that are present as - attributes of the instance's class are allowed. These could be, - for example, any mapped columns or relationships. - """ - cls_ = type(self) - for k in kwargs: - if not hasattr(cls_, k): - raise TypeError( - "%r is an invalid keyword argument for %s" % - (k, cls_.__name__)) - setattr(self, k, kwargs[k]) -_declarative_constructor.__name__ = '__init__' - -def declarative_base(bind=None, metadata=None, mapper=None, cls=object, - name='Base', constructor=_declarative_constructor, - class_registry=None, - metaclass=DeclarativeMeta): - """Construct a base class for declarative class definitions. - - The new base class will be given a metaclass that produces - appropriate :class:`~sqlalchemy.schema.Table` objects and makes - the appropriate :func:`~sqlalchemy.orm.mapper` calls based on the - information provided declaratively in the class and any subclasses - of the class. - - :param bind: An optional - :class:`~sqlalchemy.engine.base.Connectable`, will be assigned - the ``bind`` attribute on the :class:`~sqlalchemy.MetaData` - instance. - - :param metadata: - An optional :class:`~sqlalchemy.MetaData` instance. All - :class:`~sqlalchemy.schema.Table` objects implicitly declared by - subclasses of the base will share this MetaData. A MetaData instance - will be created if none is provided. The - :class:`~sqlalchemy.MetaData` instance will be available via the - `metadata` attribute of the generated declarative base class. - - :param mapper: - An optional callable, defaults to :func:`~sqlalchemy.orm.mapper`. Will - be used to map subclasses to their Tables. - - :param cls: - Defaults to :class:`object`. A type to use as the base for the generated - declarative base class. May be a class or tuple of classes. - - :param name: - Defaults to ``Base``. The display name for the generated - class. Customizing this is not required, but can improve clarity in - tracebacks and debugging. - - :param constructor: - Defaults to - :func:`~sqlalchemy.ext.declarative._declarative_constructor`, an - __init__ implementation that assigns \**kwargs for declared - fields and relationships to an instance. If ``None`` is supplied, - no __init__ will be provided and construction will fall back to - cls.__init__ by way of the normal Python semantics. - - :param class_registry: optional dictionary that will serve as the - registry of class names-> mapped classes when string names - are used to identify classes inside of :func:`.relationship` - and others. Allows two or more declarative base classes - to share the same registry of class names for simplified - inter-base relationships. - - :param metaclass: - Defaults to :class:`.DeclarativeMeta`. A metaclass or __metaclass__ - compatible callable to use as the meta type of the generated - declarative base class. - - """ - lcl_metadata = metadata or MetaData() - if bind: - lcl_metadata.bind = bind - - if class_registry is None: - class_registry = {} - - bases = not isinstance(cls, tuple) and (cls,) or cls - class_dict = dict(_decl_class_registry=class_registry, - metadata=lcl_metadata) - - if constructor: - class_dict['__init__'] = constructor - if mapper: - class_dict['__mapper_cls__'] = mapper - - return metaclass(name, bases, class_dict) - -def _undefer_column_name(key, column): - if column.key is None: - column.key = key - if column.name is None: - column.name = key - -class ConcreteBase(object): - """A helper class for 'concrete' declarative mappings. - - :class:`.ConcreteBase` will use the :func:`.polymorphic_union` - function automatically, against all tables mapped as a subclass - to this class. The function is called via the - ``__declare_last__()`` function, which is essentially - a hook for the :func:`.MapperEvents.after_configured` event. - - :class:`.ConcreteBase` produces a mapped - table for the class itself. Compare to :class:`.AbstractConcreteBase`, - which does not. - - Example:: - - from sqlalchemy.ext.declarative import ConcreteBase - - class Employee(ConcreteBase, Base): - __tablename__ = 'employee' - employee_id = Column(Integer, primary_key=True) - name = Column(String(50)) - __mapper_args__ = { - 'polymorphic_identity':'employee', - 'concrete':True} - - class Manager(Employee): - __tablename__ = 'manager' - employee_id = Column(Integer, primary_key=True) - name = Column(String(50)) - manager_data = Column(String(40)) - __mapper_args__ = { - 'polymorphic_identity':'manager', - 'concrete':True} - - """ - - @classmethod - def _create_polymorphic_union(cls, mappers): - return polymorphic_union(dict( - (mapper.polymorphic_identity, mapper.local_table) - for mapper in mappers - ), 'type', 'pjoin') - - @classmethod - def __declare_last__(cls): - m = cls.__mapper__ - if m.with_polymorphic: - return - - mappers = list(m.self_and_descendants) - pjoin = cls._create_polymorphic_union(mappers) - m._set_with_polymorphic(("*",pjoin)) - m._set_polymorphic_on(pjoin.c.type) - -class AbstractConcreteBase(ConcreteBase): - """A helper class for 'concrete' declarative mappings. - - :class:`.AbstractConcreteBase` will use the :func:`.polymorphic_union` - function automatically, against all tables mapped as a subclass - to this class. The function is called via the - ``__declare_last__()`` function, which is essentially - a hook for the :func:`.MapperEvents.after_configured` event. - - :class:`.AbstractConcreteBase` does not produce a mapped - table for the class itself. Compare to :class:`.ConcreteBase`, - which does. - - Example:: - - from sqlalchemy.ext.declarative import ConcreteBase - - class Employee(AbstractConcreteBase, Base): - pass - - class Manager(Employee): - __tablename__ = 'manager' - employee_id = Column(Integer, primary_key=True) - name = Column(String(50)) - manager_data = Column(String(40)) - __mapper_args__ = { - 'polymorphic_identity':'manager', - 'concrete':True} - - """ - - __abstract__ = True - - @classmethod - def __declare_last__(cls): - if hasattr(cls, '__mapper__'): - return - - # can't rely on 'self_and_descendants' here - # since technically an immediate subclass - # might not be mapped, but a subclass - # may be. - mappers = [] - stack = list(cls.__subclasses__()) - while stack: - klass = stack.pop() - stack.extend(klass.__subclasses__()) - mn = _mapper_or_none(klass) - if mn is not None: - mappers.append(mn) - pjoin = cls._create_polymorphic_union(mappers) - cls.__mapper__ = m = mapper(cls, pjoin, polymorphic_on=pjoin.c.type) - - for scls in cls.__subclasses__(): - sm = _mapper_or_none(scls) - if sm.concrete and cls in scls.__bases__: - sm._set_concrete_base(m) +__all__ = ['declarative_base', 'synonym_for', 'has_inherited_table', + 'comparable_using', 'instrument_declarative', 'declared_attr', + 'ConcreteBase', 'AbstractConcreteBase', 'DeclarativeMeta', + 'DeferredReflection'] diff --git a/libs/sqlalchemy/ext/declarative/api.py b/libs/sqlalchemy/ext/declarative/api.py new file mode 100644 index 00000000..2418c6e5 --- /dev/null +++ b/libs/sqlalchemy/ext/declarative/api.py @@ -0,0 +1,512 @@ +# ext/declarative/api.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +"""Public API functions and helpers for declarative.""" + + +from ...schema import Table, MetaData +from ...orm import synonym as _orm_synonym, mapper,\ + comparable_property,\ + interfaces, properties +from ...orm.util import polymorphic_union +from ...orm.base import _mapper_or_none +from ...util import compat +from ... import exc +import weakref + +from .base import _as_declarative, \ + _declarative_constructor,\ + _DeferredMapperConfig, _add_attribute +from .clsregistry import _class_resolver + + +def instrument_declarative(cls, registry, metadata): + """Given a class, configure the class declaratively, + using the given registry, which can be any dictionary, and + MetaData object. + + """ + if '_decl_class_registry' in cls.__dict__: + raise exc.InvalidRequestError( + "Class %r already has been " + "instrumented declaratively" % cls) + cls._decl_class_registry = registry + cls.metadata = metadata + _as_declarative(cls, cls.__name__, cls.__dict__) + + +def has_inherited_table(cls): + """Given a class, return True if any of the classes it inherits from has a + mapped table, otherwise return False. + """ + for class_ in cls.__mro__[1:]: + if getattr(class_, '__table__', None) is not None: + return True + return False + + +class DeclarativeMeta(type): + def __init__(cls, classname, bases, dict_): + if '_decl_class_registry' not in cls.__dict__: + _as_declarative(cls, classname, cls.__dict__) + type.__init__(cls, classname, bases, dict_) + + def __setattr__(cls, key, value): + _add_attribute(cls, key, value) + + +def synonym_for(name, map_column=False): + """Decorator, make a Python @property a query synonym for a column. + + A decorator version of :func:`~sqlalchemy.orm.synonym`. The function being + decorated is the 'descriptor', otherwise passes its arguments through to + synonym():: + + @synonym_for('col') + @property + def prop(self): + return 'special sauce' + + The regular ``synonym()`` is also usable directly in a declarative setting + and may be convenient for read/write properties:: + + prop = synonym('col', descriptor=property(_read_prop, _write_prop)) + + """ + def decorate(fn): + return _orm_synonym(name, map_column=map_column, descriptor=fn) + return decorate + + +def comparable_using(comparator_factory): + """Decorator, allow a Python @property to be used in query criteria. + + This is a decorator front end to + :func:`~sqlalchemy.orm.comparable_property` that passes + through the comparator_factory and the function being decorated:: + + @comparable_using(MyComparatorType) + @property + def prop(self): + return 'special sauce' + + The regular ``comparable_property()`` is also usable directly in a + declarative setting and may be convenient for read/write properties:: + + prop = comparable_property(MyComparatorType) + + """ + def decorate(fn): + return comparable_property(comparator_factory, fn) + return decorate + + +class declared_attr(interfaces._MappedAttribute, property): + """Mark a class-level method as representing the definition of + a mapped property or special declarative member name. + + @declared_attr turns the attribute into a scalar-like + property that can be invoked from the uninstantiated class. + Declarative treats attributes specifically marked with + @declared_attr as returning a construct that is specific + to mapping or declarative table configuration. The name + of the attribute is that of what the non-dynamic version + of the attribute would be. + + @declared_attr is more often than not applicable to mixins, + to define relationships that are to be applied to different + implementors of the class:: + + class ProvidesUser(object): + "A mixin that adds a 'user' relationship to classes." + + @declared_attr + def user(self): + return relationship("User") + + It also can be applied to mapped classes, such as to provide + a "polymorphic" scheme for inheritance:: + + class Employee(Base): + id = Column(Integer, primary_key=True) + type = Column(String(50), nullable=False) + + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() + + @declared_attr + def __mapper_args__(cls): + if cls.__name__ == 'Employee': + return { + "polymorphic_on":cls.type, + "polymorphic_identity":"Employee" + } + else: + return {"polymorphic_identity":cls.__name__} + + .. versionchanged:: 0.8 :class:`.declared_attr` can be used with + non-ORM or extension attributes, such as user-defined attributes + or :func:`.association_proxy` objects, which will be assigned + to the class at class construction time. + + + """ + + def __init__(self, fget, *arg, **kw): + super(declared_attr, self).__init__(fget, *arg, **kw) + self.__doc__ = fget.__doc__ + + def __get__(desc, self, cls): + return desc.fget(cls) + + +def declarative_base(bind=None, metadata=None, mapper=None, cls=object, + name='Base', constructor=_declarative_constructor, + class_registry=None, + metaclass=DeclarativeMeta): + """Construct a base class for declarative class definitions. + + The new base class will be given a metaclass that produces + appropriate :class:`~sqlalchemy.schema.Table` objects and makes + the appropriate :func:`~sqlalchemy.orm.mapper` calls based on the + information provided declaratively in the class and any subclasses + of the class. + + :param bind: An optional + :class:`~sqlalchemy.engine.Connectable`, will be assigned + the ``bind`` attribute on the :class:`~sqlalchemy.schema.MetaData` + instance. + + :param metadata: + An optional :class:`~sqlalchemy.schema.MetaData` instance. All + :class:`~sqlalchemy.schema.Table` objects implicitly declared by + subclasses of the base will share this MetaData. A MetaData instance + will be created if none is provided. The + :class:`~sqlalchemy.schema.MetaData` instance will be available via the + `metadata` attribute of the generated declarative base class. + + :param mapper: + An optional callable, defaults to :func:`~sqlalchemy.orm.mapper`. Will + be used to map subclasses to their Tables. + + :param cls: + Defaults to :class:`object`. A type to use as the base for the generated + declarative base class. May be a class or tuple of classes. + + :param name: + Defaults to ``Base``. The display name for the generated + class. Customizing this is not required, but can improve clarity in + tracebacks and debugging. + + :param constructor: + Defaults to + :func:`~sqlalchemy.ext.declarative._declarative_constructor`, an + __init__ implementation that assigns \**kwargs for declared + fields and relationships to an instance. If ``None`` is supplied, + no __init__ will be provided and construction will fall back to + cls.__init__ by way of the normal Python semantics. + + :param class_registry: optional dictionary that will serve as the + registry of class names-> mapped classes when string names + are used to identify classes inside of :func:`.relationship` + and others. Allows two or more declarative base classes + to share the same registry of class names for simplified + inter-base relationships. + + :param metaclass: + Defaults to :class:`.DeclarativeMeta`. A metaclass or __metaclass__ + compatible callable to use as the meta type of the generated + declarative base class. + + .. seealso:: + + :func:`.as_declarative` + + """ + lcl_metadata = metadata or MetaData() + if bind: + lcl_metadata.bind = bind + + if class_registry is None: + class_registry = weakref.WeakValueDictionary() + + bases = not isinstance(cls, tuple) and (cls,) or cls + class_dict = dict(_decl_class_registry=class_registry, + metadata=lcl_metadata) + + if constructor: + class_dict['__init__'] = constructor + if mapper: + class_dict['__mapper_cls__'] = mapper + + return metaclass(name, bases, class_dict) + +def as_declarative(**kw): + """ + Class decorator for :func:`.declarative_base`. + + Provides a syntactical shortcut to the ``cls`` argument + sent to :func:`.declarative_base`, allowing the base class + to be converted in-place to a "declarative" base:: + + from sqlalchemy.ext.declarative import as_declarative + + @as_declarative() + class Base(object): + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() + id = Column(Integer, primary_key=True) + + class MyMappedClass(Base): + # ... + + All keyword arguments passed to :func:`.as_declarative` are passed + along to :func:`.declarative_base`. + + .. versionadded:: 0.8.3 + + .. seealso:: + + :func:`.declarative_base` + + """ + def decorate(cls): + kw['cls'] = cls + kw['name'] = cls.__name__ + return declarative_base(**kw) + + return decorate + +class ConcreteBase(object): + """A helper class for 'concrete' declarative mappings. + + :class:`.ConcreteBase` will use the :func:`.polymorphic_union` + function automatically, against all tables mapped as a subclass + to this class. The function is called via the + ``__declare_last__()`` function, which is essentially + a hook for the :meth:`.after_configured` event. + + :class:`.ConcreteBase` produces a mapped + table for the class itself. Compare to :class:`.AbstractConcreteBase`, + which does not. + + Example:: + + from sqlalchemy.ext.declarative import ConcreteBase + + class Employee(ConcreteBase, Base): + __tablename__ = 'employee' + employee_id = Column(Integer, primary_key=True) + name = Column(String(50)) + __mapper_args__ = { + 'polymorphic_identity':'employee', + 'concrete':True} + + class Manager(Employee): + __tablename__ = 'manager' + employee_id = Column(Integer, primary_key=True) + name = Column(String(50)) + manager_data = Column(String(40)) + __mapper_args__ = { + 'polymorphic_identity':'manager', + 'concrete':True} + + """ + + @classmethod + def _create_polymorphic_union(cls, mappers): + return polymorphic_union(dict( + (mp.polymorphic_identity, mp.local_table) + for mp in mappers + ), 'type', 'pjoin') + + @classmethod + def __declare_last__(cls): + m = cls.__mapper__ + if m.with_polymorphic: + return + + mappers = list(m.self_and_descendants) + pjoin = cls._create_polymorphic_union(mappers) + m._set_with_polymorphic(("*", pjoin)) + m._set_polymorphic_on(pjoin.c.type) + + +class AbstractConcreteBase(ConcreteBase): + """A helper class for 'concrete' declarative mappings. + + :class:`.AbstractConcreteBase` will use the :func:`.polymorphic_union` + function automatically, against all tables mapped as a subclass + to this class. The function is called via the + ``__declare_last__()`` function, which is essentially + a hook for the :meth:`.after_configured` event. + + :class:`.AbstractConcreteBase` does not produce a mapped + table for the class itself. Compare to :class:`.ConcreteBase`, + which does. + + Example:: + + from sqlalchemy.ext.declarative import AbstractConcreteBase + + class Employee(AbstractConcreteBase, Base): + pass + + class Manager(Employee): + __tablename__ = 'manager' + employee_id = Column(Integer, primary_key=True) + name = Column(String(50)) + manager_data = Column(String(40)) + __mapper_args__ = { + 'polymorphic_identity':'manager', + 'concrete':True} + + """ + + __abstract__ = True + + @classmethod + def __declare_last__(cls): + if hasattr(cls, '__mapper__'): + return + + # can't rely on 'self_and_descendants' here + # since technically an immediate subclass + # might not be mapped, but a subclass + # may be. + mappers = [] + stack = list(cls.__subclasses__()) + while stack: + klass = stack.pop() + stack.extend(klass.__subclasses__()) + mn = _mapper_or_none(klass) + if mn is not None: + mappers.append(mn) + pjoin = cls._create_polymorphic_union(mappers) + cls.__mapper__ = m = mapper(cls, pjoin, polymorphic_on=pjoin.c.type) + + for scls in cls.__subclasses__(): + sm = _mapper_or_none(scls) + if sm.concrete and cls in scls.__bases__: + sm._set_concrete_base(m) + + +class DeferredReflection(object): + """A helper class for construction of mappings based on + a deferred reflection step. + + Normally, declarative can be used with reflection by + setting a :class:`.Table` object using autoload=True + as the ``__table__`` attribute on a declarative class. + The caveat is that the :class:`.Table` must be fully + reflected, or at the very least have a primary key column, + at the point at which a normal declarative mapping is + constructed, meaning the :class:`.Engine` must be available + at class declaration time. + + The :class:`.DeferredReflection` mixin moves the construction + of mappers to be at a later point, after a specific + method is called which first reflects all :class:`.Table` + objects created so far. Classes can define it as such:: + + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.ext.declarative import DeferredReflection + Base = declarative_base() + + class MyClass(DeferredReflection, Base): + __tablename__ = 'mytable' + + Above, ``MyClass`` is not yet mapped. After a series of + classes have been defined in the above fashion, all tables + can be reflected and mappings created using + :meth:`.prepare`:: + + engine = create_engine("someengine://...") + DeferredReflection.prepare(engine) + + The :class:`.DeferredReflection` mixin can be applied to individual + classes, used as the base for the declarative base itself, + or used in a custom abstract class. Using an abstract base + allows that only a subset of classes to be prepared for a + particular prepare step, which is necessary for applications + that use more than one engine. For example, if an application + has two engines, you might use two bases, and prepare each + separately, e.g.:: + + class ReflectedOne(DeferredReflection, Base): + __abstract__ = True + + class ReflectedTwo(DeferredReflection, Base): + __abstract__ = True + + class MyClass(ReflectedOne): + __tablename__ = 'mytable' + + class MyOtherClass(ReflectedOne): + __tablename__ = 'myothertable' + + class YetAnotherClass(ReflectedTwo): + __tablename__ = 'yetanothertable' + + # ... etc. + + Above, the class hierarchies for ``ReflectedOne`` and + ``ReflectedTwo`` can be configured separately:: + + ReflectedOne.prepare(engine_one) + ReflectedTwo.prepare(engine_two) + + .. versionadded:: 0.8 + + """ + @classmethod + def prepare(cls, engine): + """Reflect all :class:`.Table` objects for all current + :class:`.DeferredReflection` subclasses""" + + to_map = _DeferredMapperConfig.classes_for_base(cls) + for thingy in to_map: + cls._sa_decl_prepare(thingy.local_table, engine) + thingy.map() + mapper = thingy.cls.__mapper__ + metadata = mapper.class_.metadata + for rel in mapper._props.values(): + if isinstance(rel, properties.RelationshipProperty) and \ + rel.secondary is not None: + if isinstance(rel.secondary, Table): + cls._reflect_table(rel.secondary, engine) + elif isinstance(rel.secondary, _class_resolver): + rel.secondary._resolvers += ( + cls._sa_deferred_table_resolver(engine, metadata), + ) + + @classmethod + def _sa_deferred_table_resolver(cls, engine, metadata): + def _resolve(key): + t1 = Table(key, metadata) + cls._reflect_table(t1, engine) + return t1 + return _resolve + + @classmethod + def _sa_decl_prepare(cls, local_table, engine): + # autoload Table, which is already + # present in the metadata. This + # will fill in db-loaded columns + # into the existing Table object. + if local_table is not None: + cls._reflect_table(local_table, engine) + + @classmethod + def _reflect_table(cls, table, engine): + Table(table.name, + table.metadata, + extend_existing=True, + autoload_replace=False, + autoload=True, + autoload_with=engine, + schema=table.schema) diff --git a/libs/sqlalchemy/ext/declarative/base.py b/libs/sqlalchemy/ext/declarative/base.py new file mode 100644 index 00000000..a764f126 --- /dev/null +++ b/libs/sqlalchemy/ext/declarative/base.py @@ -0,0 +1,506 @@ +# ext/declarative/base.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +"""Internal implementation for declarative.""" + +from ...schema import Table, Column +from ...orm import mapper, class_mapper, synonym +from ...orm.interfaces import MapperProperty +from ...orm.properties import ColumnProperty, CompositeProperty +from ...orm.attributes import QueryableAttribute +from ...orm.base import _is_mapped_class +from ... import util, exc +from ...sql import expression +from ... import event +from . import clsregistry +import collections +import weakref + +def _declared_mapping_info(cls): + # deferred mapping + if _DeferredMapperConfig.has_cls(cls): + return _DeferredMapperConfig.config_for_cls(cls) + # regular mapping + elif _is_mapped_class(cls): + return class_mapper(cls, configure=False) + else: + return None + + +def _as_declarative(cls, classname, dict_): + from .api import declared_attr + + # dict_ will be a dictproxy, which we can't write to, and we need to! + dict_ = dict(dict_) + + column_copies = {} + potential_columns = {} + + mapper_args_fn = None + table_args = inherited_table_args = None + tablename = None + + declarative_props = (declared_attr, util.classproperty) + + for base in cls.__mro__: + _is_declarative_inherits = hasattr(base, '_decl_class_registry') + + if '__declare_last__' in base.__dict__: + @event.listens_for(mapper, "after_configured") + def go(): + cls.__declare_last__() + if '__abstract__' in base.__dict__: + if (base is cls or + (base in cls.__bases__ and not _is_declarative_inherits) + ): + return + + class_mapped = _declared_mapping_info(base) is not None + + for name, obj in vars(base).items(): + if name == '__mapper_args__': + if not mapper_args_fn and ( + not class_mapped or + isinstance(obj, declarative_props) + ): + # don't even invoke __mapper_args__ until + # after we've determined everything about the + # mapped table. + mapper_args_fn = lambda: cls.__mapper_args__ + elif name == '__tablename__': + if not tablename and ( + not class_mapped or + isinstance(obj, declarative_props) + ): + tablename = cls.__tablename__ + elif name == '__table_args__': + if not table_args and ( + not class_mapped or + isinstance(obj, declarative_props) + ): + table_args = cls.__table_args__ + if not isinstance(table_args, (tuple, dict, type(None))): + raise exc.ArgumentError( + "__table_args__ value must be a tuple, " + "dict, or None") + if base is not cls: + inherited_table_args = True + elif class_mapped: + if isinstance(obj, declarative_props): + util.warn("Regular (i.e. not __special__) " + "attribute '%s.%s' uses @declared_attr, " + "but owning class %s is mapped - " + "not applying to subclass %s." + % (base.__name__, name, base, cls)) + continue + elif base is not cls: + # we're a mixin. + if isinstance(obj, Column): + if getattr(cls, name) is not obj: + # if column has been overridden + # (like by the InstrumentedAttribute of the + # superclass), skip + continue + if obj.foreign_keys: + raise exc.InvalidRequestError( + "Columns with foreign keys to other columns " + "must be declared as @declared_attr callables " + "on declarative mixin classes. ") + if name not in dict_ and not ( + '__table__' in dict_ and + (obj.name or name) in dict_['__table__'].c + ) and name not in potential_columns: + potential_columns[name] = \ + column_copies[obj] = \ + obj.copy() + column_copies[obj]._creation_order = \ + obj._creation_order + elif isinstance(obj, MapperProperty): + raise exc.InvalidRequestError( + "Mapper properties (i.e. deferred," + "column_property(), relationship(), etc.) must " + "be declared as @declared_attr callables " + "on declarative mixin classes.") + elif isinstance(obj, declarative_props): + dict_[name] = ret = \ + column_copies[obj] = getattr(cls, name) + if isinstance(ret, (Column, MapperProperty)) and \ + ret.doc is None: + ret.doc = obj.__doc__ + + # apply inherited columns as we should + for k, v in potential_columns.items(): + dict_[k] = v + + if inherited_table_args and not tablename: + table_args = None + + clsregistry.add_class(classname, cls) + our_stuff = util.OrderedDict() + + for k in list(dict_): + + # TODO: improve this ? all dunders ? + if k in ('__table__', '__tablename__', '__mapper_args__'): + continue + + value = dict_[k] + if isinstance(value, declarative_props): + value = getattr(cls, k) + + elif isinstance(value, QueryableAttribute) and \ + value.class_ is not cls and \ + value.key != k: + # detect a QueryableAttribute that's already mapped being + # assigned elsewhere in userland, turn into a synonym() + value = synonym(value.key) + setattr(cls, k, value) + + + if (isinstance(value, tuple) and len(value) == 1 and + isinstance(value[0], (Column, MapperProperty))): + util.warn("Ignoring declarative-like tuple value of attribute " + "%s: possibly a copy-and-paste error with a comma " + "left at the end of the line?" % k) + continue + if not isinstance(value, (Column, MapperProperty)): + if not k.startswith('__'): + dict_.pop(k) + setattr(cls, k, value) + continue + if k == 'metadata': + raise exc.InvalidRequestError( + "Attribute name 'metadata' is reserved " + "for the MetaData instance when using a " + "declarative base class." + ) + prop = clsregistry._deferred_relationship(cls, value) + our_stuff[k] = prop + + # set up attributes in the order they were created + our_stuff.sort(key=lambda key: our_stuff[key]._creation_order) + + # extract columns from the class dict + declared_columns = set() + name_to_prop_key = collections.defaultdict(set) + for key, c in list(our_stuff.items()): + if isinstance(c, (ColumnProperty, CompositeProperty)): + for col in c.columns: + if isinstance(col, Column) and \ + col.table is None: + _undefer_column_name(key, col) + if not isinstance(c, CompositeProperty): + name_to_prop_key[col.name].add(key) + declared_columns.add(col) + elif isinstance(c, Column): + _undefer_column_name(key, c) + name_to_prop_key[c.name].add(key) + declared_columns.add(c) + # if the column is the same name as the key, + # remove it from the explicit properties dict. + # the normal rules for assigning column-based properties + # will take over, including precedence of columns + # in multi-column ColumnProperties. + if key == c.key: + del our_stuff[key] + + for name, keys in name_to_prop_key.items(): + if len(keys) > 1: + util.warn( + "On class %r, Column object %r named directly multiple times, " + "only one will be used: %s" % + (classname, name, (", ".join(sorted(keys)))) + ) + + declared_columns = sorted( + declared_columns, key=lambda c: c._creation_order) + table = None + + if hasattr(cls, '__table_cls__'): + table_cls = util.unbound_method_to_callable(cls.__table_cls__) + else: + table_cls = Table + + if '__table__' not in dict_: + if tablename is not None: + + args, table_kw = (), {} + if table_args: + if isinstance(table_args, dict): + table_kw = table_args + elif isinstance(table_args, tuple): + if isinstance(table_args[-1], dict): + args, table_kw = table_args[0:-1], table_args[-1] + else: + args = table_args + + autoload = dict_.get('__autoload__') + if autoload: + table_kw['autoload'] = True + + cls.__table__ = table = table_cls( + tablename, cls.metadata, + *(tuple(declared_columns) + tuple(args)), + **table_kw) + else: + table = cls.__table__ + if declared_columns: + for c in declared_columns: + if not table.c.contains_column(c): + raise exc.ArgumentError( + "Can't add additional column %r when " + "specifying __table__" % c.key + ) + + if hasattr(cls, '__mapper_cls__'): + mapper_cls = util.unbound_method_to_callable(cls.__mapper_cls__) + else: + mapper_cls = mapper + + for c in cls.__bases__: + if _declared_mapping_info(c) is not None: + inherits = c + break + else: + inherits = None + + if table is None and inherits is None: + raise exc.InvalidRequestError( + "Class %r does not have a __table__ or __tablename__ " + "specified and does not inherit from an existing " + "table-mapped class." % cls + ) + elif inherits: + inherited_mapper = _declared_mapping_info(inherits) + inherited_table = inherited_mapper.local_table + inherited_mapped_table = inherited_mapper.mapped_table + + if table is None: + # single table inheritance. + # ensure no table args + if table_args: + raise exc.ArgumentError( + "Can't place __table_args__ on an inherited class " + "with no table." + ) + # add any columns declared here to the inherited table. + for c in declared_columns: + if c.primary_key: + raise exc.ArgumentError( + "Can't place primary key columns on an inherited " + "class with no table." + ) + if c.name in inherited_table.c: + if inherited_table.c[c.name] is c: + continue + raise exc.ArgumentError( + "Column '%s' on class %s conflicts with " + "existing column '%s'" % + (c, cls, inherited_table.c[c.name]) + ) + inherited_table.append_column(c) + if inherited_mapped_table is not None and \ + inherited_mapped_table is not inherited_table: + inherited_mapped_table._refresh_for_new_column(c) + + defer_map = hasattr(cls, '_sa_decl_prepare') + if defer_map: + cfg_cls = _DeferredMapperConfig + else: + cfg_cls = _MapperConfig + mt = cfg_cls(mapper_cls, + cls, table, + inherits, + declared_columns, + column_copies, + our_stuff, + mapper_args_fn) + if not defer_map: + mt.map() + + +class _MapperConfig(object): + + mapped_table = None + + def __init__(self, mapper_cls, + cls, + table, + inherits, + declared_columns, + column_copies, + properties, mapper_args_fn): + self.mapper_cls = mapper_cls + self.cls = cls + self.local_table = table + self.inherits = inherits + self.properties = properties + self.mapper_args_fn = mapper_args_fn + self.declared_columns = declared_columns + self.column_copies = column_copies + + + def _prepare_mapper_arguments(self): + properties = self.properties + if self.mapper_args_fn: + mapper_args = self.mapper_args_fn() + else: + mapper_args = {} + + # make sure that column copies are used rather + # than the original columns from any mixins + for k in ('version_id_col', 'polymorphic_on',): + if k in mapper_args: + v = mapper_args[k] + mapper_args[k] = self.column_copies.get(v, v) + + assert 'inherits' not in mapper_args, \ + "Can't specify 'inherits' explicitly with declarative mappings" + + if self.inherits: + mapper_args['inherits'] = self.inherits + + if self.inherits and not mapper_args.get('concrete', False): + # single or joined inheritance + # exclude any cols on the inherited table which are + # not mapped on the parent class, to avoid + # mapping columns specific to sibling/nephew classes + inherited_mapper = _declared_mapping_info(self.inherits) + inherited_table = inherited_mapper.local_table + + if 'exclude_properties' not in mapper_args: + mapper_args['exclude_properties'] = exclude_properties = \ + set([c.key for c in inherited_table.c + if c not in inherited_mapper._columntoproperty]) + exclude_properties.difference_update( + [c.key for c in self.declared_columns]) + + # look through columns in the current mapper that + # are keyed to a propname different than the colname + # (if names were the same, we'd have popped it out above, + # in which case the mapper makes this combination). + # See if the superclass has a similar column property. + # If so, join them together. + for k, col in list(properties.items()): + if not isinstance(col, expression.ColumnElement): + continue + if k in inherited_mapper._props: + p = inherited_mapper._props[k] + if isinstance(p, ColumnProperty): + # note here we place the subclass column + # first. See [ticket:1892] for background. + properties[k] = [col] + p.columns + result_mapper_args = mapper_args.copy() + result_mapper_args['properties'] = properties + return result_mapper_args + + def map(self): + mapper_args = self._prepare_mapper_arguments() + self.cls.__mapper__ = self.mapper_cls( + self.cls, + self.local_table, + **mapper_args + ) + +class _DeferredMapperConfig(_MapperConfig): + _configs = util.OrderedDict() + + @property + def cls(self): + return self._cls() + + @cls.setter + def cls(self, class_): + self._cls = weakref.ref(class_, self._remove_config_cls) + self._configs[self._cls] = self + + @classmethod + def _remove_config_cls(cls, ref): + cls._configs.pop(ref, None) + + @classmethod + def has_cls(cls, class_): + # 2.6 fails on weakref if class_ is an old style class + return isinstance(class_, type) and \ + weakref.ref(class_) in cls._configs + + @classmethod + def config_for_cls(cls, class_): + return cls._configs[weakref.ref(class_)] + + + @classmethod + def classes_for_base(cls, base_cls): + return [m for m in cls._configs.values() + if issubclass(m.cls, base_cls)] + + def map(self): + self._configs.pop(self._cls, None) + super(_DeferredMapperConfig, self).map() + + +def _add_attribute(cls, key, value): + """add an attribute to an existing declarative class. + + This runs through the logic to determine MapperProperty, + adds it to the Mapper, adds a column to the mapped Table, etc. + + """ + + if '__mapper__' in cls.__dict__: + if isinstance(value, Column): + _undefer_column_name(key, value) + cls.__table__.append_column(value) + cls.__mapper__.add_property(key, value) + elif isinstance(value, ColumnProperty): + for col in value.columns: + if isinstance(col, Column) and col.table is None: + _undefer_column_name(key, col) + cls.__table__.append_column(col) + cls.__mapper__.add_property(key, value) + elif isinstance(value, MapperProperty): + cls.__mapper__.add_property( + key, + clsregistry._deferred_relationship(cls, value) + ) + elif isinstance(value, QueryableAttribute) and value.key != key: + # detect a QueryableAttribute that's already mapped being + # assigned elsewhere in userland, turn into a synonym() + value = synonym(value.key) + cls.__mapper__.add_property( + key, + clsregistry._deferred_relationship(cls, value) + ) + else: + type.__setattr__(cls, key, value) + else: + type.__setattr__(cls, key, value) + + +def _declarative_constructor(self, **kwargs): + """A simple constructor that allows initialization from kwargs. + + Sets attributes on the constructed instance using the names and + values in ``kwargs``. + + Only keys that are present as + attributes of the instance's class are allowed. These could be, + for example, any mapped columns or relationships. + """ + cls_ = type(self) + for k in kwargs: + if not hasattr(cls_, k): + raise TypeError( + "%r is an invalid keyword argument for %s" % + (k, cls_.__name__)) + setattr(self, k, kwargs[k]) +_declarative_constructor.__name__ = '__init__' + + +def _undefer_column_name(key, column): + if column.key is None: + column.key = key + if column.name is None: + column.name = key diff --git a/libs/sqlalchemy/ext/declarative/clsregistry.py b/libs/sqlalchemy/ext/declarative/clsregistry.py new file mode 100644 index 00000000..fda1cffb --- /dev/null +++ b/libs/sqlalchemy/ext/declarative/clsregistry.py @@ -0,0 +1,305 @@ +# ext/declarative/clsregistry.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +"""Routines to handle the string class registry used by declarative. + +This system allows specification of classes and expressions used in +:func:`.relationship` using strings. + +""" +from ...orm.properties import ColumnProperty, RelationshipProperty, \ + SynonymProperty +from ...schema import _get_table_key +from ...orm import class_mapper, interfaces +from ... import util +from ... import exc +import weakref + +# strong references to registries which we place in +# the _decl_class_registry, which is usually weak referencing. +# the internal registries here link to classes with weakrefs and remove +# themselves when all references to contained classes are removed. +_registries = set() + + +def add_class(classname, cls): + """Add a class to the _decl_class_registry associated with the + given declarative class. + + """ + if classname in cls._decl_class_registry: + # class already exists. + existing = cls._decl_class_registry[classname] + if not isinstance(existing, _MultipleClassMarker): + existing = \ + cls._decl_class_registry[classname] = \ + _MultipleClassMarker([cls, existing]) + else: + cls._decl_class_registry[classname] = cls + + try: + root_module = cls._decl_class_registry['_sa_module_registry'] + except KeyError: + cls._decl_class_registry['_sa_module_registry'] = \ + root_module = _ModuleMarker('_sa_module_registry', None) + + tokens = cls.__module__.split(".") + + # build up a tree like this: + # modulename: myapp.snacks.nuts + # + # myapp->snack->nuts->(classes) + # snack->nuts->(classes) + # nuts->(classes) + # + # this allows partial token paths to be used. + while tokens: + token = tokens.pop(0) + module = root_module.get_module(token) + for token in tokens: + module = module.get_module(token) + module.add_class(classname, cls) + + +class _MultipleClassMarker(object): + """refers to multiple classes of the same name + within _decl_class_registry. + + """ + + def __init__(self, classes, on_remove=None): + self.on_remove = on_remove + self.contents = set([ + weakref.ref(item, self._remove_item) for item in classes]) + _registries.add(self) + + def __iter__(self): + return (ref() for ref in self.contents) + + def attempt_get(self, path, key): + if len(self.contents) > 1: + raise exc.InvalidRequestError( + "Multiple classes found for path \"%s\" " + "in the registry of this declarative " + "base. Please use a fully module-qualified path." % + (".".join(path + [key])) + ) + else: + ref = list(self.contents)[0] + cls = ref() + if cls is None: + raise NameError(key) + return cls + + def _remove_item(self, ref): + self.contents.remove(ref) + if not self.contents: + _registries.discard(self) + if self.on_remove: + self.on_remove() + + def add_item(self, item): + modules = set([cls().__module__ for cls in self.contents]) + if item.__module__ in modules: + util.warn( + "This declarative base already contains a class with the " + "same class name and module name as %s.%s, and will " + "be replaced in the string-lookup table." % ( + item.__module__, + item.__name__ + ) + ) + self.contents.add(weakref.ref(item, self._remove_item)) + + +class _ModuleMarker(object): + """"refers to a module name within + _decl_class_registry. + + """ + def __init__(self, name, parent): + self.parent = parent + self.name = name + self.contents = {} + self.mod_ns = _ModNS(self) + if self.parent: + self.path = self.parent.path + [self.name] + else: + self.path = [] + _registries.add(self) + + def __contains__(self, name): + return name in self.contents + + def __getitem__(self, name): + return self.contents[name] + + def _remove_item(self, name): + self.contents.pop(name, None) + if not self.contents and self.parent is not None: + self.parent._remove_item(self.name) + _registries.discard(self) + + def resolve_attr(self, key): + return getattr(self.mod_ns, key) + + def get_module(self, name): + if name not in self.contents: + marker = _ModuleMarker(name, self) + self.contents[name] = marker + else: + marker = self.contents[name] + return marker + + def add_class(self, name, cls): + if name in self.contents: + existing = self.contents[name] + existing.add_item(cls) + else: + existing = self.contents[name] = \ + _MultipleClassMarker([cls], + on_remove=lambda: self._remove_item(name)) + + +class _ModNS(object): + def __init__(self, parent): + self.__parent = parent + + def __getattr__(self, key): + try: + value = self.__parent.contents[key] + except KeyError: + pass + else: + if value is not None: + if isinstance(value, _ModuleMarker): + return value.mod_ns + else: + assert isinstance(value, _MultipleClassMarker) + return value.attempt_get(self.__parent.path, key) + raise AttributeError("Module %r has no mapped classes " + "registered under the name %r" % (self.__parent.name, key)) + + +class _GetColumns(object): + def __init__(self, cls): + self.cls = cls + + def __getattr__(self, key): + mp = class_mapper(self.cls, configure=False) + if mp: + if key not in mp.all_orm_descriptors: + raise exc.InvalidRequestError( + "Class %r does not have a mapped column named %r" + % (self.cls, key)) + + desc = mp.all_orm_descriptors[key] + if desc.extension_type is interfaces.NOT_EXTENSION: + prop = desc.property + if isinstance(prop, SynonymProperty): + key = prop.name + elif not isinstance(prop, ColumnProperty): + raise exc.InvalidRequestError( + "Property %r is not an instance of" + " ColumnProperty (i.e. does not correspond" + " directly to a Column)." % key) + return getattr(self.cls, key) + + +class _GetTable(object): + def __init__(self, key, metadata): + self.key = key + self.metadata = metadata + + def __getattr__(self, key): + return self.metadata.tables[ + _get_table_key(key, self.key) + ] + + +def _determine_container(key, value): + if isinstance(value, _MultipleClassMarker): + value = value.attempt_get([], key) + return _GetColumns(value) + + +class _class_resolver(object): + def __init__(self, cls, prop, fallback, arg): + self.cls = cls + self.prop = prop + self.arg = self._declarative_arg = arg + self.fallback = fallback + self._dict = util.PopulateDict(self._access_cls) + self._resolvers = () + + def _access_cls(self, key): + cls = self.cls + if key in cls._decl_class_registry: + return _determine_container(key, cls._decl_class_registry[key]) + elif key in cls.metadata.tables: + return cls.metadata.tables[key] + elif key in cls.metadata._schemas: + return _GetTable(key, cls.metadata) + elif '_sa_module_registry' in cls._decl_class_registry and \ + key in cls._decl_class_registry['_sa_module_registry']: + registry = cls._decl_class_registry['_sa_module_registry'] + return registry.resolve_attr(key) + elif self._resolvers: + for resolv in self._resolvers: + value = resolv(key) + if value is not None: + return value + + return self.fallback[key] + + def __call__(self): + try: + x = eval(self.arg, globals(), self._dict) + + if isinstance(x, _GetColumns): + return x.cls + else: + return x + except NameError as n: + raise exc.InvalidRequestError( + "When initializing mapper %s, expression %r failed to " + "locate a name (%r). If this is a class name, consider " + "adding this relationship() to the %r class after " + "both dependent classes have been defined." % + (self.prop.parent, self.arg, n.args[0], self.cls) + ) + + +def _resolver(cls, prop): + import sqlalchemy + from sqlalchemy.orm import foreign, remote + + fallback = sqlalchemy.__dict__.copy() + fallback.update({'foreign': foreign, 'remote': remote}) + + def resolve_arg(arg): + return _class_resolver(cls, prop, fallback, arg) + return resolve_arg + + +def _deferred_relationship(cls, prop): + + if isinstance(prop, RelationshipProperty): + resolve_arg = _resolver(cls, prop) + + for attr in ('argument', 'order_by', 'primaryjoin', 'secondaryjoin', + 'secondary', '_user_defined_foreign_keys', 'remote_side'): + v = getattr(prop, attr) + if isinstance(v, util.string_types): + setattr(prop, attr, resolve_arg(v)) + + if prop.backref and isinstance(prop.backref, tuple): + key, kwargs = prop.backref + for attr in ('primaryjoin', 'secondaryjoin', 'secondary', + 'foreign_keys', 'remote_side', 'order_by'): + if attr in kwargs and isinstance(kwargs[attr], str): + kwargs[attr] = resolve_arg(kwargs[attr]) + + return prop diff --git a/libs/sqlalchemy/ext/horizontal_shard.py b/libs/sqlalchemy/ext/horizontal_shard.py index 05b45e03..8b3f968d 100644 --- a/libs/sqlalchemy/ext/horizontal_shard.py +++ b/libs/sqlalchemy/ext/horizontal_shard.py @@ -1,5 +1,5 @@ # ext/horizontal_shard.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -14,13 +14,13 @@ the source distribution. """ -from sqlalchemy import exc as sa_exc -from sqlalchemy import util -from sqlalchemy.orm.session import Session -from sqlalchemy.orm.query import Query +from .. import util +from ..orm.session import Session +from ..orm.query import Query __all__ = ['ShardedSession', 'ShardedQuery'] + class ShardedQuery(Query): def __init__(self, *args, **kwargs): super(ShardedQuery, self).__init__(*args, **kwargs) @@ -72,28 +72,29 @@ class ShardedQuery(Query): else: return None + class ShardedSession(Session): def __init__(self, shard_chooser, id_chooser, query_chooser, shards=None, query_cls=ShardedQuery, **kwargs): """Construct a ShardedSession. - :param shard_chooser: A callable which, passed a Mapper, a mapped instance, and possibly a - SQL clause, returns a shard ID. This id may be based off of the - attributes present within the object, or on some round-robin - scheme. If the scheme is based on a selection, it should set - whatever state on the instance to mark it in the future as + :param shard_chooser: A callable which, passed a Mapper, a mapped + instance, and possibly a SQL clause, returns a shard ID. This id + may be based off of the attributes present within the object, or on + some round-robin scheme. If the scheme is based on a selection, it + should set whatever state on the instance to mark it in the future as participating in that shard. - :param id_chooser: A callable, passed a query and a tuple of identity values, which - should return a list of shard ids where the ID might reside. The - databases will be queried in the order of this listing. + :param id_chooser: A callable, passed a query and a tuple of identity + values, which should return a list of shard ids where the ID might + reside. The databases will be queried in the order of this listing. - :param query_chooser: For a given Query, returns the list of shard_ids where the query - should be issued. Results from all shards returned will be combined - together into a single listing. + :param query_chooser: For a given Query, returns the list of shard_ids + where the query should be issued. Results from all shards returned + will be combined together into a single listing. - :param shards: A dictionary of string shard names to :class:`~sqlalchemy.engine.base.Engine` - objects. + :param shards: A dictionary of string shard names + to :class:`~sqlalchemy.engine.Engine` objects. """ super(ShardedSession, self).__init__(query_cls=query_cls, **kwargs) @@ -117,12 +118,11 @@ class ShardedSession(Session): shard_id=shard_id, instance=instance).contextual_connect(**kwargs) - def get_bind(self, mapper, shard_id=None, instance=None, clause=None, **kw): + def get_bind(self, mapper, shard_id=None, + instance=None, clause=None, **kw): if shard_id is None: shard_id = self.shard_chooser(mapper, instance, clause=clause) return self.__binds[shard_id] def bind_shard(self, shard_id, bind): self.__binds[shard_id] = bind - - diff --git a/libs/sqlalchemy/ext/hybrid.py b/libs/sqlalchemy/ext/hybrid.py index 038898e4..576e0bd4 100644 --- a/libs/sqlalchemy/ext/hybrid.py +++ b/libs/sqlalchemy/ext/hybrid.py @@ -1,5 +1,5 @@ # ext/hybrid.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -9,10 +9,10 @@ "hybrid" means the attribute has distinct behaviors defined at the class level and at the instance level. -The :mod:`~sqlalchemy.ext.hybrid` extension provides a special form of method -decorator, is around 50 lines of code and has almost no dependencies on the rest -of SQLAlchemy. It can, in theory, work with any descriptor-based expression -system. +The :mod:`~sqlalchemy.ext.hybrid` extension provides a special form of +method decorator, is around 50 lines of code and has almost no +dependencies on the rest of SQLAlchemy. It can, in theory, work with +any descriptor-based expression system. Consider a mapping ``Interval``, representing integer ``start`` and ``end`` values. We can define higher level functions on mapped classes that produce @@ -51,9 +51,10 @@ as the class itself:: def intersects(self, other): return self.contains(other.start) | self.contains(other.end) -Above, the ``length`` property returns the difference between the ``end`` and -``start`` attributes. With an instance of ``Interval``, this subtraction occurs -in Python, using normal Python descriptor mechanics:: +Above, the ``length`` property returns the difference between the +``end`` and ``start`` attributes. With an instance of ``Interval``, +this subtraction occurs in Python, using normal Python descriptor +mechanics:: >>> i1 = Interval(5, 10) >>> i1.length @@ -82,11 +83,12 @@ locate attributes, so can also be used with hybrid attributes:: FROM interval WHERE interval."end" - interval.start = :param_1 -The ``Interval`` class example also illustrates two methods, ``contains()`` and ``intersects()``, -decorated with :class:`.hybrid_method`. -This decorator applies the same idea to methods that :class:`.hybrid_property` applies -to attributes. The methods return boolean values, and take advantage -of the Python ``|`` and ``&`` bitwise operators to produce equivalent instance-level and +The ``Interval`` class example also illustrates two methods, +``contains()`` and ``intersects()``, decorated with +:class:`.hybrid_method`. This decorator applies the same idea to +methods that :class:`.hybrid_property` applies to attributes. The +methods return boolean values, and take advantage of the Python ``|`` +and ``&`` bitwise operators to produce equivalent instance-level and SQL expression-level boolean behavior:: >>> i1.contains(6) @@ -118,12 +120,15 @@ SQL expression-level boolean behavior:: Defining Expression Behavior Distinct from Attribute Behavior -------------------------------------------------------------- -Our usage of the ``&`` and ``|`` bitwise operators above was fortunate, considering -our functions operated on two boolean values to return a new one. In many cases, the construction -of an in-Python function and a SQLAlchemy SQL expression have enough differences that two -separate Python expressions should be defined. The :mod:`~sqlalchemy.ext.hybrid` decorators -define the :meth:`.hybrid_property.expression` modifier for this purpose. As an example we'll -define the radius of the interval, which requires the usage of the absolute value function:: +Our usage of the ``&`` and ``|`` bitwise operators above was +fortunate, considering our functions operated on two boolean values to +return a new one. In many cases, the construction of an in-Python +function and a SQLAlchemy SQL expression have enough differences that +two separate Python expressions should be defined. The +:mod:`~sqlalchemy.ext.hybrid` decorators define the +:meth:`.hybrid_property.expression` modifier for this purpose. As an +example we'll define the radius of the interval, which requires the +usage of the absolute value function:: from sqlalchemy import func @@ -138,8 +143,9 @@ define the radius of the interval, which requires the usage of the absolute valu def radius(cls): return func.abs(cls.length) / 2 -Above the Python function ``abs()`` is used for instance-level operations, the SQL function -``ABS()`` is used via the :attr:`.func` object for class-level expressions:: +Above the Python function ``abs()`` is used for instance-level +operations, the SQL function ``ABS()`` is used via the :attr:`.func` +object for class-level expressions:: >>> i1.radius 2 @@ -153,8 +159,8 @@ Above the Python function ``abs()`` is used for instance-level operations, the S Defining Setters ---------------- -Hybrid properties can also define setter methods. If we wanted ``length`` above, when -set, to modify the endpoint value:: +Hybrid properties can also define setter methods. If we wanted +``length`` above, when set, to modify the endpoint value:: class Interval(object): # ... @@ -223,7 +229,7 @@ mapping which relates a ``User`` to a ``SavingsAccount``:: account = Account(owner=self) else: account = self.accounts[0] - account.balance = balance + account.balance = value @balance.expression def balance(cls): @@ -234,8 +240,8 @@ The above hybrid property ``balance`` works with the first in-Python getter/setter methods can treat ``accounts`` as a Python list available on ``self``. -However, at the expression level, it's expected that the ``User`` class will be used -in an appropriate context such that an appropriate join to +However, at the expression level, it's expected that the ``User`` class will +be used in an appropriate context such that an appropriate join to ``SavingsAccount`` will be present:: >>> print Session().query(User, User.balance).\\ @@ -262,11 +268,10 @@ Correlated Subquery Relationship Hybrid ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We can, of course, forego being dependent on the enclosing query's usage -of joins in favor of the correlated -subquery, which can portably be packed into a single colunn expression. -A correlated subquery is more portable, but often performs more poorly -at the SQL level. -Using the same technique illustrated at :ref:`mapper_column_property_sql_expressions`, +of joins in favor of the correlated subquery, which can portably be packed +into a single column expression. A correlated subquery is more portable, but +often performs more poorly at the SQL level. Using the same technique +illustrated at :ref:`mapper_column_property_sql_expressions`, we can adjust our ``SavingsAccount`` example to aggregate the balances for *all* accounts, and use a correlated subquery for the column expression:: @@ -316,10 +321,11 @@ a correlated SELECT:: Building Custom Comparators --------------------------- -The hybrid property also includes a helper that allows construction of custom comparators. -A comparator object allows one to customize the behavior of each SQLAlchemy expression -operator individually. They are useful when creating custom types that have -some highly idiosyncratic behavior on the SQL side. +The hybrid property also includes a helper that allows construction of +custom comparators. A comparator object allows one to customize the +behavior of each SQLAlchemy expression operator individually. They +are useful when creating custom types that have some highly +idiosyncratic behavior on the SQL side. The example class below allows case-insensitive comparisons on the attribute named ``word_insensitive``:: @@ -356,9 +362,10 @@ SQL function to both sides:: FROM searchword WHERE lower(searchword.word) = lower(:lower_1) -The ``CaseInsensitiveComparator`` above implements part of the :class:`.ColumnOperators` -interface. A "coercion" operation like lowercasing can be applied to all comparison operations -(i.e. ``eq``, ``lt``, ``gt``, etc.) using :meth:`.Operators.operate`:: +The ``CaseInsensitiveComparator`` above implements part of the +:class:`.ColumnOperators` interface. A "coercion" operation like +lowercasing can be applied to all comparison operations (i.e. ``eq``, +``lt``, ``gt``, etc.) using :meth:`.Operators.operate`:: class CaseInsensitiveComparator(Comparator): def operate(self, op, other): @@ -367,17 +374,20 @@ interface. A "coercion" operation like lowercasing can be applied to all compa Hybrid Value Objects -------------------- -Note in our previous example, if we were to compare the ``word_insensitive`` attribute of -a ``SearchWord`` instance to a plain Python string, the plain Python string would not -be coerced to lower case - the ``CaseInsensitiveComparator`` we built, being returned -by ``@word_insensitive.comparator``, only applies to the SQL side. +Note in our previous example, if we were to compare the +``word_insensitive`` attribute of a ``SearchWord`` instance to a plain +Python string, the plain Python string would not be coerced to lower +case - the ``CaseInsensitiveComparator`` we built, being returned by +``@word_insensitive.comparator``, only applies to the SQL side. -A more comprehensive form of the custom comparator is to construct a *Hybrid Value Object*. -This technique applies the target value or expression to a value object which is then -returned by the accessor in all cases. The value object allows control -of all operations upon the value as well as how compared values are treated, both -on the SQL expression side as well as the Python value side. Replacing the -previous ``CaseInsensitiveComparator`` class with a new ``CaseInsensitiveWord`` class:: +A more comprehensive form of the custom comparator is to construct a +*Hybrid Value Object*. This technique applies the target value or +expression to a value object which is then returned by the accessor in +all cases. The value object allows control of all operations upon +the value as well as how compared values are treated, both on the SQL +expression side as well as the Python value side. Replacing the +previous ``CaseInsensitiveComparator`` class with a new +``CaseInsensitiveWord`` class:: class CaseInsensitiveWord(Comparator): "Hybrid value representing a lower case representation of a word." @@ -404,12 +414,13 @@ previous ``CaseInsensitiveComparator`` class with a new ``CaseInsensitiveWord`` key = 'word' "Label to apply to Query tuple results" -Above, the ``CaseInsensitiveWord`` object represents ``self.word``, which may be a SQL function, -or may be a Python native. By overriding ``operate()`` and ``__clause_element__()`` -to work in terms of ``self.word``, all comparison operations will work against the +Above, the ``CaseInsensitiveWord`` object represents ``self.word``, +which may be a SQL function, or may be a Python native. By +overriding ``operate()`` and ``__clause_element__()`` to work in terms +of ``self.word``, all comparison operations will work against the "converted" form of ``word``, whether it be SQL side or Python side. -Our ``SearchWord`` class can now deliver the ``CaseInsensitiveWord`` object unconditionally -from a single hybrid call:: +Our ``SearchWord`` class can now deliver the ``CaseInsensitiveWord`` +object unconditionally from a single hybrid call:: class SearchWord(Base): __tablename__ = 'searchword' @@ -420,9 +431,10 @@ from a single hybrid call:: def word_insensitive(self): return CaseInsensitiveWord(self.word) -The ``word_insensitive`` attribute now has case-insensitive comparison behavior -universally, including SQL expression vs. Python expression (note the Python value is -converted to lower case on the Python side here):: +The ``word_insensitive`` attribute now has case-insensitive comparison +behavior universally, including SQL expression vs. Python expression +(note the Python value is converted to lower case on the Python side +here):: >>> print Session().query(SearchWord).filter_by(word_insensitive="Trucks") SELECT searchword.id AS searchword_id, searchword.word AS searchword_word @@ -439,7 +451,8 @@ SQL expression versus SQL expression:: ... filter( ... sw1.word_insensitive > sw2.word_insensitive ... ) - SELECT lower(searchword_1.word) AS lower_1, lower(searchword_2.word) AS lower_2 + SELECT lower(searchword_1.word) AS lower_1, + lower(searchword_2.word) AS lower_2 FROM searchword AS searchword_1, searchword AS searchword_2 WHERE lower(searchword_1.word) > lower(searchword_2.word) @@ -453,30 +466,36 @@ Python only expression:: >>> print ws1.word_insensitive someword -The Hybrid Value pattern is very useful for any kind of value that may have multiple representations, -such as timestamps, time deltas, units of measurement, currencies and encrypted passwords. +The Hybrid Value pattern is very useful for any kind of value that may +have multiple representations, such as timestamps, time deltas, units +of measurement, currencies and encrypted passwords. -See Also: +.. seealso:: -`Hybrids and Value Agnostic Types `_ - on the techspot.zzzeek.org blog + `Hybrids and Value Agnostic Types + `_ - + on the techspot.zzzeek.org blog -`Value Agnostic Types, Part II `_ - on the techspot.zzzeek.org blog + `Value Agnostic Types, Part II + `_ - + on the techspot.zzzeek.org blog .. _hybrid_transformers: Building Transformers ---------------------- -A *transformer* is an object which can receive a :class:`.Query` object and return a -new one. The :class:`.Query` object includes a method :meth:`.with_transformation` -that simply returns a new :class:`.Query` transformed by the given function. +A *transformer* is an object which can receive a :class:`.Query` +object and return a new one. The :class:`.Query` object includes a +method :meth:`.with_transformation` that returns a new :class:`.Query` +transformed by the given function. We can combine this with the :class:`.Comparator` class to produce one type of recipe which can both set up the FROM clause of a query as well as assign filtering criterion. -Consider a mapped class ``Node``, which assembles using adjacency list into a hierarchical -tree pattern:: +Consider a mapped class ``Node``, which assembles using adjacency list +into a hierarchical tree pattern:: from sqlalchemy import Column, Integer, ForeignKey from sqlalchemy.orm import relationship @@ -489,8 +508,9 @@ tree pattern:: parent_id = Column(Integer, ForeignKey('node.id')) parent = relationship("Node", remote_side=id) -Suppose we wanted to add an accessor ``grandparent``. This would return the ``parent`` of -``Node.parent``. When we have an instance of ``Node``, this is simple:: +Suppose we wanted to add an accessor ``grandparent``. This would +return the ``parent`` of ``Node.parent``. When we have an instance of +``Node``, this is simple:: from sqlalchemy.ext.hybrid import hybrid_property @@ -501,11 +521,13 @@ Suppose we wanted to add an accessor ``grandparent``. This would return the ``p def grandparent(self): return self.parent.parent -For the expression, things are not so clear. We'd need to construct a :class:`.Query` where we -:meth:`~.Query.join` twice along ``Node.parent`` to get to the ``grandparent``. We can instead -return a transforming callable that we'll combine with the :class:`.Comparator` class -to receive any :class:`.Query` object, and return a new one that's joined to the ``Node.parent`` -attribute and filtered based on the given criterion:: +For the expression, things are not so clear. We'd need to construct +a :class:`.Query` where we :meth:`~.Query.join` twice along +``Node.parent`` to get to the ``grandparent``. We can instead return +a transforming callable that we'll combine with the +:class:`.Comparator` class to receive any :class:`.Query` object, and +return a new one that's joined to the ``Node.parent`` attribute and +filtered based on the given criterion:: from sqlalchemy.ext.hybrid import Comparator @@ -534,15 +556,17 @@ attribute and filtered based on the given criterion:: def grandparent(cls): return GrandparentTransformer(cls) -The ``GrandparentTransformer`` overrides the core :meth:`.Operators.operate` method -at the base of the :class:`.Comparator` hierarchy to return a query-transforming -callable, which then runs the given comparison operation in a particular context. -Such as, in the example above, the ``operate`` method is called, given the -:attr:`.Operators.eq` callable as well as the right side of the comparison -``Node(id=5)``. A function ``transform`` is then returned which will transform -a :class:`.Query` first to join to ``Node.parent``, then to compare ``parent_alias`` -using :attr:`.Operators.eq` against the left and right sides, passing into -:class:`.Query.filter`: +The ``GrandparentTransformer`` overrides the core +:meth:`.Operators.operate` method at the base of the +:class:`.Comparator` hierarchy to return a query-transforming +callable, which then runs the given comparison operation in a +particular context. Such as, in the example above, the ``operate`` +method is called, given the :attr:`.Operators.eq` callable as well as +the right side of the comparison ``Node(id=5)``. A function +``transform`` is then returned which will transform a :class:`.Query` +first to join to ``Node.parent``, then to compare ``parent_alias`` +using :attr:`.Operators.eq` against the left and right sides, passing +into :class:`.Query.filter`: .. sourcecode:: pycon+sql @@ -605,15 +629,43 @@ While it's only recommended for advanced and/or patient developers, there's probably a whole lot of amazing things it can be used for. """ -from sqlalchemy import util -from sqlalchemy.orm import attributes, interfaces +from .. import util +from ..orm import attributes, interfaces -class hybrid_method(object): +HYBRID_METHOD = util.symbol('HYBRID_METHOD') +"""Symbol indicating an :class:`_InspectionAttr` that's + of type :class:`.hybrid_method`. + + Is assigned to the :attr:`._InspectionAttr.extension_type` + attibute. + + .. seealso:: + + :attr:`.Mapper.all_orm_attributes` + +""" + +HYBRID_PROPERTY = util.symbol('HYBRID_PROPERTY') +"""Symbol indicating an :class:`_InspectionAttr` that's + of type :class:`.hybrid_method`. + + Is assigned to the :attr:`._InspectionAttr.extension_type` + attibute. + + .. seealso:: + + :attr:`.Mapper.all_orm_attributes` + +""" + +class hybrid_method(interfaces._InspectionAttr): """A decorator which allows definition of a Python object method with both instance-level and class-level behavior. """ + is_attribute = True + extension_type = HYBRID_METHOD def __init__(self, func, expr=None): """Create a new :class:`.hybrid_method`. @@ -642,17 +694,22 @@ class hybrid_method(object): return self.func.__get__(instance, owner) def expression(self, expr): - """Provide a modifying decorator that defines a SQL-expression producing method.""" + """Provide a modifying decorator that defines a + SQL-expression producing method.""" self.expr = expr return self -class hybrid_property(object): + +class hybrid_property(interfaces._InspectionAttr): """A decorator which allows definition of a Python descriptor with both instance-level and class-level behavior. """ + is_attribute = True + extension_type = HYBRID_PROPERTY + def __init__(self, fget, fset=None, fdel=None, expr=None): """Create a new :class:`.hybrid_property`. @@ -699,19 +756,22 @@ class hybrid_property(object): return self def deleter(self, fdel): - """Provide a modifying decorator that defines a value-deletion method.""" + """Provide a modifying decorator that defines a + value-deletion method.""" self.fdel = fdel return self def expression(self, expr): - """Provide a modifying decorator that defines a SQL-expression producing method.""" + """Provide a modifying decorator that defines a SQL-expression + producing method.""" self.expr = expr return self def comparator(self, comparator): - """Provide a modifying decorator that defines a custom comparator producing method. + """Provide a modifying decorator that defines a custom + comparator producing method. The return value of the decorated method should be an instance of :class:`~.hybrid.Comparator`. @@ -720,6 +780,7 @@ class hybrid_property(object): proxy_attr = attributes.\ create_proxied_attribute(self) + def expr(owner): return proxy_attr(owner, self.__name__, self, comparator(owner)) self.expr = expr @@ -727,9 +788,11 @@ class hybrid_property(object): class Comparator(interfaces.PropComparator): - """A helper class that allows easy construction of custom :class:`~.orm.interfaces.PropComparator` + """A helper class that allows easy construction of custom + :class:`~.orm.interfaces.PropComparator` classes for usage with hybrids.""" + property = None def __init__(self, expression): self.expression = expression @@ -740,8 +803,6 @@ class Comparator(interfaces.PropComparator): expr = expr.__clause_element__() return expr - def adapted(self, adapter): + def adapt_to_entity(self, adapt_to_entity): # interesting.... return self - - diff --git a/libs/sqlalchemy/ext/instrumentation.py b/libs/sqlalchemy/ext/instrumentation.py new file mode 100644 index 00000000..2cf36e9b --- /dev/null +++ b/libs/sqlalchemy/ext/instrumentation.py @@ -0,0 +1,407 @@ +"""Extensible class instrumentation. + +The :mod:`sqlalchemy.ext.instrumentation` package provides for alternate +systems of class instrumentation within the ORM. Class instrumentation +refers to how the ORM places attributes on the class which maintain +data and track changes to that data, as well as event hooks installed +on the class. + +.. note:: + The extension package is provided for the benefit of integration + with other object management packages, which already perform + their own instrumentation. It is not intended for general use. + +For examples of how the instrumentation extension is used, +see the example :ref:`examples_instrumentation`. + +.. versionchanged:: 0.8 + The :mod:`sqlalchemy.orm.instrumentation` was split out so + that all functionality having to do with non-standard + instrumentation was moved out to :mod:`sqlalchemy.ext.instrumentation`. + When imported, the module installs itself within + :mod:`sqlalchemy.orm.instrumentation` so that it + takes effect, including recognition of + ``__sa_instrumentation_manager__`` on mapped classes, as + well :data:`.instrumentation_finders` + being used to determine class instrumentation resolution. + +""" +from ..orm import instrumentation as orm_instrumentation +from ..orm.instrumentation import ( + ClassManager, InstrumentationFactory, _default_state_getter, + _default_dict_getter, _default_manager_getter +) +from ..orm import attributes, collections, base as orm_base +from .. import util +from ..orm import exc as orm_exc +import weakref + +INSTRUMENTATION_MANAGER = '__sa_instrumentation_manager__' +"""Attribute, elects custom instrumentation when present on a mapped class. + +Allows a class to specify a slightly or wildly different technique for +tracking changes made to mapped attributes and collections. + +Only one instrumentation implementation is allowed in a given object +inheritance hierarchy. + +The value of this attribute must be a callable and will be passed a class +object. The callable must return one of: + + - An instance of an InstrumentationManager or subclass + - An object implementing all or some of InstrumentationManager (TODO) + - A dictionary of callables, implementing all or some of the above (TODO) + - An instance of a ClassManager or subclass + +This attribute is consulted by SQLAlchemy instrumentation +resolution, once the :mod:`sqlalchemy.ext.instrumentation` module +has been imported. If custom finders are installed in the global +instrumentation_finders list, they may or may not choose to honor this +attribute. + +""" + + +def find_native_user_instrumentation_hook(cls): + """Find user-specified instrumentation management for a class.""" + return getattr(cls, INSTRUMENTATION_MANAGER, None) + +instrumentation_finders = [find_native_user_instrumentation_hook] +"""An extensible sequence of callables which return instrumentation +implementations + +When a class is registered, each callable will be passed a class object. +If None is returned, the +next finder in the sequence is consulted. Otherwise the return must be an +instrumentation factory that follows the same guidelines as +sqlalchemy.ext.instrumentation.INSTRUMENTATION_MANAGER. + +By default, the only finder is find_native_user_instrumentation_hook, which +searches for INSTRUMENTATION_MANAGER. If all finders return None, standard +ClassManager instrumentation is used. + +""" + + +class ExtendedInstrumentationRegistry(InstrumentationFactory): + """Extends :class:`.InstrumentationFactory` with additional + bookkeeping, to accommodate multiple types of + class managers. + + """ + _manager_finders = weakref.WeakKeyDictionary() + _state_finders = weakref.WeakKeyDictionary() + _dict_finders = weakref.WeakKeyDictionary() + _extended = False + + def _locate_extended_factory(self, class_): + for finder in instrumentation_finders: + factory = finder(class_) + if factory is not None: + manager = self._extended_class_manager(class_, factory) + return manager, factory + else: + return None, None + + def _check_conflicts(self, class_, factory): + existing_factories = self._collect_management_factories_for(class_).\ + difference([factory]) + if existing_factories: + raise TypeError( + "multiple instrumentation implementations specified " + "in %s inheritance hierarchy: %r" % ( + class_.__name__, list(existing_factories))) + + def _extended_class_manager(self, class_, factory): + manager = factory(class_) + if not isinstance(manager, ClassManager): + manager = _ClassInstrumentationAdapter(class_, manager) + + if factory != ClassManager and not self._extended: + # somebody invoked a custom ClassManager. + # reinstall global "getter" functions with the more + # expensive ones. + self._extended = True + _install_instrumented_lookups() + + self._manager_finders[class_] = manager.manager_getter() + self._state_finders[class_] = manager.state_getter() + self._dict_finders[class_] = manager.dict_getter() + return manager + + def _collect_management_factories_for(self, cls): + """Return a collection of factories in play or specified for a + hierarchy. + + Traverses the entire inheritance graph of a cls and returns a + collection of instrumentation factories for those classes. Factories + are extracted from active ClassManagers, if available, otherwise + instrumentation_finders is consulted. + + """ + hierarchy = util.class_hierarchy(cls) + factories = set() + for member in hierarchy: + manager = self.manager_of_class(member) + if manager is not None: + factories.add(manager.factory) + else: + for finder in instrumentation_finders: + factory = finder(member) + if factory is not None: + break + else: + factory = None + factories.add(factory) + factories.discard(None) + return factories + + def unregister(self, class_): + if class_ in self._manager_finders: + del self._manager_finders[class_] + del self._state_finders[class_] + del self._dict_finders[class_] + super(ExtendedInstrumentationRegistry, self).unregister(class_) + + def manager_of_class(self, cls): + if cls is None: + return None + return self._manager_finders.get(cls, _default_manager_getter)(cls) + + def state_of(self, instance): + if instance is None: + raise AttributeError("None has no persistent state.") + return self._state_finders.get( + instance.__class__, _default_state_getter)(instance) + + def dict_of(self, instance): + if instance is None: + raise AttributeError("None has no persistent state.") + return self._dict_finders.get( + instance.__class__, _default_dict_getter)(instance) + + +orm_instrumentation._instrumentation_factory = \ + _instrumentation_factory = ExtendedInstrumentationRegistry() +orm_instrumentation.instrumentation_finders = instrumentation_finders + + +class InstrumentationManager(object): + """User-defined class instrumentation extension. + + :class:`.InstrumentationManager` can be subclassed in order + to change + how class instrumentation proceeds. This class exists for + the purposes of integration with other object management + frameworks which would like to entirely modify the + instrumentation methodology of the ORM, and is not intended + for regular usage. For interception of class instrumentation + events, see :class:`.InstrumentationEvents`. + + The API for this class should be considered as semi-stable, + and may change slightly with new releases. + + .. versionchanged:: 0.8 + :class:`.InstrumentationManager` was moved from + :mod:`sqlalchemy.orm.instrumentation` to + :mod:`sqlalchemy.ext.instrumentation`. + + """ + + # r4361 added a mandatory (cls) constructor to this interface. + # given that, perhaps class_ should be dropped from all of these + # signatures. + + def __init__(self, class_): + pass + + def manage(self, class_, manager): + setattr(class_, '_default_class_manager', manager) + + def dispose(self, class_, manager): + delattr(class_, '_default_class_manager') + + def manager_getter(self, class_): + def get(cls): + return cls._default_class_manager + return get + + def instrument_attribute(self, class_, key, inst): + pass + + def post_configure_attribute(self, class_, key, inst): + pass + + def install_descriptor(self, class_, key, inst): + setattr(class_, key, inst) + + def uninstall_descriptor(self, class_, key): + delattr(class_, key) + + def install_member(self, class_, key, implementation): + setattr(class_, key, implementation) + + def uninstall_member(self, class_, key): + delattr(class_, key) + + def instrument_collection_class(self, class_, key, collection_class): + return collections.prepare_instrumentation(collection_class) + + def get_instance_dict(self, class_, instance): + return instance.__dict__ + + def initialize_instance_dict(self, class_, instance): + pass + + def install_state(self, class_, instance, state): + setattr(instance, '_default_state', state) + + def remove_state(self, class_, instance): + delattr(instance, '_default_state') + + def state_getter(self, class_): + return lambda instance: getattr(instance, '_default_state') + + def dict_getter(self, class_): + return lambda inst: self.get_instance_dict(class_, inst) + + +class _ClassInstrumentationAdapter(ClassManager): + """Adapts a user-defined InstrumentationManager to a ClassManager.""" + + def __init__(self, class_, override): + self._adapted = override + self._get_state = self._adapted.state_getter(class_) + self._get_dict = self._adapted.dict_getter(class_) + + ClassManager.__init__(self, class_) + + def manage(self): + self._adapted.manage(self.class_, self) + + def dispose(self): + self._adapted.dispose(self.class_) + + def manager_getter(self): + return self._adapted.manager_getter(self.class_) + + def instrument_attribute(self, key, inst, propagated=False): + ClassManager.instrument_attribute(self, key, inst, propagated) + if not propagated: + self._adapted.instrument_attribute(self.class_, key, inst) + + def post_configure_attribute(self, key): + super(_ClassInstrumentationAdapter, self).post_configure_attribute(key) + self._adapted.post_configure_attribute(self.class_, key, self[key]) + + def install_descriptor(self, key, inst): + self._adapted.install_descriptor(self.class_, key, inst) + + def uninstall_descriptor(self, key): + self._adapted.uninstall_descriptor(self.class_, key) + + def install_member(self, key, implementation): + self._adapted.install_member(self.class_, key, implementation) + + def uninstall_member(self, key): + self._adapted.uninstall_member(self.class_, key) + + def instrument_collection_class(self, key, collection_class): + return self._adapted.instrument_collection_class( + self.class_, key, collection_class) + + def initialize_collection(self, key, state, factory): + delegate = getattr(self._adapted, 'initialize_collection', None) + if delegate: + return delegate(key, state, factory) + else: + return ClassManager.initialize_collection(self, key, + state, factory) + + def new_instance(self, state=None): + instance = self.class_.__new__(self.class_) + self.setup_instance(instance, state) + return instance + + def _new_state_if_none(self, instance): + """Install a default InstanceState if none is present. + + A private convenience method used by the __init__ decorator. + """ + if self.has_state(instance): + return False + else: + return self.setup_instance(instance) + + def setup_instance(self, instance, state=None): + self._adapted.initialize_instance_dict(self.class_, instance) + + if state is None: + state = self._state_constructor(instance, self) + + # the given instance is assumed to have no state + self._adapted.install_state(self.class_, instance, state) + return state + + def teardown_instance(self, instance): + self._adapted.remove_state(self.class_, instance) + + def has_state(self, instance): + try: + self._get_state(instance) + except orm_exc.NO_STATE: + return False + else: + return True + + def state_getter(self): + return self._get_state + + def dict_getter(self): + return self._get_dict + + +def _install_instrumented_lookups(): + """Replace global class/object management functions + with ExtendedInstrumentationRegistry implementations, which + allow multiple types of class managers to be present, + at the cost of performance. + + This function is called only by ExtendedInstrumentationRegistry + and unit tests specific to this behavior. + + The _reinstall_default_lookups() function can be called + after this one to re-establish the default functions. + + """ + _install_lookups( + dict( + instance_state=_instrumentation_factory.state_of, + instance_dict=_instrumentation_factory.dict_of, + manager_of_class=_instrumentation_factory.manager_of_class + ) + ) + + +def _reinstall_default_lookups(): + """Restore simplified lookups.""" + _install_lookups( + dict( + instance_state=_default_state_getter, + instance_dict=_default_dict_getter, + manager_of_class=_default_manager_getter + ) + ) + + +def _install_lookups(lookups): + global instance_state, instance_dict, manager_of_class + instance_state = lookups['instance_state'] + instance_dict = lookups['instance_dict'] + manager_of_class = lookups['manager_of_class'] + orm_base.instance_state = attributes.instance_state = \ + orm_instrumentation.instance_state = instance_state + orm_base.instance_dict = attributes.instance_dict = \ + orm_instrumentation.instance_dict = instance_dict + orm_base.manager_of_class = attributes.manager_of_class = \ + orm_instrumentation.manager_of_class = manager_of_class diff --git a/libs/sqlalchemy/ext/mutable.py b/libs/sqlalchemy/ext/mutable.py index 36e8fcaf..82410031 100644 --- a/libs/sqlalchemy/ext/mutable.py +++ b/libs/sqlalchemy/ext/mutable.py @@ -1,5 +1,5 @@ # ext/mutable.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,13 +7,9 @@ """Provide support for tracking of in-place changes to scalar values, which are propagated into ORM change events on owning parent objects. -The :mod:`sqlalchemy.ext.mutable` extension replaces SQLAlchemy's legacy approach to in-place -mutations of scalar values, established by the :class:`.types.MutableType` -class as well as the ``mutable=True`` type flag, with a system that allows -change events to be propagated from the value to the owning parent, thereby -removing the need for the ORM to maintain copies of values as well as the very -expensive requirement of scanning through all "mutable" values on each flush -call, looking for changes. +.. versionadded:: 0.7 :mod:`sqlalchemy.ext.mutable` replaces SQLAlchemy's + legacy approach to in-place mutations of scalar values; see + :ref:`07_migration_mutation_extension`. .. _mutable_scalars: @@ -43,27 +39,27 @@ JSON strings before being persisted:: value = json.loads(value) return value -The usage of ``json`` is only for the purposes of example. The :mod:`sqlalchemy.ext.mutable` -extension can be used +The usage of ``json`` is only for the purposes of example. The +:mod:`sqlalchemy.ext.mutable` extension can be used with any type whose target Python type may be mutable, including :class:`.PickleType`, :class:`.postgresql.ARRAY`, etc. When using the :mod:`sqlalchemy.ext.mutable` extension, the value itself -tracks all parents which reference it. Here we will replace the usage -of plain Python dictionaries with a dict subclass that implements -the :class:`.Mutable` mixin:: +tracks all parents which reference it. Below, we illustrate the a simple +version of the :class:`.MutableDict` dictionary object, which applies +the :class:`.Mutable` mixin to a plain Python dictionary:: import collections from sqlalchemy.ext.mutable import Mutable - class MutationDict(Mutable, dict): + class MutableDict(Mutable, dict): @classmethod def coerce(cls, key, value): - "Convert plain dictionaries to MutationDict." + "Convert plain dictionaries to MutableDict." - if not isinstance(value, MutationDict): + if not isinstance(value, MutableDict): if isinstance(value, dict): - return MutationDict(value) + return MutableDict(value) # this call will raise ValueError return Mutable.coerce(key, value) @@ -84,23 +80,23 @@ the :class:`.Mutable` mixin:: The above dictionary class takes the approach of subclassing the Python built-in ``dict`` to produce a dict -subclass which routes all mutation events through ``__setitem__``. There are -many variants on this approach, such as subclassing ``UserDict.UserDict``, -the newer ``collections.MutableMapping``, etc. The part that's important to this -example is that the :meth:`.Mutable.changed` method is called whenever an in-place change to the -datastructure takes place. +subclass which routes all mutation events through ``__setitem__``. There are +variants on this approach, such as subclassing ``UserDict.UserDict`` or +``collections.MutableMapping``; the part that's important to this example is +that the :meth:`.Mutable.changed` method is called whenever an in-place +change to the datastructure takes place. We also redefine the :meth:`.Mutable.coerce` method which will be used to -convert any values that are not instances of ``MutationDict``, such +convert any values that are not instances of ``MutableDict``, such as the plain dictionaries returned by the ``json`` module, into the -appropriate type. Defining this method is optional; we could just as well created our -``JSONEncodedDict`` such that it always returns an instance of ``MutationDict``, -and additionally ensured that all calling code uses ``MutationDict`` -explicitly. When :meth:`.Mutable.coerce` is not overridden, any values -applied to a parent object which are not instances of the mutable type -will raise a ``ValueError``. +appropriate type. Defining this method is optional; we could just as well +created our ``JSONEncodedDict`` such that it always returns an instance +of ``MutableDict``, and additionally ensured that all calling code +uses ``MutableDict`` explicitly. When :meth:`.Mutable.coerce` is not +overridden, any values applied to a parent object which are not instances +of the mutable type will raise a ``ValueError``. -Our new ``MutationDict`` type offers a class method +Our new ``MutableDict`` type offers a class method :meth:`~.Mutable.as_mutable` which we can use within column metadata to associate with types. This method grabs the given type object or class and associates a listener that will detect all future mappings @@ -111,7 +107,7 @@ attribute. Such as, with classical table metadata:: my_data = Table('my_data', metadata, Column('id', Integer, primary_key=True), - Column('data', MutationDict.as_mutable(JSONEncodedDict)) + Column('data', MutableDict.as_mutable(JSONEncodedDict)) ) Above, :meth:`~.Mutable.as_mutable` returns an instance of ``JSONEncodedDict`` @@ -139,7 +135,7 @@ There's no difference in usage when using declarative:: class MyDataClass(Base): __tablename__ = 'my_data' id = Column(Integer, primary_key=True) - data = Column(MutationDict.as_mutable(JSONEncodedDict)) + data = Column(MutableDict.as_mutable(JSONEncodedDict)) Any in-place changes to the ``MyDataClass.data`` member will flag the attribute as "dirty" on the parent object:: @@ -155,13 +151,14 @@ will flag the attribute as "dirty" on the parent object:: >>> assert m1 in sess.dirty True -The ``MutationDict`` can be associated with all future instances -of ``JSONEncodedDict`` in one step, using :meth:`~.Mutable.associate_with`. This -is similar to :meth:`~.Mutable.as_mutable` except it will intercept -all occurrences of ``MutationDict`` in all mappings unconditionally, without +The ``MutableDict`` can be associated with all future instances +of ``JSONEncodedDict`` in one step, using +:meth:`~.Mutable.associate_with`. This is similar to +:meth:`~.Mutable.as_mutable` except it will intercept all occurrences +of ``MutableDict`` in all mappings unconditionally, without the need to declare it individually:: - MutationDict.associate_with(JSONEncodedDict) + MutableDict.associate_with(JSONEncodedDict) class MyDataClass(Base): __tablename__ = 'my_data' @@ -181,7 +178,7 @@ callbacks. In our case, this is a good thing, since if this dictionary were picklable, it could lead to an excessively large pickle size for our value objects that are pickled by themselves outside of the context of the parent. The developer responsibility here is only to provide a ``__getstate__`` method -that excludes the :meth:`~.MutableBase._parents` collection from the pickle +that excludes the :meth:`~MutableBase._parents` collection from the pickle stream:: class MyMutableType(Mutable): @@ -193,7 +190,7 @@ stream:: With our dictionary example, we need to return the contents of the dict itself (and also restore them on __setstate__):: - class MutationDict(Mutable, dict): + class MutableDict(Mutable, dict): # .... def __getstate__(self): @@ -331,7 +328,7 @@ Supporting Pickling As is the case with :class:`.Mutable`, the :class:`.MutableComposite` helper class uses a ``weakref.WeakKeyDictionary`` available via the -:meth:`.MutableBase._parents` attribute which isn't picklable. If we need to +:meth:`MutableBase._parents` attribute which isn't picklable. If we need to pickle instances of ``Point`` or its owning class ``Vertex``, we at least need to define a ``__getstate__`` that doesn't include the ``_parents`` dictionary. Below we define both a ``__getstate__`` and a ``__setstate__`` that package up @@ -348,17 +345,21 @@ the minimal form of our ``Point`` class:: As with :class:`.Mutable`, the :class:`.MutableComposite` augments the pickling process of the parent's object-relational state so that the -:meth:`.MutableBase._parents` collection is restored to all ``Point`` objects. +:meth:`MutableBase._parents` collection is restored to all ``Point`` objects. """ -from sqlalchemy.orm.attributes import flag_modified -from sqlalchemy import event, types -from sqlalchemy.orm import mapper, object_mapper, Mapper -from sqlalchemy.util import memoized_property +from ..orm.attributes import flag_modified +from .. import event, types +from ..orm import mapper, object_mapper, Mapper +from ..util import memoized_property import weakref + class MutableBase(object): - """Common base class to :class:`.Mutable` and :class:`.MutableComposite`.""" + """Common base class to :class:`.Mutable` + and :class:`.MutableComposite`. + + """ @memoized_property def _parents(self): @@ -398,7 +399,8 @@ class MutableBase(object): """ if value is None: return None - raise ValueError("Attribute '%s' does not accept objects of type %s" % (key, type(value))) + msg = "Attribute '%s' does not accept objects of type %s" + raise ValueError(msg % (key, type(value))) @classmethod def _listen_on_attribute(cls, attribute, coerce, parent_cls): @@ -456,12 +458,17 @@ class MutableBase(object): for val in state_dict['ext.mutable.values']: val._parents[state.obj()] = key + event.listen(parent_cls, 'load', load, + raw=True, propagate=True) + event.listen(parent_cls, 'refresh', load, + raw=True, propagate=True) + event.listen(attribute, 'set', set, + raw=True, retval=True, propagate=True) + event.listen(parent_cls, 'pickle', pickle, + raw=True, propagate=True) + event.listen(parent_cls, 'unpickle', unpickle, + raw=True, propagate=True) - event.listen(parent_cls, 'load', load, raw=True, propagate=True) - event.listen(parent_cls, 'refresh', load, raw=True, propagate=True) - event.listen(attribute, 'set', set, raw=True, retval=True, propagate=True) - event.listen(parent_cls, 'pickle', pickle, raw=True, propagate=True) - event.listen(parent_cls, 'unpickle', unpickle, raw=True, propagate=True) class Mutable(MutableBase): """Mixin that defines transparent propagation of change @@ -490,23 +497,23 @@ class Mutable(MutableBase): """Associate this wrapper with all future mapped columns of the given type. - This is a convenience method that calls ``associate_with_attribute`` automatically. + This is a convenience method that calls + ``associate_with_attribute`` automatically. .. warning:: The listeners established by this method are *global* to all mappers, and are *not* garbage collected. Only use - :meth:`.associate_with` for types that are permanent to an application, - not with ad-hoc types else this will cause unbounded growth - in memory usage. + :meth:`.associate_with` for types that are permanent to an + application, not with ad-hoc types else this will cause unbounded + growth in memory usage. """ def listen_for_type(mapper, class_): - for prop in mapper.iterate_properties: - if hasattr(prop, 'columns'): - if isinstance(prop.columns[0].type, sqltype): - cls.associate_with_attribute(getattr(class_, prop.key)) + for prop in mapper.column_attrs: + if isinstance(prop.columns[0].type, sqltype): + cls.associate_with_attribute(getattr(class_, prop.key)) event.listen(mapper, 'mapper_configured', listen_for_type) @@ -526,12 +533,12 @@ class Mutable(MutableBase): ) Note that the returned type is always an instance, even if a class - is given, and that only columns which are declared specifically with that - type instance receive additional instrumentation. + is given, and that only columns which are declared specifically with + that type instance receive additional instrumentation. To associate a particular mutable type with all occurrences of a particular type, use the :meth:`.Mutable.associate_with` classmethod - of the particular :meth:`.Mutable` subclass to establish a global + of the particular :class:`.Mutable` subclass to establish a global association. .. warning:: @@ -546,15 +553,16 @@ class Mutable(MutableBase): sqltype = types.to_instance(sqltype) def listen_for_type(mapper, class_): - for prop in mapper.iterate_properties: - if hasattr(prop, 'columns'): - if prop.columns[0].type is sqltype: - cls.associate_with_attribute(getattr(class_, prop.key)) + for prop in mapper.column_attrs: + if prop.columns[0].type is sqltype: + cls.associate_with_attribute(getattr(class_, prop.key)) event.listen(mapper, 'mapper_configured', listen_for_type) return sqltype + + class MutableComposite(MutableBase): """Mixin that defines transparent propagation of change events on a SQLAlchemy "composite" object to its @@ -562,14 +570,6 @@ class MutableComposite(MutableBase): See the example in :ref:`mutable_composites` for usage information. - .. warning:: - - The listeners established by the :class:`.MutableComposite` - class are *global* to all mappers, and are *not* garbage collected. Only use - :class:`.MutableComposite` for types that are permanent to an application, - not with ad-hoc types else this will cause unbounded growth - in memory usage. - """ def changed(self): @@ -583,14 +583,52 @@ class MutableComposite(MutableBase): prop._attribute_keys): setattr(parent, attr_name, value) - def _setup_composite_listener(): def _listen_for_type(mapper, class_): for prop in mapper.iterate_properties: if (hasattr(prop, 'composite_class') and - issubclass(prop.composite_class, MutableComposite)): + isinstance(prop.composite_class, type) and + issubclass(prop.composite_class, MutableComposite)): prop.composite_class._listen_on_attribute( getattr(class_, prop.key), False, class_) - if not Mapper.dispatch.mapper_configured._contains(Mapper, _listen_for_type): + if not event.contains(Mapper, "mapper_configured", _listen_for_type): event.listen(Mapper, 'mapper_configured', _listen_for_type) _setup_composite_listener() + + +class MutableDict(Mutable, dict): + """A dictionary type that implements :class:`.Mutable`. + + .. versionadded:: 0.8 + + """ + + def __setitem__(self, key, value): + """Detect dictionary set events and emit change events.""" + dict.__setitem__(self, key, value) + self.changed() + + def __delitem__(self, key): + """Detect dictionary del events and emit change events.""" + dict.__delitem__(self, key) + self.changed() + + def clear(self): + dict.clear(self) + self.changed() + + @classmethod + def coerce(cls, key, value): + """Convert plain dictionary to MutableDict.""" + if not isinstance(value, MutableDict): + if isinstance(value, dict): + return MutableDict(value) + return Mutable.coerce(key, value) + else: + return value + + def __getstate__(self): + return dict(self) + + def __setstate__(self, state): + self.update(state) diff --git a/libs/sqlalchemy/ext/orderinglist.py b/libs/sqlalchemy/ext/orderinglist.py index 6a9b6c39..9310c607 100644 --- a/libs/sqlalchemy/ext/orderinglist.py +++ b/libs/sqlalchemy/ext/orderinglist.py @@ -1,5 +1,5 @@ # ext/orderinglist.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -107,8 +107,8 @@ or some other integer, provide ``count_from=1``. """ -from sqlalchemy.orm.collections import collection -from sqlalchemy import util +from ..orm.collections import collection +from .. import util __all__ = ['ordering_list'] @@ -324,7 +324,7 @@ class OrderingList(list): if stop < 0: stop += len(self) - for i in xrange(start, stop, step): + for i in range(start, stop, step): self.__setitem__(i, entity[i]) else: self._order_entity(index, entity, True) @@ -334,7 +334,6 @@ class OrderingList(list): super(OrderingList, self).__delitem__(index) self._reorder() - # Py2K def __setslice__(self, start, end, values): super(OrderingList, self).__setslice__(start, end, values) self._reorder() @@ -342,13 +341,12 @@ class OrderingList(list): def __delslice__(self, start, end): super(OrderingList, self).__delslice__(start, end) self._reorder() - # end Py2K def __reduce__(self): return _reconstitute, (self.__class__, self.__dict__, list(self)) - for func_name, func in locals().items(): - if (util.callable(func) and func.func_name == func_name and + for func_name, func in list(locals().items()): + if (util.callable(func) and func.__name__ == func_name and not func.__doc__ and hasattr(list, func_name)): func.__doc__ = getattr(list, func_name).__doc__ del func_name, func diff --git a/libs/sqlalchemy/ext/serializer.py b/libs/sqlalchemy/ext/serializer.py index 47121bca..388cd404 100644 --- a/libs/sqlalchemy/ext/serializer.py +++ b/libs/sqlalchemy/ext/serializer.py @@ -1,5 +1,5 @@ # ext/serializer.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -39,46 +39,32 @@ The serializer module is only appropriate for query structures. It is not needed for: * instances of user-defined classes. These contain no references to engines, - sessions or expression constructs in the typical case and can be serialized directly. + sessions or expression constructs in the typical case and can be serialized + directly. -* Table metadata that is to be loaded entirely from the serialized structure (i.e. is - not already declared in the application). Regular pickle.loads()/dumps() can - be used to fully dump any ``MetaData`` object, typically one which was reflected - from an existing database at some previous point in time. The serializer module - is specifically for the opposite case, where the Table metadata is already present - in memory. +* Table metadata that is to be loaded entirely from the serialized structure + (i.e. is not already declared in the application). Regular + pickle.loads()/dumps() can be used to fully dump any ``MetaData`` object, + typically one which was reflected from an existing database at some previous + point in time. The serializer module is specifically for the opposite case, + where the Table metadata is already present in memory. """ -from sqlalchemy.orm import class_mapper, Query -from sqlalchemy.orm.session import Session -from sqlalchemy.orm.mapper import Mapper -from sqlalchemy.orm.attributes import QueryableAttribute -from sqlalchemy import Table, Column -from sqlalchemy.engine import Engine -from sqlalchemy.util import pickle +from ..orm import class_mapper +from ..orm.session import Session +from ..orm.mapper import Mapper +from ..orm.interfaces import MapperProperty +from ..orm.attributes import QueryableAttribute +from .. import Table, Column +from ..engine import Engine +from ..util import pickle, byte_buffer, b64encode, b64decode, text_type import re -import base64 -# Py3K -#from io import BytesIO as byte_buffer -# Py2K -from cStringIO import StringIO as byte_buffer -# end Py2K -# Py3K -#def b64encode(x): -# return base64.b64encode(x).decode('ascii') -#def b64decode(x): -# return base64.b64decode(x.encode('ascii')) -# Py2K -b64encode = base64.b64encode -b64decode = base64.b64decode -# end Py2K __all__ = ['Serializer', 'Deserializer', 'dumps', 'loads'] - def Serializer(*args, **kw): pickler = pickle.Pickler(*args, **kw) @@ -90,10 +76,13 @@ def Serializer(*args, **kw): id = "attribute:" + key + ":" + b64encode(pickle.dumps(cls)) elif isinstance(obj, Mapper) and not obj.non_primary: id = "mapper:" + b64encode(pickle.dumps(obj.class_)) + elif isinstance(obj, MapperProperty) and not obj.parent.non_primary: + id = "mapperprop:" + b64encode(pickle.dumps(obj.parent.class_)) + \ + ":" + obj.key elif isinstance(obj, Table): - id = "table:" + str(obj) + id = "table:" + text_type(obj.key) elif isinstance(obj, Column) and isinstance(obj.table, Table): - id = "column:" + str(obj.table) + ":" + obj.key + id = "column:" + text_type(obj.table.key) + ":" + text_type(obj.key) elif isinstance(obj, Session): id = "session:" elif isinstance(obj, Engine): @@ -105,7 +94,9 @@ def Serializer(*args, **kw): pickler.persistent_id = persistent_id return pickler -our_ids = re.compile(r'(mapper|table|column|session|attribute|engine):(.*)') +our_ids = re.compile( + r'(mapperprop|mapper|table|column|session|attribute|engine):(.*)') + def Deserializer(file, metadata=None, scoped_session=None, engine=None): unpickler = pickle.Unpickler(file) @@ -121,7 +112,7 @@ def Deserializer(file, metadata=None, scoped_session=None, engine=None): return None def persistent_load(id): - m = our_ids.match(id) + m = our_ids.match(text_type(id)) if not m: return None else: @@ -133,6 +124,10 @@ def Deserializer(file, metadata=None, scoped_session=None, engine=None): elif type_ == "mapper": cls = pickle.loads(b64decode(args)) return class_mapper(cls) + elif type_ == "mapperprop": + mapper, keyname = args.split(':') + cls = pickle.loads(b64decode(mapper)) + return class_mapper(cls).attrs[keyname] elif type_ == "table": return metadata.tables[args] elif type_ == "column": @@ -147,15 +142,15 @@ def Deserializer(file, metadata=None, scoped_session=None, engine=None): unpickler.persistent_load = persistent_load return unpickler + def dumps(obj, protocol=0): buf = byte_buffer() pickler = Serializer(buf, protocol) pickler.dump(obj) return buf.getvalue() + def loads(data, metadata=None, scoped_session=None, engine=None): buf = byte_buffer(data) unpickler = Deserializer(buf, metadata, scoped_session, engine) return unpickler.load() - - diff --git a/libs/sqlalchemy/ext/sqlsoup.py b/libs/sqlalchemy/ext/sqlsoup.py deleted file mode 100644 index 486b09c5..00000000 --- a/libs/sqlalchemy/ext/sqlsoup.py +++ /dev/null @@ -1,811 +0,0 @@ -# ext/sqlsoup.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -""" - -.. versionchanged:: 0.8 - SQLSoup is now its own project. Documentation - and project status are available at: - http://pypi.python.org/pypi/sqlsoup and - http://readthedocs.org/docs/sqlsoup\ . - SQLSoup will no longer be included with SQLAlchemy. - - -Introduction -============ - -SqlSoup provides a convenient way to access existing database -tables without having to declare table or mapper classes ahead -of time. It is built on top of the SQLAlchemy ORM and provides a -super-minimalistic interface to an existing database. - -SqlSoup effectively provides a coarse grained, alternative -interface to working with the SQLAlchemy ORM, providing a "self -configuring" interface for extremely rudimental operations. It's -somewhat akin to a "super novice mode" version of the ORM. While -SqlSoup can be very handy, users are strongly encouraged to use -the full ORM for non-trivial applications. - -Suppose we have a database with users, books, and loans tables -(corresponding to the PyWebOff dataset, if you're curious). - -Creating a SqlSoup gateway is just like creating an SQLAlchemy -engine:: - - >>> from sqlalchemy.ext.sqlsoup import SqlSoup - >>> db = SqlSoup('sqlite:///:memory:') - -or, you can re-use an existing engine:: - - >>> db = SqlSoup(engine) - -You can optionally specify a schema within the database for your -SqlSoup:: - - >>> db.schema = myschemaname - -Loading objects -=============== - -Loading objects is as easy as this:: - - >>> users = db.users.all() - >>> users.sort() - >>> users - [ - MappedUsers(name=u'Joe Student',email=u'student@example.edu', - password=u'student',classname=None,admin=0), - MappedUsers(name=u'Bhargan Basepair',email=u'basepair@example.edu', - password=u'basepair',classname=None,admin=1) - ] - -Of course, letting the database do the sort is better:: - - >>> db.users.order_by(db.users.name).all() - [ - MappedUsers(name=u'Bhargan Basepair',email=u'basepair@example.edu', - password=u'basepair',classname=None,admin=1), - MappedUsers(name=u'Joe Student',email=u'student@example.edu', - password=u'student',classname=None,admin=0) - ] - -Field access is intuitive:: - - >>> users[0].email - u'student@example.edu' - -Of course, you don't want to load all users very often. Let's -add a WHERE clause. Let's also switch the order_by to DESC while -we're at it:: - - >>> from sqlalchemy import or_, and_, desc - >>> where = or_(db.users.name=='Bhargan Basepair', db.users.email=='student@example.edu') - >>> db.users.filter(where).order_by(desc(db.users.name)).all() - [ - MappedUsers(name=u'Joe Student',email=u'student@example.edu', - password=u'student',classname=None,admin=0), - MappedUsers(name=u'Bhargan Basepair',email=u'basepair@example.edu', - password=u'basepair',classname=None,admin=1) - ] - -You can also use .first() (to retrieve only the first object -from a query) or .one() (like .first when you expect exactly one -user -- it will raise an exception if more were returned):: - - >>> db.users.filter(db.users.name=='Bhargan Basepair').one() - MappedUsers(name=u'Bhargan Basepair',email=u'basepair@example.edu', - password=u'basepair',classname=None,admin=1) - -Since name is the primary key, this is equivalent to - - >>> db.users.get('Bhargan Basepair') - MappedUsers(name=u'Bhargan Basepair',email=u'basepair@example.edu', - password=u'basepair',classname=None,admin=1) - -This is also equivalent to - - >>> db.users.filter_by(name='Bhargan Basepair').one() - MappedUsers(name=u'Bhargan Basepair',email=u'basepair@example.edu', - password=u'basepair',classname=None,admin=1) - -filter_by is like filter, but takes kwargs instead of full -clause expressions. This makes it more concise for simple -queries like this, but you can't do complex queries like the -or\_ above or non-equality based comparisons this way. - -Full query documentation ------------------------- - -Get, filter, filter_by, order_by, limit, and the rest of the -query methods are explained in detail in -:ref:`ormtutorial_querying`. - -Modifying objects -================= - -Modifying objects is intuitive:: - - >>> user = _ - >>> user.email = 'basepair+nospam@example.edu' - >>> db.commit() - -(SqlSoup leverages the sophisticated SQLAlchemy unit-of-work -code, so multiple updates to a single object will be turned into -a single ``UPDATE`` statement when you commit.) - -To finish covering the basics, let's insert a new loan, then -delete it:: - - >>> book_id = db.books.filter_by(title='Regional Variation in Moss').first().id - >>> db.loans.insert(book_id=book_id, user_name=user.name) - MappedLoans(book_id=2,user_name=u'Bhargan Basepair',loan_date=None) - - >>> loan = db.loans.filter_by(book_id=2, user_name='Bhargan Basepair').one() - >>> db.delete(loan) - >>> db.commit() - -You can also delete rows that have not been loaded as objects. -Let's do our insert/delete cycle once more, this time using the -loans table's delete method. (For SQLAlchemy experts: note that -no flush() call is required since this delete acts at the SQL -level, not at the Mapper level.) The same where-clause -construction rules apply here as to the select methods:: - - >>> db.loans.insert(book_id=book_id, user_name=user.name) - MappedLoans(book_id=2,user_name=u'Bhargan Basepair',loan_date=None) - >>> db.loans.delete(db.loans.book_id==2) - -You can similarly update multiple rows at once. This will change the -book_id to 1 in all loans whose book_id is 2:: - - >>> db.loans.filter_by(db.loans.book_id==2).update({'book_id':1}) - >>> db.loans.filter_by(book_id=1).all() - [MappedLoans(book_id=1,user_name=u'Joe Student', - loan_date=datetime.datetime(2006, 7, 12, 0, 0))] - - -Joins -===== - -Occasionally, you will want to pull out a lot of data from related -tables all at once. In this situation, it is far more efficient to -have the database perform the necessary join. (Here we do not have *a -lot of data* but hopefully the concept is still clear.) SQLAlchemy is -smart enough to recognize that loans has a foreign key to users, and -uses that as the join condition automatically:: - - >>> join1 = db.join(db.users, db.loans, isouter=True) - >>> join1.filter_by(name='Joe Student').all() - [ - MappedJoin(name=u'Joe Student',email=u'student@example.edu', - password=u'student',classname=None,admin=0,book_id=1, - user_name=u'Joe Student',loan_date=datetime.datetime(2006, 7, 12, 0, 0)) - ] - -If you're unfortunate enough to be using MySQL with the default MyISAM -storage engine, you'll have to specify the join condition manually, -since MyISAM does not store foreign keys. Here's the same join again, -with the join condition explicitly specified:: - - >>> db.join(db.users, db.loans, db.users.name==db.loans.user_name, isouter=True) - - -You can compose arbitrarily complex joins by combining Join objects -with tables or other joins. Here we combine our first join with the -books table:: - - >>> join2 = db.join(join1, db.books) - >>> join2.all() - [ - MappedJoin(name=u'Joe Student',email=u'student@example.edu', - password=u'student',classname=None,admin=0,book_id=1, - user_name=u'Joe Student',loan_date=datetime.datetime(2006, 7, 12, 0, 0), - id=1,title=u'Mustards I Have Known',published_year=u'1989', - authors=u'Jones') - ] - -If you join tables that have an identical column name, wrap your join -with `with_labels`, to disambiguate columns with their table name -(.c is short for .columns):: - - >>> db.with_labels(join1).c.keys() - [u'users_name', u'users_email', u'users_password', - u'users_classname', u'users_admin', u'loans_book_id', - u'loans_user_name', u'loans_loan_date'] - -You can also join directly to a labeled object:: - - >>> labeled_loans = db.with_labels(db.loans) - >>> db.join(db.users, labeled_loans, isouter=True).c.keys() - [u'name', u'email', u'password', u'classname', - u'admin', u'loans_book_id', u'loans_user_name', u'loans_loan_date'] - - -Relationships -============= - -You can define relationships on SqlSoup classes: - - >>> db.users.relate('loans', db.loans) - -These can then be used like a normal SA property: - - >>> db.users.get('Joe Student').loans - [MappedLoans(book_id=1,user_name=u'Joe Student', - loan_date=datetime.datetime(2006, 7, 12, 0, 0))] - - >>> db.users.filter(~db.users.loans.any()).all() - [MappedUsers(name=u'Bhargan Basepair', - email='basepair+nospam@example.edu', - password=u'basepair',classname=None,admin=1)] - -relate can take any options that the relationship function -accepts in normal mapper definition: - - >>> del db._cache['users'] - >>> db.users.relate('loans', db.loans, order_by=db.loans.loan_date, cascade='all, delete-orphan') - -Advanced Use -============ - -Sessions, Transactions and Application Integration ---------------------------------------------------- - -.. note:: - - Please read and understand this section thoroughly - before using SqlSoup in any web application. - -SqlSoup uses a ScopedSession to provide thread-local sessions. -You can get a reference to the current one like this:: - - >>> session = db.session - -The default session is available at the module level in SQLSoup, -via:: - - >>> from sqlalchemy.ext.sqlsoup import Session - -The configuration of this session is ``autoflush=True``, -``autocommit=False``. This means when you work with the SqlSoup -object, you need to call ``db.commit()`` in order to have -changes persisted. You may also call ``db.rollback()`` to roll -things back. - -Since the SqlSoup object's Session automatically enters into a -transaction as soon as it's used, it is *essential* that you -call ``commit()`` or ``rollback()`` on it when the work within a -thread completes. This means all the guidelines for web -application integration at :ref:`session_lifespan` must be -followed. - -The SqlSoup object can have any session or scoped session -configured onto it. This is of key importance when integrating -with existing code or frameworks such as Pylons. If your -application already has a ``Session`` configured, pass it to -your SqlSoup object:: - - >>> from myapplication import Session - >>> db = SqlSoup(session=Session) - -If the ``Session`` is configured with ``autocommit=True``, use -``flush()`` instead of ``commit()`` to persist changes - in this -case, the ``Session`` closes out its transaction immediately and -no external management is needed. ``rollback()`` is also not -available. Configuring a new SQLSoup object in "autocommit" mode -looks like:: - - >>> from sqlalchemy.orm import scoped_session, sessionmaker - >>> db = SqlSoup('sqlite://', session=scoped_session(sessionmaker(autoflush=False, expire_on_commit=False, autocommit=True))) - - -Mapping arbitrary Selectables ------------------------------ - -SqlSoup can map any SQLAlchemy :class:`.Selectable` with the map -method. Let's map an :func:`.expression.select` object that uses an aggregate -function; we'll use the SQLAlchemy :class:`.Table` that SqlSoup -introspected as the basis. (Since we're not mapping to a simple -table or join, we need to tell SQLAlchemy how to find the -*primary key* which just needs to be unique within the select, -and not necessarily correspond to a *real* PK in the database.):: - - >>> from sqlalchemy import select, func - >>> b = db.books._table - >>> s = select([b.c.published_year, func.count('*').label('n')], from_obj=[b], group_by=[b.c.published_year]) - >>> s = s.alias('years_with_count') - >>> years_with_count = db.map(s, primary_key=[s.c.published_year]) - >>> years_with_count.filter_by(published_year='1989').all() - [MappedBooks(published_year=u'1989',n=1)] - -Obviously if we just wanted to get a list of counts associated with -book years once, raw SQL is going to be less work. The advantage of -mapping a Select is reusability, both standalone and in Joins. (And if -you go to full SQLAlchemy, you can perform mappings like this directly -to your object models.) - -An easy way to save mapped selectables like this is to just hang them on -your db object:: - - >>> db.years_with_count = years_with_count - -Python is flexible like that! - -Raw SQL -------- - -SqlSoup works fine with SQLAlchemy's text construct, described -in :ref:`sqlexpression_text`. You can also execute textual SQL -directly using the `execute()` method, which corresponds to the -`execute()` method on the underlying `Session`. Expressions here -are expressed like ``text()`` constructs, using named parameters -with colons:: - - >>> rp = db.execute('select name, email from users where name like :name order by name', name='%Bhargan%') - >>> for name, email in rp.fetchall(): print name, email - Bhargan Basepair basepair+nospam@example.edu - -Or you can get at the current transaction's connection using -`connection()`. This is the raw connection object which can -accept any sort of SQL expression or raw SQL string passed to -the database:: - - >>> conn = db.connection() - >>> conn.execute("'select name, email from users where name like ? order by name'", '%Bhargan%') - -Dynamic table names -------------------- - -You can load a table whose name is specified at runtime with the -entity() method: - - >>> tablename = 'loans' - >>> db.entity(tablename) == db.loans - True - -entity() also takes an optional schema argument. If none is -specified, the default schema is used. - -""" - -from sqlalchemy import Table, MetaData, join -from sqlalchemy import schema, sql, util -from sqlalchemy.engine.base import Engine -from sqlalchemy.orm import scoped_session, sessionmaker, mapper, \ - class_mapper, relationship, session,\ - object_session, attributes -from sqlalchemy.orm.interfaces import MapperExtension, EXT_CONTINUE -from sqlalchemy.exc import SQLAlchemyError, InvalidRequestError, ArgumentError -from sqlalchemy.sql import expression - - -__all__ = ['PKNotFoundError', 'SqlSoup'] - -Session = scoped_session(sessionmaker(autoflush=True, autocommit=False)) - -class AutoAdd(MapperExtension): - def __init__(self, scoped_session): - self.scoped_session = scoped_session - - def instrument_class(self, mapper, class_): - class_.__init__ = self._default__init__(mapper) - - def _default__init__(ext, mapper): - def __init__(self, **kwargs): - for key, value in kwargs.iteritems(): - setattr(self, key, value) - return __init__ - - def init_instance(self, mapper, class_, oldinit, instance, args, kwargs): - session = self.scoped_session() - state = attributes.instance_state(instance) - session._save_impl(state) - return EXT_CONTINUE - - def init_failed(self, mapper, class_, oldinit, instance, args, kwargs): - sess = object_session(instance) - if sess: - sess.expunge(instance) - return EXT_CONTINUE - -class PKNotFoundError(SQLAlchemyError): - pass - -def _ddl_error(cls): - msg = 'SQLSoup can only modify mapped Tables (found: %s)' \ - % cls._table.__class__.__name__ - raise InvalidRequestError(msg) - -# metaclass is necessary to expose class methods with getattr, e.g. -# we want to pass db.users.select through to users._mapper.select -class SelectableClassType(type): - def insert(cls, **kwargs): - _ddl_error(cls) - - def __clause_element__(cls): - return cls._table - - def __getattr__(cls, attr): - if attr == '_query': - # called during mapper init - raise AttributeError() - return getattr(cls._query, attr) - -class TableClassType(SelectableClassType): - def insert(cls, **kwargs): - o = cls() - o.__dict__.update(kwargs) - return o - - def relate(cls, propname, *args, **kwargs): - class_mapper(cls)._configure_property(propname, relationship(*args, **kwargs)) - -def _is_outer_join(selectable): - if not isinstance(selectable, sql.Join): - return False - if selectable.isouter: - return True - return _is_outer_join(selectable.left) or _is_outer_join(selectable.right) - -def _selectable_name(selectable): - if isinstance(selectable, sql.Alias): - return _selectable_name(selectable.element) - elif isinstance(selectable, sql.Select): - return ''.join(_selectable_name(s) for s in selectable.froms) - elif isinstance(selectable, schema.Table): - return selectable.name.capitalize() - else: - x = selectable.__class__.__name__ - if x[0] == '_': - x = x[1:] - return x - -def _class_for_table(session, engine, selectable, base_cls, mapper_kwargs): - selectable = expression._clause_element_as_expr(selectable) - mapname = 'Mapped' + _selectable_name(selectable) - # Py2K - if isinstance(mapname, unicode): - engine_encoding = engine.dialect.encoding - mapname = mapname.encode(engine_encoding) - # end Py2K - - if isinstance(selectable, Table): - klass = TableClassType(mapname, (base_cls,), {}) - else: - klass = SelectableClassType(mapname, (base_cls,), {}) - - def _compare(self, o): - L = list(self.__class__.c.keys()) - L.sort() - t1 = [getattr(self, k) for k in L] - try: - t2 = [getattr(o, k) for k in L] - except AttributeError: - raise TypeError('unable to compare with %s' % o.__class__) - return t1, t2 - - # python2/python3 compatible system of - # __cmp__ - __lt__ + __eq__ - - def __lt__(self, o): - t1, t2 = _compare(self, o) - return t1 < t2 - - def __eq__(self, o): - t1, t2 = _compare(self, o) - return t1 == t2 - - def __repr__(self): - L = ["%s=%r" % (key, getattr(self, key, '')) - for key in self.__class__.c.keys()] - return '%s(%s)' % (self.__class__.__name__, ','.join(L)) - - for m in ['__eq__', '__repr__', '__lt__']: - setattr(klass, m, eval(m)) - klass._table = selectable - klass.c = expression.ColumnCollection() - mappr = mapper(klass, - selectable, - extension=AutoAdd(session), - **mapper_kwargs) - - for k in mappr.iterate_properties: - klass.c[k.key] = k.columns[0] - - klass._query = session.query_property() - return klass - -class SqlSoup(object): - """Represent an ORM-wrapped database resource.""" - - def __init__(self, engine_or_metadata, base=object, session=None): - """Initialize a new :class:`.SqlSoup`. - - :param engine_or_metadata: a string database URL, :class:`.Engine` - or :class:`.MetaData` object to associate with. If the - argument is a :class:`.MetaData`, it should be *bound* - to an :class:`.Engine`. - :param base: a class which will serve as the default class for - returned mapped classes. Defaults to ``object``. - :param session: a :class:`.ScopedSession` or :class:`.Session` with - which to associate ORM operations for this :class:`.SqlSoup` instance. - If ``None``, a :class:`.ScopedSession` that's local to this - module is used. - - """ - - self.session = session or Session - self.base=base - - if isinstance(engine_or_metadata, MetaData): - self._metadata = engine_or_metadata - elif isinstance(engine_or_metadata, (basestring, Engine)): - self._metadata = MetaData(engine_or_metadata) - else: - raise ArgumentError("invalid engine or metadata argument %r" % - engine_or_metadata) - - self._cache = {} - self.schema = None - - @property - def bind(self): - """The :class:`.Engine` associated with this :class:`.SqlSoup`.""" - return self._metadata.bind - - engine = bind - - def delete(self, instance): - """Mark an instance as deleted.""" - - self.session.delete(instance) - - def execute(self, stmt, **params): - """Execute a SQL statement. - - The statement may be a string SQL string, - an :func:`.expression.select` construct, or an :func:`.expression.text` - construct. - - """ - return self.session.execute(sql.text(stmt, bind=self.bind), **params) - - @property - def _underlying_session(self): - if isinstance(self.session, session.Session): - return self.session - else: - return self.session() - - def connection(self): - """Return the current :class:`.Connection` in use by the current transaction.""" - - return self._underlying_session._connection_for_bind(self.bind) - - def flush(self): - """Flush pending changes to the database. - - See :meth:`.Session.flush`. - - """ - self.session.flush() - - def rollback(self): - """Rollback the current transaction. - - See :meth:`.Session.rollback`. - - """ - self.session.rollback() - - def commit(self): - """Commit the current transaction. - - See :meth:`.Session.commit`. - - """ - self.session.commit() - - def clear(self): - """Synonym for :meth:`.SqlSoup.expunge_all`.""" - - self.session.expunge_all() - - def expunge(self, instance): - """Remove an instance from the :class:`.Session`. - - See :meth:`.Session.expunge`. - - """ - self.session.expunge(instance) - - def expunge_all(self): - """Clear all objects from the current :class:`.Session`. - - See :meth:`.Session.expunge_all`. - - """ - self.session.expunge_all() - - def map_to(self, attrname, tablename=None, selectable=None, - schema=None, base=None, mapper_args=util.immutabledict()): - """Configure a mapping to the given attrname. - - This is the "master" method that can be used to create any - configuration. - - .. versionadded:: 0.6.6 - - :param attrname: String attribute name which will be - established as an attribute on this :class:.`.SqlSoup` - instance. - :param base: a Python class which will be used as the - base for the mapped class. If ``None``, the "base" - argument specified by this :class:`.SqlSoup` - instance's constructor will be used, which defaults to - ``object``. - :param mapper_args: Dictionary of arguments which will - be passed directly to :func:`.orm.mapper`. - :param tablename: String name of a :class:`.Table` to be - reflected. If a :class:`.Table` is already available, - use the ``selectable`` argument. This argument is - mutually exclusive versus the ``selectable`` argument. - :param selectable: a :class:`.Table`, :class:`.Join`, or - :class:`.Select` object which will be mapped. This - argument is mutually exclusive versus the ``tablename`` - argument. - :param schema: String schema name to use if the - ``tablename`` argument is present. - - - """ - if attrname in self._cache: - raise InvalidRequestError( - "Attribute '%s' is already mapped to '%s'" % ( - attrname, - class_mapper(self._cache[attrname]).mapped_table - )) - - if tablename is not None: - if not isinstance(tablename, basestring): - raise ArgumentError("'tablename' argument must be a string." - ) - if selectable is not None: - raise ArgumentError("'tablename' and 'selectable' " - "arguments are mutually exclusive") - - selectable = Table(tablename, - self._metadata, - autoload=True, - autoload_with=self.bind, - schema=schema or self.schema) - elif schema: - raise ArgumentError("'tablename' argument is required when " - "using 'schema'.") - elif selectable is not None: - if not isinstance(selectable, expression.FromClause): - raise ArgumentError("'selectable' argument must be a " - "table, select, join, or other " - "selectable construct.") - else: - raise ArgumentError("'tablename' or 'selectable' argument is " - "required.") - - if not selectable.primary_key.columns: - if tablename: - raise PKNotFoundError( - "table '%s' does not have a primary " - "key defined" % tablename) - else: - raise PKNotFoundError( - "selectable '%s' does not have a primary " - "key defined" % selectable) - - mapped_cls = _class_for_table( - self.session, - self.engine, - selectable, - base or self.base, - mapper_args - ) - self._cache[attrname] = mapped_cls - return mapped_cls - - - def map(self, selectable, base=None, **mapper_args): - """Map a selectable directly. - - .. versionchanged:: 0.6.6 - The class and its mapping are not cached and will - be discarded once dereferenced. - - :param selectable: an :func:`.expression.select` construct. - :param base: a Python class which will be used as the - base for the mapped class. If ``None``, the "base" - argument specified by this :class:`.SqlSoup` - instance's constructor will be used, which defaults to - ``object``. - :param mapper_args: Dictionary of arguments which will - be passed directly to :func:`.orm.mapper`. - - """ - - return _class_for_table( - self.session, - self.engine, - selectable, - base or self.base, - mapper_args - ) - - def with_labels(self, selectable, base=None, **mapper_args): - """Map a selectable directly, wrapping the - selectable in a subquery with labels. - - .. versionchanged:: 0.6.6 - The class and its mapping are not cached and will - be discarded once dereferenced. - - :param selectable: an :func:`.expression.select` construct. - :param base: a Python class which will be used as the - base for the mapped class. If ``None``, the "base" - argument specified by this :class:`.SqlSoup` - instance's constructor will be used, which defaults to - ``object``. - :param mapper_args: Dictionary of arguments which will - be passed directly to :func:`.orm.mapper`. - - """ - - # TODO give meaningful aliases - return self.map( - expression._clause_element_as_expr(selectable). - select(use_labels=True). - alias('foo'), base=base, **mapper_args) - - def join(self, left, right, onclause=None, isouter=False, - base=None, **mapper_args): - """Create an :func:`.expression.join` and map to it. - - .. versionchanged:: 0.6.6 - The class and its mapping are not cached and will - be discarded once dereferenced. - - :param left: a mapped class or table object. - :param right: a mapped class or table object. - :param onclause: optional "ON" clause construct.. - :param isouter: if True, the join will be an OUTER join. - :param base: a Python class which will be used as the - base for the mapped class. If ``None``, the "base" - argument specified by this :class:`.SqlSoup` - instance's constructor will be used, which defaults to - ``object``. - :param mapper_args: Dictionary of arguments which will - be passed directly to :func:`.orm.mapper`. - - """ - - j = join(left, right, onclause=onclause, isouter=isouter) - return self.map(j, base=base, **mapper_args) - - def entity(self, attr, schema=None): - """Return the named entity from this :class:`.SqlSoup`, or - create if not present. - - For more generalized mapping, see :meth:`.map_to`. - - """ - try: - return self._cache[attr] - except KeyError, ke: - return self.map_to(attr, tablename=attr, schema=schema) - - def __getattr__(self, attr): - return self.entity(attr) - - def __repr__(self): - return 'SqlSoup(%r)' % self._metadata - diff --git a/libs/sqlalchemy/inspection.py b/libs/sqlalchemy/inspection.py new file mode 100644 index 00000000..fe9e4055 --- /dev/null +++ b/libs/sqlalchemy/inspection.py @@ -0,0 +1,92 @@ +# sqlalchemy/inspect.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""The inspection module provides the :func:`.inspect` function, +which delivers runtime information about a wide variety +of SQLAlchemy objects, both within the Core as well as the +ORM. + +The :func:`.inspect` function is the entry point to SQLAlchemy's +public API for viewing the configuration and construction +of in-memory objects. Depending on the type of object +passed to :func:`.inspect`, the return value will either be +a related object which provides a known interface, or in many +cases it will return the object itself. + +The rationale for :func:`.inspect` is twofold. One is that +it replaces the need to be aware of a large variety of "information +getting" functions in SQLAlchemy, such as :meth:`.Inspector.from_engine`, +:func:`.orm.attributes.instance_state`, :func:`.orm.class_mapper`, +and others. The other is that the return value of :func:`.inspect` +is guaranteed to obey a documented API, thus allowing third party +tools which build on top of SQLAlchemy configurations to be constructed +in a forwards-compatible way. + +.. versionadded:: 0.8 The :func:`.inspect` system is introduced + as of version 0.8. + +""" + +from . import util, exc +_registrars = util.defaultdict(list) + + +def inspect(subject, raiseerr=True): + """Produce an inspection object for the given target. + + The returned value in some cases may be the + same object as the one given, such as if a + :class:`.Mapper` object is passed. In other + cases, it will be an instance of the registered + inspection type for the given object, such as + if an :class:`.engine.Engine` is passed, an + :class:`.Inspector` object is returned. + + :param subject: the subject to be inspected. + :param raiseerr: When ``True``, if the given subject + does not + correspond to a known SQLAlchemy inspected type, + :class:`sqlalchemy.exc.NoInspectionAvailable` + is raised. If ``False``, ``None`` is returned. + + """ + type_ = type(subject) + for cls in type_.__mro__: + if cls in _registrars: + reg = _registrars[cls] + if reg is True: + return subject + ret = reg(subject) + if ret is not None: + break + else: + reg = ret = None + + if raiseerr and ( + reg is None or ret is None + ): + raise exc.NoInspectionAvailable( + "No inspection system is " + "available for object of type %s" % + type_) + return ret + + +def _inspects(*types): + def decorate(fn_or_cls): + for type_ in types: + if type_ in _registrars: + raise AssertionError( + "Type %s is already " + "registered" % type_) + _registrars[type_] = fn_or_cls + return fn_or_cls + return decorate + + +def _self_inspects(cls): + _inspects(cls)(True) + return cls diff --git a/libs/sqlalchemy/interfaces.py b/libs/sqlalchemy/interfaces.py index 92fb3154..ed50a645 100644 --- a/libs/sqlalchemy/interfaces.py +++ b/libs/sqlalchemy/interfaces.py @@ -1,18 +1,19 @@ # sqlalchemy/interfaces.py -# Copyright (C) 2007-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2007-2014 the SQLAlchemy authors and contributors # Copyright (C) 2007 Jason Kirtland jek@discorporate.us # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Interfaces and abstract types. +"""Deprecated core event interfaces. This module is **deprecated** and is superseded by the event system. """ -from sqlalchemy import event, util +from . import event, util + class PoolListener(object): """Hooks into the lifecycle of connections in a :class:`.Pool`. @@ -89,7 +90,6 @@ class PoolListener(object): if hasattr(listener, 'checkin'): event.listen(self, 'checkin', listener.checkin) - def connect(self, dbapi_con, con_record): """Called once for each new DB-API connection or Pool's ``creator()``. @@ -148,6 +148,7 @@ class PoolListener(object): """ + class ConnectionProxy(object): """Allows interception of statement execution by Connections. @@ -161,11 +162,13 @@ class ConnectionProxy(object): cursor level executions, e.g.:: class MyProxy(ConnectionProxy): - def execute(self, conn, execute, clauseelement, *multiparams, **params): + def execute(self, conn, execute, clauseelement, + *multiparams, **params): print "compiled statement:", clauseelement return execute(clauseelement, *multiparams, **params) - def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): + def cursor_execute(self, execute, cursor, statement, + parameters, context, executemany): print "raw statement:", statement return execute(cursor, statement, parameters, context) @@ -195,7 +198,7 @@ class ConnectionProxy(object): event.listen(self, 'before_execute', adapt_execute) def adapt_cursor_execute(conn, cursor, statement, - parameters,context, executemany, ): + parameters, context, executemany): def execute_wrapper( cursor, @@ -245,14 +248,13 @@ class ConnectionProxy(object): event.listen(self, 'commit_twophase', adapt_listener(listener.commit_twophase)) - def execute(self, conn, execute, clauseelement, *multiparams, **params): """Intercept high level execute() events.""" - return execute(clauseelement, *multiparams, **params) - def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): + def cursor_execute(self, execute, cursor, statement, parameters, + context, executemany): """Intercept low-level cursor execute() events.""" return execute(cursor, statement, parameters, context) @@ -306,4 +308,3 @@ class ConnectionProxy(object): """Intercept commit_twophase() events.""" return commit_twophase(xid, is_prepared) - diff --git a/libs/sqlalchemy/log.py b/libs/sqlalchemy/log.py index 24608fde..935761d5 100644 --- a/libs/sqlalchemy/log.py +++ b/libs/sqlalchemy/log.py @@ -1,5 +1,5 @@ # sqlalchemy/log.py -# Copyright (C) 2006-2011 the SQLAlchemy authors and contributors +# Copyright (C) 2006-2014 the SQLAlchemy authors and contributors # Includes alterations by Vinay Sajip vinay_sajip@yahoo.co.uk # # This module is part of SQLAlchemy and is released under @@ -19,7 +19,6 @@ instance only. import logging import sys -from sqlalchemy import util # set initial level to WARN. This so that # log statements don't occur in the absense of explicit @@ -28,24 +27,24 @@ rootlogger = logging.getLogger('sqlalchemy') if rootlogger.level == logging.NOTSET: rootlogger.setLevel(logging.WARN) + def _add_default_handler(logger): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s %(name)s %(message)s')) logger.addHandler(handler) + _logged_classes = set() -def class_logger(cls, enable=False): + + +def class_logger(cls): logger = logging.getLogger(cls.__module__ + "." + cls.__name__) - if enable == 'debug': - logger.setLevel(logging.DEBUG) - elif enable == 'info': - logger.setLevel(logging.INFO) cls._should_log_debug = lambda self: logger.isEnabledFor(logging.DEBUG) cls._should_log_info = lambda self: logger.isEnabledFor(logging.INFO) cls.logger = logger _logged_classes.add(cls) - + return cls class Identified(object): logging_name = None @@ -56,6 +55,7 @@ class Identified(object): def _should_log_info(self): return self.logger.isEnabledFor(logging.INFO) + class InstanceLogger(object): """A logger adapter (wrapper) for :class:`.Identified` subclasses. @@ -167,6 +167,7 @@ class InstanceLogger(object): level = self.logger.getEffectiveLevel() return level + def instance_logger(instance, echoflag=None): """create a logger for an instance that implements :class:`.Identified`.""" @@ -191,6 +192,7 @@ def instance_logger(instance, echoflag=None): instance.logger = logger + class echo_property(object): __doc__ = """\ When ``True``, enable log output for this element. diff --git a/libs/sqlalchemy/orm/__init__.py b/libs/sqlalchemy/orm/__init__.py index b4006be4..7825a70a 100644 --- a/libs/sqlalchemy/orm/__init__.py +++ b/libs/sqlalchemy/orm/__init__.py @@ -1,5 +1,5 @@ # orm/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -12,134 +12,60 @@ documentation for an overview of how this module is used. """ -from sqlalchemy.orm import exc -from sqlalchemy.orm.mapper import ( +from . import exc +from .mapper import ( Mapper, _mapper_registry, class_mapper, - configure_mappers + configure_mappers, + reconstructor, + validates ) -from sqlalchemy.orm.interfaces import ( +from .interfaces import ( EXT_CONTINUE, EXT_STOP, - InstrumentationManager, - MapperExtension, PropComparator, + ) +from .deprecated_interfaces import ( + MapperExtension, SessionExtension, AttributeExtension, - ) -from sqlalchemy.orm.util import ( +) +from .util import ( aliased, join, object_mapper, outerjoin, polymorphic_union, + was_deleted, with_parent, + with_polymorphic, ) -from sqlalchemy.orm.properties import ( - ColumnProperty, +from .properties import ColumnProperty +from .relationships import RelationshipProperty +from .descriptor_props import ( ComparableProperty, CompositeProperty, - RelationshipProperty, - PropertyLoader, SynonymProperty, - ) -from sqlalchemy.orm import mapper as mapperlib -from sqlalchemy.orm.mapper import reconstructor, validates -from sqlalchemy.orm import strategies -from sqlalchemy.orm.query import AliasOption, Query -from sqlalchemy.sql import util as sql_util -from sqlalchemy.orm.session import Session -from sqlalchemy.orm.session import object_session, sessionmaker, \ - make_transient -from sqlalchemy.orm.scoping import ScopedSession -from sqlalchemy import util as sa_util - -__all__ = ( - 'EXT_CONTINUE', - 'EXT_STOP', - 'InstrumentationManager', - 'MapperExtension', - 'AttributeExtension', - 'PropComparator', - 'Query', - 'Session', - 'aliased', - 'backref', - 'class_mapper', - 'clear_mappers', - 'column_property', - 'comparable_property', - 'compile_mappers', - 'configure_mappers', - 'composite', - 'contains_alias', - 'contains_eager', - 'create_session', - 'defer', - 'deferred', - 'dynamic_loader', - 'eagerload', - 'eagerload_all', - 'immediateload', - 'join', - 'joinedload', - 'joinedload_all', - 'lazyload', - 'mapper', - 'make_transient', - 'noload', - 'object_mapper', - 'object_session', - 'outerjoin', - 'polymorphic_union', - 'reconstructor', - 'relationship', - 'relation', - 'scoped_session', - 'sessionmaker', - 'subqueryload', - 'subqueryload_all', - 'synonym', - 'undefer', - 'undefer_group', - 'validates' ) - - -def scoped_session(session_factory, scopefunc=None): - """Provides thread-local or scoped management of :class:`.Session` objects. - - This is a front-end function to - :class:`.ScopedSession`:: - - Session = scoped_session(sessionmaker(autoflush=True)) - - To instantiate a Session object which is part of the scoped context, - instantiate normally:: - - session = Session() - - Most session methods are available as classmethods from the scoped - session:: - - Session.commit() - Session.close() - - See also: :ref:`unitofwork_contextual`. - - :param session_factory: a callable function that produces - :class:`.Session` instances, such as :func:`sessionmaker`. - - :param scopefunc: Optional "scope" function which would be - passed to the :class:`.ScopedRegistry`. If None, the - :class:`.ThreadLocalRegistry` is used by default. - - :returns: a :class:`.ScopedSession` instance - - - """ - return ScopedSession(session_factory, scopefunc=scopefunc) +from .relationships import ( + foreign, + remote, +) +from .session import ( + Session, + object_session, + sessionmaker, + make_transient +) +from .scoping import ( + scoped_session +) +from . import mapper as mapperlib +from .query import AliasOption, Query, Bundle +from ..util.langhelpers import public_factory +from .. import util as _sa_util +from . import strategies as _strategies def create_session(bind=None, **kwargs): """Create a new :class:`.Session` @@ -177,458 +103,14 @@ def create_session(bind=None, **kwargs): kwargs.setdefault('expire_on_commit', False) return Session(bind=bind, **kwargs) -def relationship(argument, secondary=None, **kwargs): - """Provide a relationship of a primary Mapper to a secondary Mapper. - - .. versionchanged:: 0.6 - :func:`relationship` is historically known as :func:`relation`. - - This corresponds to a parent-child or associative table relationship. The - constructed class is an instance of :class:`.RelationshipProperty`. - - A typical :func:`.relationship`, used in a classical mapping:: - - mapper(Parent, properties={ - 'children': relationship(Child) - }) - - Some arguments accepted by :func:`.relationship` optionally accept a - callable function, which when called produces the desired value. - The callable is invoked by the parent :class:`.Mapper` at "mapper initialization" - time, which happens only when mappers are first used, and is assumed - to be after all mappings have been constructed. This can be used - to resolve order-of-declaration and other dependency issues, such as - if ``Child`` is declared below ``Parent`` in the same file:: - - mapper(Parent, properties={ - "children":relationship(lambda: Child, - order_by=lambda: Child.id) - }) - - When using the :ref:`declarative_toplevel` extension, the Declarative - initializer allows string arguments to be passed to :func:`.relationship`. - These string arguments are converted into callables that evaluate - the string as Python code, using the Declarative - class-registry as a namespace. This allows the lookup of related - classes to be automatic via their string name, and removes the need to import - related classes at all into the local module space:: - - from sqlalchemy.ext.declarative import declarative_base - - Base = declarative_base() - - class Parent(Base): - __tablename__ = 'parent' - id = Column(Integer, primary_key=True) - children = relationship("Child", order_by="Child.id") - - A full array of examples and reference documentation regarding - :func:`.relationship` is at :ref:`relationship_config_toplevel`. - - :param argument: - a mapped class, or actual :class:`.Mapper` instance, representing the target of - the relationship. - - ``argument`` may also be passed as a callable function - which is evaluated at mapper initialization time, and may be passed as a - Python-evaluable string when using Declarative. - - :param secondary: - for a many-to-many relationship, specifies the intermediary - table, and is an instance of :class:`.Table`. The ``secondary`` keyword - argument should generally only - be used for a table that is not otherwise expressed in any class - mapping, unless this relationship is declared as view only, otherwise - conflicting persistence operations can occur. - - ``secondary`` may - also be passed as a callable function which is evaluated at - mapper initialization time. - - :param active_history=False: - When ``True``, indicates that the "previous" value for a - many-to-one reference should be loaded when replaced, if - not already loaded. Normally, history tracking logic for - simple many-to-ones only needs to be aware of the "new" - value in order to perform a flush. This flag is available - for applications that make use of - :func:`.attributes.get_history` which also need to know - the "previous" value of the attribute. - - :param backref: - indicates the string name of a property to be placed on the related - mapper's class that will handle this relationship in the other - direction. The other property will be created automatically - when the mappers are configured. Can also be passed as a - :func:`backref` object to control the configuration of the - new relationship. - - :param back_populates: - Takes a string name and has the same meaning as ``backref``, - except the complementing property is **not** created automatically, - and instead must be configured explicitly on the other mapper. The - complementing property should also indicate ``back_populates`` - to this relationship to ensure proper functioning. - - :param cascade: - a comma-separated list of cascade rules which determines how - Session operations should be "cascaded" from parent to child. - This defaults to ``False``, which means the default cascade - should be used. The default value is ``"save-update, merge"``. - - Available cascades are: - - * ``save-update`` - cascade the :meth:`.Session.add` - operation. This cascade applies both to future and - past calls to :meth:`~sqlalchemy.orm.session.Session.add`, - meaning new items added to a collection or scalar relationship - get placed into the same session as that of the parent, and - also applies to items which have been removed from this - relationship but are still part of unflushed history. - - * ``merge`` - cascade the :meth:`~sqlalchemy.orm.session.Session.merge` - operation - - * ``expunge`` - cascade the :meth:`.Session.expunge` - operation - - * ``delete`` - cascade the :meth:`.Session.delete` - operation - - * ``delete-orphan`` - if an item of the child's type is - detached from its parent, mark it for deletion. - - .. versionchanged:: 0.7 - This option does not prevent - a new instance of the child object from being persisted - without a parent to start with; to constrain against - that case, ensure the child's foreign key column(s) - is configured as NOT NULL - - * ``refresh-expire`` - cascade the :meth:`.Session.expire` - and :meth:`~sqlalchemy.orm.session.Session.refresh` operations - - * ``all`` - shorthand for "save-update,merge, refresh-expire, - expunge, delete" - - See the section :ref:`unitofwork_cascades` for more background - on configuring cascades. - - :param cascade_backrefs=True: - a boolean value indicating if the ``save-update`` cascade should - operate along an assignment event intercepted by a backref. - When set to ``False``, - the attribute managed by this relationship will not cascade - an incoming transient object into the session of a - persistent parent, if the event is received via backref. - - That is:: - - mapper(A, a_table, properties={ - 'bs':relationship(B, backref="a", cascade_backrefs=False) - }) - - If an ``A()`` is present in the session, assigning it to - the "a" attribute on a transient ``B()`` will not place - the ``B()`` into the session. To set the flag in the other - direction, i.e. so that ``A().bs.append(B())`` won't add - a transient ``A()`` into the session for a persistent ``B()``:: - - mapper(A, a_table, properties={ - 'bs':relationship(B, - backref=backref("a", cascade_backrefs=False) - ) - }) - - See the section :ref:`unitofwork_cascades` for more background - on configuring cascades. - - :param collection_class: - a class or callable that returns a new list-holding object. will - be used in place of a plain list for storing elements. - Behavior of this attribute is described in detail at - :ref:`custom_collections`. - - :param comparator_factory: - a class which extends :class:`.RelationshipProperty.Comparator` which - provides custom SQL clause generation for comparison operations. - - :param doc: - docstring which will be applied to the resulting descriptor. - - :param extension: - an :class:`.AttributeExtension` instance, or list of extensions, - which will be prepended to the list of attribute listeners for - the resulting descriptor placed on the class. - **Deprecated.** Please see :class:`.AttributeEvents`. - - :param foreign_keys: - a list of columns which are to be used as "foreign key" columns. - Normally, :func:`relationship` uses the :class:`.ForeignKey` - and :class:`.ForeignKeyConstraint` objects present within the - mapped or secondary :class:`.Table` to determine the "foreign" side of - the join condition. This is used to construct SQL clauses in order - to load objects, as well as to "synchronize" values from - primary key columns to referencing foreign key columns. - The ``foreign_keys`` parameter overrides the notion of what's - "foreign" in the table metadata, allowing the specification - of a list of :class:`.Column` objects that should be considered - part of the foreign key. - - There are only two use cases for ``foreign_keys`` - one, when it is not - convenient for :class:`.Table` metadata to contain its own foreign key - metadata (which should be almost never, unless reflecting a large amount of - tables from a MySQL MyISAM schema, or a schema that doesn't actually - have foreign keys on it). The other is for extremely - rare and exotic composite foreign key setups where some columns - should artificially not be considered as foreign. - - ``foreign_keys`` may also be passed as a callable function - which is evaluated at mapper initialization time, and may be passed as a - Python-evaluable string when using Declarative. - - :param innerjoin=False: - when ``True``, joined eager loads will use an inner join to join - against related tables instead of an outer join. The purpose - of this option is generally one of performance, as inner joins - generally perform better than outer joins. Another reason can be - the use of ``with_lockmode``, which does not support outer joins. - - This flag can be set to ``True`` when the relationship references an - object via many-to-one using local foreign keys that are not nullable, - or when the reference is one-to-one or a collection that is guaranteed - to have one or at least one entry. - - :param join_depth: - when non-``None``, an integer value indicating how many levels - deep "eager" loaders should join on a self-referring or cyclical - relationship. The number counts how many times the same Mapper - shall be present in the loading condition along a particular join - branch. When left at its default of ``None``, eager loaders - will stop chaining when they encounter a the same target mapper - which is already higher up in the chain. This option applies - both to joined- and subquery- eager loaders. - - :param lazy='select': specifies - how the related items should be loaded. Default value is - ``select``. Values include: - - * ``select`` - items should be loaded lazily when the property is first - accessed, using a separate SELECT statement, or identity map - fetch for simple many-to-one references. - - * ``immediate`` - items should be loaded as the parents are loaded, - using a separate SELECT statement, or identity map fetch for - simple many-to-one references. - - .. versionadded:: 0.6.5 - - * ``joined`` - items should be loaded "eagerly" in the same query as - that of the parent, using a JOIN or LEFT OUTER JOIN. Whether - the join is "outer" or not is determined by the ``innerjoin`` - parameter. - - * ``subquery`` - items should be loaded "eagerly" within the same - query as that of the parent, using a second SQL statement - which issues a JOIN to a subquery of the original - statement. - - * ``noload`` - no loading should occur at any time. This is to - support "write-only" attributes, or attributes which are - populated in some manner specific to the application. - - * ``dynamic`` - the attribute will return a pre-configured - :class:`~sqlalchemy.orm.query.Query` object for all read - operations, onto which further filtering operations can be - applied before iterating the results. See - the section :ref:`dynamic_relationship` for more details. - - * True - a synonym for 'select' - - * False - a synonym for 'joined' - - * None - a synonym for 'noload' - - Detailed discussion of loader strategies is at :ref:`loading_toplevel`. - - :param load_on_pending=False: - Indicates loading behavior for transient or pending parent objects. - - When set to ``True``, causes the lazy-loader to - issue a query for a parent object that is not persistent, meaning it has - never been flushed. This may take effect for a pending object when - autoflush is disabled, or for a transient object that has been - "attached" to a :class:`.Session` but is not part of its pending - collection. Attachment of transient objects to the session without - moving to the "pending" state is not a supported behavior at this time. - - Note that the load of related objects on a pending or transient object - also does not trigger any attribute change events - no user-defined - events will be emitted for these attributes, and if and when the - object is ultimately flushed, only the user-specific foreign key - attributes will be part of the modified state. - - The load_on_pending flag does not improve behavior - when the ORM is used normally - object references should be constructed - at the object level, not at the foreign key level, so that they - are present in an ordinary way before flush() proceeds. This flag - is not not intended for general use. - - New in 0.6.5. - - :param order_by: - indicates the ordering that should be applied when loading these - items. ``order_by`` is expected to refer to one of the :class:`.Column` - objects to which the target class is mapped, or - the attribute itself bound to the target class which refers - to the column. - - ``order_by`` may also be passed as a callable function - which is evaluated at mapper initialization time, and may be passed as a - Python-evaluable string when using Declarative. - - :param passive_deletes=False: - Indicates loading behavior during delete operations. - - A value of True indicates that unloaded child items should not - be loaded during a delete operation on the parent. Normally, - when a parent item is deleted, all child items are loaded so - that they can either be marked as deleted, or have their - foreign key to the parent set to NULL. Marking this flag as - True usually implies an ON DELETE rule is in - place which will handle updating/deleting child rows on the - database side. - - Additionally, setting the flag to the string value 'all' will - disable the "nulling out" of the child foreign keys, when there - is no delete or delete-orphan cascade enabled. This is - typically used when a triggering or error raise scenario is in - place on the database side. Note that the foreign key - attributes on in-session child objects will not be changed - after a flush occurs so this is a very special use-case - setting. - - :param passive_updates=True: - Indicates loading and INSERT/UPDATE/DELETE behavior when the - source of a foreign key value changes (i.e. an "on update" - cascade), which are typically the primary key columns of the - source row. - - When True, it is assumed that ON UPDATE CASCADE is configured on - the foreign key in the database, and that the database will - handle propagation of an UPDATE from a source column to - dependent rows. Note that with databases which enforce - referential integrity (i.e. PostgreSQL, MySQL with InnoDB tables), - ON UPDATE CASCADE is required for this operation. The - relationship() will update the value of the attribute on related - items which are locally present in the session during a flush. - - When False, it is assumed that the database does not enforce - referential integrity and will not be issuing its own CASCADE - operation for an update. The relationship() will issue the - appropriate UPDATE statements to the database in response to the - change of a referenced key, and items locally present in the - session during a flush will also be refreshed. - - This flag should probably be set to False if primary key changes - are expected and the database in use doesn't support CASCADE - (i.e. SQLite, MySQL MyISAM tables). - - Also see the passive_updates flag on ``mapper()``. - - A future SQLAlchemy release will provide a "detect" feature for - this flag. - - :param post_update: - this indicates that the relationship should be handled by a - second UPDATE statement after an INSERT or before a - DELETE. Currently, it also will issue an UPDATE after the - instance was UPDATEd as well, although this technically should - be improved. This flag is used to handle saving bi-directional - dependencies between two individual rows (i.e. each row - references the other), where it would otherwise be impossible to - INSERT or DELETE both rows fully since one row exists before the - other. Use this flag when a particular mapping arrangement will - incur two rows that are dependent on each other, such as a table - that has a one-to-many relationship to a set of child rows, and - also has a column that references a single child row within that - list (i.e. both tables contain a foreign key to each other). If - a ``flush()`` operation returns an error that a "cyclical - dependency" was detected, this is a cue that you might want to - use ``post_update`` to "break" the cycle. - - :param primaryjoin: - a SQL expression that will be used as the primary - join of this child object against the parent object, or in a - many-to-many relationship the join of the primary object to the - association table. By default, this value is computed based on the - foreign key relationships of the parent and child tables (or association - table). - - ``primaryjoin`` may also be passed as a callable function - which is evaluated at mapper initialization time, and may be passed as a - Python-evaluable string when using Declarative. - - :param remote_side: - used for self-referential relationships, indicates the column or - list of columns that form the "remote side" of the relationship. - - ``remote_side`` may also be passed as a callable function - which is evaluated at mapper initialization time, and may be passed as a - Python-evaluable string when using Declarative. - - :param query_class: - a :class:`.Query` subclass that will be used as the base of the - "appender query" returned by a "dynamic" relationship, that - is, a relationship that specifies ``lazy="dynamic"`` or was - otherwise constructed using the :func:`.orm.dynamic_loader` - function. - - :param secondaryjoin: - a SQL expression that will be used as the join of - an association table to the child object. By default, this value is - computed based on the foreign key relationships of the association and - child tables. - - ``secondaryjoin`` may also be passed as a callable function - which is evaluated at mapper initialization time, and may be passed as a - Python-evaluable string when using Declarative. - - :param single_parent=(True|False): - when True, installs a validator which will prevent objects - from being associated with more than one parent at a time. - This is used for many-to-one or many-to-many relationships that - should be treated either as one-to-one or one-to-many. Its - usage is optional unless delete-orphan cascade is also - set on this relationship(), in which case its required. - - :param uselist=(True|False): - a boolean that indicates if this property should be loaded as a - list or a scalar. In most cases, this value is determined - automatically by ``relationship()``, based on the type and direction - of the relationship - one to many forms a list, many to one - forms a scalar, many to many is a list. If a scalar is desired - where normally a list would be present, such as a bi-directional - one-to-one relationship, set uselist to False. - - :param viewonly=False: - when set to True, the relationship is used only for loading objects - within the relationship, and has no effect on the unit-of-work - flush process. Relationships with viewonly can specify any kind of - join conditions to provide additional views of related objects - onto a parent object. Note that the functionality of a viewonly - relationship has its limits - complicated join conditions may - not compile into eager or lazy loaders properly. If this is the - case, use an alternative method. - - """ - return RelationshipProperty(argument, secondary=secondary, **kwargs) +relationship = public_factory(RelationshipProperty, ".orm.relationship") def relation(*arg, **kw): """A synonym for :func:`relationship`.""" return relationship(*arg, **kw) + def dynamic_loader(argument, **kw): """Construct a dynamically-loading mapper property. @@ -648,130 +130,14 @@ def dynamic_loader(argument, **kw): kw['lazy'] = 'dynamic' return relationship(argument, **kw) -def column_property(*cols, **kw): - """Provide a column-level property for use with a Mapper. - Column-based properties can normally be applied to the mapper's - ``properties`` dictionary using the :class:`.Column` element directly. - Use this function when the given column is not directly present within the - mapper's selectable; examples include SQL expressions, functions, and - scalar SELECT queries. - - Columns that aren't present in the mapper's selectable won't be persisted - by the mapper and are effectively "read-only" attributes. - - :param \*cols: - list of Column objects to be mapped. - - :param active_history=False: - When ``True``, indicates that the "previous" value for a - scalar attribute should be loaded when replaced, if not - already loaded. Normally, history tracking logic for - simple non-primary-key scalar values only needs to be - aware of the "new" value in order to perform a flush. This - flag is available for applications that make use of - :func:`.attributes.get_history` or :meth:`.Session.is_modified` - which also need to know - the "previous" value of the attribute. - - .. versionadded:: 0.6.6 - - :param comparator_factory: a class which extends - :class:`.ColumnProperty.Comparator` which provides custom SQL clause - generation for comparison operations. - - :param group: - a group name for this property when marked as deferred. - - :param deferred: - when True, the column property is "deferred", meaning that - it does not load immediately, and is instead loaded when the - attribute is first accessed on an instance. See also - :func:`~sqlalchemy.orm.deferred`. - - :param doc: - optional string that will be applied as the doc on the - class-bound descriptor. - - :param expire_on_flush=True: - Disable expiry on flush. A column_property() which refers - to a SQL expression (and not a single table-bound column) - is considered to be a "read only" property; populating it - has no effect on the state of data, and it can only return - database state. For this reason a column_property()'s value - is expired whenever the parent object is involved in a - flush, that is, has any kind of "dirty" state within a flush. - Setting this parameter to ``False`` will have the effect of - leaving any existing value present after the flush proceeds. - Note however that the :class:`.Session` with default expiration - settings still expires - all attributes after a :meth:`.Session.commit` call, however. - - .. versionadded:: 0.7.3 - - :param extension: - an - :class:`.AttributeExtension` - instance, or list of extensions, which will be prepended - to the list of attribute listeners for the resulting - descriptor placed on the class. - **Deprecated.** Please see :class:`.AttributeEvents`. - - - """ - - return ColumnProperty(*cols, **kw) - -def composite(class_, *cols, **kwargs): - """Return a composite column-based property for use with a Mapper. - - See the mapping documentation section :ref:`mapper_composite` for a full - usage example. - - :param class\_: - The "composite type" class. - - :param \*cols: - List of Column objects to be mapped. - - :param active_history=False: - When ``True``, indicates that the "previous" value for a - scalar attribute should be loaded when replaced, if not - already loaded. See the same flag on :func:`.column_property`. - - .. versionchanged:: 0.7 - This flag specifically becomes meaningful - - previously it was a placeholder. - - :param group: - A group name for this property when marked as deferred. - - :param deferred: - When True, the column property is "deferred", meaning that it does not - load immediately, and is instead loaded when the attribute is first - accessed on an instance. See also :func:`~sqlalchemy.orm.deferred`. - - :param comparator_factory: a class which extends - :class:`.CompositeProperty.Comparator` which provides custom SQL clause - generation for comparison operations. - - :param doc: - optional string that will be applied as the doc on the - class-bound descriptor. - - :param extension: - an :class:`.AttributeExtension` instance, - or list of extensions, which will be prepended to the list of - attribute listeners for the resulting descriptor placed on the class. - **Deprecated.** Please see :class:`.AttributeEvents`. - - """ - return CompositeProperty(class_, *cols, **kwargs) +column_property = public_factory(ColumnProperty, ".orm.column_property") +composite = public_factory(CompositeProperty, ".orm.composite") def backref(name, **kwargs): - """Create a back reference with explicit keyword arguments, which are the same - arguments one can send to :func:`relationship`. + """Create a back reference with explicit keyword arguments, which are the + same arguments one can send to :func:`relationship`. Used with the ``backref`` keyword argument to :func:`relationship` in place of a string argument, e.g.:: @@ -781,467 +147,43 @@ def backref(name, **kwargs): """ return (name, kwargs) -def deferred(*columns, **kwargs): - """Return a :class:`.DeferredColumnProperty`, which indicates this - object attributes should only be loaded from its corresponding - table column when first accessed. - Used with the "properties" dictionary sent to :func:`mapper`. +def deferred(*columns, **kw): + """Indicate a column-based mapped attribute that by default will + not load unless accessed. - See also: + :param \*columns: columns to be mapped. This is typically a single + :class:`.Column` object, however a collection is supported in order + to support multiple columns mapped under the same attribute. - :ref:`deferred` + :param \**kw: additional keyword arguments passed to :class:`.ColumnProperty`. + + .. seealso:: + + :ref:`deferred` """ - return ColumnProperty(deferred=True, *columns, **kwargs) + return ColumnProperty(deferred=True, *columns, **kw) -def mapper(class_, local_table=None, *args, **params): - """Return a new :class:`~.Mapper` object. - This function is typically used behind the scenes - via the Declarative extension. When using Declarative, - many of the usual :func:`.mapper` arguments are handled - by the Declarative extension itself, including ``class_``, - ``local_table``, ``properties``, and ``inherits``. - Other options are passed to :func:`.mapper` using - the ``__mapper_args__`` class variable:: +mapper = public_factory(Mapper, ".orm.mapper") - class MyClass(Base): - __tablename__ = 'my_table' - id = Column(Integer, primary_key=True) - type = Column(String(50)) - alt = Column("some_alt", Integer) +synonym = public_factory(SynonymProperty, ".orm.synonym") - __mapper_args__ = { - 'polymorphic_on' : type - } +comparable_property = public_factory(ComparableProperty, + ".orm.comparable_property") - Explicit use of :func:`.mapper` - is often referred to as *classical mapping*. The above - declarative example is equivalent in classical form to:: - - my_table = Table("my_table", metadata, - Column('id', Integer, primary_key=True), - Column('type', String(50)), - Column("some_alt", Integer) - ) - - class MyClass(object): - pass - - mapper(MyClass, my_table, - polymorphic_on=my_table.c.type, - properties={ - 'alt':my_table.c.some_alt - }) - - See also: - - :ref:`classical_mapping` - discussion of direct usage of - :func:`.mapper` - - :param class\_: The class to be mapped. When using Declarative, - this argument is automatically passed as the declared class - itself. - - :param local_table: The :class:`.Table` or other selectable - to which the class is mapped. May be ``None`` if - this mapper inherits from another mapper using single-table - inheritance. When using Declarative, this argument is - automatically passed by the extension, based on what - is configured via the ``__table__`` argument or via the :class:`.Table` - produced as a result of the ``__tablename__`` and :class:`.Column` - arguments present. - - :param always_refresh: If True, all query operations for this mapped - class will overwrite all data within object instances that already - exist within the session, erasing any in-memory changes with - whatever information was loaded from the database. Usage of this - flag is highly discouraged; as an alternative, see the method - :meth:`.Query.populate_existing`. - - :param allow_null_pks: This flag is deprecated - this is stated as - allow_partial_pks which defaults to True. - - :param allow_partial_pks: Defaults to True. Indicates that a - composite primary key with some NULL values should be considered as - possibly existing within the database. This affects whether a - mapper will assign an incoming row to an existing identity, as well - as if :meth:`.Session.merge` will check the database first for a - particular primary key value. A "partial primary key" can occur if - one has mapped to an OUTER JOIN, for example. - - :param batch: Defaults to ``True``, indicating that save operations - of multiple entities can be batched together for efficiency. - Setting to False indicates - that an instance will be fully saved before saving the next - instance. This is used in the extremely rare case that a - :class:`.MapperEvents` listener requires being called - in between individual row persistence operations. - - :param column_prefix: A string which will be prepended - to the mapped attribute name when :class:`.Column` - objects are automatically assigned as attributes to the - mapped class. Does not affect explicitly specified - column-based properties. - - See the section :ref:`column_prefix` for an example. - - :param concrete: If True, indicates this mapper should use concrete - table inheritance with its parent mapper. - - See the section :ref:`concrete_inheritance` for an example. - - :param exclude_properties: A list or set of string column names to - be excluded from mapping. - - See :ref:`include_exclude_cols` for an example. - - :param extension: A :class:`.MapperExtension` instance or - list of :class:`.MapperExtension` - instances which will be applied to all operations by this - :class:`.Mapper`. **Deprecated.** Please see :class:`.MapperEvents`. - - :param include_properties: An inclusive list or set of string column - names to map. - - See :ref:`include_exclude_cols` for an example. - - :param inherits: A mapped class or the corresponding :class:`.Mapper` - of one indicating a superclass to which this :class:`.Mapper` - should *inherit* from. The mapped class here must be a subclass of the - other mapper's class. When using Declarative, this argument - is passed automatically as a result of the natural class - hierarchy of the declared classes. - - See also: - - :ref:`inheritance_toplevel` - - :param inherit_condition: For joined table inheritance, a SQL - expression which will - define how the two tables are joined; defaults to a natural join - between the two tables. - - :param inherit_foreign_keys: When ``inherit_condition`` is used and the - columns present are missing a :class:`.ForeignKey` configuration, - this parameter can be used to specify which columns are "foreign". - In most cases can be left as ``None``. - - :param non_primary: Specify that this :class:`.Mapper` is in addition - to the "primary" mapper, that is, the one used for persistence. - The :class:`.Mapper` created here may be used for ad-hoc - mapping of the class to an alternate selectable, for loading - only. - - The ``non_primary`` feature is rarely needed with modern - usage. - - :param order_by: A single :class:`.Column` or list of :class:`.Column` - objects for which selection operations should use as the default - ordering for entities. By default mappers have no pre-defined - ordering. - - :param passive_updates: Indicates UPDATE behavior of foreign key - columns when a primary key column changes on a joined-table inheritance - mapping. Defaults to ``True``. - - When True, it is assumed that ON UPDATE CASCADE is configured on - the foreign key in the database, and that the database will handle - propagation of an UPDATE from a source column to dependent columns - on joined-table rows. - - When False, it is assumed that the database does not enforce - referential integrity and will not be issuing its own CASCADE - operation for an update. The :class:`.Mapper` here will - emit an UPDATE statement for the dependent columns during a - primary key change. - - See also: - - :ref:`passive_updates` - description of a similar feature as - used with :func:`.relationship` - - :param polymorphic_on: Specifies the column, attribute, or - SQL expression used to determine the target class for an - incoming row, when inheriting classes are present. - - This value is commonly a :class:`.Column` object that's - present in the mapped :class:`.Table`:: - - class Employee(Base): - __tablename__ = 'employee' - - id = Column(Integer, primary_key=True) - discriminator = Column(String(50)) - - __mapper_args__ = { - "polymorphic_on":discriminator, - "polymorphic_identity":"employee" - } - - It may also be specified - as a SQL expression, as in this example where we - use the :func:`.case` construct to provide a conditional - approach:: - - class Employee(Base): - __tablename__ = 'employee' - - id = Column(Integer, primary_key=True) - discriminator = Column(String(50)) - - __mapper_args__ = { - "polymorphic_on":case([ - (discriminator == "EN", "engineer"), - (discriminator == "MA", "manager"), - ], else_="employee"), - "polymorphic_identity":"employee" - } - - It may also refer to any attribute - configured with :func:`.column_property`, or to the - string name of one:: - - class Employee(Base): - __tablename__ = 'employee' - - id = Column(Integer, primary_key=True) - discriminator = Column(String(50)) - employee_type = column_property( - case([ - (discriminator == "EN", "engineer"), - (discriminator == "MA", "manager"), - ], else_="employee") - ) - - __mapper_args__ = { - "polymorphic_on":employee_type, - "polymorphic_identity":"employee" - } - - .. versionchanged:: 0.7.4 - ``polymorphic_on`` may be specified as a SQL expression, - or refer to any attribute configured with - :func:`.column_property`, or to the string name of one. - - When setting ``polymorphic_on`` to reference an - attribute or expression that's not present in the - locally mapped :class:`.Table`, yet the value - of the discriminator should be persisted to the database, - the value of the - discriminator is not automatically set on new - instances; this must be handled by the user, - either through manual means or via event listeners. - A typical approach to establishing such a listener - looks like:: - - from sqlalchemy import event - from sqlalchemy.orm import object_mapper - - @event.listens_for(Employee, "init", propagate=True) - def set_identity(instance, *arg, **kw): - mapper = object_mapper(instance) - instance.discriminator = mapper.polymorphic_identity - - Where above, we assign the value of ``polymorphic_identity`` - for the mapped class to the ``discriminator`` attribute, - thus persisting the value to the ``discriminator`` column - in the database. - - See also: - - :ref:`inheritance_toplevel` - - :param polymorphic_identity: Specifies the value which - identifies this particular class as returned by the - column expression referred to by the ``polymorphic_on`` - setting. As rows are received, the value corresponding - to the ``polymorphic_on`` column expression is compared - to this value, indicating which subclass should - be used for the newly reconstructed object. - - :param properties: A dictionary mapping the string names of object - attributes to :class:`.MapperProperty` instances, which define the - persistence behavior of that attribute. Note that :class:`.Column` - objects present in - the mapped :class:`.Table` are automatically placed into - ``ColumnProperty`` instances upon mapping, unless overridden. - When using Declarative, this argument is passed automatically, - based on all those :class:`.MapperProperty` instances declared - in the declared class body. - - :param primary_key: A list of :class:`.Column` objects which define the - primary key to be used against this mapper's selectable unit. - This is normally simply the primary key of the ``local_table``, but - can be overridden here. - - :param version_id_col: A :class:`.Column` - that will be used to keep a running version id of mapped entities - in the database. This is used during save operations to ensure that - no other thread or process has updated the instance during the - lifetime of the entity, else a :class:`~sqlalchemy.orm.exc.StaleDataError` - exception is - thrown. By default the column must be of :class:`.Integer` type, - unless ``version_id_generator`` specifies a new generation - algorithm. - - :param version_id_generator: A callable which defines the algorithm - used to generate new version ids. Defaults to an integer - generator. Can be replaced with one that generates timestamps, - uuids, etc. e.g.:: - - import uuid - - class MyClass(Base): - __tablename__ = 'mytable' - id = Column(Integer, primary_key=True) - version_uuid = Column(String(32)) - - __mapper_args__ = { - 'version_id_col':version_uuid, - 'version_id_generator':lambda version:uuid.uuid4().hex - } - - The callable receives the current version identifier as its - single argument. - - :param with_polymorphic: A tuple in the form ``(, - )`` indicating the default style of "polymorphic" - loading, that is, which tables are queried at once. is - any single or list of mappers and/or classes indicating the - inherited classes that should be loaded at once. The special value - ``'*'`` may be used to indicate all descending classes should be - loaded immediately. The second tuple argument - indicates a selectable that will be used to query for multiple - classes. - - See also: - - :ref:`concrete_inheritance` - typically uses ``with_polymorphic`` - to specify a UNION statement to select from. - - :ref:`with_polymorphic` - usage example of the related - :meth:`.Query.with_polymorphic` method - - """ - return Mapper(class_, local_table, *args, **params) - -def synonym(name, map_column=False, descriptor=None, - comparator_factory=None, doc=None): - """Denote an attribute name as a synonym to a mapped property. - - .. versionchanged:: 0.7 - :func:`.synonym` is superseded by the :mod:`~sqlalchemy.ext.hybrid` - extension. See the documentation for hybrids - at :ref:`hybrids_toplevel`. - - Used with the ``properties`` dictionary sent to - :func:`~sqlalchemy.orm.mapper`:: - - class MyClass(object): - def _get_status(self): - return self._status - def _set_status(self, value): - self._status = value - status = property(_get_status, _set_status) - - mapper(MyClass, sometable, properties={ - "status":synonym("_status", map_column=True) - }) - - Above, the ``status`` attribute of MyClass will produce - expression behavior against the table column named ``status``, - using the Python attribute ``_status`` on the mapped class - to represent the underlying value. - - :param name: the name of the existing mapped property, which can be - any other ``MapperProperty`` including column-based properties and - relationships. - - :param map_column: if ``True``, an additional ``ColumnProperty`` is created - on the mapper automatically, using the synonym's name as the keyname of - the property, and the keyname of this ``synonym()`` as the name of the - column to map. - - """ - return SynonymProperty(name, map_column=map_column, - descriptor=descriptor, - comparator_factory=comparator_factory, - doc=doc) - -def comparable_property(comparator_factory, descriptor=None): - """Provides a method of applying a :class:`.PropComparator` - to any Python descriptor attribute. - - .. versionchanged:: 0.7 - :func:`.comparable_property` is superseded by - the :mod:`~sqlalchemy.ext.hybrid` extension. See the example - at :ref:`hybrid_custom_comparators`. - - Allows any Python descriptor to behave like a SQL-enabled - attribute when used at the class level in queries, allowing - redefinition of expression operator behavior. - - In the example below we redefine :meth:`.PropComparator.operate` - to wrap both sides of an expression in ``func.lower()`` to produce - case-insensitive comparison:: - - from sqlalchemy.orm import comparable_property - from sqlalchemy.orm.interfaces import PropComparator - from sqlalchemy.sql import func - from sqlalchemy import Integer, String, Column - from sqlalchemy.ext.declarative import declarative_base - - class CaseInsensitiveComparator(PropComparator): - def __clause_element__(self): - return self.prop - - def operate(self, op, other): - return op( - func.lower(self.__clause_element__()), - func.lower(other) - ) - - Base = declarative_base() - - class SearchWord(Base): - __tablename__ = 'search_word' - id = Column(Integer, primary_key=True) - word = Column(String) - word_insensitive = comparable_property(lambda prop, mapper: - CaseInsensitiveComparator(mapper.c.word, mapper) - ) - - - A mapping like the above allows the ``word_insensitive`` attribute - to render an expression like:: - - >>> print SearchWord.word_insensitive == "Trucks" - lower(search_word.word) = lower(:lower_1) - - :param comparator_factory: - A PropComparator subclass or factory that defines operator behavior - for this property. - - :param descriptor: - Optional when used in a ``properties={}`` declaration. The Python - descriptor or property to layer comparison behavior on top of. - - The like-named descriptor will be automatically retrieved from the - mapped class if left blank in a ``properties`` declaration. - - """ - return ComparableProperty(comparator_factory, descriptor) - -@sa_util.deprecated("0.7", message=":func:`.compile_mappers` " +@_sa_util.deprecated("0.7", message=":func:`.compile_mappers` " "is renamed to :func:`.configure_mappers`") def compile_mappers(): - """Initialize the inter-mapper relationships of all mappers that have been defined.""" + """Initialize the inter-mapper relationships of all mappers that have + been defined. + """ configure_mappers() + def clear_mappers(): """Remove all mappers from all classes. @@ -1262,7 +204,7 @@ def clear_mappers(): set of classes. """ - mapperlib._COMPILE_MUTEX.acquire() + mapperlib._CONFIGURE_MUTEX.acquire() try: while _mapper_registry: try: @@ -1272,414 +214,54 @@ def clear_mappers(): except KeyError: pass finally: - mapperlib._COMPILE_MUTEX.release() + mapperlib._CONFIGURE_MUTEX.release() -def joinedload(*keys, **kw): - """Return a ``MapperOption`` that will convert the property of the given - name or series of mapped attributes into an joined eager load. +from . import strategy_options - .. versionchanged:: 0.6beta3 - This function is known as :func:`eagerload` in all versions - of SQLAlchemy prior to version 0.6beta3, including the 0.5 and 0.4 - series. :func:`eagerload` will remain available for the foreseeable - future in order to enable cross-compatibility. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - examples:: - - # joined-load the "orders" collection on "User" - query(User).options(joinedload(User.orders)) - - # joined-load the "keywords" collection on each "Item", - # but not the "items" collection on "Order" - those - # remain lazily loaded. - query(Order).options(joinedload(Order.items, Item.keywords)) - - # to joined-load across both, use joinedload_all() - query(Order).options(joinedload_all(Order.items, Item.keywords)) - - # set the default strategy to be 'joined' - query(Order).options(joinedload('*')) - - :func:`joinedload` also accepts a keyword argument `innerjoin=True` which - indicates using an inner join instead of an outer:: - - query(Order).options(joinedload(Order.user, innerjoin=True)) - - .. note:: - - The join created by :func:`joinedload` is anonymously aliased such that - it **does not affect the query results**. An :meth:`.Query.order_by` - or :meth:`.Query.filter` call **cannot** reference these aliased - tables - so-called "user space" joins are constructed using - :meth:`.Query.join`. The rationale for this is that :func:`joinedload` is only - applied in order to affect how related objects or collections are loaded - as an optimizing detail - it can be added or removed with no impact - on actual results. See the section :ref:`zen_of_eager_loading` for - a detailed description of how this is used, including how to use a single - explicit JOIN for filtering/ordering and eager loading simultaneously. - - See also: :func:`subqueryload`, :func:`lazyload` - - """ - innerjoin = kw.pop('innerjoin', None) - if innerjoin is not None: - return ( - strategies.EagerLazyOption(keys, lazy='joined'), - strategies.EagerJoinOption(keys, innerjoin) - ) - else: - return strategies.EagerLazyOption(keys, lazy='joined') - -def joinedload_all(*keys, **kw): - """Return a ``MapperOption`` that will convert all properties along the - given dot-separated path or series of mapped attributes - into an joined eager load. - - .. versionchanged:: 0.6beta3 - This function is known as :func:`eagerload_all` in all versions - of SQLAlchemy prior to version 0.6beta3, including the 0.5 and 0.4 - series. :func:`eagerload_all` will remain available for the - foreseeable future in order to enable cross-compatibility. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - For example:: - - query.options(joinedload_all('orders.items.keywords'))... - - will set all of ``orders``, ``orders.items``, and ``orders.items.keywords`` to - load in one joined eager load. - - Individual descriptors are accepted as arguments as well:: - - query.options(joinedload_all(User.orders, Order.items, Item.keywords)) - - The keyword arguments accept a flag `innerjoin=True|False` which will - override the value of the `innerjoin` flag specified on the - relationship(). - - See also: :func:`subqueryload_all`, :func:`lazyload` - - """ - innerjoin = kw.pop('innerjoin', None) - if innerjoin is not None: - return ( - strategies.EagerLazyOption(keys, lazy='joined', chained=True), - strategies.EagerJoinOption(keys, innerjoin, chained=True) - ) - else: - return strategies.EagerLazyOption(keys, lazy='joined', chained=True) +joinedload = strategy_options.joinedload._unbound_fn +joinedload_all = strategy_options.joinedload._unbound_all_fn +contains_eager = strategy_options.contains_eager._unbound_fn +defer = strategy_options.defer._unbound_fn +undefer = strategy_options.undefer._unbound_fn +undefer_group = strategy_options.undefer_group._unbound_fn +load_only = strategy_options.load_only._unbound_fn +lazyload = strategy_options.lazyload._unbound_fn +lazyload_all = strategy_options.lazyload_all._unbound_all_fn +subqueryload = strategy_options.subqueryload._unbound_fn +subqueryload_all = strategy_options.subqueryload_all._unbound_all_fn +immediateload = strategy_options.immediateload._unbound_fn +noload = strategy_options.noload._unbound_fn +defaultload = strategy_options.defaultload._unbound_fn +from .strategy_options import Load def eagerload(*args, **kwargs): """A synonym for :func:`joinedload()`.""" return joinedload(*args, **kwargs) + def eagerload_all(*args, **kwargs): """A synonym for :func:`joinedload_all()`""" return joinedload_all(*args, **kwargs) -def subqueryload(*keys): - """Return a ``MapperOption`` that will convert the property - of the given name or series of mapped attributes - into an subquery eager load. - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - examples:: - # subquery-load the "orders" collection on "User" - query(User).options(subqueryload(User.orders)) +contains_alias = public_factory(AliasOption, ".orm.contains_alias") - # subquery-load the "keywords" collection on each "Item", - # but not the "items" collection on "Order" - those - # remain lazily loaded. - query(Order).options(subqueryload(Order.items, Item.keywords)) - # to subquery-load across both, use subqueryload_all() - query(Order).options(subqueryload_all(Order.items, Item.keywords)) - # set the default strategy to be 'subquery' - query(Order).options(subqueryload('*')) +def __go(lcls): + global __all__ + from .. import util as sa_util + from . import dynamic + from . import events + import inspect as _inspect - See also: :func:`joinedload`, :func:`lazyload` + __all__ = sorted(name for name, obj in lcls.items() + if not (name.startswith('_') or _inspect.ismodule(obj))) - """ - return strategies.EagerLazyOption(keys, lazy="subquery") + _sa_util.dependencies.resolve_all("sqlalchemy.orm") -def subqueryload_all(*keys): - """Return a ``MapperOption`` that will convert all properties along the - given dot-separated path or series of mapped attributes - into a subquery eager load. +__go(locals()) - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - For example:: - - query.options(subqueryload_all('orders.items.keywords'))... - - will set all of ``orders``, ``orders.items``, and ``orders.items.keywords`` to - load in one subquery eager load. - - Individual descriptors are accepted as arguments as well:: - - query.options(subqueryload_all(User.orders, Order.items, - Item.keywords)) - - See also: :func:`joinedload_all`, :func:`lazyload`, :func:`immediateload` - - """ - return strategies.EagerLazyOption(keys, lazy="subquery", chained=True) - -def lazyload(*keys): - """Return a ``MapperOption`` that will convert the property of the given - name or series of mapped attributes into a lazy load. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - See also: :func:`eagerload`, :func:`subqueryload`, :func:`immediateload` - - """ - return strategies.EagerLazyOption(keys, lazy=True) - -def lazyload_all(*keys): - """Return a ``MapperOption`` that will convert all the properties - along the given dot-separated path or series of mapped attributes - into a lazy load. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - See also: :func:`eagerload`, :func:`subqueryload`, :func:`immediateload` - - """ - return strategies.EagerLazyOption(keys, lazy=True, chained=True) - -def noload(*keys): - """Return a ``MapperOption`` that will convert the property of the - given name or series of mapped attributes into a non-load. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - See also: :func:`lazyload`, :func:`eagerload`, - :func:`subqueryload`, :func:`immediateload` - - """ - return strategies.EagerLazyOption(keys, lazy=None) - -def immediateload(*keys): - """Return a ``MapperOption`` that will convert the property of the given - name or series of mapped attributes into an immediate load. - - The "immediate" load means the attribute will be fetched - with a separate SELECT statement per parent in the - same way as lazy loading - except the loader is guaranteed - to be called at load time before the parent object - is returned in the result. - - The normal behavior of lazy loading applies - if - the relationship is a simple many-to-one, and the child - object is already present in the :class:`.Session`, - no SELECT statement will be emitted. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - See also: :func:`lazyload`, :func:`eagerload`, :func:`subqueryload` - - .. versionadded:: 0.6.5 - - """ - return strategies.EagerLazyOption(keys, lazy='immediate') - -def contains_alias(alias): - """Return a :class:`.MapperOption` that will indicate to the query that - the main table has been aliased. - - This is used in the very rare case that :func:`.contains_eager` - is being used in conjunction with a user-defined SELECT - statement that aliases the parent table. E.g.:: - - # define an aliased UNION called 'ulist' - statement = users.select(users.c.user_id==7).\\ - union(users.select(users.c.user_id>7)).\\ - alias('ulist') - - # add on an eager load of "addresses" - statement = statement.outerjoin(addresses).\\ - select().apply_labels() - - # create query, indicating "ulist" will be an - # alias for the main table, "addresses" - # property should be eager loaded - query = session.query(User).options( - contains_alias('ulist'), - contains_eager('addresses')) - - # then get results via the statement - results = query.from_statement(statement).all() - - :param alias: is the string name of an alias, or a - :class:`~.sql.expression.Alias` object representing - the alias. - - """ - return AliasOption(alias) - -def contains_eager(*keys, **kwargs): - """Return a ``MapperOption`` that will indicate to the query that - the given attribute should be eagerly loaded from columns currently - in the query. - - Used with :meth:`~sqlalchemy.orm.query.Query.options`. - - The option is used in conjunction with an explicit join that loads - the desired rows, i.e.:: - - sess.query(Order).\\ - join(Order.user).\\ - options(contains_eager(Order.user)) - - The above query would join from the ``Order`` entity to its related - ``User`` entity, and the returned ``Order`` objects would have the - ``Order.user`` attribute pre-populated. - - :func:`contains_eager` also accepts an `alias` argument, which is the - string name of an alias, an :func:`~sqlalchemy.sql.expression.alias` - construct, or an :func:`~sqlalchemy.orm.aliased` construct. Use this when - the eagerly-loaded rows are to come from an aliased table:: - - user_alias = aliased(User) - sess.query(Order).\\ - join((user_alias, Order.user)).\\ - options(contains_eager(Order.user, alias=user_alias)) - - See also :func:`eagerload` for the "automatic" version of this - functionality. - - For additional examples of :func:`contains_eager` see - :ref:`contains_eager`. - - """ - alias = kwargs.pop('alias', None) - if kwargs: - raise exceptions.ArgumentError('Invalid kwargs for contains_eag' - 'er: %r' % kwargs.keys()) - return strategies.EagerLazyOption(keys, lazy='joined', - propagate_to_loaders=False, chained=True), \ - strategies.LoadEagerFromAliasOption(keys, alias=alias, chained=True) - -def defer(*key): - """Return a :class:`.MapperOption` that will convert the column property - of the given name into a deferred load. - - Used with :meth:`.Query.options`. - - e.g.:: - - from sqlalchemy.orm import defer - - query(MyClass).options(defer("attribute_one"), - defer("attribute_two")) - - A class bound descriptor is also accepted:: - - query(MyClass).options( - defer(MyClass.attribute_one), - defer(MyClass.attribute_two)) - - A "path" can be specified onto a related or collection object using a - dotted name. The :func:`.orm.defer` option will be applied to that object - when loaded:: - - query(MyClass).options( - defer("related.attribute_one"), - defer("related.attribute_two")) - - To specify a path via class, send multiple arguments:: - - query(MyClass).options( - defer(MyClass.related, MyOtherClass.attribute_one), - defer(MyClass.related, MyOtherClass.attribute_two)) - - See also: - - :ref:`deferred` - - :param \*key: A key representing an individual path. Multiple entries - are accepted to allow a multiple-token path for a single target, not - multiple targets. - - """ - return strategies.DeferredOption(key, defer=True) - -def undefer(*key): - """Return a :class:`.MapperOption` that will convert the column property - of the given name into a non-deferred (regular column) load. - - Used with :meth:`.Query.options`. - - e.g.:: - - from sqlalchemy.orm import undefer - - query(MyClass).options(undefer("attribute_one"), - undefer("attribute_two")) - - A class bound descriptor is also accepted:: - - query(MyClass).options( - undefer(MyClass.attribute_one), - undefer(MyClass.attribute_two)) - - A "path" can be specified onto a related or collection object using a - dotted name. The :func:`.orm.undefer` option will be applied to that - object when loaded:: - - query(MyClass).options( - undefer("related.attribute_one"), - undefer("related.attribute_two")) - - To specify a path via class, send multiple arguments:: - - query(MyClass).options( - undefer(MyClass.related, MyOtherClass.attribute_one), - undefer(MyClass.related, MyOtherClass.attribute_two)) - - See also: - - :func:`.orm.undefer_group` as a means to "undefer" a group - of attributes at once. - - :ref:`deferred` - - :param \*key: A key representing an individual path. Multiple entries - are accepted to allow a multiple-token path for a single target, not - multiple targets. - - """ - return strategies.DeferredOption(key, defer=False) - -def undefer_group(name): - """Return a :class:`.MapperOption` that will convert the given group of deferred - column properties into a non-deferred (regular column) load. - - Used with :meth:`.Query.options`. - - e.g.:: - - query(MyClass).options(undefer("group_one")) - - See also: - - :ref:`deferred` - - :param name: String name of the deferred group. This name is - established using the "group" name to the :func:`.orm.deferred` - configurational function. - - """ - return strategies.UndeferGroupOption(name) - -from sqlalchemy import util as _sa_util -_sa_util.importlater.resolve_all() diff --git a/libs/sqlalchemy/orm/attributes.py b/libs/sqlalchemy/orm/attributes.py index b40abd57..e5f8550a 100644 --- a/libs/sqlalchemy/orm/attributes.py +++ b/libs/sqlalchemy/orm/attributes.py @@ -1,5 +1,5 @@ # orm/attributes.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -14,80 +14,51 @@ defines a large part of the ORM's interactivity. """ import operator -from operator import itemgetter +from .. import util, event, inspection +from . import interfaces, collections, exc as orm_exc -from sqlalchemy import util, event, exc as sa_exc -from sqlalchemy.orm import interfaces, collections, events, exc as orm_exc +from .base import instance_state, instance_dict, manager_of_class + +from .base import PASSIVE_NO_RESULT, ATTR_WAS_SET, ATTR_EMPTY, NO_VALUE,\ + NEVER_SET, NO_CHANGE, CALLABLES_OK, SQL_OK, RELATED_OBJECT_OK,\ + INIT_OK, NON_PERSISTENT_OK, LOAD_AGAINST_COMMITTED, PASSIVE_OFF,\ + PASSIVE_RETURN_NEVER_SET, PASSIVE_NO_INITIALIZE, PASSIVE_NO_FETCH,\ + PASSIVE_NO_FETCH_RELATED, PASSIVE_ONLY_PERSISTENT +from .base import state_str, instance_str + +@inspection._self_inspects +class QueryableAttribute(interfaces._MappedAttribute, + interfaces._InspectionAttr, + interfaces.PropComparator): + """Base class for :term:`descriptor` objects that intercept + attribute events on behalf of a :class:`.MapperProperty` + object. The actual :class:`.MapperProperty` is accessible + via the :attr:`.QueryableAttribute.property` + attribute. -mapperutil = util.importlater("sqlalchemy.orm", "util") + .. seealso:: -PASSIVE_NO_RESULT = util.symbol('PASSIVE_NO_RESULT') -ATTR_WAS_SET = util.symbol('ATTR_WAS_SET') -ATTR_EMPTY = util.symbol('ATTR_EMPTY') -NO_VALUE = util.symbol('NO_VALUE') -NEVER_SET = util.symbol('NEVER_SET') + :class:`.InstrumentedAttribute` -PASSIVE_RETURN_NEVER_SET = util.symbol('PASSIVE_RETURN_NEVER_SET', -"""Symbol indicating that loader callables can be -fired off, but if no callable is applicable and no value is -present, the attribute should remain non-initialized. -NEVER_SET is returned in this case. -""") + :class:`.MapperProperty` -PASSIVE_NO_INITIALIZE = util.symbol('PASSIVE_NO_INITIALIZE', -"""Symbol indicating that loader callables should - not be fired off, and a non-initialized attribute - should remain that way. -""") + :attr:`.Mapper.all_orm_descriptors` -PASSIVE_NO_FETCH = util.symbol('PASSIVE_NO_FETCH', -"""Symbol indicating that loader callables should not emit SQL, - but a value can be fetched from the current session. + :attr:`.Mapper.attrs` + """ - Non-initialized attributes should be initialized to an empty value. - -""") - -PASSIVE_NO_FETCH_RELATED = util.symbol('PASSIVE_NO_FETCH_RELATED', -"""Symbol indicating that loader callables should not emit SQL for - loading a related object, but can refresh the attributes of the local - instance in order to locate a related object in the current session. - - Non-initialized attributes should be initialized to an empty value. - - The unit of work uses this mode to check if history is present - on many-to-one attributes with minimal SQL emitted. - -""") - -PASSIVE_ONLY_PERSISTENT = util.symbol('PASSIVE_ONLY_PERSISTENT', -"""Symbol indicating that loader callables should only fire off for - parent objects which are persistent (i.e., have a database - identity). - - Load operations for the "previous" value of an attribute make - use of this flag during change events. - -""") - -PASSIVE_OFF = util.symbol('PASSIVE_OFF', -"""Symbol indicating that loader callables should be executed - normally. - -""") - - -class QueryableAttribute(interfaces.PropComparator): - """Base class for class-bound attributes. """ + is_attribute = True def __init__(self, class_, key, impl=None, - comparator=None, parententity=None): + comparator=None, parententity=None, + of_type=None): self.class_ = class_ self.key = key self.impl = impl self.comparator = comparator - self.parententity = parententity + self._parententity = parententity + self._of_type = of_type manager = manager_of_class(class_) # manager is None in the case of AliasedClass @@ -98,9 +69,6 @@ class QueryableAttribute(interfaces.PropComparator): if key in base: self.dispatch._update(base[key].dispatch) - dispatch = event.dispatcher(events.AttributeEvents) - dispatch.dispatch_cls._active_history = False - @util.memoized_property def _supports_population(self): return self.impl.supports_population @@ -113,11 +81,91 @@ class QueryableAttribute(interfaces.PropComparator): # TODO: conditionally attach this method based on clause_element ? return self + + @util.memoized_property + def info(self): + """Return the 'info' dictionary for the underlying SQL element. + + The behavior here is as follows: + + * If the attribute is a column-mapped property, i.e. + :class:`.ColumnProperty`, which is mapped directly + to a schema-level :class:`.Column` object, this attribute + will return the :attr:`.SchemaItem.info` dictionary associated + with the core-level :class:`.Column` object. + + * If the attribute is a :class:`.ColumnProperty` but is mapped to + any other kind of SQL expression other than a :class:`.Column`, + the attribute will refer to the :attr:`.MapperProperty.info` dictionary + associated directly with the :class:`.ColumnProperty`, assuming the SQL + expression itself does not have it's own ``.info`` attribute + (which should be the case, unless a user-defined SQL construct + has defined one). + + * If the attribute refers to any other kind of :class:`.MapperProperty`, + including :class:`.RelationshipProperty`, the attribute will refer + to the :attr:`.MapperProperty.info` dictionary associated with + that :class:`.MapperProperty`. + + * To access the :attr:`.MapperProperty.info` dictionary of the :class:`.MapperProperty` + unconditionally, including for a :class:`.ColumnProperty` that's + associated directly with a :class:`.schema.Column`, the attribute + can be referred to using :attr:`.QueryableAttribute.property` + attribute, as ``MyClass.someattribute.property.info``. + + .. versionadded:: 0.8.0 + + .. seealso:: + + :attr:`.SchemaItem.info` + + :attr:`.MapperProperty.info` + + """ + return self.comparator.info + + @util.memoized_property + def parent(self): + """Return an inspection instance representing the parent. + + This will be either an instance of :class:`.Mapper` + or :class:`.AliasedInsp`, depending upon the nature + of the parent entity which this attribute is associated + with. + + """ + return inspection.inspect(self._parententity) + + @property + def expression(self): + return self.comparator.__clause_element__() + def __clause_element__(self): return self.comparator.__clause_element__() + def _query_clause_element(self): + """like __clause_element__(), but called specifically + by :class:`.Query` to allow special behavior.""" + + return self.comparator._query_clause_element() + + def adapt_to_entity(self, adapt_to_entity): + assert not self._of_type + return self.__class__(adapt_to_entity.entity, self.key, impl=self.impl, + comparator=self.comparator.adapt_to_entity(adapt_to_entity), + parententity=adapt_to_entity) + + def of_type(self, cls): + return QueryableAttribute( + self.class_, + self.key, + self.impl, + self.comparator.of_type(cls), + self._parententity, + of_type=cls) + def label(self, name): - return self.__clause_element__().label(name) + return self._query_clause_element().label(name) def operate(self, op, *other, **kwargs): return op(self.comparator, *other, **kwargs) @@ -133,9 +181,11 @@ class QueryableAttribute(interfaces.PropComparator): return getattr(self.comparator, key) except AttributeError: raise AttributeError( - 'Neither %r object nor %r object has an attribute %r' % ( + 'Neither %r object nor %r object associated with %s ' + 'has an attribute %r' % ( type(self).__name__, type(self.comparator).__name__, + self, key) ) @@ -144,11 +194,26 @@ class QueryableAttribute(interfaces.PropComparator): @util.memoized_property def property(self): + """Return the :class:`.MapperProperty` associated with this + :class:`.QueryableAttribute`. + + + Return values here will commonly be instances of + :class:`.ColumnProperty` or :class:`.RelationshipProperty`. + + + """ return self.comparator.property class InstrumentedAttribute(QueryableAttribute): - """Class bound instrumented attribute which adds descriptor methods.""" + """Class bound instrumented attribute which adds basic + :term:`descriptor` methods. + + See :class:`.QueryableAttribute` for a description of most features. + + + """ def __set__(self, instance, value): self.impl.set(instance_state(instance), @@ -165,7 +230,8 @@ class InstrumentedAttribute(QueryableAttribute): if self._supports_population and self.key in dict_: return dict_[self.key] else: - return self.impl.get(instance_state(instance),dict_) + return self.impl.get(instance_state(instance), dict_) + def create_proxied_attribute(descriptor): """Create an QueryableAttribute / user descriptor hybrid. @@ -184,13 +250,16 @@ def create_proxied_attribute(descriptor): """ - def __init__(self, class_, key, descriptor, comparator, - adapter=None, doc=None): + def __init__(self, class_, key, descriptor, + comparator, + adapt_to_entity=None, doc=None, + original_property=None): self.class_ = class_ self.key = key self.descriptor = descriptor + self.original_property = original_property self._comparator = comparator - self.adapter = adapter + self._adapt_to_entity = adapt_to_entity self.__doc__ = doc @property @@ -201,16 +270,15 @@ def create_proxied_attribute(descriptor): def comparator(self): if util.callable(self._comparator): self._comparator = self._comparator() - if self.adapter: - self._comparator = self._comparator.adapted(self.adapter) + if self._adapt_to_entity: + self._comparator = self._comparator.adapt_to_entity( + self._adapt_to_entity) return self._comparator - def adapted(self, adapter): - """Proxy adapted() for the use case of AliasedClass calling adapted.""" - - return self.__class__(self.class_, self.key, self.descriptor, + def adapt_to_entity(self, adapt_to_entity): + return self.__class__(adapt_to_entity.entity, self.key, self.descriptor, self._comparator, - adapter) + adapt_to_entity) def __get__(self, instance, owner): if instance is None: @@ -219,7 +287,7 @@ def create_proxied_attribute(descriptor): return self.descriptor.__get__(instance, owner) def __str__(self): - return self.key + return "%s.%s" % (self.class_.__name__, self.key) def __getattr__(self, attribute): """Delegate __getattr__ to the original descriptor and/or @@ -232,9 +300,11 @@ def create_proxied_attribute(descriptor): return getattr(self.comparator, attribute) except AttributeError: raise AttributeError( - 'Neither %r object nor %r object has an attribute %r' % ( + 'Neither %r object nor %r object associated with %s ' + 'has an attribute %r' % ( type(descriptor).__name__, type(self.comparator).__name__, + self, attribute) ) @@ -245,6 +315,54 @@ def create_proxied_attribute(descriptor): from_instance=descriptor) return Proxy +OP_REMOVE = util.symbol("REMOVE") +OP_APPEND = util.symbol("APPEND") +OP_REPLACE = util.symbol("REPLACE") + +class Event(object): + """A token propagated throughout the course of a chain of attribute + events. + + Serves as an indicator of the source of the event and also provides + a means of controlling propagation across a chain of attribute + operations. + + The :class:`.Event` object is sent as the ``initiator`` argument + when dealing with the :meth:`.AttributeEvents.append`, + :meth:`.AttributeEvents.set`, + and :meth:`.AttributeEvents.remove` events. + + The :class:`.Event` object is currently interpreted by the backref + event handlers, and is used to control the propagation of operations + across two mutually-dependent attributes. + + .. versionadded:: 0.9.0 + + """ + + impl = None + """The :class:`.AttributeImpl` which is the current event initiator. + """ + + op = None + """The symbol :attr:`.OP_APPEND`, :attr:`.OP_REMOVE` or :attr:`.OP_REPLACE`, + indicating the source operation. + + """ + + def __init__(self, attribute_impl, op): + self.impl = attribute_impl + self.op = op + self.parent_token = self.impl.parent_token + + + @property + def key(self): + return self.impl.key + + def hasparent(self, state): + return self.impl.hasparent(state) + class AttributeImpl(object): """internal implementation for instrumented attributes.""" @@ -252,6 +370,7 @@ class AttributeImpl(object): callable_, dispatch, trackparent=False, extension=None, compare_function=None, active_history=False, parent_token=None, expire_missing=True, + send_modified_events=True, **kwargs): """Construct an AttributeImpl. @@ -295,6 +414,10 @@ class AttributeImpl(object): during state.expire_attributes(None), if no value is present for this key. + send_modified_events + if False, the InstanceState._modified_event method will have no effect; + this means the attribute will never show up as changed in a + history entry. """ self.class_ = class_ self.key = key @@ -302,6 +425,7 @@ class AttributeImpl(object): self.dispatch = dispatch self.trackparent = trackparent self.parent_token = parent_token or self + self.send_modified_events = send_modified_events if compare_function is None: self.is_equal = operator.eq else: @@ -319,6 +443,9 @@ class AttributeImpl(object): self.expire_missing = expire_missing + def __str__(self): + return "%s.%s" % (self.class_.__name__, self.key) + def _get_active_history(self): """Backwards compat for impl.active_history""" @@ -329,7 +456,6 @@ class AttributeImpl(object): active_history = property(_get_active_history, _set_active_history) - def hasparent(self, state, optimistic=False): """Return the boolean value of a `hasparent` flag attached to the given state. @@ -346,7 +472,8 @@ class AttributeImpl(object): will also not have a `hasparent` flag. """ - assert self.trackparent, "This AttributeImpl is not configured to track parents." + msg = "This AttributeImpl is not configured to track parents." + assert self.trackparent, msg return state.parents.get(id(self.parent_token), optimistic) \ is not False @@ -357,7 +484,8 @@ class AttributeImpl(object): attribute represented by this ``InstrumentedAttribute``. """ - assert self.trackparent, "This AttributeImpl is not configured to track parents." + msg = "This AttributeImpl is not configured to track parents." + assert self.trackparent, msg id_ = id(self.parent_token) if value: @@ -376,15 +504,14 @@ class AttributeImpl(object): "but the parent record " "has gone stale, can't be sure this " "is the most recent parent." % - (mapperutil.state_str(state), - mapperutil.state_str(parent_state), + (state_str(state), + state_str(parent_state), self.key)) return state.parents[id_] = False - def set_callable(self, state, callable_): """Set a callable function for this attribute on the given object. @@ -431,7 +558,6 @@ class AttributeImpl(object): def get(self, state, dict_, passive=PASSIVE_OFF): """Retrieve a value from the given object. - If a callable is assembled on this object's attribute, and passive is False, the callable will be executed and the resulting value will be set as the new value for this attribute. @@ -443,12 +569,12 @@ class AttributeImpl(object): key = self.key if key not in state.committed_state or \ state.committed_state[key] is NEVER_SET: - if passive is PASSIVE_NO_INITIALIZE: + if not passive & CALLABLES_OK: return PASSIVE_NO_RESULT if key in state.callables: callable_ = state.callables[key] - value = callable_(passive) + value = callable_(state, passive) elif self.callable_: value = self.callable_(state, passive) else: @@ -468,7 +594,7 @@ class AttributeImpl(object): elif value is not ATTR_EMPTY: return self.set_committed_value(state, dict_, value) - if passive is PASSIVE_RETURN_NEVER_SET: + if not passive & INIT_OK: return NEVER_SET else: # Return a new, empty value @@ -505,15 +631,17 @@ class AttributeImpl(object): """set an attribute value on the given instance and 'commit' it.""" dict_[self.key] = value - state.commit(dict_, [self.key]) + state._commit(dict_, [self.key]) return value + class ScalarAttributeImpl(AttributeImpl): """represents a scalar value-holding InstrumentedAttribute.""" accepts_scalar_loader = True uses_objects = False supports_population = True + collection = False def delete(self, state, dict_): @@ -524,19 +652,24 @@ class ScalarAttributeImpl(AttributeImpl): old = dict_.get(self.key, NO_VALUE) if self.dispatch.remove: - self.fire_remove_event(state, dict_, old, None) - state.modified_event(dict_, self, old) + self.fire_remove_event(state, dict_, old, self._remove_token) + state._modified_event(dict_, self, old) del dict_[self.key] def get_history(self, state, dict_, passive=PASSIVE_OFF): - return History.from_scalar_attribute( - self, state, dict_.get(self.key, NO_VALUE)) + if self.key in dict_: + return History.from_scalar_attribute(self, state, dict_[self.key]) + else: + if passive & INIT_OK: + passive ^= INIT_OK + current = self.get(state, dict_, passive=passive) + if current is PASSIVE_NO_RESULT: + return HISTORY_BLANK + else: + return History.from_scalar_attribute(self, state, current) def set(self, state, dict_, value, initiator, passive=PASSIVE_OFF, check_old=None, pop=False): - if initiator and initiator.parent_token is self.parent_token: - return - if self.dispatch._active_history: old = self.get(state, dict_, PASSIVE_RETURN_NEVER_SET) else: @@ -545,79 +678,35 @@ class ScalarAttributeImpl(AttributeImpl): if self.dispatch.set: value = self.fire_replace_event(state, dict_, value, old, initiator) - state.modified_event(dict_, self, old) + state._modified_event(dict_, self, old) dict_[self.key] = value + @util.memoized_property + def _replace_token(self): + return Event(self, OP_REPLACE) + + @util.memoized_property + def _append_token(self): + return Event(self, OP_REPLACE) + + @util.memoized_property + def _remove_token(self): + return Event(self, OP_REMOVE) + def fire_replace_event(self, state, dict_, value, previous, initiator): for fn in self.dispatch.set: - value = fn(state, value, previous, initiator or self) + value = fn(state, value, previous, initiator or self._replace_token) return value def fire_remove_event(self, state, dict_, value, initiator): for fn in self.dispatch.remove: - fn(state, value, initiator or self) + fn(state, value, initiator or self._remove_token) @property def type(self): self.property.columns[0].type -class MutableScalarAttributeImpl(ScalarAttributeImpl): - """represents a scalar value-holding InstrumentedAttribute, which can - detect changes within the value itself. - - """ - - uses_objects = False - supports_population = True - - def __init__(self, class_, key, callable_, dispatch, - class_manager, copy_function=None, - compare_function=None, **kwargs): - super(ScalarAttributeImpl, self).__init__( - class_, - key, - callable_, dispatch, - compare_function=compare_function, - **kwargs) - class_manager.mutable_attributes.add(key) - if copy_function is None: - raise sa_exc.ArgumentError( - "MutableScalarAttributeImpl requires a copy function") - self.copy = copy_function - - def get_history(self, state, dict_, passive=PASSIVE_OFF): - if not dict_: - v = state.committed_state.get(self.key, NO_VALUE) - else: - v = dict_.get(self.key, NO_VALUE) - - return History.from_scalar_attribute(self, state, v) - - def check_mutable_modified(self, state, dict_): - a, u, d = self.get_history(state, dict_) - return bool(a or d) - - def get(self, state, dict_, passive=PASSIVE_OFF): - if self.key not in state.mutable_dict: - ret = ScalarAttributeImpl.get(self, state, dict_, passive=passive) - if ret is not PASSIVE_NO_RESULT: - state.mutable_dict[self.key] = ret - return ret - else: - return state.mutable_dict[self.key] - - def delete(self, state, dict_): - ScalarAttributeImpl.delete(self, state, dict_) - state.mutable_dict.pop(self.key) - - def set(self, state, dict_, value, initiator, - passive=PASSIVE_OFF, check_old=None, pop=False): - ScalarAttributeImpl.set(self, state, dict_, value, - initiator, passive, check_old=check_old, pop=pop) - state.mutable_dict[self.key] = value - - class ScalarObjectAttributeImpl(ScalarAttributeImpl): """represents a scalar-holding InstrumentedAttribute, where the target object is also instrumented. @@ -629,18 +718,19 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): accepts_scalar_loader = False uses_objects = True supports_population = True + collection = False def delete(self, state, dict_): old = self.get(state, dict_) - self.fire_remove_event(state, dict_, old, self) + self.fire_remove_event(state, dict_, old, self._remove_token) del dict_[self.key] def get_history(self, state, dict_, passive=PASSIVE_OFF): if self.key in dict_: return History.from_object_attribute(self, state, dict_[self.key]) else: - if passive is PASSIVE_OFF: - passive = PASSIVE_RETURN_NEVER_SET + if passive & INIT_OK: + passive ^= INIT_OK current = self.get(state, dict_, passive=passive) if current is PASSIVE_NO_RESULT: return HISTORY_BLANK @@ -669,14 +759,7 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): passive=PASSIVE_OFF, check_old=None, pop=False): """Set a value on the given InstanceState. - `initiator` is the ``InstrumentedAttribute`` that initiated the - ``set()`` operation and is used to control the depth of a circular - setter operation. - """ - if initiator and initiator.parent_token is self.parent_token: - return - if self.dispatch._active_history: old = self.get(state, dict_, passive=PASSIVE_ONLY_PERSISTENT) else: @@ -690,21 +773,22 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): else: raise ValueError( "Object %s not associated with %s on attribute '%s'" % ( - mapperutil.instance_str(check_old), - mapperutil.state_str(state), + instance_str(check_old), + state_str(state), self.key )) value = self.fire_replace_event(state, dict_, value, old, initiator) dict_[self.key] = value + def fire_remove_event(self, state, dict_, value, initiator): if self.trackparent and value is not None: self.sethasparent(instance_state(value), state, False) for fn in self.dispatch.remove: - fn(state, value, initiator or self) + fn(state, value, initiator or self._remove_token) - state.modified_event(dict_, self, value) + state._modified_event(dict_, self, value) def fire_replace_event(self, state, dict_, value, previous, initiator): if self.trackparent: @@ -714,9 +798,9 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl): self.sethasparent(instance_state(previous), state, False) for fn in self.dispatch.set: - value = fn(state, value, previous, initiator or self) + value = fn(state, value, previous, initiator or self._replace_token) - state.modified_event(dict_, self, previous) + state._modified_event(dict_, self, previous) if self.trackparent: if value is not None: @@ -739,6 +823,7 @@ class CollectionAttributeImpl(AttributeImpl): accepts_scalar_loader = False uses_objects = True supports_population = True + collection = True def __init__(self, class_, key, callable_, dispatch, typecallable=None, trackparent=False, extension=None, @@ -758,7 +843,7 @@ class CollectionAttributeImpl(AttributeImpl): self.collection_factory = typecallable def __copy(self, item): - return [y for y in list(collections.collection_adapter(item))] + return [y for y in collections.collection_adapter(item)] def get_history(self, state, dict_, passive=PASSIVE_OFF): current = self.get(state, dict_, passive=passive) @@ -776,7 +861,7 @@ class CollectionAttributeImpl(AttributeImpl): if self.key in state.committed_state: original = state.committed_state[self.key] - if original is not NO_VALUE: + if original not in (NO_VALUE, NEVER_SET): current_states = [((c is not None) and instance_state(c) or None, c) for c in current] @@ -788,18 +873,28 @@ class CollectionAttributeImpl(AttributeImpl): original_set = dict(original_states) return \ - [(s, o) for s, o in current_states if s not in original_set] + \ - [(s, o) for s, o in current_states if s in original_set] + \ - [(s, o) for s, o in original_states if s not in current_set] + [(s, o) for s, o in current_states + if s not in original_set] + \ + [(s, o) for s, o in current_states + if s in original_set] + \ + [(s, o) for s, o in original_states + if s not in current_set] return [(instance_state(o), o) for o in current] + @util.memoized_property + def _append_token(self): + return Event(self, OP_APPEND) + + @util.memoized_property + def _remove_token(self): + return Event(self, OP_REMOVE) def fire_append_event(self, state, dict_, value, initiator): for fn in self.dispatch.append: - value = fn(state, value, initiator or self) + value = fn(state, value, initiator or self._append_token) - state.modified_event(dict_, self, NEVER_SET, True) + state._modified_event(dict_, self, NEVER_SET, True) if self.trackparent and value is not None: self.sethasparent(instance_state(value), state, True) @@ -807,22 +902,22 @@ class CollectionAttributeImpl(AttributeImpl): return value def fire_pre_remove_event(self, state, dict_, initiator): - state.modified_event(dict_, self, NEVER_SET, True) + state._modified_event(dict_, self, NEVER_SET, True) def fire_remove_event(self, state, dict_, value, initiator): if self.trackparent and value is not None: self.sethasparent(instance_state(value), state, False) for fn in self.dispatch.remove: - fn(state, value, initiator or self) + fn(state, value, initiator or self._remove_token) - state.modified_event(dict_, self, NEVER_SET, True) + state._modified_event(dict_, self, NEVER_SET, True) def delete(self, state, dict_): if self.key not in dict_: return - state.modified_event(dict_, self, NEVER_SET, True) + state._modified_event(dict_, self, NEVER_SET, True) collection = self.get_collection(state, state.dict) collection.clear_with_event() @@ -841,28 +936,22 @@ class CollectionAttributeImpl(AttributeImpl): self.key, state, self.collection_factory) def append(self, state, dict_, value, initiator, passive=PASSIVE_OFF): - if initiator and initiator.parent_token is self.parent_token: - return - collection = self.get_collection(state, dict_, passive=passive) if collection is PASSIVE_NO_RESULT: value = self.fire_append_event(state, dict_, value, initiator) assert self.key not in dict_, \ "Collection was loaded during event handling." - state.get_pending(self.key).append(value) + state._get_pending_mutation(self.key).append(value) else: collection.append_with_event(value, initiator) def remove(self, state, dict_, value, initiator, passive=PASSIVE_OFF): - if initiator and initiator.parent_token is self.parent_token: - return - collection = self.get_collection(state, state.dict, passive=passive) if collection is PASSIVE_NO_RESULT: self.fire_remove_event(state, dict_, value, initiator) assert self.key not in dict_, \ "Collection was loaded during event handling." - state.get_pending(self.key).remove(value) + state._get_pending_mutation(self.key).remove(value) else: collection.remove_with_event(value, initiator) @@ -879,14 +968,8 @@ class CollectionAttributeImpl(AttributeImpl): passive=PASSIVE_OFF, pop=False): """Set a value on the given object. - `initiator` is the ``InstrumentedAttribute`` that initiated the - ``set()`` operation and is used to control the depth of a circular - setter operation. """ - if initiator and initiator.parent_token is self.parent_token: - return - self._set_iterable( state, dict_, value, lambda adapter, i: adapter.adapt_like_to_iterable(i)) @@ -918,7 +1001,7 @@ class CollectionAttributeImpl(AttributeImpl): return # place a copy of "old" in state.committed_state - state.modified_event(dict_, self, old, True) + state._modified_event(dict_, self, old, True) old_collection = getattr(old, '_sa_adapter') @@ -927,6 +1010,10 @@ class CollectionAttributeImpl(AttributeImpl): collections.bulk_replace(new_values, old_collection, new_collection) old_collection.unlink(old) + def _invalidate_collection(self, collection): + adapter = getattr(collection, '_sa_adapter') + adapter.invalidated = True + def set_committed_value(self, state, dict_, value): """Set an attribute value on the given instance and 'commit' it.""" @@ -937,14 +1024,14 @@ class CollectionAttributeImpl(AttributeImpl): state.dict[self.key] = user_data - state.commit(dict_, [self.key]) + state._commit(dict_, [self.key]) - if self.key in state.pending: + if self.key in state._pending_mutations: # pending items exist. issue a modified event, # add/remove new items. - state.modified_event(dict_, self, user_data, True) + state._modified_event(dict_, self, user_data, True) - pending = state.pending.pop(self.key) + pending = state._pending_mutations.pop(self.key) added = pending.added_items removed = pending.deleted_items for item in added: @@ -968,57 +1055,77 @@ class CollectionAttributeImpl(AttributeImpl): return getattr(user_data, '_sa_adapter') + def backref_listeners(attribute, key, uselist): """Apply listeners to synchronize a two-way relationship.""" # use easily recognizable names for stack traces parent_token = attribute.impl.parent_token + parent_impl = attribute.impl - def _acceptable_key_err(child_state, initiator): + def _acceptable_key_err(child_state, initiator, child_impl): raise ValueError( - "Object %s not associated with attribute of " - "type %s" % (mapperutil.state_str(child_state), - manager_of_class(initiator.class_)[initiator.key])) + "Bidirectional attribute conflict detected: " + 'Passing object %s to attribute "%s" ' + 'triggers a modify event on attribute "%s" ' + 'via the backref "%s".' % ( + state_str(child_state), + initiator.parent_token, + child_impl.parent_token, + attribute.impl.parent_token + ) + ) def emit_backref_from_scalar_set_event(state, child, oldchild, initiator): if oldchild is child: return child - if oldchild is not None and oldchild is not PASSIVE_NO_RESULT: # With lazy=None, there's no guarantee that the full collection is # present when updating via a backref. old_state, old_dict = instance_state(oldchild),\ instance_dict(oldchild) impl = old_state.manager[key].impl - impl.pop(old_state, - old_dict, - state.obj(), - initiator, passive=PASSIVE_NO_FETCH) + + if initiator.impl is not impl or \ + initiator.op not in (OP_REPLACE, OP_REMOVE): + impl.pop(old_state, + old_dict, + state.obj(), + parent_impl._append_token, + passive=PASSIVE_NO_FETCH) if child is not None: child_state, child_dict = instance_state(child),\ instance_dict(child) child_impl = child_state.manager[key].impl if initiator.parent_token is not parent_token and \ - initiator.parent_token is not child_impl.parent_token: - _acceptable_key_err(state, initiator) - child_impl.append( - child_state, - child_dict, - state.obj(), - initiator, - passive=PASSIVE_NO_FETCH) + initiator.parent_token is not child_impl.parent_token: + _acceptable_key_err(state, initiator, child_impl) + elif initiator.impl is not child_impl or \ + initiator.op not in (OP_APPEND, OP_REPLACE): + child_impl.append( + child_state, + child_dict, + state.obj(), + initiator, + passive=PASSIVE_NO_FETCH) return child def emit_backref_from_collection_append_event(state, child, initiator): + if child is None: + return + child_state, child_dict = instance_state(child), \ instance_dict(child) child_impl = child_state.manager[key].impl + if initiator.parent_token is not parent_token and \ - initiator.parent_token is not child_impl.parent_token: - _acceptable_key_err(state, initiator) - child_impl.append( + initiator.parent_token is not child_impl.parent_token: + _acceptable_key_err(state, initiator, child_impl) + elif initiator.impl is not child_impl or \ + initiator.op not in (OP_APPEND, OP_REPLACE): + child_impl.append( child_state, child_dict, state.obj(), @@ -1031,10 +1138,9 @@ def backref_listeners(attribute, key, uselist): child_state, child_dict = instance_state(child),\ instance_dict(child) child_impl = child_state.manager[key].impl - # can't think of a path that would produce an initiator - # mismatch here, as it would require an existing collection - # mismatch. - child_impl.pop( + if initiator.impl is not child_impl or \ + initiator.op not in (OP_REMOVE, OP_REPLACE): + child_impl.pop( child_state, child_dict, state.obj(), @@ -1059,35 +1165,40 @@ _NO_STATE_SYMBOLS = frozenset([ id(PASSIVE_NO_RESULT), id(NO_VALUE), id(NEVER_SET)]) -class History(tuple): + +History = util.namedtuple("History", [ + "added", "unchanged", "deleted" + ]) + + +class History(History): """A 3-tuple of added, unchanged and deleted values, representing the changes which have occurred on an instrumented attribute. - Each tuple member is an iterable sequence. + The easiest way to get a :class:`.History` object for a particular + attribute on an object is to use the :func:`.inspect` function:: + + from sqlalchemy import inspect + + hist = inspect(myobject).attrs.myattribute.history + + Each tuple member is an iterable sequence: + + * ``added`` - the collection of items added to the attribute (the first + tuple element). + + * ``unchanged`` - the collection of items that have not changed on the + attribute (the second tuple element). + + * ``deleted`` - the collection of items that have been removed from the + attribute (the third tuple element). """ - __slots__ = () - - added = property(itemgetter(0)) - """Return the collection of items added to the attribute (the first tuple - element).""" - - unchanged = property(itemgetter(1)) - """Return the collection of items that have not changed on the attribute - (the second tuple element).""" - - - deleted = property(itemgetter(2)) - """Return the collection of items that have been removed from the - attribute (the third tuple element).""" - - def __new__(cls, added, unchanged, deleted): - return tuple.__new__(cls, (added, unchanged, deleted)) - - def __nonzero__(self): + def __bool__(self): return self != HISTORY_BLANK + __nonzero__ = __bool__ def empty(self): """Return True if this :class:`.History` has no changes @@ -1142,7 +1253,7 @@ class History(tuple): original = state.committed_state.get(attribute.key, _NO_HISTORY) if original is _NO_HISTORY: - if current is NO_VALUE: + if current is NEVER_SET: return cls((), (), ()) else: return cls((), [current], ()) @@ -1159,7 +1270,7 @@ class History(tuple): deleted = () else: deleted = [original] - if current is NO_VALUE: + if current is NEVER_SET: return cls((), (), deleted) else: return cls([current], (), deleted) @@ -1193,18 +1304,23 @@ class History(tuple): @classmethod def from_collection(cls, attribute, state, current): original = state.committed_state.get(attribute.key, _NO_HISTORY) - current = getattr(current, '_sa_adapter') - if original is NO_VALUE: + if current is NO_VALUE or current is NEVER_SET: + return cls((), (), ()) + + current = getattr(current, '_sa_adapter') + if original in (NO_VALUE, NEVER_SET): return cls(list(current), (), ()) elif original is _NO_HISTORY: return cls((), list(current), ()) else: - current_states = [((c is not None) and instance_state(c) or None, c) + current_states = [((c is not None) and instance_state(c) + or None, c) for c in current ] - original_states = [((c is not None) and instance_state(c) or None, c) + original_states = [((c is not None) and instance_state(c) + or None, c) for c in original ] @@ -1219,6 +1335,7 @@ class History(tuple): HISTORY_BLANK = History(None, None, None) + def get_history(obj, key, passive=PASSIVE_OFF): """Return a :class:`.History` record for the given object and attribute key. @@ -1228,10 +1345,11 @@ def get_history(obj, key, passive=PASSIVE_OFF): :param key: string attribute name. - :param passive: indicates if the attribute should be - loaded from the database if not already present (:attr:`.PASSIVE_NO_FETCH`), and - if the attribute should be not initialized to a blank value otherwise - (:attr:`.PASSIVE_NO_INITIALIZE`). Default is :attr:`PASSIVE_OFF`. + :param passive: indicates loading behavior for the attribute + if the value is not already present. This is a + bitflag attribute, which defaults to the symbol + :attr:`.PASSIVE_OFF` indicating all necessary SQL + should be emitted. """ if passive is True: @@ -1245,6 +1363,7 @@ def get_history(obj, key, passive=PASSIVE_OFF): return get_state_history(instance_state(obj), key, passive) + def get_state_history(state, key, passive=PASSIVE_OFF): return state.get_history(key, passive) @@ -1255,6 +1374,7 @@ def has_parent(cls, obj, key, optimistic=False): state = instance_state(obj) return manager.has_parent(state, key, optimistic) + def register_attribute(class_, key, **kw): comparator = kw.pop('comparator', None) parententity = kw.pop('parententity', None) @@ -1264,9 +1384,10 @@ def register_attribute(class_, key, **kw): register_attribute_impl(class_, key, **kw) return desc + def register_attribute_impl(class_, key, uselist=False, callable_=None, - useobject=False, mutable_scalars=False, + useobject=False, impl_class=None, backref=None, **kw): manager = manager_of_class(class_) @@ -1286,10 +1407,7 @@ def register_attribute_impl(class_, key, typecallable=typecallable, **kw) elif useobject: impl = ScalarObjectAttributeImpl(class_, key, callable_, - dispatch,**kw) - elif mutable_scalars: - impl = MutableScalarAttributeImpl(class_, key, callable_, dispatch, - class_manager=manager, **kw) + dispatch, **kw) else: impl = ScalarAttributeImpl(class_, key, callable_, dispatch, **kw) @@ -1301,6 +1419,7 @@ def register_attribute_impl(class_, key, manager.post_configure_attribute(key) return manager[key] + def register_descriptor(class_, key, comparator=None, parententity=None, doc=None): manager = manager_of_class(class_) @@ -1313,9 +1432,11 @@ def register_descriptor(class_, key, comparator=None, manager.instrument_attribute(key, descriptor) return descriptor + def unregister_attribute(class_, key): manager_of_class(class_).uninstrument_attribute(key) + def init_collection(obj, key): """Initialize a collection attribute and return the collection adapter. @@ -1327,7 +1448,7 @@ def init_collection(obj, key): collection_adapter.append_without_event(elem) For an easier way to do the above, see - :func:`~sqlalchemy.orm.attributes.set_committed_value`. + :func:`~sqlalchemy.orm.attributes.set_committed_value`. obj is an instrumented object instance. An InstanceState is accepted directly for backwards compatibility but @@ -1338,6 +1459,7 @@ def init_collection(obj, key): dict_ = state.dict return init_state_collection(state, dict_, key) + def init_state_collection(state, dict_, key): """Initialize a collection attribute and return the collection adapter.""" @@ -1345,6 +1467,7 @@ def init_state_collection(state, dict_, key): user_data = attr.initialize(state, dict_) return attr.get_collection(state, dict_, user_data) + def set_committed_value(instance, key, value): """Set the value of an attribute with no history events. @@ -1363,6 +1486,7 @@ def set_committed_value(instance, key, value): state, dict_ = instance_state(instance), instance_dict(instance) state.manager[key].impl.set_committed_value(state, dict_, value) + def set_attribute(instance, key, value): """Set the value of an attribute, firing history events. @@ -1376,6 +1500,7 @@ def set_attribute(instance, key, value): state, dict_ = instance_state(instance), instance_dict(instance) state.manager[key].impl.set(state, dict_, value, None) + def get_attribute(instance, key): """Get the value of an attribute, firing any callables required. @@ -1389,6 +1514,7 @@ def get_attribute(instance, key): state, dict_ = instance_state(instance), instance_dict(instance) return state.manager[key].impl.get(state, dict_) + def del_attribute(instance, key): """Delete the value of an attribute, firing history events. @@ -1402,6 +1528,7 @@ def del_attribute(instance, key): state, dict_ = instance_state(instance), instance_dict(instance) state.manager[key].impl.delete(state, dict_) + def flag_modified(instance, key): """Mark an attribute on an instance as 'modified'. @@ -1411,5 +1538,4 @@ def flag_modified(instance, key): """ state, dict_ = instance_state(instance), instance_dict(instance) impl = state.manager[key].impl - state.modified_event(dict_, impl, NO_VALUE) - + state._modified_event(dict_, impl, NO_VALUE) diff --git a/libs/sqlalchemy/orm/base.py b/libs/sqlalchemy/orm/base.py new file mode 100644 index 00000000..e3b75eda --- /dev/null +++ b/libs/sqlalchemy/orm/base.py @@ -0,0 +1,453 @@ +# orm/base.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Constants and rudimental functions used throughout the ORM. + +""" + +from .. import util, inspection, exc as sa_exc +from ..sql import expression +from . import exc +import operator + +PASSIVE_NO_RESULT = util.symbol('PASSIVE_NO_RESULT', +"""Symbol returned by a loader callable or other attribute/history +retrieval operation when a value could not be determined, based +on loader callable flags. +""" +) + +ATTR_WAS_SET = util.symbol('ATTR_WAS_SET', +"""Symbol returned by a loader callable to indicate the +retrieved value, or values, were assigned to their attributes +on the target object. +""") + +ATTR_EMPTY = util.symbol('ATTR_EMPTY', +"""Symbol used internally to indicate an attribute had no callable. +""") + +NO_VALUE = util.symbol('NO_VALUE', +"""Symbol which may be placed as the 'previous' value of an attribute, +indicating no value was loaded for an attribute when it was modified, +and flags indicated we were not to load it. +""" +) + +NEVER_SET = util.symbol('NEVER_SET', +"""Symbol which may be placed as the 'previous' value of an attribute +indicating that the attribute had not been assigned to previously. +""" +) + +NO_CHANGE = util.symbol("NO_CHANGE", +"""No callables or SQL should be emitted on attribute access +and no state should change""", canonical=0 +) + +CALLABLES_OK = util.symbol("CALLABLES_OK", +"""Loader callables can be fired off if a value +is not present.""", canonical=1 +) + +SQL_OK = util.symbol("SQL_OK", +"""Loader callables can emit SQL at least on scalar value +attributes.""", canonical=2) + +RELATED_OBJECT_OK = util.symbol("RELATED_OBJECT_OK", +"""callables can use SQL to load related objects as well +as scalar value attributes. +""", canonical=4 +) + +INIT_OK = util.symbol("INIT_OK", +"""Attributes should be initialized with a blank +value (None or an empty collection) upon get, if no other +value can be obtained. +""", canonical=8 +) + +NON_PERSISTENT_OK = util.symbol("NON_PERSISTENT_OK", +"""callables can be emitted if the parent is not persistent.""", +canonical=16 +) + +LOAD_AGAINST_COMMITTED = util.symbol("LOAD_AGAINST_COMMITTED", +"""callables should use committed values as primary/foreign keys during a load +""", canonical=32 +) + +# pre-packaged sets of flags used as inputs +PASSIVE_OFF = util.symbol("PASSIVE_OFF", + "Callables can be emitted in all cases.", + canonical=(RELATED_OBJECT_OK | NON_PERSISTENT_OK | + INIT_OK | CALLABLES_OK | SQL_OK) +) +PASSIVE_RETURN_NEVER_SET = util.symbol("PASSIVE_RETURN_NEVER_SET", + """PASSIVE_OFF ^ INIT_OK""", + canonical=PASSIVE_OFF ^ INIT_OK +) +PASSIVE_NO_INITIALIZE = util.symbol("PASSIVE_NO_INITIALIZE", + "PASSIVE_RETURN_NEVER_SET ^ CALLABLES_OK", + canonical=PASSIVE_RETURN_NEVER_SET ^ CALLABLES_OK +) +PASSIVE_NO_FETCH = util.symbol("PASSIVE_NO_FETCH", + "PASSIVE_OFF ^ SQL_OK", + canonical=PASSIVE_OFF ^ SQL_OK +) +PASSIVE_NO_FETCH_RELATED = util.symbol("PASSIVE_NO_FETCH_RELATED", + "PASSIVE_OFF ^ RELATED_OBJECT_OK", + canonical=PASSIVE_OFF ^ RELATED_OBJECT_OK +) +PASSIVE_ONLY_PERSISTENT = util.symbol("PASSIVE_ONLY_PERSISTENT", + "PASSIVE_OFF ^ NON_PERSISTENT_OK", + canonical=PASSIVE_OFF ^ NON_PERSISTENT_OK +) + +DEFAULT_MANAGER_ATTR = '_sa_class_manager' +DEFAULT_STATE_ATTR = '_sa_instance_state' +_INSTRUMENTOR = ('mapper', 'instrumentor') + +EXT_CONTINUE = util.symbol('EXT_CONTINUE') +EXT_STOP = util.symbol('EXT_STOP') + +ONETOMANY = util.symbol('ONETOMANY', +"""Indicates the one-to-many direction for a :func:`.relationship`. + +This symbol is typically used by the internals but may be exposed within +certain API features. + +""") + +MANYTOONE = util.symbol('MANYTOONE', +"""Indicates the many-to-one direction for a :func:`.relationship`. + +This symbol is typically used by the internals but may be exposed within +certain API features. + +""") + +MANYTOMANY = util.symbol('MANYTOMANY', +"""Indicates the many-to-many direction for a :func:`.relationship`. + +This symbol is typically used by the internals but may be exposed within +certain API features. + +""") + +NOT_EXTENSION = util.symbol('NOT_EXTENSION', +"""Symbol indicating an :class:`_InspectionAttr` that's + not part of sqlalchemy.ext. + + Is assigned to the :attr:`._InspectionAttr.extension_type` + attibute. + +""") + +_none_set = frozenset([None]) + + +def _generative(*assertions): + """Mark a method as generative, e.g. method-chained.""" + + @util.decorator + def generate(fn, *args, **kw): + self = args[0]._clone() + for assertion in assertions: + assertion(self, fn.__name__) + fn(self, *args[1:], **kw) + return self + return generate + + +# these can be replaced by sqlalchemy.ext.instrumentation +# if augmented class instrumentation is enabled. +def manager_of_class(cls): + return cls.__dict__.get(DEFAULT_MANAGER_ATTR, None) + +instance_state = operator.attrgetter(DEFAULT_STATE_ATTR) + +instance_dict = operator.attrgetter('__dict__') + +def instance_str(instance): + """Return a string describing an instance.""" + + return state_str(instance_state(instance)) + +def state_str(state): + """Return a string describing an instance via its InstanceState.""" + + if state is None: + return "None" + else: + return '<%s at 0x%x>' % (state.class_.__name__, id(state.obj())) + +def state_class_str(state): + """Return a string describing an instance's class via its InstanceState.""" + + if state is None: + return "None" + else: + return '<%s>' % (state.class_.__name__, ) + + +def attribute_str(instance, attribute): + return instance_str(instance) + "." + attribute + + +def state_attribute_str(state, attribute): + return state_str(state) + "." + attribute + +def object_mapper(instance): + """Given an object, return the primary Mapper associated with the object + instance. + + Raises :class:`sqlalchemy.orm.exc.UnmappedInstanceError` + if no mapping is configured. + + This function is available via the inspection system as:: + + inspect(instance).mapper + + Using the inspection system will raise + :class:`sqlalchemy.exc.NoInspectionAvailable` if the instance is + not part of a mapping. + + """ + return object_state(instance).mapper + + +def object_state(instance): + """Given an object, return the :class:`.InstanceState` + associated with the object. + + Raises :class:`sqlalchemy.orm.exc.UnmappedInstanceError` + if no mapping is configured. + + Equivalent functionality is available via the :func:`.inspect` + function as:: + + inspect(instance) + + Using the inspection system will raise + :class:`sqlalchemy.exc.NoInspectionAvailable` if the instance is + not part of a mapping. + + """ + state = _inspect_mapped_object(instance) + if state is None: + raise exc.UnmappedInstanceError(instance) + else: + return state + + +@inspection._inspects(object) +def _inspect_mapped_object(instance): + try: + return instance_state(instance) + # TODO: whats the py-2/3 syntax to catch two + # different kinds of exceptions at once ? + except exc.UnmappedClassError: + return None + except exc.NO_STATE: + return None + + + +def _class_to_mapper(class_or_mapper): + insp = inspection.inspect(class_or_mapper, False) + if insp is not None: + return insp.mapper + else: + raise exc.UnmappedClassError(class_or_mapper) + + +def _mapper_or_none(entity): + """Return the :class:`.Mapper` for the given class or None if the + class is not mapped.""" + + insp = inspection.inspect(entity, False) + if insp is not None: + return insp.mapper + else: + return None + + +def _is_mapped_class(entity): + """Return True if the given object is a mapped class, + :class:`.Mapper`, or :class:`.AliasedClass`.""" + + insp = inspection.inspect(entity, False) + return insp is not None and \ + hasattr(insp, "mapper") and \ + ( + insp.is_mapper + or insp.is_aliased_class + ) + +def _attr_as_key(attr): + if hasattr(attr, 'key'): + return attr.key + else: + return expression._column_as_key(attr) + + + +def _orm_columns(entity): + insp = inspection.inspect(entity, False) + if hasattr(insp, 'selectable'): + return [c for c in insp.selectable.c] + else: + return [entity] + + + +def _is_aliased_class(entity): + insp = inspection.inspect(entity, False) + return insp is not None and \ + getattr(insp, "is_aliased_class", False) + + +def _entity_descriptor(entity, key): + """Return a class attribute given an entity and string name. + + May return :class:`.InstrumentedAttribute` or user-defined + attribute. + + """ + insp = inspection.inspect(entity) + if insp.is_selectable: + description = entity + entity = insp.c + elif insp.is_aliased_class: + entity = insp.entity + description = entity + elif hasattr(insp, "mapper"): + description = entity = insp.mapper.class_ + else: + description = entity + + try: + return getattr(entity, key) + except AttributeError: + raise sa_exc.InvalidRequestError( + "Entity '%s' has no property '%s'" % + (description, key) + ) + +_state_mapper = util.dottedgetter('manager.mapper') + +@inspection._inspects(type) +def _inspect_mapped_class(class_, configure=False): + try: + class_manager = manager_of_class(class_) + if not class_manager.is_mapped: + return None + mapper = class_manager.mapper + if configure and mapper._new_mappers: + mapper._configure_all() + return mapper + + except exc.NO_STATE: + return None + +def class_mapper(class_, configure=True): + """Given a class, return the primary :class:`.Mapper` associated + with the key. + + Raises :exc:`.UnmappedClassError` if no mapping is configured + on the given class, or :exc:`.ArgumentError` if a non-class + object is passed. + + Equivalent functionality is available via the :func:`.inspect` + function as:: + + inspect(some_mapped_class) + + Using the inspection system will raise + :class:`sqlalchemy.exc.NoInspectionAvailable` if the class is not mapped. + + """ + mapper = _inspect_mapped_class(class_, configure=configure) + if mapper is None: + if not isinstance(class_, type): + raise sa_exc.ArgumentError( + "Class object expected, got '%r'." % class_) + raise exc.UnmappedClassError(class_) + else: + return mapper + + +class _InspectionAttr(object): + """A base class applied to all ORM objects that can be returned + by the :func:`.inspect` function. + + The attributes defined here allow the usage of simple boolean + checks to test basic facts about the object returned. + + While the boolean checks here are basically the same as using + the Python isinstance() function, the flags here can be used without + the need to import all of these classes, and also such that + the SQLAlchemy class system can change while leaving the flags + here intact for forwards-compatibility. + + """ + + is_selectable = False + """Return True if this object is an instance of :class:`.Selectable`.""" + + is_aliased_class = False + """True if this object is an instance of :class:`.AliasedClass`.""" + + is_instance = False + """True if this object is an instance of :class:`.InstanceState`.""" + + is_mapper = False + """True if this object is an instance of :class:`.Mapper`.""" + + is_property = False + """True if this object is an instance of :class:`.MapperProperty`.""" + + is_attribute = False + """True if this object is a Python :term:`descriptor`. + + This can refer to one of many types. Usually a + :class:`.QueryableAttribute` which handles attributes events on behalf + of a :class:`.MapperProperty`. But can also be an extension type + such as :class:`.AssociationProxy` or :class:`.hybrid_property`. + The :attr:`._InspectionAttr.extension_type` will refer to a constant + identifying the specific subtype. + + .. seealso:: + + :attr:`.Mapper.all_orm_descriptors` + + """ + + is_clause_element = False + """True if this object is an instance of :class:`.ClauseElement`.""" + + extension_type = NOT_EXTENSION + """The extension type, if any. + Defaults to :data:`.interfaces.NOT_EXTENSION` + + .. versionadded:: 0.8.0 + + .. seealso:: + + :data:`.HYBRID_METHOD` + + :data:`.HYBRID_PROPERTY` + + :data:`.ASSOCIATION_PROXY` + + """ + +class _MappedAttribute(object): + """Mixin for attributes which should be replaced by mapper-assigned + attributes. + + """ diff --git a/libs/sqlalchemy/orm/collections.py b/libs/sqlalchemy/orm/collections.py index e26a5973..87e351b6 100644 --- a/libs/sqlalchemy/orm/collections.py +++ b/libs/sqlalchemy/orm/collections.py @@ -1,5 +1,5 @@ # orm/collections.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -12,7 +12,7 @@ used, allowing arbitrary types (including built-ins) to be used as entity collections without requiring inheritance from a base class. Instrumentation decoration relays membership change events to the -``InstrumentedCollectionAttribute`` that is currently managing the collection. +:class:`.CollectionAttributeImpl` that is currently managing the collection. The decorators observe function call arguments and return values, tracking entities entering or leaving the collection. Two decorator approaches are provided. One is a bundle of generic decorators that map function arguments @@ -97,19 +97,19 @@ instrumentation may be the answer. Within your method, ``collection_adapter(self)`` will retrieve an object that you can use for explicit control over triggering append and remove events. -The owning object and InstrumentedCollectionAttribute are also reachable +The owning object and :class:`.CollectionAttributeImpl` are also reachable through the adapter, allowing for some very sophisticated behavior. """ -import copy import inspect import operator -import sys import weakref -from sqlalchemy.sql import expression -from sqlalchemy import schema, util, exc as sa_exc +from ..sql import expression +from .. import util, exc as sa_exc +from . import base + __all__ = ['collection', 'collection_adapter', 'mapped_collection', 'column_mapped_collection', @@ -138,8 +138,8 @@ class _PlainColumnGetter(object): return self.cols def __call__(self, value): - state = instance_state(value) - m = _state_mapper(state) + state = base.instance_state(value) + m = base._state_mapper(state) key = [ m._get_state_attr_by_column(state, state.dict, col) @@ -151,6 +151,7 @@ class _PlainColumnGetter(object): else: return key[0] + class _SerializableColumnGetter(object): """Column-based getter used in version 0.7.6 only. @@ -160,11 +161,13 @@ class _SerializableColumnGetter(object): def __init__(self, colkeys): self.colkeys = colkeys self.composite = len(colkeys) > 1 + def __reduce__(self): return _SerializableColumnGetter, (self.colkeys,) + def __call__(self, value): - state = instance_state(value) - m = _state_mapper(state) + state = base.instance_state(value) + m = base._state_mapper(state) key = [m._get_state_attr_by_column( state, state.dict, m.mapped_table.columns[k]) @@ -174,6 +177,7 @@ class _SerializableColumnGetter(object): else: return key[0] + class _SerializableColumnGetterV2(_PlainColumnGetter): """Updated serializable getter which deals with multi-table mapped classes. @@ -219,8 +223,9 @@ class _SerializableColumnGetterV2(_PlainColumnGetter): def column_mapped_collection(mapping_spec): """A dictionary-based collection type with column-based keying. - Returns a :class:`.MappedCollection` factory with a keying function generated - from mapping_spec, which may be a Column or a sequence of Columns. + Returns a :class:`.MappedCollection` factory with a keying function + generated from mapping_spec, which may be a Column or a sequence + of Columns. The key value must be immutable for the lifetime of the object. You can not, for example, map on foreign key values if those key values will @@ -228,16 +233,13 @@ def column_mapped_collection(mapping_spec): after a session flush. """ - global _state_mapper, instance_state - from sqlalchemy.orm.util import _state_mapper - from sqlalchemy.orm.attributes import instance_state - cols = [expression._only_column_elements(q, "mapping_spec") for q in util.to_list(mapping_spec) ] keyfunc = _PlainColumnGetter(cols) return lambda: MappedCollection(keyfunc) + class _SerializableAttrGetter(object): def __init__(self, name): self.name = name @@ -249,6 +251,7 @@ class _SerializableAttrGetter(object): def __reduce__(self): return _SerializableAttrGetter, (self.name, ) + def attribute_mapped_collection(attr_name): """A dictionary-based collection type with attribute-based keying. @@ -269,8 +272,9 @@ def attribute_mapped_collection(attr_name): def mapped_collection(keyfunc): """A dictionary-based collection type with arbitrary keying. - Returns a :class:`.MappedCollection` factory with a keying function generated - from keyfunc, a callable that takes an entity and returns a key value. + Returns a :class:`.MappedCollection` factory with a keying function + generated from keyfunc, a callable that takes an entity and returns a + key value. The key value must be immutable for the lifetime of the object. You can not, for example, map on foreign key values if those key values will @@ -280,13 +284,14 @@ def mapped_collection(keyfunc): """ return lambda: MappedCollection(keyfunc) + class collection(object): """Decorators for entity collection classes. The decorators fall into two groups: annotations and interception recipes. - The annotating decorators (appender, remover, iterator, - internally_instrumented, link) indicate the method's purpose and take no + The annotating decorators (appender, remover, iterator, linker, converter, + internally_instrumented) indicate the method's purpose and take no arguments. They are not written with parens:: @collection.appender @@ -346,7 +351,7 @@ class collection(object): promulgation to collection events. """ - setattr(fn, '_sa_instrument_role', 'appender') + fn._sa_instrument_role = 'appender' return fn @staticmethod @@ -373,7 +378,7 @@ class collection(object): promulgation to collection events. """ - setattr(fn, '_sa_instrument_role', 'remover') + fn._sa_instrument_role = 'remover' return fn @staticmethod @@ -387,17 +392,18 @@ class collection(object): def __iter__(self): ... """ - setattr(fn, '_sa_instrument_role', 'iterator') + fn._sa_instrument_role = 'iterator' return fn @staticmethod def internally_instrumented(fn): """Tag the method as instrumented. - This tag will prevent any decoration from being applied to the method. - Use this if you are orchestrating your own calls to :func:`.collection_adapter` - in one of the basic SQLAlchemy interface methods, or to prevent - an automatic ABC method decoration from wrapping your implementation:: + This tag will prevent any decoration from being applied to the + method. Use this if you are orchestrating your own calls to + :func:`.collection_adapter` in one of the basic SQLAlchemy + interface methods, or to prevent an automatic ABC method + decoration from wrapping your implementation:: # normally an 'extend' method on a list-like class would be # automatically intercepted and re-implemented in terms of @@ -407,12 +413,12 @@ class collection(object): def extend(self, items): ... """ - setattr(fn, '_sa_instrumented', True) + fn._sa_instrumented = True return fn @staticmethod - def link(fn): - """Tag the method as a the "linked to attribute" event handler. + def linker(fn): + """Tag the method as a "linked to attribute" event handler. This optional event handler will be called when the collection class is linked to or unlinked from the InstrumentedAttribute. It is @@ -421,9 +427,12 @@ class collection(object): that has been linked, or None if unlinking. """ - setattr(fn, '_sa_instrument_role', 'link') + fn._sa_instrument_role = 'linker' return fn + link = linker + """deprecated; synonym for :meth:`.collection.linker`.""" + @staticmethod def converter(fn): """Tag the method as the collection converter. @@ -454,7 +463,7 @@ class collection(object): validation on the values about to be assigned. """ - setattr(fn, '_sa_instrument_role', 'converter') + fn._sa_instrument_role = 'converter' return fn @staticmethod @@ -474,7 +483,7 @@ class collection(object): """ def decorator(fn): - setattr(fn, '_sa_instrument_before', ('fire_append_event', arg)) + fn._sa_instrument_before = ('fire_append_event', arg) return fn return decorator @@ -494,8 +503,8 @@ class collection(object): """ def decorator(fn): - setattr(fn, '_sa_instrument_before', ('fire_append_event', arg)) - setattr(fn, '_sa_instrument_after', 'fire_remove_event') + fn._sa_instrument_before = ('fire_append_event', arg) + fn._sa_instrument_after = 'fire_remove_event' return fn return decorator @@ -516,7 +525,7 @@ class collection(object): """ def decorator(fn): - setattr(fn, '_sa_instrument_before', ('fire_remove_event', arg)) + fn._sa_instrument_before = ('fire_remove_event', arg) return fn return decorator @@ -536,31 +545,13 @@ class collection(object): """ def decorator(fn): - setattr(fn, '_sa_instrument_after', 'fire_remove_event') + fn._sa_instrument_after = 'fire_remove_event' return fn return decorator -# public instrumentation interface for 'internally instrumented' -# implementations -def collection_adapter(collection): - """Fetch the :class:`.CollectionAdapter` for a collection.""" - - return getattr(collection, '_sa_adapter', None) - -def collection_iter(collection): - """Iterate over an object supporting the @iterator or __iter__ protocols. - - If the collection is an ORM collection, it need not be attached to an - object to be iterable. - - """ - try: - return getattr(collection, '_sa_iterator', - getattr(collection, '__iter__'))() - except AttributeError: - raise TypeError("'%s' object is not iterable" % - type(collection).__name__) +collection_adapter = operator.attrgetter('_sa_adapter') +"""Fetch the :class:`.CollectionAdapter` for a collection.""" class CollectionAdapter(object): @@ -573,16 +564,19 @@ class CollectionAdapter(object): The ORM uses :class:`.CollectionAdapter` exclusively for interaction with entity collections. - The usage of getattr()/setattr() is currently to allow injection - of custom methods, such as to unwrap Zope security proxies. """ + invalidated = False + def __init__(self, attr, owner_state, data): self._key = attr.key self._data = weakref.ref(data) self.owner_state = owner_state self.link_to_self(data) + def _warn_invalidated(self): + util.warn("This collection has been invalidated.") + @property def data(self): "The entity collection being adapted." @@ -593,16 +587,19 @@ class CollectionAdapter(object): return self.owner_state.manager[self._key].impl def link_to_self(self, data): - """Link a collection to this adapter, and fire a link event.""" - setattr(data, '_sa_adapter', self) - if hasattr(data, '_sa_on_link'): - getattr(data, '_sa_on_link')(self) + """Link a collection to this adapter""" + + data._sa_adapter = self + if data._sa_linker: + data._sa_linker(self) + def unlink(self, data): - """Unlink a collection from any adapter, and fire a link event.""" - setattr(data, '_sa_adapter', None) - if hasattr(data, '_sa_on_link'): - getattr(data, '_sa_on_link')(None) + """Unlink a collection from any adapter""" + + del data._sa_adapter + if data._sa_linker: + data._sa_linker(None) def adapt_like_to_iterable(self, obj): """Converts collection-compatible objects to an iterable of values. @@ -618,7 +615,7 @@ class CollectionAdapter(object): a default duck-typing-based implementation is used. """ - converter = getattr(self._data(), '_sa_converter', None) + converter = self._data()._sa_converter if converter is not None: return converter(obj) @@ -639,75 +636,78 @@ class CollectionAdapter(object): # If the object is an adapted collection, return the (iterable) # adapter. if getattr(obj, '_sa_adapter', None) is not None: - return getattr(obj, '_sa_adapter') + return obj._sa_adapter elif setting_type == dict: - # Py3K - #return obj.values() - # Py2K - return getattr(obj, 'itervalues', getattr(obj, 'values'))() - # end Py2K + if util.py3k: + return obj.values() + else: + return getattr(obj, 'itervalues', obj.values)() else: return iter(obj) def append_with_event(self, item, initiator=None): """Add an entity to the collection, firing mutation events.""" - getattr(self._data(), '_sa_appender')(item, _sa_initiator=initiator) + self._data()._sa_appender(item, _sa_initiator=initiator) def append_without_event(self, item): """Add or restore an entity to the collection, firing no events.""" - getattr(self._data(), '_sa_appender')(item, _sa_initiator=False) + self._data()._sa_appender(item, _sa_initiator=False) def append_multiple_without_event(self, items): """Add or restore an entity to the collection, firing no events.""" - appender = getattr(self._data(), '_sa_appender') + appender = self._data()._sa_appender for item in items: appender(item, _sa_initiator=False) def remove_with_event(self, item, initiator=None): """Remove an entity from the collection, firing mutation events.""" - getattr(self._data(), '_sa_remover')(item, _sa_initiator=initiator) + self._data()._sa_remover(item, _sa_initiator=initiator) def remove_without_event(self, item): """Remove an entity from the collection, firing no events.""" - getattr(self._data(), '_sa_remover')(item, _sa_initiator=False) + self._data()._sa_remover(item, _sa_initiator=False) def clear_with_event(self, initiator=None): """Empty the collection, firing a mutation event for each entity.""" - remover = getattr(self._data(), '_sa_remover') + remover = self._data()._sa_remover for item in list(self): remover(item, _sa_initiator=initiator) def clear_without_event(self): """Empty the collection, firing no events.""" - remover = getattr(self._data(), '_sa_remover') + remover = self._data()._sa_remover for item in list(self): remover(item, _sa_initiator=False) def __iter__(self): """Iterate over entities in the collection.""" - # Py3K requires iter() here - return iter(getattr(self._data(), '_sa_iterator')()) + return iter(self._data()._sa_iterator()) def __len__(self): """Count entities in the collection.""" - return len(list(getattr(self._data(), '_sa_iterator')())) + return len(list(self._data()._sa_iterator())) - def __nonzero__(self): + def __bool__(self): return True + __nonzero__ = __bool__ + def fire_append_event(self, item, initiator=None): """Notify that a entity has entered the collection. - Initiator is a token owned by the InstrumentedAttribute that initiated the membership - mutation, and should be left as None unless you are passing along - an initiator value from a chained operation. + Initiator is a token owned by the InstrumentedAttribute that + initiated the membership mutation, and should be left as None + unless you are passing along an initiator value from a chained + operation. """ - if initiator is not False and item is not None: + if initiator is not False: + if self.invalidated: + self._warn_invalidated() return self.attr.fire_append_event( self.owner_state, self.owner_state.dict, @@ -723,7 +723,9 @@ class CollectionAdapter(object): an initiator value from a chained operation. """ - if initiator is not False and item is not None: + if initiator is not False: + if self.invalidated: + self._warn_invalidated() self.attr.fire_remove_event( self.owner_state, self.owner_state.dict, @@ -736,6 +738,8 @@ class CollectionAdapter(object): fire_remove_event(). """ + if self.invalidated: + self._warn_invalidated() self.attr.fire_pre_remove_event( self.owner_state, self.owner_state.dict, @@ -762,9 +766,11 @@ def bulk_replace(values, existing_adapter, new_adapter): :param values: An iterable of collection member instances - :param existing_adapter: A :class:`.CollectionAdapter` of instances to be replaced + :param existing_adapter: A :class:`.CollectionAdapter` of + instances to be replaced - :param new_adapter: An empty :class:`.CollectionAdapter` to load with ``values`` + :param new_adapter: An empty :class:`.CollectionAdapter` + to load with ``values`` """ @@ -772,9 +778,10 @@ def bulk_replace(values, existing_adapter, new_adapter): values = list(values) idset = util.IdentitySet - constants = idset(existing_adapter or ()).intersection(values or ()) + existing_idset = idset(existing_adapter or ()) + constants = existing_idset.intersection(values or ()) additions = idset(values or ()).difference(constants) - removals = idset(existing_adapter or ()).difference(constants) + removals = existing_idset.difference(constants) for member in values or (): if member in additions: @@ -786,6 +793,7 @@ def bulk_replace(values, existing_adapter, new_adapter): for member in removals: existing_adapter.remove_with_event(member) + def prepare_instrumentation(factory): """Prepare a callable for future use as a collection class factory. @@ -807,7 +815,7 @@ def prepare_instrumentation(factory): # Did factory callable return a builtin? if cls in __canned_instrumentation: # Wrap it so that it returns our 'Instrumented*' - factory = __converting_factory(factory) + factory = __converting_factory(cls, factory) cls = factory() # Instrument the class if needed. @@ -820,51 +828,28 @@ def prepare_instrumentation(factory): return factory -def __converting_factory(original_factory): - """Convert the type returned by collection factories on the fly. - Given a collection factory that returns a builtin type (e.g. a list), - return a wrapped function that converts that type to one of our - instrumented types. +def __converting_factory(specimen_cls, original_factory): + """Return a wrapper that converts a "canned" collection like + set, dict, list into the Instrumented* version. """ + + instrumented_cls = __canned_instrumentation[specimen_cls] + def wrapper(): collection = original_factory() - type_ = type(collection) - if type_ in __canned_instrumentation: - # return an instrumented type initialized from the factory's - # collection - return __canned_instrumentation[type_](collection) - else: - raise sa_exc.InvalidRequestError( - "Collection class factories must produce instances of a " - "single class.") - try: - # often flawed but better than nothing - wrapper.__name__ = "%sWrapper" % original_factory.__name__ - wrapper.__doc__ = original_factory.__doc__ - except: - pass + return instrumented_cls(collection) + + # often flawed but better than nothing + wrapper.__name__ = "%sWrapper" % original_factory.__name__ + wrapper.__doc__ = original_factory.__doc__ + return wrapper def _instrument_class(cls): """Modify methods in a class and install instrumentation.""" - # TODO: more formally document this as a decoratorless/Python 2.3 - # option for specifying instrumentation. (likely doc'd here in code only, - # not in online docs.) Useful for C types too. - # - # __instrumentation__ = { - # 'rolename': 'methodname', # ... - # 'methods': { - # 'methodname': ('fire_{append,remove}_event', argspec, - # 'fire_{append,remove}_event'), - # 'append': ('fire_append_event', 1, None), - # '__setitem__': ('fire_append_event', 1, 'fire_remove_event'), - # 'pop': (None, None, 'fire_remove_event'), - # } - # } - # In the normal call flow, a request for any of the 3 basic collection # types is transformed into one of our trivial subclasses # (e.g. InstrumentedList). Catch anything else that sneaks in here... @@ -873,53 +858,55 @@ def _instrument_class(cls): "Can not instrument a built-in type. Use a " "subclass, even a trivial one.") + roles = {} + methods = {} + + # search for _sa_instrument_role-decorated methods in + # method resolution order, assign to roles + for supercls in cls.__mro__: + for name, method in vars(supercls).items(): + if not util.callable(method): + continue + + # note role declarations + if hasattr(method, '_sa_instrument_role'): + role = method._sa_instrument_role + assert role in ('appender', 'remover', 'iterator', + 'linker', 'converter') + roles.setdefault(role, name) + + # transfer instrumentation requests from decorated function + # to the combined queue + before, after = None, None + if hasattr(method, '_sa_instrument_before'): + op, argument = method._sa_instrument_before + assert op in ('fire_append_event', 'fire_remove_event') + before = op, argument + if hasattr(method, '_sa_instrument_after'): + op = method._sa_instrument_after + assert op in ('fire_append_event', 'fire_remove_event') + after = op + if before: + methods[name] = before[0], before[1], after + elif after: + methods[name] = None, None, after + + # see if this class has "canned" roles based on a known + # collection type (dict, set, list). Apply those roles + # as needed to the "roles" dictionary, and also + # prepare "decorator" methods collection_type = util.duck_type_collection(cls) if collection_type in __interfaces: - roles = __interfaces[collection_type].copy() - decorators = roles.pop('_decorators', {}) - else: - roles, decorators = {}, {} + canned_roles, decorators = __interfaces[collection_type] + for role, name in canned_roles.items(): + roles.setdefault(role, name) - if hasattr(cls, '__instrumentation__'): - roles.update(copy.deepcopy(getattr(cls, '__instrumentation__'))) - - methods = roles.pop('methods', {}) - - for name in dir(cls): - method = getattr(cls, name, None) - if not util.callable(method): - continue - - # note role declarations - if hasattr(method, '_sa_instrument_role'): - role = method._sa_instrument_role - assert role in ('appender', 'remover', 'iterator', - 'link', 'converter') - roles[role] = name - - # transfer instrumentation requests from decorated function - # to the combined queue - before, after = None, None - if hasattr(method, '_sa_instrument_before'): - op, argument = method._sa_instrument_before - assert op in ('fire_append_event', 'fire_remove_event') - before = op, argument - if hasattr(method, '_sa_instrument_after'): - op = method._sa_instrument_after - assert op in ('fire_append_event', 'fire_remove_event') - after = op - if before: - methods[name] = before[0], before[1], after - elif after: - methods[name] = None, None, after - - # apply ABC auto-decoration to methods that need it - - for method, decorator in decorators.items(): - fn = getattr(cls, method, None) - if (fn and method not in methods and - not hasattr(fn, '_sa_instrumented')): - setattr(cls, method, decorator(fn)) + # apply ABC auto-decoration to methods that need it + for method, decorator in decorators.items(): + fn = getattr(cls, method, None) + if (fn and method not in methods and + not hasattr(fn, '_sa_instrumented')): + setattr(cls, method, decorator(fn)) # ensure all roles are present, and apply implicit instrumentation if # needed @@ -946,15 +933,21 @@ def _instrument_class(cls): # apply ad-hoc instrumentation from decorators, class-level defaults # and implicit role declarations - for method, (before, argument, after) in methods.items(): - setattr(cls, method, - _instrument_membership_mutator(getattr(cls, method), + for method_name, (before, argument, after) in methods.items(): + setattr(cls, method_name, + _instrument_membership_mutator(getattr(cls, method_name), before, argument, after)) # intern the role map - for role, method in roles.items(): - setattr(cls, '_sa_%s' % role, getattr(cls, method)) + for role, method_name in roles.items(): + setattr(cls, '_sa_%s' % role, getattr(cls, method_name)) + + cls._sa_adapter = None + if not hasattr(cls, '_sa_linker'): + cls._sa_linker = None + if not hasattr(cls, '_sa_converter'): + cls._sa_converter = None + cls._sa_instrumented = id(cls) - setattr(cls, '_sa_instrumented', id(cls)) def _instrument_membership_mutator(method, before, argument, after): """Route method args and/or return value through the collection adapter.""" @@ -992,7 +985,7 @@ def _instrument_membership_mutator(method, before, argument, after): if initiator is False: executor = None else: - executor = getattr(args[0], '_sa_adapter', None) + executor = args[0]._sa_adapter if before and executor: getattr(executor, before)(value, initiator) @@ -1004,42 +997,46 @@ def _instrument_membership_mutator(method, before, argument, after): if res is not None: getattr(executor, after)(res, initiator) return res - try: - wrapper._sa_instrumented = True - wrapper.__name__ = method.__name__ - wrapper.__doc__ = method.__doc__ - except: - pass + + wrapper._sa_instrumented = True + if hasattr(method, "_sa_instrument_role"): + wrapper._sa_instrument_role = method._sa_instrument_role + wrapper.__name__ = method.__name__ + wrapper.__doc__ = method.__doc__ return wrapper + def __set(collection, item, _sa_initiator=None): """Run set events, may eventually be inlined into decorators.""" - if _sa_initiator is not False and item is not None: - executor = getattr(collection, '_sa_adapter', None) + if _sa_initiator is not False: + executor = collection._sa_adapter if executor: - item = getattr(executor, 'fire_append_event')(item, _sa_initiator) + item = executor.fire_append_event(item, _sa_initiator) return item + def __del(collection, item, _sa_initiator=None): """Run del events, may eventually be inlined into decorators.""" - if _sa_initiator is not False and item is not None: - executor = getattr(collection, '_sa_adapter', None) + if _sa_initiator is not False: + executor = collection._sa_adapter if executor: - getattr(executor, 'fire_remove_event')(item, _sa_initiator) + executor.fire_remove_event(item, _sa_initiator) + def __before_delete(collection, _sa_initiator=None): """Special method to run 'commit existing value' methods""" - executor = getattr(collection, '_sa_adapter', None) + executor = collection._sa_adapter if executor: - getattr(executor, 'fire_pre_remove_event')(_sa_initiator) + executor.fire_pre_remove_event(_sa_initiator) + def _list_decorators(): """Tailored instrumentation wrappers for any list-like class.""" def _tidy(fn): - setattr(fn, '_sa_instrumented', True) - fn.__doc__ = getattr(getattr(list, fn.__name__), '__doc__') + fn._sa_instrumented = True + fn.__doc__ = getattr(list, fn.__name__).__doc__ def append(fn): def append(self, item, _sa_initiator=None): @@ -1078,19 +1075,22 @@ def _list_decorators(): start = index.start or 0 if start < 0: start += len(self) - stop = index.stop or len(self) + if index.stop is not None: + stop = index.stop + else: + stop = len(self) if stop < 0: stop += len(self) if step == 1: - for i in xrange(start, stop, step): + for i in range(start, stop, step): if len(self) > start: del self[start] for i, item in enumerate(value): self.insert(i + start, item) else: - rng = range(start, stop, step) + rng = list(range(start, stop, step)) if len(value) != len(rng): raise ValueError( "attempt to assign sequence of size %s to " @@ -1117,24 +1117,23 @@ def _list_decorators(): _tidy(__delitem__) return __delitem__ - # Py2K - def __setslice__(fn): - def __setslice__(self, start, end, values): - for value in self[start:end]: - __del(self, value) - values = [__set(self, value) for value in values] - fn(self, start, end, values) - _tidy(__setslice__) - return __setslice__ + if util.py2k: + def __setslice__(fn): + def __setslice__(self, start, end, values): + for value in self[start:end]: + __del(self, value) + values = [__set(self, value) for value in values] + fn(self, start, end, values) + _tidy(__setslice__) + return __setslice__ - def __delslice__(fn): - def __delslice__(self, start, end): - for value in self[start:end]: - __del(self, value) - fn(self, start, end) - _tidy(__delslice__) - return __delslice__ - # end Py2K + def __delslice__(fn): + def __delslice__(self, start, end): + for value in self[start:end]: + __del(self, value) + fn(self, start, end) + _tidy(__delslice__) + return __delslice__ def extend(fn): def extend(self, iterable): @@ -1162,6 +1161,15 @@ def _list_decorators(): _tidy(pop) return pop + if not util.py2k: + def clear(fn): + def clear(self, index=-1): + for item in self: + __del(self, item) + fn(self) + _tidy(clear) + return clear + # __imul__ : not wrapping this. all members of the collection are already # present, so no need to fire appends... wrapping it with an explicit # decorator is still possible, so events on *= can be had if they're @@ -1171,12 +1179,13 @@ def _list_decorators(): l.pop('_tidy') return l + def _dict_decorators(): """Tailored instrumentation wrappers for any dict-like mapping class.""" def _tidy(fn): - setattr(fn, '_sa_instrumented', True) - fn.__doc__ = getattr(getattr(dict, fn.__name__), '__doc__') + fn._sa_instrumented = True + fn.__doc__ = getattr(dict, fn.__name__).__doc__ Unspecified = util.symbol('Unspecified') @@ -1235,48 +1244,38 @@ def _dict_decorators(): _tidy(setdefault) return setdefault - if sys.version_info < (2, 4): - def update(fn): - def update(self, other): - for key in other.keys(): - if key not in self or self[key] is not other[key]: - self[key] = other[key] - _tidy(update) - return update - else: - def update(fn): - def update(self, __other=Unspecified, **kw): - if __other is not Unspecified: - if hasattr(__other, 'keys'): - for key in __other.keys(): - if (key not in self or - self[key] is not __other[key]): - self[key] = __other[key] - else: - for key, value in __other: - if key not in self or self[key] is not value: - self[key] = value - for key in kw: - if key not in self or self[key] is not kw[key]: - self[key] = kw[key] - _tidy(update) - return update + def update(fn): + def update(self, __other=Unspecified, **kw): + if __other is not Unspecified: + if hasattr(__other, 'keys'): + for key in list(__other): + if (key not in self or + self[key] is not __other[key]): + self[key] = __other[key] + else: + for key, value in __other: + if key not in self or self[key] is not value: + self[key] = value + for key in kw: + if key not in self or self[key] is not kw[key]: + self[key] = kw[key] + _tidy(update) + return update l = locals().copy() l.pop('_tidy') l.pop('Unspecified') return l -if util.py3k_warning: - _set_binop_bases = (set, frozenset) -else: - import sets - _set_binop_bases = (set, frozenset, sets.BaseSet) +_set_binop_bases = (set, frozenset) + def _set_binops_check_strict(self, obj): - """Allow only set, frozenset and self.__class__-derived objects in binops.""" + """Allow only set, frozenset and self.__class__-derived + objects in binops.""" return isinstance(obj, _set_binop_bases + (self.__class__,)) + def _set_binops_check_loose(self, obj): """Allow anything set-like to participate in set binops.""" return (isinstance(obj, _set_binop_bases + (self.__class__,)) or @@ -1287,8 +1286,8 @@ def _set_decorators(): """Tailored instrumentation wrappers for any set-like class.""" def _tidy(fn): - setattr(fn, '_sa_instrumented', True) - fn.__doc__ = getattr(getattr(set, fn.__name__), '__doc__') + fn._sa_instrumented = True + fn.__doc__ = getattr(set, fn.__name__).__doc__ Unspecified = util.symbol('Unspecified') @@ -1301,23 +1300,15 @@ def _set_decorators(): _tidy(add) return add - if sys.version_info < (2, 4): - def discard(fn): - def discard(self, value, _sa_initiator=None): - if value in self: - self.remove(value, _sa_initiator) - _tidy(discard) - return discard - else: - def discard(fn): - def discard(self, value, _sa_initiator=None): + def discard(fn): + def discard(self, value, _sa_initiator=None): + # testlib.pragma exempt:__hash__ + if value in self: + __del(self, value, _sa_initiator) # testlib.pragma exempt:__hash__ - if value in self: - __del(self, value, _sa_initiator) - # testlib.pragma exempt:__hash__ - fn(self, value) - _tidy(discard) - return discard + fn(self, value) + _tidy(discard) + return discard def remove(fn): def remove(self, value, _sa_initiator=None): @@ -1442,29 +1433,14 @@ def _set_decorators(): class InstrumentedList(list): """An instrumented version of the built-in list.""" - __instrumentation__ = { - 'appender': 'append', - 'remover': 'remove', - 'iterator': '__iter__', } class InstrumentedSet(set): """An instrumented version of the built-in set.""" - __instrumentation__ = { - 'appender': 'add', - 'remover': 'remove', - 'iterator': '__iter__', } class InstrumentedDict(dict): """An instrumented version of the built-in dict.""" - # Py3K - #__instrumentation__ = { - # 'iterator': 'values', } - # Py2K - __instrumentation__ = { - 'iterator': 'itervalues', } - # end Py2K __canned_instrumentation = { list: InstrumentedList, @@ -1473,33 +1449,29 @@ __canned_instrumentation = { } __interfaces = { - list: {'appender': 'append', - 'remover': 'remove', - 'iterator': '__iter__', - '_decorators': _list_decorators(), }, - set: {'appender': 'add', + list: ( + {'appender': 'append', 'remover': 'remove', + 'iterator': '__iter__'}, _list_decorators() + ), + + set: ({'appender': 'add', 'remover': 'remove', - 'iterator': '__iter__', - '_decorators': _set_decorators(), }, + 'iterator': '__iter__'}, _set_decorators() + ), + # decorators are required for dicts and object collections. - # Py3K - #dict: {'iterator': 'values', - # '_decorators': _dict_decorators(), }, - # Py2K - dict: {'iterator': 'itervalues', - '_decorators': _dict_decorators(), }, - # end Py2K - # < 0.4 compatible naming, deprecated- use decorators instead. - None: {} + dict: ({'iterator': 'values'}, _dict_decorators()) if util.py3k + else ({'iterator': 'itervalues'}, _dict_decorators()), } + class MappedCollection(dict): """A basic dictionary-based collection class. - Extends dict with the minimal bag semantics that collection classes require. - ``set`` and ``remove`` are implemented in terms of a keying function: any - callable that takes an object and returns an object for use as a dictionary - key. + Extends dict with the minimal bag semantics that collection + classes require. ``set`` and ``remove`` are implemented in terms + of a keying function: any callable that takes an object and + returns an object for use as a dictionary key. """ @@ -1519,14 +1491,16 @@ class MappedCollection(dict): """ self.keyfunc = keyfunc + @collection.appender + @collection.internally_instrumented def set(self, value, _sa_initiator=None): """Add an item by value, consulting the keyfunc for the key.""" key = self.keyfunc(value) self.__setitem__(key, value, _sa_initiator) - set = collection.internally_instrumented(set) - set = collection.appender(set) + @collection.remover + @collection.internally_instrumented def remove(self, value, _sa_initiator=None): """Remove an item by value, consulting the keyfunc for the key.""" @@ -1541,9 +1515,8 @@ class MappedCollection(dict): "values after flush?" % (value, self[key], key)) self.__delitem__(key, _sa_initiator) - remove = collection.internally_instrumented(remove) - remove = collection.remover(remove) + @collection.converter def _convert(self, dictlike): """Validate and convert a dict-like object into values for set()ing. @@ -1561,11 +1534,11 @@ class MappedCollection(dict): new_key = self.keyfunc(value) if incoming_key != new_key: raise TypeError( - "Found incompatible key %r for value %r; this collection's " + "Found incompatible key %r for value %r; this " + "collection's " "keying function requires a key of %r for this value." % ( incoming_key, value, new_key)) yield value - _convert = collection.converter(_convert) # ensure instrumentation is associated with # these built-in classes; if a user-defined class @@ -1575,4 +1548,3 @@ class MappedCollection(dict): _instrument_class(MappedCollection) _instrument_class(InstrumentedList) _instrument_class(InstrumentedSet) - diff --git a/libs/sqlalchemy/orm/dependency.py b/libs/sqlalchemy/orm/dependency.py index c46969f2..34a2af39 100644 --- a/libs/sqlalchemy/orm/dependency.py +++ b/libs/sqlalchemy/orm/dependency.py @@ -1,5 +1,5 @@ # orm/dependency.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,10 +8,11 @@ """ -from sqlalchemy import sql, util, exc as sa_exc -from sqlalchemy.orm import attributes, exc, sync, unitofwork, \ +from .. import sql, util, exc as sa_exc +from . import attributes, exc, sync, unitofwork, \ util as mapperutil -from sqlalchemy.orm.interfaces import ONETOMANY, MANYTOONE, MANYTOMANY +from .interfaces import ONETOMANY, MANYTOONE, MANYTOMANY + class DependencyProcessor(object): def __init__(self, prop): @@ -32,7 +33,7 @@ class DependencyProcessor(object): if self.passive_updates: self._passive_update_flag = attributes.PASSIVE_NO_INITIALIZE else: - self._passive_update_flag= attributes.PASSIVE_OFF + self._passive_update_flag = attributes.PASSIVE_OFF self.key = prop.key if not self.prop.synchronize_pairs: @@ -63,7 +64,6 @@ class DependencyProcessor(object): """ uow.register_preprocessor(self, True) - def per_property_flush_actions(self, uow): after_save = unitofwork.ProcessAll(uow, self, False, True) before_delete = unitofwork.ProcessAll(uow, self, True, True) @@ -95,7 +95,6 @@ class DependencyProcessor(object): before_delete ) - def per_state_flush_actions(self, uow, states, isdelete): """establish actions and dependencies related to a flush. @@ -159,7 +158,8 @@ class DependencyProcessor(object): # detect if there's anything changed or loaded # by a preprocessor on this state/attribute. if not, # we should be able to skip it entirely. - sum_ = state.manager[self.key].impl.get_all_pending(state, state.dict) + sum_ = state.manager[self.key].impl.get_all_pending( + state, state.dict) if not sum_: continue @@ -210,7 +210,6 @@ class DependencyProcessor(object): after_save, before_delete, isdelete, childisdelete) - def presort_deletes(self, uowcommit, states): return False @@ -247,7 +246,11 @@ class DependencyProcessor(object): self.mapper in uowcommit.mappers def _verify_canload(self, state): - if state is not None and \ + if self.prop.uselist and state is None: + raise exc.FlushError( + "Can't flush None value found in " + "collection %s" % (self.prop, )) + elif state is not None and \ not self.mapper._canload(state, allow_subtypes=not self.enable_typechecks): if self.mapper._canload(state, allow_subtypes=True): @@ -310,6 +313,7 @@ class DependencyProcessor(object): def __repr__(self): return "%s(%s)" % (self.__class__.__name__, self.prop) + class OneToManyDP(DependencyProcessor): def per_property_dependencies(self, uow, parent_saves, @@ -433,8 +437,6 @@ class OneToManyDP(DependencyProcessor): uowcommit.register_object(child, operation="delete", prop=self.prop) - - def presort_saves(self, uowcommit, states): children_added = uowcommit.memo(('children_added', self), set) @@ -559,10 +561,10 @@ class OneToManyDP(DependencyProcessor): pks_changed): source = state dest = child + self._verify_canload(child) if dest is None or \ (not self.post_update and uowcommit.is_deleted(dest)): return - self._verify_canload(child) if clearkeys: sync.clear(dest, self.mapper, self.prop.synchronize_pairs) else: @@ -577,6 +579,7 @@ class OneToManyDP(DependencyProcessor): self.parent, self.prop.synchronize_pairs) + class ManyToOneDP(DependencyProcessor): def __init__(self, prop): DependencyProcessor.__init__(self, prop) @@ -690,8 +693,8 @@ class ManyToOneDP(DependencyProcessor): continue uowcommit.register_object(child, isdelete=True, operation="delete", prop=self.prop) - for c, m, st_, dct_ in self.mapper.cascade_iterator( - 'delete', child): + t = self.mapper.cascade_iterator('delete', child) + for c, m, st_, dct_ in t: uowcommit.register_object( st_, isdelete=True) @@ -704,17 +707,14 @@ class ManyToOneDP(DependencyProcessor): self.key, self._passive_delete_flag) if history: - ret = True for child in history.deleted: if self.hasparent(child) is False: uowcommit.register_object(child, isdelete=True, operation="delete", prop=self.prop) - for c, m, st_, dct_ in self.mapper.cascade_iterator( - 'delete', child): - uowcommit.register_object( - st_, - isdelete=True) + t = self.mapper.cascade_iterator('delete', child) + for c, m, st_, dct_ in t: + uowcommit.register_object(st_, isdelete=True) def process_deletes(self, uowcommit, states): if self.post_update and \ @@ -773,6 +773,7 @@ class ManyToOneDP(DependencyProcessor): uowcommit, False) + class DetectKeySwitch(DependencyProcessor): """For many-to-one relationships with no one-to-many backref, searches for parents through the unit of work when a primary @@ -862,7 +863,7 @@ class DetectKeySwitch(DependencyProcessor): related = state.get_impl(self.key).get(state, dict_, passive=self._passive_update_flag) if related is not attributes.PASSIVE_NO_RESULT and \ - related is not None: + related is not None: related_state = attributes.instance_state(dict_[self.key]) if related_state in switchers: uowcommit.register_object(state, @@ -932,12 +933,14 @@ class ManyToManyDP(DependencyProcessor): ]) def presort_deletes(self, uowcommit, states): + # TODO: no tests fail if this whole + # thing is removed !!!! if not self.passive_deletes: # if no passive deletes, load history on # the collection, so that prop_has_changes() # returns True for state in states: - history = uowcommit.get_attribute_history( + uowcommit.get_attribute_history( state, self.key, self._passive_delete_flag) @@ -1031,8 +1034,7 @@ class ManyToManyDP(DependencyProcessor): passive) if history: for child in history.added: - if child is None or \ - (processed is not None and + if (processed is not None and (state, child) in processed): continue associationrow = {} @@ -1043,8 +1045,7 @@ class ManyToManyDP(DependencyProcessor): continue secondary_insert.append(associationrow) for child in history.deleted: - if child is None or \ - (processed is not None and + if (processed is not None and (state, child) in processed): continue associationrow = {} @@ -1097,11 +1098,11 @@ class ManyToManyDP(DependencyProcessor): if result.supports_sane_multi_rowcount() and \ result.rowcount != len(secondary_delete): raise exc.StaleDataError( - "DELETE statement on table '%s' expected to delete %d row(s); " - "Only %d were matched." % - (self.secondary.description, len(secondary_delete), - result.rowcount) - ) + "DELETE statement on table '%s' expected to delete " + "%d row(s); Only %d were matched." % + (self.secondary.description, len(secondary_delete), + result.rowcount) + ) if secondary_update: associationrow = secondary_update[0] @@ -1114,11 +1115,11 @@ class ManyToManyDP(DependencyProcessor): if result.supports_sane_multi_rowcount() and \ result.rowcount != len(secondary_update): raise exc.StaleDataError( - "UPDATE statement on table '%s' expected to update %d row(s); " - "Only %d were matched." % - (self.secondary.description, len(secondary_update), - result.rowcount) - ) + "UPDATE statement on table '%s' expected to update " + "%d row(s); Only %d were matched." % + (self.secondary.description, len(secondary_update), + result.rowcount) + ) if secondary_insert: statement = self.secondary.insert() @@ -1126,8 +1127,14 @@ class ManyToManyDP(DependencyProcessor): def _synchronize(self, state, child, associationrow, clearkeys, uowcommit, operation): - if associationrow is None: - return + + # this checks for None if uselist=True + self._verify_canload(child) + + # but if uselist=False we get here. If child is None, + # no association row can be generated, so return. + if child is None: + return False if child is not None and not uowcommit.session._contains_state(child): if not child.deleted: @@ -1137,8 +1144,6 @@ class ManyToManyDP(DependencyProcessor): (mapperutil.state_class_str(child), operation, self.prop)) return False - self._verify_canload(child) - sync.populate_dict(state, self.parent, associationrow, self.prop.synchronize_pairs) sync.populate_dict(child, self.mapper, associationrow, @@ -1154,8 +1159,7 @@ class ManyToManyDP(DependencyProcessor): self.prop.synchronize_pairs) _direction_to_processor = { - ONETOMANY : OneToManyDP, + ONETOMANY: OneToManyDP, MANYTOONE: ManyToOneDP, - MANYTOMANY : ManyToManyDP, + MANYTOMANY: ManyToManyDP, } - diff --git a/libs/sqlalchemy/orm/deprecated_interfaces.py b/libs/sqlalchemy/orm/deprecated_interfaces.py index d251f52e..020b7c71 100644 --- a/libs/sqlalchemy/orm/deprecated_interfaces.py +++ b/libs/sqlalchemy/orm/deprecated_interfaces.py @@ -1,13 +1,13 @@ # orm/deprecated_interfaces.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy import event, util -from interfaces import EXT_CONTINUE - +from .. import event, util +from .interfaces import EXT_CONTINUE +@util.langhelpers.dependency_for("sqlalchemy.orm.interfaces") class MapperExtension(object): """Base implementation for :class:`.Mapper` event hooks. @@ -116,7 +116,6 @@ class MapperExtension(object): event.listen(self, "%s" % meth, ls_meth, raw=False, retval=True, propagate=True) - def instrument_class(self, mapper, class_): """Receive a class when the mapper is first constructed, and has applied instrumentation to the mapped class. @@ -374,6 +373,8 @@ class MapperExtension(object): return EXT_CONTINUE + +@util.langhelpers.dependency_for("sqlalchemy.orm.interfaces") class SessionExtension(object): """Base implementation for :class:`.Session` event hooks. @@ -385,7 +386,7 @@ class SessionExtension(object): :class:`.SessionEvents`. Subclasses may be installed into a :class:`.Session` (or - :func:`.sessionmaker`) using the ``extension`` keyword + :class:`.sessionmaker`) using the ``extension`` keyword argument:: from sqlalchemy.orm.interfaces import SessionExtension @@ -439,7 +440,7 @@ class SessionExtension(object): Note that this may not be per-flush if a longer running transaction is ongoing.""" - def before_flush( self, session, flush_context, instances): + def before_flush(self, session, flush_context, instances): """Execute before flush process has started. `instances` is an optional list of objects which were passed to @@ -462,7 +463,7 @@ class SessionExtension(object): occurred, depending on whether or not the flush started its own transaction or participated in a larger transaction. """ - def after_begin( self, session, transaction, connection): + def after_begin(self, session, transaction, connection): """Execute after a transaction is begun on a connection `transaction` is the SessionTransaction. This method is called @@ -473,7 +474,7 @@ class SessionExtension(object): This is called after an add, delete or merge. """ - def after_bulk_update( self, session, query, query_context, result): + def after_bulk_update(self, session, query, query_context, result): """Execute after a bulk update operation to the session. This is called after a session.query(...).update() @@ -483,7 +484,7 @@ class SessionExtension(object): `result` is the result object returned from the bulk operation. """ - def after_bulk_delete( self, session, query, query_context, result): + def after_bulk_delete(self, session, query, query_context, result): """Execute after a bulk delete operation to the session. This is called after a session.query(...).delete() @@ -494,6 +495,7 @@ class SessionExtension(object): """ +@util.langhelpers.dependency_for("sqlalchemy.orm.interfaces") class AttributeExtension(object): """Base implementation for :class:`.AttributeImpl` event hooks, events that fire upon attribute mutations in user code. @@ -586,5 +588,3 @@ class AttributeExtension(object): """ return value - - diff --git a/libs/sqlalchemy/orm/descriptor_props.py b/libs/sqlalchemy/orm/descriptor_props.py index 70cf0e7c..24b0a15e 100644 --- a/libs/sqlalchemy/orm/descriptor_props.py +++ b/libs/sqlalchemy/orm/descriptor_props.py @@ -1,5 +1,5 @@ # orm/descriptor_props.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -10,13 +10,14 @@ as actively in the load/persist ORM loop. """ -from sqlalchemy.orm.interfaces import \ - MapperProperty, PropComparator, StrategizedProperty -from sqlalchemy.orm.mapper import _none_set -from sqlalchemy.orm import attributes, strategies -from sqlalchemy import util, sql, exc as sa_exc, event, schema -from sqlalchemy.sql import expression -properties = util.importlater('sqlalchemy.orm', 'properties') +from .interfaces import MapperProperty, PropComparator +from .util import _none_set +from . import attributes +from .. import util, sql, exc as sa_exc, event, schema +from ..sql import expression +from . import properties +from . import query + class DescriptorProperty(MapperProperty): """:class:`.MapperProperty` which proxies access to a @@ -30,6 +31,7 @@ class DescriptorProperty(MapperProperty): class _ProxyImpl(object): accepts_scalar_loader = False expire_missing = True + collection = False def __init__(self, key): self.key = key @@ -47,8 +49,10 @@ class DescriptorProperty(MapperProperty): if self.descriptor is None: def fset(obj, value): setattr(obj, self.name, value) + def fdel(obj): delattr(obj, self.name) + def fget(obj): return getattr(obj, self.name) @@ -65,15 +69,79 @@ class DescriptorProperty(MapperProperty): self.key, self.descriptor, lambda: self._comparator_factory(mapper), - doc=self.doc + doc=self.doc, + original_property=self ) proxy_attr.impl = _ProxyImpl(self.key) mapper.class_manager.instrument_attribute(self.key, proxy_attr) +@util.langhelpers.dependency_for("sqlalchemy.orm.properties") class CompositeProperty(DescriptorProperty): + """Defines a "composite" mapped attribute, representing a collection + of columns as one attribute. + :class:`.CompositeProperty` is constructed using the :func:`.composite` + function. + + .. seealso:: + + :ref:`mapper_composite` + + """ def __init__(self, class_, *attrs, **kwargs): + """Return a composite column-based property for use with a Mapper. + + See the mapping documentation section :ref:`mapper_composite` for a full + usage example. + + The :class:`.MapperProperty` returned by :func:`.composite` + is the :class:`.CompositeProperty`. + + :param class\_: + The "composite type" class. + + :param \*cols: + List of Column objects to be mapped. + + :param active_history=False: + When ``True``, indicates that the "previous" value for a + scalar attribute should be loaded when replaced, if not + already loaded. See the same flag on :func:`.column_property`. + + .. versionchanged:: 0.7 + This flag specifically becomes meaningful + - previously it was a placeholder. + + :param group: + A group name for this property when marked as deferred. + + :param deferred: + When True, the column property is "deferred", meaning that it does not + load immediately, and is instead loaded when the attribute is first + accessed on an instance. See also :func:`~sqlalchemy.orm.deferred`. + + :param comparator_factory: a class which extends + :class:`.CompositeProperty.Comparator` which provides custom SQL clause + generation for comparison operations. + + :param doc: + optional string that will be applied as the doc on the + class-bound descriptor. + + :param info: Optional data dictionary which will be populated into the + :attr:`.MapperProperty.info` attribute of this object. + + .. versionadded:: 0.8 + + :param extension: + an :class:`.AttributeExtension` instance, + or list of extensions, which will be prepended to the list of + attribute listeners for the resulting descriptor placed on the class. + **Deprecated.** Please see :class:`.AttributeEvents`. + + """ + self.attrs = attrs self.composite_class = class_ self.active_history = kwargs.get('active_history', False) @@ -81,9 +149,13 @@ class CompositeProperty(DescriptorProperty): self.group = kwargs.get('group', None) self.comparator_factory = kwargs.pop('comparator_factory', self.__class__.Comparator) + if 'info' in kwargs: + self.info = kwargs.pop('info') + util.set_creation_order(self) self._create_descriptor() + def instrument_class(self, mapper): super(CompositeProperty, self).instrument_class(mapper) self._setup_event_handlers() @@ -110,7 +182,10 @@ class CompositeProperty(DescriptorProperty): # key not present. Iterate through related # attributes, retrieve their values. This # ensures they all load. - values = [getattr(instance, key) for key in self._attribute_keys] + values = [ + getattr(instance, key) + for key in self._attribute_keys + ] # current expected behavior here is that the composite is # created on access if the object is persistent or if @@ -164,12 +239,17 @@ class CompositeProperty(DescriptorProperty): def _init_props(self): self.props = props = [] for attr in self.attrs: - if isinstance(attr, basestring): + if isinstance(attr, str): prop = self.parent.get_property(attr) elif isinstance(attr, schema.Column): prop = self.parent._columntoproperty[attr] elif isinstance(attr, attributes.InstrumentedAttribute): prop = attr.property + else: + raise sa_exc.ArgumentError( + "Composite expects Column objects or mapped " + "attributes/attribute names as arguments, got: %r" + % (attr,)) props.append(prop) @property @@ -185,7 +265,9 @@ class CompositeProperty(DescriptorProperty): prop.active_history = self.active_history if self.deferred: prop.deferred = self.deferred - prop.strategy_class = strategies.DeferredColumnLoader + prop.strategy_class = prop._strategy_lookup( + ("deferred", True), + ("instrument", True)) prop.group = self.group def _setup_event_handlers(self): @@ -225,12 +307,15 @@ class CompositeProperty(DescriptorProperty): state.dict.pop(self.key, None) event.listen(self.parent, 'after_insert', - insert_update_handler, raw=True) + insert_update_handler, raw=True) event.listen(self.parent, 'after_update', - insert_update_handler, raw=True) - event.listen(self.parent, 'load', load_handler, raw=True, propagate=True) - event.listen(self.parent, 'refresh', load_handler, raw=True, propagate=True) - event.listen(self.parent, "expire", expire_handler, raw=True, propagate=True) + insert_update_handler, raw=True) + event.listen(self.parent, 'load', + load_handler, raw=True, propagate=True) + event.listen(self.parent, 'refresh', + load_handler, raw=True, propagate=True) + event.listen(self.parent, 'expire', + expire_handler, raw=True, propagate=True) # TODO: need a deserialize hook here @@ -271,34 +356,80 @@ class CompositeProperty(DescriptorProperty): ) else: return attributes.History( - (),[self.composite_class(*added)], () + (), [self.composite_class(*added)], () ) def _comparator_factory(self, mapper): - return self.comparator_factory(self) + return self.comparator_factory(self, mapper) + + class CompositeBundle(query.Bundle): + def __init__(self, property, expr): + self.property = property + super(CompositeProperty.CompositeBundle, self).__init__( + property.key, *expr) + + def create_row_processor(self, query, procs, labels): + def proc(row, result): + return self.property.composite_class(*[proc(row, result) for proc in procs]) + return proc + class Comparator(PropComparator): - def __init__(self, prop, adapter=None): - self.prop = self.property = prop - self.adapter = adapter + """Produce boolean, comparison, and other operators for + :class:`.CompositeProperty` attributes. + + See the example in :ref:`composite_operations` for an overview + of usage , as well as the documentation for :class:`.PropComparator`. + + See also: + + :class:`.PropComparator` + + :class:`.ColumnOperators` + + :ref:`types_operators` + + :attr:`.TypeEngine.comparator_factory` + + """ - def __clause_element__(self): - if self.adapter: - # TODO: test coverage for adapted composite comparison - return expression.ClauseList( - *[self.adapter(x) for x in self.prop._comparable_elements]) - else: - return expression.ClauseList(*self.prop._comparable_elements) __hash__ = None + @property + def clauses(self): + return self.__clause_element__() + + def __clause_element__(self): + return expression.ClauseList(group=False, *self._comparable_elements) + + def _query_clause_element(self): + return CompositeProperty.CompositeBundle(self.prop, self.__clause_element__()) + + @util.memoized_property + def _comparable_elements(self): + if self._adapt_to_entity: + return [ + getattr( + self._adapt_to_entity.entity, + prop.key + ) for prop in self.prop._comparable_elements + ] + else: + return self.prop._comparable_elements + def __eq__(self, other): if other is None: values = [None] * len(self.prop._comparable_elements) else: values = other.__composite_values__() - return sql.and_( - *[a==b for a, b in zip(self.prop._comparable_elements, values)]) + comparisons = [ + a == b + for a, b in zip(self.prop._comparable_elements, values) + ] + if self._adapt_to_entity: + comparisons = [self.adapter(x) for x in comparisons] + return sql.and_(*comparisons) def __ne__(self, other): return sql.not_(self.__eq__(other)) @@ -306,6 +437,8 @@ class CompositeProperty(DescriptorProperty): def __str__(self): return str(self.parent.class_.__name__) + "." + self.key + +@util.langhelpers.dependency_for("sqlalchemy.orm.properties") class ConcreteInheritedProperty(DescriptorProperty): """A 'do nothing' :class:`.MapperProperty` that disables an attribute on a concrete subclass that is only present @@ -343,8 +476,10 @@ class ConcreteInheritedProperty(DescriptorProperty): class NoninheritedConcreteProp(object): def __set__(s, obj, value): warn() + def __delete__(s, obj): warn() + def __get__(s, obj, owner): if obj is None: return self.descriptor @@ -352,11 +487,66 @@ class ConcreteInheritedProperty(DescriptorProperty): self.descriptor = NoninheritedConcreteProp() +@util.langhelpers.dependency_for("sqlalchemy.orm.properties") class SynonymProperty(DescriptorProperty): def __init__(self, name, map_column=None, descriptor=None, comparator_factory=None, doc=None): + """Denote an attribute name as a synonym to a mapped property, + in that the attribute will mirror the value and expression behavior + of another attribute. + + :param name: the name of the existing mapped property. This + can refer to the string name of any :class:`.MapperProperty` + configured on the class, including column-bound attributes + and relationships. + + :param descriptor: a Python :term:`descriptor` that will be used + as a getter (and potentially a setter) when this attribute is + accessed at the instance level. + + :param map_column: if ``True``, the :func:`.synonym` construct will + locate the existing named :class:`.MapperProperty` based on the + attribute name of this :func:`.synonym`, and assign it to a new + attribute linked to the name of this :func:`.synonym`. + That is, given a mapping like:: + + class MyClass(Base): + __tablename__ = 'my_table' + + id = Column(Integer, primary_key=True) + job_status = Column(String(50)) + + job_status = synonym("_job_status", map_column=True) + + The above class ``MyClass`` will now have the ``job_status`` + :class:`.Column` object mapped to the attribute named ``_job_status``, + and the attribute named ``job_status`` will refer to the synonym + itself. This feature is typically used in conjunction with the + ``descriptor`` argument in order to link a user-defined descriptor + as a "wrapper" for an existing column. + + :param comparator_factory: A subclass of :class:`.PropComparator` + that will provide custom comparison behavior at the SQL expression + level. + + .. note:: + + For the use case of providing an attribute which redefines both + Python-level and SQL-expression level behavior of an attribute, + please refer to the Hybrid attribute introduced at + :ref:`mapper_hybrids` for a more effective technique. + + .. seealso:: + + :ref:`synonyms` - examples of functionality. + + :ref:`mapper_hybrids` - Hybrids provide a better approach for + more complicated attribute-wrapping schemes than synonyms. + + """ + self.name = name self.map_column = map_column self.descriptor = descriptor @@ -409,10 +599,73 @@ class SynonymProperty(DescriptorProperty): self.parent = parent + +@util.langhelpers.dependency_for("sqlalchemy.orm.properties") class ComparableProperty(DescriptorProperty): """Instruments a Python property for use in query expressions.""" def __init__(self, comparator_factory, descriptor=None, doc=None): + """Provides a method of applying a :class:`.PropComparator` + to any Python descriptor attribute. + + .. versionchanged:: 0.7 + :func:`.comparable_property` is superseded by + the :mod:`~sqlalchemy.ext.hybrid` extension. See the example + at :ref:`hybrid_custom_comparators`. + + Allows any Python descriptor to behave like a SQL-enabled + attribute when used at the class level in queries, allowing + redefinition of expression operator behavior. + + In the example below we redefine :meth:`.PropComparator.operate` + to wrap both sides of an expression in ``func.lower()`` to produce + case-insensitive comparison:: + + from sqlalchemy.orm import comparable_property + from sqlalchemy.orm.interfaces import PropComparator + from sqlalchemy.sql import func + from sqlalchemy import Integer, String, Column + from sqlalchemy.ext.declarative import declarative_base + + class CaseInsensitiveComparator(PropComparator): + def __clause_element__(self): + return self.prop + + def operate(self, op, other): + return op( + func.lower(self.__clause_element__()), + func.lower(other) + ) + + Base = declarative_base() + + class SearchWord(Base): + __tablename__ = 'search_word' + id = Column(Integer, primary_key=True) + word = Column(String) + word_insensitive = comparable_property(lambda prop, mapper: + CaseInsensitiveComparator(mapper.c.word, mapper) + ) + + + A mapping like the above allows the ``word_insensitive`` attribute + to render an expression like:: + + >>> print SearchWord.word_insensitive == "Trucks" + lower(search_word.word) = lower(:lower_1) + + :param comparator_factory: + A PropComparator subclass or factory that defines operator behavior + for this property. + + :param descriptor: + Optional when used in a ``properties={}`` declaration. The Python + descriptor or property to layer comparison behavior on top of. + + The like-named descriptor will be automatically retrieved from the + mapped class if left blank in a ``properties`` declaration. + + """ self.descriptor = descriptor self.comparator_factory = comparator_factory self.doc = doc or (descriptor and descriptor.__doc__) or None @@ -420,3 +673,5 @@ class ComparableProperty(DescriptorProperty): def _comparator_factory(self, mapper): return self.comparator_factory(self, mapper) + + diff --git a/libs/sqlalchemy/orm/dynamic.py b/libs/sqlalchemy/orm/dynamic.py index e3773659..bae09d32 100644 --- a/libs/sqlalchemy/orm/dynamic.py +++ b/libs/sqlalchemy/orm/dynamic.py @@ -1,5 +1,5 @@ # orm/dynamic.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -11,39 +11,40 @@ basic add/delete mutation. """ -from sqlalchemy import log, util -from sqlalchemy.orm import exc as orm_exc -from sqlalchemy.sql import operators -from sqlalchemy.orm import ( - attributes, object_session, util as mapperutil, strategies, object_mapper +from .. import log, util, exc +from ..sql import operators +from . import ( + attributes, object_session, util as orm_util, strategies, + object_mapper, exc as orm_exc, properties ) -from sqlalchemy.orm.query import Query -from sqlalchemy.orm.util import has_identity -from sqlalchemy.orm import collections +from .query import Query +@log.class_logger +@properties.RelationshipProperty.strategy_for(lazy="dynamic") class DynaLoader(strategies.AbstractRelationshipLoader): def init_class_attribute(self, mapper): self.is_class_level = True if not self.uselist: - util.warn( - "On relationship %s, 'dynamic' loaders cannot be used with " - "many-to-one/one-to-one relationships and/or " - "uselist=False." % self.parent_property) + raise exc.InvalidRequestError( + "On relationship %s, 'dynamic' loaders cannot be used with " + "many-to-one/one-to-one relationships and/or " + "uselist=False." % self.parent_property) strategies._register_attribute(self, mapper, useobject=True, + uselist=True, impl_class=DynamicAttributeImpl, target_mapper=self.parent_property.mapper, order_by=self.parent_property.order_by, - query_class=self.parent_property.query_class + query_class=self.parent_property.query_class, + backref=self.parent_property.back_populates, ) -log.class_logger(DynaLoader) - class DynamicAttributeImpl(attributes.AttributeImpl): uses_objects = True accepts_scalar_loader = False supports_population = False + collection = False def __init__(self, class_, key, typecallable, dispatch, @@ -60,7 +61,7 @@ class DynamicAttributeImpl(attributes.AttributeImpl): self.query_class = mixin_user_query(query_class) def get(self, state, dict_, passive=attributes.PASSIVE_OFF): - if passive is not attributes.PASSIVE_OFF: + if not passive & attributes.SQL_OK: return self._get_collection_history(state, attributes.PASSIVE_NO_INITIALIZE).added_items else: @@ -68,39 +69,53 @@ class DynamicAttributeImpl(attributes.AttributeImpl): def get_collection(self, state, dict_, user_data=None, passive=attributes.PASSIVE_NO_INITIALIZE): - if passive is not attributes.PASSIVE_OFF: + if not passive & attributes.SQL_OK: return self._get_collection_history(state, passive).added_items else: history = self._get_collection_history(state, passive) - return history.added_items + history.unchanged_items + return history.added_plus_unchanged - def fire_append_event(self, state, dict_, value, initiator): - collection_history = self._modified_event(state, dict_) - collection_history.added_items.append(value) + @util.memoized_property + def _append_token(self): + return attributes.Event(self, attributes.OP_APPEND) + + @util.memoized_property + def _remove_token(self): + return attributes.Event(self, attributes.OP_REMOVE) + + def fire_append_event(self, state, dict_, value, initiator, + collection_history=None): + if collection_history is None: + collection_history = self._modified_event(state, dict_) + + collection_history.add_added(value) for fn in self.dispatch.append: - value = fn(state, value, initiator or self) + value = fn(state, value, initiator or self._append_token) if self.trackparent and value is not None: self.sethasparent(attributes.instance_state(value), state, True) - def fire_remove_event(self, state, dict_, value, initiator): - collection_history = self._modified_event(state, dict_) - collection_history.deleted_items.append(value) + def fire_remove_event(self, state, dict_, value, initiator, + collection_history=None): + if collection_history is None: + collection_history = self._modified_event(state, dict_) + + collection_history.add_removed(value) if self.trackparent and value is not None: self.sethasparent(attributes.instance_state(value), state, False) for fn in self.dispatch.remove: - fn(state, value, initiator or self) + fn(state, value, initiator or self._remove_token) def _modified_event(self, state, dict_): if self.key not in state.committed_state: state.committed_state[self.key] = CollectionHistory(self, state) - state.modified_event(dict_, + state._modified_event(dict_, self, attributes.NEVER_SET) @@ -119,18 +134,31 @@ class DynamicAttributeImpl(attributes.AttributeImpl): return self._set_iterable(state, dict_, value) - def _set_iterable(self, state, dict_, iterable, adapter=None): - collection_history = self._modified_event(state, dict_) new_values = list(iterable) if state.has_identity: - old_collection = list(self.get(state, dict_)) + old_collection = util.IdentitySet(self.get(state, dict_)) + + collection_history = self._modified_event(state, dict_) + if not state.has_identity: + old_collection = collection_history.added_items else: - old_collection = [] - collections.bulk_replace(new_values, DynCollectionAdapter(self, - state, old_collection), - DynCollectionAdapter(self, state, - new_values)) + old_collection = old_collection.union( + collection_history.added_items) + + idset = util.IdentitySet + constants = old_collection.intersection(new_values) + additions = idset(new_values).difference(constants) + removals = old_collection.difference(constants) + + for member in new_values: + if member in additions: + self.fire_append_event(state, dict_, member, None, + collection_history=collection_history) + + for member in removals: + self.fire_remove_event(state, dict_, member, None, + collection_history=collection_history) def delete(self, *args, **kwargs): raise NotImplementedError() @@ -141,15 +169,15 @@ class DynamicAttributeImpl(attributes.AttributeImpl): def get_history(self, state, dict_, passive=attributes.PASSIVE_OFF): c = self._get_collection_history(state, passive) - return attributes.History(c.added_items, c.unchanged_items, - c.deleted_items) + return c.as_history() def get_all_pending(self, state, dict_): - c = self._get_collection_history(state, True) + c = self._get_collection_history( + state, attributes.PASSIVE_NO_INITIALIZE) return [ (attributes.instance_state(x), x) for x in - c.added_items + c.unchanged_items + c.deleted_items + c.all_items ] def _get_collection_history(self, state, passive=attributes.PASSIVE_OFF): @@ -158,7 +186,7 @@ class DynamicAttributeImpl(attributes.AttributeImpl): else: c = CollectionHistory(self, state) - if passive is attributes.PASSIVE_OFF: + if state.has_identity and (passive & attributes.INIT_OK): return CollectionHistory(self, state, apply_to=c) else: return c @@ -173,34 +201,16 @@ class DynamicAttributeImpl(attributes.AttributeImpl): if initiator is not self: self.fire_remove_event(state, dict_, value, initiator) -class DynCollectionAdapter(object): - """the dynamic analogue to orm.collections.CollectionAdapter""" + def pop(self, state, dict_, value, initiator, + passive=attributes.PASSIVE_OFF): + self.remove(state, dict_, value, initiator, passive=passive) - def __init__(self, attr, owner_state, data): - self.attr = attr - self.state = owner_state - self.data = data - - def __iter__(self): - return iter(self.data) - - def append_with_event(self, item, initiator=None): - self.attr.append(self.state, self.state.dict, item, initiator) - - def remove_with_event(self, item, initiator=None): - self.attr.remove(self.state, self.state.dict, item, initiator) - - def append_without_event(self, item): - pass - - def remove_without_event(self, item): - pass class AppenderMixin(object): query_class = None def __init__(self, attr, state): - Query.__init__(self, attr.target_mapper, None) + super(AppenderMixin, self).__init__(attr.target_mapper, None) self.instance = instance = state.obj() self.attr = attr @@ -215,22 +225,19 @@ class AppenderMixin(object): if self.attr.order_by: self._order_by = self.attr.order_by - def __session(self): + def session(self): sess = object_session(self.instance) if sess is not None and self.autoflush and sess.autoflush \ and self.instance in sess: sess.flush() - if not has_identity(self.instance): + if not orm_util.has_identity(self.instance): return None else: return sess - - def session(self): - return self.__session() - session = property(session, lambda s, x:None) + session = property(session, lambda s, x: None) def __iter__(self): - sess = self.__session() + sess = self.session if sess is None: return iter(self.attr._get_collection_history( attributes.instance_state(self.instance), @@ -239,17 +246,16 @@ class AppenderMixin(object): return iter(self._clone(sess)) def __getitem__(self, index): - sess = self.__session() + sess = self.session if sess is None: return self.attr._get_collection_history( attributes.instance_state(self.instance), - attributes.PASSIVE_NO_INITIALIZE).added_items.\ - __getitem__(index) + attributes.PASSIVE_NO_INITIALIZE).indexed(index) else: return self._clone(sess).__getitem__(index) def count(self): - sess = self.__session() + sess = self.session if sess is None: return len(self.attr._get_collection_history( attributes.instance_state(self.instance), @@ -269,7 +275,7 @@ class AppenderMixin(object): "Parent instance %s is not bound to a Session, and no " "contextual session is established; lazy load operation " "of attribute '%s' cannot proceed" % ( - mapperutil.instance_str(instance), self.attr.key)) + orm_util.instance_str(instance), self.attr.key)) if self.query_class: query = self.query_class(self.attr.target_mapper, session=sess) @@ -281,6 +287,12 @@ class AppenderMixin(object): return query + def extend(self, iterator): + for item in iterator: + self.attr.append( + attributes.instance_state(self.instance), + attributes.instance_dict(self.instance), item, None) + def append(self, item): self.attr.append( attributes.instance_state(self.instance), @@ -301,20 +313,56 @@ def mixin_user_query(cls): name = 'Appender' + cls.__name__ return type(name, (AppenderMixin, cls), {'query_class': cls}) + class CollectionHistory(object): """Overrides AttributeHistory to receive append/remove events directly.""" def __init__(self, attr, state, apply_to=None): if apply_to: - deleted = util.IdentitySet(apply_to.deleted_items) - added = apply_to.added_items coll = AppenderQuery(attr, state).autoflush(False) - self.unchanged_items = [o for o in util.IdentitySet(coll) - if o not in deleted] + self.unchanged_items = util.OrderedIdentitySet(coll) self.added_items = apply_to.added_items self.deleted_items = apply_to.deleted_items + self._reconcile_collection = True else: - self.deleted_items = [] - self.added_items = [] - self.unchanged_items = [] + self.deleted_items = util.OrderedIdentitySet() + self.added_items = util.OrderedIdentitySet() + self.unchanged_items = util.OrderedIdentitySet() + self._reconcile_collection = False + + @property + def added_plus_unchanged(self): + return list(self.added_items.union(self.unchanged_items)) + + @property + def all_items(self): + return list(self.added_items.union( + self.unchanged_items).union(self.deleted_items)) + + def as_history(self): + if self._reconcile_collection: + added = self.added_items.difference(self.unchanged_items) + deleted = self.deleted_items.intersection(self.unchanged_items) + unchanged = self.unchanged_items.difference(deleted) + else: + added, unchanged, deleted = self.added_items,\ + self.unchanged_items,\ + self.deleted_items + return attributes.History( + list(added), + list(unchanged), + list(deleted), + ) + + def indexed(self, index): + return list(self.added_items)[index] + + def add_added(self, value): + self.added_items.add(value) + + def add_removed(self, value): + if value in self.added_items: + self.added_items.remove(value) + else: + self.deleted_items.add(value) diff --git a/libs/sqlalchemy/orm/evaluator.py b/libs/sqlalchemy/orm/evaluator.py index 5eaac6c3..e1dd9606 100644 --- a/libs/sqlalchemy/orm/evaluator.py +++ b/libs/sqlalchemy/orm/evaluator.py @@ -1,12 +1,11 @@ # orm/evaluator.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php import operator -from sqlalchemy.sql import operators, functions -from sqlalchemy.sql import expression as sql +from ..sql import operators class UnevaluatableError(Exception): @@ -14,23 +13,23 @@ class UnevaluatableError(Exception): _straight_ops = set(getattr(operators, op) for op in ('add', 'mul', 'sub', - # Py2K 'div', - # end Py2K 'mod', 'truediv', 'lt', 'le', 'ne', 'gt', 'ge', 'eq')) _notimplemented_ops = set(getattr(operators, op) - for op in ('like_op', 'notlike_op', 'ilike_op', - 'notilike_op', 'between_op', 'in_op', - 'notin_op', 'endswith_op', 'concat_op')) + for op in ('like_op', 'notlike_op', 'ilike_op', + 'notilike_op', 'between_op', 'in_op', + 'notin_op', 'endswith_op', 'concat_op')) + class EvaluatorCompiler(object): def process(self, clause): meth = getattr(self, "visit_%s" % clause.__visit_name__, None) if not meth: - raise UnevaluatableError("Cannot evaluate %s" % type(clause).__name__) + raise UnevaluatableError( + "Cannot evaluate %s" % type(clause).__name__) return meth(clause) def visit_grouping(self, clause): @@ -39,6 +38,12 @@ class EvaluatorCompiler(object): def visit_null(self, clause): return lambda obj: None + def visit_false(self, clause): + return lambda obj: False + + def visit_true(self, clause): + return lambda obj: True + def visit_column(self, clause): if 'parentmapper' in clause._annotations: key = clause._annotations['parentmapper'].\ @@ -49,7 +54,7 @@ class EvaluatorCompiler(object): return lambda obj: get_corresponding_attr(obj) def visit_clauselist(self, clause): - evaluators = map(self.process, clause.clauses) + evaluators = list(map(self.process, clause.clauses)) if clause.operator is operators.or_: def evaluate(obj): has_null = False @@ -71,12 +76,15 @@ class EvaluatorCompiler(object): return False return True else: - raise UnevaluatableError("Cannot evaluate clauselist with operator %s" % clause.operator) + raise UnevaluatableError( + "Cannot evaluate clauselist with operator %s" % + clause.operator) return evaluate def visit_binary(self, clause): - eval_left,eval_right = map(self.process, [clause.left, clause.right]) + eval_left, eval_right = list(map(self.process, + [clause.left, clause.right])) operator = clause.operator if operator is operators.is_: def evaluate(obj): @@ -92,7 +100,9 @@ class EvaluatorCompiler(object): return None return operator(eval_left(obj), eval_right(obj)) else: - raise UnevaluatableError("Cannot evaluate %s with operator %s" % (type(clause).__name__, clause.operator)) + raise UnevaluatableError( + "Cannot evaluate %s with operator %s" % + (type(clause).__name__, clause.operator)) return evaluate def visit_unary(self, clause): @@ -104,7 +114,9 @@ class EvaluatorCompiler(object): return None return not value return evaluate - raise UnevaluatableError("Cannot evaluate %s with operator %s" % (type(clause).__name__, clause.operator)) + raise UnevaluatableError( + "Cannot evaluate %s with operator %s" % + (type(clause).__name__, clause.operator)) def visit_bindparam(self, clause): val = clause.value diff --git a/libs/sqlalchemy/orm/events.py b/libs/sqlalchemy/orm/events.py index 3c868d14..a09154dd 100644 --- a/libs/sqlalchemy/orm/events.py +++ b/libs/sqlalchemy/orm/events.py @@ -1,5 +1,5 @@ # orm/events.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,9 +7,15 @@ """ORM event interfaces. """ -from sqlalchemy import event, exc, util -orm = util.importlater("sqlalchemy", "orm") +from .. import event, exc, util +from .base import _mapper_or_none import inspect +import weakref +from . import interfaces +from . import mapperlib, instrumentation +from .session import Session, sessionmaker +from .scoping import scoped_session +from .attributes import QueryableAttribute class InstrumentationEvents(event.Events): """Events related to class instrumentation events. @@ -17,26 +23,68 @@ class InstrumentationEvents(event.Events): The listeners here support being established against any new style class, that is any object that is a subclass of 'type'. Events will then be fired off for events - against that class as well as all subclasses. - 'type' itself is also accepted as a target - in which case the events fire for all classes. + against that class. If the "propagate=True" flag is passed + to event.listen(), the event will fire off for subclasses + of that class as well. + + The Python ``type`` builtin is also accepted as a target, + which when used has the effect of events being emitted + for all classes. + + Note the "propagate" flag here is defaulted to ``True``, + unlike the other class level events where it defaults + to ``False``. This means that new subclasses will also + be the subject of these events, when a listener + is established on a superclass. + + .. versionchanged:: 0.8 - events here will emit based + on comparing the incoming class to the type of class + passed to :func:`.event.listen`. Previously, the + event would fire for any class unconditionally regardless + of what class was sent for listening, despite + documentation which stated the contrary. """ + _target_class_doc = "SomeBaseClass" + _dispatch_target = instrumentation.InstrumentationFactory + + @classmethod def _accept_with(cls, target): if isinstance(target, type): - return orm.instrumentation.instrumentation_registry + return _InstrumentationEventsHold(target) else: return None @classmethod - def _listen(cls, target, identifier, fn, propagate=False): - event.Events._listen(target, identifier, fn, propagate=propagate) + def _listen(cls, event_key, propagate=True): + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn + + def listen(target_cls, *arg): + listen_cls = target() + if propagate and issubclass(target_cls, listen_cls): + return fn(target_cls, *arg) + elif not propagate and target_cls is listen_cls: + return fn(target_cls, *arg) + + def remove(ref): + key = event.registry._EventKey(None, identifier, listen, + instrumentation._instrumentation_factory) + getattr(instrumentation._instrumentation_factory.dispatch, + identifier).remove(key) + + target = weakref.ref(target.class_, remove) + + event_key.\ + with_dispatch_target(instrumentation._instrumentation_factory).\ + with_wrapper(listen).base_listen() @classmethod - def _remove(cls, identifier, target, fn): - raise NotImplementedError("Removal of instrumentation events not yet implemented") + def _clear(cls): + super(InstrumentationEvents, cls)._clear() + instrumentation._instrumentation_factory.dispatch._clear() def class_instrument(self, cls): """Called after the given class is instrumented. @@ -54,10 +102,21 @@ class InstrumentationEvents(event.Events): """ - def attribute_instrument(self, cls, key, inst): """Called when an attribute is instrumented.""" + + +class _InstrumentationEventsHold(object): + """temporary marker object used to transfer from _accept_with() to + _listen() on the InstrumentationEvents class. + + """ + def __init__(self, class_): + self.class_ = class_ + + dispatch = event.dispatcher(InstrumentationEvents) + class InstanceEvents(event.Events): """Define events specific to object lifecycle. @@ -68,21 +127,19 @@ class InstanceEvents(event.Events): def my_load_listener(target, context): print "on load!" - event.listen(SomeMappedClass, 'load', my_load_listener) + event.listen(SomeClass, 'load', my_load_listener) - Available targets include mapped classes, instances of - :class:`.Mapper` (i.e. returned by :func:`.mapper`, - :func:`.class_mapper` and similar), as well as the - :class:`.Mapper` class and :func:`.mapper` function itself - for global event reception:: + Available targets include: - from sqlalchemy.orm import mapper + * mapped classes + * unmapped superclasses of mapped or to-be-mapped classes + (using the ``propagate=True`` flag) + * :class:`.Mapper` objects + * the :class:`.Mapper` class itself and the :func:`.mapper` + function indicate listening for all mappers. - def some_listener(target, context): - log.debug("Instance %s being loaded" % target) - - # attach to all mappers - event.listen(mapper, 'load', some_listener) + .. versionchanged:: 0.8.0 instance events can be associated with + unmapped superclasses of mapped classes. Instance events are closely related to mapper events, but are more specific to the instance and its instrumentation, @@ -92,47 +149,66 @@ class InstanceEvents(event.Events): available to the :func:`.event.listen` function. :param propagate=False: When True, the event listener should - be applied to all inheriting mappers as well as the - mapper which is the target of this listener. + be applied to all inheriting classes as well as the + class which is the target of this listener. :param raw=False: When True, the "target" argument passed to applicable event listener functions will be the instance's :class:`.InstanceState` management object, rather than the mapped instance itself. """ + + _target_class_doc = "SomeClass" + + _dispatch_target = instrumentation.ClassManager + @classmethod - def _accept_with(cls, target): - if isinstance(target, orm.instrumentation.ClassManager): + def _new_classmanager_instance(cls, class_, classmanager): + _InstanceEventsHold.populate(class_, classmanager) + + @classmethod + @util.dependencies("sqlalchemy.orm") + def _accept_with(cls, orm, target): + if isinstance(target, instrumentation.ClassManager): return target - elif isinstance(target, orm.Mapper): + elif isinstance(target, mapperlib.Mapper): return target.class_manager elif target is orm.mapper: - return orm.instrumentation.ClassManager + return instrumentation.ClassManager elif isinstance(target, type): - if issubclass(target, orm.Mapper): - return orm.instrumentation.ClassManager + if issubclass(target, mapperlib.Mapper): + return instrumentation.ClassManager else: - manager = orm.instrumentation.manager_of_class(target) + manager = instrumentation.manager_of_class(target) if manager: return manager + else: + return _InstanceEventsHold(target) return None @classmethod - def _listen(cls, target, identifier, fn, raw=False, propagate=False): + def _listen(cls, event_key, raw=False, propagate=False): + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn + if not raw: orig_fn = fn + def wrap(state, *arg, **kw): return orig_fn(state.obj(), *arg, **kw) fn = wrap + event_key = event_key.with_wrapper(fn) + + event_key.base_listen(propagate=propagate) - event.Events._listen(target, identifier, fn, propagate=propagate) if propagate: for mgr in target.subclass_managers(True): - event.Events._listen(mgr, identifier, fn, True) + event_key.with_dispatch_target(mgr).base_listen(propagate=True) @classmethod - def _remove(cls, identifier, target, fn): - raise NotImplementedError("Removal of instance events not yet implemented") + def _clear(cls): + super(InstanceEvents, cls)._clear() + _InstanceEventsHold._clear() def first_init(self, manager, cls): """Called when the first instance of a particular mapping is called. @@ -256,6 +332,81 @@ class InstanceEvents(event.Events): """ +class _EventsHold(event.RefCollection): + """Hold onto listeners against unmapped, uninstrumented classes. + + Establish _listen() for that class' mapper/instrumentation when + those objects are created for that class. + + """ + def __init__(self, class_): + self.class_ = class_ + + @classmethod + def _clear(cls): + cls.all_holds.clear() + + class HoldEvents(object): + _dispatch_target = None + + @classmethod + def _listen(cls, event_key, raw=False, propagate=False): + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn + + if target.class_ in target.all_holds: + collection = target.all_holds[target.class_] + else: + collection = target.all_holds[target.class_] = {} + + event.registry._stored_in_collection(event_key, target) + collection[event_key._key] = (event_key, raw, propagate) + + if propagate: + stack = list(target.class_.__subclasses__()) + while stack: + subclass = stack.pop(0) + stack.extend(subclass.__subclasses__()) + subject = target.resolve(subclass) + if subject is not None: + event_key.with_dispatch_target(subject).\ + listen(raw=raw, propagate=propagate) + + def remove(self, event_key): + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn + + collection = target.all_holds[target.class_] + del collection[event_key._key] + + @classmethod + def populate(cls, class_, subject): + for subclass in class_.__mro__: + if subclass in cls.all_holds: + collection = cls.all_holds[subclass] + for event_key, raw, propagate in collection.values(): + if propagate or subclass is class_: + # since we can't be sure in what order different classes + # in a hierarchy are triggered with populate(), + # we rely upon _EventsHold for all event + # assignment, instead of using the generic propagate + # flag. + event_key.with_dispatch_target(subject).\ + listen(raw=raw, propagate=False) + + +class _InstanceEventsHold(_EventsHold): + all_holds = weakref.WeakKeyDictionary() + + def resolve(self, class_): + return instrumentation.manager_of_class(class_) + + class HoldInstanceEvents(_EventsHold.HoldEvents, InstanceEvents): + pass + + dispatch = event.dispatcher(HoldInstanceEvents) + + class MapperEvents(event.Events): """Define events specific to mappings. @@ -270,23 +421,22 @@ class MapperEvents(event.Events): "select my_special_function(%d)" % target.special_number) - # associate the listener function with SomeMappedClass, + # associate the listener function with SomeClass, # to execute during the "before_insert" hook - event.listen(SomeMappedClass, 'before_insert', my_before_insert_listener) + event.listen( + SomeClass, 'before_insert', my_before_insert_listener) - Available targets include mapped classes, instances of - :class:`.Mapper` (i.e. returned by :func:`.mapper`, - :func:`.class_mapper` and similar), as well as the - :class:`.Mapper` class and :func:`.mapper` function itself - for global event reception:: + Available targets include: - from sqlalchemy.orm import mapper + * mapped classes + * unmapped superclasses of mapped or to-be-mapped classes + (using the ``propagate=True`` flag) + * :class:`.Mapper` objects + * the :class:`.Mapper` class itself and the :func:`.mapper` + function indicate listening for all mappers. - def some_listener(mapper, connection, target): - log.debug("Instance %s being inserted" % target) - - # attach to all mappers - event.listen(mapper, 'before_insert', some_listener) + .. versionchanged:: 0.8.0 mapper events can be associated with + unmapped superclasses of mapped classes. Mapper events provide hooks into critical sections of the mapper, including those related to object instrumentation, @@ -305,7 +455,8 @@ class MapperEvents(event.Events): available to the :func:`.event.listen` function. :param propagate=False: When True, the event listener should - be applied to all inheriting mappers as well as the + be applied to all inheriting mappers and/or the mappers of + inheriting classes, as well as any mapper which is the target of this listener. :param raw=False: When True, the "target" argument passed to applicable event listener functions will be the @@ -327,47 +478,69 @@ class MapperEvents(event.Events): """ + _target_class_doc = "SomeClass" + _dispatch_target = mapperlib.Mapper + @classmethod - def _accept_with(cls, target): + def _new_mapper_instance(cls, class_, mapper): + _MapperEventsHold.populate(class_, mapper) + + @classmethod + @util.dependencies("sqlalchemy.orm") + def _accept_with(cls, orm, target): if target is orm.mapper: - return orm.Mapper + return mapperlib.Mapper elif isinstance(target, type): - if issubclass(target, orm.Mapper): + if issubclass(target, mapperlib.Mapper): return target else: - return orm.class_mapper(target, compile=False) + mapper = _mapper_or_none(target) + if mapper is not None: + return mapper + else: + return _MapperEventsHold(target) else: return target @classmethod - def _listen(cls, target, identifier, fn, + def _listen(cls, event_key, raw=False, retval=False, propagate=False): + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn if not raw or not retval: if not raw: meth = getattr(cls, identifier) try: - target_index = inspect.getargspec(meth)[0].index('target') - 1 + target_index = \ + inspect.getargspec(meth)[0].index('target') - 1 except ValueError: target_index = None wrapped_fn = fn + def wrap(*arg, **kw): if not raw and target_index is not None: arg = list(arg) arg[target_index] = arg[target_index].obj() if not retval: wrapped_fn(*arg, **kw) - return orm.interfaces.EXT_CONTINUE + return interfaces.EXT_CONTINUE else: return wrapped_fn(*arg, **kw) fn = wrap + event_key = event_key.with_wrapper(wrap) if propagate: for mapper in target.self_and_descendants: - event.Events._listen(mapper, identifier, fn, propagate=True) + event_key.with_dispatch_target(mapper).base_listen(propagate=True) else: - event.Events._listen(target, identifier, fn) + event_key.base_listen() + + @classmethod + def _clear(cls): + super(MapperEvents, cls)._clear() + _MapperEventsHold._clear() def instrument_class(self, mapper, class_): """Receive a class when the mapper is first constructed, @@ -376,8 +549,15 @@ class MapperEvents(event.Events): This event is the earliest phase of mapper construction. Most attributes of the mapper are not yet initialized. - This listener can generally only be applied to the :class:`.Mapper` - class overall. + This listener can either be applied to the :class:`.Mapper` + class overall, or to any un-mapped class which serves as a base + for classes that will be mapped (using the ``propagate=True`` flag):: + + Base = declarative_base() + + @event.listens_for(Base, "instrument_class", propagate=True) + def on_new_class(mapper, cls_): + " ... " :param mapper: the :class:`.Mapper` which is the target of this event. @@ -389,9 +569,9 @@ class MapperEvents(event.Events): """Called when the mapper for the class is fully configured. This event is the latest phase of mapper construction, and - is invoked when the mapped classes are first used, so that relationships - between mappers can be resolved. When the event is called, - the mapper should be in its final state. + is invoked when the mapped classes are first used, so that + relationships between mappers can be resolved. When the event is + called, the mapper should be in its final state. While the configuration event normally occurs automatically, it can be forced to occur ahead of time, in the case where the event @@ -415,9 +595,9 @@ class MapperEvents(event.Events): Theoretically this event is called once per application, but is actually called any time new mappers - have been affected by a :func:`.orm.configure_mappers` call. If new mappings - are constructed after existing ones have already been used, - this event can be called again. + have been affected by a :func:`.orm.configure_mappers` + call. If new mappings are constructed after existing ones have + already been used, this event can be called again. """ @@ -505,7 +685,6 @@ class MapperEvents(event.Events): """ - def populate_instance(self, mapper, context, row, target, **flags): """Receive an instance before that instance has @@ -561,24 +740,26 @@ class MapperEvents(event.Events): .. warning:: Mapper-level flush events are designed to operate **on attributes local to the immediate object being handled - and via SQL operations with the given** :class:`.Connection` **only.** - Handlers here should **not** make alterations to the state of - the :class:`.Session` overall, and in general should not - affect any :func:`.relationship` -mapped attributes, as - session cascade rules will not function properly, nor is it - always known if the related class has already been handled. - Operations that **are not supported in mapper events** include: + and via SQL operations with the given** + :class:`.Connection` **only.** Handlers here should **not** make + alterations to the state of the :class:`.Session` overall, and + in general should not affect any :func:`.relationship` -mapped + attributes, as session cascade rules will not function properly, + nor is it always known if the related class has already been + handled. Operations that **are not supported in mapper + events** include: * :meth:`.Session.add` * :meth:`.Session.delete` * Mapped collection append, add, remove, delete, discard, etc. - * Mapped relationship attribute set/del events, i.e. ``someobject.related = someotherobject`` + * Mapped relationship attribute set/del events, + i.e. ``someobject.related = someotherobject`` Operations which manipulate the state of the object relative to other objects are better handled: - * In the ``__init__()`` method of the mapped object itself, or another method - designed to establish some particular state. + * In the ``__init__()`` method of the mapped object itself, or + another method designed to establish some particular state. * In a ``@validates`` handler, see :ref:`simple_validators` * Within the :meth:`.SessionEvents.before_flush` event. @@ -617,24 +798,26 @@ class MapperEvents(event.Events): .. warning:: Mapper-level flush events are designed to operate **on attributes local to the immediate object being handled - and via SQL operations with the given** :class:`.Connection` **only.** - Handlers here should **not** make alterations to the state of - the :class:`.Session` overall, and in general should not - affect any :func:`.relationship` -mapped attributes, as - session cascade rules will not function properly, nor is it - always known if the related class has already been handled. - Operations that **are not supported in mapper events** include: + and via SQL operations with the given** + :class:`.Connection` **only.** Handlers here should **not** make + alterations to the state of the :class:`.Session` overall, and in + general should not affect any :func:`.relationship` -mapped + attributes, as session cascade rules will not function properly, + nor is it always known if the related class has already been + handled. Operations that **are not supported in mapper + events** include: * :meth:`.Session.add` * :meth:`.Session.delete` * Mapped collection append, add, remove, delete, discard, etc. - * Mapped relationship attribute set/del events, i.e. ``someobject.related = someotherobject`` + * Mapped relationship attribute set/del events, + i.e. ``someobject.related = someotherobject`` Operations which manipulate the state of the object relative to other objects are better handled: - * In the ``__init__()`` method of the mapped object itself, or another method - designed to establish some particular state. + * In the ``__init__()`` method of the mapped object itself, + or another method designed to establish some particular state. * In a ``@validates`` handler, see :ref:`simple_validators` * Within the :meth:`.SessionEvents.before_flush` event. @@ -692,9 +875,9 @@ class MapperEvents(event.Events): .. warning:: Mapper-level flush events are designed to operate **on attributes local to the immediate object being handled - and via SQL operations with the given** :class:`.Connection` **only.** - Handlers here should **not** make alterations to the state of - the :class:`.Session` overall, and in general should not + and via SQL operations with the given** :class:`.Connection` + **only.** Handlers here should **not** make alterations to the + state of the :class:`.Session` overall, and in general should not affect any :func:`.relationship` -mapped attributes, as session cascade rules will not function properly, nor is it always known if the related class has already been handled. @@ -703,13 +886,14 @@ class MapperEvents(event.Events): * :meth:`.Session.add` * :meth:`.Session.delete` * Mapped collection append, add, remove, delete, discard, etc. - * Mapped relationship attribute set/del events, i.e. ``someobject.related = someotherobject`` + * Mapped relationship attribute set/del events, + i.e. ``someobject.related = someotherobject`` Operations which manipulate the state of the object relative to other objects are better handled: - * In the ``__init__()`` method of the mapped object itself, or another method - designed to establish some particular state. + * In the ``__init__()`` method of the mapped object itself, + or another method designed to establish some particular state. * In a ``@validates`` handler, see :ref:`simple_validators` * Within the :meth:`.SessionEvents.before_flush` event. @@ -765,9 +949,9 @@ class MapperEvents(event.Events): .. warning:: Mapper-level flush events are designed to operate **on attributes local to the immediate object being handled - and via SQL operations with the given** :class:`.Connection` **only.** - Handlers here should **not** make alterations to the state of - the :class:`.Session` overall, and in general should not + and via SQL operations with the given** :class:`.Connection` + **only.** Handlers here should **not** make alterations to the + state of the :class:`.Session` overall, and in general should not affect any :func:`.relationship` -mapped attributes, as session cascade rules will not function properly, nor is it always known if the related class has already been handled. @@ -776,13 +960,14 @@ class MapperEvents(event.Events): * :meth:`.Session.add` * :meth:`.Session.delete` * Mapped collection append, add, remove, delete, discard, etc. - * Mapped relationship attribute set/del events, i.e. ``someobject.related = someotherobject`` + * Mapped relationship attribute set/del events, + i.e. ``someobject.related = someotherobject`` Operations which manipulate the state of the object relative to other objects are better handled: - * In the ``__init__()`` method of the mapped object itself, or another method - designed to establish some particular state. + * In the ``__init__()`` method of the mapped object itself, + or another method designed to establish some particular state. * In a ``@validates`` handler, see :ref:`simple_validators` * Within the :meth:`.SessionEvents.before_flush` event. @@ -815,9 +1000,9 @@ class MapperEvents(event.Events): .. warning:: Mapper-level flush events are designed to operate **on attributes local to the immediate object being handled - and via SQL operations with the given** :class:`.Connection` **only.** - Handlers here should **not** make alterations to the state of - the :class:`.Session` overall, and in general should not + and via SQL operations with the given** :class:`.Connection` + **only.** Handlers here should **not** make alterations to the + state of the :class:`.Session` overall, and in general should not affect any :func:`.relationship` -mapped attributes, as session cascade rules will not function properly, nor is it always known if the related class has already been handled. @@ -826,13 +1011,14 @@ class MapperEvents(event.Events): * :meth:`.Session.add` * :meth:`.Session.delete` * Mapped collection append, add, remove, delete, discard, etc. - * Mapped relationship attribute set/del events, i.e. ``someobject.related = someotherobject`` + * Mapped relationship attribute set/del events, + i.e. ``someobject.related = someotherobject`` Operations which manipulate the state of the object relative to other objects are better handled: - * In the ``__init__()`` method of the mapped object itself, or another method - designed to establish some particular state. + * In the ``__init__()`` method of the mapped object itself, + or another method designed to establish some particular state. * In a ``@validates`` handler, see :ref:`simple_validators` * Within the :meth:`.SessionEvents.before_flush` event. @@ -865,9 +1051,9 @@ class MapperEvents(event.Events): .. warning:: Mapper-level flush events are designed to operate **on attributes local to the immediate object being handled - and via SQL operations with the given** :class:`.Connection` **only.** - Handlers here should **not** make alterations to the state of - the :class:`.Session` overall, and in general should not + and via SQL operations with the given** :class:`.Connection` + **only.** Handlers here should **not** make alterations to the + state of the :class:`.Session` overall, and in general should not affect any :func:`.relationship` -mapped attributes, as session cascade rules will not function properly, nor is it always known if the related class has already been handled. @@ -876,13 +1062,14 @@ class MapperEvents(event.Events): * :meth:`.Session.add` * :meth:`.Session.delete` * Mapped collection append, add, remove, delete, discard, etc. - * Mapped relationship attribute set/del events, i.e. ``someobject.related = someotherobject`` + * Mapped relationship attribute set/del events, + i.e. ``someobject.related = someotherobject`` Operations which manipulate the state of the object relative to other objects are better handled: - * In the ``__init__()`` method of the mapped object itself, or another method - designed to establish some particular state. + * In the ``__init__()`` method of the mapped object itself, + or another method designed to establish some particular state. * In a ``@validates`` handler, see :ref:`simple_validators` * Within the :meth:`.SessionEvents.before_flush` event. @@ -900,9 +1087,17 @@ class MapperEvents(event.Events): """ - @classmethod - def _remove(cls, identifier, target, fn): - raise NotImplementedError("Removal of mapper events not yet implemented") +class _MapperEventsHold(_EventsHold): + all_holds = weakref.WeakKeyDictionary() + + def resolve(self, class_): + return _mapper_or_none(class_) + + class HoldMapperEvents(_EventsHold.HoldEvents, MapperEvents): + pass + + dispatch = event.dispatcher(HoldMapperEvents) + class SessionEvents(event.Events): """Define events specific to :class:`.Session` lifecycle. @@ -921,7 +1116,7 @@ class SessionEvents(event.Events): The :func:`~.event.listen` function will accept :class:`.Session` objects as well as the return result - of :func:`.sessionmaker` and :func:`.scoped_session`. + of :class:`~.sessionmaker()` and :class:`~.scoped_session()`. Additionally, it accepts the :class:`.Session` class which will apply listeners to all :class:`.Session` instances @@ -929,48 +1124,137 @@ class SessionEvents(event.Events): """ + _target_class_doc = "SomeSessionOrFactory" + + _dispatch_target = Session + @classmethod def _accept_with(cls, target): - if isinstance(target, orm.ScopedSession): - if not isinstance(target.session_factory, type) or \ - not issubclass(target.session_factory, orm.Session): + if isinstance(target, scoped_session): + + target = target.session_factory + if not isinstance(target, sessionmaker) and \ + ( + not isinstance(target, type) or + not issubclass(target, Session) + ): raise exc.ArgumentError( - "Session event listen on a ScopedSession " + "Session event listen on a scoped_session " "requires that its creation callable " - "is a Session subclass.") - return target.session_factory + "is associated with the Session class.") + + if isinstance(target, sessionmaker): + return target.class_ elif isinstance(target, type): - if issubclass(target, orm.ScopedSession): - return orm.Session - elif issubclass(target, orm.Session): + if issubclass(target, scoped_session): + return Session + elif issubclass(target, Session): return target - elif isinstance(target, orm.Session): + elif isinstance(target, Session): return target else: return None - @classmethod - def _remove(cls, identifier, target, fn): - raise NotImplementedError("Removal of session events not yet implemented") + def after_transaction_create(self, session, transaction): + """Execute when a new :class:`.SessionTransaction` is created. + + This event differs from :meth:`~.SessionEvents.after_begin` + in that it occurs for each :class:`.SessionTransaction` + overall, as opposed to when transactions are begun + on individual database connections. It is also invoked + for nested transactions and subtransactions, and is always + matched by a corresponding + :meth:`~.SessionEvents.after_transaction_end` event + (assuming normal operation of the :class:`.Session`). + + :param session: the target :class:`.Session`. + :param transaction: the target :class:`.SessionTransaction`. + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`~.SessionEvents.after_transaction_end` + + """ + + def after_transaction_end(self, session, transaction): + """Execute when the span of a :class:`.SessionTransaction` ends. + + This event differs from :meth:`~.SessionEvents.after_commit` + in that it corresponds to all :class:`.SessionTransaction` + objects in use, including those for nested transactions + and subtransactions, and is always matched by a corresponding + :meth:`~.SessionEvents.after_transaction_create` event. + + :param session: the target :class:`.Session`. + :param transaction: the target :class:`.SessionTransaction`. + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`~.SessionEvents.after_transaction_create` + + """ def before_commit(self, session): """Execute before commit is called. - Note that this may not be per-flush if a longer running - transaction is ongoing. + .. note:: + + The :meth:`~.SessionEvents.before_commit` hook is *not* per-flush, + that is, the :class:`.Session` can emit SQL to the database + many times within the scope of a transaction. + For interception of these events, use the :meth:`~.SessionEvents.before_flush`, + :meth:`~.SessionEvents.after_flush`, or :meth:`~.SessionEvents.after_flush_postexec` + events. :param session: The target :class:`.Session`. + .. seealso:: + + :meth:`~.SessionEvents.after_commit` + + :meth:`~.SessionEvents.after_begin` + + :meth:`~.SessionEvents.after_transaction_create` + + :meth:`~.SessionEvents.after_transaction_end` + """ def after_commit(self, session): """Execute after a commit has occurred. - Note that this may not be per-flush if a longer running - transaction is ongoing. + .. note:: + + The :meth:`~.SessionEvents.after_commit` hook is *not* per-flush, + that is, the :class:`.Session` can emit SQL to the database + many times within the scope of a transaction. + For interception of these events, use the :meth:`~.SessionEvents.before_flush`, + :meth:`~.SessionEvents.after_flush`, or :meth:`~.SessionEvents.after_flush_postexec` + events. + + .. note:: + + The :class:`.Session` is not in an active tranasction + when the :meth:`~.SessionEvents.after_commit` event is invoked, and therefore + can not emit SQL. To emit SQL corresponding to every transaction, + use the :meth:`~.SessionEvents.before_commit` event. :param session: The target :class:`.Session`. + .. seealso:: + + :meth:`~.SessionEvents.before_commit` + + :meth:`~.SessionEvents.after_begin` + + :meth:`~.SessionEvents.after_transaction_create` + + :meth:`~.SessionEvents.after_transaction_end` + """ def after_rollback(self, session): @@ -1010,16 +1294,16 @@ class SessionEvents(event.Events): session.execute("select * from some_table") :param session: The target :class:`.Session`. - :param previous_transaction: The :class:`.SessionTransaction` transactional - marker object which was just closed. The current :class:`.SessionTransaction` - for the given :class:`.Session` is available via the - :attr:`.Session.transaction` attribute. + :param previous_transaction: The :class:`.SessionTransaction` + transactional marker object which was just closed. The current + :class:`.SessionTransaction` for the given :class:`.Session` is + available via the :attr:`.Session.transaction` attribute. .. versionadded:: 0.7.3 """ - def before_flush( self, session, flush_context, instances): + def before_flush(self, session, flush_context, instances): """Execute before flush process has started. :param session: The target :class:`.Session`. @@ -1029,6 +1313,12 @@ class SessionEvents(event.Events): objects which can be passed to the :meth:`.Session.flush` method (note this usage is deprecated). + .. seealso:: + + :meth:`~.SessionEvents.after_flush` + + :meth:`~.SessionEvents.after_flush_postexec` + """ def after_flush(self, session, flush_context): @@ -1043,6 +1333,12 @@ class SessionEvents(event.Events): :param flush_context: Internal :class:`.UOWTransaction` object which handles the details of the flush. + .. seealso:: + + :meth:`~.SessionEvents.before_flush` + + :meth:`~.SessionEvents.after_flush_postexec` + """ def after_flush_postexec(self, session, flush_context): @@ -1057,48 +1353,124 @@ class SessionEvents(event.Events): :param session: The target :class:`.Session`. :param flush_context: Internal :class:`.UOWTransaction` object which handles the details of the flush. + + + .. seealso:: + + :meth:`~.SessionEvents.before_flush` + + :meth:`~.SessionEvents.after_flush` + """ - def after_begin( self, session, transaction, connection): + def after_begin(self, session, transaction, connection): """Execute after a transaction is begun on a connection :param session: The target :class:`.Session`. :param transaction: The :class:`.SessionTransaction`. - :param connection: The :class:`~.engine.base.Connection` object + :param connection: The :class:`~.engine.Connection` object which will be used for SQL statements. + .. seealso:: + + :meth:`~.SessionEvents.before_commit` + + :meth:`~.SessionEvents.after_commit` + + :meth:`~.SessionEvents.after_transaction_create` + + :meth:`~.SessionEvents.after_transaction_end` + + """ + + def before_attach(self, session, instance): + """Execute before an instance is attached to a session. + + This is called before an add, delete or merge causes + the object to be part of the session. + + .. versionadded:: 0.8. Note that :meth:`~.SessionEvents.after_attach` now + fires off after the item is part of the session. + :meth:`.before_attach` is provided for those cases where + the item should not yet be part of the session state. + + .. seealso:: + + :meth:`~.SessionEvents.after_attach` + """ def after_attach(self, session, instance): """Execute after an instance is attached to a session. - This is called after an add, delete or merge. """ + This is called after an add, delete or merge. - def after_bulk_update( self, session, query, query_context, result): + .. note:: + + As of 0.8, this event fires off *after* the item + has been fully associated with the session, which is + different than previous releases. For event + handlers that require the object not yet + be part of session state (such as handlers which + may autoflush while the target object is not + yet complete) consider the + new :meth:`.before_attach` event. + + .. seealso:: + + :meth:`~.SessionEvents.before_attach` + + """ + + @event._legacy_signature("0.9", + ["session", "query", "query_context", "result"], + lambda update_context: ( + update_context.session, + update_context.query, + update_context.context, + update_context.result)) + def after_bulk_update(self, update_context): """Execute after a bulk update operation to the session. This is called as a result of the :meth:`.Query.update` method. - :param query: the :class:`.Query` object that this update operation was - called upon. - :param query_context: The :class:`.QueryContext` object, corresponding - to the invocation of an ORM query. - :param result: the :class:`.ResultProxy` returned as a result of the - bulk UPDATE operation. + :param update_context: an "update context" object which contains + details about the update, including these attributes: + + * ``session`` - the :class:`.Session` involved + * ``query`` -the :class:`.Query` object that this update operation was + called upon. + * ``context`` The :class:`.QueryContext` object, corresponding + to the invocation of an ORM query. + * ``result`` the :class:`.ResultProxy` returned as a result of the + bulk UPDATE operation. + """ - def after_bulk_delete( self, session, query, query_context, result): + @event._legacy_signature("0.9", + ["session", "query", "query_context", "result"], + lambda delete_context: ( + delete_context.session, + delete_context.query, + delete_context.context, + delete_context.result)) + def after_bulk_delete(self, delete_context): """Execute after a bulk delete operation to the session. This is called as a result of the :meth:`.Query.delete` method. - :param query: the :class:`.Query` object that this update operation was - called upon. - :param query_context: The :class:`.QueryContext` object, corresponding - to the invocation of an ORM query. - :param result: the :class:`.ResultProxy` returned as a result of the - bulk DELETE operation. + :param delete_context: a "delete context" object which contains + details about the update, including these attributes: + + * ``session`` - the :class:`.Session` involved + * ``query`` -the :class:`.Query` object that this update operation was + called upon. + * ``context`` The :class:`.QueryContext` object, corresponding + to the invocation of an ORM query. + * ``result`` the :class:`.ResultProxy` returned as a result of the + bulk DELETE operation. + """ @@ -1132,7 +1504,7 @@ class AttributeEvents(event.Events): listen(UserContact.phone, 'set', validate_phone, retval=True) A validation function like the above can also raise an exception - such as :class:`.ValueError` to halt the operation. + such as :exc:`ValueError` to halt the operation. Several modifiers are available to the :func:`~.event.listen` function. @@ -1160,26 +1532,36 @@ class AttributeEvents(event.Events): """ + _target_class_doc = "SomeClass.some_attribute" + _dispatch_target = QueryableAttribute + + @staticmethod + def _set_dispatch(cls, dispatch_cls): + event.Events._set_dispatch(cls, dispatch_cls) + dispatch_cls._active_history = False + @classmethod def _accept_with(cls, target): # TODO: coverage - if isinstance(target, orm.interfaces.MapperProperty): + if isinstance(target, interfaces.MapperProperty): return getattr(target.parent.class_, target.key) else: return target @classmethod - def _listen(cls, target, identifier, fn, active_history=False, + def _listen(cls, event_key, active_history=False, raw=False, retval=False, propagate=False): + + target, identifier, fn = \ + event_key.dispatch_target, event_key.identifier, event_key.fn + if active_history: target.dispatch._active_history = True - # TODO: for removal, need to package the identity - # of the wrapper with the original function. - if not raw or not retval: orig_fn = fn + def wrap(target, value, *arg): if not raw: target = target.obj() @@ -1189,18 +1571,15 @@ class AttributeEvents(event.Events): else: return orig_fn(target, value, *arg) fn = wrap + event_key = event_key.with_wrapper(wrap) - event.Events._listen(target, identifier, fn, propagate) + event_key.base_listen(propagate=propagate) if propagate: - manager = orm.instrumentation.manager_of_class(target.class_) + manager = instrumentation.manager_of_class(target.class_) for mgr in manager.subclass_managers(True): - event.Events._listen(mgr[target.key], identifier, fn, True) - - @classmethod - def _remove(cls, identifier, target, fn): - raise NotImplementedError("Removal of attribute events not yet implemented") + event_key.with_dispatch_target(mgr[target.key]).base_listen(propagate=True) def append(self, target, value, initiator): """Receive a collection append event. @@ -1212,8 +1591,15 @@ class AttributeEvents(event.Events): is registered with ``retval=True``, the listener function must return this value, or a new value which replaces it. - :param initiator: the attribute implementation object - which initiated this event. + :param initiator: An instance of :class:`.attributes.Event` + representing the initiation of the event. May be modified + from it's original value by backref handlers in order to control + chained event propagation. + + .. versionchanged:: 0.9.0 the ``initiator`` argument is now + passed as a :class:`.attributes.Event` object, and may be modified + by backref handlers within a chain of backref-linked events. + :return: if the event was registered with ``retval=True``, the given value, or a new effective value, should be returned. @@ -1226,8 +1612,15 @@ class AttributeEvents(event.Events): If the listener is registered with ``raw=True``, this will be the :class:`.InstanceState` object. :param value: the value being removed. - :param initiator: the attribute implementation object - which initiated this event. + :param initiator: An instance of :class:`.attributes.Event` + representing the initiation of the event. May be modified + from it's original value by backref handlers in order to control + chained event propagation. + + .. versionchanged:: 0.9.0 the ``initiator`` argument is now + passed as a :class:`.attributes.Event` object, and may be modified + by backref handlers within a chain of backref-linked events. + :return: No return value is defined for this event. """ @@ -1247,8 +1640,15 @@ class AttributeEvents(event.Events): the previous value of the attribute will be loaded from the database if the existing value is currently unloaded or expired. - :param initiator: the attribute implementation object - which initiated this event. + :param initiator: An instance of :class:`.attributes.Event` + representing the initiation of the event. May be modified + from it's original value by backref handlers in order to control + chained event propagation. + + .. versionchanged:: 0.9.0 the ``initiator`` argument is now + passed as a :class:`.attributes.Event` object, and may be modified + by backref handlers within a chain of backref-linked events. + :return: if the event was registered with ``retval=True``, the given value, or a new effective value, should be returned. diff --git a/libs/sqlalchemy/orm/exc.py b/libs/sqlalchemy/orm/exc.py index a116d204..d1ef1ded 100644 --- a/libs/sqlalchemy/orm/exc.py +++ b/libs/sqlalchemy/orm/exc.py @@ -1,18 +1,17 @@ # orm/exc.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """SQLAlchemy ORM exceptions.""" - -import sqlalchemy as sa -orm_util = sa.util.importlater('sqlalchemy.orm', 'util') +from .. import exc as sa_exc, util NO_STATE = (AttributeError, KeyError) """Exception types that may be raised by instrumentation implementations.""" -class StaleDataError(sa.exc.SQLAlchemyError): + +class StaleDataError(sa_exc.SQLAlchemyError): """An operation encountered database state that is unaccounted for. Conditions which cause this to happen include: @@ -41,31 +40,39 @@ class StaleDataError(sa.exc.SQLAlchemyError): ConcurrentModificationError = StaleDataError -class FlushError(sa.exc.SQLAlchemyError): +class FlushError(sa_exc.SQLAlchemyError): """A invalid condition was detected during flush().""" -class UnmappedError(sa.exc.InvalidRequestError): +class UnmappedError(sa_exc.InvalidRequestError): """Base for exceptions that involve expected mappings not present.""" -class ObjectDereferencedError(sa.exc.SQLAlchemyError): - """An operation cannot complete due to an object being garbage collected.""" -class DetachedInstanceError(sa.exc.SQLAlchemyError): +class ObjectDereferencedError(sa_exc.SQLAlchemyError): + """An operation cannot complete due to an object being garbage + collected. + + """ + + +class DetachedInstanceError(sa_exc.SQLAlchemyError): """An attempt to access unloaded attributes on a mapped instance that is detached.""" + class UnmappedInstanceError(UnmappedError): """An mapping operation was requested for an unknown instance.""" - def __init__(self, obj, msg=None): + @util.dependencies("sqlalchemy.orm.base") + def __init__(self, base, obj, msg=None): if not msg: try: - mapper = sa.orm.class_mapper(type(obj)) + base.class_mapper(type(obj)) name = _safe_cls_name(type(obj)) msg = ("Class %r is mapped, but this instance lacks " - "instrumentation. This occurs when the instance is created " - "before sqlalchemy.orm.mapper(%s) was called." % (name, name)) + "instrumentation. This occurs when the instance" + "is created before sqlalchemy.orm.mapper(%s) " + "was called." % (name, name)) except UnmappedClassError: msg = _default_unmapped(type(obj)) if isinstance(obj, type): @@ -77,6 +84,7 @@ class UnmappedInstanceError(UnmappedError): def __reduce__(self): return self.__class__, (None, self.args[0]) + class UnmappedClassError(UnmappedError): """An mapping operation was requested for an unknown class.""" @@ -88,7 +96,8 @@ class UnmappedClassError(UnmappedError): def __reduce__(self): return self.__class__, (None, self.args[0]) -class ObjectDeletedError(sa.exc.InvalidRequestError): + +class ObjectDeletedError(sa_exc.InvalidRequestError): """A refresh operation failed to retrieve the database row corresponding to an object's known primary key identity. @@ -107,33 +116,30 @@ class ObjectDeletedError(sa.exc.InvalidRequestError): object. """ - def __init__(self, state, msg=None): + @util.dependencies("sqlalchemy.orm.base") + def __init__(self, base, state, msg=None): if not msg: msg = "Instance '%s' has been deleted, or its "\ - "row is otherwise not present." % orm_util.state_str(state) + "row is otherwise not present." % base.state_str(state) - sa.exc.InvalidRequestError.__init__(self, msg) + sa_exc.InvalidRequestError.__init__(self, msg) def __reduce__(self): return self.__class__, (None, self.args[0]) -class UnmappedColumnError(sa.exc.InvalidRequestError): + +class UnmappedColumnError(sa_exc.InvalidRequestError): """Mapping operation was requested on an unknown column.""" -class NoResultFound(sa.exc.InvalidRequestError): +class NoResultFound(sa_exc.InvalidRequestError): """A database result was required but none was found.""" -class MultipleResultsFound(sa.exc.InvalidRequestError): +class MultipleResultsFound(sa_exc.InvalidRequestError): """A single database result was required but more than one were found.""" -# Legacy compat until 0.6. -sa.exc.ConcurrentModificationError = ConcurrentModificationError -sa.exc.FlushError = FlushError -sa.exc.UnmappedColumnError - def _safe_cls_name(cls): try: cls_name = '.'.join((cls.__module__, cls.__name__)) @@ -143,9 +149,10 @@ def _safe_cls_name(cls): cls_name = repr(cls) return cls_name -def _default_unmapped(cls): +@util.dependencies("sqlalchemy.orm.base") +def _default_unmapped(base, cls): try: - mappers = sa.orm.attributes.manager_of_class(cls).mappers + mappers = base.manager_of_class(cls).mappers except NO_STATE: mappers = {} except TypeError: diff --git a/libs/sqlalchemy/orm/identity.py b/libs/sqlalchemy/orm/identity.py index 29e13f2c..a91085d2 100644 --- a/libs/sqlalchemy/orm/identity.py +++ b/libs/sqlalchemy/orm/identity.py @@ -1,16 +1,15 @@ # orm/identity.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php import weakref -from sqlalchemy.orm import attributes - +from . import attributes +from .. import util class IdentityMap(dict): def __init__(self): - self._mutable_attrs = set() self._modified = set() self._wr = weakref.ref(self) @@ -31,28 +30,20 @@ class IdentityMap(dict): if state.modified: self._modified.add(state) - if state.manager.mutable_attributes: - self._mutable_attrs.add(state) def _manage_removed_state(self, state): del state._instance_dict - self._mutable_attrs.discard(state) self._modified.discard(state) def _dirty_states(self): - return self._modified.union(s for s in self._mutable_attrs.copy() - if s.modified) + return self._modified def check_modified(self): - """return True if any InstanceStates present have been marked as 'modified'.""" + """return True if any InstanceStates present have been marked + as 'modified'. - if self._modified: - return True - else: - for state in self._mutable_attrs.copy(): - if state.modified: - return True - return False + """ + return bool(self._modified) def has_key(self, key): return key in self @@ -75,6 +66,7 @@ class IdentityMap(dict): def __delitem__(self, key): raise NotImplementedError("IdentityMap uses remove() to remove data") + class WeakInstanceDict(IdentityMap): def __init__(self): IdentityMap.__init__(self) @@ -83,9 +75,7 @@ class WeakInstanceDict(IdentityMap): state = dict.__getitem__(self, key) o = state.obj() if o is None: - o = state._is_really_none() - if o is None: - raise KeyError, key + raise KeyError(key) return o def __contains__(self, key): @@ -93,8 +83,6 @@ class WeakInstanceDict(IdentityMap): if dict.__contains__(self, key): state = dict.__getitem__(self, key) o = state.obj() - if o is None: - o = state._is_really_none() else: return False except KeyError: @@ -124,12 +112,11 @@ class WeakInstanceDict(IdentityMap): existing_state = dict.__getitem__(self, key) if existing_state is not state: o = existing_state.obj() - if o is None: - o = existing_state._is_really_none() if o is not None: - raise AssertionError("A conflicting state is already " - "present in the identity map for key %r" - % (key, )) + raise AssertionError( + "A conflicting state is already " + "present in the identity map for key %r" + % (key, )) else: return except KeyError: @@ -143,9 +130,7 @@ class WeakInstanceDict(IdentityMap): return default o = state.obj() if o is None: - o = state._is_really_none() - if o is None: - return default + return default return o def _items(self): @@ -167,28 +152,27 @@ class WeakInstanceDict(IdentityMap): return result - # Py3K - #def items(self): - # return iter(self._items()) - # - #def values(self): - # return iter(self._values()) - # Py2K - items = _items - def iteritems(self): - return iter(self.items()) + if util.py2k: + items = _items + values = _values - values = _values - def itervalues(self): - return iter(self.values()) - # end Py2K + def iteritems(self): + return iter(self.items()) + + def itervalues(self): + return iter(self.values()) + else: + def items(self): + return iter(self._items()) + + def values(self): + return iter(self._values()) def all_states(self): - # Py3K - # return list(dict.values(self)) - # Py2K - return dict.values(self) - # end Py2K + if util.py2k: + return dict.values(self) + else: + return list(dict.values(self)) def discard(self, state): st = dict.get(self, state.key, None) @@ -199,12 +183,15 @@ class WeakInstanceDict(IdentityMap): def prune(self): return 0 + class StrongInstanceDict(IdentityMap): def all_states(self): - return [attributes.instance_state(o) for o in self.itervalues()] + return [attributes.instance_state(o) for o in self.values()] def contains_state(self, state): - return state.key in self and attributes.instance_state(self[state.key]) is state + return ( + state.key in self and + attributes.instance_state(self[state.key]) is state) def replace(self, state): if dict.__contains__(self, state.key): @@ -251,4 +238,3 @@ class StrongInstanceDict(IdentityMap): dict.update(self, keepers) self.modified = bool(dirty) return ref_count - len(self) - diff --git a/libs/sqlalchemy/orm/instrumentation.py b/libs/sqlalchemy/orm/instrumentation.py index 0006accb..68b4f061 100644 --- a/libs/sqlalchemy/orm/instrumentation.py +++ b/libs/sqlalchemy/orm/instrumentation.py @@ -1,5 +1,5 @@ # orm/instrumentation.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -14,79 +14,41 @@ for state tracking. It interacts closely with state.py and attributes.py which establish per-instance and per-class-attribute instrumentation, respectively. -SQLA's instrumentation system is completely customizable, in which -case an understanding of the general mechanics of this module is helpful. -An example of full customization is in /examples/custom_attributes. +The class instrumentation system can be customized on a per-class +or global basis using the :mod:`sqlalchemy.ext.instrumentation` +module, which provides the means to build and specify +alternate instrumentation forms. + +.. versionchanged: 0.8 + The instrumentation extension system was moved out of the + ORM and into the external :mod:`sqlalchemy.ext.instrumentation` + package. When that package is imported, it installs + itself within sqlalchemy.orm so that its more comprehensive + resolution mechanics take effect. """ -from sqlalchemy.orm import exc, collections, events -from operator import attrgetter, itemgetter -from sqlalchemy import event, util -import weakref -from sqlalchemy.orm import state, attributes - - -INSTRUMENTATION_MANAGER = '__sa_instrumentation_manager__' -"""Attribute, elects custom instrumentation when present on a mapped class. - -Allows a class to specify a slightly or wildly different technique for -tracking changes made to mapped attributes and collections. - -Only one instrumentation implementation is allowed in a given object -inheritance hierarchy. - -The value of this attribute must be a callable and will be passed a class -object. The callable must return one of: - - - An instance of an interfaces.InstrumentationManager or subclass - - An object implementing all or some of InstrumentationManager (TODO) - - A dictionary of callables, implementing all or some of the above (TODO) - - An instance of a ClassManager or subclass - -interfaces.InstrumentationManager is public API and will remain stable -between releases. ClassManager is not public and no guarantees are made -about stability. Caveat emptor. - -This attribute is consulted by the default SQLAlchemy instrumentation -resolution code. If custom finders are installed in the global -instrumentation_finders list, they may or may not choose to honor this -attribute. - -""" - -instrumentation_finders = [] -"""An extensible sequence of instrumentation implementation finding callables. - -Finders callables will be passed a class object. If None is returned, the -next finder in the sequence is consulted. Otherwise the return must be an -instrumentation factory that follows the same guidelines as -INSTRUMENTATION_MANAGER. - -By default, the only finder is find_native_user_instrumentation_hook, which -searches for INSTRUMENTATION_MANAGER. If all finders return None, standard -ClassManager instrumentation is used. - -""" - +from . import exc, collections, interfaces, state +from .. import util +from . import base class ClassManager(dict): """tracks state information at the class level.""" - MANAGER_ATTR = '_sa_class_manager' - STATE_ATTR = '_sa_instance_state' + MANAGER_ATTR = base.DEFAULT_MANAGER_ATTR + STATE_ATTR = base.DEFAULT_STATE_ATTR deferred_scalar_loader = None original_init = object.__init__ + factory = None + def __init__(self, class_): self.class_ = class_ - self.factory = None # where we came from, for inheritance bookkeeping self.info = {} self.new_init = None - self.mutable_attributes = set() self.local_attrs = {} self.originals = {} @@ -99,10 +61,28 @@ class ClassManager(dict): for base in self._bases: self.update(base) + self.dispatch._events._new_classmanager_instance(class_, self) + #events._InstanceEventsHold.populate(class_, self) + + for basecls in class_.__mro__: + mgr = manager_of_class(basecls) + if mgr is not None: + self.dispatch._update(mgr.dispatch) self.manage() self._instrument_init() - dispatch = event.dispatcher(events.InstanceEvents) + if '__del__' in class_.__dict__: + util.warn("__del__() method on class %s will " + "cause unreachable cycles and memory leaks, " + "as SQLAlchemy instrumentation often creates " + "reference cycles. Please remove this method." % + class_) + + def __hash__(self): + return id(self) + + def __eq__(self, other): + return other is self @property def is_mapped(self): @@ -113,6 +93,24 @@ class ClassManager(dict): # raises unless self.mapper has been assigned raise exc.UnmappedClassError(self.class_) + def _all_sqla_attributes(self, exclude=None): + """return an iterator of all classbound attributes that are + implement :class:`._InspectionAttr`. + + This includes :class:`.QueryableAttribute` as well as extension + types such as :class:`.hybrid_property` and :class:`.AssociationProxy`. + + """ + if exclude is None: + exclude = set() + for supercls in self.class_.__mro__: + for key in set(supercls.__dict__).difference(exclude): + exclude.add(key) + val = supercls.__dict__[key] + if isinstance(val, interfaces._InspectionAttr): + yield key, val + + def _attr_has_impl(self, key): """Return True if the given attribute is fully initialized. @@ -134,7 +132,7 @@ class ClassManager(dict): """ manager = manager_of_class(cls) if manager is None: - manager = _create_manager_for_cls(cls, _source=self) + manager = _instrumentation_factory.create_manager_for_cls(cls) return manager def _instrument_init(self): @@ -155,10 +153,7 @@ class ClassManager(dict): @util.memoized_property def _state_constructor(self): self.dispatch.first_init(self, self.class_) - if self.mutable_attributes: - return state.MutableAttrInstanceState - else: - return state.InstanceState + return state.InstanceState def manage(self): """Mark this instance as the manager for its class.""" @@ -170,8 +165,25 @@ class ClassManager(dict): delattr(self.class_, self.MANAGER_ATTR) + @util.hybridmethod def manager_getter(self): - return attrgetter(self.MANAGER_ATTR) + return _default_manager_getter + + @util.hybridmethod + def state_getter(self): + """Return a (instance) -> InstanceState callable. + + "state getter" callables should raise either KeyError or + AttributeError if no InstanceState could be found for the + instance. + """ + + return _default_state_getter + + @util.hybridmethod + def dict_getter(self): + return _default_dict_getter + def instrument_attribute(self, key, inst, propagated=False): if propagated: @@ -196,7 +208,7 @@ class ClassManager(dict): yield m def post_configure_attribute(self, key): - instrumentation_registry.dispatch.\ + _instrumentation_factory.dispatch.\ attribute_instrument(self.class_, key, self[key]) def uninstrument_attribute(self, key, propagated=False): @@ -209,8 +221,6 @@ class ClassManager(dict): del self.local_attrs[key] self.uninstall_descriptor(key) del self[key] - if key in self.mutable_attributes: - self.mutable_attributes.remove(key) for cls in self.class_.__subclasses__(): manager = manager_of_class(cls) if manager: @@ -271,7 +281,7 @@ class ClassManager(dict): @property def attributes(self): - return self.itervalues() + return iter(self.values()) ## InstanceState management @@ -288,6 +298,9 @@ class ClassManager(dict): def teardown_instance(self, instance): delattr(instance, self.STATE_ATTR) + def _serialize(self, state, state_dict): + return _SerializeManager(state, state_dict) + def _new_state_if_none(self, instance): """Install a default InstanceState if none is present. @@ -310,19 +323,6 @@ class ClassManager(dict): setattr(instance, self.STATE_ATTR, state) return state - def state_getter(self): - """Return a (instance) -> InstanceState callable. - - "state getter" callables should raise either KeyError or - AttributeError if no InstanceState could be found for the - instance. - """ - - return attrgetter(self.STATE_ATTR) - - def dict_getter(self): - return attrgetter('__dict__') - def has_state(self, instance): return hasattr(instance, self.STATE_ATTR) @@ -330,123 +330,116 @@ class ClassManager(dict): """TODO""" return self.get_impl(key).hasparent(state, optimistic=optimistic) - def __nonzero__(self): + def __bool__(self): """All ClassManagers are non-zero regardless of attribute state.""" return True + __nonzero__ = __bool__ + def __repr__(self): return '<%s of %r at %x>' % ( self.__class__.__name__, self.class_, id(self)) -class _ClassInstrumentationAdapter(ClassManager): - """Adapts a user-defined InstrumentationManager to a ClassManager.""" +class _SerializeManager(object): + """Provide serialization of a :class:`.ClassManager`. - def __init__(self, class_, override, **kw): - self._adapted = override - self._get_state = self._adapted.state_getter(class_) - self._get_dict = self._adapted.dict_getter(class_) + The :class:`.InstanceState` uses ``__init__()`` on serialize + and ``__call__()`` on deserialize. - ClassManager.__init__(self, class_, **kw) + """ + def __init__(self, state, d): + self.class_ = state.class_ + manager = state.manager + manager.dispatch.pickle(state, d) - def manage(self): - self._adapted.manage(self.class_, self) + def __call__(self, state, inst, state_dict): + state.manager = manager = manager_of_class(self.class_) + if manager is None: + raise exc.UnmappedInstanceError( + inst, + "Cannot deserialize object of type %r - " + "no mapper() has " + "been configured for this class within the current " + "Python process!" % + self.class_) + elif manager.is_mapped and not manager.mapper.configured: + manager.mapper._configure_all() - def dispose(self): - self._adapted.dispose(self.class_) + # setup _sa_instance_state ahead of time so that + # unpickle events can access the object normally. + # see [ticket:2362] + if inst is not None: + manager.setup_instance(inst, state) + manager.dispatch.unpickle(state, state_dict) - def manager_getter(self): - return self._adapted.manager_getter(self.class_) +class InstrumentationFactory(object): + """Factory for new ClassManager instances.""" - def instrument_attribute(self, key, inst, propagated=False): - ClassManager.instrument_attribute(self, key, inst, propagated) - if not propagated: - self._adapted.instrument_attribute(self.class_, key, inst) + def create_manager_for_cls(self, class_): + assert class_ is not None + assert manager_of_class(class_) is None - def post_configure_attribute(self, key): - super(_ClassInstrumentationAdapter, self).post_configure_attribute(key) - self._adapted.post_configure_attribute(self.class_, key, self[key]) + # give a more complicated subclass + # a chance to do what it wants here + manager, factory = self._locate_extended_factory(class_) - def install_descriptor(self, key, inst): - self._adapted.install_descriptor(self.class_, key, inst) + if factory is None: + factory = ClassManager + manager = factory(class_) - def uninstall_descriptor(self, key): - self._adapted.uninstall_descriptor(self.class_, key) + self._check_conflicts(class_, factory) - def install_member(self, key, implementation): - self._adapted.install_member(self.class_, key, implementation) + manager.factory = factory - def uninstall_member(self, key): - self._adapted.uninstall_member(self.class_, key) + self.dispatch.class_instrument(class_) + return manager - def instrument_collection_class(self, key, collection_class): - return self._adapted.instrument_collection_class( - self.class_, key, collection_class) + def _locate_extended_factory(self, class_): + """Overridden by a subclass to do an extended lookup.""" + return None, None - def initialize_collection(self, key, state, factory): - delegate = getattr(self._adapted, 'initialize_collection', None) - if delegate: - return delegate(key, state, factory) - else: - return ClassManager.initialize_collection(self, key, - state, factory) + def _check_conflicts(self, class_, factory): + """Overridden by a subclass to test for conflicting factories.""" + return - def new_instance(self, state=None): - instance = self.class_.__new__(self.class_) - self.setup_instance(instance, state) - return instance + def unregister(self, class_): + manager = manager_of_class(class_) + manager.unregister() + manager.dispose() + self.dispatch.class_uninstrument(class_) + if ClassManager.MANAGER_ATTR in class_.__dict__: + delattr(class_, ClassManager.MANAGER_ATTR) - def _new_state_if_none(self, instance): - """Install a default InstanceState if none is present. +# this attribute is replaced by sqlalchemy.ext.instrumentation +# when importred. +_instrumentation_factory = InstrumentationFactory() - A private convenience method used by the __init__ decorator. - """ - if self.has_state(instance): - return False - else: - return self.setup_instance(instance) +# these attributes are replaced by sqlalchemy.ext.instrumentation +# when a non-standard InstrumentationManager class is first +# used to instrument a class. +instance_state = _default_state_getter = base.instance_state - def setup_instance(self, instance, state=None): - self._adapted.initialize_instance_dict(self.class_, instance) +instance_dict = _default_dict_getter = base.instance_dict - if state is None: - state = self._state_constructor(instance, self) +manager_of_class = _default_manager_getter = base.manager_of_class - # the given instance is assumed to have no state - self._adapted.install_state(self.class_, instance, state) - return state - - def teardown_instance(self, instance): - self._adapted.remove_state(self.class_, instance) - - def has_state(self, instance): - try: - state = self._get_state(instance) - except exc.NO_STATE: - return False - else: - return True - - def state_getter(self): - return self._get_state - - def dict_getter(self): - return self._get_dict - -def register_class(class_, **kw): +def register_class(class_): """Register class instrumentation. Returns the existing or newly created class manager. + """ manager = manager_of_class(class_) if manager is None: - manager = _create_manager_for_cls(class_, **kw) + manager = _instrumentation_factory.create_manager_for_cls(class_) return manager + def unregister_class(class_): """Unregister class instrumentation.""" - instrumentation_registry.unregister(class_) + _instrumentation_factory.unregister(class_) def is_instrumented(instance, key): @@ -460,174 +453,6 @@ def is_instrumented(instance, key): return manager_of_class(instance.__class__).\ is_instrumented(key, search=True) -class InstrumentationRegistry(object): - """Private instrumentation registration singleton. - - All classes are routed through this registry - when first instrumented, however the InstrumentationRegistry - is not actually needed unless custom ClassManagers are in use. - - """ - - _manager_finders = weakref.WeakKeyDictionary() - _state_finders = util.WeakIdentityMapping() - _dict_finders = util.WeakIdentityMapping() - _extended = False - - dispatch = event.dispatcher(events.InstrumentationEvents) - - def create_manager_for_cls(self, class_, **kw): - assert class_ is not None - assert manager_of_class(class_) is None - - for finder in instrumentation_finders: - factory = finder(class_) - if factory is not None: - break - else: - factory = ClassManager - - existing_factories = self._collect_management_factories_for(class_).\ - difference([factory]) - if existing_factories: - raise TypeError( - "multiple instrumentation implementations specified " - "in %s inheritance hierarchy: %r" % ( - class_.__name__, list(existing_factories))) - - manager = factory(class_) - if not isinstance(manager, ClassManager): - manager = _ClassInstrumentationAdapter(class_, manager) - - if factory != ClassManager and not self._extended: - # somebody invoked a custom ClassManager. - # reinstall global "getter" functions with the more - # expensive ones. - self._extended = True - _install_lookup_strategy(self) - - manager.factory = factory - self._manager_finders[class_] = manager.manager_getter() - self._state_finders[class_] = manager.state_getter() - self._dict_finders[class_] = manager.dict_getter() - - self.dispatch.class_instrument(class_) - - return manager - - def _collect_management_factories_for(self, cls): - """Return a collection of factories in play or specified for a - hierarchy. - - Traverses the entire inheritance graph of a cls and returns a - collection of instrumentation factories for those classes. Factories - are extracted from active ClassManagers, if available, otherwise - instrumentation_finders is consulted. - - """ - hierarchy = util.class_hierarchy(cls) - factories = set() - for member in hierarchy: - manager = manager_of_class(member) - if manager is not None: - factories.add(manager.factory) - else: - for finder in instrumentation_finders: - factory = finder(member) - if factory is not None: - break - else: - factory = None - factories.add(factory) - factories.discard(None) - return factories - - def manager_of_class(self, cls): - # this is only called when alternate instrumentation - # has been established - if cls is None: - return None - try: - finder = self._manager_finders[cls] - except KeyError: - return None - else: - return finder(cls) - - def state_of(self, instance): - # this is only called when alternate instrumentation - # has been established - if instance is None: - raise AttributeError("None has no persistent state.") - try: - return self._state_finders[instance.__class__](instance) - except KeyError: - raise AttributeError("%r is not instrumented" % - instance.__class__) - - def dict_of(self, instance): - # this is only called when alternate instrumentation - # has been established - if instance is None: - raise AttributeError("None has no persistent state.") - try: - return self._dict_finders[instance.__class__](instance) - except KeyError: - raise AttributeError("%r is not instrumented" % - instance.__class__) - - def unregister(self, class_): - if class_ in self._manager_finders: - manager = self.manager_of_class(class_) - self.dispatch.class_uninstrument(class_) - manager.unregister() - manager.dispose() - del self._manager_finders[class_] - del self._state_finders[class_] - del self._dict_finders[class_] - if ClassManager.MANAGER_ATTR in class_.__dict__: - delattr(class_, ClassManager.MANAGER_ATTR) - -instrumentation_registry = InstrumentationRegistry() - - -def _install_lookup_strategy(implementation): - """Replace global class/object management functions - with either faster or more comprehensive implementations, - based on whether or not extended class instrumentation - has been detected. - - This function is called only by InstrumentationRegistry() - and unit tests specific to this behavior. - - """ - global instance_state, instance_dict, manager_of_class - if implementation is util.symbol('native'): - instance_state = attrgetter(ClassManager.STATE_ATTR) - instance_dict = attrgetter("__dict__") - def manager_of_class(cls): - return cls.__dict__.get(ClassManager.MANAGER_ATTR, None) - else: - instance_state = instrumentation_registry.state_of - instance_dict = instrumentation_registry.dict_of - manager_of_class = instrumentation_registry.manager_of_class - attributes.instance_state = instance_state - attributes.instance_dict = instance_dict - attributes.manager_of_class = manager_of_class - -_create_manager_for_cls = instrumentation_registry.create_manager_for_cls - -# Install default "lookup" strategies. These are basically -# very fast attrgetters for key attributes. -# When a custom ClassManager is installed, more expensive per-class -# strategies are copied over these. -_install_lookup_strategy(util.symbol('native')) - - -def find_native_user_instrumentation_hook(cls): - """Find user-specified instrumentation management for a class.""" - return getattr(cls, INSTRUMENTATION_MANAGER, None) -instrumentation_finders.append(find_native_user_instrumentation_hook) def _generate_init(class_, class_manager): """Build an __init__ decorator that triggers ClassManager events.""" @@ -647,28 +472,28 @@ def _generate_init(class_, class_manager): def __init__(%(apply_pos)s): new_state = class_manager._new_state_if_none(%(self_arg)s) if new_state: - return new_state.initialize_instance(%(apply_kw)s) + return new_state._initialize_instance(%(apply_kw)s) else: return original__init__(%(apply_kw)s) """ func_vars = util.format_argspec_init(original__init__, grouped=False) func_text = func_body % func_vars - # Py3K - #func_defaults = getattr(original__init__, '__defaults__', None) - #func_kw_defaults = getattr(original__init__, '__kwdefaults__', None) - # Py2K - func = getattr(original__init__, 'im_func', original__init__) - func_defaults = getattr(func, 'func_defaults', None) - # end Py2K + if util.py2k: + func = getattr(original__init__, 'im_func', original__init__) + func_defaults = getattr(func, 'func_defaults', None) + else: + func_defaults = getattr(original__init__, '__defaults__', None) + func_kw_defaults = getattr(original__init__, '__kwdefaults__', None) env = locals().copy() - exec func_text in env + exec(func_text, env) __init__ = env['__init__'] __init__.__doc__ = original__init__.__doc__ + if func_defaults: - __init__.func_defaults = func_defaults - # Py3K - #if func_kw_defaults: - # __init__.__kwdefaults__ = func_kw_defaults + __init__.__defaults__ = func_defaults + if not util.py2k and func_kw_defaults: + __init__.__kwdefaults__ = func_kw_defaults + return __init__ diff --git a/libs/sqlalchemy/orm/interfaces.py b/libs/sqlalchemy/orm/interfaces.py index b911ac29..3d5559be 100644 --- a/libs/sqlalchemy/orm/interfaces.py +++ b/libs/sqlalchemy/orm/interfaces.py @@ -1,5 +1,5 @@ # orm/interfaces.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -16,16 +16,16 @@ classes within should be considered mostly private. """ -from itertools import chain +from __future__ import absolute_import -from sqlalchemy import exc as sa_exc -from sqlalchemy import util -from sqlalchemy.sql import operators -deque = __import__('collections').deque +from .. import exc as sa_exc, util, inspect +from ..sql import operators +from collections import deque +from .base import ONETOMANY, MANYTOONE, MANYTOMANY, EXT_CONTINUE, EXT_STOP, NOT_EXTENSION +from .base import _InspectionAttr, _MappedAttribute +from .path_registry import PathRegistry +import collections -mapperutil = util.importlater('sqlalchemy.orm', 'util') - -collections = None __all__ = ( 'AttributeExtension', @@ -42,21 +42,11 @@ __all__ = ( 'SessionExtension', 'StrategizedOption', 'StrategizedProperty', - 'build_path', ) -EXT_CONTINUE = util.symbol('EXT_CONTINUE') -EXT_STOP = util.symbol('EXT_STOP') - -ONETOMANY = util.symbol('ONETOMANY') -MANYTOONE = util.symbol('MANYTOONE') -MANYTOMANY = util.symbol('MANYTOMANY') - -from deprecated_interfaces import AttributeExtension, SessionExtension, \ - MapperExtension -class MapperProperty(object): +class MapperProperty(_MappedAttribute, _InspectionAttr): """Manage the relationship of a ``Mapper`` to a single class attribute, as well as that attribute as it appears on individual instances of the class, including attribute instrumentation, @@ -66,18 +56,21 @@ class MapperProperty(object): mapped :class:`.Column`, which is represented in a mapping as an instance of :class:`.ColumnProperty`, and a reference to another class produced by :func:`.relationship`, - represented in the mapping as an instance of :class:`.RelationshipProperty`. + represented in the mapping as an instance of + :class:`.RelationshipProperty`. """ - cascade = () + cascade = frozenset() """The set of 'cascade' attribute names. This collection is checked before the 'cascade_iterator' method is called. """ - def setup(self, context, entity, path, reduced_path, adapter, **kwargs): + is_property = True + + def setup(self, context, entity, path, adapter, **kwargs): """Called by Query for the purposes of constructing a SQL statement. Each MapperProperty associated with the target mapper processes the @@ -87,7 +80,7 @@ class MapperProperty(object): pass - def create_row_processor(self, context, path, reduced_path, + def create_row_processor(self, context, path, mapper, row, adapter): """Return a 3-tuple consisting of three row processing functions. @@ -112,11 +105,33 @@ class MapperProperty(object): def set_parent(self, parent, init): self.parent = parent - def instrument_class(self, mapper): + def instrument_class(self, mapper): # pragma: no-coverage raise NotImplementedError() - _compile_started = False - _compile_finished = False + @util.memoized_property + def info(self): + """Info dictionary associated with the object, allowing user-defined + data to be associated with this :class:`.MapperProperty`. + + The dictionary is generated when first accessed. Alternatively, + it can be specified as a constructor argument to the + :func:`.column_property`, :func:`.relationship`, or :func:`.composite` + functions. + + .. versionadded:: 0.8 Added support for .info to all + :class:`.MapperProperty` subclasses. + + .. seealso:: + + :attr:`.QueryableAttribute.info` + + :attr:`.SchemaItem.info` + + """ + return {} + + _configure_started = False + _configure_finished = False def init(self): """Called after all mappers are created to assemble @@ -124,14 +139,33 @@ class MapperProperty(object): initialization steps. """ - self._compile_started = True + self._configure_started = True self.do_init() - self._compile_finished = True + self._configure_finished = True @property def class_attribute(self): """Return the class-bound descriptor corresponding to this - MapperProperty.""" + :class:`.MapperProperty`. + + This is basically a ``getattr()`` call:: + + return getattr(self.parent.class_, self.key) + + I.e. if this :class:`.MapperProperty` were named ``addresses``, + and the class to which it is mapped is ``User``, this sequence + is possible:: + + >>> from sqlalchemy import inspect + >>> mapper = inspect(User) + >>> addresses_property = mapper.attrs.addresses + >>> addresses_property.class_attribute is User.addresses + True + >>> User.addresses.property is addresses_property + True + + + """ return getattr(self.parent.class_, self.key) @@ -153,9 +187,6 @@ class MapperProperty(object): """ pass - def per_property_preprocessors(self, uow): - pass - def is_primary(self): """Return True if this ``MapperProperty``'s mapper is the primary mapper for its class. @@ -186,47 +217,131 @@ class MapperProperty(object): return operator(self.comparator, value) + def __repr__(self): + return '<%s at 0x%x; %s>' % ( + self.__class__.__name__, + id(self), getattr(self, 'key', 'no key')) + class PropComparator(operators.ColumnOperators): - """Defines comparison operations for MapperProperty objects. + """Defines boolean, comparison, and other operators for + :class:`.MapperProperty` objects. + + SQLAlchemy allows for operators to + be redefined at both the Core and ORM level. :class:`.PropComparator` + is the base class of operator redefinition for ORM-level operations, + including those of :class:`.ColumnProperty`, + :class:`.RelationshipProperty`, and :class:`.CompositeProperty`. + + .. note:: With the advent of Hybrid properties introduced in SQLAlchemy + 0.7, as well as Core-level operator redefinition in + SQLAlchemy 0.8, the use case for user-defined :class:`.PropComparator` + instances is extremely rare. See :ref:`hybrids_toplevel` as well + as :ref:`types_operators`. User-defined subclasses of :class:`.PropComparator` may be created. The built-in Python comparison and math operator methods, such as - ``__eq__()``, ``__lt__()``, ``__add__()``, can be overridden to provide + :meth:`.operators.ColumnOperators.__eq__`, + :meth:`.operators.ColumnOperators.__lt__`, and + :meth:`.operators.ColumnOperators.__add__`, can be overridden to provide new operator behavior. The custom :class:`.PropComparator` is passed to - the mapper property via the ``comparator_factory`` argument. In each case, + the :class:`.MapperProperty` instance via the ``comparator_factory`` + argument. In each case, the appropriate subclass of :class:`.PropComparator` should be used:: + # definition of custom PropComparator subclasses + from sqlalchemy.orm.properties import \\ ColumnProperty,\\ CompositeProperty,\\ RelationshipProperty class MyColumnComparator(ColumnProperty.Comparator): - pass - - class MyCompositeComparator(CompositeProperty.Comparator): - pass + def __eq__(self, other): + return self.__clause_element__() == other class MyRelationshipComparator(RelationshipProperty.Comparator): - pass + def any(self, expression): + "define the 'any' operation" + # ... + + class MyCompositeComparator(CompositeProperty.Comparator): + def __gt__(self, other): + "redefine the 'greater than' operation" + + return sql.and_(*[a>b for a, b in + zip(self.__clause_element__().clauses, + other.__composite_values__())]) + + + # application of custom PropComparator subclasses + + from sqlalchemy.orm import column_property, relationship, composite + from sqlalchemy import Column, String + + class SomeMappedClass(Base): + some_column = column_property(Column("some_column", String), + comparator_factory=MyColumnComparator) + + some_relationship = relationship(SomeOtherClass, + comparator_factory=MyRelationshipComparator) + + some_composite = composite( + Column("a", String), Column("b", String), + comparator_factory=MyCompositeComparator + ) + + Note that for column-level operator redefinition, it's usually + simpler to define the operators at the Core level, using the + :attr:`.TypeEngine.comparator_factory` attribute. See + :ref:`types_operators` for more detail. + + See also: + + :class:`.ColumnProperty.Comparator` + + :class:`.RelationshipProperty.Comparator` + + :class:`.CompositeProperty.Comparator` + + :class:`.ColumnOperators` + + :ref:`types_operators` + + :attr:`.TypeEngine.comparator_factory` """ - def __init__(self, prop, mapper, adapter=None): + def __init__(self, prop, parentmapper, adapt_to_entity=None): self.prop = self.property = prop - self.mapper = mapper - self.adapter = adapter + self._parentmapper = parentmapper + self._adapt_to_entity = adapt_to_entity def __clause_element__(self): raise NotImplementedError("%r" % self) - def adapted(self, adapter): + def _query_clause_element(self): + return self.__clause_element__() + + def adapt_to_entity(self, adapt_to_entity): """Return a copy of this PropComparator which will use the given - adaption function on the local side of generated expressions. + :class:`.AliasedInsp` to produce corresponding expressions. + """ + return self.__class__(self.prop, self._parentmapper, adapt_to_entity) + + @property + def adapter(self): + """Produce a callable that adapts column expressions + to suit an aliased version of this comparator. """ + if self._adapt_to_entity is None: + return None + else: + return self._adapt_to_entity._adapt_element - return self.__class__(self.prop, self.mapper, adapter) + @util.memoized_property + def info(self): + return self.property.info @staticmethod def any_op(a, b, **kwargs): @@ -251,8 +366,8 @@ class PropComparator(operators.ColumnOperators): query.join(Company.employees.of_type(Engineer)).\\ filter(Engineer.name=='foo') - :param \class_: a class or mapper indicating that criterion will be against - this specific subclass. + :param \class_: a class or mapper indicating that criterion will be + against this specific subclass. """ @@ -269,9 +384,9 @@ class PropComparator(operators.ColumnOperators): :param criterion: an optional ClauseElement formulated against the member class' table or attributes. - :param \**kwargs: key/value pairs corresponding to member class attribute - names which will be compared via equality to the corresponding - values. + :param \**kwargs: key/value pairs corresponding to member class + attribute names which will be compared via equality to the + corresponding values. """ @@ -287,9 +402,9 @@ class PropComparator(operators.ColumnOperators): :param criterion: an optional ClauseElement formulated against the member class' table or attributes. - :param \**kwargs: key/value pairs corresponding to member class attribute - names which will be compared via equality to the corresponding - values. + :param \**kwargs: key/value pairs corresponding to member class + attribute names which will be compared via equality to the + corresponding values. """ @@ -308,75 +423,87 @@ class StrategizedProperty(MapperProperty): strategy_wildcard_key = None - def _get_context_strategy(self, context, reduced_path): - key = ('loaderstrategy', reduced_path) - cls = None - if key in context.attributes: - cls = context.attributes[key] - elif self.strategy_wildcard_key: - key = ('loaderstrategy', (self.strategy_wildcard_key,)) - if key in context.attributes: - cls = context.attributes[key] + def _get_context_loader(self, context, path): + load = None - if cls: - try: - return self._strategies[cls] - except KeyError: - return self.__init_strategy(cls) - return self.strategy + # use EntityRegistry.__getitem__()->PropRegistry here so + # that the path is stated in terms of our base + search_path = dict.__getitem__(path, self) - def _get_strategy(self, cls): + # search among: exact match, "attr.*", "default" strategy + # if any. + for path_key in ( + search_path._loader_key, + search_path._wildcard_path_loader_key, + search_path._default_path_loader_key + ): + if path_key in context.attributes: + load = context.attributes[path_key] + break + + return load + + def _get_strategy(self, key): try: - return self._strategies[cls] + return self._strategies[key] except KeyError: - return self.__init_strategy(cls) + cls = self._strategy_lookup(*key) + self._strategies[key] = self._strategies[cls] = strategy = cls(self) + return strategy - def __init_strategy(self, cls): - self._strategies[cls] = strategy = cls(self) - return strategy + def _get_strategy_by_cls(self, cls): + return self._get_strategy(cls._strategy_keys[0]) - def setup(self, context, entity, path, reduced_path, adapter, **kwargs): - self._get_context_strategy(context, reduced_path + (self.key,)).\ - setup_query(context, entity, path, - reduced_path, adapter, **kwargs) + def setup(self, context, entity, path, adapter, **kwargs): + loader = self._get_context_loader(context, path) + if loader and loader.strategy: + strat = self._get_strategy(loader.strategy) + else: + strat = self.strategy + strat.setup_query(context, entity, path, loader, adapter, **kwargs) - def create_row_processor(self, context, path, reduced_path, mapper, row, adapter): - return self._get_context_strategy(context, reduced_path + (self.key,)).\ - create_row_processor(context, path, - reduced_path, mapper, row, adapter) + def create_row_processor(self, context, path, mapper, row, adapter): + loader = self._get_context_loader(context, path) + if loader and loader.strategy: + strat = self._get_strategy(loader.strategy) + else: + strat = self.strategy + return strat.create_row_processor(context, path, loader, + mapper, row, adapter) def do_init(self): self._strategies = {} - self.strategy = self.__init_strategy(self.strategy_class) + self.strategy = self._get_strategy_by_cls(self.strategy_class) def post_instrument_class(self, mapper): if self.is_primary() and \ not mapper.class_manager._attr_has_impl(self.key): self.strategy.init_class_attribute(mapper) -def build_path(entity, key, prev=None): - if prev: - return prev + (entity, key) - else: - return (entity, key) -def serialize_path(path): - if path is None: - return None + _strategies = collections.defaultdict(dict) - return zip( - [m.class_ for m in [path[i] for i in range(0, len(path), 2)]], - [path[i] for i in range(1, len(path), 2)] + [None] - ) + @classmethod + def strategy_for(cls, **kw): + def decorate(dec_cls): + dec_cls._strategy_keys = [] + key = tuple(sorted(kw.items())) + cls._strategies[cls][key] = dec_cls + dec_cls._strategy_keys.append(key) + return dec_cls + return decorate -def deserialize_path(path): - if path is None: - return None + @classmethod + def _strategy_lookup(cls, *key): + for prop_cls in cls.__mro__: + if prop_cls in cls._strategies: + strategies = cls._strategies[prop_cls] + try: + return strategies[key] + except KeyError: + pass + raise Exception("can't locate strategy for %s %s" % (cls, key)) - p = tuple(chain(*[(mapperutil.class_mapper(cls), key) for cls, key in path])) - if p and p[-1] is None: - p = p[0:-1] - return p class MapperOption(object): """Describe a modification to a Query.""" @@ -398,242 +525,8 @@ class MapperOption(object): self.process_query(query) -class PropertyOption(MapperOption): - """A MapperOption that is applied to a property off the mapper or - one of its child mappers, identified by a dot-separated key - or list of class-bound attributes. """ - def __init__(self, key, mapper=None): - self.key = key - self.mapper = mapper - def process_query(self, query): - self._process(query, True) - - def process_query_conditionally(self, query): - self._process(query, False) - - def _process(self, query, raiseerr): - paths, mappers = self._get_paths(query, raiseerr) - if paths: - self.process_query_property(query, paths, mappers) - - def process_query_property(self, query, paths, mappers): - pass - - def __getstate__(self): - d = self.__dict__.copy() - d['key'] = ret = [] - for token in util.to_list(self.key): - if isinstance(token, PropComparator): - ret.append((token.mapper.class_, token.key)) - else: - ret.append(token) - return d - - def __setstate__(self, state): - ret = [] - for key in state['key']: - if isinstance(key, tuple): - cls, propkey = key - ret.append(getattr(cls, propkey)) - else: - ret.append(key) - state['key'] = tuple(ret) - self.__dict__ = state - - def _find_entity_prop_comparator(self, query, token, mapper, raiseerr): - if mapperutil._is_aliased_class(mapper): - searchfor = mapper - isa = False - else: - searchfor = mapperutil._class_to_mapper(mapper) - isa = True - for ent in query._mapper_entities: - if searchfor is ent.path_entity or isa \ - and searchfor.common_parent(ent.path_entity): - return ent - else: - if raiseerr: - if not list(query._mapper_entities): - raise sa_exc.ArgumentError( - "Query has only expression-based entities - " - "can't find property named '%s'." - % (token, ) - ) - else: - raise sa_exc.ArgumentError( - "Can't find property '%s' on any entity " - "specified in this Query. Note the full path " - "from root (%s) to target entity must be specified." - % (token, ",".join(str(x) for - x in query._mapper_entities)) - ) - else: - return None - - def _find_entity_basestring(self, query, token, raiseerr): - for ent in query._mapper_entities: - # return only the first _MapperEntity when searching - # based on string prop name. Ideally object - # attributes are used to specify more exactly. - return ent - else: - if raiseerr: - raise sa_exc.ArgumentError( - "Query has only expression-based entities - " - "can't find property named '%s'." - % (token, ) - ) - else: - return None - - def _get_paths(self, query, raiseerr): - path = None - entity = None - l = [] - mappers = [] - - # _current_path implies we're in a - # secondary load with an existing path - current_path = list(query._current_path) - - tokens = deque(self.key) - while tokens: - token = tokens.popleft() - if isinstance(token, basestring): - # wildcard token - if token.endswith(':*'): - return [(token,)], [] - sub_tokens = token.split(".", 1) - token = sub_tokens[0] - tokens.extendleft(sub_tokens[1:]) - - # exhaust current_path before - # matching tokens to entities - if current_path: - if current_path[1] == token: - current_path = current_path[2:] - continue - else: - return [], [] - - if not entity: - entity = self._find_entity_basestring( - query, - token, - raiseerr) - if entity is None: - return [], [] - path_element = entity.path_entity - mapper = entity.mapper - mappers.append(mapper) - if hasattr(mapper.class_, token): - prop = getattr(mapper.class_, token).property - else: - if raiseerr: - raise sa_exc.ArgumentError( - "Can't find property named '%s' on the " - "mapped entity %s in this Query. " % ( - token, mapper) - ) - else: - return [], [] - elif isinstance(token, PropComparator): - prop = token.property - - # exhaust current_path before - # matching tokens to entities - if current_path: - if current_path[0:2] == \ - [token.parententity, prop.key]: - current_path = current_path[2:] - continue - else: - return [], [] - - if not entity: - entity = self._find_entity_prop_comparator( - query, - prop.key, - token.parententity, - raiseerr) - if not entity: - return [], [] - path_element = entity.path_entity - mapper = entity.mapper - mappers.append(prop.parent) - else: - raise sa_exc.ArgumentError( - "mapper option expects " - "string key or list of attributes") - assert prop is not None - if raiseerr and not prop.parent.common_parent(mapper): - raise sa_exc.ArgumentError("Attribute '%s' does not " - "link from element '%s'" % (token, path_element)) - - path = build_path(path_element, prop.key, path) - - l.append(path) - if getattr(token, '_of_type', None): - path_element = mapper = token._of_type - else: - path_element = mapper = getattr(prop, 'mapper', None) - if mapper is None and tokens: - raise sa_exc.ArgumentError( - "Attribute '%s' of entity '%s' does not " - "refer to a mapped entity" % - (token, entity) - ) - - if current_path: - # ran out of tokens before - # current_path was exhausted. - assert not tokens - return [], [] - - return l, mappers - -class StrategizedOption(PropertyOption): - """A MapperOption that affects which LoaderStrategy will be used - for an operation by a StrategizedProperty. - """ - - chained = False - - def process_query_property(self, query, paths, mappers): - - # _get_context_strategy may receive the path in terms of a base - # mapper - e.g. options(eagerload_all(Company.employees, - # Engineer.machines)) in the polymorphic tests leads to - # "(Person, 'machines')" in the path due to the mechanics of how - # the eager strategy builds up the path - - if self.chained: - for path in paths: - query._attributes[('loaderstrategy', - _reduce_path(path))] = \ - self.get_strategy_class() - else: - query._attributes[('loaderstrategy', - _reduce_path(paths[-1]))] = \ - self.get_strategy_class() - - def get_strategy_class(self): - raise NotImplementedError() - -def _reduce_path(path): - """Convert a (mapper, path) path to use base mappers. - - This is used to allow more open ended selection of loader strategies, i.e. - Mapper -> prop1 -> Subclass -> prop2, where Subclass is a sub-mapper - of the mapper referenced by Mapper.prop1. - - """ - return tuple([i % 2 != 0 and - element or - getattr(element, 'base_mapper', element) - for i, element in enumerate(path)]) class LoaderStrategy(object): """Describe the loading behavior of a StrategizedProperty object. @@ -663,22 +556,14 @@ class LoaderStrategy(object): self.is_class_level = False self.parent = self.parent_property.parent self.key = self.parent_property.key - # TODO: there's no particular reason we need - # the separate .init() method at this point. - # It's possible someone has written their - # own LS object. - self.init() - - def init(self): - raise NotImplementedError("LoaderStrategy") def init_class_attribute(self, mapper): pass - def setup_query(self, context, entity, path, reduced_path, adapter, **kwargs): + def setup_query(self, context, entity, path, loadopt, adapter, **kwargs): pass - def create_row_processor(self, context, path, reduced_path, mapper, + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): """Return row processing functions which fulfill the contract specified by MapperProperty.create_row_processor. @@ -690,94 +575,3 @@ class LoaderStrategy(object): def __str__(self): return str(self.parent_property) - - def debug_callable(self, fn, logger, announcement, logfn): - if announcement: - logger.debug(announcement) - if logfn: - def call(*args, **kwargs): - logger.debug(logfn(*args, **kwargs)) - return fn(*args, **kwargs) - return call - else: - return fn - -class InstrumentationManager(object): - """User-defined class instrumentation extension. - - :class:`.InstrumentationManager` can be subclassed in order - to change - how class instrumentation proceeds. This class exists for - the purposes of integration with other object management - frameworks which would like to entirely modify the - instrumentation methodology of the ORM, and is not intended - for regular usage. For interception of class instrumentation - events, see :class:`.InstrumentationEvents`. - - For an example of :class:`.InstrumentationManager`, see the - example :ref:`examples_instrumentation`. - - The API for this class should be considered as semi-stable, - and may change slightly with new releases. - - """ - - # r4361 added a mandatory (cls) constructor to this interface. - # given that, perhaps class_ should be dropped from all of these - # signatures. - - def __init__(self, class_): - pass - - def manage(self, class_, manager): - setattr(class_, '_default_class_manager', manager) - - def dispose(self, class_, manager): - delattr(class_, '_default_class_manager') - - def manager_getter(self, class_): - def get(cls): - return cls._default_class_manager - return get - - def instrument_attribute(self, class_, key, inst): - pass - - def post_configure_attribute(self, class_, key, inst): - pass - - def install_descriptor(self, class_, key, inst): - setattr(class_, key, inst) - - def uninstall_descriptor(self, class_, key): - delattr(class_, key) - - def install_member(self, class_, key, implementation): - setattr(class_, key, implementation) - - def uninstall_member(self, class_, key): - delattr(class_, key) - - def instrument_collection_class(self, class_, key, collection_class): - global collections - if collections is None: - from sqlalchemy.orm import collections - return collections.prepare_instrumentation(collection_class) - - def get_instance_dict(self, class_, instance): - return instance.__dict__ - - def initialize_instance_dict(self, class_, instance): - pass - - def install_state(self, class_, instance, state): - setattr(instance, '_default_state', state) - - def remove_state(self, class_, instance): - delattr(instance, '_default_state') - - def state_getter(self, class_): - return lambda instance: getattr(instance, '_default_state') - - def dict_getter(self, class_): - return lambda inst: self.get_instance_dict(class_, inst) diff --git a/libs/sqlalchemy/orm/loading.py b/libs/sqlalchemy/orm/loading.py new file mode 100644 index 00000000..af77fe3e --- /dev/null +++ b/libs/sqlalchemy/orm/loading.py @@ -0,0 +1,610 @@ +# orm/loading.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""private module containing functions used to convert database +rows into object instances and associated state. + +the functions here are called primarily by Query, Mapper, +as well as some of the attribute loading strategies. + +""" + + +from .. import util +from . import attributes, exc as orm_exc, state as statelib +from .interfaces import EXT_CONTINUE +from ..sql import util as sql_util +from .util import _none_set, state_str +from .. import exc as sa_exc + +_new_runid = util.counter() + + +def instances(query, cursor, context): + """Return an ORM result as an iterator.""" + session = query.session + + context.runid = _new_runid() + + filter_fns = [ent.filter_fn + for ent in query._entities] + filtered = id in filter_fns + + single_entity = len(query._entities) == 1 and \ + query._entities[0].supports_single_entity + + if filtered: + if single_entity: + filter_fn = id + else: + def filter_fn(row): + return tuple(fn(x) for x, fn in zip(row, filter_fns)) + + custom_rows = single_entity and \ + query._entities[0].custom_rows + + (process, labels) = \ + list(zip(*[ + query_entity.row_processor(query, + context, custom_rows) + for query_entity in query._entities + ])) + + while True: + context.progress = {} + context.partials = {} + + if query._yield_per: + fetch = cursor.fetchmany(query._yield_per) + if not fetch: + break + else: + fetch = cursor.fetchall() + + if custom_rows: + rows = [] + for row in fetch: + process[0](row, rows) + elif single_entity: + rows = [process[0](row, None) for row in fetch] + else: + rows = [util.KeyedTuple([proc(row, None) for proc in process], + labels) for row in fetch] + + if filtered: + rows = util.unique_list(rows, filter_fn) + + if context.refresh_state and query._only_load_props \ + and context.refresh_state in context.progress: + context.refresh_state._commit( + context.refresh_state.dict, query._only_load_props) + context.progress.pop(context.refresh_state) + + statelib.InstanceState._commit_all_states( + list(context.progress.items()), + session.identity_map + ) + + for state, (dict_, attrs) in context.partials.items(): + state._commit(dict_, attrs) + + for row in rows: + yield row + + if not query._yield_per: + break + + +@util.dependencies("sqlalchemy.orm.query") +def merge_result(querylib, query, iterator, load=True): + """Merge a result into this :class:`.Query` object's Session.""" + + session = query.session + if load: + # flush current contents if we expect to load data + session._autoflush() + + autoflush = session.autoflush + try: + session.autoflush = False + single_entity = len(query._entities) == 1 + if single_entity: + if isinstance(query._entities[0], querylib._MapperEntity): + result = [session._merge( + attributes.instance_state(instance), + attributes.instance_dict(instance), + load=load, _recursive={}) + for instance in iterator] + else: + result = list(iterator) + else: + mapped_entities = [i for i, e in enumerate(query._entities) + if isinstance(e, querylib._MapperEntity)] + result = [] + keys = [ent._label_name for ent in query._entities] + for row in iterator: + newrow = list(row) + for i in mapped_entities: + if newrow[i] is not None: + newrow[i] = session._merge( + attributes.instance_state(newrow[i]), + attributes.instance_dict(newrow[i]), + load=load, _recursive={}) + result.append(util.KeyedTuple(newrow, keys)) + + return iter(result) + finally: + session.autoflush = autoflush + + +def get_from_identity(session, key, passive): + """Look up the given key in the given session's identity map, + check the object for expired state if found. + + """ + instance = session.identity_map.get(key) + if instance is not None: + + state = attributes.instance_state(instance) + + # expired - ensure it still exists + if state.expired: + if not passive & attributes.SQL_OK: + # TODO: no coverage here + return attributes.PASSIVE_NO_RESULT + elif not passive & attributes.RELATED_OBJECT_OK: + # this mode is used within a flush and the instance's + # expired state will be checked soon enough, if necessary + return instance + try: + state(state, passive) + except orm_exc.ObjectDeletedError: + session._remove_newly_deleted([state]) + return None + return instance + else: + return None + + +def load_on_ident(query, key, + refresh_state=None, lockmode=None, + only_load_props=None): + """Load the given identity key from the database.""" + + if key is not None: + ident = key[1] + else: + ident = None + + if refresh_state is None: + q = query._clone() + q._get_condition() + else: + q = query._clone() + + if ident is not None: + mapper = query._mapper_zero() + + (_get_clause, _get_params) = mapper._get_clause + + # None present in ident - turn those comparisons + # into "IS NULL" + if None in ident: + nones = set([ + _get_params[col].key for col, value in + zip(mapper.primary_key, ident) if value is None + ]) + _get_clause = sql_util.adapt_criterion_to_null( + _get_clause, nones) + + _get_clause = q._adapt_clause(_get_clause, True, False) + q._criterion = _get_clause + + params = dict([ + (_get_params[primary_key].key, id_val) + for id_val, primary_key in zip(ident, mapper.primary_key) + ]) + + q._params = params + + if lockmode is not None: + version_check = True + q = q.with_lockmode(lockmode) + elif query._for_update_arg is not None: + version_check = True + q._for_update_arg = query._for_update_arg + else: + version_check = False + + q._get_options( + populate_existing=bool(refresh_state), + version_check=version_check, + only_load_props=only_load_props, + refresh_state=refresh_state) + q._order_by = None + + try: + return q.one() + except orm_exc.NoResultFound: + return None + + +def instance_processor(mapper, context, path, adapter, + polymorphic_from=None, + only_load_props=None, + refresh_state=None, + polymorphic_discriminator=None): + + """Produce a mapper level row processor callable + which processes rows into mapped instances.""" + + # note that this method, most of which exists in a closure + # called _instance(), resists being broken out, as + # attempts to do so tend to add significant function + # call overhead. _instance() is the most + # performance-critical section in the whole ORM. + + pk_cols = mapper.primary_key + + if polymorphic_from or refresh_state: + polymorphic_on = None + else: + if polymorphic_discriminator is not None: + polymorphic_on = polymorphic_discriminator + else: + polymorphic_on = mapper.polymorphic_on + polymorphic_instances = util.PopulateDict( + _configure_subclass_mapper( + mapper, + context, path, adapter) + ) + + version_id_col = mapper.version_id_col + + if adapter: + pk_cols = [adapter.columns[c] for c in pk_cols] + if polymorphic_on is not None: + polymorphic_on = adapter.columns[polymorphic_on] + if version_id_col is not None: + version_id_col = adapter.columns[version_id_col] + + identity_class = mapper._identity_class + + new_populators = [] + existing_populators = [] + eager_populators = [] + + load_path = context.query._current_path + path \ + if context.query._current_path.path \ + else path + + def populate_state(state, dict_, row, isnew, only_load_props): + if isnew: + if context.propagate_options: + state.load_options = context.propagate_options + if state.load_options: + state.load_path = load_path + + if not new_populators: + _populators(mapper, context, path, row, adapter, + new_populators, + existing_populators, + eager_populators + ) + + if isnew: + populators = new_populators + else: + populators = existing_populators + + if only_load_props is None: + for key, populator in populators: + populator(state, dict_, row) + elif only_load_props: + for key, populator in populators: + if key in only_load_props: + populator(state, dict_, row) + + session_identity_map = context.session.identity_map + + listeners = mapper.dispatch + + translate_row = listeners.translate_row or None + create_instance = listeners.create_instance or None + populate_instance = listeners.populate_instance or None + append_result = listeners.append_result or None + populate_existing = context.populate_existing or mapper.always_refresh + invoke_all_eagers = context.invoke_all_eagers + + if mapper.allow_partial_pks: + is_not_primary_key = _none_set.issuperset + else: + is_not_primary_key = _none_set.issubset + + def _instance(row, result): + if not new_populators and invoke_all_eagers: + _populators(mapper, context, path, row, adapter, + new_populators, + existing_populators, + eager_populators + ) + + if translate_row: + for fn in translate_row: + ret = fn(mapper, context, row) + if ret is not EXT_CONTINUE: + row = ret + break + + if polymorphic_on is not None: + discriminator = row[polymorphic_on] + if discriminator is not None: + _instance = polymorphic_instances[discriminator] + if _instance: + return _instance(row, result) + + # determine identity key + if refresh_state: + identitykey = refresh_state.key + if identitykey is None: + # super-rare condition; a refresh is being called + # on a non-instance-key instance; this is meant to only + # occur within a flush() + identitykey = mapper._identity_key_from_state(refresh_state) + else: + identitykey = ( + identity_class, + tuple([row[column] for column in pk_cols]) + ) + + instance = session_identity_map.get(identitykey) + if instance is not None: + state = attributes.instance_state(instance) + dict_ = attributes.instance_dict(instance) + + isnew = state.runid != context.runid + currentload = not isnew + loaded_instance = False + + if not currentload and \ + version_id_col is not None and \ + context.version_check and \ + mapper._get_state_attr_by_column( + state, + dict_, + mapper.version_id_col) != \ + row[version_id_col]: + + raise orm_exc.StaleDataError( + "Instance '%s' has version id '%s' which " + "does not match database-loaded version id '%s'." + % (state_str(state), + mapper._get_state_attr_by_column( + state, dict_, + mapper.version_id_col), + row[version_id_col])) + elif refresh_state: + # out of band refresh_state detected (i.e. its not in the + # session.identity_map) honor it anyway. this can happen + # if a _get() occurs within save_obj(), such as + # when eager_defaults is True. + state = refresh_state + instance = state.obj() + dict_ = attributes.instance_dict(instance) + isnew = state.runid != context.runid + currentload = True + loaded_instance = False + else: + # check for non-NULL values in the primary key columns, + # else no entity is returned for the row + if is_not_primary_key(identitykey[1]): + return None + + isnew = True + currentload = True + loaded_instance = True + + if create_instance: + for fn in create_instance: + instance = fn(mapper, context, + row, mapper.class_) + if instance is not EXT_CONTINUE: + manager = attributes.manager_of_class( + instance.__class__) + # TODO: if manager is None, raise a friendly error + # about returning instances of unmapped types + manager.setup_instance(instance) + break + else: + instance = mapper.class_manager.new_instance() + else: + instance = mapper.class_manager.new_instance() + + dict_ = attributes.instance_dict(instance) + state = attributes.instance_state(instance) + state.key = identitykey + + # attach instance to session. + state.session_id = context.session.hash_key + session_identity_map.add(state) + + if currentload or populate_existing: + # state is being fully loaded, so populate. + # add to the "context.progress" collection. + if isnew: + state.runid = context.runid + context.progress[state] = dict_ + + if populate_instance: + for fn in populate_instance: + ret = fn(mapper, context, row, state, + only_load_props=only_load_props, + instancekey=identitykey, isnew=isnew) + if ret is not EXT_CONTINUE: + break + else: + populate_state(state, dict_, row, isnew, only_load_props) + else: + populate_state(state, dict_, row, isnew, only_load_props) + + if loaded_instance: + state.manager.dispatch.load(state, context) + elif isnew: + state.manager.dispatch.refresh(state, context, only_load_props) + + elif state in context.partials or state.unloaded or eager_populators: + # state is having a partial set of its attributes + # refreshed. Populate those attributes, + # and add to the "context.partials" collection. + if state in context.partials: + isnew = False + (d_, attrs) = context.partials[state] + else: + isnew = True + attrs = state.unloaded + context.partials[state] = (dict_, attrs) + + if populate_instance: + for fn in populate_instance: + ret = fn(mapper, context, row, state, + only_load_props=attrs, + instancekey=identitykey, isnew=isnew) + if ret is not EXT_CONTINUE: + break + else: + populate_state(state, dict_, row, isnew, attrs) + else: + populate_state(state, dict_, row, isnew, attrs) + + for key, pop in eager_populators: + if key not in state.unloaded: + pop(state, dict_, row) + + if isnew: + state.manager.dispatch.refresh(state, context, attrs) + + if result is not None: + if append_result: + for fn in append_result: + if fn(mapper, context, row, state, + result, instancekey=identitykey, + isnew=isnew) is not EXT_CONTINUE: + break + else: + result.append(instance) + else: + result.append(instance) + + return instance + return _instance + + +def _populators(mapper, context, path, row, adapter, + new_populators, existing_populators, eager_populators): + """Produce a collection of attribute level row processor + callables.""" + + delayed_populators = [] + pops = (new_populators, existing_populators, delayed_populators, + eager_populators) + + for prop in mapper._props.values(): + + for i, pop in enumerate(prop.create_row_processor( + context, + path, + mapper, row, adapter)): + if pop is not None: + pops[i].append((prop.key, pop)) + + if delayed_populators: + new_populators.extend(delayed_populators) + + +def _configure_subclass_mapper(mapper, context, path, adapter): + """Produce a mapper level row processor callable factory for mappers + inheriting this one.""" + + def configure_subclass_mapper(discriminator): + try: + sub_mapper = mapper.polymorphic_map[discriminator] + except KeyError: + raise AssertionError( + "No such polymorphic_identity %r is defined" % + discriminator) + if sub_mapper is mapper: + return None + + return instance_processor( + sub_mapper, + context, + path, + adapter, + polymorphic_from=mapper) + return configure_subclass_mapper + + +def load_scalar_attributes(mapper, state, attribute_names): + """initiate a column-based attribute refresh operation.""" + + #assert mapper is _state_mapper(state) + session = state.session + if not session: + raise orm_exc.DetachedInstanceError( + "Instance %s is not bound to a Session; " + "attribute refresh operation cannot proceed" % + (state_str(state))) + + has_key = bool(state.key) + + result = False + + if mapper.inherits and not mapper.concrete: + statement = mapper._optimized_get_statement(state, attribute_names) + if statement is not None: + result = load_on_ident( + session.query(mapper).from_statement(statement), + None, + only_load_props=attribute_names, + refresh_state=state + ) + + if result is False: + if has_key: + identity_key = state.key + else: + # this codepath is rare - only valid when inside a flush, and the + # object is becoming persistent but hasn't yet been assigned + # an identity_key. + # check here to ensure we have the attrs we need. + pk_attrs = [mapper._columntoproperty[col].key + for col in mapper.primary_key] + if state.expired_attributes.intersection(pk_attrs): + raise sa_exc.InvalidRequestError( + "Instance %s cannot be refreshed - it's not " + " persistent and does not " + "contain a full primary key." % state_str(state)) + identity_key = mapper._identity_key_from_state(state) + + if (_none_set.issubset(identity_key) and \ + not mapper.allow_partial_pks) or \ + _none_set.issuperset(identity_key): + util.warn("Instance %s to be refreshed doesn't " + "contain a full primary key - can't be refreshed " + "(and shouldn't be expired, either)." + % state_str(state)) + return + + result = load_on_ident( + session.query(mapper), + identity_key, + refresh_state=state, + only_load_props=attribute_names) + + # if instance is pending, a refresh operation + # may not complete (even if PK attributes are assigned) + if has_key and result is None: + raise orm_exc.ObjectDeletedError(state) diff --git a/libs/sqlalchemy/orm/mapper.py b/libs/sqlalchemy/orm/mapper.py index de4d351b..a853efc3 100644 --- a/libs/sqlalchemy/orm/mapper.py +++ b/libs/sqlalchemy/orm/mapper.py @@ -1,5 +1,5 @@ # orm/mapper.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -13,38 +13,28 @@ This is a semi-private module; the main configurational API of the ORM is available in :class:`~sqlalchemy.orm.`. """ +from __future__ import absolute_import import types import weakref -import operator -from itertools import chain, groupby -deque = __import__('collections').deque +from itertools import chain +from collections import deque -from sqlalchemy import sql, util, log, exc as sa_exc, event, schema -from sqlalchemy.sql import expression, visitors, operators, util as sqlutil -from sqlalchemy.orm import instrumentation, attributes, sync, \ - exc as orm_exc, unitofwork, events -from sqlalchemy.orm.interfaces import MapperProperty, EXT_CONTINUE, \ - PropComparator +from .. import sql, util, log, exc as sa_exc, event, schema, inspection +from ..sql import expression, visitors, operators, util as sql_util +from . import instrumentation, attributes, exc as orm_exc, loading +from . import properties +from .interfaces import MapperProperty, _InspectionAttr, _MappedAttribute -from sqlalchemy.orm.util import _INSTRUMENTOR, _class_to_mapper, \ - _state_mapper, class_mapper, instance_str, state_str +from .base import _class_to_mapper, _state_mapper, class_mapper, \ + state_str, _INSTRUMENTOR +from .path_registry import PathRegistry import sys -sessionlib = util.importlater("sqlalchemy.orm", "session") -properties = util.importlater("sqlalchemy.orm", "properties") -__all__ = ( - 'Mapper', - '_mapper_registry', - 'class_mapper', - 'object_mapper', - ) _mapper_registry = weakref.WeakKeyDictionary() -_new_mappers = False _already_compiling = False -_none_set = frozenset([None]) _memoized_configured_property = util.group_expirable_memoized_property() @@ -54,37 +44,66 @@ _memoized_configured_property = util.group_expirable_memoized_property() # column NO_ATTRIBUTE = util.symbol('NO_ATTRIBUTE') -# lock used to synchronize the "mapper compile" step -_COMPILE_MUTEX = util.threading.RLock() +# lock used to synchronize the "mapper configure" step +_CONFIGURE_MUTEX = util.threading.RLock() -class Mapper(object): + +@inspection._self_inspects +@log.class_logger +class Mapper(_InspectionAttr): """Define the correlation of class attributes to database table columns. - Instances of this class should be constructed via the - :func:`~sqlalchemy.orm.mapper` function. + The :class:`.Mapper` object is instantiated using the + :func:`~sqlalchemy.orm.mapper` function. For information + about instantiating new :class:`.Mapper` objects, see + that function's documentation. + + + When :func:`.mapper` is used + explicitly to link a user defined class with table + metadata, this is referred to as *classical mapping*. + Modern SQLAlchemy usage tends to favor the + :mod:`sqlalchemy.ext.declarative` extension for class + configuration, which + makes usage of :func:`.mapper` behind the scenes. + + Given a particular class known to be mapped by the ORM, + the :class:`.Mapper` which maintains it can be acquired + using the :func:`.inspect` function:: + + from sqlalchemy import inspect + + mapper = inspect(MyClass) + + A class which was mapped by the :mod:`sqlalchemy.ext.declarative` + extension will also have its mapper available via the ``__mapper__`` + attribute. + """ + + _new_mappers = False + def __init__(self, class_, - local_table, - properties = None, - primary_key = None, - non_primary = False, - inherits = None, - inherit_condition = None, - inherit_foreign_keys = None, - extension = None, - order_by = False, - always_refresh = False, - version_id_col = None, - version_id_generator = None, + local_table=None, + properties=None, + primary_key=None, + non_primary=False, + inherits=None, + inherit_condition=None, + inherit_foreign_keys=None, + extension=None, + order_by=False, + always_refresh=False, + version_id_col=None, + version_id_generator=None, polymorphic_on=None, _polymorphic_map=None, polymorphic_identity=None, concrete=False, with_polymorphic=None, - allow_null_pks=None, allow_partial_pks=True, batch=True, column_prefix=None, @@ -92,12 +111,383 @@ class Mapper(object): exclude_properties=None, passive_updates=True, eager_defaults=False, + legacy_is_orphan=False, _compiled_cache_size=100, ): - """Construct a new mapper. + """Return a new :class:`~.Mapper` object. - Mappers are normally constructed via the - :func:`~sqlalchemy.orm.mapper` function. See for details. + This function is typically used behind the scenes + via the Declarative extension. When using Declarative, + many of the usual :func:`.mapper` arguments are handled + by the Declarative extension itself, including ``class_``, + ``local_table``, ``properties``, and ``inherits``. + Other options are passed to :func:`.mapper` using + the ``__mapper_args__`` class variable:: + + class MyClass(Base): + __tablename__ = 'my_table' + id = Column(Integer, primary_key=True) + type = Column(String(50)) + alt = Column("some_alt", Integer) + + __mapper_args__ = { + 'polymorphic_on' : type + } + + + Explicit use of :func:`.mapper` + is often referred to as *classical mapping*. The above + declarative example is equivalent in classical form to:: + + my_table = Table("my_table", metadata, + Column('id', Integer, primary_key=True), + Column('type', String(50)), + Column("some_alt", Integer) + ) + + class MyClass(object): + pass + + mapper(MyClass, my_table, + polymorphic_on=my_table.c.type, + properties={ + 'alt':my_table.c.some_alt + }) + + .. seealso:: + + :ref:`classical_mapping` - discussion of direct usage of + :func:`.mapper` + + :param class\_: The class to be mapped. When using Declarative, + this argument is automatically passed as the declared class + itself. + + :param local_table: The :class:`.Table` or other selectable + to which the class is mapped. May be ``None`` if + this mapper inherits from another mapper using single-table + inheritance. When using Declarative, this argument is + automatically passed by the extension, based on what + is configured via the ``__table__`` argument or via the + :class:`.Table` produced as a result of the ``__tablename__`` + and :class:`.Column` arguments present. + + :param always_refresh: If True, all query operations for this mapped + class will overwrite all data within object instances that already + exist within the session, erasing any in-memory changes with + whatever information was loaded from the database. Usage of this + flag is highly discouraged; as an alternative, see the method + :meth:`.Query.populate_existing`. + + :param allow_partial_pks: Defaults to True. Indicates that a + composite primary key with some NULL values should be considered as + possibly existing within the database. This affects whether a + mapper will assign an incoming row to an existing identity, as well + as if :meth:`.Session.merge` will check the database first for a + particular primary key value. A "partial primary key" can occur if + one has mapped to an OUTER JOIN, for example. + + :param batch: Defaults to ``True``, indicating that save operations + of multiple entities can be batched together for efficiency. + Setting to False indicates + that an instance will be fully saved before saving the next + instance. This is used in the extremely rare case that a + :class:`.MapperEvents` listener requires being called + in between individual row persistence operations. + + :param column_prefix: A string which will be prepended + to the mapped attribute name when :class:`.Column` + objects are automatically assigned as attributes to the + mapped class. Does not affect explicitly specified + column-based properties. + + See the section :ref:`column_prefix` for an example. + + :param concrete: If True, indicates this mapper should use concrete + table inheritance with its parent mapper. + + See the section :ref:`concrete_inheritance` for an example. + + :param eager_defaults: if True, the ORM will immediately fetch the + value of server-generated default values after an INSERT or UPDATE, + rather than leaving them as expired to be fetched on next access. + This can be used for event schemes where the server-generated values + are needed immediately before the flush completes. By default, + this scheme will emit an individual ``SELECT`` statement per row + inserted or updated, which note can add significant performance + overhead. However, if the + target database supports :term:`RETURNING`, the default values will be + returned inline with the INSERT or UPDATE statement, which can + greatly enhance performance for an application that needs frequent + access to just-generated server defaults. + + .. versionchanged:: 0.9.0 The ``eager_defaults`` option can now + make use of :term:`RETURNING` for backends which support it. + + :param exclude_properties: A list or set of string column names to + be excluded from mapping. + + See :ref:`include_exclude_cols` for an example. + + :param extension: A :class:`.MapperExtension` instance or + list of :class:`.MapperExtension` instances which will be applied + to all operations by this :class:`.Mapper`. **Deprecated.** + Please see :class:`.MapperEvents`. + + :param include_properties: An inclusive list or set of string column + names to map. + + See :ref:`include_exclude_cols` for an example. + + :param inherits: A mapped class or the corresponding :class:`.Mapper` + of one indicating a superclass to which this :class:`.Mapper` + should *inherit* from. The mapped class here must be a subclass + of the other mapper's class. When using Declarative, this argument + is passed automatically as a result of the natural class + hierarchy of the declared classes. + + .. seealso:: + + :ref:`inheritance_toplevel` + + :param inherit_condition: For joined table inheritance, a SQL + expression which will + define how the two tables are joined; defaults to a natural join + between the two tables. + + :param inherit_foreign_keys: When ``inherit_condition`` is used and the + columns present are missing a :class:`.ForeignKey` configuration, + this parameter can be used to specify which columns are "foreign". + In most cases can be left as ``None``. + + :param legacy_is_orphan: Boolean, defaults to ``False``. + When ``True``, specifies that "legacy" orphan consideration + is to be applied to objects mapped by this mapper, which means + that a pending (that is, not persistent) object is auto-expunged + from an owning :class:`.Session` only when it is de-associated + from *all* parents that specify a ``delete-orphan`` cascade towards + this mapper. The new default behavior is that the object is auto-expunged + when it is de-associated with *any* of its parents that specify + ``delete-orphan`` cascade. This behavior is more consistent with + that of a persistent object, and allows behavior to be consistent + in more scenarios independently of whether or not an orphanable + object has been flushed yet or not. + + See the change note and example at :ref:`legacy_is_orphan_addition` + for more detail on this change. + + .. versionadded:: 0.8 - the consideration of a pending object as + an "orphan" has been modified to more closely match the + behavior as that of persistent objects, which is that the object + is expunged from the :class:`.Session` as soon as it is + de-associated from any of its orphan-enabled parents. Previously, + the pending object would be expunged only if de-associated + from all of its orphan-enabled parents. The new flag ``legacy_is_orphan`` + is added to :func:`.orm.mapper` which re-establishes the + legacy behavior. + + :param non_primary: Specify that this :class:`.Mapper` is in addition + to the "primary" mapper, that is, the one used for persistence. + The :class:`.Mapper` created here may be used for ad-hoc + mapping of the class to an alternate selectable, for loading + only. + + The ``non_primary`` feature is rarely needed with modern + usage. + + :param order_by: A single :class:`.Column` or list of :class:`.Column` + objects for which selection operations should use as the default + ordering for entities. By default mappers have no pre-defined + ordering. + + :param passive_updates: Indicates UPDATE behavior of foreign key + columns when a primary key column changes on a joined-table + inheritance mapping. Defaults to ``True``. + + When True, it is assumed that ON UPDATE CASCADE is configured on + the foreign key in the database, and that the database will handle + propagation of an UPDATE from a source column to dependent columns + on joined-table rows. + + When False, it is assumed that the database does not enforce + referential integrity and will not be issuing its own CASCADE + operation for an update. The :class:`.Mapper` here will + emit an UPDATE statement for the dependent columns during a + primary key change. + + ..seealso:: + + :ref:`passive_updates` - description of a similar feature as + used with :func:`.relationship` + + :param polymorphic_on: Specifies the column, attribute, or + SQL expression used to determine the target class for an + incoming row, when inheriting classes are present. + + This value is commonly a :class:`.Column` object that's + present in the mapped :class:`.Table`:: + + class Employee(Base): + __tablename__ = 'employee' + + id = Column(Integer, primary_key=True) + discriminator = Column(String(50)) + + __mapper_args__ = { + "polymorphic_on":discriminator, + "polymorphic_identity":"employee" + } + + It may also be specified + as a SQL expression, as in this example where we + use the :func:`.case` construct to provide a conditional + approach:: + + class Employee(Base): + __tablename__ = 'employee' + + id = Column(Integer, primary_key=True) + discriminator = Column(String(50)) + + __mapper_args__ = { + "polymorphic_on":case([ + (discriminator == "EN", "engineer"), + (discriminator == "MA", "manager"), + ], else_="employee"), + "polymorphic_identity":"employee" + } + + It may also refer to any attribute + configured with :func:`.column_property`, or to the + string name of one:: + + class Employee(Base): + __tablename__ = 'employee' + + id = Column(Integer, primary_key=True) + discriminator = Column(String(50)) + employee_type = column_property( + case([ + (discriminator == "EN", "engineer"), + (discriminator == "MA", "manager"), + ], else_="employee") + ) + + __mapper_args__ = { + "polymorphic_on":employee_type, + "polymorphic_identity":"employee" + } + + .. versionchanged:: 0.7.4 + ``polymorphic_on`` may be specified as a SQL expression, + or refer to any attribute configured with + :func:`.column_property`, or to the string name of one. + + When setting ``polymorphic_on`` to reference an + attribute or expression that's not present in the + locally mapped :class:`.Table`, yet the value + of the discriminator should be persisted to the database, + the value of the + discriminator is not automatically set on new + instances; this must be handled by the user, + either through manual means or via event listeners. + A typical approach to establishing such a listener + looks like:: + + from sqlalchemy import event + from sqlalchemy.orm import object_mapper + + @event.listens_for(Employee, "init", propagate=True) + def set_identity(instance, *arg, **kw): + mapper = object_mapper(instance) + instance.discriminator = mapper.polymorphic_identity + + Where above, we assign the value of ``polymorphic_identity`` + for the mapped class to the ``discriminator`` attribute, + thus persisting the value to the ``discriminator`` column + in the database. + + .. seealso:: + + :ref:`inheritance_toplevel` + + :param polymorphic_identity: Specifies the value which + identifies this particular class as returned by the + column expression referred to by the ``polymorphic_on`` + setting. As rows are received, the value corresponding + to the ``polymorphic_on`` column expression is compared + to this value, indicating which subclass should + be used for the newly reconstructed object. + + :param properties: A dictionary mapping the string names of object + attributes to :class:`.MapperProperty` instances, which define the + persistence behavior of that attribute. Note that :class:`.Column` + objects present in + the mapped :class:`.Table` are automatically placed into + ``ColumnProperty`` instances upon mapping, unless overridden. + When using Declarative, this argument is passed automatically, + based on all those :class:`.MapperProperty` instances declared + in the declared class body. + + :param primary_key: A list of :class:`.Column` objects which define the + primary key to be used against this mapper's selectable unit. + This is normally simply the primary key of the ``local_table``, but + can be overridden here. + + :param version_id_col: A :class:`.Column` + that will be used to keep a running version id of rows + in the table. This is used to detect concurrent updates or + the presence of stale data in a flush. The methodology is to + detect if an UPDATE statement does not match the last known + version id, a + :class:`~sqlalchemy.orm.exc.StaleDataError` exception is + thrown. + By default, the column must be of :class:`.Integer` type, + unless ``version_id_generator`` specifies an alternative version + generator. + + .. seealso:: + + :ref:`mapper_version_counter` - discussion of version counting + and rationale. + + :param version_id_generator: Define how new version ids should + be generated. Defaults to ``None``, which indicates that + a simple integer counting scheme be employed. To provide a custom + versioning scheme, provide a callable function of the form:: + + def generate_version(version): + return next_version + + Alternatively, server-side versioning functions such as triggers, + or programmatic versioning schemes outside of the version id generator + may be used, by specifying the value ``False``. + Please see :ref:`server_side_version_counter` for a discussion + of important points when using this option. + + .. versionadded:: 0.9.0 ``version_id_generator`` supports server-side + version number generation. + + .. seealso:: + + :ref:`custom_version_counter` + + :ref:`server_side_version_counter` + + + :param with_polymorphic: A tuple in the form ``(, + )`` indicating the default style of "polymorphic" + loading, that is, which tables are queried at once. is + any single or list of mappers and/or classes indicating the + inherited classes that should be loaded at once. The special value + ``'*'`` may be used to indicate all descending classes should be + loaded immediately. The second tuple argument + indicates a selectable that will be used to query for multiple + classes. + + .. seealso:: + + :ref:`with_polymorphic` - discussion of polymorphic querying techniques. """ @@ -114,9 +504,19 @@ class Mapper(object): self.order_by = order_by self.always_refresh = always_refresh - self.version_id_col = version_id_col - self.version_id_generator = version_id_generator or \ - (lambda x:(x or 0) + 1) + + if isinstance(version_id_col, MapperProperty): + self.version_id_prop = version_id_col + self.version_id_col = None + else: + self.version_id_col = version_id_col + if version_id_generator is False: + self.version_id_generator = False + elif version_id_generator is None: + self.version_id_generator = lambda x: (x or 0) + 1 + else: + self.version_id_generator = version_id_generator + self.concrete = concrete self.single = False self.inherits = inherits @@ -124,14 +524,16 @@ class Mapper(object): self.inherit_condition = inherit_condition self.inherit_foreign_keys = inherit_foreign_keys self._init_properties = properties or {} - self.delete_orphans = [] + self._delete_orphans = [] self.batch = batch self.eager_defaults = eager_defaults self.column_prefix = column_prefix - self.polymorphic_on = expression._clause_element_as_expr(polymorphic_on) + self.polymorphic_on = expression._clause_element_as_expr( + polymorphic_on) self._dependency_processors = [] self.validators = util.immutabledict() self.passive_updates = passive_updates + self.legacy_is_orphan = legacy_is_orphan self._clause_adapter = None self._requires_row_aliasing = False self._inherits_equated_pairs = None @@ -140,18 +542,11 @@ class Mapper(object): self._reconstructor = None self._deprecated_extensions = util.to_list(extension or []) - if allow_null_pks: - util.warn_deprecated( - "the allow_null_pks option to Mapper() is " - "deprecated. It is now allow_partial_pks=False|True, " - "defaults to True.") - allow_partial_pks = allow_null_pks - self.allow_partial_pks = allow_partial_pks self._set_with_polymorphic(with_polymorphic) - if isinstance(self.local_table, expression._SelectBase): + if isinstance(self.local_table, expression.SelectBase): raise sa_exc.InvalidRequestError( "When mapping against a select() construct, map against " "an alias() of the construct instead." @@ -161,7 +556,7 @@ class Mapper(object): if self.with_polymorphic and \ isinstance(self.with_polymorphic[1], - expression._SelectBase): + expression.SelectBase): self.with_polymorphic = (self.with_polymorphic[0], self.with_polymorphic[1].alias()) @@ -190,10 +585,11 @@ class Mapper(object): self.configured = False # prevent this mapper from being constructed - # while a configure_mappers() is occurring (and defer a configure_mappers() - # until construction succeeds) - _COMPILE_MUTEX.acquire() + # while a configure_mappers() is occurring (and defer a + # configure_mappers() until construction succeeds) + _CONFIGURE_MUTEX.acquire() try: + self.dispatch._events._new_mapper_instance(class_, self) self._configure_inheritance() self._configure_legacy_instrument_class() self._configure_class_instrumentation() @@ -201,16 +597,36 @@ class Mapper(object): self._configure_properties() self._configure_polymorphic_setter() self._configure_pks() - global _new_mappers - _new_mappers = True + Mapper._new_mappers = True self._log("constructed") self._expire_memoizations() finally: - _COMPILE_MUTEX.release() + _CONFIGURE_MUTEX.release() # major attributes initialized at the classlevel so that # they can be Sphinx-documented. + is_mapper = True + """Part of the inspection API.""" + + @property + def mapper(self): + """Part of the inspection API. + + Returns self. + + """ + return self + + @property + def entity(self): + """Part of the inspection API. + + Returns self.class\_. + + """ + return self.class_ + local_table = None """The :class:`.Selectable` which this :class:`.Mapper` manages. @@ -226,7 +642,9 @@ class Mapper(object): this :class:`.Mapper` represents. If this mapper is a single-table inheriting mapper, local_table will be ``None``. - See also :attr:`~.Mapper.mapped_table`. + .. seealso:: + + :attr:`~.Mapper.mapped_table`. """ @@ -244,7 +662,9 @@ class Mapper(object): subclass. For single-table inheritance mappers, mapped_table references the base table. - See also :attr:`~.Mapper.local_table`. + .. seealso:: + + :attr:`~.Mapper.local_table`. """ @@ -263,7 +683,9 @@ class Mapper(object): This is a *read only* attribute determined during mapper construction. Behavior is undefined if directly modified. - See also :func:`.configure_mappers`. + .. seealso:: + + :func:`.configure_mappers`. """ @@ -294,11 +716,11 @@ class Mapper(object): which comprise the 'primary key' of the mapped table, from the perspective of this :class:`.Mapper`. - This list is against the selectable in :attr:`~.Mapper.mapped_table`. In the - case of inheriting mappers, some columns may be managed by a superclass - mapper. For example, in the case of a :class:`.Join`, the primary - key is determined by all of the primary key columns across all tables - referenced by the :class:`.Join`. + This list is against the selectable in :attr:`~.Mapper.mapped_table`. In + the case of inheriting mappers, some columns may be managed by a + superclass mapper. For example, in the case of a :class:`.Join`, the + primary key is determined by all of the primary key columns across all + tables referenced by the :class:`.Join`. The list is also not necessarily the same as the primary key column collection associated with the underlying tables; the :class:`.Mapper` @@ -349,11 +771,13 @@ class Mapper(object): """ polymorphic_on = None - """The :class:`.Column` specified as the ``polymorphic_on`` column + """The :class:`.Column` or SQL expression specified as the + ``polymorphic_on`` argument for this :class:`.Mapper`, within an inheritance scenario. - This attribute may also be of other types besides :class:`.Column` - in a future SQLAlchemy release. + This attribute is normally a :class:`.Column` instance but + may also be an expression, such as one derived from + :func:`.cast`. This is a *read only* attribute determined during mapper construction. Behavior is undefined if directly modified. @@ -361,8 +785,8 @@ class Mapper(object): """ polymorphic_map = None - """A mapping of "polymorphic identity" identifiers mapped to :class:`.Mapper` - instances, within an inheritance scenario. + """A mapping of "polymorphic identity" identifiers mapped to + :class:`.Mapper` instances, within an inheritance scenario. The identifiers can be of any type which is comparable to the type of column represented by :attr:`~.Mapper.polymorphic_on`. @@ -377,11 +801,12 @@ class Mapper(object): """ polymorphic_identity = None - """Represent an identifier which is matched against the :attr:`~.Mapper.polymorphic_on` - column during result row loading. + """Represent an identifier which is matched against the + :attr:`~.Mapper.polymorphic_on` column during result row loading. Used only with inheritance, this object can be of any type which is - comparable to the type of column represented by :attr:`~.Mapper.polymorphic_on`. + comparable to the type of column represented by + :attr:`~.Mapper.polymorphic_on`. This is a *read only* attribute determined during mapper construction. Behavior is undefined if directly modified. @@ -429,18 +854,20 @@ class Mapper(object): c = None """A synonym for :attr:`~.Mapper.columns`.""" - dispatch = event.dispatcher(events.MapperEvents) + @util.memoized_property + def _path_registry(self): + return PathRegistry.per_mapper(self) def _configure_inheritance(self): """Configure settings related to inherting and/or inherited mappers being present.""" # a set of all mappers which inherit from this one. - self._inheriting_mappers = set() + self._inheriting_mappers = util.WeakSequence() if self.inherits: if isinstance(self.inherits, type): - self.inherits = class_mapper(self.inherits, compile=False) + self.inherits = class_mapper(self.inherits, configure=False) if not issubclass(self.class_, self.inherits.class_): raise sa_exc.ArgumentError( "Class '%s' does not inherit from '%s'" % @@ -468,7 +895,7 @@ class Mapper(object): # immediate table of the inherited mapper, not its # full table which could pull in other stuff we dont # want (allows test/inheritance.InheritTest4 to pass) - self.inherit_condition = sqlutil.join_condition( + self.inherit_condition = sql_util.join_condition( self.inherits.local_table, self.local_table) self.mapped_table = sql.join( @@ -477,7 +904,7 @@ class Mapper(object): self.inherit_condition) fks = util.to_set(self.inherit_foreign_keys) - self._inherits_equated_pairs = sqlutil.criterion_as_pairs( + self._inherits_equated_pairs = sql_util.criterion_as_pairs( self.mapped_table.onclause, consider_as_foreign_keys=fks) else: @@ -510,7 +937,7 @@ class Mapper(object): self.polymorphic_map = self.inherits.polymorphic_map self.batch = self.inherits.batch - self.inherits._inheriting_mappers.add(self) + self.inherits._inheriting_mappers.append(self) self.base_mapper = self.inherits.base_mapper self.passive_updates = self.inherits.passive_updates self._all_tables = self.inherits._all_tables @@ -535,7 +962,7 @@ class Mapper(object): if with_polymorphic == '*': self.with_polymorphic = ('*', None) elif isinstance(with_polymorphic, (tuple, list)): - if isinstance(with_polymorphic[0], (basestring, tuple, list)): + if isinstance(with_polymorphic[0], util.string_types + (tuple, list)): self.with_polymorphic = with_polymorphic else: self.with_polymorphic = (with_polymorphic, None) @@ -544,7 +971,7 @@ class Mapper(object): else: self.with_polymorphic = None - if isinstance(self.local_table, expression._SelectBase): + if isinstance(self.local_table, expression.SelectBase): raise sa_exc.InvalidRequestError( "When mapping against a select() construct, map against " "an alias() of the construct instead." @@ -554,16 +981,16 @@ class Mapper(object): if self.with_polymorphic and \ isinstance(self.with_polymorphic[1], - expression._SelectBase): + expression.SelectBase): self.with_polymorphic = (self.with_polymorphic[0], self.with_polymorphic[1].alias()) if self.configured: self._expire_memoizations() def _set_concrete_base(self, mapper): - """Set the given :class:`.Mapper` as the 'inherits' for this :class:`.Mapper`, - assuming this :class:`.Mapper` is concrete and does not already have - an inherits.""" + """Set the given :class:`.Mapper` as the 'inherits' for this + :class:`.Mapper`, assuming this :class:`.Mapper` is concrete + and does not already have an inherits.""" assert self.concrete assert not self.inherits @@ -577,16 +1004,15 @@ class Mapper(object): self.batch = self.inherits.batch for mp in self.self_and_descendants: mp.base_mapper = self.inherits.base_mapper - self.inherits._inheriting_mappers.add(self) + self.inherits._inheriting_mappers.append(self) self.passive_updates = self.inherits.passive_updates self._all_tables = self.inherits._all_tables - for key, prop in mapper._props.iteritems(): + for key, prop in mapper._props.items(): if key not in self._props and \ not self._should_exclude(key, key, local=False, column=None): self._adapt_inherited_property(key, prop, False) - def _set_polymorphic_on(self, polymorphic_on): self.polymorphic_on = polymorphic_on self._configure_polymorphic_setter(True) @@ -595,8 +1021,9 @@ class Mapper(object): if self.inherits: self.dispatch._update(self.inherits.dispatch) - super_extensions = set(chain(*[m._deprecated_extensions - for m in self.inherits.iterate_to_root()])) + super_extensions = set( + chain(*[m._deprecated_extensions + for m in self.inherits.iterate_to_root()])) else: super_extensions = set() @@ -606,8 +1033,9 @@ class Mapper(object): def _configure_listeners(self): if self.inherits: - super_extensions = set(chain(*[m._deprecated_extensions - for m in self.inherits.iterate_to_root()])) + super_extensions = set( + chain(*[m._deprecated_extensions + for m in self.inherits.iterate_to_root()])) else: super_extensions = set() @@ -615,10 +1043,6 @@ class Mapper(object): if ext not in super_extensions: ext._adapt_listener(self, ext) - if self.inherits: - self.class_manager.dispatch._update( - self.inherits.class_manager.dispatch) - def _configure_class_instrumentation(self): """If this mapper is to be a primary mapper (i.e. the non_primary flag is not set), associate this Mapper with the @@ -667,7 +1091,8 @@ class Mapper(object): self.class_manager = manager manager.mapper = self - manager.deferred_scalar_loader = self._load_scalar_attributes + manager.deferred_scalar_loader = util.partial( + loading.load_scalar_attributes, self) # The remaining members can be added by any mapper, # e_name None or not. @@ -684,30 +1109,20 @@ class Mapper(object): self._reconstructor = method event.listen(manager, 'load', _event_on_load, raw=True) elif hasattr(method, '__sa_validators__'): - include_removes = getattr(method, "__sa_include_removes__", False) + validation_opts = method.__sa_validation_opts__ for name in method.__sa_validators__: self.validators = self.validators.union( - {name : (method, include_removes)} + {name: (method, validation_opts)} ) manager.info[_INSTRUMENTOR] = self - @util.deprecated("0.7", message=":meth:`.Mapper.compile` " - "is replaced by :func:`.configure_mappers`") - def compile(self): - """Initialize the inter-mapper relationships of all mappers that - have been constructed thus far. + @classmethod + def _configure_all(cls): + """Class-level path to the :func:`.configure_mappers` call. """ configure_mappers() - return self - - - @property - @util.deprecated("0.7", message=":attr:`.Mapper.compiled` " - "is replaced by :attr:`.Mapper.configured`") - def compiled(self): - return self.configured def dispose(self): # Disable any attribute-based compilation. @@ -717,13 +1132,14 @@ class Mapper(object): del self._configure_failed if not self.non_primary and \ + self.class_manager is not None and \ self.class_manager.is_mapped and \ - self.class_manager.mapper is self: + self.class_manager.mapper is self: instrumentation.unregister_class(self.class_) def _configure_pks(self): - self.tables = sqlutil.find_tables(self.mapped_table) + self.tables = sql_util.find_tables(self.mapped_table) self._pks_by_table = {} self._cols_by_table = {} @@ -741,7 +1157,7 @@ class Mapper(object): if t.primary_key and pk_cols.issuperset(t.primary_key): # ordering is important since it determines the ordering of # mapper.primary_key (and therefore query.get()) - self._pks_by_table[t] =\ + self._pks_by_table[t] = \ util.ordered_column_set(t.primary_key).\ intersection(pk_cols) self._cols_by_table[t] = \ @@ -788,12 +1204,12 @@ class Mapper(object): # determine primary key from argument or mapped_table pks - # reduce to the minimal set of columns if self._primary_key_argument: - primary_key = sqlutil.reduce_columns( + primary_key = sql_util.reduce_columns( [self.mapped_table.corresponding_column(c) for c in self._primary_key_argument], ignore_nonexistent_tables=True) else: - primary_key = sqlutil.reduce_columns( + primary_key = sql_util.reduce_columns( self._pks_by_table[self.mapped_table], ignore_nonexistent_tables=True) @@ -821,14 +1237,15 @@ class Mapper(object): # load custom properties if self._init_properties: - for key, prop in self._init_properties.iteritems(): + for key, prop in self._init_properties.items(): self._configure_property(key, prop, False) # pull properties from the inherited mapper if any. if self.inherits: - for key, prop in self.inherits._props.iteritems(): + for key, prop in self.inherits._props.items(): if key not in self._props and \ - not self._should_exclude(key, key, local=False, column=None): + not self._should_exclude(key, key, local=False, + column=None): self._adapt_inherited_property(key, prop, False) # create properties for each column in the mapped table, @@ -873,7 +1290,7 @@ class Mapper(object): if self.polymorphic_on is not None: setter = True - if isinstance(self.polymorphic_on, basestring): + if isinstance(self.polymorphic_on, util.string_types): # polymorphic_on specified as as string - link # it to mapped ColumnProperty try: @@ -894,7 +1311,8 @@ class Mapper(object): elif isinstance(self.polymorphic_on, MapperProperty): # polymorphic_on is directly a MapperProperty, # ensure it's a ColumnProperty - if not isinstance(self.polymorphic_on, properties.ColumnProperty): + if not isinstance(self.polymorphic_on, + properties.ColumnProperty): raise sa_exc.ArgumentError( "Only direct column-mapped " "property or SQL expression " @@ -902,7 +1320,7 @@ class Mapper(object): prop = self.polymorphic_on self.polymorphic_on = prop.columns[0] polymorphic_key = prop.key - elif not expression.is_column(self.polymorphic_on): + elif not expression._is_column(self.polymorphic_on): # polymorphic_on is not a Column and not a ColumnProperty; # not supported right now. raise sa_exc.ArgumentError( @@ -911,27 +1329,31 @@ class Mapper(object): "can be passed for polymorphic_on" ) else: - # polymorphic_on is a Column or SQL expression and doesn't - # appear to be mapped. - # this means it can be 1. only present in the with_polymorphic - # selectable or 2. a totally standalone SQL expression which we'd + # polymorphic_on is a Column or SQL expression and + # doesn't appear to be mapped. this means it can be 1. + # only present in the with_polymorphic selectable or + # 2. a totally standalone SQL expression which we'd # hope is compatible with this mapper's mapped_table - col = self.mapped_table.corresponding_column(self.polymorphic_on) + col = self.mapped_table.corresponding_column( + self.polymorphic_on) if col is None: - # polymorphic_on doesn't derive from any column/expression - # isn't present in the mapped table. - # we will make a "hidden" ColumnProperty for it. - # Just check that if it's directly a schema.Column and we - # have with_polymorphic, it's likely a user error if the - # schema.Column isn't represented somehow in either mapped_table or - # with_polymorphic. Otherwise as of 0.7.4 we just go with it - # and assume the user wants it that way (i.e. a CASE statement) + # polymorphic_on doesn't derive from any + # column/expression isn't present in the mapped + # table. we will make a "hidden" ColumnProperty + # for it. Just check that if it's directly a + # schema.Column and we have with_polymorphic, it's + # likely a user error if the schema.Column isn't + # represented somehow in either mapped_table or + # with_polymorphic. Otherwise as of 0.7.4 we + # just go with it and assume the user wants it + # that way (i.e. a CASE statement) setter = False instrument = False col = self.polymorphic_on if isinstance(col, schema.Column) and ( self.with_polymorphic is None or \ - self.with_polymorphic[1].corresponding_column(col) is None + self.with_polymorphic[1].\ + corresponding_column(col) is None ): raise sa_exc.InvalidRequestError( "Could not map polymorphic_on column " @@ -944,22 +1366,26 @@ class Mapper(object): # and is probably mapped, but polymorphic_on itself # is not. This happens when # the polymorphic_on is only directly present in the - # with_polymorphic selectable, as when use polymorphic_union. + # with_polymorphic selectable, as when use + # polymorphic_union. # we'll make a separate ColumnProperty for it. instrument = True key = getattr(col, 'key', None) if key: if self._should_exclude(col.key, col.key, False, col): raise sa_exc.InvalidRequestError( - "Cannot exclude or override the discriminator column %r" % + "Cannot exclude or override the " + "discriminator column %r" % col.key) else: - self.polymorphic_on = col = col.label("_sa_polymorphic_on") + self.polymorphic_on = col = \ + col.label("_sa_polymorphic_on") key = col.key self._configure_property( key, - properties.ColumnProperty(col, _instrument=instrument), + properties.ColumnProperty(col, + _instrument=instrument), init=init, setparent=True) polymorphic_key = key else: @@ -982,7 +1408,10 @@ class Mapper(object): # directly; it ensures the polymorphic_identity of the # instance's mapper is used so is portable to subclasses. if self.polymorphic_on is not None: - self._set_polymorphic_identity = mapper._set_polymorphic_identity + self._set_polymorphic_identity = \ + mapper._set_polymorphic_identity + self._validate_polymorphic_identity = \ + mapper._validate_polymorphic_identity else: self._set_polymorphic_identity = None return @@ -993,11 +1422,46 @@ class Mapper(object): state.get_impl(polymorphic_key).set(state, dict_, state.manager.mapper.polymorphic_identity, None) + def _validate_polymorphic_identity(mapper, state, dict_): + if polymorphic_key in dict_ and \ + dict_[polymorphic_key] not in \ + mapper._acceptable_polymorphic_identities: + util.warn( + "Flushing object %s with " + "incompatible polymorphic identity %r; the " + "object may not refresh and/or load correctly" % ( + state_str(state), + dict_[polymorphic_key] + ) + ) + self._set_polymorphic_identity = _set_polymorphic_identity + self._validate_polymorphic_identity = _validate_polymorphic_identity else: self._set_polymorphic_identity = None + _validate_polymorphic_identity = None + + @_memoized_configured_property + def _version_id_prop(self): + if self.version_id_col is not None: + return self._columntoproperty[self.version_id_col] + else: + return None + + @_memoized_configured_property + def _acceptable_polymorphic_identities(self): + identities = set() + + stack = deque([self]) + while stack: + item = stack.popleft() + if item.mapped_table is self.mapped_table: + identities.add(item.polymorphic_identity) + stack.extend(item._inheriting_mappers) + + return identities def _adapt_inherited_property(self, key, prop, init): if not self.concrete: @@ -1012,67 +1476,7 @@ class Mapper(object): self._log("_configure_property(%s, %s)", key, prop.__class__.__name__) if not isinstance(prop, MapperProperty): - # we were passed a Column or a list of Columns; - # generate a properties.ColumnProperty - columns = util.to_list(prop) - column = columns[0] - if not expression.is_column(column): - raise sa_exc.ArgumentError( - "%s=%r is not an instance of MapperProperty or Column" - % (key, prop)) - - prop = self._props.get(key, None) - - if isinstance(prop, properties.ColumnProperty): - if prop.parent is self: - raise sa_exc.InvalidRequestError( - "Implicitly combining column %s with column " - "%s under attribute '%s'. Please configure one " - "or more attributes for these same-named columns " - "explicitly." - % (prop.columns[-1], column, key)) - - # existing properties.ColumnProperty from an inheriting - # mapper. make a copy and append our column to it - prop = prop.copy() - prop.columns.insert(0, column) - self._log("inserting column to existing list " - "in properties.ColumnProperty %s" % (key)) - - elif prop is None or isinstance(prop, properties.ConcreteInheritedProperty): - mapped_column = [] - for c in columns: - mc = self.mapped_table.corresponding_column(c) - if mc is None: - mc = self.local_table.corresponding_column(c) - if mc is not None: - # if the column is in the local table but not the - # mapped table, this corresponds to adding a - # column after the fact to the local table. - # [ticket:1523] - self.mapped_table._reset_exported() - mc = self.mapped_table.corresponding_column(c) - if mc is None: - raise sa_exc.ArgumentError( - "When configuring property '%s' on %s, " - "column '%s' is not represented in the mapper's " - "table. Use the `column_property()` function to " - "force this column to be mapped as a read-only " - "attribute." % (key, self, c)) - mapped_column.append(mc) - prop = properties.ColumnProperty(*mapped_column) - else: - raise sa_exc.ArgumentError( - "WARNING: when configuring property '%s' on %s, " - "column '%s' conflicts with property '%r'. " - "To resolve this, map the column to the class under a " - "different name in the 'properties' dictionary. Or, " - "to remove all awareness of the column entirely " - "(including its availability as a foreign key), " - "use the 'include_properties' or 'exclude_properties' " - "mapper arguments to control specifically which table " - "columns get mapped." % - (key, self, column.key, prop)) + prop = self._property_from_column(key, prop) if isinstance(prop, properties.ColumnProperty): col = self.mapped_table.corresponding_column(prop.columns[0]) @@ -1101,7 +1505,7 @@ class Mapper(object): # initialized; check for 'readonly' if hasattr(self, '_readonly_props') and \ (not hasattr(col, 'table') or - col.table not in self._cols_by_table): + col.table not in self._cols_by_table): self._readonly_props.add(prop) else: @@ -1139,6 +1543,16 @@ class Mapper(object): "%r for column %r" % (syn, key, key, syn) ) + if key in self._props and \ + not isinstance(prop, properties.ColumnProperty) and \ + not isinstance(self._props[key], properties.ColumnProperty): + util.warn("Property %s on %s being replaced with new " + "property %s; the old property will be discarded" % ( + self._props[key], + self, + prop, + )) + self._props[key] = prop if not self.non_primary: @@ -1154,6 +1568,73 @@ class Mapper(object): if self.configured: self._expire_memoizations() + def _property_from_column(self, key, prop): + """generate/update a :class:`.ColumnProprerty` given a + :class:`.Column` object. """ + + # we were passed a Column or a list of Columns; + # generate a properties.ColumnProperty + columns = util.to_list(prop) + column = columns[0] + if not expression._is_column(column): + raise sa_exc.ArgumentError( + "%s=%r is not an instance of MapperProperty or Column" + % (key, prop)) + + prop = self._props.get(key, None) + + if isinstance(prop, properties.ColumnProperty): + if prop.parent is self: + raise sa_exc.InvalidRequestError( + "Implicitly combining column %s with column " + "%s under attribute '%s'. Please configure one " + "or more attributes for these same-named columns " + "explicitly." + % (prop.columns[-1], column, key)) + + # existing properties.ColumnProperty from an inheriting + # mapper. make a copy and append our column to it + prop = prop.copy() + prop.columns.insert(0, column) + self._log("inserting column to existing list " + "in properties.ColumnProperty %s" % (key)) + return prop + elif prop is None or isinstance(prop, + properties.ConcreteInheritedProperty): + mapped_column = [] + for c in columns: + mc = self.mapped_table.corresponding_column(c) + if mc is None: + mc = self.local_table.corresponding_column(c) + if mc is not None: + # if the column is in the local table but not the + # mapped table, this corresponds to adding a + # column after the fact to the local table. + # [ticket:1523] + self.mapped_table._reset_exported() + mc = self.mapped_table.corresponding_column(c) + if mc is None: + raise sa_exc.ArgumentError( + "When configuring property '%s' on %s, " + "column '%s' is not represented in the mapper's " + "table. Use the `column_property()` function to " + "force this column to be mapped as a read-only " + "attribute." % (key, self, c)) + mapped_column.append(mc) + return properties.ColumnProperty(*mapped_column) + else: + raise sa_exc.ArgumentError( + "WARNING: when configuring property '%s' on %s, " + "column '%s' conflicts with property '%r'. " + "To resolve this, map the column to the class under a " + "different name in the 'properties' dictionary. Or, " + "to remove all awareness of the column entirely " + "(including its availability as a foreign key), " + "use the 'include_properties' or 'exclude_properties' " + "mapper arguments to control specifically which table " + "columns get mapped." % + (key, self, column.key, prop)) + def _post_configure_properties(self): """Call the ``init()`` method on all ``MapperProperties`` attached to this mapper. @@ -1164,14 +1645,14 @@ class Mapper(object): """ self._log("_post_configure_properties() started") - l = [(key, prop) for key, prop in self._props.iteritems()] + l = [(key, prop) for key, prop in self._props.items()] for key, prop in l: self._log("initialize prop %s", key) - if prop.parent is self and not prop._compile_started: + if prop.parent is self and not prop._configure_started: prop.init() - if prop._compile_finished: + if prop._configure_finished: prop.post_instrument_class(self) self._log("_post_configure_properties() complete") @@ -1182,7 +1663,7 @@ class Mapper(object): using `add_property`. """ - for key, value in dict_of_properties.iteritems(): + for key, value in dict_of_properties.items(): self.add_property(key, value) def add_property(self, key, prop): @@ -1212,7 +1693,6 @@ class Mapper(object): "|non-primary" or "") + ")" def _log(self, msg, *args): - self.logger.info( "%s " + msg, *((self._log_desc,) + args) ) @@ -1235,23 +1715,32 @@ class Mapper(object): ) def _is_orphan(self, state): - o = False + orphan_possible = False for mapper in self.iterate_to_root(): - for (key, cls) in mapper.delete_orphans: - if attributes.manager_of_class(cls).has_parent( - state, key, optimistic=bool(state.key)): + for (key, cls) in mapper._delete_orphans: + orphan_possible = True + + has_parent = attributes.manager_of_class(cls).has_parent( + state, key, optimistic=state.has_identity) + + if self.legacy_is_orphan and has_parent: return False - o = o or bool(mapper.delete_orphans) - return o + elif not self.legacy_is_orphan and not has_parent: + return True + + if self.legacy_is_orphan: + return orphan_possible + else: + return False def has_property(self, key): return key in self._props - def get_property(self, key, _compile_mappers=True): + def get_property(self, key, _configure_mappers=True): """return a MapperProperty associated with the given key. """ - if _compile_mappers and _new_mappers: + if _configure_mappers and Mapper._new_mappers: configure_mappers() try: @@ -1260,12 +1749,6 @@ class Mapper(object): raise sa_exc.InvalidRequestError( "Mapper '%s' has no property '%s'" % (self, key)) - @util.deprecated('0.6.4', - 'Call to deprecated function mapper._get_col_to_pr' - 'op(). Use mapper.get_property_by_column()') - def _get_col_to_prop(self, col): - return self._columntoproperty[col] - def get_property_by_column(self, column): """Given a :class:`.Column` object, return the :class:`.MapperProperty` which maps this column.""" @@ -1275,9 +1758,9 @@ class Mapper(object): @property def iterate_properties(self): """return an iterator of all MapperProperty objects.""" - if _new_mappers: + if Mapper._new_mappers: configure_mappers() - return self._props.itervalues() + return iter(self._props.values()) def _mappers_from_spec(self, spec, selectable): """given a with_polymorphic() argument, return the set of mappers it @@ -1290,29 +1773,34 @@ class Mapper(object): if spec == '*': mappers = list(self.self_and_descendants) elif spec: - mappers = [_class_to_mapper(m) for m in util.to_list(spec)] - for m in mappers: + mappers = set() + for m in util.to_list(spec): + m = _class_to_mapper(m) if not m.isa(self): raise sa_exc.InvalidRequestError( - "%r does not inherit from %r" % + "%r does not inherit from %r" % (m, self)) + + if selectable is None: + mappers.update(m.iterate_to_root()) + else: + mappers.add(m) + mappers = [m for m in self.self_and_descendants if m in mappers] else: mappers = [] if selectable is not None: - tables = set(sqlutil.find_tables(selectable, + tables = set(sql_util.find_tables(selectable, include_aliases=True)) mappers = [m for m in mappers if m.local_table in tables] - return mappers - def _selectable_from_mappers(self, mappers): + def _selectable_from_mappers(self, mappers, innerjoin): """given a list of mappers (assumed to be within this mapper's inheritance hierarchy), construct an outerjoin amongst those mapper's mapped tables. """ - from_obj = self.mapped_table for m in mappers: if m is self: @@ -1322,7 +1810,11 @@ class Mapper(object): "'with_polymorphic()' requires 'selectable' argument " "when concrete-inheriting mappers are used.") elif not m.single: - from_obj = from_obj.outerjoin(m.local_table, + if innerjoin: + from_obj = from_obj.join(m.local_table, + m.inherit_condition) + else: + from_obj = from_obj.outerjoin(m.local_table, m.inherit_condition) return from_obj @@ -1340,8 +1832,10 @@ class Mapper(object): @_memoized_configured_property def _with_polymorphic_mappers(self): + if Mapper._new_mappers: + configure_mappers() if not self.with_polymorphic: - return [self] + return [] return self._mappers_from_spec(*self.with_polymorphic) @_memoized_configured_property @@ -1354,30 +1848,52 @@ class Mapper(object): return selectable else: return self._selectable_from_mappers( - self._mappers_from_spec(spec, selectable)) + self._mappers_from_spec(spec, selectable), + False) - def _with_polymorphic_args(self, spec=None, selectable=False): + with_polymorphic_mappers = _with_polymorphic_mappers + """The list of :class:`.Mapper` objects included in the + default "polymorphic" query. + + """ + + @property + def selectable(self): + """The :func:`.select` construct this :class:`.Mapper` selects from + by default. + + Normally, this is equivalent to :attr:`.mapped_table`, unless + the ``with_polymorphic`` feature is in use, in which case the + full "polymorphic" selectable is returned. + + """ + return self._with_polymorphic_selectable + + def _with_polymorphic_args(self, spec=None, selectable=False, + innerjoin=False): if self.with_polymorphic: if not spec: spec = self.with_polymorphic[0] if selectable is False: selectable = self.with_polymorphic[1] - + elif selectable is False: + selectable = None mappers = self._mappers_from_spec(spec, selectable) if selectable is not None: return mappers, selectable else: - return mappers, self._selectable_from_mappers(mappers) + return mappers, self._selectable_from_mappers(mappers, + innerjoin) @_memoized_configured_property def _polymorphic_properties(self): - return tuple(self._iterate_polymorphic_properties( + return list(self._iterate_polymorphic_properties( self._with_polymorphic_mappers)) + def _iterate_polymorphic_properties(self, mappers=None): """Return an iterator of MapperProperty objects which will render into a SELECT.""" - if mappers is None: mappers = self._with_polymorphic_mappers @@ -1389,8 +1905,10 @@ class Mapper(object): # from other mappers, as these are sometimes dependent on that # mapper's polymorphic selectable (which we don't want rendered) for c in util.unique_list( - chain(*[list(mapper.iterate_properties) for mapper in [self] + - mappers]) + chain(*[ + list(mapper.iterate_properties) for mapper in + [self] + mappers + ]) ): if getattr(c, '_is_polymorphic_discriminator', False) and \ (self.polymorphic_on is None or @@ -1398,12 +1916,126 @@ class Mapper(object): continue yield c - @property - def properties(self): - raise NotImplementedError( - "Public collection of MapperProperty objects is " - "provided by the get_property() and iterate_properties " - "accessors.") + @util.memoized_property + def attrs(self): + """A namespace of all :class:`.MapperProperty` objects + associated this mapper. + + This is an object that provides each property based on + its key name. For instance, the mapper for a + ``User`` class which has ``User.name`` attribute would + provide ``mapper.attrs.name``, which would be the + :class:`.ColumnProperty` representing the ``name`` + column. The namespace object can also be iterated, + which would yield each :class:`.MapperProperty`. + + :class:`.Mapper` has several pre-filtered views + of this attribute which limit the types of properties + returned, inclding :attr:`.synonyms`, :attr:`.column_attrs`, + :attr:`.relationships`, and :attr:`.composites`. + + .. seealso:: + + :attr:`.Mapper.all_orm_descriptors` + + """ + if Mapper._new_mappers: + configure_mappers() + return util.ImmutableProperties(self._props) + + @util.memoized_property + def all_orm_descriptors(self): + """A namespace of all :class:`._InspectionAttr` attributes associated + with the mapped class. + + These attributes are in all cases Python :term:`descriptors` associated + with the mapped class or its superclasses. + + This namespace includes attributes that are mapped to the class + as well as attributes declared by extension modules. + It includes any Python descriptor type that inherits from + :class:`._InspectionAttr`. This includes :class:`.QueryableAttribute`, + as well as extension types such as :class:`.hybrid_property`, + :class:`.hybrid_method` and :class:`.AssociationProxy`. + + To distinguish between mapped attributes and extension attributes, + the attribute :attr:`._InspectionAttr.extension_type` will refer + to a constant that distinguishes between different extension types. + + When dealing with a :class:`.QueryableAttribute`, the + :attr:`.QueryableAttribute.property` attribute refers to the + :class:`.MapperProperty` property, which is what you get when referring + to the collection of mapped properties via :attr:`.Mapper.attrs`. + + .. versionadded:: 0.8.0 + + .. seealso:: + + :attr:`.Mapper.attrs` + + """ + return util.ImmutableProperties( + dict(self.class_manager._all_sqla_attributes())) + + @_memoized_configured_property + def synonyms(self): + """Return a namespace of all :class:`.SynonymProperty` + properties maintained by this :class:`.Mapper`. + + .. seealso:: + + :attr:`.Mapper.attrs` - namespace of all :class:`.MapperProperty` + objects. + + """ + return self._filter_properties(properties.SynonymProperty) + + @_memoized_configured_property + def column_attrs(self): + """Return a namespace of all :class:`.ColumnProperty` + properties maintained by this :class:`.Mapper`. + + .. seealso:: + + :attr:`.Mapper.attrs` - namespace of all :class:`.MapperProperty` + objects. + + """ + return self._filter_properties(properties.ColumnProperty) + + @_memoized_configured_property + def relationships(self): + """Return a namespace of all :class:`.RelationshipProperty` + properties maintained by this :class:`.Mapper`. + + .. seealso:: + + :attr:`.Mapper.attrs` - namespace of all :class:`.MapperProperty` + objects. + + """ + return self._filter_properties(properties.RelationshipProperty) + + @_memoized_configured_property + def composites(self): + """Return a namespace of all :class:`.CompositeProperty` + properties maintained by this :class:`.Mapper`. + + .. seealso:: + + :attr:`.Mapper.attrs` - namespace of all :class:`.MapperProperty` + objects. + + """ + return self._filter_properties(properties.CompositeProperty) + + def _filter_properties(self, type_): + if Mapper._new_mappers: + configure_mappers() + return util.ImmutableProperties(util.OrderedDict( + (k, v) for k, v in self._props.items() + if isinstance(v, type_) + )) @_memoized_configured_property def _get_clause(self): @@ -1414,7 +2046,7 @@ class Mapper(object): """ params = [(primary_key, sql.bindparam(None, type_=primary_key.type)) for primary_key in self.primary_key] - return sql.and_(*[k==v for (k, v) in params]), \ + return sql.and_(*[k == v for (k, v) in params]), \ util.column_dict(params) @_memoized_configured_property @@ -1439,6 +2071,7 @@ class Mapper(object): """ result = util.column_dict() + def visit_binary(binary): if binary.operator == operators.eq: if binary.left in result: @@ -1453,24 +2086,17 @@ class Mapper(object): if mapper.inherit_condition is not None: visitors.traverse( mapper.inherit_condition, {}, - {'binary':visit_binary}) + {'binary': visit_binary}) return result def _is_userland_descriptor(self, obj): - if isinstance(obj, (MapperProperty, - attributes.QueryableAttribute)): - return False - elif not hasattr(obj, '__get__'): + if isinstance(obj, (_MappedAttribute, + instrumentation.ClassManager, + expression.ColumnElement)): return False else: - obj = util.unbound_method_to_callable(obj) - if isinstance( - obj.__get__(None, obj), - attributes.QueryableAttribute - ): - return False - return True + return True def _should_exclude(self, name, assigned_name, local, column): """determine whether a particular property should be implicitly @@ -1481,8 +2107,8 @@ class Mapper(object): """ - # check for descriptors, either local or from - # an inherited class + # check for class-bound attributes and/or descriptors, + # either local or from an inherited class if local: if self.class_.__dict__.get(assigned_name, None) is not None \ and self._is_userland_descriptor( @@ -1551,7 +2177,7 @@ class Mapper(object): item = stack.popleft() descendants.append(item) stack.extend(item._inheriting_mappers) - return tuple(descendants) + return util.WeakSequence(descendants) def polymorphic_iterator(self): """Iterate through the collection including this mapper and @@ -1580,10 +2206,11 @@ class Mapper(object): """Return an identity-map key for use in storing/retrieving an item from the identity map. - row - A ``sqlalchemy.engine.base.RowProxy`` instance or a - dictionary corresponding result-set ``ColumnElement`` - instances to their values within a row. + :param row: A :class:`.RowProxy` instance. The columns which are mapped + by this :class:`.Mapper` should be locatable in the row, preferably + via the :class:`.Column` object directly (as is the case when a + :func:`.select` construct is executed), or via string names of the form + ``_``. """ pk_cols = self.primary_key @@ -1597,8 +2224,7 @@ class Mapper(object): """Return an identity-map key for use in storing/retrieving an item from an identity map. - primary_key - A list of values indicating the identifier. + :param primary_key: A list of values indicating the identifier. """ return self._identity_class, tuple(primary_key) @@ -1607,6 +2233,11 @@ class Mapper(object): """Return the identity key for the given instance, based on its primary key attributes. + If the instance's state is expired, calling this method + will result in a database check to see if the object has been deleted. + If the row no longer exists, + :class:`~sqlalchemy.orm.exc.ObjectDeletedError` is raised. + This value is typically also found on the instance state under the attribute name `key`. @@ -1627,6 +2258,11 @@ class Mapper(object): """Return the list of primary key values for the given instance. + If the instance's state is expired, calling this method + will result in a database check to see if the object has been deleted. + If the row no longer exists, + :class:`~sqlalchemy.orm.exc.ObjectDeletedError` is raised. + """ state = attributes.instance_state(instance) return self._primary_key_from_state(state) @@ -1661,63 +2297,6 @@ class Mapper(object): return state.manager[prop.key].impl.\ get_committed_value(state, dict_, passive=passive) - def _load_scalar_attributes(self, state, attribute_names): - """initiate a column-based attribute refresh operation.""" - - #assert mapper is _state_mapper(state) - session = sessionlib._state_session(state) - if not session: - raise orm_exc.DetachedInstanceError( - "Instance %s is not bound to a Session; " - "attribute refresh operation cannot proceed" % - (state_str(state))) - - has_key = bool(state.key) - - result = False - - if self.inherits and not self.concrete: - statement = self._optimized_get_statement(state, attribute_names) - if statement is not None: - result = session.query(self).from_statement(statement).\ - _load_on_ident(None, - only_load_props=attribute_names, - refresh_state=state) - - if result is False: - if has_key: - identity_key = state.key - else: - # this codepath is rare - only valid when inside a flush, and the - # object is becoming persistent but hasn't yet been assigned an identity_key. - # check here to ensure we have the attrs we need. - pk_attrs = [self._columntoproperty[col].key - for col in self.primary_key] - if state.expired_attributes.intersection(pk_attrs): - raise sa_exc.InvalidRequestError("Instance %s cannot be refreshed - it's not " - " persistent and does not " - "contain a full primary key." % state_str(state)) - identity_key = self._identity_key_from_state(state) - - if (_none_set.issubset(identity_key) and \ - not self.allow_partial_pks) or \ - _none_set.issuperset(identity_key): - util.warn("Instance %s to be refreshed doesn't " - "contain a full primary key - can't be refreshed " - "(and shouldn't be expired, either)." - % state_str(state)) - return - - result = session.query(self)._load_on_ident( - identity_key, - refresh_state=state, - only_load_props=attribute_names) - - # if instance is pending, a refresh operation - # may not complete (even if PK attributes are assigned) - if has_key and result is None: - raise orm_exc.ObjectDeletedError(state) - def _optimized_get_statement(self, state, attribute_names): """assemble a WHERE clause which retrieves a given state by primary key, using a minimized set of tables. @@ -1731,7 +2310,7 @@ class Mapper(object): props = self._props tables = set(chain( - *[sqlutil.find_tables(c, check_columns=True) + *[sql_util.find_tables(c, check_columns=True) for key in attribute_names for c in props[key].columns] )) @@ -1762,7 +2341,8 @@ class Mapper(object): state, state.dict, rightcol, passive=attributes.PASSIVE_NO_INITIALIZE) - if rightval is attributes.PASSIVE_NO_RESULT or rightval is None: + if rightval is attributes.PASSIVE_NO_RESULT or \ + rightval is None: raise ColumnsNotAvailable() binary.right = sql.bindparam(None, rightval, type_=binary.right.type) @@ -1774,11 +2354,13 @@ class Mapper(object): for mapper in reversed(list(self.iterate_to_root())): if mapper.local_table in tables: start = True + elif not isinstance(mapper.local_table, expression.TableClause): + return None if start and not mapper.single: allconds.append(visitors.cloned_traverse( mapper.inherit_condition, {}, - {'binary':visit_binary} + {'binary': visit_binary} ) ) except ColumnsNotAvailable: @@ -1826,7 +2408,7 @@ class Mapper(object): queue = deque(prop.cascade_iterator(type_, parent_state, parent_dict, visited_states, halt_on)) if queue: - visitables.append((queue,mpp, None, None)) + visitables.append((queue, mpp, None, None)) elif item_type is mpp: instance, instance_mapper, corresponding_state, \ corresponding_dict = iterator.popleft() @@ -1843,44 +2425,50 @@ class Mapper(object): @_memoized_configured_property def _sorted_tables(self): table_to_mapper = {} + for mapper in self.base_mapper.self_and_descendants: for t in mapper.tables: - table_to_mapper[t] = mapper + table_to_mapper.setdefault(t, mapper) + + extra_dependencies = [] + for table, mapper in table_to_mapper.items(): + super_ = mapper.inherits + if super_: + extra_dependencies.extend([ + (super_table, table) + for super_table in super_.tables + ]) + + def skip(fk): + # attempt to skip dependencies that are not + # significant to the inheritance chain + # for two tables that are related by inheritance. + # while that dependency may be important, it's techinically + # not what we mean to sort on here. + parent = table_to_mapper.get(fk.parent.table) + dep = table_to_mapper.get(fk.column.table) + if parent is not None and \ + dep is not None and \ + dep is not parent and \ + dep.inherit_condition is not None: + cols = set(sql_util._find_columns(dep.inherit_condition)) + if parent.inherit_condition is not None: + cols = cols.union(sql_util._find_columns( + parent.inherit_condition)) + return fk.parent not in cols and fk.column not in cols + else: + return fk.parent not in cols + return False + + sorted_ = sql_util.sort_tables(table_to_mapper, + skip_fn=skip, + extra_dependencies=extra_dependencies) - sorted_ = sqlutil.sort_tables(table_to_mapper.iterkeys()) ret = util.OrderedDict() for t in sorted_: ret[t] = table_to_mapper[t] return ret - def _per_mapper_flush_actions(self, uow): - saves = unitofwork.SaveUpdateAll(uow, self.base_mapper) - deletes = unitofwork.DeleteAll(uow, self.base_mapper) - uow.dependencies.add((saves, deletes)) - - for dep in self._dependency_processors: - dep.per_property_preprocessors(uow) - - for prop in self._props.values(): - prop.per_property_preprocessors(uow) - - def _per_state_flush_actions(self, uow, states, isdelete): - - base_mapper = self.base_mapper - save_all = unitofwork.SaveUpdateAll(uow, base_mapper) - delete_all = unitofwork.DeleteAll(uow, base_mapper) - for state in states: - # keep saves before deletes - - # this ensures 'row switch' operations work - if isdelete: - action = unitofwork.DeleteState(uow, state, base_mapper) - uow.dependencies.add((save_all, action)) - else: - action = unitofwork.SaveUpdateState(uow, state, base_mapper) - uow.dependencies.add((action, delete_all)) - - yield action - def _memo(self, key, callable_): if key in self._memoized_values: return self._memoized_values[key] @@ -1900,325 +2488,14 @@ class Mapper(object): for m in self.iterate_to_root(): if m._inherits_equated_pairs and \ cols.intersection( - [l for l, r in m._inherits_equated_pairs]): + util.reduce(set.union, + [l.proxy_set for l, r in m._inherits_equated_pairs]) + ): result[table].append((m, m._inherits_equated_pairs)) return result - def _instance_processor(self, context, path, reduced_path, adapter, - polymorphic_from=None, - only_load_props=None, refresh_state=None, - polymorphic_discriminator=None): - - """Produce a mapper level row processor callable - which processes rows into mapped instances.""" - - # note that this method, most of which exists in a closure - # called _instance(), resists being broken out, as - # attempts to do so tend to add significant function - # call overhead. _instance() is the most - # performance-critical section in the whole ORM. - - pk_cols = self.primary_key - - if polymorphic_from or refresh_state: - polymorphic_on = None - else: - if polymorphic_discriminator is not None: - polymorphic_on = polymorphic_discriminator - else: - polymorphic_on = self.polymorphic_on - polymorphic_instances = util.PopulateDict( - self._configure_subclass_mapper( - context, path, reduced_path, adapter) - ) - - version_id_col = self.version_id_col - - if adapter: - pk_cols = [adapter.columns[c] for c in pk_cols] - if polymorphic_on is not None: - polymorphic_on = adapter.columns[polymorphic_on] - if version_id_col is not None: - version_id_col = adapter.columns[version_id_col] - - identity_class = self._identity_class - - new_populators = [] - existing_populators = [] - eager_populators = [] - load_path = context.query._current_path + path - - def populate_state(state, dict_, row, isnew, only_load_props): - if isnew: - if context.propagate_options: - state.load_options = context.propagate_options - if state.load_options: - state.load_path = load_path - - if not new_populators: - self._populators(context, path, reduced_path, row, adapter, - new_populators, - existing_populators, - eager_populators - ) - - if isnew: - populators = new_populators - else: - populators = existing_populators - - if only_load_props is None: - for key, populator in populators: - populator(state, dict_, row) - elif only_load_props: - for key, populator in populators: - if key in only_load_props: - populator(state, dict_, row) - - session_identity_map = context.session.identity_map - - listeners = self.dispatch - - translate_row = listeners.translate_row or None - create_instance = listeners.create_instance or None - populate_instance = listeners.populate_instance or None - append_result = listeners.append_result or None - populate_existing = context.populate_existing or self.always_refresh - invoke_all_eagers = context.invoke_all_eagers - - if self.allow_partial_pks: - is_not_primary_key = _none_set.issuperset - else: - is_not_primary_key = _none_set.issubset - - def _instance(row, result): - if not new_populators and invoke_all_eagers: - self._populators(context, path, reduced_path, row, adapter, - new_populators, - existing_populators, - eager_populators - ) - - if translate_row: - for fn in translate_row: - ret = fn(self, context, row) - if ret is not EXT_CONTINUE: - row = ret - break - - if polymorphic_on is not None: - discriminator = row[polymorphic_on] - if discriminator is not None: - _instance = polymorphic_instances[discriminator] - if _instance: - return _instance(row, result) - - # determine identity key - if refresh_state: - identitykey = refresh_state.key - if identitykey is None: - # super-rare condition; a refresh is being called - # on a non-instance-key instance; this is meant to only - # occur within a flush() - identitykey = self._identity_key_from_state(refresh_state) - else: - identitykey = ( - identity_class, - tuple([row[column] for column in pk_cols]) - ) - - instance = session_identity_map.get(identitykey) - if instance is not None: - state = attributes.instance_state(instance) - dict_ = attributes.instance_dict(instance) - - isnew = state.runid != context.runid - currentload = not isnew - loaded_instance = False - - if not currentload and \ - version_id_col is not None and \ - context.version_check and \ - self._get_state_attr_by_column( - state, - dict_, - self.version_id_col) != \ - row[version_id_col]: - - raise orm_exc.StaleDataError( - "Instance '%s' has version id '%s' which " - "does not match database-loaded version id '%s'." - % (state_str(state), - self._get_state_attr_by_column( - state, dict_, - self.version_id_col), - row[version_id_col])) - elif refresh_state: - # out of band refresh_state detected (i.e. its not in the - # session.identity_map) honor it anyway. this can happen - # if a _get() occurs within save_obj(), such as - # when eager_defaults is True. - state = refresh_state - instance = state.obj() - dict_ = attributes.instance_dict(instance) - isnew = state.runid != context.runid - currentload = True - loaded_instance = False - else: - # check for non-NULL values in the primary key columns, - # else no entity is returned for the row - if is_not_primary_key(identitykey[1]): - return None - - isnew = True - currentload = True - loaded_instance = True - - if create_instance: - for fn in create_instance: - instance = fn(self, context, - row, self.class_) - if instance is not EXT_CONTINUE: - manager = attributes.manager_of_class( - instance.__class__) - # TODO: if manager is None, raise a friendly error - # about returning instances of unmapped types - manager.setup_instance(instance) - break - else: - instance = self.class_manager.new_instance() - else: - instance = self.class_manager.new_instance() - - dict_ = attributes.instance_dict(instance) - state = attributes.instance_state(instance) - state.key = identitykey - - # attach instance to session. - state.session_id = context.session.hash_key - session_identity_map.add(state) - - if currentload or populate_existing: - # state is being fully loaded, so populate. - # add to the "context.progress" collection. - if isnew: - state.runid = context.runid - context.progress[state] = dict_ - - if populate_instance: - for fn in populate_instance: - ret = fn(self, context, row, state, - only_load_props=only_load_props, - instancekey=identitykey, isnew=isnew) - if ret is not EXT_CONTINUE: - break - else: - populate_state(state, dict_, row, isnew, only_load_props) - else: - populate_state(state, dict_, row, isnew, only_load_props) - - if loaded_instance: - state.manager.dispatch.load(state, context) - elif isnew: - state.manager.dispatch.refresh(state, context, only_load_props) - - elif state in context.partials or state.unloaded or eager_populators: - # state is having a partial set of its attributes - # refreshed. Populate those attributes, - # and add to the "context.partials" collection. - if state in context.partials: - isnew = False - (d_, attrs) = context.partials[state] - else: - isnew = True - attrs = state.unloaded - context.partials[state] = (dict_, attrs) - - if populate_instance: - for fn in populate_instance: - ret = fn(self, context, row, state, - only_load_props=attrs, - instancekey=identitykey, isnew=isnew) - if ret is not EXT_CONTINUE: - break - else: - populate_state(state, dict_, row, isnew, attrs) - else: - populate_state(state, dict_, row, isnew, attrs) - - for key, pop in eager_populators: - if key not in state.unloaded: - pop(state, dict_, row) - - if isnew: - state.manager.dispatch.refresh(state, context, attrs) - - - if result is not None: - if append_result: - for fn in append_result: - if fn(self, context, row, state, - result, instancekey=identitykey, - isnew=isnew) is not EXT_CONTINUE: - break - else: - result.append(instance) - else: - result.append(instance) - - return instance - return _instance - - def _populators(self, context, path, reduced_path, row, adapter, - new_populators, existing_populators, eager_populators): - """Produce a collection of attribute level row processor callables.""" - - delayed_populators = [] - pops = (new_populators, existing_populators, delayed_populators, eager_populators) - for prop in self._props.itervalues(): - for i, pop in enumerate(prop.create_row_processor( - context, path, - reduced_path, - self, row, adapter)): - if pop is not None: - pops[i].append((prop.key, pop)) - - if delayed_populators: - new_populators.extend(delayed_populators) - - def _configure_subclass_mapper(self, context, path, reduced_path, adapter): - """Produce a mapper level row processor callable factory for mappers - inheriting this one.""" - - def configure_subclass_mapper(discriminator): - try: - mapper = self.polymorphic_map[discriminator] - except KeyError: - raise AssertionError( - "No such polymorphic_identity %r is defined" % - discriminator) - if mapper is self: - return None - - # replace the tip of the path info with the subclass mapper - # being used. that way accurate "load_path" info is available - # for options invoked during deferred loads. - # we lose AliasedClass path elements this way, but currently, - # those are not needed at this stage. - - # this asserts to true - #assert mapper.isa(_class_to_mapper(path[-1])) - - return mapper._instance_processor(context, path[0:-1] + (mapper,), - reduced_path[0:-1] + (mapper.base_mapper,), - adapter, - polymorphic_from=self) - return configure_subclass_mapper - -log.class_logger(Mapper) - def configure_mappers(): """Initialize the inter-mapper relationships of all mappers that have been constructed thus far. @@ -2228,12 +2505,11 @@ def configure_mappers(): """ - global _new_mappers - if not _new_mappers: + if not Mapper._new_mappers: return _call_configured = None - _COMPILE_MUTEX.acquire() + _CONFIGURE_MUTEX.acquire() try: global _already_compiling if _already_compiling: @@ -2242,7 +2518,7 @@ def configure_mappers(): try: # double-check inside mutex - if not _new_mappers: + if not Mapper._new_mappers: return # initialize properties on all mappers @@ -2262,7 +2538,8 @@ def configure_mappers(): try: mapper._post_configure_properties() mapper._expire_memoizations() - mapper.dispatch.mapper_configured(mapper, mapper.class_) + mapper.dispatch.mapper_configured( + mapper, mapper.class_) _call_configured = mapper except: exc = sys.exc_info()[1] @@ -2270,14 +2547,15 @@ def configure_mappers(): mapper._configure_failed = exc raise - _new_mappers = False + Mapper._new_mappers = False finally: _already_compiling = False finally: - _COMPILE_MUTEX.release() + _CONFIGURE_MUTEX.release() if _call_configured is not None: _call_configured.dispatch.after_configured() + def reconstructor(fn): """Decorate a method as the 'reconstructor' hook. @@ -2297,16 +2575,18 @@ def reconstructor(fn): fn.__sa_reconstructor__ = True return fn + def validates(*names, **kw): """Decorate a method as a 'validator' for one or more named properties. Designates a method as a validator, a method which receives the name of the attribute as well as a value to be assigned, or in the - case of a collection, the value to be added to the collection. The function - can then raise validation exceptions to halt the process from continuing - (where Python's built-in ``ValueError`` and ``AssertionError`` exceptions are - reasonable choices), or can modify or replace the value before proceeding. - The function should otherwise return the given value. + case of a collection, the value to be added to the collection. + The function can then raise validation exceptions to halt the + process from continuing (where Python's built-in ``ValueError`` + and ``AssertionError`` exceptions are reasonable choices), or can + modify or replace the value before proceeding. The function should + otherwise return the given value. Note that a validator for a collection **cannot** issue a load of that collection within the validation routine - this usage raises @@ -2319,20 +2599,38 @@ def validates(*names, **kw): argument "is_remove" which will be a boolean. .. versionadded:: 0.7.7 + :param include_backrefs: defaults to ``True``; if ``False``, the + validation function will not emit if the originator is an attribute + event related via a backref. This can be used for bi-directional + :func:`.validates` usage where only one validator should emit per + attribute operation. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :ref:`simple_validators` - usage examples for :func:`.validates` """ include_removes = kw.pop('include_removes', False) + include_backrefs = kw.pop('include_backrefs', True) + def wrap(fn): fn.__sa_validators__ = names - fn.__sa_include_removes__ = include_removes + fn.__sa_validation_opts__ = { + "include_removes": include_removes, + "include_backrefs": include_backrefs + } return fn return wrap + def _event_on_load(state, ctx): instrumenting_mapper = state.manager.info[_INSTRUMENTOR] if instrumenting_mapper._reconstructor: instrumenting_mapper._reconstructor(state.obj()) + def _event_on_first_init(manager, cls): """Initial mapper compilation trigger. @@ -2343,9 +2641,10 @@ def _event_on_first_init(manager, cls): instrumenting_mapper = manager.info.get(_INSTRUMENTOR) if instrumenting_mapper: - if _new_mappers: + if Mapper._new_mappers: configure_mappers() + def _event_on_init(state, args, kwargs): """Run init_instance hooks. @@ -2357,11 +2656,12 @@ def _event_on_init(state, args, kwargs): instrumenting_mapper = state.manager.info.get(_INSTRUMENTOR) if instrumenting_mapper: - if _new_mappers: + if Mapper._new_mappers: configure_mappers() if instrumenting_mapper._set_polymorphic_identity: instrumenting_mapper._set_polymorphic_identity(state) + def _event_on_resurrect(state): # re-populate the primary key elements # of the dict based on the mapping. @@ -2372,7 +2672,7 @@ def _event_on_resurrect(state): state, state.dict, col, val) -class _ColumnMapping(util.py25_dict): +class _ColumnMapping(dict): """Error reporting helper for mapper._columntoproperty.""" def __init__(self, mapper): diff --git a/libs/sqlalchemy/orm/path_registry.py b/libs/sqlalchemy/orm/path_registry.py new file mode 100644 index 00000000..3397626b --- /dev/null +++ b/libs/sqlalchemy/orm/path_registry.py @@ -0,0 +1,261 @@ +# orm/path_registry.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +"""Path tracking utilities, representing mapper graph traversals. + +""" + +from .. import inspection +from .. import util +from .. import exc +from itertools import chain +from .base import class_mapper + +def _unreduce_path(path): + return PathRegistry.deserialize(path) + + +_WILDCARD_TOKEN = "*" +_DEFAULT_TOKEN = "_sa_default" + +class PathRegistry(object): + """Represent query load paths and registry functions. + + Basically represents structures like: + + (, "orders", , "items", ) + + These structures are generated by things like + query options (joinedload(), subqueryload(), etc.) and are + used to compose keys stored in the query._attributes dictionary + for various options. + + They are then re-composed at query compile/result row time as + the query is formed and as rows are fetched, where they again + serve to compose keys to look up options in the context.attributes + dictionary, which is copied from query._attributes. + + The path structure has a limited amount of caching, where each + "root" ultimately pulls from a fixed registry associated with + the first mapper, that also contains elements for each of its + property keys. However paths longer than two elements, which + are the exception rather than the rule, are generated on an + as-needed basis. + + """ + + def __eq__(self, other): + return other is not None and \ + self.path == other.path + + def set(self, attributes, key, value): + attributes[(key, self.path)] = value + + def setdefault(self, attributes, key, value): + attributes.setdefault((key, self.path), value) + + def get(self, attributes, key, value=None): + key = (key, self.path) + if key in attributes: + return attributes[key] + else: + return value + + def __len__(self): + return len(self.path) + + @property + def length(self): + return len(self.path) + + def pairs(self): + path = self.path + for i in range(0, len(path), 2): + yield path[i], path[i + 1] + + def contains_mapper(self, mapper): + for path_mapper in [ + self.path[i] for i in range(0, len(self.path), 2) + ]: + if path_mapper.is_mapper and \ + path_mapper.isa(mapper): + return True + else: + return False + + def contains(self, attributes, key): + return (key, self.path) in attributes + + def __reduce__(self): + return _unreduce_path, (self.serialize(), ) + + def serialize(self): + path = self.path + return list(zip( + [m.class_ for m in [path[i] for i in range(0, len(path), 2)]], + [path[i].key for i in range(1, len(path), 2)] + [None] + )) + + @classmethod + def deserialize(cls, path): + if path is None: + return None + + p = tuple(chain(*[(class_mapper(mcls), + class_mapper(mcls).attrs[key] + if key is not None else None) + for mcls, key in path])) + if p and p[-1] is None: + p = p[0:-1] + return cls.coerce(p) + + @classmethod + def per_mapper(cls, mapper): + return EntityRegistry( + cls.root, mapper + ) + + @classmethod + def coerce(cls, raw): + return util.reduce(lambda prev, next: prev[next], raw, cls.root) + + def token(self, token): + if token.endswith(':' + _WILDCARD_TOKEN): + return TokenRegistry(self, token) + elif token.endswith(":" + _DEFAULT_TOKEN): + return TokenRegistry(self.root, token) + else: + raise exc.ArgumentError("invalid token: %s" % token) + + def __add__(self, other): + return util.reduce( + lambda prev, next: prev[next], + other.path, self) + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self.path, ) + + +class RootRegistry(PathRegistry): + """Root registry, defers to mappers so that + paths are maintained per-root-mapper. + + """ + path = () + has_entity = False + def __getitem__(self, entity): + return entity._path_registry + +PathRegistry.root = RootRegistry() + +class TokenRegistry(PathRegistry): + def __init__(self, parent, token): + self.token = token + self.parent = parent + self.path = parent.path + (token,) + + has_entity = False + + def __getitem__(self, entity): + raise NotImplementedError() + +class PropRegistry(PathRegistry): + def __init__(self, parent, prop): + # restate this path in terms of the + # given MapperProperty's parent. + insp = inspection.inspect(parent[-1]) + if not insp.is_aliased_class or insp._use_mapper_path: + parent = parent.parent[prop.parent] + elif insp.is_aliased_class and insp.with_polymorphic_mappers: + if prop.parent is not insp.mapper and \ + prop.parent in insp.with_polymorphic_mappers: + subclass_entity = parent[-1]._entity_for_mapper(prop.parent) + parent = parent.parent[subclass_entity] + + self.prop = prop + self.parent = parent + self.path = parent.path + (prop,) + + @util.memoized_property + def has_entity(self): + return hasattr(self.prop, "mapper") + + @util.memoized_property + def entity(self): + return self.prop.mapper + + @util.memoized_property + def _wildcard_path_loader_key(self): + """Given a path (mapper A, prop X), replace the prop with the wildcard, + e.g. (mapper A, 'relationship:.*') or (mapper A, 'column:.*'), then + return within the ("loader", path) structure. + + """ + return ("loader", + self.parent.token( + "%s:%s" % (self.prop.strategy_wildcard_key, _WILDCARD_TOKEN) + ).path + ) + + @util.memoized_property + def _default_path_loader_key(self): + return ("loader", + self.parent.token( + "%s:%s" % (self.prop.strategy_wildcard_key, _DEFAULT_TOKEN) + ).path + ) + + @util.memoized_property + def _loader_key(self): + return ("loader", self.path) + + @property + def mapper(self): + return self.entity + + @property + def entity_path(self): + return self[self.entity] + + def __getitem__(self, entity): + if isinstance(entity, (int, slice)): + return self.path[entity] + else: + return EntityRegistry( + self, entity + ) + +class EntityRegistry(PathRegistry, dict): + is_aliased_class = False + has_entity = True + + def __init__(self, parent, entity): + self.key = entity + self.parent = parent + self.is_aliased_class = entity.is_aliased_class + self.entity = entity + self.path = parent.path + (entity,) + self.entity_path = self + + @property + def mapper(self): + return inspection.inspect(self.entity).mapper + + def __bool__(self): + return True + __nonzero__ = __bool__ + + def __getitem__(self, entity): + if isinstance(entity, (int, slice)): + return self.path[entity] + else: + return dict.__getitem__(self, entity) + + def __missing__(self, key): + self[key] = item = PropRegistry(self, key) + return item + + + diff --git a/libs/sqlalchemy/orm/persistence.py b/libs/sqlalchemy/orm/persistence.py index 5be57cce..35631988 100644 --- a/libs/sqlalchemy/orm/persistence.py +++ b/libs/sqlalchemy/orm/persistence.py @@ -1,5 +1,5 @@ # orm/persistence.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -15,12 +15,12 @@ in unitofwork.py. import operator from itertools import groupby +from .. import sql, util, exc as sa_exc, schema +from . import attributes, sync, exc as orm_exc, evaluator +from .base import _state_mapper, state_str, _attr_as_key +from ..sql import expression +from . import loading -from sqlalchemy import sql, util, exc as sa_exc -from sqlalchemy.orm import attributes, sync, \ - exc as orm_exc - -from sqlalchemy.orm.util import _state_mapper, state_str def save_obj(base_mapper, states, uowtransaction, single=False): """Issue ``INSERT`` and/or ``UPDATE`` statements for a list @@ -46,7 +46,7 @@ def save_obj(base_mapper, states, uowtransaction, single=False): cached_connections = _cached_connection_dict(base_mapper) - for table, mapper in base_mapper._sorted_tables.iteritems(): + for table, mapper in base_mapper._sorted_tables.items(): insert = _collect_insert_commands(base_mapper, uowtransaction, table, states_to_insert) @@ -61,11 +61,12 @@ def save_obj(base_mapper, states, uowtransaction, single=False): if insert: _emit_insert_statements(base_mapper, uowtransaction, cached_connections, - table, insert) + mapper, table, insert) _finalize_insert_update_commands(base_mapper, uowtransaction, states_to_insert, states_to_update) + def post_update(base_mapper, states, uowtransaction, post_update_cols): """Issue UPDATE statements on behalf of a relationship() which specifies post_update. @@ -77,8 +78,7 @@ def post_update(base_mapper, states, uowtransaction, post_update_cols): base_mapper, states, uowtransaction) - - for table, mapper in base_mapper._sorted_tables.iteritems(): + for table, mapper in base_mapper._sorted_tables.items(): update = _collect_post_update_commands(base_mapper, uowtransaction, table, states_to_update, post_update_cols) @@ -88,6 +88,7 @@ def post_update(base_mapper, states, uowtransaction, post_update_cols): cached_connections, mapper, table, update) + def delete_obj(base_mapper, states, uowtransaction): """Issue ``DELETE`` statements for a list of objects. @@ -105,7 +106,7 @@ def delete_obj(base_mapper, states, uowtransaction): table_to_mapper = base_mapper._sorted_tables - for table in reversed(table_to_mapper.keys()): + for table in reversed(list(table_to_mapper.keys())): delete = _collect_delete_commands(base_mapper, uowtransaction, table, states_to_delete) @@ -118,6 +119,7 @@ def delete_obj(base_mapper, states, uowtransaction): in states_to_delete: mapper.dispatch.after_delete(mapper, connection, state) + def _organize_states_for_save(base_mapper, states, uowtransaction): """Make an initial pass across a set of states for INSERT or UPDATE. @@ -148,12 +150,15 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): else: mapper.dispatch.before_update(mapper, connection, state) + if mapper._validate_polymorphic_identity: + mapper._validate_polymorphic_identity(mapper, state, dict_) + # detect if we have a "pending" instance (i.e. has # no instance_key attached to it), and another instance # with the same identity key already exists as persistent. # convert to an UPDATE if so. if not has_identity and \ - instance_key in uowtransaction.session.identity_map: + instance_key in uowtransaction.session.identity_map: instance = \ uowtransaction.session.identity_map[instance_key] existing = attributes.instance_state(instance) @@ -187,6 +192,7 @@ def _organize_states_for_save(base_mapper, states, uowtransaction): return states_to_insert, states_to_update + def _organize_states_for_post_update(base_mapper, states, uowtransaction): """Make an initial pass across a set of states for UPDATE @@ -200,6 +206,7 @@ def _organize_states_for_post_update(base_mapper, states, return list(_connections_for_states(base_mapper, uowtransaction, states)) + def _organize_states_for_delete(base_mapper, states, uowtransaction): """Make an initial pass across a set of states for DELETE. @@ -220,6 +227,7 @@ def _organize_states_for_delete(base_mapper, states, uowtransaction): bool(state.key), connection)) return states_to_delete + def _collect_insert_commands(base_mapper, uowtransaction, table, states_to_insert): """Identify sets of values to use in INSERT statements for a @@ -238,9 +246,12 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, value_params = {} has_all_pks = True + has_all_defaults = True for col in mapper._cols_by_table[table]: - if col is mapper.version_id_col: - params[col.key] = mapper.version_id_generator(None) + if col is mapper.version_id_col and \ + mapper.version_id_generator is not False: + val = mapper.version_id_generator(None) + params[col.key] = val else: # pull straight from the dict for # pending objects @@ -253,6 +264,9 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, elif col.default is None and \ col.server_default is None: params[col.key] = value + elif col.server_default is not None and \ + mapper.base_mapper.eager_defaults: + has_all_defaults = False elif isinstance(value, sql.ClauseElement): value_params[col] = value @@ -260,9 +274,11 @@ def _collect_insert_commands(base_mapper, uowtransaction, table, params[col.key] = value insert.append((state, state_dict, params, mapper, - connection, value_params, has_all_pks)) + connection, value_params, has_all_pks, + has_all_defaults)) return insert + def _collect_update_commands(base_mapper, uowtransaction, table, states_to_update): """Identify sets of values to use in UPDATE statements for a @@ -306,19 +322,20 @@ def _collect_update_commands(base_mapper, uowtransaction, params[col.key] = history.added[0] hasdata = True else: - params[col.key] = mapper.version_id_generator( - params[col._label]) + if mapper.version_id_generator is not False: + val = mapper.version_id_generator(params[col._label]) + params[col.key] = val - # HACK: check for history, in case the - # history is only - # in a different table than the one - # where the version_id_col is. - for prop in mapper._columntoproperty.itervalues(): - history = attributes.get_state_history( - state, prop.key, - attributes.PASSIVE_NO_INITIALIZE) - if history.added: - hasdata = True + # HACK: check for history, in case the + # history is only + # in a different table than the one + # where the version_id_col is. + for prop in mapper._columntoproperty.values(): + history = attributes.get_state_history( + state, prop.key, + attributes.PASSIVE_NO_INITIALIZE) + if history.added: + hasdata = True else: prop = mapper._columntoproperty[col] history = attributes.get_state_history( @@ -370,7 +387,7 @@ def _collect_update_commands(base_mapper, uowtransaction, params[col._label] = value if hasdata: if hasnull: - raise sa_exc.FlushError( + raise orm_exc.FlushError( "Can't update table " "using NULL for primary " "key value") @@ -400,6 +417,7 @@ def _collect_post_update_commands(base_mapper, uowtransaction, table, mapper._get_state_attr_by_column( state, state_dict, col) + elif col in post_update_cols: prop = mapper._columntoproperty[col] history = attributes.get_state_history( @@ -414,6 +432,7 @@ def _collect_post_update_commands(base_mapper, uowtransaction, table, connection)) return update + def _collect_delete_commands(base_mapper, uowtransaction, table, states_to_delete): """Identify values to use in DELETE statements for a list of @@ -434,7 +453,7 @@ def _collect_delete_commands(base_mapper, uowtransaction, table, mapper._get_state_attr_by_column( state, state_dict, col) if value is None: - raise sa_exc.FlushError( + raise orm_exc.FlushError( "Can't delete from table " "using NULL for primary " "key value") @@ -468,7 +487,13 @@ def _emit_update_statements(base_mapper, uowtransaction, sql.bindparam(mapper.version_id_col._label, type_=mapper.version_id_col.type)) - return table.update(clause) + stmt = table.update(clause) + if mapper.base_mapper.eager_defaults: + stmt = stmt.return_defaults() + elif mapper.version_id_col is not None: + stmt = stmt.return_defaults(mapper.version_id_col) + + return stmt statement = base_mapper._memo(('update', table), update_stmt) @@ -490,8 +515,7 @@ def _emit_update_statements(base_mapper, uowtransaction, table, state, state_dict, - c.context.prefetch_cols, - c.context.postfetch_cols, + c, c.context.compiled_parameters[0], value_params) rows += c.rowcount @@ -509,45 +533,57 @@ def _emit_update_statements(base_mapper, uowtransaction, c.dialect.dialect_description, stacklevel=12) + def _emit_insert_statements(base_mapper, uowtransaction, - cached_connections, table, insert): + cached_connections, mapper, table, insert): """Emit INSERT statements corresponding to value lists collected by _collect_insert_commands().""" statement = base_mapper._memo(('insert', table), table.insert) - for (connection, pkeys, hasvalue, has_all_pks), \ + for (connection, pkeys, hasvalue, has_all_pks, has_all_defaults), \ records in groupby(insert, lambda rec: (rec[4], - rec[2].keys(), + list(rec[2].keys()), bool(rec[5]), - rec[6]) + rec[6], rec[7]) ): - if has_all_pks and not hasvalue: + if \ + ( + has_all_defaults + or not base_mapper.eager_defaults + or not connection.dialect.implicit_returning + ) and has_all_pks and not hasvalue: + records = list(records) multiparams = [rec[2] for rec in records] + c = cached_connections[connection].\ execute(statement, multiparams) - for (state, state_dict, params, mapper, - conn, value_params, has_all_pks), \ + for (state, state_dict, params, mapper_rec, + conn, value_params, has_all_pks, has_all_defaults), \ last_inserted_params in \ zip(records, c.context.compiled_parameters): _postfetch( - mapper, + mapper_rec, uowtransaction, table, state, state_dict, - c.context.prefetch_cols, - c.context.postfetch_cols, + c, last_inserted_params, value_params) else: - for state, state_dict, params, mapper, \ + if not has_all_defaults and base_mapper.eager_defaults: + statement = statement.return_defaults() + elif mapper.version_id_col is not None: + statement = statement.return_defaults(mapper.version_id_col) + + for state, state_dict, params, mapper_rec, \ connection, value_params, \ - has_all_pks in records: + has_all_pks, has_all_defaults in records: if value_params: result = connection.execute( @@ -563,28 +599,26 @@ def _emit_insert_statements(base_mapper, uowtransaction, # set primary key attributes for pk, col in zip(primary_key, mapper._pks_by_table[table]): - prop = mapper._columntoproperty[col] + prop = mapper_rec._columntoproperty[col] if state_dict.get(prop.key) is None: # TODO: would rather say: #state_dict[prop.key] = pk - mapper._set_state_attr_by_column( + mapper_rec._set_state_attr_by_column( state, state_dict, col, pk) _postfetch( - mapper, + mapper_rec, uowtransaction, table, state, state_dict, - result.context.prefetch_cols, - result.context.postfetch_cols, + result, result.context.compiled_parameters[0], value_params) - def _emit_post_update_statements(base_mapper, uowtransaction, cached_connections, mapper, table, update): """Emit UPDATE statements corresponding to value lists collected @@ -606,7 +640,7 @@ def _emit_post_update_statements(base_mapper, uowtransaction, # also group them into common (connection, cols) sets # to support executemany(). for key, grouper in groupby( - update, lambda rec: (rec[4], rec[2].keys()) + update, lambda rec: (rec[4], list(rec[2].keys())) ): connection = key[0] multiparams = [params for state, state_dict, @@ -640,7 +674,7 @@ def _emit_delete_statements(base_mapper, uowtransaction, cached_connections, return table.delete(clause) - for connection, del_objects in delete.iteritems(): + for connection, del_objects in delete.items(): statement = base_mapper._memo(('delete', table), delete_stmt) connection = cached_connections[connection] @@ -687,15 +721,27 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, if p.expire_on_flush or p.key not in state.dict] ) if readonly: - state.expire_attributes(state.dict, readonly) + state._expire_attributes(state.dict, readonly) - # if eager_defaults option is enabled, - # refresh whatever has been expired. - if base_mapper.eager_defaults and state.unloaded: + # if eager_defaults option is enabled, load + # all expired cols. Else if we have a version_id_col, make sure + # it isn't expired. + toload_now = [] + + if base_mapper.eager_defaults: + toload_now.extend(state._unloaded_non_object) + elif mapper.version_id_col is not None and \ + mapper.version_id_generator is False: + prop = mapper._columntoproperty[mapper.version_id_col] + if prop.key in state.unloaded: + toload_now.extend([prop.key]) + + if toload_now: state.key = base_mapper._identity_key_from_state(state) - uowtransaction.session.query(base_mapper)._load_on_ident( + loading.load_on_ident( + uowtransaction.session.query(base_mapper), state.key, refresh_state=state, - only_load_props=state.unloaded) + only_load_props=toload_now) # call after_XXX extensions if not has_identity: @@ -703,22 +749,34 @@ def _finalize_insert_update_commands(base_mapper, uowtransaction, else: mapper.dispatch.after_update(mapper, connection, state) + def _postfetch(mapper, uowtransaction, table, - state, dict_, prefetch_cols, postfetch_cols, - params, value_params): + state, dict_, result, params, value_params): """Expire attributes in need of newly persisted database state, after an INSERT or UPDATE statement has proceeded for that state.""" + prefetch_cols = result.context.prefetch_cols + postfetch_cols = result.context.postfetch_cols + returning_cols = result.context.returning_cols + if mapper.version_id_col is not None: prefetch_cols = list(prefetch_cols) + [mapper.version_id_col] + if returning_cols: + row = result.context.returned_defaults + if row is not None: + for col in returning_cols: + if col.primary_key: + continue + mapper._set_state_attr_by_column(state, dict_, col, row[col]) + for c in prefetch_cols: if c.key in params and c in mapper._columntoproperty: mapper._set_state_attr_by_column(state, dict_, c, params[c.key]) if postfetch_cols: - state.expire_attributes(state.dict, + state._expire_attributes(state.dict, [mapper._columntoproperty[c].key for c in postfetch_cols if c in mapper._columntoproperty] @@ -733,6 +791,7 @@ def _postfetch(mapper, uowtransaction, table, uowtransaction, mapper.passive_updates) + def _connections_for_states(base_mapper, uowtransaction, states): """Return an iterator of (state, state.dict, mapper, connection). @@ -762,18 +821,265 @@ def _connections_for_states(base_mapper, uowtransaction, states): yield state, state.dict, mapper, connection + def _cached_connection_dict(base_mapper): # dictionary of connection->connection_with_cache_options. return util.PopulateDict( - lambda conn:conn.execution_options( + lambda conn: conn.execution_options( compiled_cache=base_mapper._compiled_cache )) + def _sort_states(states): pending = set(states) persistent = set(s for s in pending if s.key is not None) pending.difference_update(persistent) return sorted(pending, key=operator.attrgetter("insert_order")) + \ - sorted(persistent, key=lambda q:q.key[1]) + sorted(persistent, key=lambda q: q.key[1]) +class BulkUD(object): + """Handle bulk update and deletes via a :class:`.Query`.""" + + def __init__(self, query): + self.query = query.enable_eagerloads(False) + + @property + def session(self): + return self.query.session + + @classmethod + def _factory(cls, lookup, synchronize_session, *arg): + try: + klass = lookup[synchronize_session] + except KeyError: + raise sa_exc.ArgumentError( + "Valid strategies for session synchronization " + "are %s" % (", ".join(sorted(repr(x) + for x in lookup)))) + else: + return klass(*arg) + + def exec_(self): + self._do_pre() + self._do_pre_synchronize() + self._do_exec() + self._do_post_synchronize() + self._do_post() + + def _do_pre(self): + query = self.query + self.context = context = query._compile_context() + if len(context.statement.froms) != 1 or \ + not isinstance(context.statement.froms[0], schema.Table): + + self.primary_table = query._only_entity_zero( + "This operation requires only one Table or " + "entity be specified as the target." + ).mapper.local_table + else: + self.primary_table = context.statement.froms[0] + + session = query.session + + if query._autoflush: + session._autoflush() + + def _do_pre_synchronize(self): + pass + + def _do_post_synchronize(self): + pass + + +class BulkEvaluate(BulkUD): + """BulkUD which does the 'evaluate' method of session state resolution.""" + + def _additional_evaluators(self, evaluator_compiler): + pass + + def _do_pre_synchronize(self): + query = self.query + try: + evaluator_compiler = evaluator.EvaluatorCompiler() + if query.whereclause is not None: + eval_condition = evaluator_compiler.process( + query.whereclause) + else: + def eval_condition(obj): + return True + + self._additional_evaluators(evaluator_compiler) + + except evaluator.UnevaluatableError: + raise sa_exc.InvalidRequestError( + "Could not evaluate current criteria in Python. " + "Specify 'fetch' or False for the " + "synchronize_session parameter.") + target_cls = query._mapper_zero().class_ + + #TODO: detect when the where clause is a trivial primary key match + self.matched_objects = [ + obj for (cls, pk), obj in + query.session.identity_map.items() + if issubclass(cls, target_cls) and + eval_condition(obj)] + + +class BulkFetch(BulkUD): + """BulkUD which does the 'fetch' method of session state resolution.""" + + def _do_pre_synchronize(self): + query = self.query + session = query.session + select_stmt = self.context.statement.with_only_columns( + self.primary_table.primary_key) + self.matched_rows = session.execute( + select_stmt, + params=query._params).fetchall() + + +class BulkUpdate(BulkUD): + """BulkUD which handles UPDATEs.""" + + def __init__(self, query, values): + super(BulkUpdate, self).__init__(query) + self.query._no_select_modifiers("update") + self.values = values + + @classmethod + def factory(cls, query, synchronize_session, values): + return BulkUD._factory({ + "evaluate": BulkUpdateEvaluate, + "fetch": BulkUpdateFetch, + False: BulkUpdate + }, synchronize_session, query, values) + + def _do_exec(self): + update_stmt = sql.update(self.primary_table, + self.context.whereclause, self.values) + + self.result = self.query.session.execute( + update_stmt, params=self.query._params) + self.rowcount = self.result.rowcount + + def _do_post(self): + session = self.query.session + session.dispatch.after_bulk_update(self) + + +class BulkDelete(BulkUD): + """BulkUD which handles DELETEs.""" + + def __init__(self, query): + super(BulkDelete, self).__init__(query) + self.query._no_select_modifiers("delete") + + @classmethod + def factory(cls, query, synchronize_session): + return BulkUD._factory({ + "evaluate": BulkDeleteEvaluate, + "fetch": BulkDeleteFetch, + False: BulkDelete + }, synchronize_session, query) + + def _do_exec(self): + delete_stmt = sql.delete(self.primary_table, + self.context.whereclause) + + self.result = self.query.session.execute(delete_stmt, + params=self.query._params) + self.rowcount = self.result.rowcount + + def _do_post(self): + session = self.query.session + session.dispatch.after_bulk_delete(self) + + +class BulkUpdateEvaluate(BulkEvaluate, BulkUpdate): + """BulkUD which handles UPDATEs using the "evaluate" + method of session resolution.""" + + def _additional_evaluators(self, evaluator_compiler): + self.value_evaluators = {} + for key, value in self.values.items(): + key = _attr_as_key(key) + self.value_evaluators[key] = evaluator_compiler.process( + expression._literal_as_binds(value)) + + def _do_post_synchronize(self): + session = self.query.session + states = set() + evaluated_keys = list(self.value_evaluators.keys()) + for obj in self.matched_objects: + state, dict_ = attributes.instance_state(obj),\ + attributes.instance_dict(obj) + + # only evaluate unmodified attributes + to_evaluate = state.unmodified.intersection( + evaluated_keys) + for key in to_evaluate: + dict_[key] = self.value_evaluators[key](obj) + + state._commit(dict_, list(to_evaluate)) + + # expire attributes with pending changes + # (there was no autoflush, so they are overwritten) + state._expire_attributes(dict_, + set(evaluated_keys). + difference(to_evaluate)) + states.add(state) + session._register_altered(states) + + +class BulkDeleteEvaluate(BulkEvaluate, BulkDelete): + """BulkUD which handles DELETEs using the "evaluate" + method of session resolution.""" + + def _do_post_synchronize(self): + self.query.session._remove_newly_deleted( + [attributes.instance_state(obj) + for obj in self.matched_objects]) + + +class BulkUpdateFetch(BulkFetch, BulkUpdate): + """BulkUD which handles UPDATEs using the "fetch" + method of session resolution.""" + + def _do_post_synchronize(self): + session = self.query.session + target_mapper = self.query._mapper_zero() + + states = set([ + attributes.instance_state(session.identity_map[identity_key]) + for identity_key in [ + target_mapper.identity_key_from_primary_key( + list(primary_key)) + for primary_key in self.matched_rows + ] + if identity_key in session.identity_map + ]) + attrib = [_attr_as_key(k) for k in self.values] + for state in states: + session._expire_state(state, attrib) + session._register_altered(states) + + +class BulkDeleteFetch(BulkFetch, BulkDelete): + """BulkUD which handles DELETEs using the "fetch" + method of session resolution.""" + + def _do_post_synchronize(self): + session = self.query.session + target_mapper = self.query._mapper_zero() + for primary_key in self.matched_rows: + # TODO: inline this and call remove_newly_deleted + # once + identity_key = target_mapper.identity_key_from_primary_key( + list(primary_key)) + if identity_key in session.identity_map: + session._remove_newly_deleted( + [attributes.instance_state( + session.identity_map[identity_key] + )] + ) diff --git a/libs/sqlalchemy/orm/properties.py b/libs/sqlalchemy/orm/properties.py index 204232cf..a0def7d3 100644 --- a/libs/sqlalchemy/orm/properties.py +++ b/libs/sqlalchemy/orm/properties.py @@ -1,5 +1,5 @@ # orm/properties.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -10,27 +10,20 @@ This is a private module which defines the behavior of invidual ORM- mapped attributes. """ +from __future__ import absolute_import -from sqlalchemy import sql, util, log, exc as sa_exc -from sqlalchemy.sql.util import ClauseAdapter, criterion_as_pairs, \ - join_condition, _shallow_annotate -from sqlalchemy.sql import operators, expression -from sqlalchemy.orm import attributes, dependency, mapper, \ - object_mapper, strategies, configure_mappers -from sqlalchemy.orm.util import CascadeOptions, _class_to_mapper, \ - _orm_annotate, _orm_deannotate +from .. import util, log +from ..sql import expression +from . import attributes +from .util import _orm_full_deannotate -from sqlalchemy.orm.interfaces import MANYTOMANY, MANYTOONE, \ - MapperProperty, ONETOMANY, PropComparator, StrategizedProperty -mapperlib = util.importlater("sqlalchemy.orm", "mapperlib") -NoneType = type(None) +from .interfaces import PropComparator, StrategizedProperty -__all__ = ('ColumnProperty', 'CompositeProperty', 'SynonymProperty', - 'ComparableProperty', 'RelationshipProperty', 'RelationProperty') +__all__ = ['ColumnProperty', 'CompositeProperty', 'SynonymProperty', + 'ComparableProperty', 'RelationshipProperty'] -from descriptor_props import CompositeProperty, SynonymProperty, \ - ComparableProperty,ConcreteInheritedProperty +@log.class_logger class ColumnProperty(StrategizedProperty): """Describes an object attribute that corresponds to a table column. @@ -38,31 +31,85 @@ class ColumnProperty(StrategizedProperty): """ + strategy_wildcard_key = 'column' + def __init__(self, *columns, **kwargs): - """Construct a ColumnProperty. + """Provide a column-level property for use with a Mapper. - Note the public constructor is the :func:`.orm.column_property` function. + Column-based properties can normally be applied to the mapper's + ``properties`` dictionary using the :class:`.Column` element directly. + Use this function when the given column is not directly present within the + mapper's selectable; examples include SQL expressions, functions, and + scalar SELECT queries. - :param \*columns: The list of `columns` describes a single - object property. If there are multiple tables joined - together for the mapper, this list represents the equivalent - column as it appears across each table. + Columns that aren't present in the mapper's selectable won't be persisted + by the mapper and are effectively "read-only" attributes. + + :param \*cols: + list of Column objects to be mapped. + + :param active_history=False: + When ``True``, indicates that the "previous" value for a + scalar attribute should be loaded when replaced, if not + already loaded. Normally, history tracking logic for + simple non-primary-key scalar values only needs to be + aware of the "new" value in order to perform a flush. This + flag is available for applications that make use of + :func:`.attributes.get_history` or :meth:`.Session.is_modified` + which also need to know + the "previous" value of the attribute. + + .. versionadded:: 0.6.6 + + :param comparator_factory: a class which extends + :class:`.ColumnProperty.Comparator` which provides custom SQL clause + generation for comparison operations. :param group: + a group name for this property when marked as deferred. :param deferred: + when True, the column property is "deferred", meaning that + it does not load immediately, and is instead loaded when the + attribute is first accessed on an instance. See also + :func:`~sqlalchemy.orm.deferred`. - :param comparator_factory: + :param doc: + optional string that will be applied as the doc on the + class-bound descriptor. - :param descriptor: + :param expire_on_flush=True: + Disable expiry on flush. A column_property() which refers + to a SQL expression (and not a single table-bound column) + is considered to be a "read only" property; populating it + has no effect on the state of data, and it can only return + database state. For this reason a column_property()'s value + is expired whenever the parent object is involved in a + flush, that is, has any kind of "dirty" state within a flush. + Setting this parameter to ``False`` will have the effect of + leaving any existing value present after the flush proceeds. + Note however that the :class:`.Session` with default expiration + settings still expires + all attributes after a :meth:`.Session.commit` call, however. - :param expire_on_flush: + .. versionadded:: 0.7.3 + + :param info: Optional data dictionary which will be populated into the + :attr:`.MapperProperty.info` attribute of this object. + + .. versionadded:: 0.8 :param extension: + an + :class:`.AttributeExtension` + instance, or list of extensions, which will be prepended + to the list of attribute listeners for the resulting + descriptor placed on the class. + **Deprecated.** Please see :class:`.AttributeEvents`. """ self._orig_columns = [expression._labeled(c) for c in columns] - self.columns = [expression._labeled(_orm_deannotate(c)) + self.columns = [expression._labeled(_orm_full_deannotate(c)) for c in columns] self.group = kwargs.pop('group', None) self.deferred = kwargs.pop('deferred', False) @@ -74,6 +121,9 @@ class ColumnProperty(StrategizedProperty): self.active_history = kwargs.pop('active_history', False) self.expire_on_flush = kwargs.pop('expire_on_flush', True) + if 'info' in kwargs: + self.info = kwargs.pop('info') + if 'doc' in kwargs: self.doc = kwargs.pop('doc') else: @@ -92,12 +142,18 @@ class ColumnProperty(StrategizedProperty): ', '.join(sorted(kwargs.keys())))) util.set_creation_order(self) - if not self.instrument: - self.strategy_class = strategies.UninstrumentedColumnLoader - elif self.deferred: - self.strategy_class = strategies.DeferredColumnLoader - else: - self.strategy_class = strategies.ColumnLoader + + self.strategy_class = self._strategy_lookup( + ("deferred", self.deferred), + ("instrument", self.instrument) + ) + + @property + def expression(self): + """Return the primary column or expression for this ColumnProperty. + + """ + return self.columns[0] def instrument_class(self, mapper): if not self.instrument: @@ -147,17 +203,49 @@ class ColumnProperty(StrategizedProperty): impl = dest_state.get_impl(self.key) impl.set(dest_state, dest_dict, value, None) elif dest_state.has_identity and self.key not in dest_dict: - dest_state.expire_attributes(dest_dict, [self.key]) + dest_state._expire_attributes(dest_dict, [self.key]) class Comparator(PropComparator): + """Produce boolean, comparison, and other operators for + :class:`.ColumnProperty` attributes. + + See the documentation for :class:`.PropComparator` for a brief + overview. + + See also: + + :class:`.PropComparator` + + :class:`.ColumnOperators` + + :ref:`types_operators` + + :attr:`.TypeEngine.comparator_factory` + + """ @util.memoized_instancemethod def __clause_element__(self): if self.adapter: return self.adapter(self.prop.columns[0]) else: return self.prop.columns[0]._annotate({ - "parententity": self.mapper, - "parentmapper":self.mapper}) + "parententity": self._parentmapper, + "parentmapper": self._parentmapper}) + + @util.memoized_property + def info(self): + ce = self.__clause_element__() + try: + return ce.info + except AttributeError: + return self.prop.info + + def __getattr__(self, key): + """proxy attribute access down to the mapped column. + + this allows user-defined comparison methods to be accessed. + """ + return getattr(self.__clause_element__(), key) def operate(self, op, *other, **kwargs): return op(self.__clause_element__(), *other, **kwargs) @@ -166,1400 +254,6 @@ class ColumnProperty(StrategizedProperty): col = self.__clause_element__() return op(col._bind_param(op, other), col, **kwargs) - # TODO: legacy..do we need this ? (0.5) - ColumnComparator = Comparator - def __str__(self): return str(self.parent.class_.__name__) + "." + self.key -log.class_logger(ColumnProperty) - -class RelationshipProperty(StrategizedProperty): - """Describes an object property that holds a single item or list - of items that correspond to a related database table. - - Public constructor is the :func:`.orm.relationship` function. - - Of note here is the :class:`.RelationshipProperty.Comparator` - class, which implements comparison operations for scalar- - and collection-referencing mapped attributes. - - """ - - strategy_wildcard_key = 'relationship:*' - - def __init__(self, argument, - secondary=None, primaryjoin=None, - secondaryjoin=None, - foreign_keys=None, - uselist=None, - order_by=False, - backref=None, - back_populates=None, - post_update=False, - cascade=False, extension=None, - viewonly=False, lazy=True, - collection_class=None, passive_deletes=False, - passive_updates=True, remote_side=None, - enable_typechecks=True, join_depth=None, - comparator_factory=None, - single_parent=False, innerjoin=False, - doc=None, - active_history=False, - cascade_backrefs=True, - load_on_pending=False, - strategy_class=None, _local_remote_pairs=None, - query_class=None): - - self.uselist = uselist - self.argument = argument - self.secondary = secondary - self.primaryjoin = primaryjoin - self.secondaryjoin = secondaryjoin - self.post_update = post_update - self.direction = None - self.viewonly = viewonly - self.lazy = lazy - self.single_parent = single_parent - self._user_defined_foreign_keys = foreign_keys - self.collection_class = collection_class - self.passive_deletes = passive_deletes - self.cascade_backrefs = cascade_backrefs - self.passive_updates = passive_updates - self.remote_side = remote_side - self.enable_typechecks = enable_typechecks - self.query_class = query_class - self.innerjoin = innerjoin - self.doc = doc - self.active_history = active_history - self.join_depth = join_depth - self.local_remote_pairs = _local_remote_pairs - self.extension = extension - self.load_on_pending = load_on_pending - self.comparator_factory = comparator_factory or \ - RelationshipProperty.Comparator - self.comparator = self.comparator_factory(self, None) - util.set_creation_order(self) - - if strategy_class: - self.strategy_class = strategy_class - elif self.lazy== 'dynamic': - from sqlalchemy.orm import dynamic - self.strategy_class = dynamic.DynaLoader - else: - self.strategy_class = strategies.factory(self.lazy) - - self._reverse_property = set() - - if cascade is not False: - self.cascade = CascadeOptions(cascade) - else: - self.cascade = CascadeOptions("save-update, merge") - - if self.passive_deletes == 'all' and \ - ("delete" in self.cascade or - "delete-orphan" in self.cascade): - raise sa_exc.ArgumentError( - "Can't set passive_deletes='all' in conjunction " - "with 'delete' or 'delete-orphan' cascade") - - self.order_by = order_by - - self.back_populates = back_populates - - if self.back_populates: - if backref: - raise sa_exc.ArgumentError( - "backref and back_populates keyword arguments " - "are mutually exclusive") - self.backref = None - else: - self.backref = backref - - - def instrument_class(self, mapper): - attributes.register_descriptor( - mapper.class_, - self.key, - comparator=self.comparator_factory(self, mapper), - parententity=mapper, - doc=self.doc, - ) - - class Comparator(PropComparator): - """Produce comparison operations for :func:`~.orm.relationship`-based - attributes.""" - - def __init__(self, prop, mapper, of_type=None, adapter=None): - """Construction of :class:`.RelationshipProperty.Comparator` - is internal to the ORM's attribute mechanics. - - """ - self.prop = prop - self.mapper = mapper - self.adapter = adapter - if of_type: - self._of_type = _class_to_mapper(of_type) - - def adapted(self, adapter): - """Return a copy of this PropComparator which will use the - given adaption function on the local side of generated - expressions. - - """ - - return self.__class__(self.property, self.mapper, - getattr(self, '_of_type', None), - adapter) - - @property - def parententity(self): - return self.property.parent - - def __clause_element__(self): - elem = self.property.parent._with_polymorphic_selectable - if self.adapter: - return self.adapter(elem) - else: - return elem - - def of_type(self, cls): - """Produce a construct that represents a particular 'subtype' of - attribute for the parent class. - - Currently this is usable in conjunction with :meth:`.Query.join` - and :meth:`.Query.outerjoin`. - - """ - return RelationshipProperty.Comparator( - self.property, - self.mapper, - cls, adapter=self.adapter) - - def in_(self, other): - """Produce an IN clause - this is not implemented - for :func:`~.orm.relationship`-based attributes at this time. - - """ - raise NotImplementedError('in_() not yet supported for ' - 'relationships. For a simple many-to-one, use ' - 'in_() against the set of foreign key values.') - - __hash__ = None - - def __eq__(self, other): - """Implement the ``==`` operator. - - In a many-to-one context, such as:: - - MyClass.some_prop == - - this will typically produce a - clause such as:: - - mytable.related_id == - - Where ```` is the primary key of the given - object. - - The ``==`` operator provides partial functionality for non- - many-to-one comparisons: - - * Comparisons against collections are not supported. - Use :meth:`~.RelationshipProperty.Comparator.contains`. - * Compared to a scalar one-to-many, will produce a - clause that compares the target columns in the parent to - the given target. - * Compared to a scalar many-to-many, an alias - of the association table will be rendered as - well, forming a natural join that is part of the - main body of the query. This will not work for - queries that go beyond simple AND conjunctions of - comparisons, such as those which use OR. Use - explicit joins, outerjoins, or - :meth:`~.RelationshipProperty.Comparator.has` for - more comprehensive non-many-to-one scalar - membership tests. - * Comparisons against ``None`` given in a one-to-many - or many-to-many context produce a NOT EXISTS clause. - - """ - if isinstance(other, (NoneType, expression._Null)): - if self.property.direction in [ONETOMANY, MANYTOMANY]: - return ~self._criterion_exists() - else: - return _orm_annotate(self.property._optimized_compare( - None, adapt_source=self.adapter)) - elif self.property.uselist: - raise sa_exc.InvalidRequestError("Can't compare a colle" - "ction to an object or collection; use " - "contains() to test for membership.") - else: - return _orm_annotate(self.property._optimized_compare(other, - adapt_source=self.adapter)) - - def _criterion_exists(self, criterion=None, **kwargs): - if getattr(self, '_of_type', None): - target_mapper = self._of_type - to_selectable = target_mapper._with_polymorphic_selectable - if self.property._is_self_referential: - to_selectable = to_selectable.alias() - - single_crit = target_mapper._single_table_criterion - if single_crit is not None: - if criterion is not None: - criterion = single_crit & criterion - else: - criterion = single_crit - else: - to_selectable = None - - if self.adapter: - source_selectable = self.__clause_element__() - else: - source_selectable = None - - pj, sj, source, dest, secondary, target_adapter = \ - self.property._create_joins(dest_polymorphic=True, - dest_selectable=to_selectable, - source_selectable=source_selectable) - - for k in kwargs: - crit = getattr(self.property.mapper.class_, k) == kwargs[k] - if criterion is None: - criterion = crit - else: - criterion = criterion & crit - - # annotate the *local* side of the join condition, in the case - # of pj + sj this is the full primaryjoin, in the case of just - # pj its the local side of the primaryjoin. - if sj is not None: - j = _orm_annotate(pj) & sj - else: - j = _orm_annotate(pj, exclude=self.property.remote_side) - - if criterion is not None and target_adapter: - # limit this adapter to annotated only? - criterion = target_adapter.traverse(criterion) - - # only have the "joined left side" of what we - # return be subject to Query adaption. The right - # side of it is used for an exists() subquery and - # should not correlate or otherwise reach out - # to anything in the enclosing query. - if criterion is not None: - criterion = criterion._annotate({'no_replacement_traverse': True}) - - crit = j & criterion - - return sql.exists([1], crit, from_obj=dest).\ - correlate(source._annotate({'_orm_adapt':True})) - - def any(self, criterion=None, **kwargs): - """Produce an expression that tests a collection against - particular criterion, using EXISTS. - - An expression like:: - - session.query(MyClass).filter( - MyClass.somereference.any(SomeRelated.x==2) - ) - - - Will produce a query like:: - - SELECT * FROM my_table WHERE - EXISTS (SELECT 1 FROM related WHERE related.my_id=my_table.id - AND related.x=2) - - Because :meth:`~.RelationshipProperty.Comparator.any` uses - a correlated subquery, its performance is not nearly as - good when compared against large target tables as that of - using a join. - - :meth:`~.RelationshipProperty.Comparator.any` is particularly - useful for testing for empty collections:: - - session.query(MyClass).filter( - ~MyClass.somereference.any() - ) - - will produce:: - - SELECT * FROM my_table WHERE - NOT EXISTS (SELECT 1 FROM related WHERE related.my_id=my_table.id) - - :meth:`~.RelationshipProperty.Comparator.any` is only - valid for collections, i.e. a :func:`.relationship` - that has ``uselist=True``. For scalar references, - use :meth:`~.RelationshipProperty.Comparator.has`. - - """ - if not self.property.uselist: - raise sa_exc.InvalidRequestError( - "'any()' not implemented for scalar " - "attributes. Use has()." - ) - - return self._criterion_exists(criterion, **kwargs) - - def has(self, criterion=None, **kwargs): - """Produce an expression that tests a scalar reference against - particular criterion, using EXISTS. - - An expression like:: - - session.query(MyClass).filter( - MyClass.somereference.has(SomeRelated.x==2) - ) - - - Will produce a query like:: - - SELECT * FROM my_table WHERE - EXISTS (SELECT 1 FROM related WHERE related.id==my_table.related_id - AND related.x=2) - - Because :meth:`~.RelationshipProperty.Comparator.has` uses - a correlated subquery, its performance is not nearly as - good when compared against large target tables as that of - using a join. - - :meth:`~.RelationshipProperty.Comparator.has` is only - valid for scalar references, i.e. a :func:`.relationship` - that has ``uselist=False``. For collection references, - use :meth:`~.RelationshipProperty.Comparator.any`. - - """ - if self.property.uselist: - raise sa_exc.InvalidRequestError( - "'has()' not implemented for collections. " - "Use any().") - return self._criterion_exists(criterion, **kwargs) - - def contains(self, other, **kwargs): - """Return a simple expression that tests a collection for - containment of a particular item. - - :meth:`~.RelationshipProperty.Comparator.contains` is - only valid for a collection, i.e. a - :func:`~.orm.relationship` that implements - one-to-many or many-to-many with ``uselist=True``. - - When used in a simple one-to-many context, an - expression like:: - - MyClass.contains(other) - - Produces a clause like:: - - mytable.id == - - Where ```` is the value of the foreign key - attribute on ``other`` which refers to the primary - key of its parent object. From this it follows that - :meth:`~.RelationshipProperty.Comparator.contains` is - very useful when used with simple one-to-many - operations. - - For many-to-many operations, the behavior of - :meth:`~.RelationshipProperty.Comparator.contains` - has more caveats. The association table will be - rendered in the statement, producing an "implicit" - join, that is, includes multiple tables in the FROM - clause which are equated in the WHERE clause:: - - query(MyClass).filter(MyClass.contains(other)) - - Produces a query like:: - - SELECT * FROM my_table, my_association_table AS - my_association_table_1 WHERE - my_table.id = my_association_table_1.parent_id - AND my_association_table_1.child_id = - - Where ```` would be the primary key of - ``other``. From the above, it is clear that - :meth:`~.RelationshipProperty.Comparator.contains` - will **not** work with many-to-many collections when - used in queries that move beyond simple AND - conjunctions, such as multiple - :meth:`~.RelationshipProperty.Comparator.contains` - expressions joined by OR. In such cases subqueries or - explicit "outer joins" will need to be used instead. - See :meth:`~.RelationshipProperty.Comparator.any` for - a less-performant alternative using EXISTS, or refer - to :meth:`.Query.outerjoin` as well as :ref:`ormtutorial_joins` - for more details on constructing outer joins. - - """ - if not self.property.uselist: - raise sa_exc.InvalidRequestError( - "'contains' not implemented for scalar " - "attributes. Use ==") - clause = self.property._optimized_compare(other, - adapt_source=self.adapter) - - if self.property.secondaryjoin is not None: - clause.negation_clause = \ - self.__negated_contains_or_equals(other) - - return clause - - def __negated_contains_or_equals(self, other): - if self.property.direction == MANYTOONE: - state = attributes.instance_state(other) - - def state_bindparam(x, state, col): - o = state.obj() # strong ref - return sql.bindparam(x, unique=True, callable_=lambda : \ - self.property.mapper._get_committed_attr_by_column(o, - col)) - - def adapt(col): - if self.adapter: - return self.adapter(col) - else: - return col - - if self.property._use_get: - return sql.and_(*[ - sql.or_( - adapt(x) != state_bindparam(adapt(x), state, y), - adapt(x) == None) - for (x, y) in self.property.local_remote_pairs]) - - criterion = sql.and_(*[x==y for (x, y) in - zip( - self.property.mapper.primary_key, - self.property.\ - mapper.\ - primary_key_from_instance(other)) - ]) - return ~self._criterion_exists(criterion) - - def __ne__(self, other): - """Implement the ``!=`` operator. - - In a many-to-one context, such as:: - - MyClass.some_prop != - - This will typically produce a clause such as:: - - mytable.related_id != - - Where ```` is the primary key of the - given object. - - The ``!=`` operator provides partial functionality for non- - many-to-one comparisons: - - * Comparisons against collections are not supported. - Use - :meth:`~.RelationshipProperty.Comparator.contains` - in conjunction with :func:`~.expression.not_`. - * Compared to a scalar one-to-many, will produce a - clause that compares the target columns in the parent to - the given target. - * Compared to a scalar many-to-many, an alias - of the association table will be rendered as - well, forming a natural join that is part of the - main body of the query. This will not work for - queries that go beyond simple AND conjunctions of - comparisons, such as those which use OR. Use - explicit joins, outerjoins, or - :meth:`~.RelationshipProperty.Comparator.has` in - conjunction with :func:`~.expression.not_` for - more comprehensive non-many-to-one scalar - membership tests. - * Comparisons against ``None`` given in a one-to-many - or many-to-many context produce an EXISTS clause. - - """ - if isinstance(other, (NoneType, expression._Null)): - if self.property.direction == MANYTOONE: - return sql.or_(*[x != None for x in - self.property._calculated_foreign_keys]) - else: - return self._criterion_exists() - elif self.property.uselist: - raise sa_exc.InvalidRequestError("Can't compare a collection" - " to an object or collection; use " - "contains() to test for membership.") - else: - return self.__negated_contains_or_equals(other) - - @util.memoized_property - def property(self): - if mapperlib.module._new_mappers: - configure_mappers() - return self.prop - - def compare(self, op, value, - value_is_parent=False, - alias_secondary=True): - if op == operators.eq: - if value is None: - if self.uselist: - return ~sql.exists([1], self.primaryjoin) - else: - return self._optimized_compare(None, - value_is_parent=value_is_parent, - alias_secondary=alias_secondary) - else: - return self._optimized_compare(value, - value_is_parent=value_is_parent, - alias_secondary=alias_secondary) - else: - return op(self.comparator, value) - - def _optimized_compare(self, value, value_is_parent=False, - adapt_source=None, - alias_secondary=True): - if value is not None: - value = attributes.instance_state(value) - return self._get_strategy(strategies.LazyLoader).lazy_clause(value, - reverse_direction=not value_is_parent, - alias_secondary=alias_secondary, - adapt_source=adapt_source) - - def __str__(self): - return str(self.parent.class_.__name__) + "." + self.key - - def merge(self, - session, - source_state, - source_dict, - dest_state, - dest_dict, - load, _recursive): - - if load: - for r in self._reverse_property: - if (source_state, r) in _recursive: - return - - - if not "merge" in self.cascade: - return - - if self.key not in source_dict: - return - - if self.uselist: - instances = source_state.get_impl(self.key).\ - get(source_state, source_dict) - if hasattr(instances, '_sa_adapter'): - # convert collections to adapters to get a true iterator - instances = instances._sa_adapter - - if load: - # for a full merge, pre-load the destination collection, - # so that individual _merge of each item pulls from identity - # map for those already present. - # also assumes CollectionAttrbiuteImpl behavior of loading - # "old" list in any case - dest_state.get_impl(self.key).get(dest_state, dest_dict) - - dest_list = [] - for current in instances: - current_state = attributes.instance_state(current) - current_dict = attributes.instance_dict(current) - _recursive[(current_state, self)] = True - obj = session._merge(current_state, current_dict, - load=load, _recursive=_recursive) - if obj is not None: - dest_list.append(obj) - - if not load: - coll = attributes.init_state_collection(dest_state, - dest_dict, self.key) - for c in dest_list: - coll.append_without_event(c) - else: - dest_state.get_impl(self.key)._set_iterable(dest_state, - dest_dict, dest_list) - else: - current = source_dict[self.key] - if current is not None: - current_state = attributes.instance_state(current) - current_dict = attributes.instance_dict(current) - _recursive[(current_state, self)] = True - obj = session._merge(current_state, current_dict, - load=load, _recursive=_recursive) - else: - obj = None - - if not load: - dest_dict[self.key] = obj - else: - dest_state.get_impl(self.key).set(dest_state, - dest_dict, obj, None) - - def cascade_iterator(self, type_, state, dict_, visited_states, halt_on=None): - #assert type_ in self.cascade - - # only actively lazy load on the 'delete' cascade - if type_ != 'delete' or self.passive_deletes: - passive = attributes.PASSIVE_NO_INITIALIZE - else: - passive = attributes.PASSIVE_OFF - - if type_ == 'save-update': - tuples = state.manager[self.key].impl.\ - get_all_pending(state, dict_) - - else: - tuples = state.value_as_iterable(dict_, self.key, - passive=passive) - - skip_pending = type_ == 'refresh-expire' and 'delete-orphan' \ - not in self.cascade - - for instance_state, c in tuples: - if instance_state in visited_states: - continue - - if c is None: - # would like to emit a warning here, but - # would not be consistent with collection.append(None) - # current behavior of silently skipping. - # see [ticket:2229] - continue - - instance_dict = attributes.instance_dict(c) - - if halt_on and halt_on(instance_state): - continue - - if skip_pending and not instance_state.key: - continue - - instance_mapper = instance_state.manager.mapper - - if not instance_mapper.isa(self.mapper.class_manager.mapper): - raise AssertionError("Attribute '%s' on class '%s' " - "doesn't handle objects " - "of type '%s'" % ( - self.key, - self.parent.class_, - c.__class__ - )) - - visited_states.add(instance_state) - - yield c, instance_mapper, instance_state, instance_dict - - - def _add_reverse_property(self, key): - other = self.mapper.get_property(key, _compile_mappers=False) - self._reverse_property.add(other) - other._reverse_property.add(self) - - if not other.mapper.common_parent(self.parent): - raise sa_exc.ArgumentError('reverse_property %r on ' - 'relationship %s references relationship %s, which ' - 'does not reference mapper %s' % (key, self, other, - self.parent)) - if self.direction in (ONETOMANY, MANYTOONE) and self.direction \ - == other.direction: - raise sa_exc.ArgumentError('%s and back-reference %s are ' - 'both of the same direction %r. Did you mean to ' - 'set remote_side on the many-to-one side ?' - % (other, self, self.direction)) - - @util.memoized_property - def mapper(self): - """Return the targeted :class:`.Mapper` for this - :class:`.RelationshipProperty`. - - This is a lazy-initializing static attribute. - - """ - if isinstance(self.argument, type): - mapper_ = mapper.class_mapper(self.argument, - compile=False) - elif isinstance(self.argument, mapper.Mapper): - mapper_ = self.argument - elif util.callable(self.argument): - - # accept a callable to suit various deferred- - # configurational schemes - - mapper_ = mapper.class_mapper(self.argument(), - compile=False) - else: - raise sa_exc.ArgumentError("relationship '%s' expects " - "a class or a mapper argument (received: %s)" - % (self.key, type(self.argument))) - assert isinstance(mapper_, mapper.Mapper), mapper_ - return mapper_ - - @util.memoized_property - @util.deprecated("0.7", "Use .target") - def table(self): - """Return the selectable linked to this - :class:`.RelationshipProperty` object's target - :class:`.Mapper`.""" - return self.target - - def do_init(self): - self._check_conflicts() - self._process_dependent_arguments() - self._determine_joins() - self._determine_synchronize_pairs() - self._determine_direction() - self._determine_local_remote_pairs() - self._post_init() - self._generate_backref() - super(RelationshipProperty, self).do_init() - - def _check_conflicts(self): - """Test that this relationship is legal, warn about - inheritance conflicts.""" - - if not self.is_primary() \ - and not mapper.class_mapper( - self.parent.class_, - compile=False).has_property(self.key): - raise sa_exc.ArgumentError("Attempting to assign a new " - "relationship '%s' to a non-primary mapper on " - "class '%s'. New relationships can only be added " - "to the primary mapper, i.e. the very first mapper " - "created for class '%s' " % (self.key, - self.parent.class_.__name__, - self.parent.class_.__name__)) - - # check for conflicting relationship() on superclass - if not self.parent.concrete: - for inheriting in self.parent.iterate_to_root(): - if inheriting is not self.parent \ - and inheriting.has_property(self.key): - util.warn("Warning: relationship '%s' on mapper " - "'%s' supersedes the same relationship " - "on inherited mapper '%s'; this can " - "cause dependency issues during flush" - % (self.key, self.parent, inheriting)) - - def _process_dependent_arguments(self): - """Convert incoming configuration arguments to their - proper form. - - Callables are resolved, ORM annotations removed. - - """ - # accept callables for other attributes which may require - # deferred initialization. This technique is used - # by declarative "string configs" and some recipes. - for attr in ( - 'order_by', - 'primaryjoin', - 'secondaryjoin', - 'secondary', - '_user_defined_foreign_keys', - 'remote_side', - ): - attr_value = getattr(self, attr) - if util.callable(attr_value): - setattr(self, attr, attr_value()) - - # remove "annotations" which are present if mapped class - # descriptors are used to create the join expression. - for attr in 'primaryjoin', 'secondaryjoin': - val = getattr(self, attr) - if val is not None: - setattr(self, attr, _orm_deannotate( - expression._only_column_elements(val, attr)) - ) - - # ensure expressions in self.order_by, foreign_keys, - # remote_side are all columns, not strings. - if self.order_by is not False and self.order_by is not None: - self.order_by = [ - expression._only_column_elements(x, "order_by") - for x in - util.to_list(self.order_by)] - - self._user_defined_foreign_keys = \ - util.column_set( - expression._only_column_elements(x, "foreign_keys") - for x in util.to_column_set( - self._user_defined_foreign_keys - )) - - self.remote_side = \ - util.column_set( - expression._only_column_elements(x, "remote_side") - for x in - util.to_column_set(self.remote_side)) - - self.target = self.mapper.mapped_table - - if self.cascade.delete_orphan: - self.mapper.primary_mapper().delete_orphans.append( - (self.key, self.parent.class_) - ) - - def _determine_joins(self): - """Determine the 'primaryjoin' and 'secondaryjoin' attributes, - if not passed to the constructor already. - - This is based on analysis of the foreign key relationships - between the parent and target mapped selectables. - - """ - if self.secondaryjoin is not None and self.secondary is None: - raise sa_exc.ArgumentError("Property '" + self.key - + "' specified with secondary join condition but " - "no secondary argument") - - # if join conditions were not specified, figure them out based - # on foreign keys - - def _search_for_join(mapper, table): - # find a join between the given mapper's mapped table and - # the given table. will try the mapper's local table first - # for more specificity, then if not found will try the more - # general mapped table, which in the case of inheritance is - # a join. - return join_condition(mapper.mapped_table, table, - a_subset=mapper.local_table) - - try: - if self.secondary is not None: - if self.secondaryjoin is None: - self.secondaryjoin = _search_for_join(self.mapper, - self.secondary) - if self.primaryjoin is None: - self.primaryjoin = _search_for_join(self.parent, - self.secondary) - else: - if self.primaryjoin is None: - self.primaryjoin = _search_for_join(self.parent, - self.target) - except sa_exc.ArgumentError, e: - raise sa_exc.ArgumentError("Could not determine join " - "condition between parent/child tables on " - "relationship %s. Specify a 'primaryjoin' " - "expression. If 'secondary' is present, " - "'secondaryjoin' is needed as well." - % self) - - def _columns_are_mapped(self, *cols): - """Return True if all columns in the given collection are - mapped by the tables referenced by this :class:`.Relationship`. - - """ - for c in cols: - if self.secondary is not None \ - and self.secondary.c.contains_column(c): - continue - if not self.parent.mapped_table.c.contains_column(c) and \ - not self.target.c.contains_column(c): - return False - return True - - def _sync_pairs_from_join(self, join_condition, primary): - """Determine a list of "source"/"destination" column pairs - based on the given join condition, as well as the - foreign keys argument. - - "source" would be a column referenced by a foreign key, - and "destination" would be the column who has a foreign key - reference to "source". - - """ - - fks = self._user_defined_foreign_keys - # locate pairs - eq_pairs = criterion_as_pairs(join_condition, - consider_as_foreign_keys=fks, - any_operator=self.viewonly) - - # couldn't find any fks, but we have - # "secondary" - assume the "secondary" columns - # are the fks - if not eq_pairs and \ - self.secondary is not None and \ - not fks: - fks = set(self.secondary.c) - eq_pairs = criterion_as_pairs(join_condition, - consider_as_foreign_keys=fks, - any_operator=self.viewonly) - - if eq_pairs: - util.warn("No ForeignKey objects were present " - "in secondary table '%s'. Assumed referenced " - "foreign key columns %s for join condition '%s' " - "on relationship %s" % ( - self.secondary.description, - ", ".join(sorted(["'%s'" % col for col in fks])), - join_condition, - self - )) - - # Filter out just to columns that are mapped. - # If viewonly, allow pairs where the FK col - # was part of "foreign keys" - the column it references - # may be in an un-mapped table - see - # test.orm.test_relationships.ViewOnlyComplexJoin.test_basic - # for an example of this. - eq_pairs = [(l, r) for (l, r) in eq_pairs - if self._columns_are_mapped(l, r) - or self.viewonly and - r in fks] - - if eq_pairs: - return eq_pairs - - # from here below is just determining the best error message - # to report. Check for a join condition using any operator - # (not just ==), perhaps they need to turn on "viewonly=True". - if not self.viewonly and criterion_as_pairs(join_condition, - consider_as_foreign_keys=self._user_defined_foreign_keys, - any_operator=True): - - err = "Could not locate any "\ - "foreign-key-equated, locally mapped column "\ - "pairs for %s "\ - "condition '%s' on relationship %s." % ( - primary and 'primaryjoin' or 'secondaryjoin', - join_condition, - self - ) - - if not self._user_defined_foreign_keys: - err += " Ensure that the "\ - "referencing Column objects have a "\ - "ForeignKey present, or are otherwise part "\ - "of a ForeignKeyConstraint on their parent "\ - "Table, or specify the foreign_keys parameter "\ - "to this relationship." - - err += " For more "\ - "relaxed rules on join conditions, the "\ - "relationship may be marked as viewonly=True." - - raise sa_exc.ArgumentError(err) - else: - if self._user_defined_foreign_keys: - raise sa_exc.ArgumentError("Could not determine " - "relationship direction for %s condition " - "'%s', on relationship %s, using manual " - "'foreign_keys' setting. Do the columns " - "in 'foreign_keys' represent all, and " - "only, the 'foreign' columns in this join " - "condition? Does the %s Table already " - "have adequate ForeignKey and/or " - "ForeignKeyConstraint objects established " - "(in which case 'foreign_keys' is usually " - "unnecessary)?" - % ( - primary and 'primaryjoin' or 'secondaryjoin', - join_condition, - self, - primary and 'mapped' or 'secondary' - )) - else: - raise sa_exc.ArgumentError("Could not determine " - "relationship direction for %s condition " - "'%s', on relationship %s. Ensure that the " - "referencing Column objects have a " - "ForeignKey present, or are otherwise part " - "of a ForeignKeyConstraint on their parent " - "Table, or specify the foreign_keys parameter " - "to this relationship." - % ( - primary and 'primaryjoin' or 'secondaryjoin', - join_condition, - self - )) - - def _determine_synchronize_pairs(self): - """Resolve 'primary'/foreign' column pairs from the primaryjoin - and secondaryjoin arguments. - - """ - if self.local_remote_pairs: - if not self._user_defined_foreign_keys: - raise sa_exc.ArgumentError( - "foreign_keys argument is " - "required with _local_remote_pairs argument") - self.synchronize_pairs = [] - for l, r in self.local_remote_pairs: - if r in self._user_defined_foreign_keys: - self.synchronize_pairs.append((l, r)) - elif l in self._user_defined_foreign_keys: - self.synchronize_pairs.append((r, l)) - else: - self.synchronize_pairs = self._sync_pairs_from_join( - self.primaryjoin, - True) - - self._calculated_foreign_keys = util.column_set( - r for (l, r) in - self.synchronize_pairs) - - if self.secondaryjoin is not None: - self.secondary_synchronize_pairs = self._sync_pairs_from_join( - self.secondaryjoin, - False) - self._calculated_foreign_keys.update( - r for (l, r) in - self.secondary_synchronize_pairs) - else: - self.secondary_synchronize_pairs = None - - def _determine_direction(self): - """Determine if this relationship is one to many, many to one, - many to many. - - This is derived from the primaryjoin, presence of "secondary", - and in the case of self-referential the "remote side". - - """ - if self.secondaryjoin is not None: - self.direction = MANYTOMANY - elif self._refers_to_parent_table(): - - # self referential defaults to ONETOMANY unless the "remote" - # side is present and does not reference any foreign key - # columns - - if self.local_remote_pairs: - remote = [r for (l, r) in self.local_remote_pairs] - elif self.remote_side: - remote = self.remote_side - else: - remote = None - if not remote or self._calculated_foreign_keys.difference(l for (l, - r) in self.synchronize_pairs).intersection(remote): - self.direction = ONETOMANY - else: - self.direction = MANYTOONE - else: - parentcols = util.column_set(self.parent.mapped_table.c) - targetcols = util.column_set(self.mapper.mapped_table.c) - - # fk collection which suggests ONETOMANY. - onetomany_fk = targetcols.intersection( - self._calculated_foreign_keys) - - # fk collection which suggests MANYTOONE. - - manytoone_fk = parentcols.intersection( - self._calculated_foreign_keys) - - if onetomany_fk and manytoone_fk: - # fks on both sides. do the same test only based on the - # local side. - referents = [c for (c, f) in self.synchronize_pairs] - onetomany_local = parentcols.intersection(referents) - manytoone_local = targetcols.intersection(referents) - - if onetomany_local and not manytoone_local: - self.direction = ONETOMANY - elif manytoone_local and not onetomany_local: - self.direction = MANYTOONE - else: - raise sa_exc.ArgumentError( - "Can't determine relationship" - " direction for relationship '%s' - foreign " - "key columns are present in both the parent " - "and the child's mapped tables. Specify " - "'foreign_keys' argument." % self) - elif onetomany_fk: - self.direction = ONETOMANY - elif manytoone_fk: - self.direction = MANYTOONE - else: - raise sa_exc.ArgumentError("Can't determine relationship " - "direction for relationship '%s' - foreign " - "key columns are present in neither the parent " - "nor the child's mapped tables" % self) - - if self.cascade.delete_orphan and not self.single_parent \ - and (self.direction is MANYTOMANY or self.direction - is MANYTOONE): - util.warn('On %s, delete-orphan cascade is not supported ' - 'on a many-to-many or many-to-one relationship ' - 'when single_parent is not set. Set ' - 'single_parent=True on the relationship().' - % self) - if self.direction is MANYTOONE and self.passive_deletes: - util.warn("On %s, 'passive_deletes' is normally configured " - "on one-to-many, one-to-one, many-to-many " - "relationships only." - % self) - - def _determine_local_remote_pairs(self): - """Determine pairs of columns representing "local" to - "remote", where "local" columns are on the parent mapper, - "remote" are on the target mapper. - - These pairs are used on the load side only to generate - lazy loading clauses. - - """ - if not self.local_remote_pairs and not self.remote_side: - # the most common, trivial case. Derive - # local/remote pairs from the synchronize pairs. - eq_pairs = util.unique_list( - self.synchronize_pairs + - (self.secondary_synchronize_pairs or [])) - if self.direction is MANYTOONE: - self.local_remote_pairs = [(r, l) for l, r in eq_pairs] - else: - self.local_remote_pairs = eq_pairs - - # "remote_side" specified, derive from the primaryjoin - # plus remote_side, similarly to how synchronize_pairs - # were determined. - elif self.remote_side: - if self.local_remote_pairs: - raise sa_exc.ArgumentError('remote_side argument is ' - 'redundant against more detailed ' - '_local_remote_side argument.') - if self.direction is MANYTOONE: - self.local_remote_pairs = [(r, l) for (l, r) in - criterion_as_pairs(self.primaryjoin, - consider_as_referenced_keys=self.remote_side, - any_operator=True)] - - else: - self.local_remote_pairs = \ - criterion_as_pairs(self.primaryjoin, - consider_as_foreign_keys=self.remote_side, - any_operator=True) - if not self.local_remote_pairs: - raise sa_exc.ArgumentError('Relationship %s could ' - 'not determine any local/remote column ' - 'pairs from remote side argument %r' - % (self, self.remote_side)) - # else local_remote_pairs were sent explcitly via - # ._local_remote_pairs. - - # create local_side/remote_side accessors - self.local_side = util.ordered_column_set( - l for l, r in self.local_remote_pairs) - self.remote_side = util.ordered_column_set( - r for l, r in self.local_remote_pairs) - - # check that the non-foreign key column in the local/remote - # collection is mapped. The foreign key - # which the individual mapped column references directly may - # itself be in a non-mapped table; see - # test.orm.test_relationships.ViewOnlyComplexJoin.test_basic - # for an example of this. - if self.direction is ONETOMANY: - for col in self.local_side: - if not self._columns_are_mapped(col): - raise sa_exc.ArgumentError( - "Local column '%s' is not " - "part of mapping %s. Specify remote_side " - "argument to indicate which column lazy join " - "condition should compare against." % (col, - self.parent)) - elif self.direction is MANYTOONE: - for col in self.remote_side: - if not self._columns_are_mapped(col): - raise sa_exc.ArgumentError( - "Remote column '%s' is not " - "part of mapping %s. Specify remote_side " - "argument to indicate which column lazy join " - "condition should bind." % (col, self.mapper)) - - def _generate_backref(self): - if not self.is_primary(): - return - if self.backref is not None and not self.back_populates: - if isinstance(self.backref, basestring): - backref_key, kwargs = self.backref, {} - else: - backref_key, kwargs = self.backref - mapper = self.mapper.primary_mapper() - if mapper.has_property(backref_key): - raise sa_exc.ArgumentError("Error creating backref " - "'%s' on relationship '%s': property of that " - "name exists on mapper '%s'" % (backref_key, - self, mapper)) - if self.secondary is not None: - pj = kwargs.pop('primaryjoin', self.secondaryjoin) - sj = kwargs.pop('secondaryjoin', self.primaryjoin) - else: - pj = kwargs.pop('primaryjoin', self.primaryjoin) - sj = kwargs.pop('secondaryjoin', None) - if sj: - raise sa_exc.InvalidRequestError( - "Can't assign 'secondaryjoin' on a backref against " - "a non-secondary relationship." - ) - foreign_keys = kwargs.pop('foreign_keys', - self._user_defined_foreign_keys) - parent = self.parent.primary_mapper() - kwargs.setdefault('viewonly', self.viewonly) - kwargs.setdefault('post_update', self.post_update) - kwargs.setdefault('passive_updates', self.passive_updates) - self.back_populates = backref_key - relationship = RelationshipProperty( - parent, - self.secondary, - pj, - sj, - foreign_keys=foreign_keys, - back_populates=self.key, - **kwargs - ) - mapper._configure_property(backref_key, relationship) - - if self.back_populates: - self._add_reverse_property(self.back_populates) - - def _post_init(self): - self.logger.info('%s setup primary join %s', self, - self.primaryjoin) - self.logger.info('%s setup secondary join %s', self, - self.secondaryjoin) - self.logger.info('%s synchronize pairs [%s]', self, - ','.join('(%s => %s)' % (l, r) for (l, r) in - self.synchronize_pairs)) - self.logger.info('%s secondary synchronize pairs [%s]', self, - ','.join('(%s => %s)' % (l, r) for (l, r) in - self.secondary_synchronize_pairs or [])) - self.logger.info('%s local/remote pairs [%s]', self, - ','.join('(%s / %s)' % (l, r) for (l, r) in - self.local_remote_pairs)) - self.logger.info('%s relationship direction %s', self, - self.direction) - if self.uselist is None: - self.uselist = self.direction is not MANYTOONE - if not self.viewonly: - self._dependency_processor = \ - dependency.DependencyProcessor.from_relationship(self) - - @util.memoized_property - def _use_get(self): - """memoize the 'use_get' attribute of this RelationshipLoader's - lazyloader.""" - - strategy = self._get_strategy(strategies.LazyLoader) - return strategy.use_get - - def _refers_to_parent_table(self): - pt = self.parent.mapped_table - mt = self.mapper.mapped_table - for c, f in self.synchronize_pairs: - if ( - pt.is_derived_from(c.table) and \ - pt.is_derived_from(f.table) and \ - mt.is_derived_from(c.table) and \ - mt.is_derived_from(f.table) - ): - return True - else: - return False - - @util.memoized_property - def _is_self_referential(self): - return self.mapper.common_parent(self.parent) - - def per_property_preprocessors(self, uow): - if not self.viewonly and self._dependency_processor: - self._dependency_processor.per_property_preprocessors(uow) - - def _create_joins(self, source_polymorphic=False, - source_selectable=None, dest_polymorphic=False, - dest_selectable=None, of_type=None): - if source_selectable is None: - if source_polymorphic and self.parent.with_polymorphic: - source_selectable = self.parent._with_polymorphic_selectable - - aliased = False - if dest_selectable is None: - if dest_polymorphic and self.mapper.with_polymorphic: - dest_selectable = self.mapper._with_polymorphic_selectable - aliased = True - else: - dest_selectable = self.mapper.mapped_table - - if self._is_self_referential and source_selectable is None: - dest_selectable = dest_selectable.alias() - aliased = True - else: - aliased = True - - # place a barrier on the destination such that - # replacement traversals won't ever dig into it. - # its internal structure remains fixed - # regardless of context. - dest_selectable = _shallow_annotate( - dest_selectable, - {'no_replacement_traverse':True}) - - aliased = aliased or (source_selectable is not None) - - primaryjoin, secondaryjoin, secondary = self.primaryjoin, \ - self.secondaryjoin, self.secondary - - # adjust the join condition for single table inheritance, - # in the case that the join is to a subclass - # this is analogous to the "_adjust_for_single_table_inheritance()" - # method in Query. - - dest_mapper = of_type or self.mapper - - single_crit = dest_mapper._single_table_criterion - if single_crit is not None: - if secondaryjoin is not None: - secondaryjoin = secondaryjoin & single_crit - else: - primaryjoin = primaryjoin & single_crit - - if aliased: - if secondary is not None: - secondary = secondary.alias() - primary_aliasizer = ClauseAdapter(secondary) - secondary_aliasizer = \ - ClauseAdapter(dest_selectable, - equivalents=self.mapper._equivalent_columns).\ - chain(primary_aliasizer) - if source_selectable is not None: - primary_aliasizer = \ - ClauseAdapter(secondary).\ - chain(ClauseAdapter(source_selectable, - equivalents=self.parent._equivalent_columns)) - secondaryjoin = \ - secondary_aliasizer.traverse(secondaryjoin) - else: - primary_aliasizer = ClauseAdapter(dest_selectable, - exclude=self.local_side, - equivalents=self.mapper._equivalent_columns) - if source_selectable is not None: - primary_aliasizer.chain( - ClauseAdapter(source_selectable, - exclude=self.remote_side, - equivalents=self.parent._equivalent_columns)) - secondary_aliasizer = None - primaryjoin = primary_aliasizer.traverse(primaryjoin) - target_adapter = secondary_aliasizer or primary_aliasizer - target_adapter.include = target_adapter.exclude = None - else: - target_adapter = None - if source_selectable is None: - source_selectable = self.parent.local_table - if dest_selectable is None: - dest_selectable = self.mapper.local_table - return ( - primaryjoin, - secondaryjoin, - source_selectable, - dest_selectable, - secondary, - target_adapter, - ) - -PropertyLoader = RelationProperty = RelationshipProperty -log.class_logger(RelationshipProperty) - diff --git a/libs/sqlalchemy/orm/query.py b/libs/sqlalchemy/orm/query.py index 286dbf6b..6bd465e9 100644 --- a/libs/sqlalchemy/orm/query.py +++ b/libs/sqlalchemy/orm/query.py @@ -1,5 +1,5 @@ # orm/query.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -19,38 +19,33 @@ database to return iterable result sets. """ from itertools import chain -from operator import itemgetter -from sqlalchemy import sql, util, log, schema -from sqlalchemy import exc as sa_exc -from sqlalchemy.orm import exc as orm_exc -from sqlalchemy.sql import util as sql_util -from sqlalchemy.sql import expression, visitors, operators -from sqlalchemy.orm import ( - attributes, interfaces, mapper, object_mapper, evaluator, +from . import ( + attributes, interfaces, object_mapper, persistence, + exc as orm_exc, loading ) -from sqlalchemy.orm.util import ( - AliasedClass, ORMAdapter, _entity_descriptor, _entity_info, - _is_aliased_class, _is_mapped_class, _orm_columns, _orm_selectable, - join as orm_join,with_parent, _attr_as_key, aliased +from .base import _entity_descriptor, _is_aliased_class, \ + _is_mapped_class, _orm_columns, _generative +from .path_registry import PathRegistry +from .util import ( + AliasedClass, ORMAdapter, join as orm_join, with_parent, aliased ) - +from .. import sql, util, log, exc as sa_exc, inspect, inspection +from ..sql.expression import _interpret_as_from +from ..sql import ( + util as sql_util, + expression, visitors + ) +from ..sql.base import ColumnCollection +from . import properties __all__ = ['Query', 'QueryContext', 'aliased'] -def _generative(*assertions): - """Mark a method as generative.""" - - @util.decorator - def generate(fn, *args, **kw): - self = args[0]._clone() - for assertion in assertions: - assertion(self, fn.func_name) - fn(self, *args[1:], **kw) - return self - return generate +_path_registry = PathRegistry.root +@inspection._self_inspects +@log.class_logger class Query(object): """ORM-level SQL construction object. @@ -62,8 +57,9 @@ class Query(object): criteria and options associated with it. :class:`.Query` objects are normally initially generated using the - :meth:`~.Session.query` method of :class:`.Session`. For a full walkthrough - of :class:`.Query` usage, see the :ref:`ormtutorial_toplevel`. + :meth:`~.Session.query` method of :class:`.Session`. For a full + walkthrough of :class:`.Query` usage, see the + :ref:`ormtutorial_toplevel`. """ @@ -72,7 +68,6 @@ class Query(object): _with_labels = False _criterion = None _yield_per = None - _lockmode = None _order_by = False _group_by = False _having = None @@ -80,18 +75,19 @@ class Query(object): _prefixes = None _offset = None _limit = None + _for_update_arg = None _statement = None _correlate = frozenset() _populate_existing = False _invoke_all_eagers = True _version_check = False _autoflush = True - _current_path = () _only_load_props = None _refresh_state = None _from_obj = () _join_entities = () _select_from_entity = None + _mapper_adapter_map = {} _filter_aliases = None _from_obj_alias = None _joinpath = _joinpoint = util.immutabledict() @@ -102,6 +98,8 @@ class Query(object): _with_hints = () _enable_single_crit = True + _current_path = _path_registry + def __init__(self, entities, session=None): self.session = session self._polymorphic_adapters = {} @@ -111,90 +109,108 @@ class Query(object): if entity_wrapper is None: entity_wrapper = _QueryEntity self._entities = [] + self._primary_entity = None for ent in util.to_list(entities): entity_wrapper(self, ent) - self._setup_aliasizers(self._entities) + self._set_entity_selectables(self._entities) - def _setup_aliasizers(self, entities): - if hasattr(self, '_mapper_adapter_map'): - # usually safe to share a single map, but copying to prevent - # subtle leaks if end-user is reusing base query with arbitrary - # number of aliased() objects - self._mapper_adapter_map = d = self._mapper_adapter_map.copy() - else: - self._mapper_adapter_map = d = {} + def _set_entity_selectables(self, entities): + self._mapper_adapter_map = d = self._mapper_adapter_map.copy() for ent in entities: for entity in ent.entities: if entity not in d: - mapper, selectable, is_aliased_class = \ - _entity_info(entity) - if not is_aliased_class and mapper.with_polymorphic: - with_polymorphic = mapper._with_polymorphic_mappers - if mapper.mapped_table not in \ + ext_info = inspect(entity) + if not ext_info.is_aliased_class and \ + ext_info.mapper.with_polymorphic: + if ext_info.mapper.mapped_table not in \ self._polymorphic_adapters: - self._mapper_loads_polymorphically_with(mapper, + self._mapper_loads_polymorphically_with( + ext_info.mapper, sql_util.ColumnAdapter( - selectable, - mapper._equivalent_columns)) - adapter = None - elif is_aliased_class: - adapter = sql_util.ColumnAdapter( - selectable, - mapper._equivalent_columns) - with_polymorphic = None + ext_info.selectable, + ext_info.mapper._equivalent_columns + ) + ) + aliased_adapter = None + elif ext_info.is_aliased_class: + aliased_adapter = sql_util.ColumnAdapter( + ext_info.selectable, + ext_info.mapper._equivalent_columns + ) else: - with_polymorphic = adapter = None + aliased_adapter = None - d[entity] = (mapper, adapter, selectable, - is_aliased_class, with_polymorphic) - ent.setup_entity(entity, *d[entity]) + d[entity] = ( + ext_info, + aliased_adapter + ) + ent.setup_entity(*d[entity]) def _mapper_loads_polymorphically_with(self, mapper, adapter): - for m2 in mapper._with_polymorphic_mappers: + for m2 in mapper._with_polymorphic_mappers or [mapper]: self._polymorphic_adapters[m2] = adapter for m in m2.iterate_to_root(): - self._polymorphic_adapters[m.mapped_table] = \ - self._polymorphic_adapters[m.local_table] = \ - adapter - - def _set_select_from(self, *obj): + self._polymorphic_adapters[m.local_table] = adapter + def _set_select_from(self, obj, set_base_alias): fa = [] + select_from_alias = None + for from_obj in obj: - if isinstance(from_obj, expression._SelectBase): - from_obj = from_obj.alias() - fa.append(from_obj) + info = inspect(from_obj) + + if hasattr(info, 'mapper') and \ + (info.is_mapper or info.is_aliased_class): + if set_base_alias: + raise sa_exc.ArgumentError( + "A selectable (FromClause) instance is " + "expected when the base alias is being set.") + fa.append(info.selectable) + elif not info.is_selectable: + raise sa_exc.ArgumentError( + "argument is not a mapped class, mapper, " + "aliased(), or FromClause instance.") + else: + if isinstance(from_obj, expression.SelectBase): + from_obj = from_obj.alias() + if set_base_alias: + select_from_alias = from_obj + fa.append(from_obj) self._from_obj = tuple(fa) - if len(self._from_obj) == 1 and \ - isinstance(self._from_obj[0], expression.Alias): + if set_base_alias and \ + len(self._from_obj) == 1 and \ + isinstance(select_from_alias, expression.Alias): equivs = self.__all_equivs() self._from_obj_alias = sql_util.ColumnAdapter( self._from_obj[0], equivs) - def _reset_polymorphic_adapter(self, mapper): for m2 in mapper._with_polymorphic_mappers: self._polymorphic_adapters.pop(m2, None) for m in m2.iterate_to_root(): - self._polymorphic_adapters.pop(m.mapped_table, None) self._polymorphic_adapters.pop(m.local_table, None) - def __adapt_polymorphic_element(self, element): + def _adapt_polymorphic_element(self, element): + if "parententity" in element._annotations: + search = element._annotations['parententity'] + alias = self._polymorphic_adapters.get(search, None) + if alias: + return alias.adapt_clause(element) + if isinstance(element, expression.FromClause): search = element elif hasattr(element, 'table'): search = element.table else: - search = None + return None - if search is not None: - alias = self._polymorphic_adapters.get(search, None) - if alias: - return alias.adapt_clause(element) + alias = self._polymorphic_adapters.get(search, None) + if alias: + return alias.adapt_clause(element) def _adapt_col_list(self, cols): return [ @@ -209,8 +225,8 @@ class Query(object): self._orm_only_adapt = False def _adapt_clause(self, clause, as_filter, orm_only): - """Adapt incoming clauses to transformations which have been applied - within this query.""" + """Adapt incoming clauses to transformations which + have been applied within this query.""" adapters = [] @@ -241,7 +257,7 @@ class Query(object): if self._polymorphic_adapters: adapters.append( ( - orm_only, self.__adapt_polymorphic_element + orm_only, self._adapt_polymorphic_element ) ) @@ -273,14 +289,10 @@ class Query(object): return self._select_from_entity or \ self._entity_zero().entity_zero - @property def _mapper_entities(self): - # TODO: this is wrong, its hardcoded to "primary entity" when - # for the case of __all_equivs() it should not be - # the name of this accessor is wrong too for ent in self._entities: - if hasattr(ent, 'primary_entity'): + if isinstance(ent, _MapperEntity): yield ent def _joinpoint_zero(self): @@ -290,39 +302,36 @@ class Query(object): ) def _mapper_zero_or_none(self): - if not getattr(self._entities[0], 'primary_entity', False): + if self._primary_entity: + return self._primary_entity.mapper + else: return None - return self._entities[0].mapper def _only_mapper_zero(self, rationale=None): if len(self._entities) > 1: raise sa_exc.InvalidRequestError( rationale or - "This operation requires a Query against a single mapper." + "This operation requires a Query " + "against a single mapper." ) return self._mapper_zero() def _only_full_mapper_zero(self, methname): - if len(self._entities) != 1: + if self._entities != [self._primary_entity]: raise sa_exc.InvalidRequestError( "%s() can only be used against " "a single mapped class." % methname) - entity = self._entity_zero() - if not hasattr(entity, 'primary_entity'): - raise sa_exc.InvalidRequestError( - "%s() can only be used against " - "a single mapped class." % methname) - return entity.entity_zero + return self._primary_entity.entity_zero def _only_entity_zero(self, rationale=None): if len(self._entities) > 1: raise sa_exc.InvalidRequestError( rationale or - "This operation requires a Query against a single mapper." + "This operation requires a Query " + "against a single mapper." ) return self._entity_zero() - def __all_equivs(self): equivs = {} for ent in self._mapper_entities: @@ -373,7 +382,8 @@ class Query(object): "Query.%s() being called on a Query which already has LIMIT " "or OFFSET applied. To modify the row-limited results of a " " Query, call from_self() first. " - "Otherwise, call %s() before limit() or offset() are applied." + "Otherwise, call %s() before limit() or offset() " + "are applied." % (meth, meth) ) @@ -427,47 +437,63 @@ class Query(object): statement if self._params: stmt = stmt.params(self._params) + + # TODO: there's no tests covering effects of # the annotation not being there return stmt._annotate({'no_replacement_traverse': True}) - def subquery(self, name=None): - """return the full SELECT statement represented by this :class:`.Query`, - embedded within an :class:`.Alias`. + def subquery(self, name=None, with_labels=False, reduce_columns=False): + """return the full SELECT statement represented by + this :class:`.Query`, embedded within an :class:`.Alias`. Eager JOIN generation within the query is disabled. - The statement will not have disambiguating labels - applied to the list of selected columns unless the - :meth:`.Query.with_labels` method is used to generate a new - :class:`.Query` with the option enabled. - :param name: string name to be assigned as the alias; this is passed through to :meth:`.FromClause.alias`. If ``None``, a name will be deterministically generated at compile time. + :param with_labels: if True, :meth:`.with_labels` will be called + on the :class:`.Query` first to apply table-qualified labels + to all columns. + + :param reduce_columns: if True, :meth:`.Select.reduce_columns` will + be called on the resulting :func:`.select` construct, + to remove same-named columns where one also refers to the other + via foreign key or WHERE clause equivalence. + + .. versionchanged:: 0.8 the ``with_labels`` and ``reduce_columns`` + keyword arguments were added. """ - return self.enable_eagerloads(False).statement.alias(name=name) + q = self.enable_eagerloads(False) + if with_labels: + q = q.with_labels() + q = q.statement + if reduce_columns: + q = q.reduce_columns() + return q.alias(name=name) def cte(self, name=None, recursive=False): - """Return the full SELECT statement represented by this :class:`.Query` - represented as a common table expression (CTE). + """Return the full SELECT statement represented by this + :class:`.Query` represented as a common table expression (CTE). .. versionadded:: 0.7.6 Parameters and usage are the same as those of the - :meth:`._SelectBase.cte` method; see that method for + :meth:`.SelectBase.cte` method; see that method for further details. Here is the `Postgresql WITH - RECURSIVE example `_. - Note that, in this example, the ``included_parts`` cte and the ``incl_alias`` alias - of it are Core selectables, which - means the columns are accessed via the ``.c.`` attribute. The ``parts_alias`` - object is an :func:`.orm.aliased` instance of the ``Part`` entity, so column-mapped - attributes are available directly:: + RECURSIVE example + `_. + Note that, in this example, the ``included_parts`` cte and the + ``incl_alias`` alias of it are Core selectables, which + means the columns are accessed via the ``.c.`` attribute. The + ``parts_alias`` object is an :func:`.orm.aliased` instance of the + ``Part`` entity, so column-mapped attributes are available + directly:: from sqlalchemy.orm import aliased @@ -478,11 +504,11 @@ class Query(object): quantity = Column(Integer) included_parts = session.query( - Part.sub_part, - Part.part, - Part.quantity).\\ - filter(Part.part=="our part").\\ - cte(name="included_parts", recursive=True) + Part.sub_part, + Part.part, + Part.quantity).\\ + filter(Part.part=="our part").\\ + cte(name="included_parts", recursive=True) incl_alias = aliased(included_parts, name="pr") parts_alias = aliased(Part, name="p") @@ -496,22 +522,25 @@ class Query(object): q = session.query( included_parts.c.sub_part, - func.sum(included_parts.c.quantity).label('total_quantity') + func.sum(included_parts.c.quantity). + label('total_quantity') ).\\ group_by(included_parts.c.sub_part) See also: - :meth:`._SelectBase.cte` + :meth:`.SelectBase.cte` """ - return self.enable_eagerloads(False).statement.cte(name=name, recursive=recursive) + return self.enable_eagerloads(False).\ + statement.cte(name=name, recursive=recursive) def label(self, name): - """Return the full SELECT statement represented by this :class:`.Query`, converted + """Return the full SELECT statement represented by this + :class:`.Query`, converted to a scalar subquery with a label of the given name. - Analogous to :meth:`sqlalchemy.sql._SelectBaseMixin.label`. + Analogous to :meth:`sqlalchemy.sql.expression.SelectBase.label`. .. versionadded:: 0.6.5 @@ -519,12 +548,11 @@ class Query(object): return self.enable_eagerloads(False).statement.label(name) - def as_scalar(self): - """Return the full SELECT statement represented by this :class:`.Query`, converted - to a scalar subquery. + """Return the full SELECT statement represented by this + :class:`.Query`, converted to a scalar subquery. - Analogous to :meth:`sqlalchemy.sql._SelectBaseMixin.as_scalar`. + Analogous to :meth:`sqlalchemy.sql.expression.SelectBase.as_scalar`. .. versionadded:: 0.6.5 @@ -532,6 +560,16 @@ class Query(object): return self.enable_eagerloads(False).statement.as_scalar() + @property + def selectable(self): + """Return the :class:`.Select` object emitted by this :class:`.Query`. + + Used for :func:`.inspect` compatibility, this is equivalent to:: + + query.enable_eagerloads(False).with_labels().statement + + """ + return self.__clause_element__() def __clause_element__(self): return self.enable_eagerloads(False).with_labels().statement @@ -594,7 +632,8 @@ class Query(object): @property def whereclause(self): - """A readonly attribute which returns the current WHERE criterion for this Query. + """A readonly attribute which returns the current WHERE criterion for + this Query. This returned value is a SQL expression construct, or ``None`` if no criterion has been established. @@ -617,39 +656,34 @@ class Query(object): @_generative(_no_clauseelement_condition) def with_polymorphic(self, cls_or_mappers, - selectable=None, discriminator=None): - """Load columns for descendant mappers of this Query's mapper. + selectable=None, + polymorphic_on=None): + """Load columns for inheriting classes. - Using this method will ensure that each descendant mapper's - tables are included in the FROM clause, and will allow filter() - criterion to be used against those tables. The resulting - instances will also have those columns already loaded so that - no "post fetch" of those columns will be required. + :meth:`.Query.with_polymorphic` applies transformations + to the "main" mapped class represented by this :class:`.Query`. + The "main" mapped class here means the :class:`.Query` + object's first argument is a full class, i.e. + ``session.query(SomeClass)``. These transformations allow additional + tables to be present in the FROM clause so that columns for a + joined-inheritance subclass are available in the query, both for the + purposes of load-time efficiency as well as the ability to use + these columns at query time. - :param cls_or_mappers: a single class or mapper, or list of - class/mappers, which inherit from this Query's mapper. - Alternatively, it may also be the string ``'*'``, in which case - all descending mappers will be added to the FROM clause. + See the documentation section :ref:`with_polymorphic` for + details on how this method is used. - :param selectable: a table or select() statement that will - be used in place of the generated FROM clause. This argument is - required if any of the desired mappers use concrete table - inheritance, since SQLAlchemy currently cannot generate UNIONs - among tables automatically. If used, the ``selectable`` argument - must represent the full set of tables and columns mapped by every - desired mapper. Otherwise, the unaccounted mapped columns will - result in their table being appended directly to the FROM clause - which will usually lead to incorrect results. - - :param discriminator: a column to be used as the "discriminator" - column for the given selectable. If not given, the polymorphic_on - attribute of the mapper will be used, if any. This is useful for - mappers that don't have polymorphic loading behavior by default, - such as concrete table mappers. + .. versionchanged:: 0.8 + A new and more flexible function + :func:`.orm.with_polymorphic` supersedes + :meth:`.Query.with_polymorphic`, as it can apply the equivalent + functionality to any set of columns or classes in the + :class:`.Query`, not just the "zero mapper". See that + function for a description of arguments. """ - if not getattr(self._entities[0], 'primary_entity', False): + if not self._primary_entity: raise sa_exc.InvalidRequestError( "No primary mapper set up for this Query.") entity = self._entities[0]._clone() @@ -657,7 +691,7 @@ class Query(object): entity.set_with_polymorphic(self, cls_or_mappers, selectable=selectable, - discriminator=discriminator) + polymorphic_on=polymorphic_on) @_generative() def yield_per(self, count): @@ -674,17 +708,17 @@ class Query(object): loading, the full result for all rows is fetched which generally defeats the purpose of :meth:`~sqlalchemy.orm.query.Query.yield_per`. - Also note that many DBAPIs do not "stream" results, pre-buffering - all rows before making them available, including mysql-python and - psycopg2. :meth:`~sqlalchemy.orm.query.Query.yield_per` will also - set the ``stream_results`` execution - option to ``True``, which currently is only understood by psycopg2 - and causes server side cursors to be used. + Also note that while :meth:`~sqlalchemy.orm.query.Query.yield_per` + will set the ``stream_results`` execution option to True, currently + this is only understood by :mod:`~sqlalchemy.dialects.postgresql.psycopg2` dialect + which will stream results using server side cursors instead of pre-buffer + all rows for this query. Other DBAPIs pre-buffer all rows before + making them available. """ self._yield_per = count - self._execution_options = self._execution_options.copy() - self._execution_options['stream_results'] = True + self._execution_options = self._execution_options.union( + {"stream_results": True}) def get(self, ident): """Return an instance based on the given primary key identifier, @@ -728,7 +762,7 @@ class Query(object): foreign-key-to-primary-key criterion, will also use an operation equivalent to :meth:`~.Query.get` in order to retrieve the target value from the local identity map - before querying the database. See :ref:`loading_toplevel` + before querying the database. See :doc:`/orm/loading` for further details on relationship loading. :param ident: A scalar or tuple value representing @@ -762,9 +796,10 @@ class Query(object): if not self._populate_existing and \ not mapper.always_refresh and \ - self._lockmode is None: + self._for_update_arg is None: - instance = self._get_from_identity(self.session, key, False) + instance = loading.get_from_identity( + self.session, key, attributes.PASSIVE_OFF) if instance is not None: # reject calls for id in identity map but class # mismatch. @@ -772,7 +807,7 @@ class Query(object): return None return instance - return self._load_on_ident(key) + return loading.load_on_ident(self, key) @_generative() def correlate(self, *args): @@ -796,7 +831,8 @@ class Query(object): """ self._correlate = self._correlate.union( - _orm_selectable(s) + _interpret_as_from(s) + if s is not None else None for s in args) @_generative() @@ -852,11 +888,10 @@ class Query(object): """ if property is None: - from sqlalchemy.orm import properties mapper = object_mapper(instance) for prop in mapper.iterate_properties: - if isinstance(prop, properties.PropertyLoader) and \ + if isinstance(prop, properties.RelationshipProperty) and \ prop.mapper is self._mapper_zero(): property = prop break @@ -881,11 +916,11 @@ class Query(object): self._entities = list(self._entities) m = _MapperEntity(self, entity) - self._setup_aliasizers([m]) + self._set_entity_selectables([m]) @_generative() def with_session(self, session): - """Return a :class:`Query` that will use the given :class:`.Session`. + """Return a :class:`.Query` that will use the given :class:`.Session`. """ @@ -922,7 +957,7 @@ class Query(object): '_prefixes', ): self.__dict__.pop(attr, None) - self._set_select_from(fromclause) + self._set_select_from([fromclause], True) # this enables clause adaptation for non-ORM # expressions. @@ -950,18 +985,14 @@ class Query(object): """Return a scalar result corresponding to the given column expression.""" try: - # Py3K - #return self.values(column).__next__()[0] - # Py2K - return self.values(column).next()[0] - # end Py2K + return next(self.values(column))[0] except StopIteration: return None @_generative() def with_entities(self, *entities): - """Return a new :class:`.Query` replacing the SELECT list with the given - entities. + """Return a new :class:`.Query` replacing the SELECT list with the + given entities. e.g.:: @@ -986,7 +1017,6 @@ class Query(object): """ self._set_entities(entities) - @_generative() def add_columns(self, *column): """Add one or more column expressions to the list @@ -998,19 +1028,19 @@ class Query(object): _ColumnEntity(self, c) # _ColumnEntity may add many entities if the # given arg is a FROM clause - self._setup_aliasizers(self._entities[l:]) + self._set_entity_selectables(self._entities[l:]) @util.pending_deprecation("0.7", ":meth:`.add_column` is superseded by :meth:`.add_columns`", False) def add_column(self, column): - """Add a column expression to the list of result columns to be returned. + """Add a column expression to the list of result columns to be + returned. Pending deprecation: :meth:`.add_column` will be superseded by :meth:`.add_columns`. """ - return self.add_columns(column) def options(self, *args): @@ -1019,7 +1049,7 @@ class Query(object): Most supplied options regard changing how column- and relationship-mapped attributes are loaded. See the sections - :ref:`deferred` and :ref:`loading_toplevel` for reference + :ref:`deferred` and :doc:`/orm/loading` for reference documentation. """ @@ -1074,7 +1104,7 @@ class Query(object): :class:`.Table`, :class:`.Alias`, or ORM entity / mapped class /etc. """ - mapper, selectable, is_aliased_class = _entity_info(selectable) + selectable = inspect(selectable).selectable self._with_hints += ((selectable, text, dialect_name),) @@ -1094,32 +1124,63 @@ class Query(object): @_generative() def with_lockmode(self, mode): - """Return a new Query object with the specified locking mode. + """Return a new :class:`.Query` object with the specified "locking mode", + which essentially refers to the ``FOR UPDATE`` clause. - :param mode: a string representing the desired locking mode. A - corresponding value is passed to the ``for_update`` parameter of - :meth:`~sqlalchemy.sql.expression.select` when the query is - executed. Valid values are: + .. deprecated:: 0.9.0 superseded by :meth:`.Query.with_for_update`. - ``'update'`` - passes ``for_update=True``, which translates to - ``FOR UPDATE`` (standard SQL, supported by most dialects) + :param mode: a string representing the desired locking mode. + Valid values are: - ``'update_nowait'`` - passes ``for_update='nowait'``, which - translates to ``FOR UPDATE NOWAIT`` (supported by Oracle, - PostgreSQL 8.1 upwards) + * ``None`` - translates to no lockmode - ``'read'`` - passes ``for_update='read'``, which translates to - ``LOCK IN SHARE MODE`` (for MySQL), and ``FOR SHARE`` (for - PostgreSQL) + * ``'update'`` - translates to ``FOR UPDATE`` + (standard SQL, supported by most dialects) - ``'read_nowait'`` - passes ``for_update='read_nowait'``, which - translates to ``FOR SHARE NOWAIT`` (supported by PostgreSQL). + * ``'update_nowait'`` - translates to ``FOR UPDATE NOWAIT`` + (supported by Oracle, PostgreSQL 8.1 upwards) + + * ``'read'`` - translates to ``LOCK IN SHARE MODE`` (for MySQL), + and ``FOR SHARE`` (for PostgreSQL) + + .. seealso:: + + :meth:`.Query.with_for_update` - improved API for + specifying the ``FOR UPDATE`` clause. - .. versionadded:: 0.7.7 - ``FOR SHARE`` and ``FOR SHARE NOWAIT`` (PostgreSQL). """ + self._for_update_arg = LockmodeArg.parse_legacy_query(mode) - self._lockmode = mode + @_generative() + def with_for_update(self, read=False, nowait=False, of=None): + """return a new :class:`.Query` with the specified options for the + ``FOR UPDATE`` clause. + + The behavior of this method is identical to that of + :meth:`.SelectBase.with_for_update`. When called with no arguments, + the resulting ``SELECT`` statement will have a ``FOR UPDATE`` clause + appended. When additional arguments are specified, backend-specific + options such as ``FOR UPDATE NOWAIT`` or ``LOCK IN SHARE MODE`` + can take effect. + + E.g.:: + + q = sess.query(User).with_for_update(nowait=True, of=User) + + The above query on a Postgresql backend will render like:: + + SELECT users.id AS users_id FROM users FOR UPDATE OF users NOWAIT + + .. versionadded:: 0.9.0 :meth:`.Query.with_for_update` supersedes + the :meth:`.Query.with_lockmode` method. + + .. seealso:: + + :meth:`.GenerativeSelect.with_for_update` - Core level method with + full argument and behavioral description. + + """ + self._for_update_arg = LockmodeArg(read=read, nowait=nowait, of=of) @_generative() def params(self, *args, **kwargs): @@ -1168,14 +1229,7 @@ class Query(object): """ for criterion in list(criterion): - if isinstance(criterion, basestring): - criterion = sql.text(criterion) - - if criterion is not None and \ - not isinstance(criterion, sql.ClauseElement): - raise sa_exc.ArgumentError( - "filter() argument must be of type " - "sqlalchemy.sql.ClauseElement or string") + criterion = expression._literal_as_text(criterion) criterion = self._adapt_clause(criterion, True, True) @@ -1208,7 +1262,7 @@ class Query(object): """ clauses = [_entity_descriptor(self._joinpoint_zero(), key) == value - for key, value in kwargs.iteritems()] + for key, value in kwargs.items()] return self.filter(sql.and_(*clauses)) @_generative(_no_statement_condition, _no_limit_offset) @@ -1249,7 +1303,6 @@ class Query(object): the newly resulting :class:`.Query`""" criterion = list(chain(*[_orm_columns(c) for c in criterion])) - criterion = self._adapt_col_list(criterion) if self._group_by is False: @@ -1262,7 +1315,7 @@ class Query(object): """apply a HAVING criterion to the query and return the newly resulting :class:`.Query`. - :meth:`having` is used in conjunction with :meth:`group_by`. + :meth:`~.Query.having` is used in conjunction with :meth:`~.Query.group_by`. HAVING criterion makes it possible to use filters on aggregate functions like COUNT, SUM, AVG, MAX, and MIN, eg.:: @@ -1274,7 +1327,7 @@ class Query(object): """ - if isinstance(criterion, basestring): + if isinstance(criterion, util.string_types): criterion = sql.text(criterion) if criterion is not None and \ @@ -1328,9 +1381,8 @@ class Query(object): """ - return self._from_selectable( - expression.union(*([self]+ list(q)))) + expression.union(*([self] + list(q)))) def union_all(self, *q): """Produce a UNION ALL of this Query against one or more queries. @@ -1340,7 +1392,7 @@ class Query(object): """ return self._from_selectable( - expression.union_all(*([self]+ list(q))) + expression.union_all(*([self] + list(q))) ) def intersect(self, *q): @@ -1351,7 +1403,7 @@ class Query(object): """ return self._from_selectable( - expression.intersect(*([self]+ list(q))) + expression.intersect(*([self] + list(q))) ) def intersect_all(self, *q): @@ -1362,7 +1414,7 @@ class Query(object): """ return self._from_selectable( - expression.intersect_all(*([self]+ list(q))) + expression.intersect_all(*([self] + list(q))) ) def except_(self, *q): @@ -1373,7 +1425,7 @@ class Query(object): """ return self._from_selectable( - expression.except_(*([self]+ list(q))) + expression.except_(*([self] + list(q))) ) def except_all(self, *q): @@ -1384,7 +1436,7 @@ class Query(object): """ return self._from_selectable( - expression.except_all(*([self]+ list(q))) + expression.except_all(*([self] + list(q))) ) def join(self, *props, **kwargs): @@ -1410,9 +1462,9 @@ class Query(object): In the above example we refer to ``User.addresses`` as passed to :meth:`~.Query.join` as the *on clause*, that is, it indicates how the "ON" portion of the JOIN should be constructed. For a - single-entity query such as the one above (i.e. we start by selecting only from - ``User`` and nothing else), the relationship can also be specified by its - string name:: + single-entity query such as the one above (i.e. we start by selecting + only from ``User`` and nothing else), the relationship can also be + specified by its string name:: q = session.query(User).join("addresses") @@ -1422,8 +1474,9 @@ class Query(object): q = session.query(User).join("orders", "items", "keywords") - The above would be shorthand for three separate calls to :meth:`~.Query.join`, - each using an explicit attribute to indicate the source entity:: + The above would be shorthand for three separate calls to + :meth:`~.Query.join`, each using an explicit attribute to indicate + the source entity:: q = session.query(User).\\ join(User.orders).\\ @@ -1440,7 +1493,7 @@ class Query(object): q = session.query(User).join(Address) - The above calling form of :meth:`.join` will raise an error if + The above calling form of :meth:`~.Query.join` will raise an error if either there are no foreign keys between the two entities, or if there are multiple foreign key linkages between them. In the above calling form, :meth:`~.Query.join` is called upon to @@ -1499,25 +1552,26 @@ class Query(object): There is a lot of flexibility in what the "target" can be when using :meth:`~.Query.join`. As noted previously, it also accepts - :class:`.Table` constructs and other selectables such as :func:`.alias` - and :func:`.select` constructs, with either the one or two-argument forms:: + :class:`.Table` constructs and other selectables such as + :func:`.alias` and :func:`.select` constructs, with either the one + or two-argument forms:: addresses_q = select([Address.user_id]).\\ - where(Address.email_address.endswith("@bar.com")).\\ - alias() + where(Address.email_address.endswith("@bar.com")).\\ + alias() q = session.query(User).\\ join(addresses_q, addresses_q.c.user_id==User.id) :meth:`~.Query.join` also features the ability to *adapt* a - :meth:`~sqlalchemy.orm.relationship` -driven ON clause to the target selectable. - Below we construct a JOIN from ``User`` to a subquery against ``Address``, allowing - the relationship denoted by ``User.addresses`` to *adapt* itself - to the altered target:: + :meth:`~sqlalchemy.orm.relationship` -driven ON clause to the target + selectable. Below we construct a JOIN from ``User`` to a subquery + against ``Address``, allowing the relationship denoted by + ``User.addresses`` to *adapt* itself to the altered target:: address_subq = session.query(Address).\\ - filter(Address.email_address == 'ed@foo.com').\\ - subquery() + filter(Address.email_address == 'ed@foo.com').\\ + subquery() q = session.query(User).join(address_subq, User.addresses) @@ -1601,14 +1655,14 @@ class Query(object): example :ref:`examples_xmlpersistence` which illustrates an XPath-like query system using algorithmic joins. - :param *props: A collection of one or more join conditions, + :param \*props: A collection of one or more join conditions, each consisting of a relationship-bound attribute or string relationship name representing an "on clause", or a single target entity, or a tuple in the form of ``(target, onclause)``. A special two-argument calling form of the form ``target, onclause`` is also accepted. :param aliased=False: If True, indicate that the JOIN target should be - anonymously aliased. Subsequent calls to :class:`~.Query.filter` + anonymously aliased. Subsequent calls to :meth:`~.Query.filter` and similar will adapt the incoming criterion to the target alias, until :meth:`~.Query.reset_joinpoint` is called. :param from_joinpoint=False: When using ``aliased=True``, a setting @@ -1632,7 +1686,7 @@ class Query(object): kwargs.pop('from_joinpoint', False) if kwargs: raise TypeError("unknown arguments: %s" % - ','.join(kwargs.iterkeys())) + ','.join(kwargs.keys)) return self._join(props, outerjoin=False, create_aliases=aliased, from_joinpoint=from_joinpoint) @@ -1648,7 +1702,7 @@ class Query(object): kwargs.pop('from_joinpoint', False) if kwargs: raise TypeError("unknown arguments: %s" % - ','.join(kwargs.iterkeys())) + ','.join(kwargs)) return self._join(props, outerjoin=True, create_aliases=aliased, from_joinpoint=from_joinpoint) @@ -1678,7 +1732,7 @@ class Query(object): if len(keys) == 2 and \ isinstance(keys[0], (expression.FromClause, type, AliasedClass)) and \ - isinstance(keys[1], (basestring, expression.ClauseElement, + isinstance(keys[1], (str, expression.ClauseElement, interfaces.PropComparator)): # detect 2-arg form of join and # convert to a tuple. @@ -1698,14 +1752,14 @@ class Query(object): # is a little bit of legacy behavior still at work here # which means they might be in either order. may possibly # lock this down to (right_entity, onclause) in 0.6. - if isinstance(arg1, (interfaces.PropComparator, basestring)): + if isinstance(arg1, (interfaces.PropComparator, util.string_types)): right_entity, onclause = arg2, arg1 else: right_entity, onclause = arg1, arg2 left_entity = prop = None - if isinstance(onclause, basestring): + if isinstance(onclause, util.string_types): left_entity = self._joinpoint_zero() descriptor = _entity_descriptor(left_entity, onclause) @@ -1715,10 +1769,14 @@ class Query(object): # and Class is that of the current joinpoint elif from_joinpoint and \ isinstance(onclause, interfaces.PropComparator): - left_entity = onclause.parententity + left_entity = onclause._parententity + info = inspect(self._joinpoint_zero()) left_mapper, left_selectable, left_is_aliased = \ - _entity_info(self._joinpoint_zero()) + getattr(info, 'mapper', None), \ + info.selectable, \ + getattr(info, 'is_aliased_class', None) + if left_mapper is left_entity: left_entity = self._joinpoint_zero() descriptor = _entity_descriptor(left_entity, @@ -1734,7 +1792,7 @@ class Query(object): else: right_entity = onclause.property.mapper - left_entity = onclause.parententity + left_entity = onclause._parententity prop = onclause.property if not isinstance(onclause, attributes.QueryableAttribute): @@ -1766,6 +1824,7 @@ class Query(object): right_entity, onclause, outerjoin, create_aliases, prop) + def _join_left_to_right(self, left, right, onclause, outerjoin, create_aliases, prop): """append a JOIN to the query's from clause.""" @@ -1783,35 +1842,63 @@ class Query(object): raise sa_exc.InvalidRequestError( "Can't construct a join from %s to %s, they " "are the same entity" % - (left, right)) + (left, right)) - right, right_is_aliased, onclause = self._prepare_right_side( - right, onclause, - outerjoin, create_aliases, - prop) + l_info = inspect(left) + r_info = inspect(right) + + + overlap = False + if not create_aliases: + right_mapper = getattr(r_info, "mapper", None) + # if the target is a joined inheritance mapping, + # be more liberal about auto-aliasing. + if right_mapper and ( + right_mapper.with_polymorphic or + isinstance(right_mapper.mapped_table, expression.Join) + ): + for from_obj in self._from_obj or [l_info.selectable]: + if sql_util.selectables_overlap(l_info.selectable, from_obj) and \ + sql_util.selectables_overlap(from_obj, r_info.selectable): + overlap = True + break + elif sql_util.selectables_overlap(l_info.selectable, r_info.selectable): + overlap = True + + + if overlap and l_info.selectable is r_info.selectable: + raise sa_exc.InvalidRequestError( + "Can't join table/selectable '%s' to itself" % + l_info.selectable) + + right, onclause = self._prepare_right_side( + r_info, right, onclause, + create_aliases, + prop, overlap) # if joining on a MapperProperty path, # track the path to prevent redundant joins if not create_aliases and prop: self._update_joinpoint({ - '_joinpoint_entity':right, - 'prev':((left, right, prop.key), self._joinpoint) + '_joinpoint_entity': right, + 'prev': ((left, right, prop.key), self._joinpoint) }) else: - self._joinpoint = { - '_joinpoint_entity':right - } + self._joinpoint = {'_joinpoint_entity': right} - self._join_to_left(left, right, - right_is_aliased, - onclause, outerjoin) + self._join_to_left(l_info, left, right, onclause, outerjoin) - def _prepare_right_side(self, right, onclause, outerjoin, - create_aliases, prop): - right_mapper, right_selectable, right_is_aliased = _entity_info(right) + def _prepare_right_side(self, r_info, right, onclause, create_aliases, + prop, overlap): + info = r_info + + right_mapper, right_selectable, right_is_aliased = \ + getattr(info, 'mapper', None), \ + info.selectable, \ + getattr(info, 'is_aliased_class', False) if right_mapper: - self._join_entities += (right, ) + self._join_entities += (info, ) if right_mapper and prop and \ not right_mapper.common_parent(prop.mapper): @@ -1833,23 +1920,27 @@ class Query(object): (right_selectable.description, right_mapper.mapped_table.description)) - if not isinstance(right_selectable, expression.Alias): + if isinstance(right_selectable, expression.SelectBase): + # TODO: this isn't even covered now! right_selectable = right_selectable.alias() + need_adapter = True right = aliased(right_mapper, right_selectable) - need_adapter = True aliased_entity = right_mapper and \ not right_is_aliased and \ ( - right_mapper.with_polymorphic or - isinstance( - right_mapper.mapped_table, - expression.Join) + right_mapper.with_polymorphic and isinstance( + right_mapper._with_polymorphic_selectable, + expression.Alias) + or + overlap # test for overlap: + # orm/inheritance/relationships.py + # SelfReferentialM2MTest ) if not need_adapter and (create_aliases or aliased_entity): - right = aliased(right) + right = aliased(right, flat=True) need_adapter = True # if an alias() of the right side was generated here, @@ -1879,44 +1970,23 @@ class Query(object): ) ) - return right, right_is_aliased, onclause + return right, onclause - def _join_to_left(self, left, right, right_is_aliased, onclause, outerjoin): - left_mapper, left_selectable, left_is_aliased = _entity_info(left) + def _join_to_left(self, l_info, left, right, onclause, outerjoin): + info = l_info + left_mapper = getattr(info, 'mapper', None) + left_selectable = info.selectable - # this is an overly broad assumption here, but there's a - # very wide variety of situations where we rely upon orm.join's - # adaption to glue clauses together, with joined-table inheritance's - # wide array of variables taking up most of the space. - # Setting the flag here is still a guess, so it is a bug - # that we don't have definitive criterion to determine when - # adaption should be enabled (or perhaps that we're even doing the - # whole thing the way we are here). - join_to_left = not right_is_aliased and not left_is_aliased - - if self._from_obj and left_selectable is not None: + if self._from_obj: replace_clause_index, clause = sql_util.find_join_source( self._from_obj, left_selectable) if clause is not None: - # the entire query's FROM clause is an alias of itself (i.e. - # from_self(), similar). if the left clause is that one, - # ensure it adapts to the left side. - if self._from_obj_alias and clause is self._from_obj[0]: - join_to_left = True - - # An exception case where adaption to the left edge is not - # desirable. See above note on join_to_left. - if join_to_left and isinstance(clause, expression.Join) and \ - sql_util.clause_is_present(left_selectable, clause): - join_to_left = False - try: clause = orm_join(clause, right, - onclause, isouter=outerjoin, - join_to_left=join_to_left) - except sa_exc.ArgumentError, ae: + onclause, isouter=outerjoin) + except sa_exc.ArgumentError as ae: raise sa_exc.InvalidRequestError( "Could not find a FROM clause to join from. " "Tried joining to %s, but got: %s" % (right, ae)) @@ -1934,23 +2004,16 @@ class Query(object): break else: clause = left - elif left_selectable is not None: - clause = left_selectable else: - clause = None - - if clause is None: - raise sa_exc.InvalidRequestError( - "Could not find a FROM clause to join from") + clause = left_selectable + assert clause is not None try: - clause = orm_join(clause, right, onclause, - isouter=outerjoin, join_to_left=join_to_left) - except sa_exc.ArgumentError, ae: + clause = orm_join(clause, right, onclause, isouter=outerjoin) + except sa_exc.ArgumentError as ae: raise sa_exc.InvalidRequestError( "Could not find a FROM clause to join from. " "Tried joining to %s, but got: %s" % (right, ae)) - self._from_obj = self._from_obj + (clause,) def _reset_joinpoint(self): @@ -1974,32 +2037,134 @@ class Query(object): def select_from(self, *from_obj): """Set the FROM clause of this :class:`.Query` explicitly. - Sending a mapped class or entity here effectively replaces the + :meth:`.Query.select_from` is often used in conjunction with + :meth:`.Query.join` in order to control which entity is selected + from on the "left" side of the join. + + The entity or selectable object here effectively replaces the "left edge" of any calls to :meth:`~.Query.join`, when no joinpoint is otherwise established - usually, the default "join point" is the leftmost entity in the :class:`~.Query` object's list of entities to be selected. - Mapped entities or plain :class:`~.Table` or other selectables - can be sent here which will form the default FROM clause. + A typical example:: - See the example in :meth:`~.Query.join` for a typical - usage of :meth:`~.Query.select_from`. + q = session.query(Address).select_from(User).\\ + join(User.addresses).\\ + filter(User.name == 'ed') + + Which produces SQL equivalent to:: + + SELECT address.* FROM user + JOIN address ON user.id=address.user_id + WHERE user.name = :name_1 + + :param \*from_obj: collection of one or more entities to apply + to the FROM clause. Entities can be mapped classes, + :class:`.AliasedClass` objects, :class:`.Mapper` objects + as well as core :class:`.FromClause` elements like subqueries. + + .. versionchanged:: 0.9 + This method no longer applies the given FROM object + to be the selectable from which matching entities + select from; the :meth:`.select_entity_from` method + now accomplishes this. See that method for a description + of this behavior. + + .. seealso:: + + :meth:`~.Query.join` + + :meth:`.Query.select_entity_from` """ - obj = [] - for fo in from_obj: - if _is_mapped_class(fo): - mapper, selectable, is_aliased_class = _entity_info(fo) - self._select_from_entity = fo - obj.append(selectable) - elif not isinstance(fo, expression.FromClause): - raise sa_exc.ArgumentError( - "select_from() accepts FromClause objects only.") - else: - obj.append(fo) - self._set_select_from(*obj) + self._set_select_from(from_obj, False) + + @_generative(_no_clauseelement_condition) + def select_entity_from(self, from_obj): + """Set the FROM clause of this :class:`.Query` to a + core selectable, applying it as a replacement FROM clause + for corresponding mapped entities. + + This method is similar to the :meth:`.Query.select_from` + method, in that it sets the FROM clause of the query. However, + where :meth:`.Query.select_from` only affects what is placed + in the FROM, this method also applies the given selectable + to replace the FROM which the selected entities would normally + select from. + + The given ``from_obj`` must be an instance of a :class:`.FromClause`, + e.g. a :func:`.select` or :class:`.Alias` construct. + + An example would be a :class:`.Query` that selects ``User`` entities, + but uses :meth:`.Query.select_entity_from` to have the entities + selected from a :func:`.select` construct instead of the + base ``user`` table:: + + select_stmt = select([User]).where(User.id == 7) + + q = session.query(User).\\ + select_entity_from(select_stmt).\\ + filter(User.name == 'ed') + + The query generated will select ``User`` entities directly + from the given :func:`.select` construct, and will be:: + + SELECT anon_1.id AS anon_1_id, anon_1.name AS anon_1_name + FROM (SELECT "user".id AS id, "user".name AS name + FROM "user" + WHERE "user".id = :id_1) AS anon_1 + WHERE anon_1.name = :name_1 + + Notice above that even the WHERE criterion was "adapted" such that + the ``anon_1`` subquery effectively replaces all references to the + ``user`` table, except for the one that it refers to internally. + + Compare this to :meth:`.Query.select_from`, which as of + version 0.9, does not affect existing entities. The + statement below:: + + q = session.query(User).\\ + select_from(select_stmt).\\ + filter(User.name == 'ed') + + Produces SQL where both the ``user`` table as well as the + ``select_stmt`` construct are present as separate elements + in the FROM clause. No "adaptation" of the ``user`` table + is applied:: + + SELECT "user".id AS user_id, "user".name AS user_name + FROM "user", (SELECT "user".id AS id, "user".name AS name + FROM "user" + WHERE "user".id = :id_1) AS anon_1 + WHERE "user".name = :name_1 + + :meth:`.Query.select_entity_from` maintains an older + behavior of :meth:`.Query.select_from`. In modern usage, + similar results can also be achieved using :func:`.aliased`:: + + select_stmt = select([User]).where(User.id == 7) + user_from_select = aliased(User, select_stmt.alias()) + + q = session.query(user_from_select) + + :param from_obj: a :class:`.FromClause` object that will replace + the FROM clause of this :class:`.Query`. + + .. seealso:: + + :meth:`.Query.select_from` + + .. versionadded:: 0.8 + :meth:`.Query.select_entity_from` was added to specify + the specific behavior of entity replacement, however + the :meth:`.Query.select_from` maintains this behavior + as well until 0.9. + + """ + + self._set_select_from([from_obj], True) def __getitem__(self, item): if isinstance(item, slice): @@ -2025,7 +2190,7 @@ class Query(object): if item == -1: return list(self)[-1] else: - return list(self[item:item+1])[0] + return list(self[item:item + 1])[0] @_generative(_no_statement_condition) def slice(self, start, stop): @@ -2085,7 +2250,7 @@ class Query(object): ``Query``. :param \*prefixes: optional prefixes, typically strings, - not using any commas. In particular is useful for MySQL keywords. + not using any commas. In particular is useful for MySQL keywords. e.g.:: @@ -2126,12 +2291,12 @@ class Query(object): appropriate to the entity class represented by this ``Query``. """ - if isinstance(statement, basestring): + if isinstance(statement, util.string_types): statement = sql.text(statement) if not isinstance(statement, - (expression._TextClause, - expression._SelectBase)): + (expression.TextClause, + expression.SelectBase)): raise sa_exc.ArgumentError( "from_statement accepts text(), select(), " "and union() objects only.") @@ -2235,12 +2400,12 @@ class Query(object): def _execute_and_instances(self, querycontext): conn = self._connection_from_session( - mapper = self._mapper_zero_or_none(), - clause = querycontext.statement, + mapper=self._mapper_zero_or_none(), + clause=querycontext.statement, close_with_result=True) result = conn.execute(querycontext.statement, self._params) - return self.instances(result, querycontext) + return loading.instances(self, result, querycontext) @property def column_descriptions(self): @@ -2280,10 +2445,10 @@ class Query(object): """ return [ { - 'name':ent._label_name, - 'type':ent.type, - 'aliased':getattr(ent, 'is_aliased_class', False), - 'expr':ent.expr + 'name': ent._label_name, + 'type': ent.type, + 'aliased': getattr(ent, 'is_aliased_class', False), + 'expr': ent.expr } for ent in self._entities ] @@ -2298,91 +2463,26 @@ class Query(object): for u in session.query(User).instances(result): print u """ - session = self.session - context = __context if context is None: context = QueryContext(self) - context.runid = _new_runid() - - filter_fns = [ent.filter_fn - for ent in self._entities] - filtered = id in filter_fns - - single_entity = filtered and len(self._entities) == 1 - - if filtered: - if single_entity: - filter_fn = id - else: - def filter_fn(row): - return tuple(fn(x) for x, fn in zip(row, filter_fns)) - - custom_rows = single_entity and \ - self._entities[0].mapper.dispatch.append_result - - (process, labels) = \ - zip(*[ - query_entity.row_processor(self, context, custom_rows) - for query_entity in self._entities - ]) - - - while True: - context.progress = {} - context.partials = {} - - if self._yield_per: - fetch = cursor.fetchmany(self._yield_per) - if not fetch: - break - else: - fetch = cursor.fetchall() - - if custom_rows: - rows = [] - for row in fetch: - process[0](row, rows) - elif single_entity: - rows = [process[0](row, None) for row in fetch] - else: - rows = [util.NamedTuple([proc(row, None) for proc in process], - labels) for row in fetch] - - if filtered: - rows = util.unique_list(rows, filter_fn) - - if context.refresh_state and self._only_load_props \ - and context.refresh_state in context.progress: - context.refresh_state.commit( - context.refresh_state.dict, self._only_load_props) - context.progress.pop(context.refresh_state) - - session._finalize_loaded(context.progress) - - for ii, (dict_, attrs) in context.partials.iteritems(): - ii.commit(dict_, attrs) - - for row in rows: - yield row - - if not self._yield_per: - break + return loading.instances(self, cursor, context) def merge_result(self, iterator, load=True): """Merge a result into this :class:`.Query` object's Session. - Given an iterator returned by a :class:`.Query` of the same structure as this - one, return an identical iterator of results, with all mapped - instances merged into the session using :meth:`.Session.merge`. This is an - optimized method which will merge all mapped instances, preserving the - structure of the result rows and unmapped columns with less method - overhead than that of calling :meth:`.Session.merge` explicitly for each - value. + Given an iterator returned by a :class:`.Query` of the same structure + as this one, return an identical iterator of results, with all mapped + instances merged into the session using :meth:`.Session.merge`. This + is an optimized method which will merge all mapped instances, + preserving the structure of the result rows and unmapped columns with + less method overhead than that of calling :meth:`.Session.merge` + explicitly for each value. The structure of the results is determined based on the column list of - this :class:`.Query` - if these do not correspond, unchecked errors will occur. + this :class:`.Query` - if these do not correspond, unchecked errors + will occur. The 'load' argument is the same as that of :meth:`.Session.merge`. @@ -2393,137 +2493,17 @@ class Query(object): """ - session = self.session - if load: - # flush current contents if we expect to load data - session._autoflush() - - autoflush = session.autoflush - try: - session.autoflush = False - single_entity = len(self._entities) == 1 - if single_entity: - if isinstance(self._entities[0], _MapperEntity): - result = [session._merge( - attributes.instance_state(instance), - attributes.instance_dict(instance), - load=load, _recursive={}) - for instance in iterator] - else: - result = list(iterator) - else: - mapped_entities = [i for i, e in enumerate(self._entities) - if isinstance(e, _MapperEntity)] - result = [] - keys = [ent._label_name for ent in self._entities] - for row in iterator: - newrow = list(row) - for i in mapped_entities: - if newrow[i] is not None: - newrow[i] = session._merge( - attributes.instance_state(newrow[i]), - attributes.instance_dict(newrow[i]), - load=load, _recursive={}) - result.append(util.NamedTuple(newrow, keys)) - - return iter(result) - finally: - session.autoflush = autoflush - - @classmethod - def _get_from_identity(cls, session, key, passive): - """Look up the given key in the given session's identity map, - check the object for expired state if found. - - """ - instance = session.identity_map.get(key) - if instance is not None: - - state = attributes.instance_state(instance) - - # expired - ensure it still exists - if state.expired: - if passive is attributes.PASSIVE_NO_FETCH: - # TODO: no coverage here - return attributes.PASSIVE_NO_RESULT - elif passive is attributes.PASSIVE_NO_FETCH_RELATED: - # this mode is used within a flush and the instance's - # expired state will be checked soon enough, if necessary - return instance - try: - state(passive) - except orm_exc.ObjectDeletedError: - session._remove_newly_deleted(state) - return None - return instance - else: - return None - - def _load_on_ident(self, key, refresh_state=None, lockmode=None, - only_load_props=None): - """Load the given identity key from the database.""" - - lockmode = lockmode or self._lockmode - - if key is not None: - ident = key[1] - else: - ident = None - - if refresh_state is None: - q = self._clone() - q._get_condition() - else: - q = self._clone() - - if ident is not None: - mapper = self._mapper_zero() - - (_get_clause, _get_params) = mapper._get_clause - - # None present in ident - turn those comparisons - # into "IS NULL" - if None in ident: - nones = set([ - _get_params[col].key for col, value in - zip(mapper.primary_key, ident) if value is None - ]) - _get_clause = sql_util.adapt_criterion_to_null( - _get_clause, nones) - - _get_clause = q._adapt_clause(_get_clause, True, False) - q._criterion = _get_clause - - params = dict([ - (_get_params[primary_key].key, id_val) - for id_val, primary_key in zip(ident, mapper.primary_key) - ]) - - q._params = params - - if lockmode is not None: - q._lockmode = lockmode - q._get_options( - populate_existing=bool(refresh_state), - version_check=(lockmode is not None), - only_load_props=only_load_props, - refresh_state=refresh_state) - q._order_by = None - - try: - return q.one() - except orm_exc.NoResultFound: - return None + return loading.merge_result(self, iterator, load) @property def _select_args(self): return { - 'limit':self._limit, - 'offset':self._offset, - 'distinct':self._distinct, - 'prefixes':self._prefixes, - 'group_by':self._group_by or None, - 'having':self._having + 'limit': self._limit, + 'offset': self._offset, + 'distinct': self._distinct, + 'prefixes': self._prefixes, + 'group_by': self._group_by or None, + 'having': self._having } @property @@ -2533,6 +2513,26 @@ class Query(object): kwargs.get('offset') is not None or kwargs.get('distinct', False)) + def exists(self): + """A convenience method that turns a query into an EXISTS subquery + of the form EXISTS (SELECT 1 FROM ... WHERE ...). + + e.g.:: + + q = session.query(User).filter(User.name == 'fred') + session.query(q.exists()) + + Producing SQL similar to:: + + SELECT EXISTS ( + SELECT 1 FROM users WHERE users.name = :name_1 + ) AS anon_1 + + .. versionadded:: 0.8.1 + + """ + return sql.exists(self.with_labels().statement.with_only_columns(['1'])) + def count(self): """Return a count of rows this Query would return. @@ -2549,7 +2549,8 @@ class Query(object): to count, to skip the usage of a subquery or otherwise control of the FROM clause, or to use other aggregate functions, - use :attr:`~sqlalchemy.sql.expression.func` expressions in conjunction + use :attr:`~sqlalchemy.sql.expression.func` + expressions in conjunction with :meth:`~.Session.query`, i.e.:: from sqlalchemy import func @@ -2593,109 +2594,53 @@ class Query(object): removed from the session. Matched objects are removed from the session. - ``'evaluate'`` - Evaluate the query's criteria in Python straight on - the objects in the session. If evaluation of the criteria isn't + ``'evaluate'`` - Evaluate the query's criteria in Python straight + on the objects in the session. If evaluation of the criteria isn't implemented, an error is raised. In that case you probably want to use the 'fetch' strategy as a fallback. The expression evaluator currently doesn't account for differing string collations between the database and Python. - Returns the number of rows deleted, excluding any cascades. + :return: the count of rows matched as returned by the database's + "row count" feature. - The method does *not* offer in-Python cascading of relationships - it - is assumed that ON DELETE CASCADE is configured for any foreign key - references which require it. The Session needs to be expired (occurs - automatically after commit(), or call expire_all()) in order for the - state of dependent objects subject to delete or delete-orphan cascade - to be correctly represented. + This method has several key caveats: - Note that the :meth:`.MapperEvents.before_delete` and - :meth:`.MapperEvents.after_delete` - events are **not** invoked from this method. It instead - invokes :meth:`.SessionEvents.after_bulk_delete`. + * The method does **not** offer in-Python cascading of relationships - it + is assumed that ON DELETE CASCADE/SET NULL/etc. is configured for any foreign key + references which require it, otherwise the database may emit an + integrity violation if foreign key references are being enforced. + + After the DELETE, dependent objects in the :class:`.Session` which + were impacted by an ON DELETE may not contain the current + state, or may have been deleted. This issue is resolved once the + :class:`.Session` is expired, + which normally occurs upon :meth:`.Session.commit` or can be forced + by using :meth:`.Session.expire_all`. Accessing an expired object + whose row has been deleted will invoke a SELECT to locate the + row; when the row is not found, an :class:`~sqlalchemy.orm.exc.ObjectDeletedError` + is raised. + + * The :meth:`.MapperEvents.before_delete` and + :meth:`.MapperEvents.after_delete` + events are **not** invoked from this method. Instead, the + :meth:`.SessionEvents.after_bulk_delete` method is provided to act + upon a mass DELETE of entity rows. + + .. seealso:: + + :meth:`.Query.update` + + :ref:`inserts_and_updates` - Core SQL tutorial """ - #TODO: lots of duplication and ifs - probably needs to be - # refactored to strategies #TODO: cascades need handling. - if synchronize_session not in [False, 'evaluate', 'fetch']: - raise sa_exc.ArgumentError( - "Valid strategies for session " - "synchronization are False, 'evaluate' and " - "'fetch'") - self._no_select_modifiers("delete") - - self = self.enable_eagerloads(False) - - context = self._compile_context() - if len(context.statement.froms) != 1 or \ - not isinstance(context.statement.froms[0], schema.Table): - raise sa_exc.ArgumentError("Only deletion via a single table " - "query is currently supported") - primary_table = context.statement.froms[0] - - session = self.session - - if self._autoflush: - session._autoflush() - - if synchronize_session == 'evaluate': - try: - evaluator_compiler = evaluator.EvaluatorCompiler() - if self.whereclause is not None: - eval_condition = evaluator_compiler.process( - self.whereclause) - else: - def eval_condition(obj): - return True - - except evaluator.UnevaluatableError: - raise sa_exc.InvalidRequestError( - "Could not evaluate current criteria in Python. " - "Specify 'fetch' or False for the synchronize_session " - "parameter.") - - target_cls = self._mapper_zero().class_ - - #TODO: detect when the where clause is a trivial primary key match - objs_to_expunge = [ - obj for (cls, pk),obj in - session.identity_map.iteritems() - if issubclass(cls, target_cls) and - eval_condition(obj)] - - elif synchronize_session == 'fetch': - #TODO: use RETURNING when available - select_stmt = context.statement.with_only_columns( - primary_table.primary_key) - matched_rows = session.execute( - select_stmt, - params=self._params).fetchall() - - delete_stmt = sql.delete(primary_table, context.whereclause) - - result = session.execute(delete_stmt, params=self._params) - - if synchronize_session == 'evaluate': - for obj in objs_to_expunge: - session._remove_newly_deleted(attributes.instance_state(obj)) - elif synchronize_session == 'fetch': - target_mapper = self._mapper_zero() - for primary_key in matched_rows: - identity_key = target_mapper.identity_key_from_primary_key( - list(primary_key)) - if identity_key in session.identity_map: - session._remove_newly_deleted( - attributes.instance_state( - session.identity_map[identity_key] - ) - ) - - session.dispatch.after_bulk_delete(session, self, context, result) - - return result.rowcount + delete_op = persistence.BulkDelete.factory( + self, synchronize_session) + delete_op.exec_() + return delete_op.rowcount def update(self, values, synchronize_session='evaluate'): """Perform a bulk update query. @@ -2719,27 +2664,57 @@ class Query(object): objects that are matched by the update query. The updated attributes are expired on matched objects. - ``'evaluate'`` - Evaluate the Query's criteria in Python straight on - the objects in the session. If evaluation of the criteria isn't + ``'evaluate'`` - Evaluate the Query's criteria in Python straight + on the objects in the session. If evaluation of the criteria isn't implemented, an exception is raised. The expression evaluator currently doesn't account for differing string collations between the database and Python. - Returns the number of rows matched by the update. + :return: the count of rows matched as returned by the database's + "row count" feature. - The method does *not* offer in-Python cascading of relationships - it - is assumed that ON UPDATE CASCADE is configured for any foreign key - references which require it. + This method has several key caveats: - The Session needs to be expired (occurs automatically after commit(), - or call expire_all()) in order for the state of dependent objects - subject foreign key cascade to be correctly represented. + * The method does **not** offer in-Python cascading of relationships - it + is assumed that ON UPDATE CASCADE is configured for any foreign key + references which require it, otherwise the database may emit an + integrity violation if foreign key references are being enforced. - Note that the :meth:`.MapperEvents.before_update` and - :meth:`.MapperEvents.after_update` - events are **not** invoked from this method. It instead - invokes :meth:`.SessionEvents.after_bulk_update`. + After the UPDATE, dependent objects in the :class:`.Session` which + were impacted by an ON UPDATE CASCADE may not contain the current + state; this issue is resolved once the :class:`.Session` is expired, + which normally occurs upon :meth:`.Session.commit` or can be forced + by using :meth:`.Session.expire_all`. + + * As of 0.8, this method will support multiple table updates, as detailed + in :ref:`multi_table_updates`, and this behavior does extend to support + updates of joined-inheritance and other multiple table mappings. However, + the **join condition of an inheritance mapper is currently not + automatically rendered**. + Care must be taken in any multiple-table update to explicitly include + the joining condition between those tables, even in mappings where + this is normally automatic. + E.g. if a class ``Engineer`` subclasses ``Employee``, an UPDATE of the + ``Engineer`` local table using criteria against the ``Employee`` + local table might look like:: + + session.query(Engineer).\\ + filter(Engineer.id == Employee.id).\\ + filter(Employee.name == 'dilbert').\\ + update({"engineer_type": "programmer"}) + + * The :meth:`.MapperEvents.before_update` and + :meth:`.MapperEvents.after_update` + events are **not** invoked from this method. Instead, the + :meth:`.SessionEvents.after_bulk_update` method is provided to act + upon a mass UPDATE of entity rows. + + .. seealso:: + + :meth:`.Query.delete` + + :ref:`inserts_and_updates` - Core SQL tutorial """ @@ -2749,108 +2724,11 @@ class Query(object): # fk assignments #TODO: cascades need handling. - if synchronize_session == 'expire': - util.warn_deprecated("The 'expire' value as applied to " - "the synchronize_session argument of " - "query.update() is now called 'fetch'") - synchronize_session = 'fetch' + update_op = persistence.BulkUpdate.factory( + self, synchronize_session, values) + update_op.exec_() + return update_op.rowcount - if synchronize_session not in [False, 'evaluate', 'fetch']: - raise sa_exc.ArgumentError( - "Valid strategies for session synchronization " - "are False, 'evaluate' and 'fetch'") - self._no_select_modifiers("update") - - self = self.enable_eagerloads(False) - - context = self._compile_context() - if len(context.statement.froms) != 1 or \ - not isinstance(context.statement.froms[0], schema.Table): - raise sa_exc.ArgumentError( - "Only update via a single table query is " - "currently supported") - primary_table = context.statement.froms[0] - - session = self.session - - if self._autoflush: - session._autoflush() - - if synchronize_session == 'evaluate': - try: - evaluator_compiler = evaluator.EvaluatorCompiler() - if self.whereclause is not None: - eval_condition = evaluator_compiler.process( - self.whereclause) - else: - def eval_condition(obj): - return True - - value_evaluators = {} - for key,value in values.iteritems(): - key = _attr_as_key(key) - value_evaluators[key] = evaluator_compiler.process( - expression._literal_as_binds(value)) - except evaluator.UnevaluatableError: - raise sa_exc.InvalidRequestError( - "Could not evaluate current criteria in Python. " - "Specify 'fetch' or False for the " - "synchronize_session parameter.") - target_cls = self._mapper_zero().class_ - matched_objects = [] - for (cls, pk),obj in session.identity_map.iteritems(): - evaluated_keys = value_evaluators.keys() - - if issubclass(cls, target_cls) and eval_condition(obj): - matched_objects.append(obj) - - elif synchronize_session == 'fetch': - select_stmt = context.statement.with_only_columns( - primary_table.primary_key) - matched_rows = session.execute( - select_stmt, - params=self._params).fetchall() - - update_stmt = sql.update(primary_table, context.whereclause, values) - - result = session.execute(update_stmt, params=self._params) - - if synchronize_session == 'evaluate': - target_cls = self._mapper_zero().class_ - - for obj in matched_objects: - state, dict_ = attributes.instance_state(obj),\ - attributes.instance_dict(obj) - - # only evaluate unmodified attributes - to_evaluate = state.unmodified.intersection( - evaluated_keys) - for key in to_evaluate: - dict_[key] = value_evaluators[key](obj) - - state.commit(dict_, list(to_evaluate)) - - # expire attributes with pending changes - # (there was no autoflush, so they are overwritten) - state.expire_attributes(dict_, - set(evaluated_keys). - difference(to_evaluate)) - - elif synchronize_session == 'fetch': - target_mapper = self._mapper_zero() - - for primary_key in matched_rows: - identity_key = target_mapper.identity_key_from_primary_key( - list(primary_key)) - if identity_key in session.identity_map: - session.expire( - session.identity_map[identity_key], - [_attr_as_key(k) for k in values] - ) - - session.dispatch.after_bulk_update(session, self, context, result) - - return result.rowcount def _compile_context(self, labels=True): context = QueryContext(self) @@ -2858,18 +2736,9 @@ class Query(object): if context.statement is not None: return context - if self._lockmode: - try: - for_update = {'read': 'read', - 'read_nowait': 'read_nowait', - 'update': True, - 'update_nowait': 'nowait', - None: False}[self._lockmode] - except KeyError: - raise sa_exc.ArgumentError( - "Unknown lockmode %r" % self._lockmode) - else: - for_update = False + context.labels = labels + + context._for_update_arg = self._for_update_arg for entity in self._entities: entity.setup_context(self, context) @@ -2878,16 +2747,19 @@ class Query(object): strategy = rec[0] strategy(*rec[1:]) - eager_joins = context.eager_joins.values() - if context.from_clause: # "load from explicit FROMs" mode, # i.e. when select_from() or join() is used - froms = list(context.from_clause) + context.froms = list(context.from_clause) + # this would fix... + #import pdb + #pdb.set_trace() + #context.froms += tuple(context.from_clause) + else: # "load from discrete FROMs" mode, # i.e. when each _MappedEntity has its own FROM - froms = context.froms + context.froms = context.froms if self._enable_single_crit: self._adjust_for_single_inheritance(context) @@ -2904,134 +2776,160 @@ class Query(object): "SELECT from.") if context.multi_row_eager_loaders and self._should_nest_selectable: - # for eager joins present and LIMIT/OFFSET/DISTINCT, - # wrap the query inside a select, - # then append eager joins onto that + context.statement = self._compound_eager_statement(context) + else: + context.statement = self._simple_statement(context) + return context - if context.order_by: - order_by_col_expr = list( - chain(*[ - sql_util.unwrap_order_by(o) - for o in context.order_by - ]) - ) - else: - context.order_by = None - order_by_col_expr = [] + def _compound_eager_statement(self, context): + # for eager joins present and LIMIT/OFFSET/DISTINCT, + # wrap the query inside a select, + # then append eager joins onto that - inner = sql.select( - context.primary_columns + order_by_col_expr, + if context.order_by: + order_by_col_expr = list( + chain(*[ + sql_util.unwrap_order_by(o) + for o in context.order_by + ]) + ) + else: + context.order_by = None + order_by_col_expr = [] + + inner = sql.select( + context.primary_columns + order_by_col_expr, + context.whereclause, + from_obj=context.froms, + use_labels=context.labels, + # TODO: this order_by is only needed if + # LIMIT/OFFSET is present in self._select_args, + # else the application on the outside is enough + order_by=context.order_by, + **self._select_args + ) + + for hint in self._with_hints: + inner = inner.with_hint(*hint) + + if self._correlate: + inner = inner.correlate(*self._correlate) + + inner = inner.alias() + + equivs = self.__all_equivs() + + context.adapter = sql_util.ColumnAdapter(inner, equivs) + + statement = sql.select( + [inner] + context.secondary_columns, + use_labels=context.labels) + + statement._for_update_arg = context._for_update_arg + + from_clause = inner + for eager_join in context.eager_joins.values(): + # EagerLoader places a 'stop_on' attribute on the join, + # giving us a marker as to where the "splice point" of + # the join should be + from_clause = sql_util.splice_joins( + from_clause, + eager_join, eager_join.stop_on) + + statement.append_from(from_clause) + + if context.order_by: + statement.append_order_by( + *context.adapter.copy_and_process( + context.order_by + ) + ) + + statement.append_order_by(*context.eager_order_by) + return statement + + def _simple_statement(self, context): + if not context.order_by: + context.order_by = None + + if self._distinct and context.order_by: + order_by_col_expr = list( + chain(*[ + sql_util.unwrap_order_by(o) + for o in context.order_by + ]) + ) + context.primary_columns += order_by_col_expr + + context.froms += tuple(context.eager_joins.values()) + + statement = sql.select( + context.primary_columns + + context.secondary_columns, context.whereclause, - from_obj=froms, - use_labels=labels, - correlate=False, - # TODO: this order_by is only needed if - # LIMIT/OFFSET is present in self._select_args, - # else the application on the outside is enough + from_obj=context.froms, + use_labels=context.labels, order_by=context.order_by, **self._select_args ) - for hint in self._with_hints: - inner = inner.with_hint(*hint) + statement._for_update_arg = context._for_update_arg - if self._correlate: - inner = inner.correlate(*self._correlate) + for hint in self._with_hints: + statement = statement.with_hint(*hint) - inner = inner.alias() - - equivs = self.__all_equivs() - - context.adapter = sql_util.ColumnAdapter(inner, equivs) - - statement = sql.select( - [inner] + context.secondary_columns, - for_update=for_update, - use_labels=labels) - - from_clause = inner - for eager_join in eager_joins: - # EagerLoader places a 'stop_on' attribute on the join, - # giving us a marker as to where the "splice point" of - # the join should be - from_clause = sql_util.splice_joins( - from_clause, - eager_join, eager_join.stop_on) - - statement.append_from(from_clause) - - if context.order_by: - statement.append_order_by( - *context.adapter.copy_and_process( - context.order_by - ) - ) + if self._correlate: + statement = statement.correlate(*self._correlate) + if context.eager_order_by: statement.append_order_by(*context.eager_order_by) - else: - if not context.order_by: - context.order_by = None - - if self._distinct and context.order_by: - order_by_col_expr = list( - chain(*[ - sql_util.unwrap_order_by(o) - for o in context.order_by - ]) - ) - context.primary_columns += order_by_col_expr - - froms += tuple(context.eager_joins.values()) - - statement = sql.select( - context.primary_columns + - context.secondary_columns, - context.whereclause, - from_obj=froms, - use_labels=labels, - for_update=for_update, - correlate=False, - order_by=context.order_by, - **self._select_args - ) - - for hint in self._with_hints: - statement = statement.with_hint(*hint) - - if self._correlate: - statement = statement.correlate(*self._correlate) - - if context.eager_order_by: - statement.append_order_by(*context.eager_order_by) - - context.statement = statement - - return context + return statement def _adjust_for_single_inheritance(self, context): """Apply single-table-inheritance filtering. - For all distinct single-table-inheritance mappers represented in the - columns clause of this query, add criterion to the WHERE clause of the - given QueryContext such that only the appropriate subtypes are - selected from the total results. + For all distinct single-table-inheritance mappers represented in + the columns clause of this query, add criterion to the WHERE + clause of the given QueryContext such that only the appropriate + subtypes are selected from the total results. """ - for entity, (mapper, adapter, s, i, w) in \ - self._mapper_adapter_map.iteritems(): - if entity in self._join_entities: + for (ext_info, adapter) in self._mapper_adapter_map.values(): + if ext_info in self._join_entities: continue - single_crit = mapper._single_table_criterion + single_crit = ext_info.mapper._single_table_criterion if single_crit is not None: if adapter: single_crit = adapter.traverse(single_crit) single_crit = self._adapt_clause(single_crit, False, False) context.whereclause = sql.and_( - context.whereclause, single_crit) + sql.True_._ifnone(context.whereclause), + single_crit) def __str__(self): return str(self._compile_context().statement) +from ..sql.selectable import ForUpdateArg + +class LockmodeArg(ForUpdateArg): + @classmethod + def parse_legacy_query(self, mode): + if mode in (None, False): + return None + + if mode == "read": + read = True + nowait = False + elif mode == "update": + read = nowait = False + elif mode == "update_nowait": + nowait = True + read = False + else: + raise sa_exc.ArgumentError( + "Unknown with_lockmode argument: %r" % mode) + + return LockmodeArg(read=read, nowait=nowait) class _QueryEntity(object): """represent an entity column returned within a Query result.""" @@ -3039,9 +2937,11 @@ class _QueryEntity(object): def __new__(cls, *args, **kwargs): if cls is _QueryEntity: entity = args[1] - if not isinstance(entity, basestring) and \ + if not isinstance(entity, util.string_types) and \ _is_mapped_class(entity): cls = _MapperEntity + elif isinstance(entity, Bundle): + cls = _BundleEntity else: cls = _ColumnEntity return object.__new__(cls) @@ -3051,39 +2951,52 @@ class _QueryEntity(object): q.__dict__ = self.__dict__.copy() return q + class _MapperEntity(_QueryEntity): """mapper/class/AliasedClass entity""" def __init__(self, query, entity): - self.primary_entity = not query._entities + if not query._primary_entity: + query._primary_entity = self query._entities.append(self) self.entities = [entity] - self.entity_zero = self.expr = entity + self.expr = entity - def setup_entity(self, entity, mapper, adapter, - from_obj, is_aliased_class, with_polymorphic): - self.mapper = mapper - self.adapter = adapter - self.selectable = from_obj - self._with_polymorphic = with_polymorphic - self._polymorphic_discriminator = None - self.is_aliased_class = is_aliased_class - if is_aliased_class: - self.path_entity = self.entity_zero = entity - self._path = (entity,) - self._label_name = self.entity_zero._sa_label_name - self._reduced_path = (self.path_entity, ) + supports_single_entity = True + + def setup_entity(self, ext_info, aliased_adapter): + self.mapper = ext_info.mapper + self.aliased_adapter = aliased_adapter + self.selectable = ext_info.selectable + self.is_aliased_class = ext_info.is_aliased_class + self._with_polymorphic = ext_info.with_polymorphic_mappers + self._polymorphic_discriminator = \ + ext_info.polymorphic_on + self.entity_zero = ext_info + if ext_info.is_aliased_class: + self._label_name = self.entity_zero.name else: - self.path_entity = mapper - self._path = (mapper,) - self._reduced_path = (mapper.base_mapper, ) - self.entity_zero = mapper self._label_name = self.mapper.class_.__name__ - + self.path = self.entity_zero._path_registry + self.custom_rows = bool(self.mapper.dispatch.append_result) def set_with_polymorphic(self, query, cls_or_mappers, - selectable, discriminator): + selectable, polymorphic_on): + """Receive an update from a call to query.with_polymorphic(). + + Note the newer style of using a free standing with_polymporphic() + construct doesn't make use of this method. + + + """ + if self.is_aliased_class: + # TODO: invalidrequest ? + raise NotImplementedError( + "Can't use with_polymorphic() against " + "an Aliased object" + ) + if cls_or_mappers is None: query._reset_polymorphic_adapter(self.mapper) return @@ -3091,15 +3004,12 @@ class _MapperEntity(_QueryEntity): mappers, from_obj = self.mapper._with_polymorphic_args( cls_or_mappers, selectable) self._with_polymorphic = mappers - self._polymorphic_discriminator = discriminator + self._polymorphic_discriminator = polymorphic_on - # TODO: do the wrapped thing here too so that - # with_polymorphic() can be applied to aliases - if not self.is_aliased_class: - self.selectable = from_obj - query._mapper_loads_polymorphically_with(self.mapper, - sql_util.ColumnAdapter(from_obj, - self.mapper._equivalent_columns)) + self.selectable = from_obj + query._mapper_loads_polymorphically_with(self.mapper, + sql_util.ColumnAdapter(from_obj, + self.mapper._equivalent_columns)) filter_fn = id @@ -3112,10 +3022,18 @@ class _MapperEntity(_QueryEntity): return self.entity_zero def corresponds_to(self, entity): - if _is_aliased_class(entity) or self.is_aliased_class: - return entity is self.path_entity - else: - return entity.common_parent(self.path_entity) + if entity.is_aliased_class: + if self.is_aliased_class: + if entity._base_alias is self.entity_zero._base_alias: + return True + return False + elif self.is_aliased_class: + if self.entity_zero._use_mapper_path: + return entity in self._with_polymorphic + else: + return entity is self.entity_zero + + return entity.common_parent(self.entity_zero) def adapt_to_selectable(self, query, sel): query._entities.append(self) @@ -3123,11 +3041,12 @@ class _MapperEntity(_QueryEntity): def _get_entity_clauses(self, query, context): adapter = None - if not self.is_aliased_class and query._polymorphic_adapters: - adapter = query._polymorphic_adapters.get(self.mapper, None) - if not adapter and self.adapter: - adapter = self.adapter + if not self.is_aliased_class: + if query._polymorphic_adapters: + adapter = query._polymorphic_adapters.get(self.mapper, None) + else: + adapter = self.aliased_adapter if adapter: if query._from_obj_alias: @@ -3152,34 +3071,34 @@ class _MapperEntity(_QueryEntity): # require row aliasing unconditionally. if not adapter and self.mapper._requires_row_aliasing: adapter = sql_util.ColumnAdapter( - self.selectable, - self.mapper._equivalent_columns) + self.selectable, + self.mapper._equivalent_columns) - if self.primary_entity: - _instance = self.mapper._instance_processor( - context, - self._path, - self._reduced_path, - adapter, - only_load_props=query._only_load_props, - refresh_state=context.refresh_state, - polymorphic_discriminator= - self._polymorphic_discriminator + if query._primary_entity is self: + _instance = loading.instance_processor( + self.mapper, + context, + self.path, + adapter, + only_load_props=query._only_load_props, + refresh_state=context.refresh_state, + polymorphic_discriminator=self._polymorphic_discriminator ) else: - _instance = self.mapper._instance_processor( - context, - self._path, - self._reduced_path, - adapter, - polymorphic_discriminator= - self._polymorphic_discriminator) + _instance = loading.instance_processor( + self.mapper, + context, + self.path, + adapter, + polymorphic_discriminator=self._polymorphic_discriminator + ) return _instance, self._label_name def setup_context(self, query, context): adapter = self._get_entity_clauses(query, context) + #if self._adapted_selectable is None: context.froms += (self.selectable,) if context.order_by is False and self.mapper.order_by: @@ -3206,14 +3125,16 @@ class _MapperEntity(_QueryEntity): value.setup( context, self, - self._path, - self._reduced_path, + self.path, adapter, only_load_props=query._only_load_props, column_collection=context.primary_columns ) - if self._polymorphic_discriminator is not None: + if self._polymorphic_discriminator is not None and \ + self._polymorphic_discriminator \ + is not self.mapper.polymorphic_on: + if adapter: pd = adapter.columns[self._polymorphic_discriminator] else: @@ -3223,6 +3144,188 @@ class _MapperEntity(_QueryEntity): def __str__(self): return str(self.mapper) +@inspection._self_inspects +class Bundle(object): + """A grouping of SQL expressions that are returned by a :class:`.Query` + under one namespace. + + The :class:`.Bundle` essentially allows nesting of the tuple-based + results returned by a column-oriented :class:`.Query` object. It also + is extensible via simple subclassing, where the primary capability + to override is that of how the set of expressions should be returned, + allowing post-processing as well as custom return types, without + involving ORM identity-mapped classes. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :ref:`bundles` + + """ + + single_entity = False + """If True, queries for a single Bundle will be returned as a single + entity, rather than an element within a keyed tuple.""" + + def __init__(self, name, *exprs, **kw): + """Construct a new :class:`.Bundle`. + + e.g.:: + + bn = Bundle("mybundle", MyClass.x, MyClass.y) + + for row in session.query(bn).filter(bn.c.x == 5).filter(bn.c.y == 4): + print(row.mybundle.x, row.mybundle.y) + + :param name: name of the bundle. + :param \*exprs: columns or SQL expressions comprising the bundle. + :param single_entity=False: if True, rows for this :class:`.Bundle` + can be returned as a "single entity" outside of any enclosing tuple + in the same manner as a mapped entity. + + """ + self.name = self._label = name + self.exprs = exprs + self.c = self.columns = ColumnCollection() + self.columns.update((getattr(col, "key", col._label), col) + for col in exprs) + self.single_entity = kw.pop('single_entity', self.single_entity) + + columns = None + """A namespace of SQL expressions referred to by this :class:`.Bundle`. + + e.g.:: + + bn = Bundle("mybundle", MyClass.x, MyClass.y) + + q = sess.query(bn).filter(bn.c.x == 5) + + Nesting of bundles is also supported:: + + b1 = Bundle("b1", + Bundle('b2', MyClass.a, MyClass.b), + Bundle('b3', MyClass.x, MyClass.y) + ) + + q = sess.query(b1).filter(b1.c.b2.c.a == 5).filter(b1.c.b3.c.y == 9) + + .. seealso:: + + :attr:`.Bundle.c` + + """ + + c = None + """An alias for :attr:`.Bundle.columns`.""" + + def _clone(self): + cloned = self.__class__.__new__(self.__class__) + cloned.__dict__.update(self.__dict__) + return cloned + + def __clause_element__(self): + return expression.ClauseList(group=False, *self.c) + + @property + def clauses(self): + return self.__clause_element__().clauses + + def label(self, name): + """Provide a copy of this :class:`.Bundle` passing a new label.""" + + cloned = self._clone() + cloned.name = name + return cloned + + def create_row_processor(self, query, procs, labels): + """Produce the "row processing" function for this :class:`.Bundle`. + + May be overridden by subclasses. + + .. seealso:: + + :ref:`bundles` - includes an example of subclassing. + + """ + def proc(row, result): + return util.KeyedTuple([proc(row, None) for proc in procs], labels) + return proc + + +class _BundleEntity(_QueryEntity): + def __init__(self, query, bundle, setup_entities=True): + query._entities.append(self) + self.bundle = self.expr = bundle + self.type = type(bundle) + self._label_name = bundle.name + self._entities = [] + + if setup_entities: + for expr in bundle.exprs: + if isinstance(expr, Bundle): + _BundleEntity(self, expr) + else: + _ColumnEntity(self, expr, namespace=self) + + self.entities = () + + self.filter_fn = lambda item: item + + self.supports_single_entity = self.bundle.single_entity + + custom_rows = False + + @property + def entity_zero(self): + for ent in self._entities: + ezero = ent.entity_zero + if ezero is not None: + return ezero + else: + return None + + def corresponds_to(self, entity): + # TODO: this seems to have no effect for + # _ColumnEntity either + return False + + @property + def entity_zero_or_selectable(self): + for ent in self._entities: + ezero = ent.entity_zero_or_selectable + if ezero is not None: + return ezero + else: + return None + + def adapt_to_selectable(self, query, sel): + c = _BundleEntity(query, self.bundle, setup_entities=False) + #c._label_name = self._label_name + #c.entity_zero = self.entity_zero + #c.entities = self.entities + + for ent in self._entities: + ent.adapt_to_selectable(c, sel) + + def setup_entity(self, ext_info, aliased_adapter): + for ent in self._entities: + ent.setup_entity(ext_info, aliased_adapter) + + def setup_context(self, query, context): + for ent in self._entities: + ent.setup_context(query, context) + + def row_processor(self, query, context, custom_rows): + procs, labels = zip( + *[ent.row_processor(query, context, custom_rows) + for ent in self._entities] + ) + + proc = self.bundle.create_row_processor(query, procs, labels) + + return proc, self._label_name + class _ColumnEntity(_QueryEntity): """Column/expression based entity.""" @@ -3230,7 +3333,7 @@ class _ColumnEntity(_QueryEntity): self.expr = column self.namespace = namespace - if isinstance(column, basestring): + if isinstance(column, util.string_types): column = sql.literal_column(column) self._label_name = column.name elif isinstance(column, ( @@ -3238,7 +3341,7 @@ class _ColumnEntity(_QueryEntity): interfaces.PropComparator )): self._label_name = column.key - column = column.__clause_element__() + column = column._query_clause_element() else: self._label_name = getattr(column, 'key', None) @@ -3251,6 +3354,9 @@ class _ColumnEntity(_QueryEntity): if c is not column: return + elif isinstance(column, Bundle): + _BundleEntity(query, column) + return if not isinstance(column, sql.ColumnElement): raise sa_exc.InvalidRequestError( @@ -3258,13 +3364,21 @@ class _ColumnEntity(_QueryEntity): "expected - got '%r'" % (column, ) ) + self.type = type_ = column.type + if type_.hashable: + self.filter_fn = lambda item: item + else: + counter = util.counter() + self.filter_fn = lambda item: counter() + # If the Column is unnamed, give it a # label() so that mutable column expressions # can be located in the result even # if the expression's identity has been changed # due to adaption. - if not column._label: - column = column.label(None) + + if not column._label and not getattr(column, 'is_literal', False): + column = column.label(self._label_name) query._entities.append(self) @@ -3293,6 +3407,9 @@ class _ColumnEntity(_QueryEntity): else: self.entity_zero = None + supports_single_entity = False + custom_rows = False + @property def entity_zero_or_selectable(self): if self.entity_zero is not None: @@ -3302,29 +3419,24 @@ class _ColumnEntity(_QueryEntity): else: return None - @property - def type(self): - return self.column.type - - def filter_fn(self, item): - return item - def adapt_to_selectable(self, query, sel): c = _ColumnEntity(query, sel.corresponding_column(self.column)) c._label_name = self._label_name c.entity_zero = self.entity_zero c.entities = self.entities - def setup_entity(self, entity, mapper, adapter, from_obj, - is_aliased_class, with_polymorphic): + def setup_entity(self, ext_info, aliased_adapter): if 'selectable' not in self.__dict__: - self.selectable = from_obj - self.froms.add(from_obj) + self.selectable = ext_info.selectable + self.froms.add(ext_info.selectable) def corresponds_to(self, entity): + # TODO: just returning False here, + # no tests fail if self.entity_zero is None: return False elif _is_aliased_class(entity): + # TODO: polymorphic subclasses ? return entity is self.entity_zero else: return not _is_aliased_class(self.entity_zero) and \ @@ -3354,17 +3466,17 @@ class _ColumnEntity(_QueryEntity): def __str__(self): return str(self.column) -log.class_logger(Query) class QueryContext(object): multi_row_eager_loaders = False adapter = None froms = () + for_update = None def __init__(self, query): if query._statement is not None: - if isinstance(query._statement, expression._SelectBase) and \ + if isinstance(query._statement, expression.SelectBase) and \ not query._statement.use_labels: self.statement = query._statement.apply_labels() else: @@ -3390,17 +3502,49 @@ class QueryContext(object): o.propagate_to_loaders) self.attributes = query._attributes.copy() + class AliasOption(interfaces.MapperOption): def __init__(self, alias): + """Return a :class:`.MapperOption` that will indicate to the :class:`.Query` + that the main table has been aliased. + + This is a seldom-used option to suit the + very rare case that :func:`.contains_eager` + is being used in conjunction with a user-defined SELECT + statement that aliases the parent table. E.g.:: + + # define an aliased UNION called 'ulist' + ulist = users.select(users.c.user_id==7).\\ + union(users.select(users.c.user_id>7)).\\ + alias('ulist') + + # add on an eager load of "addresses" + statement = ulist.outerjoin(addresses).\\ + select().apply_labels() + + # create query, indicating "ulist" will be an + # alias for the main table, "addresses" + # property should be eager loaded + query = session.query(User).options( + contains_alias(ulist), + contains_eager(User.addresses)) + + # then get results via the statement + results = query.from_statement(statement).all() + + :param alias: is the string name of an alias, or a + :class:`~.sql.expression.Alias` object representing + the alias. + + """ self.alias = alias def process_query(self, query): - if isinstance(self.alias, basestring): + if isinstance(self.alias, util.string_types): alias = query._mapper_zero().mapped_table.alias(self.alias) else: alias = self.alias query._from_obj_alias = sql_util.ColumnAdapter(alias) -_new_runid = util.counter() diff --git a/libs/sqlalchemy/orm/relationships.py b/libs/sqlalchemy/orm/relationships.py new file mode 100644 index 00000000..982f10a4 --- /dev/null +++ b/libs/sqlalchemy/orm/relationships.py @@ -0,0 +1,2511 @@ +# orm/relationships.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Heuristics related to join conditions as used in +:func:`.relationship`. + +Provides the :class:`.JoinCondition` object, which encapsulates +SQL annotation and aliasing behavior focused on the `primaryjoin` +and `secondaryjoin` aspects of :func:`.relationship`. + +""" + +from .. import sql, util, exc as sa_exc, schema, log + +from .util import CascadeOptions, _orm_annotate, _orm_deannotate +from . import dependency +from . import attributes +from ..sql.util import ( + ClauseAdapter, + join_condition, _shallow_annotate, visit_binary_product, + _deep_deannotate, selectables_overlap + ) +from ..sql import operators, expression, visitors +from .interfaces import MANYTOMANY, MANYTOONE, ONETOMANY, StrategizedProperty, PropComparator +from ..inspection import inspect +from . import mapper as mapperlib + +def remote(expr): + """Annotate a portion of a primaryjoin expression + with a 'remote' annotation. + + See the section :ref:`relationship_custom_foreign` for a + description of use. + + .. versionadded:: 0.8 + + .. seealso:: + + :ref:`relationship_custom_foreign` + + :func:`.foreign` + + """ + return _annotate_columns(expression._clause_element_as_expr(expr), + {"remote": True}) + + +def foreign(expr): + """Annotate a portion of a primaryjoin expression + with a 'foreign' annotation. + + See the section :ref:`relationship_custom_foreign` for a + description of use. + + .. versionadded:: 0.8 + + .. seealso:: + + :ref:`relationship_custom_foreign` + + :func:`.remote` + + """ + + return _annotate_columns(expression._clause_element_as_expr(expr), + {"foreign": True}) + + +@log.class_logger +@util.langhelpers.dependency_for("sqlalchemy.orm.properties") +class RelationshipProperty(StrategizedProperty): + """Describes an object property that holds a single item or list + of items that correspond to a related database table. + + Public constructor is the :func:`.orm.relationship` function. + + See also: + + :ref:`relationship_config_toplevel` + + """ + + strategy_wildcard_key = 'relationship' + + _dependency_processor = None + + def __init__(self, argument, + secondary=None, primaryjoin=None, + secondaryjoin=None, + foreign_keys=None, + uselist=None, + order_by=False, + backref=None, + back_populates=None, + post_update=False, + cascade=False, extension=None, + viewonly=False, lazy=True, + collection_class=None, passive_deletes=False, + passive_updates=True, remote_side=None, + enable_typechecks=True, join_depth=None, + comparator_factory=None, + single_parent=False, innerjoin=False, + distinct_target_key=None, + doc=None, + active_history=False, + cascade_backrefs=True, + load_on_pending=False, + strategy_class=None, _local_remote_pairs=None, + query_class=None, + info=None): + """Provide a relationship of a primary Mapper to a secondary Mapper. + + This corresponds to a parent-child or associative table relationship. The + constructed class is an instance of :class:`.RelationshipProperty`. + + A typical :func:`.relationship`, used in a classical mapping:: + + mapper(Parent, properties={ + 'children': relationship(Child) + }) + + Some arguments accepted by :func:`.relationship` optionally accept a + callable function, which when called produces the desired value. + The callable is invoked by the parent :class:`.Mapper` at "mapper + initialization" time, which happens only when mappers are first used, and + is assumed to be after all mappings have been constructed. This can be + used to resolve order-of-declaration and other dependency issues, such as + if ``Child`` is declared below ``Parent`` in the same file:: + + mapper(Parent, properties={ + "children":relationship(lambda: Child, + order_by=lambda: Child.id) + }) + + When using the :ref:`declarative_toplevel` extension, the Declarative + initializer allows string arguments to be passed to :func:`.relationship`. + These string arguments are converted into callables that evaluate + the string as Python code, using the Declarative + class-registry as a namespace. This allows the lookup of related + classes to be automatic via their string name, and removes the need to + import related classes at all into the local module space:: + + from sqlalchemy.ext.declarative import declarative_base + + Base = declarative_base() + + class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + children = relationship("Child", order_by="Child.id") + + A full array of examples and reference documentation regarding + :func:`.relationship` is at :ref:`relationship_config_toplevel`. + + :param argument: + a mapped class, or actual :class:`.Mapper` instance, representing the + target of the relationship. + + ``argument`` may also be passed as a callable function + which is evaluated at mapper initialization time, and may be passed as a + Python-evaluable string when using Declarative. + + :param secondary: + for a many-to-many relationship, specifies the intermediary + table, and is an instance of :class:`.Table`. The ``secondary`` keyword + argument should generally only + be used for a table that is not otherwise expressed in any class + mapping, unless this relationship is declared as view only, otherwise + conflicting persistence operations can occur. + + ``secondary`` may + also be passed as a callable function which is evaluated at + mapper initialization time. + + :param active_history=False: + When ``True``, indicates that the "previous" value for a + many-to-one reference should be loaded when replaced, if + not already loaded. Normally, history tracking logic for + simple many-to-ones only needs to be aware of the "new" + value in order to perform a flush. This flag is available + for applications that make use of + :func:`.attributes.get_history` which also need to know + the "previous" value of the attribute. + + :param backref: + indicates the string name of a property to be placed on the related + mapper's class that will handle this relationship in the other + direction. The other property will be created automatically + when the mappers are configured. Can also be passed as a + :func:`backref` object to control the configuration of the + new relationship. + + :param back_populates: + Takes a string name and has the same meaning as ``backref``, + except the complementing property is **not** created automatically, + and instead must be configured explicitly on the other mapper. The + complementing property should also indicate ``back_populates`` + to this relationship to ensure proper functioning. + + :param cascade: + a comma-separated list of cascade rules which determines how + Session operations should be "cascaded" from parent to child. + This defaults to ``False``, which means the default cascade + should be used. The default value is ``"save-update, merge"``. + + Available cascades are: + + * ``save-update`` - cascade the :meth:`.Session.add` + operation. This cascade applies both to future and + past calls to :meth:`~sqlalchemy.orm.session.Session.add`, + meaning new items added to a collection or scalar relationship + get placed into the same session as that of the parent, and + also applies to items which have been removed from this + relationship but are still part of unflushed history. + + * ``merge`` - cascade the :meth:`~sqlalchemy.orm.session.Session.merge` + operation + + * ``expunge`` - cascade the :meth:`.Session.expunge` + operation + + * ``delete`` - cascade the :meth:`.Session.delete` + operation + + * ``delete-orphan`` - if an item of the child's type is + detached from its parent, mark it for deletion. + + .. versionchanged:: 0.7 + This option does not prevent + a new instance of the child object from being persisted + without a parent to start with; to constrain against + that case, ensure the child's foreign key column(s) + is configured as NOT NULL + + * ``refresh-expire`` - cascade the :meth:`.Session.expire` + and :meth:`~sqlalchemy.orm.session.Session.refresh` operations + + * ``all`` - shorthand for "save-update,merge, refresh-expire, + expunge, delete" + + See the section :ref:`unitofwork_cascades` for more background + on configuring cascades. + + :param cascade_backrefs=True: + a boolean value indicating if the ``save-update`` cascade should + operate along an assignment event intercepted by a backref. + When set to ``False``, + the attribute managed by this relationship will not cascade + an incoming transient object into the session of a + persistent parent, if the event is received via backref. + + That is:: + + mapper(A, a_table, properties={ + 'bs':relationship(B, backref="a", cascade_backrefs=False) + }) + + If an ``A()`` is present in the session, assigning it to + the "a" attribute on a transient ``B()`` will not place + the ``B()`` into the session. To set the flag in the other + direction, i.e. so that ``A().bs.append(B())`` won't add + a transient ``A()`` into the session for a persistent ``B()``:: + + mapper(A, a_table, properties={ + 'bs':relationship(B, + backref=backref("a", cascade_backrefs=False) + ) + }) + + See the section :ref:`unitofwork_cascades` for more background + on configuring cascades. + + :param collection_class: + a class or callable that returns a new list-holding object. will + be used in place of a plain list for storing elements. + Behavior of this attribute is described in detail at + :ref:`custom_collections`. + + :param comparator_factory: + a class which extends :class:`.RelationshipProperty.Comparator` which + provides custom SQL clause generation for comparison operations. + + :param distinct_target_key=None: + Indicate if a "subquery" eager load should apply the DISTINCT + keyword to the innermost SELECT statement. When left as ``None``, + the DISTINCT keyword will be applied in those cases when the target + columns do not comprise the full primary key of the target table. + When set to ``True``, the DISTINCT keyword is applied to the innermost + SELECT unconditionally. + + It may be desirable to set this flag to False when the DISTINCT is + reducing performance of the innermost subquery beyond that of what + duplicate innermost rows may be causing. + + .. versionadded:: 0.8.3 - distinct_target_key allows the + subquery eager loader to apply a DISTINCT modifier to the + innermost SELECT. + + .. versionchanged:: 0.9.0 - distinct_target_key now defaults to + ``None``, so that the feature enables itself automatically for + those cases where the innermost query targets a non-unique + key. + + :param doc: + docstring which will be applied to the resulting descriptor. + + :param extension: + an :class:`.AttributeExtension` instance, or list of extensions, + which will be prepended to the list of attribute listeners for + the resulting descriptor placed on the class. + **Deprecated.** Please see :class:`.AttributeEvents`. + + :param foreign_keys: + a list of columns which are to be used as "foreign key" columns, + or columns which refer to the value in a remote column, within the + context of this :func:`.relationship` object's ``primaryjoin`` + condition. That is, if the ``primaryjoin`` condition of this + :func:`.relationship` is ``a.id == b.a_id``, and the values in ``b.a_id`` + are required to be present in ``a.id``, then the "foreign key" column + of this :func:`.relationship` is ``b.a_id``. + + In normal cases, the ``foreign_keys`` parameter is **not required.** + :func:`.relationship` will **automatically** determine which columns + in the ``primaryjoin`` conditition are to be considered "foreign key" + columns based on those :class:`.Column` objects that specify + :class:`.ForeignKey`, or are otherwise listed as referencing columns + in a :class:`.ForeignKeyConstraint` construct. ``foreign_keys`` is only + needed when: + + 1. There is more than one way to construct a join from the local + table to the remote table, as there are multiple foreign key + references present. Setting ``foreign_keys`` will limit the + :func:`.relationship` to consider just those columns specified + here as "foreign". + + .. versionchanged:: 0.8 + A multiple-foreign key join ambiguity can be resolved by + setting the ``foreign_keys`` parameter alone, without the + need to explicitly set ``primaryjoin`` as well. + + 2. The :class:`.Table` being mapped does not actually have + :class:`.ForeignKey` or :class:`.ForeignKeyConstraint` + constructs present, often because the table + was reflected from a database that does not support foreign key + reflection (MySQL MyISAM). + + 3. The ``primaryjoin`` argument is used to construct a non-standard + join condition, which makes use of columns or expressions that do + not normally refer to their "parent" column, such as a join condition + expressed by a complex comparison using a SQL function. + + The :func:`.relationship` construct will raise informative error messages + that suggest the use of the ``foreign_keys`` parameter when presented + with an ambiguous condition. In typical cases, if :func:`.relationship` + doesn't raise any exceptions, the ``foreign_keys`` parameter is usually + not needed. + + ``foreign_keys`` may also be passed as a callable function + which is evaluated at mapper initialization time, and may be passed as a + Python-evaluable string when using Declarative. + + .. seealso:: + + :ref:`relationship_foreign_keys` + + :ref:`relationship_custom_foreign` + + :func:`.foreign` - allows direct annotation of the "foreign" columns + within a ``primaryjoin`` condition. + + .. versionadded:: 0.8 + The :func:`.foreign` annotation can also be applied + directly to the ``primaryjoin`` expression, which is an alternate, + more specific system of describing which columns in a particular + ``primaryjoin`` should be considered "foreign". + + :param info: Optional data dictionary which will be populated into the + :attr:`.MapperProperty.info` attribute of this object. + + .. versionadded:: 0.8 + + :param innerjoin=False: + when ``True``, joined eager loads will use an inner join to join + against related tables instead of an outer join. The purpose + of this option is generally one of performance, as inner joins + generally perform better than outer joins. Another reason can be + the use of ``with_lockmode``, which does not support outer joins. + + This flag can be set to ``True`` when the relationship references an + object via many-to-one using local foreign keys that are not nullable, + or when the reference is one-to-one or a collection that is guaranteed + to have one or at least one entry. + + :param join_depth: + when non-``None``, an integer value indicating how many levels + deep "eager" loaders should join on a self-referring or cyclical + relationship. The number counts how many times the same Mapper + shall be present in the loading condition along a particular join + branch. When left at its default of ``None``, eager loaders + will stop chaining when they encounter a the same target mapper + which is already higher up in the chain. This option applies + both to joined- and subquery- eager loaders. + + :param lazy='select': specifies + how the related items should be loaded. Default value is + ``select``. Values include: + + * ``select`` - items should be loaded lazily when the property is first + accessed, using a separate SELECT statement, or identity map + fetch for simple many-to-one references. + + * ``immediate`` - items should be loaded as the parents are loaded, + using a separate SELECT statement, or identity map fetch for + simple many-to-one references. + + .. versionadded:: 0.6.5 + + * ``joined`` - items should be loaded "eagerly" in the same query as + that of the parent, using a JOIN or LEFT OUTER JOIN. Whether + the join is "outer" or not is determined by the ``innerjoin`` + parameter. + + * ``subquery`` - items should be loaded "eagerly" as the parents are + loaded, using one additional SQL statement, which issues a JOIN to a + subquery of the original statement, for each collection requested. + + * ``noload`` - no loading should occur at any time. This is to + support "write-only" attributes, or attributes which are + populated in some manner specific to the application. + + * ``dynamic`` - the attribute will return a pre-configured + :class:`~sqlalchemy.orm.query.Query` object for all read + operations, onto which further filtering operations can be + applied before iterating the results. See + the section :ref:`dynamic_relationship` for more details. + + * True - a synonym for 'select' + + * False - a synonym for 'joined' + + * None - a synonym for 'noload' + + Detailed discussion of loader strategies is at :doc:`/orm/loading`. + + :param load_on_pending=False: + Indicates loading behavior for transient or pending parent objects. + + When set to ``True``, causes the lazy-loader to + issue a query for a parent object that is not persistent, meaning it has + never been flushed. This may take effect for a pending object when + autoflush is disabled, or for a transient object that has been + "attached" to a :class:`.Session` but is not part of its pending + collection. + + The load_on_pending flag does not improve behavior + when the ORM is used normally - object references should be constructed + at the object level, not at the foreign key level, so that they + are present in an ordinary way before flush() proceeds. This flag + is not not intended for general use. + + .. versionadded:: 0.6.5 + + .. seealso:: + + :meth:`.Session.enable_relationship_loading` - this method establishes + "load on pending" behavior for the whole object, and also allows + loading on objects that remain transient or detached. + + :param order_by: + indicates the ordering that should be applied when loading these + items. ``order_by`` is expected to refer to one of the :class:`.Column` + objects to which the target class is mapped, or + the attribute itself bound to the target class which refers + to the column. + + ``order_by`` may also be passed as a callable function + which is evaluated at mapper initialization time, and may be passed as a + Python-evaluable string when using Declarative. + + :param passive_deletes=False: + Indicates loading behavior during delete operations. + + A value of True indicates that unloaded child items should not + be loaded during a delete operation on the parent. Normally, + when a parent item is deleted, all child items are loaded so + that they can either be marked as deleted, or have their + foreign key to the parent set to NULL. Marking this flag as + True usually implies an ON DELETE rule is in + place which will handle updating/deleting child rows on the + database side. + + Additionally, setting the flag to the string value 'all' will + disable the "nulling out" of the child foreign keys, when there + is no delete or delete-orphan cascade enabled. This is + typically used when a triggering or error raise scenario is in + place on the database side. Note that the foreign key + attributes on in-session child objects will not be changed + after a flush occurs so this is a very special use-case + setting. + + :param passive_updates=True: + Indicates loading and INSERT/UPDATE/DELETE behavior when the + source of a foreign key value changes (i.e. an "on update" + cascade), which are typically the primary key columns of the + source row. + + When True, it is assumed that ON UPDATE CASCADE is configured on + the foreign key in the database, and that the database will + handle propagation of an UPDATE from a source column to + dependent rows. Note that with databases which enforce + referential integrity (i.e. PostgreSQL, MySQL with InnoDB tables), + ON UPDATE CASCADE is required for this operation. The + relationship() will update the value of the attribute on related + items which are locally present in the session during a flush. + + When False, it is assumed that the database does not enforce + referential integrity and will not be issuing its own CASCADE + operation for an update. The relationship() will issue the + appropriate UPDATE statements to the database in response to the + change of a referenced key, and items locally present in the + session during a flush will also be refreshed. + + This flag should probably be set to False if primary key changes + are expected and the database in use doesn't support CASCADE + (i.e. SQLite, MySQL MyISAM tables). + + Also see the passive_updates flag on ``mapper()``. + + A future SQLAlchemy release will provide a "detect" feature for + this flag. + + :param post_update: + this indicates that the relationship should be handled by a + second UPDATE statement after an INSERT or before a + DELETE. Currently, it also will issue an UPDATE after the + instance was UPDATEd as well, although this technically should + be improved. This flag is used to handle saving bi-directional + dependencies between two individual rows (i.e. each row + references the other), where it would otherwise be impossible to + INSERT or DELETE both rows fully since one row exists before the + other. Use this flag when a particular mapping arrangement will + incur two rows that are dependent on each other, such as a table + that has a one-to-many relationship to a set of child rows, and + also has a column that references a single child row within that + list (i.e. both tables contain a foreign key to each other). If + a ``flush()`` operation returns an error that a "cyclical + dependency" was detected, this is a cue that you might want to + use ``post_update`` to "break" the cycle. + + :param primaryjoin: + a SQL expression that will be used as the primary + join of this child object against the parent object, or in a + many-to-many relationship the join of the primary object to the + association table. By default, this value is computed based on the + foreign key relationships of the parent and child tables (or association + table). + + ``primaryjoin`` may also be passed as a callable function + which is evaluated at mapper initialization time, and may be passed as a + Python-evaluable string when using Declarative. + + :param remote_side: + used for self-referential relationships, indicates the column or + list of columns that form the "remote side" of the relationship. + + ``remote_side`` may also be passed as a callable function + which is evaluated at mapper initialization time, and may be passed as a + Python-evaluable string when using Declarative. + + .. versionchanged:: 0.8 + The :func:`.remote` annotation can also be applied + directly to the ``primaryjoin`` expression, which is an alternate, + more specific system of describing which columns in a particular + ``primaryjoin`` should be considered "remote". + + :param query_class: + a :class:`.Query` subclass that will be used as the base of the + "appender query" returned by a "dynamic" relationship, that + is, a relationship that specifies ``lazy="dynamic"`` or was + otherwise constructed using the :func:`.orm.dynamic_loader` + function. + + :param secondaryjoin: + a SQL expression that will be used as the join of + an association table to the child object. By default, this value is + computed based on the foreign key relationships of the association and + child tables. + + ``secondaryjoin`` may also be passed as a callable function + which is evaluated at mapper initialization time, and may be passed as a + Python-evaluable string when using Declarative. + + :param single_parent=(True|False): + when True, installs a validator which will prevent objects + from being associated with more than one parent at a time. + This is used for many-to-one or many-to-many relationships that + should be treated either as one-to-one or one-to-many. Its + usage is optional unless delete-orphan cascade is also + set on this relationship(), in which case its required. + + :param uselist=(True|False): + a boolean that indicates if this property should be loaded as a + list or a scalar. In most cases, this value is determined + automatically by ``relationship()``, based on the type and direction + of the relationship - one to many forms a list, many to one + forms a scalar, many to many is a list. If a scalar is desired + where normally a list would be present, such as a bi-directional + one-to-one relationship, set uselist to False. + + :param viewonly=False: + when set to True, the relationship is used only for loading objects + within the relationship, and has no effect on the unit-of-work + flush process. Relationships with viewonly can specify any kind of + join conditions to provide additional views of related objects + onto a parent object. Note that the functionality of a viewonly + relationship has its limits - complicated join conditions may + not compile into eager or lazy loaders properly. If this is the + case, use an alternative method. + + .. versionchanged:: 0.6 + :func:`relationship` was renamed from its previous name + :func:`relation`. + + """ + + self.uselist = uselist + self.argument = argument + self.secondary = secondary + self.primaryjoin = primaryjoin + self.secondaryjoin = secondaryjoin + self.post_update = post_update + self.direction = None + self.viewonly = viewonly + self.lazy = lazy + self.single_parent = single_parent + self._user_defined_foreign_keys = foreign_keys + self.collection_class = collection_class + self.passive_deletes = passive_deletes + self.cascade_backrefs = cascade_backrefs + self.passive_updates = passive_updates + self.remote_side = remote_side + self.enable_typechecks = enable_typechecks + self.query_class = query_class + self.innerjoin = innerjoin + self.distinct_target_key = distinct_target_key + self.doc = doc + self.active_history = active_history + self.join_depth = join_depth + self.local_remote_pairs = _local_remote_pairs + self.extension = extension + self.load_on_pending = load_on_pending + self.comparator_factory = comparator_factory or \ + RelationshipProperty.Comparator + self.comparator = self.comparator_factory(self, None) + util.set_creation_order(self) + + if info is not None: + self.info = info + + if strategy_class: + self.strategy_class = strategy_class + else: + self.strategy_class = self._strategy_lookup(("lazy", self.lazy)) + + self._reverse_property = set() + + self.cascade = cascade if cascade is not False \ + else "save-update, merge" + + self.order_by = order_by + + self.back_populates = back_populates + + if self.back_populates: + if backref: + raise sa_exc.ArgumentError( + "backref and back_populates keyword arguments " + "are mutually exclusive") + self.backref = None + else: + self.backref = backref + + def instrument_class(self, mapper): + attributes.register_descriptor( + mapper.class_, + self.key, + comparator=self.comparator_factory(self, mapper), + parententity=mapper, + doc=self.doc, + ) + + class Comparator(PropComparator): + """Produce boolean, comparison, and other operators for + :class:`.RelationshipProperty` attributes. + + See the documentation for :class:`.PropComparator` for a brief overview + of ORM level operator definition. + + See also: + + :class:`.PropComparator` + + :class:`.ColumnProperty.Comparator` + + :class:`.ColumnOperators` + + :ref:`types_operators` + + :attr:`.TypeEngine.comparator_factory` + + """ + + _of_type = None + + def __init__(self, prop, parentmapper, adapt_to_entity=None, of_type=None): + """Construction of :class:`.RelationshipProperty.Comparator` + is internal to the ORM's attribute mechanics. + + """ + self.prop = prop + self._parentmapper = parentmapper + self._adapt_to_entity = adapt_to_entity + if of_type: + self._of_type = of_type + + def adapt_to_entity(self, adapt_to_entity): + return self.__class__(self.property, self._parentmapper, + adapt_to_entity=adapt_to_entity, + of_type=self._of_type) + + @util.memoized_property + def mapper(self): + """The target :class:`.Mapper` referred to by this + :class:`.RelationshipProperty.Comparator`. + + This is the "target" or "remote" side of the + :func:`.relationship`. + + """ + return self.property.mapper + + @util.memoized_property + def _parententity(self): + return self.property.parent + + def _source_selectable(self): + elem = self.property.parent._with_polymorphic_selectable + if self.adapter: + return self.adapter(elem) + else: + return elem + + def __clause_element__(self): + adapt_from = self._source_selectable() + if self._of_type: + of_type = inspect(self._of_type).mapper + else: + of_type = None + + pj, sj, source, dest, \ + secondary, target_adapter = self.property._create_joins( + source_selectable=adapt_from, + source_polymorphic=True, + of_type=of_type) + if sj is not None: + return pj & sj + else: + return pj + + def of_type(self, cls): + """Produce a construct that represents a particular 'subtype' of + attribute for the parent class. + + Currently this is usable in conjunction with :meth:`.Query.join` + and :meth:`.Query.outerjoin`. + + """ + return RelationshipProperty.Comparator( + self.property, + self._parentmapper, + adapt_to_entity=self._adapt_to_entity, + of_type=cls) + + def in_(self, other): + """Produce an IN clause - this is not implemented + for :func:`~.orm.relationship`-based attributes at this time. + + """ + raise NotImplementedError('in_() not yet supported for ' + 'relationships. For a simple many-to-one, use ' + 'in_() against the set of foreign key values.') + + __hash__ = None + + def __eq__(self, other): + """Implement the ``==`` operator. + + In a many-to-one context, such as:: + + MyClass.some_prop == + + this will typically produce a + clause such as:: + + mytable.related_id == + + Where ```` is the primary key of the given + object. + + The ``==`` operator provides partial functionality for non- + many-to-one comparisons: + + * Comparisons against collections are not supported. + Use :meth:`~.RelationshipProperty.Comparator.contains`. + * Compared to a scalar one-to-many, will produce a + clause that compares the target columns in the parent to + the given target. + * Compared to a scalar many-to-many, an alias + of the association table will be rendered as + well, forming a natural join that is part of the + main body of the query. This will not work for + queries that go beyond simple AND conjunctions of + comparisons, such as those which use OR. Use + explicit joins, outerjoins, or + :meth:`~.RelationshipProperty.Comparator.has` for + more comprehensive non-many-to-one scalar + membership tests. + * Comparisons against ``None`` given in a one-to-many + or many-to-many context produce a NOT EXISTS clause. + + """ + if isinstance(other, (util.NoneType, expression.Null)): + if self.property.direction in [ONETOMANY, MANYTOMANY]: + return ~self._criterion_exists() + else: + return _orm_annotate(self.property._optimized_compare( + None, adapt_source=self.adapter)) + elif self.property.uselist: + raise sa_exc.InvalidRequestError("Can't compare a colle" + "ction to an object or collection; use " + "contains() to test for membership.") + else: + return _orm_annotate(self.property._optimized_compare(other, + adapt_source=self.adapter)) + + def _criterion_exists(self, criterion=None, **kwargs): + if getattr(self, '_of_type', None): + info = inspect(self._of_type) + target_mapper, to_selectable, is_aliased_class = \ + info.mapper, info.selectable, info.is_aliased_class + if self.property._is_self_referential and not is_aliased_class: + to_selectable = to_selectable.alias() + + single_crit = target_mapper._single_table_criterion + if single_crit is not None: + if criterion is not None: + criterion = single_crit & criterion + else: + criterion = single_crit + else: + is_aliased_class = False + to_selectable = None + + if self.adapter: + source_selectable = self._source_selectable() + else: + source_selectable = None + + pj, sj, source, dest, secondary, target_adapter = \ + self.property._create_joins(dest_polymorphic=True, + dest_selectable=to_selectable, + source_selectable=source_selectable) + + for k in kwargs: + crit = getattr(self.property.mapper.class_, k) == kwargs[k] + if criterion is None: + criterion = crit + else: + criterion = criterion & crit + + # annotate the *local* side of the join condition, in the case + # of pj + sj this is the full primaryjoin, in the case of just + # pj its the local side of the primaryjoin. + if sj is not None: + j = _orm_annotate(pj) & sj + else: + j = _orm_annotate(pj, exclude=self.property.remote_side) + + if criterion is not None and target_adapter and not is_aliased_class: + # limit this adapter to annotated only? + criterion = target_adapter.traverse(criterion) + + # only have the "joined left side" of what we + # return be subject to Query adaption. The right + # side of it is used for an exists() subquery and + # should not correlate or otherwise reach out + # to anything in the enclosing query. + if criterion is not None: + criterion = criterion._annotate( + {'no_replacement_traverse': True}) + + crit = j & sql.True_._ifnone(criterion) + + ex = sql.exists([1], crit, from_obj=dest).correlate_except(dest) + if secondary is not None: + ex = ex.correlate_except(secondary) + return ex + + def any(self, criterion=None, **kwargs): + """Produce an expression that tests a collection against + particular criterion, using EXISTS. + + An expression like:: + + session.query(MyClass).filter( + MyClass.somereference.any(SomeRelated.x==2) + ) + + + Will produce a query like:: + + SELECT * FROM my_table WHERE + EXISTS (SELECT 1 FROM related WHERE related.my_id=my_table.id + AND related.x=2) + + Because :meth:`~.RelationshipProperty.Comparator.any` uses + a correlated subquery, its performance is not nearly as + good when compared against large target tables as that of + using a join. + + :meth:`~.RelationshipProperty.Comparator.any` is particularly + useful for testing for empty collections:: + + session.query(MyClass).filter( + ~MyClass.somereference.any() + ) + + will produce:: + + SELECT * FROM my_table WHERE + NOT EXISTS (SELECT 1 FROM related WHERE + related.my_id=my_table.id) + + :meth:`~.RelationshipProperty.Comparator.any` is only + valid for collections, i.e. a :func:`.relationship` + that has ``uselist=True``. For scalar references, + use :meth:`~.RelationshipProperty.Comparator.has`. + + """ + if not self.property.uselist: + raise sa_exc.InvalidRequestError( + "'any()' not implemented for scalar " + "attributes. Use has()." + ) + + return self._criterion_exists(criterion, **kwargs) + + def has(self, criterion=None, **kwargs): + """Produce an expression that tests a scalar reference against + particular criterion, using EXISTS. + + An expression like:: + + session.query(MyClass).filter( + MyClass.somereference.has(SomeRelated.x==2) + ) + + + Will produce a query like:: + + SELECT * FROM my_table WHERE + EXISTS (SELECT 1 FROM related WHERE + related.id==my_table.related_id AND related.x=2) + + Because :meth:`~.RelationshipProperty.Comparator.has` uses + a correlated subquery, its performance is not nearly as + good when compared against large target tables as that of + using a join. + + :meth:`~.RelationshipProperty.Comparator.has` is only + valid for scalar references, i.e. a :func:`.relationship` + that has ``uselist=False``. For collection references, + use :meth:`~.RelationshipProperty.Comparator.any`. + + """ + if self.property.uselist: + raise sa_exc.InvalidRequestError( + "'has()' not implemented for collections. " + "Use any().") + return self._criterion_exists(criterion, **kwargs) + + def contains(self, other, **kwargs): + """Return a simple expression that tests a collection for + containment of a particular item. + + :meth:`~.RelationshipProperty.Comparator.contains` is + only valid for a collection, i.e. a + :func:`~.orm.relationship` that implements + one-to-many or many-to-many with ``uselist=True``. + + When used in a simple one-to-many context, an + expression like:: + + MyClass.contains(other) + + Produces a clause like:: + + mytable.id == + + Where ```` is the value of the foreign key + attribute on ``other`` which refers to the primary + key of its parent object. From this it follows that + :meth:`~.RelationshipProperty.Comparator.contains` is + very useful when used with simple one-to-many + operations. + + For many-to-many operations, the behavior of + :meth:`~.RelationshipProperty.Comparator.contains` + has more caveats. The association table will be + rendered in the statement, producing an "implicit" + join, that is, includes multiple tables in the FROM + clause which are equated in the WHERE clause:: + + query(MyClass).filter(MyClass.contains(other)) + + Produces a query like:: + + SELECT * FROM my_table, my_association_table AS + my_association_table_1 WHERE + my_table.id = my_association_table_1.parent_id + AND my_association_table_1.child_id = + + Where ```` would be the primary key of + ``other``. From the above, it is clear that + :meth:`~.RelationshipProperty.Comparator.contains` + will **not** work with many-to-many collections when + used in queries that move beyond simple AND + conjunctions, such as multiple + :meth:`~.RelationshipProperty.Comparator.contains` + expressions joined by OR. In such cases subqueries or + explicit "outer joins" will need to be used instead. + See :meth:`~.RelationshipProperty.Comparator.any` for + a less-performant alternative using EXISTS, or refer + to :meth:`.Query.outerjoin` as well as :ref:`ormtutorial_joins` + for more details on constructing outer joins. + + """ + if not self.property.uselist: + raise sa_exc.InvalidRequestError( + "'contains' not implemented for scalar " + "attributes. Use ==") + clause = self.property._optimized_compare(other, + adapt_source=self.adapter) + + if self.property.secondaryjoin is not None: + clause.negation_clause = \ + self.__negated_contains_or_equals(other) + + return clause + + def __negated_contains_or_equals(self, other): + if self.property.direction == MANYTOONE: + state = attributes.instance_state(other) + + def state_bindparam(x, state, col): + o = state.obj() # strong ref + return sql.bindparam(x, unique=True, callable_=lambda: \ + self.property.mapper._get_committed_attr_by_column(o, col)) + + def adapt(col): + if self.adapter: + return self.adapter(col) + else: + return col + + if self.property._use_get: + return sql.and_(*[ + sql.or_( + adapt(x) != state_bindparam(adapt(x), state, y), + adapt(x) == None) + for (x, y) in self.property.local_remote_pairs]) + + criterion = sql.and_(*[x == y for (x, y) in + zip( + self.property.mapper.primary_key, + self.property.\ + mapper.\ + primary_key_from_instance(other)) + ]) + return ~self._criterion_exists(criterion) + + def __ne__(self, other): + """Implement the ``!=`` operator. + + In a many-to-one context, such as:: + + MyClass.some_prop != + + This will typically produce a clause such as:: + + mytable.related_id != + + Where ```` is the primary key of the + given object. + + The ``!=`` operator provides partial functionality for non- + many-to-one comparisons: + + * Comparisons against collections are not supported. + Use + :meth:`~.RelationshipProperty.Comparator.contains` + in conjunction with :func:`~.expression.not_`. + * Compared to a scalar one-to-many, will produce a + clause that compares the target columns in the parent to + the given target. + * Compared to a scalar many-to-many, an alias + of the association table will be rendered as + well, forming a natural join that is part of the + main body of the query. This will not work for + queries that go beyond simple AND conjunctions of + comparisons, such as those which use OR. Use + explicit joins, outerjoins, or + :meth:`~.RelationshipProperty.Comparator.has` in + conjunction with :func:`~.expression.not_` for + more comprehensive non-many-to-one scalar + membership tests. + * Comparisons against ``None`` given in a one-to-many + or many-to-many context produce an EXISTS clause. + + """ + if isinstance(other, (util.NoneType, expression.Null)): + if self.property.direction == MANYTOONE: + return sql.or_(*[x != None for x in + self.property._calculated_foreign_keys]) + else: + return self._criterion_exists() + elif self.property.uselist: + raise sa_exc.InvalidRequestError("Can't compare a collection" + " to an object or collection; use " + "contains() to test for membership.") + else: + return self.__negated_contains_or_equals(other) + + @util.memoized_property + def property(self): + if mapperlib.Mapper._new_mappers: + mapperlib.Mapper._configure_all() + return self.prop + + def compare(self, op, value, + value_is_parent=False, + alias_secondary=True): + if op == operators.eq: + if value is None: + if self.uselist: + return ~sql.exists([1], self.primaryjoin) + else: + return self._optimized_compare(None, + value_is_parent=value_is_parent, + alias_secondary=alias_secondary) + else: + return self._optimized_compare(value, + value_is_parent=value_is_parent, + alias_secondary=alias_secondary) + else: + return op(self.comparator, value) + + def _optimized_compare(self, value, value_is_parent=False, + adapt_source=None, + alias_secondary=True): + if value is not None: + value = attributes.instance_state(value) + return self._lazy_strategy.lazy_clause(value, + reverse_direction=not value_is_parent, + alias_secondary=alias_secondary, + adapt_source=adapt_source) + + def __str__(self): + return str(self.parent.class_.__name__) + "." + self.key + + def merge(self, + session, + source_state, + source_dict, + dest_state, + dest_dict, + load, _recursive): + + if load: + for r in self._reverse_property: + if (source_state, r) in _recursive: + return + + if not "merge" in self._cascade: + return + + if self.key not in source_dict: + return + + if self.uselist: + instances = source_state.get_impl(self.key).\ + get(source_state, source_dict) + if hasattr(instances, '_sa_adapter'): + # convert collections to adapters to get a true iterator + instances = instances._sa_adapter + + if load: + # for a full merge, pre-load the destination collection, + # so that individual _merge of each item pulls from identity + # map for those already present. + # also assumes CollectionAttrbiuteImpl behavior of loading + # "old" list in any case + dest_state.get_impl(self.key).get(dest_state, dest_dict) + + dest_list = [] + for current in instances: + current_state = attributes.instance_state(current) + current_dict = attributes.instance_dict(current) + _recursive[(current_state, self)] = True + obj = session._merge(current_state, current_dict, + load=load, _recursive=_recursive) + if obj is not None: + dest_list.append(obj) + + if not load: + coll = attributes.init_state_collection(dest_state, + dest_dict, self.key) + for c in dest_list: + coll.append_without_event(c) + else: + dest_state.get_impl(self.key)._set_iterable(dest_state, + dest_dict, dest_list) + else: + current = source_dict[self.key] + if current is not None: + current_state = attributes.instance_state(current) + current_dict = attributes.instance_dict(current) + _recursive[(current_state, self)] = True + obj = session._merge(current_state, current_dict, + load=load, _recursive=_recursive) + else: + obj = None + + if not load: + dest_dict[self.key] = obj + else: + dest_state.get_impl(self.key).set(dest_state, + dest_dict, obj, None) + + def _value_as_iterable(self, state, dict_, key, + passive=attributes.PASSIVE_OFF): + """Return a list of tuples (state, obj) for the given + key. + + returns an empty list if the value is None/empty/PASSIVE_NO_RESULT + """ + + impl = state.manager[key].impl + x = impl.get(state, dict_, passive=passive) + if x is attributes.PASSIVE_NO_RESULT or x is None: + return [] + elif hasattr(impl, 'get_collection'): + return [ + (attributes.instance_state(o), o) for o in + impl.get_collection(state, dict_, x, passive=passive) + ] + else: + return [(attributes.instance_state(x), x)] + + def cascade_iterator(self, type_, state, dict_, + visited_states, halt_on=None): + #assert type_ in self._cascade + + # only actively lazy load on the 'delete' cascade + if type_ != 'delete' or self.passive_deletes: + passive = attributes.PASSIVE_NO_INITIALIZE + else: + passive = attributes.PASSIVE_OFF + + if type_ == 'save-update': + tuples = state.manager[self.key].impl.\ + get_all_pending(state, dict_) + + else: + tuples = self._value_as_iterable(state, dict_, self.key, + passive=passive) + + skip_pending = type_ == 'refresh-expire' and 'delete-orphan' \ + not in self._cascade + + for instance_state, c in tuples: + if instance_state in visited_states: + continue + + if c is None: + # would like to emit a warning here, but + # would not be consistent with collection.append(None) + # current behavior of silently skipping. + # see [ticket:2229] + continue + + instance_dict = attributes.instance_dict(c) + + if halt_on and halt_on(instance_state): + continue + + if skip_pending and not instance_state.key: + continue + + instance_mapper = instance_state.manager.mapper + + if not instance_mapper.isa(self.mapper.class_manager.mapper): + raise AssertionError("Attribute '%s' on class '%s' " + "doesn't handle objects " + "of type '%s'" % ( + self.key, + self.parent.class_, + c.__class__ + )) + + visited_states.add(instance_state) + + yield c, instance_mapper, instance_state, instance_dict + + def _add_reverse_property(self, key): + other = self.mapper.get_property(key, _configure_mappers=False) + self._reverse_property.add(other) + other._reverse_property.add(self) + + if not other.mapper.common_parent(self.parent): + raise sa_exc.ArgumentError('reverse_property %r on ' + 'relationship %s references relationship %s, which ' + 'does not reference mapper %s' % (key, self, other, + self.parent)) + if self.direction in (ONETOMANY, MANYTOONE) and self.direction \ + == other.direction: + raise sa_exc.ArgumentError('%s and back-reference %s are ' + 'both of the same direction %r. Did you mean to ' + 'set remote_side on the many-to-one side ?' + % (other, self, self.direction)) + + @util.memoized_property + def mapper(self): + """Return the targeted :class:`.Mapper` for this + :class:`.RelationshipProperty`. + + This is a lazy-initializing static attribute. + + """ + if util.callable(self.argument) and \ + not isinstance(self.argument, (type, mapperlib.Mapper)): + argument = self.argument() + else: + argument = self.argument + + if isinstance(argument, type): + mapper_ = mapperlib.class_mapper(argument, + configure=False) + elif isinstance(self.argument, mapperlib.Mapper): + mapper_ = argument + else: + raise sa_exc.ArgumentError("relationship '%s' expects " + "a class or a mapper argument (received: %s)" + % (self.key, type(argument))) + return mapper_ + + @util.memoized_property + @util.deprecated("0.7", "Use .target") + def table(self): + """Return the selectable linked to this + :class:`.RelationshipProperty` object's target + :class:`.Mapper`. + """ + return self.target + + def do_init(self): + self._check_conflicts() + self._process_dependent_arguments() + self._setup_join_conditions() + self._check_cascade_settings(self._cascade) + self._post_init() + self._generate_backref() + super(RelationshipProperty, self).do_init() + self._lazy_strategy = self._get_strategy((("lazy", "select"),)) + + + def _process_dependent_arguments(self): + """Convert incoming configuration arguments to their + proper form. + + Callables are resolved, ORM annotations removed. + + """ + # accept callables for other attributes which may require + # deferred initialization. This technique is used + # by declarative "string configs" and some recipes. + for attr in ( + 'order_by', 'primaryjoin', 'secondaryjoin', + 'secondary', '_user_defined_foreign_keys', 'remote_side', + ): + attr_value = getattr(self, attr) + if util.callable(attr_value): + setattr(self, attr, attr_value()) + + # remove "annotations" which are present if mapped class + # descriptors are used to create the join expression. + for attr in 'primaryjoin', 'secondaryjoin': + val = getattr(self, attr) + if val is not None: + setattr(self, attr, _orm_deannotate( + expression._only_column_elements(val, attr)) + ) + + # ensure expressions in self.order_by, foreign_keys, + # remote_side are all columns, not strings. + if self.order_by is not False and self.order_by is not None: + self.order_by = [ + expression._only_column_elements(x, "order_by") + for x in + util.to_list(self.order_by)] + + self._user_defined_foreign_keys = \ + util.column_set( + expression._only_column_elements(x, "foreign_keys") + for x in util.to_column_set( + self._user_defined_foreign_keys + )) + + self.remote_side = \ + util.column_set( + expression._only_column_elements(x, "remote_side") + for x in + util.to_column_set(self.remote_side)) + + self.target = self.mapper.mapped_table + + + def _setup_join_conditions(self): + self._join_condition = jc = JoinCondition( + parent_selectable=self.parent.mapped_table, + child_selectable=self.mapper.mapped_table, + parent_local_selectable=self.parent.local_table, + child_local_selectable=self.mapper.local_table, + primaryjoin=self.primaryjoin, + secondary=self.secondary, + secondaryjoin=self.secondaryjoin, + parent_equivalents=self.parent._equivalent_columns, + child_equivalents=self.mapper._equivalent_columns, + consider_as_foreign_keys=self._user_defined_foreign_keys, + local_remote_pairs=self.local_remote_pairs, + remote_side=self.remote_side, + self_referential=self._is_self_referential, + prop=self, + support_sync=not self.viewonly, + can_be_synced_fn=self._columns_are_mapped + ) + self.primaryjoin = jc.deannotated_primaryjoin + self.secondaryjoin = jc.deannotated_secondaryjoin + self.direction = jc.direction + self.local_remote_pairs = jc.local_remote_pairs + self.remote_side = jc.remote_columns + self.local_columns = jc.local_columns + self.synchronize_pairs = jc.synchronize_pairs + self._calculated_foreign_keys = jc.foreign_key_columns + self.secondary_synchronize_pairs = jc.secondary_synchronize_pairs + + def _check_conflicts(self): + """Test that this relationship is legal, warn about + inheritance conflicts.""" + + if not self.is_primary() \ + and not mapperlib.class_mapper( + self.parent.class_, + configure=False).has_property(self.key): + raise sa_exc.ArgumentError("Attempting to assign a new " + "relationship '%s' to a non-primary mapper on " + "class '%s'. New relationships can only be added " + "to the primary mapper, i.e. the very first mapper " + "created for class '%s' " % (self.key, + self.parent.class_.__name__, + self.parent.class_.__name__)) + + # check for conflicting relationship() on superclass + if not self.parent.concrete: + for inheriting in self.parent.iterate_to_root(): + if inheriting is not self.parent \ + and inheriting.has_property(self.key): + util.warn("Warning: relationship '%s' on mapper " + "'%s' supersedes the same relationship " + "on inherited mapper '%s'; this can " + "cause dependency issues during flush" + % (self.key, self.parent, inheriting)) + + def _get_cascade(self): + """Return the current cascade setting for this + :class:`.RelationshipProperty`. + """ + return self._cascade + + def _set_cascade(self, cascade): + cascade = CascadeOptions(cascade) + if 'mapper' in self.__dict__: + self._check_cascade_settings(cascade) + self._cascade = cascade + + if self._dependency_processor: + self._dependency_processor.cascade = cascade + + cascade = property(_get_cascade, _set_cascade) + + def _check_cascade_settings(self, cascade): + if cascade.delete_orphan and not self.single_parent \ + and (self.direction is MANYTOMANY or self.direction + is MANYTOONE): + raise sa_exc.ArgumentError( + 'On %s, delete-orphan cascade is not supported ' + 'on a many-to-many or many-to-one relationship ' + 'when single_parent is not set. Set ' + 'single_parent=True on the relationship().' + % self) + if self.direction is MANYTOONE and self.passive_deletes: + util.warn("On %s, 'passive_deletes' is normally configured " + "on one-to-many, one-to-one, many-to-many " + "relationships only." + % self) + + if self.passive_deletes == 'all' and \ + ("delete" in cascade or + "delete-orphan" in cascade): + raise sa_exc.ArgumentError( + "On %s, can't set passive_deletes='all' in conjunction " + "with 'delete' or 'delete-orphan' cascade" % self) + + if cascade.delete_orphan: + self.mapper.primary_mapper()._delete_orphans.append( + (self.key, self.parent.class_) + ) + + def _columns_are_mapped(self, *cols): + """Return True if all columns in the given collection are + mapped by the tables referenced by this :class:`.Relationship`. + + """ + for c in cols: + if self.secondary is not None \ + and self.secondary.c.contains_column(c): + continue + if not self.parent.mapped_table.c.contains_column(c) and \ + not self.target.c.contains_column(c): + return False + return True + + def _generate_backref(self): + """Interpret the 'backref' instruction to create a + :func:`.relationship` complementary to this one.""" + + if not self.is_primary(): + return + if self.backref is not None and not self.back_populates: + if isinstance(self.backref, util.string_types): + backref_key, kwargs = self.backref, {} + else: + backref_key, kwargs = self.backref + mapper = self.mapper.primary_mapper() + + check = set(mapper.iterate_to_root()).\ + union(mapper.self_and_descendants) + for m in check: + if m.has_property(backref_key): + raise sa_exc.ArgumentError("Error creating backref " + "'%s' on relationship '%s': property of that " + "name exists on mapper '%s'" % (backref_key, + self, m)) + + # determine primaryjoin/secondaryjoin for the + # backref. Use the one we had, so that + # a custom join doesn't have to be specified in + # both directions. + if self.secondary is not None: + # for many to many, just switch primaryjoin/ + # secondaryjoin. use the annotated + # pj/sj on the _join_condition. + pj = kwargs.pop('primaryjoin', + self._join_condition.secondaryjoin_minus_local) + sj = kwargs.pop('secondaryjoin', + self._join_condition.primaryjoin_minus_local) + else: + pj = kwargs.pop('primaryjoin', + self._join_condition.primaryjoin_reverse_remote) + sj = kwargs.pop('secondaryjoin', None) + if sj: + raise sa_exc.InvalidRequestError( + "Can't assign 'secondaryjoin' on a backref " + "against a non-secondary relationship." + ) + + foreign_keys = kwargs.pop('foreign_keys', + self._user_defined_foreign_keys) + parent = self.parent.primary_mapper() + kwargs.setdefault('viewonly', self.viewonly) + kwargs.setdefault('post_update', self.post_update) + kwargs.setdefault('passive_updates', self.passive_updates) + self.back_populates = backref_key + relationship = RelationshipProperty( + parent, self.secondary, + pj, sj, + foreign_keys=foreign_keys, + back_populates=self.key, + **kwargs) + mapper._configure_property(backref_key, relationship) + + if self.back_populates: + self._add_reverse_property(self.back_populates) + + def _post_init(self): + if self.uselist is None: + self.uselist = self.direction is not MANYTOONE + if not self.viewonly: + self._dependency_processor = \ + dependency.DependencyProcessor.from_relationship(self) + + @util.memoized_property + def _use_get(self): + """memoize the 'use_get' attribute of this RelationshipLoader's + lazyloader.""" + + strategy = self._lazy_strategy + return strategy.use_get + + @util.memoized_property + def _is_self_referential(self): + return self.mapper.common_parent(self.parent) + + def _create_joins(self, source_polymorphic=False, + source_selectable=None, dest_polymorphic=False, + dest_selectable=None, of_type=None): + if source_selectable is None: + if source_polymorphic and self.parent.with_polymorphic: + source_selectable = self.parent._with_polymorphic_selectable + + aliased = False + if dest_selectable is None: + if dest_polymorphic and self.mapper.with_polymorphic: + dest_selectable = self.mapper._with_polymorphic_selectable + aliased = True + else: + dest_selectable = self.mapper.mapped_table + + if self._is_self_referential and source_selectable is None: + dest_selectable = dest_selectable.alias() + aliased = True + else: + aliased = True + + dest_mapper = of_type or self.mapper + + single_crit = dest_mapper._single_table_criterion + aliased = aliased or (source_selectable is not None) + + primaryjoin, secondaryjoin, secondary, target_adapter, dest_selectable = \ + self._join_condition.join_targets( + source_selectable, dest_selectable, aliased, single_crit + ) + if source_selectable is None: + source_selectable = self.parent.local_table + if dest_selectable is None: + dest_selectable = self.mapper.local_table + return (primaryjoin, secondaryjoin, source_selectable, + dest_selectable, secondary, target_adapter) + +def _annotate_columns(element, annotations): + def clone(elem): + if isinstance(elem, expression.ColumnClause): + elem = elem._annotate(annotations.copy()) + elem._copy_internals(clone=clone) + return elem + + if element is not None: + element = clone(element) + return element + + +class JoinCondition(object): + def __init__(self, + parent_selectable, + child_selectable, + parent_local_selectable, + child_local_selectable, + primaryjoin=None, + secondary=None, + secondaryjoin=None, + parent_equivalents=None, + child_equivalents=None, + consider_as_foreign_keys=None, + local_remote_pairs=None, + remote_side=None, + self_referential=False, + prop=None, + support_sync=True, + can_be_synced_fn=lambda *c: True + ): + self.parent_selectable = parent_selectable + self.parent_local_selectable = parent_local_selectable + self.child_selectable = child_selectable + self.child_local_selectable = child_local_selectable + self.parent_equivalents = parent_equivalents + self.child_equivalents = child_equivalents + self.primaryjoin = primaryjoin + self.secondaryjoin = secondaryjoin + self.secondary = secondary + self.consider_as_foreign_keys = consider_as_foreign_keys + self._local_remote_pairs = local_remote_pairs + self._remote_side = remote_side + self.prop = prop + self.self_referential = self_referential + self.support_sync = support_sync + self.can_be_synced_fn = can_be_synced_fn + self._determine_joins() + self._annotate_fks() + self._annotate_remote() + self._annotate_local() + self._setup_pairs() + self._check_foreign_cols(self.primaryjoin, True) + if self.secondaryjoin is not None: + self._check_foreign_cols(self.secondaryjoin, False) + self._determine_direction() + self._check_remote_side() + self._log_joins() + + def _log_joins(self): + if self.prop is None: + return + log = self.prop.logger + log.info('%s setup primary join %s', self.prop, + self.primaryjoin) + log.info('%s setup secondary join %s', self.prop, + self.secondaryjoin) + log.info('%s synchronize pairs [%s]', self.prop, + ','.join('(%s => %s)' % (l, r) for (l, r) in + self.synchronize_pairs)) + log.info('%s secondary synchronize pairs [%s]', self.prop, + ','.join('(%s => %s)' % (l, r) for (l, r) in + self.secondary_synchronize_pairs or [])) + log.info('%s local/remote pairs [%s]', self.prop, + ','.join('(%s / %s)' % (l, r) for (l, r) in + self.local_remote_pairs)) + log.info('%s remote columns [%s]', self.prop, + ','.join('%s' % col for col in self.remote_columns) + ) + log.info('%s local columns [%s]', self.prop, + ','.join('%s' % col for col in self.local_columns) + ) + log.info('%s relationship direction %s', self.prop, + self.direction) + + def _determine_joins(self): + """Determine the 'primaryjoin' and 'secondaryjoin' attributes, + if not passed to the constructor already. + + This is based on analysis of the foreign key relationships + between the parent and target mapped selectables. + + """ + if self.secondaryjoin is not None and self.secondary is None: + raise sa_exc.ArgumentError( + "Property %s specified with secondary " + "join condition but " + "no secondary argument" % self.prop) + + # find a join between the given mapper's mapped table and + # the given table. will try the mapper's local table first + # for more specificity, then if not found will try the more + # general mapped table, which in the case of inheritance is + # a join. + try: + consider_as_foreign_keys = self.consider_as_foreign_keys or None + if self.secondary is not None: + if self.secondaryjoin is None: + self.secondaryjoin = \ + join_condition( + self.child_selectable, + self.secondary, + a_subset=self.child_local_selectable, + consider_as_foreign_keys=consider_as_foreign_keys + ) + if self.primaryjoin is None: + self.primaryjoin = \ + join_condition( + self.parent_selectable, + self.secondary, + a_subset=self.parent_local_selectable, + consider_as_foreign_keys=consider_as_foreign_keys + ) + else: + if self.primaryjoin is None: + self.primaryjoin = \ + join_condition( + self.parent_selectable, + self.child_selectable, + a_subset=self.parent_local_selectable, + consider_as_foreign_keys=consider_as_foreign_keys + ) + except sa_exc.NoForeignKeysError: + if self.secondary is not None: + raise sa_exc.NoForeignKeysError("Could not determine join " + "condition between parent/child tables on " + "relationship %s - there are no foreign keys " + "linking these tables via secondary table '%s'. " + "Ensure that referencing columns are associated " + "with a ForeignKey or ForeignKeyConstraint, or " + "specify 'primaryjoin' and 'secondaryjoin' " + "expressions." + % (self.prop, self.secondary)) + else: + raise sa_exc.NoForeignKeysError("Could not determine join " + "condition between parent/child tables on " + "relationship %s - there are no foreign keys " + "linking these tables. " + "Ensure that referencing columns are associated " + "with a ForeignKey or ForeignKeyConstraint, or " + "specify a 'primaryjoin' expression." + % self.prop) + except sa_exc.AmbiguousForeignKeysError: + if self.secondary is not None: + raise sa_exc.AmbiguousForeignKeysError( + "Could not determine join " + "condition between parent/child tables on " + "relationship %s - there are multiple foreign key " + "paths linking the tables via secondary table '%s'. " + "Specify the 'foreign_keys' " + "argument, providing a list of those columns which " + "should be counted as containing a foreign key " + "reference from the secondary table to each of the " + "parent and child tables." + % (self.prop, self.secondary)) + else: + raise sa_exc.AmbiguousForeignKeysError( + "Could not determine join " + "condition between parent/child tables on " + "relationship %s - there are multiple foreign key " + "paths linking the tables. Specify the " + "'foreign_keys' argument, providing a list of those " + "columns which should be counted as containing a " + "foreign key reference to the parent table." + % self.prop) + + @property + def primaryjoin_minus_local(self): + return _deep_deannotate(self.primaryjoin, values=("local", "remote")) + + @property + def secondaryjoin_minus_local(self): + return _deep_deannotate(self.secondaryjoin, values=("local", "remote")) + + @util.memoized_property + def primaryjoin_reverse_remote(self): + """Return the primaryjoin condition suitable for the + "reverse" direction. + + If the primaryjoin was delivered here with pre-existing + "remote" annotations, the local/remote annotations + are reversed. Otherwise, the local/remote annotations + are removed. + + """ + if self._has_remote_annotations: + def replace(element): + if "remote" in element._annotations: + v = element._annotations.copy() + del v['remote'] + v['local'] = True + return element._with_annotations(v) + elif "local" in element._annotations: + v = element._annotations.copy() + del v['local'] + v['remote'] = True + return element._with_annotations(v) + return visitors.replacement_traverse( + self.primaryjoin, {}, replace) + else: + if self._has_foreign_annotations: + # TODO: coverage + return _deep_deannotate(self.primaryjoin, + values=("local", "remote")) + else: + return _deep_deannotate(self.primaryjoin) + + def _has_annotation(self, clause, annotation): + for col in visitors.iterate(clause, {}): + if annotation in col._annotations: + return True + else: + return False + + @util.memoized_property + def _has_foreign_annotations(self): + return self._has_annotation(self.primaryjoin, "foreign") + + @util.memoized_property + def _has_remote_annotations(self): + return self._has_annotation(self.primaryjoin, "remote") + + def _annotate_fks(self): + """Annotate the primaryjoin and secondaryjoin + structures with 'foreign' annotations marking columns + considered as foreign. + + """ + if self._has_foreign_annotations: + return + + if self.consider_as_foreign_keys: + self._annotate_from_fk_list() + else: + self._annotate_present_fks() + + def _annotate_from_fk_list(self): + def check_fk(col): + if col in self.consider_as_foreign_keys: + return col._annotate({"foreign": True}) + self.primaryjoin = visitors.replacement_traverse( + self.primaryjoin, + {}, + check_fk + ) + if self.secondaryjoin is not None: + self.secondaryjoin = visitors.replacement_traverse( + self.secondaryjoin, + {}, + check_fk + ) + + def _annotate_present_fks(self): + if self.secondary is not None: + secondarycols = util.column_set(self.secondary.c) + else: + secondarycols = set() + + def is_foreign(a, b): + if isinstance(a, schema.Column) and \ + isinstance(b, schema.Column): + if a.references(b): + return a + elif b.references(a): + return b + + if secondarycols: + if a in secondarycols and b not in secondarycols: + return a + elif b in secondarycols and a not in secondarycols: + return b + + def visit_binary(binary): + if not isinstance(binary.left, sql.ColumnElement) or \ + not isinstance(binary.right, sql.ColumnElement): + return + + if "foreign" not in binary.left._annotations and \ + "foreign" not in binary.right._annotations: + col = is_foreign(binary.left, binary.right) + if col is not None: + if col.compare(binary.left): + binary.left = binary.left._annotate( + {"foreign": True}) + elif col.compare(binary.right): + binary.right = binary.right._annotate( + {"foreign": True}) + + self.primaryjoin = visitors.cloned_traverse( + self.primaryjoin, + {}, + {"binary": visit_binary} + ) + if self.secondaryjoin is not None: + self.secondaryjoin = visitors.cloned_traverse( + self.secondaryjoin, + {}, + {"binary": visit_binary} + ) + + def _refers_to_parent_table(self): + """Return True if the join condition contains column + comparisons where both columns are in both tables. + + """ + pt = self.parent_selectable + mt = self.child_selectable + result = [False] + + def visit_binary(binary): + c, f = binary.left, binary.right + if ( + isinstance(c, expression.ColumnClause) and \ + isinstance(f, expression.ColumnClause) and \ + pt.is_derived_from(c.table) and \ + pt.is_derived_from(f.table) and \ + mt.is_derived_from(c.table) and \ + mt.is_derived_from(f.table) + ): + result[0] = True + visitors.traverse( + self.primaryjoin, + {}, + {"binary": visit_binary} + ) + return result[0] + + def _tables_overlap(self): + """Return True if parent/child tables have some overlap.""" + + return selectables_overlap(self.parent_selectable, self.child_selectable) + + def _annotate_remote(self): + """Annotate the primaryjoin and secondaryjoin + structures with 'remote' annotations marking columns + considered as part of the 'remote' side. + + """ + if self._has_remote_annotations: + return + + if self.secondary is not None: + self._annotate_remote_secondary() + elif self._local_remote_pairs or self._remote_side: + self._annotate_remote_from_args() + elif self._refers_to_parent_table(): + self._annotate_selfref(lambda col: "foreign" in col._annotations) + elif self._tables_overlap(): + self._annotate_remote_with_overlap() + else: + self._annotate_remote_distinct_selectables() + + def _annotate_remote_secondary(self): + """annotate 'remote' in primaryjoin, secondaryjoin + when 'secondary' is present. + + """ + def repl(element): + if self.secondary.c.contains_column(element): + return element._annotate({"remote": True}) + self.primaryjoin = visitors.replacement_traverse( + self.primaryjoin, {}, repl) + self.secondaryjoin = visitors.replacement_traverse( + self.secondaryjoin, {}, repl) + + def _annotate_selfref(self, fn): + """annotate 'remote' in primaryjoin, secondaryjoin + when the relationship is detected as self-referential. + + """ + def visit_binary(binary): + equated = binary.left.compare(binary.right) + if isinstance(binary.left, expression.ColumnClause) and \ + isinstance(binary.right, expression.ColumnClause): + # assume one to many - FKs are "remote" + if fn(binary.left): + binary.left = binary.left._annotate({"remote": True}) + if fn(binary.right) and not equated: + binary.right = binary.right._annotate( + {"remote": True}) + else: + self._warn_non_column_elements() + + self.primaryjoin = visitors.cloned_traverse( + self.primaryjoin, {}, + {"binary": visit_binary}) + + def _annotate_remote_from_args(self): + """annotate 'remote' in primaryjoin, secondaryjoin + when the 'remote_side' or '_local_remote_pairs' + arguments are used. + + """ + if self._local_remote_pairs: + if self._remote_side: + raise sa_exc.ArgumentError( + "remote_side argument is redundant " + "against more detailed _local_remote_side " + "argument.") + + remote_side = [r for (l, r) in self._local_remote_pairs] + else: + remote_side = self._remote_side + + if self._refers_to_parent_table(): + self._annotate_selfref(lambda col: col in remote_side) + else: + def repl(element): + if element in remote_side: + return element._annotate({"remote": True}) + self.primaryjoin = visitors.replacement_traverse( + self.primaryjoin, {}, repl) + + def _annotate_remote_with_overlap(self): + """annotate 'remote' in primaryjoin, secondaryjoin + when the parent/child tables have some set of + tables in common, though is not a fully self-referential + relationship. + + """ + def visit_binary(binary): + binary.left, binary.right = proc_left_right(binary.left, + binary.right) + binary.right, binary.left = proc_left_right(binary.right, + binary.left) + + def proc_left_right(left, right): + if isinstance(left, expression.ColumnClause) and \ + isinstance(right, expression.ColumnClause): + if self.child_selectable.c.contains_column(right) and \ + self.parent_selectable.c.contains_column(left): + right = right._annotate({"remote": True}) + else: + self._warn_non_column_elements() + + return left, right + + self.primaryjoin = visitors.cloned_traverse( + self.primaryjoin, {}, + {"binary": visit_binary}) + + def _annotate_remote_distinct_selectables(self): + """annotate 'remote' in primaryjoin, secondaryjoin + when the parent/child tables are entirely + separate. + + """ + def repl(element): + if self.child_selectable.c.contains_column(element) and \ + ( + not self.parent_local_selectable.c.\ + contains_column(element) + or self.child_local_selectable.c.\ + contains_column(element)): + return element._annotate({"remote": True}) + self.primaryjoin = visitors.replacement_traverse( + self.primaryjoin, {}, repl) + + def _warn_non_column_elements(self): + util.warn( + "Non-simple column elements in primary " + "join condition for property %s - consider using " + "remote() annotations to mark the remote side." + % self.prop + ) + + def _annotate_local(self): + """Annotate the primaryjoin and secondaryjoin + structures with 'local' annotations. + + This annotates all column elements found + simultaneously in the parent table + and the join condition that don't have a + 'remote' annotation set up from + _annotate_remote() or user-defined. + + """ + if self._has_annotation(self.primaryjoin, "local"): + return + + if self._local_remote_pairs: + local_side = util.column_set([l for (l, r) + in self._local_remote_pairs]) + else: + local_side = util.column_set(self.parent_selectable.c) + + def locals_(elem): + if "remote" not in elem._annotations and \ + elem in local_side: + return elem._annotate({"local": True}) + self.primaryjoin = visitors.replacement_traverse( + self.primaryjoin, {}, locals_ + ) + + def _check_remote_side(self): + if not self.local_remote_pairs: + raise sa_exc.ArgumentError('Relationship %s could ' + 'not determine any unambiguous local/remote column ' + 'pairs based on join condition and remote_side ' + 'arguments. ' + 'Consider using the remote() annotation to ' + 'accurately mark those elements of the join ' + 'condition that are on the remote side of ' + 'the relationship.' + % (self.prop, )) + + def _check_foreign_cols(self, join_condition, primary): + """Check the foreign key columns collected and emit error + messages.""" + + can_sync = False + + foreign_cols = self._gather_columns_with_annotation( + join_condition, "foreign") + + has_foreign = bool(foreign_cols) + + if primary: + can_sync = bool(self.synchronize_pairs) + else: + can_sync = bool(self.secondary_synchronize_pairs) + + if self.support_sync and can_sync or \ + (not self.support_sync and has_foreign): + return + + # from here below is just determining the best error message + # to report. Check for a join condition using any operator + # (not just ==), perhaps they need to turn on "viewonly=True". + if self.support_sync and has_foreign and not can_sync: + err = "Could not locate any simple equality expressions "\ + "involving locally mapped foreign key columns for "\ + "%s join condition "\ + "'%s' on relationship %s." % ( + primary and 'primary' or 'secondary', + join_condition, + self.prop + ) + err += \ + " Ensure that referencing columns are associated "\ + "with a ForeignKey or ForeignKeyConstraint, or are "\ + "annotated in the join condition with the foreign() "\ + "annotation. To allow comparison operators other than "\ + "'==', the relationship can be marked as viewonly=True." + + raise sa_exc.ArgumentError(err) + else: + err = "Could not locate any relevant foreign key columns "\ + "for %s join condition '%s' on relationship %s." % ( + primary and 'primary' or 'secondary', + join_condition, + self.prop + ) + err += \ + ' Ensure that referencing columns are associated '\ + 'with a ForeignKey or ForeignKeyConstraint, or are '\ + 'annotated in the join condition with the foreign() '\ + 'annotation.' + raise sa_exc.ArgumentError(err) + + def _determine_direction(self): + """Determine if this relationship is one to many, many to one, + many to many. + + """ + if self.secondaryjoin is not None: + self.direction = MANYTOMANY + else: + parentcols = util.column_set(self.parent_selectable.c) + targetcols = util.column_set(self.child_selectable.c) + + # fk collection which suggests ONETOMANY. + onetomany_fk = targetcols.intersection( + self.foreign_key_columns) + + # fk collection which suggests MANYTOONE. + + manytoone_fk = parentcols.intersection( + self.foreign_key_columns) + + if onetomany_fk and manytoone_fk: + # fks on both sides. test for overlap of local/remote + # with foreign key + self_equated = self.remote_columns.intersection( + self.local_columns + ) + onetomany_local = self.remote_columns.\ + intersection(self.foreign_key_columns).\ + difference(self_equated) + manytoone_local = self.local_columns.\ + intersection(self.foreign_key_columns).\ + difference(self_equated) + if onetomany_local and not manytoone_local: + self.direction = ONETOMANY + elif manytoone_local and not onetomany_local: + self.direction = MANYTOONE + else: + raise sa_exc.ArgumentError( + "Can't determine relationship" + " direction for relationship '%s' - foreign " + "key columns within the join condition are present " + "in both the parent and the child's mapped tables. " + "Ensure that only those columns referring " + "to a parent column are marked as foreign, " + "either via the foreign() annotation or " + "via the foreign_keys argument." % self.prop) + elif onetomany_fk: + self.direction = ONETOMANY + elif manytoone_fk: + self.direction = MANYTOONE + else: + raise sa_exc.ArgumentError("Can't determine relationship " + "direction for relationship '%s' - foreign " + "key columns are present in neither the parent " + "nor the child's mapped tables" % self.prop) + + def _deannotate_pairs(self, collection): + """provide deannotation for the various lists of + pairs, so that using them in hashes doesn't incur + high-overhead __eq__() comparisons against + original columns mapped. + + """ + return [(x._deannotate(), y._deannotate()) + for x, y in collection] + + def _setup_pairs(self): + sync_pairs = [] + lrp = util.OrderedSet([]) + secondary_sync_pairs = [] + + def go(joincond, collection): + def visit_binary(binary, left, right): + if "remote" in right._annotations and \ + "remote" not in left._annotations and \ + self.can_be_synced_fn(left): + lrp.add((left, right)) + elif "remote" in left._annotations and \ + "remote" not in right._annotations and \ + self.can_be_synced_fn(right): + lrp.add((right, left)) + if binary.operator is operators.eq and \ + self.can_be_synced_fn(left, right): + if "foreign" in right._annotations: + collection.append((left, right)) + elif "foreign" in left._annotations: + collection.append((right, left)) + visit_binary_product(visit_binary, joincond) + + for joincond, collection in [ + (self.primaryjoin, sync_pairs), + (self.secondaryjoin, secondary_sync_pairs) + ]: + if joincond is None: + continue + go(joincond, collection) + + self.local_remote_pairs = self._deannotate_pairs(lrp) + self.synchronize_pairs = self._deannotate_pairs(sync_pairs) + self.secondary_synchronize_pairs = \ + self._deannotate_pairs(secondary_sync_pairs) + + @util.memoized_property + def remote_columns(self): + return self._gather_join_annotations("remote") + + @util.memoized_property + def local_columns(self): + return self._gather_join_annotations("local") + + @util.memoized_property + def foreign_key_columns(self): + return self._gather_join_annotations("foreign") + + @util.memoized_property + def deannotated_primaryjoin(self): + return _deep_deannotate(self.primaryjoin) + + @util.memoized_property + def deannotated_secondaryjoin(self): + if self.secondaryjoin is not None: + return _deep_deannotate(self.secondaryjoin) + else: + return None + + def _gather_join_annotations(self, annotation): + s = set( + self._gather_columns_with_annotation( + self.primaryjoin, annotation) + ) + if self.secondaryjoin is not None: + s.update( + self._gather_columns_with_annotation( + self.secondaryjoin, annotation) + ) + return set([x._deannotate() for x in s]) + + def _gather_columns_with_annotation(self, clause, *annotation): + annotation = set(annotation) + return set([ + col for col in visitors.iterate(clause, {}) + if annotation.issubset(col._annotations) + ]) + + def join_targets(self, source_selectable, + dest_selectable, + aliased, + single_crit=None): + """Given a source and destination selectable, create a + join between them. + + This takes into account aliasing the join clause + to reference the appropriate corresponding columns + in the target objects, as well as the extra child + criterion, equivalent column sets, etc. + + """ + + # place a barrier on the destination such that + # replacement traversals won't ever dig into it. + # its internal structure remains fixed + # regardless of context. + dest_selectable = _shallow_annotate( + dest_selectable, + {'no_replacement_traverse': True}) + + primaryjoin, secondaryjoin, secondary = self.primaryjoin, \ + self.secondaryjoin, self.secondary + + # adjust the join condition for single table inheritance, + # in the case that the join is to a subclass + # this is analogous to the + # "_adjust_for_single_table_inheritance()" method in Query. + + if single_crit is not None: + if secondaryjoin is not None: + secondaryjoin = secondaryjoin & single_crit + else: + primaryjoin = primaryjoin & single_crit + + if aliased: + if secondary is not None: + secondary = secondary.alias() + primary_aliasizer = ClauseAdapter(secondary) + secondary_aliasizer = \ + ClauseAdapter(dest_selectable, + equivalents=self.child_equivalents).\ + chain(primary_aliasizer) + if source_selectable is not None: + primary_aliasizer = \ + ClauseAdapter(secondary).\ + chain(ClauseAdapter(source_selectable, + equivalents=self.parent_equivalents)) + secondaryjoin = \ + secondary_aliasizer.traverse(secondaryjoin) + else: + primary_aliasizer = ClauseAdapter(dest_selectable, + exclude_fn=_ColInAnnotations("local"), + equivalents=self.child_equivalents) + if source_selectable is not None: + primary_aliasizer.chain( + ClauseAdapter(source_selectable, + exclude_fn=_ColInAnnotations("remote"), + equivalents=self.parent_equivalents)) + secondary_aliasizer = None + + primaryjoin = primary_aliasizer.traverse(primaryjoin) + target_adapter = secondary_aliasizer or primary_aliasizer + target_adapter.exclude_fn = None + else: + target_adapter = None + return primaryjoin, secondaryjoin, secondary, \ + target_adapter, dest_selectable + + def create_lazy_clause(self, reverse_direction=False): + binds = util.column_dict() + lookup = util.column_dict() + equated_columns = util.column_dict() + + if reverse_direction and self.secondaryjoin is None: + for l, r in self.local_remote_pairs: + _list = lookup.setdefault(r, []) + _list.append((r, l)) + equated_columns[l] = r + else: + for l, r in self.local_remote_pairs: + _list = lookup.setdefault(l, []) + _list.append((l, r)) + equated_columns[r] = l + + def col_to_bind(col): + if col in lookup: + for tobind, equated in lookup[col]: + if equated in binds: + return None + if col not in binds: + binds[col] = sql.bindparam( + None, None, type_=col.type, unique=True) + return binds[col] + return None + + lazywhere = self.deannotated_primaryjoin + + if self.deannotated_secondaryjoin is None or not reverse_direction: + lazywhere = visitors.replacement_traverse( + lazywhere, {}, col_to_bind) + + if self.deannotated_secondaryjoin is not None: + secondaryjoin = self.deannotated_secondaryjoin + if reverse_direction: + secondaryjoin = visitors.replacement_traverse( + secondaryjoin, {}, col_to_bind) + lazywhere = sql.and_(lazywhere, secondaryjoin) + + bind_to_col = dict((binds[col].key, col) for col in binds) + + return lazywhere, bind_to_col, equated_columns + +class _ColInAnnotations(object): + """Seralizable equivalent to: + + lambda c: "name" in c._annotations + """ + def __init__(self, name): + self.name = name + + def __call__(self, c): + return self.name in c._annotations diff --git a/libs/sqlalchemy/orm/scoping.py b/libs/sqlalchemy/orm/scoping.py index b5bd65b2..c1f8f319 100644 --- a/libs/sqlalchemy/orm/scoping.py +++ b/libs/sqlalchemy/orm/scoping.py @@ -1,67 +1,98 @@ # orm/scoping.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy import exc as sa_exc -from sqlalchemy.util import ScopedRegistry, ThreadLocalRegistry, warn -from sqlalchemy.orm import class_mapper -from sqlalchemy.orm import exc as orm_exc -from sqlalchemy.orm.session import Session +from .. import exc as sa_exc +from ..util import ScopedRegistry, ThreadLocalRegistry, warn +from . import class_mapper, exc as orm_exc +from .session import Session -__all__ = ['ScopedSession'] +__all__ = ['scoped_session'] -class ScopedSession(object): - """Provides thread-local management of Sessions. +class scoped_session(object): + """Provides scoped management of :class:`.Session` objects. - Typical invocation is via the :func:`.scoped_session` - function:: - - Session = scoped_session(sessionmaker()) - - The internal registry is accessible, - and by default is an instance of :class:`.ThreadLocalRegistry`. - - See also: :ref:`unitofwork_contextual`. + See :ref:`unitofwork_contextual` for a tutorial. """ def __init__(self, session_factory, scopefunc=None): + """Construct a new :class:`.scoped_session`. + + :param session_factory: a factory to create new :class:`.Session` + instances. This is usually, but not necessarily, an instance + of :class:`.sessionmaker`. + :param scopefunc: optional function which defines + the current scope. If not passed, the :class:`.scoped_session` + object assumes "thread-local" scope, and will use + a Python ``threading.local()`` in order to maintain the current + :class:`.Session`. If passed, the function should return + a hashable token; this token will be used as the key in a + dictionary in order to store and retrieve the current + :class:`.Session`. + + """ self.session_factory = session_factory if scopefunc: self.registry = ScopedRegistry(session_factory, scopefunc) else: self.registry = ThreadLocalRegistry(session_factory) - def __call__(self, **kwargs): - if kwargs: - scope = kwargs.pop('scope', False) + def __call__(self, **kw): + """Return the current :class:`.Session`, creating it + using the session factory if not present. + + :param \**kw: Keyword arguments will be passed to the + session factory callable, if an existing :class:`.Session` + is not present. If the :class:`.Session` is present and + keyword arguments have been passed, + :exc:`~sqlalchemy.exc.InvalidRequestError` is raised. + + """ + if kw: + scope = kw.pop('scope', False) if scope is not None: if self.registry.has(): raise sa_exc.InvalidRequestError( "Scoped session is already present; " "no new arguments may be specified.") else: - sess = self.session_factory(**kwargs) + sess = self.session_factory(**kw) self.registry.set(sess) return sess else: - return self.session_factory(**kwargs) + return self.session_factory(**kw) else: return self.registry() def remove(self): - """Dispose of the current contextual session.""" + """Dispose of the current :class:`.Session`, if present. + + This will first call :meth:`.Session.close` method + on the current :class:`.Session`, which releases any existing + transactional/connection resources still being held; transactions + specifically are rolled back. The :class:`.Session` is then + discarded. Upon next usage within the same scope, + the :class:`.scoped_session` will produce a new + :class:`.Session` object. + + """ if self.registry.has(): self.registry().close() self.registry.clear() def configure(self, **kwargs): - """reconfigure the sessionmaker used by this ScopedSession.""" + """reconfigure the :class:`.sessionmaker` used by this + :class:`.scoped_session`. + + See :meth:`.sessionmaker.configure`. + + """ if self.registry.has(): warn('At least one scoped session is already present. ' @@ -71,8 +102,8 @@ class ScopedSession(object): self.session_factory.configure(**kwargs) def query_property(self, query_cls=None): - """return a class property which produces a `Query` object - against the class when called. + """return a class property which produces a :class:`.Query` object + against the class and the current :class:`.Session` when called. e.g.:: @@ -109,27 +140,37 @@ class ScopedSession(object): return None return query() +ScopedSession = scoped_session +"""Old name for backwards compatibility.""" + + def instrument(name): def do(self, *args, **kwargs): return getattr(self.registry(), name)(*args, **kwargs) return do + for meth in Session.public_methods: - setattr(ScopedSession, meth, instrument(meth)) + setattr(scoped_session, meth, instrument(meth)) + def makeprop(name): def set(self, attr): setattr(self.registry(), name, attr) + def get(self): return getattr(self.registry(), name) + return property(get, set) + for prop in ('bind', 'dirty', 'deleted', 'new', 'identity_map', - 'is_active', 'autoflush', 'no_autoflush'): - setattr(ScopedSession, prop, makeprop(prop)) + 'is_active', 'autoflush', 'no_autoflush', 'info'): + setattr(scoped_session, prop, makeprop(prop)) + def clslevel(name): def do(cls, *args, **kwargs): return getattr(Session, name)(*args, **kwargs) return classmethod(do) -for prop in ('close_all', 'object_session', 'identity_key'): - setattr(ScopedSession, prop, clslevel(prop)) +for prop in ('close_all', 'object_session', 'identity_key'): + setattr(scoped_session, prop, clslevel(prop)) diff --git a/libs/sqlalchemy/orm/session.py b/libs/sqlalchemy/orm/session.py index 8994a339..c10a0efc 100644 --- a/libs/sqlalchemy/orm/session.py +++ b/libs/sqlalchemy/orm/session.py @@ -1,106 +1,84 @@ # orm/session.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php - """Provides the Session class and related utilities.""" -import weakref -from itertools import chain -from sqlalchemy import util, sql, engine, log, exc as sa_exc -from sqlalchemy.sql import util as sql_util, expression -from sqlalchemy.orm import ( - SessionExtension, attributes, exc, query, unitofwork, util as mapperutil, state - ) -from sqlalchemy.orm.util import object_mapper as _object_mapper -from sqlalchemy.orm.util import class_mapper as _class_mapper -from sqlalchemy.orm.util import ( - _class_to_mapper, _state_mapper, - ) -from sqlalchemy.orm.mapper import Mapper, _none_set -from sqlalchemy.orm.unitofwork import UOWTransaction -from sqlalchemy.orm import identity -from sqlalchemy import event -from sqlalchemy.orm.events import SessionEvents + +import weakref +from .. import util, sql, engine, exc as sa_exc +from ..sql import util as sql_util, expression +from . import ( + SessionExtension, attributes, exc, query, + loading, identity + ) +from ..inspection import inspect +from .base import ( + object_mapper, class_mapper, + _class_to_mapper, _state_mapper, object_state, + _none_set, state_str, instance_str + ) +from .unitofwork import UOWTransaction +from . import state as statelib import sys -__all__ = ['Session', 'SessionTransaction', 'SessionExtension'] +__all__ = ['Session', 'SessionTransaction', 'SessionExtension', 'sessionmaker'] +_sessions = weakref.WeakValueDictionary() +"""Weak-referencing dictionary of :class:`.Session` objects. +""" -def sessionmaker(bind=None, class_=None, autoflush=True, autocommit=False, - expire_on_commit=True, **kwargs): - """Generate a custom-configured :class:`.Session` class. - - The returned object is a subclass of :class:`.Session`, which, when instantiated - with no arguments, uses the keyword arguments configured here as its - constructor arguments. - - It is intended that the :func:`.sessionmaker()` function be called within the - global scope of an application, and the returned class be made available - to the rest of the application as the single class used to instantiate - sessions. - - e.g.:: - - # global scope - Session = sessionmaker(autoflush=False) - - # later, in a local scope, create and use a session: - sess = Session() - - Any keyword arguments sent to the constructor itself will override the - "configured" keywords:: - - Session = sessionmaker() - - # bind an individual session to a connection - sess = Session(bind=connection) - - The class also includes a special classmethod ``configure()``, which - allows additional configurational options to take place after the custom - ``Session`` class has been generated. This is useful particularly for - defining the specific ``Engine`` (or engines) to which new instances of - ``Session`` should be bound:: - - Session = sessionmaker() - Session.configure(bind=create_engine('sqlite:///foo.db')) - - sess = Session() - - For options, see the constructor options for :class:`.Session`. - +def _state_session(state): + """Given an :class:`.InstanceState`, return the :class:`.Session` + associated, if any. """ - kwargs['bind'] = bind - kwargs['autoflush'] = autoflush - kwargs['autocommit'] = autocommit - kwargs['expire_on_commit'] = expire_on_commit - - if class_ is None: - class_ = Session - - class Sess(object): - def __init__(self, **local_kwargs): - for k in kwargs: - local_kwargs.setdefault(k, kwargs[k]) - super(Sess, self).__init__(**local_kwargs) - - @classmethod - def configure(self, **new_kwargs): - """(Re)configure the arguments for this sessionmaker. - - e.g.:: - - Session = sessionmaker() - - Session.configure(bind=create_engine('sqlite://')) - """ - kwargs.update(new_kwargs) + if state.session_id: + try: + return _sessions[state.session_id] + except KeyError: + pass + return None - return type("SessionMaker", (Sess, class_), {}) +class _SessionClassMethods(object): + """Class-level methods for :class:`.Session`, :class:`.sessionmaker`.""" + + @classmethod + def close_all(cls): + """Close *all* sessions in memory.""" + + for sess in _sessions.values(): + sess.close() + + @classmethod + @util.dependencies("sqlalchemy.orm.util") + def identity_key(cls, orm_util, *args, **kwargs): + """Return an identity key. + + This is an alias of :func:`.util.identity_key`. + + """ + return orm_util.identity_key(*args, **kwargs) + + @classmethod + def object_session(cls, instance): + """Return the :class:`.Session` to which an object belongs. + + This is an alias of :func:`.object_session`. + + """ + + return object_session(instance) + + +ACTIVE = util.symbol('ACTIVE') +PREPARED = util.symbol('PREPARED') +COMMITTED = util.symbol('COMMITTED') +DEACTIVE = util.symbol('DEACTIVE') +CLOSED = util.symbol('CLOSED') class SessionTransaction(object): """A :class:`.Session`-level transaction. @@ -112,52 +90,55 @@ class SessionTransaction(object): back all at once. It also provides optional two-phase commit behavior which can augment this coordination operation. - The :attr:`.Session.transaction` attribute of :class:`.Session` refers to the - current :class:`.SessionTransaction` object in use, if any. + The :attr:`.Session.transaction` attribute of :class:`.Session` + refers to the current :class:`.SessionTransaction` object in use, if any. A :class:`.SessionTransaction` is associated with a :class:`.Session` in its default mode of ``autocommit=False`` immediately, associated with no database connections. As the :class:`.Session` is called upon to emit SQL on behalf of various :class:`.Engine` or :class:`.Connection` - objects, a corresponding :class:`.Connection` and associated :class:`.Transaction` - is added to a collection within the :class:`.SessionTransaction` object, - becoming one of the connection/transaction pairs maintained by the + objects, a corresponding :class:`.Connection` and associated + :class:`.Transaction` is added to a collection within the + :class:`.SessionTransaction` object, becoming one of the + connection/transaction pairs maintained by the :class:`.SessionTransaction`. The lifespan of the :class:`.SessionTransaction` ends when the - :meth:`.Session.commit`, :meth:`.Session.rollback` or :meth:`.Session.close` - methods are called. At this point, the :class:`.SessionTransaction` removes - its association with its parent :class:`.Session`. A :class:`.Session` - that is in ``autocommit=False`` mode will create a new - :class:`.SessionTransaction` to replace it immediately, whereas a - :class:`.Session` that's in ``autocommit=True`` + :meth:`.Session.commit`, :meth:`.Session.rollback` or + :meth:`.Session.close` methods are called. At this point, the + :class:`.SessionTransaction` removes its association with its parent + :class:`.Session`. A :class:`.Session` that is in ``autocommit=False`` + mode will create a new :class:`.SessionTransaction` to replace it + immediately, whereas a :class:`.Session` that's in ``autocommit=True`` mode will remain without a :class:`.SessionTransaction` until the :meth:`.Session.begin` method is called. Another detail of :class:`.SessionTransaction` behavior is that it is - capable of "nesting". This means that the :meth:`.begin` method can - be called while an existing :class:`.SessionTransaction` is already present, - producing a new :class:`.SessionTransaction` that temporarily replaces - the parent :class:`.SessionTransaction`. When a :class:`.SessionTransaction` - is produced as nested, it assigns itself to the :attr:`.Session.transaction` - attribute. When it is ended via :meth:`.Session.commit` or :meth:`.Session.rollback`, - it restores its parent :class:`.SessionTransaction` back onto the - :attr:`.Session.transaction` attribute. The - behavior is effectively a stack, where :attr:`.Session.transaction` refers - to the current head of the stack. + capable of "nesting". This means that the :meth:`.Session.begin` method + can be called while an existing :class:`.SessionTransaction` is already + present, producing a new :class:`.SessionTransaction` that temporarily + replaces the parent :class:`.SessionTransaction`. When a + :class:`.SessionTransaction` is produced as nested, it assigns itself to + the :attr:`.Session.transaction` attribute. When it is ended via + :meth:`.Session.commit` or :meth:`.Session.rollback`, it restores its + parent :class:`.SessionTransaction` back onto the + :attr:`.Session.transaction` attribute. The behavior is effectively a + stack, where :attr:`.Session.transaction` refers to the current head of + the stack. - The purpose of this stack is to allow nesting of :meth:`.rollback` or - :meth:`.commit` calls in context with various flavors of :meth:`.begin`. - This nesting behavior applies to when :meth:`.Session.begin_nested` - is used to emit a SAVEPOINT transaction, and is also used to produce - a so-called "subtransaction" which allows a block of code to use a + The purpose of this stack is to allow nesting of + :meth:`.Session.rollback` or :meth:`.Session.commit` calls in context + with various flavors of :meth:`.Session.begin`. This nesting behavior + applies to when :meth:`.Session.begin_nested` is used to emit a + SAVEPOINT transaction, and is also used to produce a so-called + "subtransaction" which allows a block of code to use a begin/rollback/commit sequence regardless of whether or not its enclosing - code block has begun a transaction. The :meth:`.flush` method, whether called - explicitly or via autoflush, is the primary consumer of the "subtransaction" - feature, in that it wishes to guarantee that it works within in a transaction block - regardless of whether or not the :class:`.Session` is in transactional mode - when the method is called. + code block has begun a transaction. The :meth:`.flush` method, whether + called explicitly or via autoflush, is the primary consumer of the + "subtransaction" feature, in that it wishes to guarantee that it works + within in a transaction block regardless of whether or not the + :class:`.Session` is in transactional mode when the method is called. See also: @@ -186,8 +167,7 @@ class SessionTransaction(object): self._connections = {} self._parent = parent self.nested = nested - self._active = True - self._prepared = False + self._state = ACTIVE if not parent and nested: raise sa_exc.InvalidRequestError( "Can't start a SAVEPOINT transaction when no existing " @@ -196,44 +176,59 @@ class SessionTransaction(object): if self.session._enable_transaction_accounting: self._take_snapshot() + if self.session.dispatch.after_transaction_create: + self.session.dispatch.after_transaction_create(self.session, self) + @property def is_active(self): - return self.session is not None and self._active + return self.session is not None and self._state is ACTIVE - def _assert_is_active(self): - self._assert_is_open() - if not self._active: - if self._rollback_exception: - raise sa_exc.InvalidRequestError( - "This Session's transaction has been rolled back " - "due to a previous exception during flush." - " To begin a new transaction with this Session, " - "first issue Session.rollback()." - " Original exception was: %s" - % self._rollback_exception + def _assert_active(self, prepared_ok=False, + rollback_ok=False, + deactive_ok=False, + closed_msg="This transaction is closed"): + if self._state is COMMITTED: + raise sa_exc.InvalidRequestError( + "This session is in 'committed' state; no further " + "SQL can be emitted within this transaction." ) - else: + elif self._state is PREPARED: + if not prepared_ok: raise sa_exc.InvalidRequestError( - "This Session's transaction has been rolled back " - "by a nested rollback() call. To begin a new " - "transaction, issue Session.rollback() first." + "This session is in 'prepared' state; no further " + "SQL can be emitted within this transaction." ) - - def _assert_is_open(self, error_msg="The transaction is closed"): - if self.session is None: - raise sa_exc.ResourceClosedError(error_msg) + elif self._state is DEACTIVE: + if not deactive_ok and not rollback_ok: + if self._rollback_exception: + raise sa_exc.InvalidRequestError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + " Original exception was: %s" + % self._rollback_exception + ) + elif not deactive_ok: + raise sa_exc.InvalidRequestError( + "This Session's transaction has been rolled back " + "by a nested rollback() call. To begin a new " + "transaction, issue Session.rollback() first." + ) + elif self._state is CLOSED: + raise sa_exc.ResourceClosedError(closed_msg) @property def _is_transaction_boundary(self): return self.nested or not self._parent def connection(self, bindkey, **kwargs): - self._assert_is_active() - engine = self.session.get_bind(bindkey, **kwargs) - return self._connection_for_bind(engine) + self._assert_active() + bind = self.session.get_bind(bindkey, **kwargs) + return self._connection_for_bind(bind) def _begin(self, nested=False): - self._assert_is_active() + self._assert_active() return SessionTransaction( self.session, self, nested=nested) @@ -251,6 +246,7 @@ class SessionTransaction(object): if not self._is_transaction_boundary: self._new = self._parent._new self._deleted = self._parent._deleted + self._dirty = self._parent._dirty self._key_switches = self._parent._key_switches return @@ -259,9 +255,10 @@ class SessionTransaction(object): self._new = weakref.WeakKeyDictionary() self._deleted = weakref.WeakKeyDictionary() + self._dirty = weakref.WeakKeyDictionary() self._key_switches = weakref.WeakKeyDictionary() - def _restore_snapshot(self): + def _restore_snapshot(self, dirty_only=False): assert self._is_transaction_boundary for s in set(self._new).union(self.session._new): @@ -283,17 +280,22 @@ class SessionTransaction(object): assert not self.session._deleted for s in self.session.identity_map.all_states(): - s.expire(s.dict, self.session.identity_map._modified) + if not dirty_only or s.modified or s in self._dirty: + s._expire(s.dict, self.session.identity_map._modified) def _remove_snapshot(self): assert self._is_transaction_boundary if not self.nested and self.session.expire_on_commit: for s in self.session.identity_map.all_states(): - s.expire(s.dict, self.session.identity_map._modified) + s._expire(s.dict, self.session.identity_map._modified) + for s in self._deleted: + s.session_id = None + self._deleted.clear() + def _connection_for_bind(self, bind): - self._assert_is_active() + self._assert_active() if bind in self._connections: return self._connections[bind][0] @@ -327,11 +329,12 @@ class SessionTransaction(object): def prepare(self): if self._parent is not None or not self.session.twophase: raise sa_exc.InvalidRequestError( - "Only root two phase transactions of can be prepared") + "'twophase' mode not enabled, or not root transaction; " + "can't prepare.") self._prepare_impl() def _prepare_impl(self): - self._assert_is_active() + self._assert_active() if self._parent is None or self.nested: self.session.dispatch.before_commit(self.session) @@ -341,7 +344,7 @@ class SessionTransaction(object): subtransaction.commit() if not self.session._flushing: - for _flush_guard in xrange(100): + for _flush_guard in range(100): if self.session._is_clean(): break self.session.flush() @@ -356,21 +359,21 @@ class SessionTransaction(object): for t in set(self._connections.values()): t[1].prepare() except: - self.rollback() - raise + with util.safe_reraise(): + self.rollback() - self._deactivate() - self._prepared = True + self._state = PREPARED def commit(self): - self._assert_is_open() - if not self._prepared: + self._assert_active(prepared_ok=True) + if self._state is not PREPARED: self._prepare_impl() if self._parent is None or self.nested: for t in set(self._connections.values()): t[1].commit() + self._state = COMMITTED self.session.dispatch.after_commit(self.session) if self.session._enable_transaction_accounting: @@ -380,33 +383,33 @@ class SessionTransaction(object): return self._parent def rollback(self, _capture_exception=False): - self._assert_is_open() + self._assert_active(prepared_ok=True, rollback_ok=True) stx = self.session.transaction if stx is not self: for subtransaction in stx._iterate_parents(upto=self): subtransaction.close() - if self.is_active or self._prepared: + if self._state in (ACTIVE, PREPARED): for transaction in self._iterate_parents(): if transaction._parent is None or transaction.nested: transaction._rollback_impl() - transaction._deactivate() + transaction._state = DEACTIVE break else: - transaction._deactivate() + transaction._state = DEACTIVE sess = self.session if self.session._enable_transaction_accounting and \ - not sess._is_clean(): + not sess._is_clean(): # if items were added, deleted, or mutated # here, we need to re-restore the snapshot util.warn( "Session's state has been changed on " "a non-active transaction - this state " "will be discarded.") - self._restore_snapshot() + self._restore_snapshot(dirty_only=self.nested) self.close() if self._parent and _capture_exception: @@ -421,25 +424,27 @@ class SessionTransaction(object): t[1].rollback() if self.session._enable_transaction_accounting: - self._restore_snapshot() + self._restore_snapshot(dirty_only=self.nested) self.session.dispatch.after_rollback(self.session) - def _deactivate(self): - self._active = False - def close(self): self.session.transaction = self._parent if self._parent is None: for connection, transaction, autoclose in \ - set(self._connections.values()): + set(self._connections.values()): if autoclose: connection.close() else: transaction.close() + + self._state = CLOSED + if self.session.dispatch.after_transaction_end: + self.session.dispatch.after_transaction_end(self.session, self) + + if self._parent is None: if not self.session.autocommit: self.session.begin() - self._deactivate() self.session = None self._connections = None @@ -447,23 +452,23 @@ class SessionTransaction(object): return self def __exit__(self, type, value, traceback): - self._assert_is_open("Cannot end transaction context. The transaction " - "was closed from within the context") + self._assert_active(deactive_ok=True, prepared_ok=True) if self.session.transaction is None: return if type is None: try: self.commit() except: - self.rollback() - raise + with util.safe_reraise(): + self.rollback() else: self.rollback() -class Session(object): + +class Session(_SessionClassMethods): """Manages persistence operations for ORM-mapped objects. - The Session's usage paradigm is described at :ref:`session_toplevel`. + The Session's usage paradigm is described at :doc:`/orm/session`. """ @@ -476,54 +481,65 @@ class Session(object): 'merge', 'query', 'refresh', 'rollback', 'scalar') - def __init__(self, bind=None, autoflush=True, expire_on_commit=True, _enable_transaction_accounting=True, autocommit=False, twophase=False, weak_identity_map=True, binds=None, extension=None, + info=None, query_cls=query.Query): """Construct a new Session. - See also the :func:`.sessionmaker` function which is used to + See also the :class:`.sessionmaker` function which is used to generate a :class:`.Session`-producing callable with a given set of arguments. - :param autocommit: Defaults to ``False``. When ``True``, the ``Session`` - does not keep a persistent transaction running, and will acquire - connections from the engine on an as-needed basis, returning them - immediately after their use. Flushes will begin and commit (or possibly - rollback) their own transaction if no transaction is present. When using - this mode, the `session.begin()` method may be used to begin a - transaction explicitly. + :param autocommit: - Leaving it on its default value of ``False`` means that the ``Session`` - will acquire a connection and begin a transaction the first time it is - used, which it will maintain persistently until ``rollback()``, - ``commit()``, or ``close()`` is called. When the transaction is released - by any of these methods, the ``Session`` is ready for the next usage, - which will again acquire and maintain a new connection/transaction. + .. warning:: + + The autocommit flag is **not for general use**, and if it is used, + queries should only be invoked within the span of a + :meth:`.Session.begin` / :meth:`.Session.commit` pair. Executing + queries outside of a demarcated transaction is a legacy mode + of usage, and can in some cases lead to concurrent connection + checkouts. + + Defaults to ``False``. When ``True``, the + :class:`.Session` does not keep a persistent transaction running, and + will acquire connections from the engine on an as-needed basis, + returning them immediately after their use. Flushes will begin and + commit (or possibly rollback) their own transaction if no + transaction is present. When using this mode, the + :meth:`.Session.begin` method is used to explicitly start + transactions. + + .. seealso:: + + :ref:`session_autocommit` :param autoflush: When ``True``, all query operations will issue a ``flush()`` call to this ``Session`` before proceeding. This is a - convenience feature so that ``flush()`` need not be called repeatedly - in order for database queries to retrieve results. It's typical that - ``autoflush`` is used in conjunction with ``autocommit=False``. In this - scenario, explicit calls to ``flush()`` are rarely needed; you usually - only need to call ``commit()`` (which flushes) to finalize changes. + convenience feature so that ``flush()`` need not be called + repeatedly in order for database queries to retrieve results. It's + typical that ``autoflush`` is used in conjunction with + ``autocommit=False``. In this scenario, explicit calls to + ``flush()`` are rarely needed; you usually only need to call + ``commit()`` (which flushes) to finalize changes. :param bind: An optional ``Engine`` or ``Connection`` to which this ``Session`` should be bound. When specified, all SQL operations performed by this session will execute via this connectable. - :param binds: An optional dictionary which contains more granular "bind" - information than the ``bind`` parameter provides. This dictionary can - map individual ``Table`` instances as well as ``Mapper`` instances to - individual ``Engine`` or ``Connection`` objects. Operations which - proceed relative to a particular ``Mapper`` will consult this - dictionary for the direct ``Mapper`` instance as well as the mapper's - ``mapped_table`` attribute in order to locate an connectable to use. - The full resolution is described in the ``get_bind()`` method of - ``Session``. Usage looks like:: + :param binds: An optional dictionary which contains more granular + "bind" information than the ``bind`` parameter provides. This + dictionary can map individual ``Table`` instances as well as + ``Mapper`` instances to individual ``Engine`` or ``Connection`` + objects. Operations which proceed relative to a particular + ``Mapper`` will consult this dictionary for the direct ``Mapper`` + instance as well as the mapper's ``mapped_table`` attribute in + order to locate an connectable to use. The full resolution is + described in the ``get_bind()`` method of ``Session``. + Usage looks like:: Session = sessionmaker(binds={ SomeMappedClass: create_engine('postgresql://engine1'), @@ -531,42 +547,51 @@ class Session(object): some_table: create_engine('postgresql://engine3'), }) - Also see the :meth:`.Session.bind_mapper` and :meth:`.Session.bind_table` methods. + Also see the :meth:`.Session.bind_mapper` + and :meth:`.Session.bind_table` methods. :param \class_: Specify an alternate class other than - ``sqlalchemy.orm.session.Session`` which should be used by the returned - class. This is the only argument that is local to the + ``sqlalchemy.orm.session.Session`` which should be used by the + returned class. This is the only argument that is local to the ``sessionmaker()`` function, and is not sent directly to the constructor for ``Session``. :param _enable_transaction_accounting: Defaults to ``True``. A - legacy-only flag which when ``False`` disables *all* 0.5-style object - accounting on transaction boundaries, including auto-expiry of - instances on rollback and commit, maintenance of the "new" and + legacy-only flag which when ``False`` disables *all* 0.5-style + object accounting on transaction boundaries, including auto-expiry + of instances on rollback and commit, maintenance of the "new" and "deleted" lists upon rollback, and autoflush of pending changes upon begin(), all of which are interdependent. :param expire_on_commit: Defaults to ``True``. When ``True``, all - instances will be fully expired after each ``commit()``, so that all - attribute/object access subsequent to a completed transaction will load - from the most recent database state. + instances will be fully expired after each ``commit()``, so that + all attribute/object access subsequent to a completed transaction + will load from the most recent database state. :param extension: An optional :class:`~.SessionExtension` instance, or a list - of such instances, which will receive pre- and post- commit and flush - events, as well as a post-rollback event. **Deprecated.** + of such instances, which will receive pre- and post- commit and + flush events, as well as a post-rollback event. **Deprecated.** Please see :class:`.SessionEvents`. - :param query_cls: Class which should be used to create new Query objects, - as returned by the ``query()`` method. Defaults to + :param info: optional dictionary of arbitrary data to be associated + with this :class:`.Session`. Is available via the :attr:`.Session.info` + attribute. Note the dictionary is copied at construction time so + that modifications to the per-:class:`.Session` dictionary will be local + to that :class:`.Session`. + + .. versionadded:: 0.9.0 + + :param query_cls: Class which should be used to create new Query + objects, as returned by the ``query()`` method. Defaults to :class:`~sqlalchemy.orm.query.Query`. :param twophase: When ``True``, all transactions will be started as a "two phase" transaction, i.e. using the "two phase" semantics of the database in use along with an XID. During a ``commit()``, after ``flush()`` has been issued for all attached databases, the - ``prepare()`` method on each database's ``TwoPhaseTransaction`` will - be called. This allows each database to roll back the entire + ``prepare()`` method on each database's ``TwoPhaseTransaction`` + will be called. This allows each database to roll back the entire transaction, before each transaction is committed. :param weak_identity_map: Defaults to ``True`` - when set to @@ -590,6 +615,7 @@ class Session(object): self.bind = bind self.__binds = {} self._flushing = False + self._warn_on_events = False self.transaction = None self.hash_key = _new_sessionid() self.autoflush = autoflush @@ -598,29 +624,48 @@ class Session(object): self._enable_transaction_accounting = _enable_transaction_accounting self.twophase = twophase self._query_cls = query_cls + if info: + self.info.update(info) if extension: for ext in util.to_list(extension): SessionExtension._adapt_listener(self, ext) if binds is not None: - for mapperortable, bind in binds.iteritems(): - if isinstance(mapperortable, (type, Mapper)): + for mapperortable, bind in binds.items(): + insp = inspect(mapperortable) + if insp.is_selectable: + self.bind_table(mapperortable, bind) + elif insp.is_mapper: self.bind_mapper(mapperortable, bind) else: - self.bind_table(mapperortable, bind) + assert False + if not self.autocommit: self.begin() _sessions[self.hash_key] = self - dispatch = event.dispatcher(SessionEvents) - connection_callable = None transaction = None """The current active or inactive :class:`.SessionTransaction`.""" + @util.memoized_property + def info(self): + """A user-modifiable dictionary. + + The initial value of this dictioanry can be populated using the + ``info`` argument to the :class:`.Session` constructor or + :class:`.sessionmaker` constructor or factory methods. The dictionary + here is always local to this :class:`.Session` and can be modified + independently of all other :class:`.Session` objects. + + .. versionadded:: 0.9.0 + + """ + return {} + def begin(self, subtransactions=False, nested=False): """Begin a transaction on this Session. @@ -628,13 +673,14 @@ class Session(object): transaction or nested transaction, an error is raised, unless ``subtransactions=True`` or ``nested=True`` is specified. - The ``subtransactions=True`` flag indicates that this :meth:`~.Session.begin` - can create a subtransaction if a transaction is already in progress. - For documentation on subtransactions, please see :ref:`session_subtransactions`. + The ``subtransactions=True`` flag indicates that this + :meth:`~.Session.begin` can create a subtransaction if a transaction + is already in progress. For documentation on subtransactions, please + see :ref:`session_subtransactions`. The ``nested`` flag begins a SAVEPOINT transaction and is equivalent - to calling :meth:`~.Session.begin_nested`. For documentation on SAVEPOINT - transactions, please see :ref:`session_begin_nested`. + to calling :meth:`~.Session.begin_nested`. For documentation on + SAVEPOINT transactions, please see :ref:`session_begin_nested`. """ if self.transaction is not None: @@ -643,8 +689,8 @@ class Session(object): nested=nested) else: raise sa_exc.InvalidRequestError( - "A transaction is already begun. Use subtransactions=True " - "to allow subtransactions.") + "A transaction is already begun. Use " + "subtransactions=True to allow subtransactions.") else: self.transaction = SessionTransaction( self, nested=nested) @@ -672,6 +718,10 @@ class Session(object): to the first real transaction are closed. Subtransactions occur when begin() is called multiple times. + .. seealso:: + + :ref:`session_rollback` + """ if self.transaction is None: pass @@ -682,24 +732,29 @@ class Session(object): """Flush pending changes and commit the current transaction. If no transaction is in progress, this method raises an - InvalidRequestError. + :exc:`~sqlalchemy.exc.InvalidRequestError`. By default, the :class:`.Session` also expires all database loaded state on all ORM-managed attributes after transaction commit. This so that subsequent operations load the most recent data from the database. This behavior can be disabled using - the ``expire_on_commit=False`` option to :func:`.sessionmaker` or + the ``expire_on_commit=False`` option to :class:`.sessionmaker` or the :class:`.Session` constructor. If a subtransaction is in effect (which occurs when begin() is called multiple times), the subtransaction will be closed, and the next call to ``commit()`` will operate on the enclosing transaction. - For a session configured with autocommit=False, a new transaction will + When using the :class:`.Session` in its default mode of + ``autocommit=False``, a new transaction will be begun immediately after the commit, but note that the newly begun transaction does *not* use any connection resources until the first SQL is actually emitted. + .. seealso:: + + :ref:`session_committing` + """ if self.transaction is None: if not self.autocommit: @@ -713,10 +768,11 @@ class Session(object): """Prepare the current transaction in progress for two phase commit. If no transaction is in progress, this method raises an - InvalidRequestError. + :exc:`~sqlalchemy.exc.InvalidRequestError`. Only root transactions of two phase sessions can be prepared. If the - current transaction is not such, an InvalidRequestError is raised. + current transaction is not such, an + :exc:`~sqlalchemy.exc.InvalidRequestError` is raised. """ if self.transaction is None: @@ -735,18 +791,20 @@ class Session(object): :class:`.Session` object's transactional state. If this :class:`.Session` is configured with ``autocommit=False``, - either the :class:`.Connection` corresponding to the current transaction - is returned, or if no transaction is in progress, a new one is begun - and the :class:`.Connection` returned (note that no transactional state - is established with the DBAPI until the first SQL statement is emitted). + either the :class:`.Connection` corresponding to the current + transaction is returned, or if no transaction is in progress, a new + one is begun and the :class:`.Connection` returned (note that no + transactional state is established with the DBAPI until the first + SQL statement is emitted). - Alternatively, if this :class:`.Session` is configured with ``autocommit=True``, - an ad-hoc :class:`.Connection` is returned using :meth:`.Engine.contextual_connect` - on the underlying :class:`.Engine`. + Alternatively, if this :class:`.Session` is configured with + ``autocommit=True``, an ad-hoc :class:`.Connection` is returned + using :meth:`.Engine.contextual_connect` on the underlying + :class:`.Engine`. - Ambiguity in multi-bind or unbound :class:`.Session` objects can be resolved through - any of the optional keyword arguments. This ultimately makes usage of the - :meth:`.get_bind` method for resolution. + Ambiguity in multi-bind or unbound :class:`.Session` objects can be + resolved through any of the optional keyword arguments. This + ultimately makes usage of the :meth:`.get_bind` method for resolution. :param bind: Optional :class:`.Engine` to be used as the bind. If @@ -765,11 +823,12 @@ class Session(object): etc.) which will be used to locate a bind, if a bind cannot otherwise be identified. - :param close_with_result: Passed to :meth:`Engine.connect`, indicating - the :class:`.Connection` should be considered "single use", automatically - closing when the first result set is closed. This flag only has - an effect if this :class:`.Session` is configured with ``autocommit=True`` - and does not already have a transaction in progress. + :param close_with_result: Passed to :meth:`.Engine.connect`, indicating + the :class:`.Connection` should be considered "single use", + automatically closing when the first result set is closed. This + flag only has an effect if this :class:`.Session` is configured with + ``autocommit=True`` and does not already have a transaction + in progress. :param \**kw: Additional keyword arguments are sent to :meth:`get_bind()`, @@ -811,8 +870,8 @@ class Session(object): :func:`~.sql.expression.delete`, and :func:`~.sql.expression.text`. Plain SQL strings can be passed as well, which in the case of :meth:`.Session.execute` only - will be interpreted the same as if it were passed via a :func:`~.expression.text` - construct. That is, the following usage:: + will be interpreted the same as if it were passed via a + :func:`~.expression.text` construct. That is, the following usage:: result = session.execute( "SELECT * FROM user WHERE id=:param", @@ -828,9 +887,10 @@ class Session(object): ) The second positional argument to :meth:`.Session.execute` is an - optional parameter set. Similar to that of :meth:`.Connection.execute`, whether this - is passed as a single dictionary, or a list of dictionaries, determines - whether the DBAPI cursor's ``execute()`` or ``executemany()`` is used to execute the + optional parameter set. Similar to that of + :meth:`.Connection.execute`, whether this is passed as a single + dictionary, or a list of dictionaries, determines whether the DBAPI + cursor's ``execute()`` or ``executemany()`` is used to execute the statement. An INSERT construct may be invoked for a single row:: result = session.execute(users.insert(), {"id": 7, "name": "somename"}) @@ -859,12 +919,13 @@ class Session(object): The :class:`.ResultProxy` returned by the :meth:`.Session.execute` method is returned with the "close_with_result" flag set to true; the significance of this flag is that if this :class:`.Session` is - autocommitting and does not have a transaction-dedicated :class:`.Connection` - available, a temporary :class:`.Connection` is established for the - statement execution, which is closed (meaning, returned to the connection - pool) when the :class:`.ResultProxy` has consumed all available data. - This applies *only* when the :class:`.Session` is configured with - autocommit=True and no transaction has been started. + autocommitting and does not have a transaction-dedicated + :class:`.Connection` available, a temporary :class:`.Connection` is + established for the statement execution, which is closed (meaning, + returned to the connection pool) when the :class:`.ResultProxy` has + consumed all available data. This applies *only* when the + :class:`.Session` is configured with autocommit=True and no + transaction has been started. :param clause: An executable statement (i.e. an :class:`.Executable` expression @@ -919,7 +980,8 @@ class Session(object): def scalar(self, clause, params=None, mapper=None, bind=None, **kw): """Like :meth:`~.Session.execute` but return a scalar result.""" - return self.execute(clause, params=params, mapper=mapper, bind=bind, **kw).scalar() + return self.execute( + clause, params=params, mapper=mapper, bind=bind, **kw).scalar() def close(self): """Close this Session. @@ -936,13 +998,6 @@ class Session(object): for transaction in self.transaction._iterate_parents(): transaction.close() - @classmethod - def close_all(cls): - """Close *all* sessions in memory.""" - - for sess in _sessions.values(): - sess.close() - def expunge_all(self): """Remove all object instances from this ``Session``. @@ -951,14 +1006,15 @@ class Session(object): """ for state in self.identity_map.all_states() + list(self._new): - state.detach() + state._detach() self.identity_map = self._identity_cls() self._new = {} self._deleted = {} # TODO: need much more test coverage for bind_mapper() and similar ! - # TODO: + crystalize + document resolution order vis. bind_mapper/bind_table + # TODO: + crystalize + document resolution order + # vis. bind_mapper/bind_table def bind_mapper(self, mapper, bind): """Bind operations for a mapper to a Connectable. @@ -974,7 +1030,7 @@ class Session(object): """ if isinstance(mapper, type): - mapper = _class_mapper(mapper) + mapper = class_mapper(mapper) self.__binds[mapper.base_mapper] = bind for t in mapper._all_tables: @@ -1027,7 +1083,7 @@ class Session(object): linked to the :class:`.MetaData` ultimately associated with the :class:`.Table` or other selectable to which the mapper is mapped. - 6. No bind can be found, :class:`.UnboundExecutionError` + 6. No bind can be found, :exc:`~sqlalchemy.exc.UnboundExecutionError` is raised. :param mapper: @@ -1041,10 +1097,10 @@ class Session(object): :param clause: A :class:`.ClauseElement` (i.e. :func:`~.sql.expression.select`, :func:`~.sql.expression.text`, - etc.). If the ``mapper`` argument is not present or could not produce - a bind, the given expression construct will be searched for a bound - element, typically a :class:`.Table` associated with bound - :class:`.MetaData`. + etc.). If the ``mapper`` argument is not present or could not + produce a bind, the given expression construct will be searched + for a bound element, typically a :class:`.Table` associated with + bound :class:`.MetaData`. """ if mapper is clause is None: @@ -1124,11 +1180,18 @@ class Session(object): def _autoflush(self): if self.autoflush and not self._flushing: - self.flush() - - def _finalize_loaded(self, states): - for state, dict_ in states.items(): - state.commit_all(dict_, self.identity_map) + try: + self.flush() + except sa_exc.StatementError as e: + # note we are reraising StatementError as opposed to + # raising FlushError with "chaining" to remain compatible + # with code that catches StatementError, IntegrityError, + # etc. + e.add_detail( + "raised as a result of Query-invoked autoflush; " + "consider using a session.no_autoflush block if this " + "flush is occuring prematurely") + util.raise_from_cause(e) def refresh(self, instance, attribute_names=None, lockmode=None): """Expire and refresh the attributes on the given instance. @@ -1165,13 +1228,14 @@ class Session(object): self._expire_state(state, attribute_names) - if self.query(_object_mapper(instance))._load_on_ident( + if loading.load_on_ident( + self.query(object_mapper(instance)), state.key, refresh_state=state, lockmode=lockmode, only_load_props=attribute_names) is None: raise sa_exc.InvalidRequestError( "Could not refresh instance '%s'" % - mapperutil.instance_str(instance)) + instance_str(instance)) def expire_all(self): """Expires all persistent instances within this Session. @@ -1196,7 +1260,7 @@ class Session(object): """ for state in self.identity_map.all_states(): - state.expire(state.dict, self.identity_map._modified) + state._expire(state.dict, self.identity_map._modified) def expire(self, instance, attribute_names=None): """Expire the attributes on an instance. @@ -1234,7 +1298,7 @@ class Session(object): def _expire_state(self, state, attribute_names): self._validate_persistent(state) if attribute_names: - state.expire_attributes(state.dict, attribute_names) + state._expire_attributes(state.dict, attribute_names) else: # pre-fetch the full cascade since the expire is going to # remove associations @@ -1248,10 +1312,10 @@ class Session(object): """Expire a state if persistent, else expunge if pending""" if state.key: - state.expire(state.dict, self.identity_map._modified) + state._expire(state.dict, self.identity_map._modified) elif state in self._new: self._new.pop(state) - state.detach() + state._detach() @util.deprecated("0.7", "The non-weak-referencing identity map " "feature is no longer needed.") @@ -1282,7 +1346,7 @@ class Session(object): if state.session_id is not self.hash_key: raise sa_exc.InvalidRequestError( "Instance %s is not present in this Session" % - mapperutil.state_str(state)) + state_str(state)) cascaded = list(state.manager.mapper.cascade_iterator( 'expunge', state)) @@ -1293,68 +1357,83 @@ class Session(object): def _expunge_state(self, state): if state in self._new: self._new.pop(state) - state.detach() + state._detach() elif self.identity_map.contains_state(state): self.identity_map.discard(state) self._deleted.pop(state, None) - state.detach() + state._detach() elif self.transaction: self.transaction._deleted.pop(state, None) - def _register_newly_persistent(self, state): - mapper = _state_mapper(state) + def _register_newly_persistent(self, states): + for state in states: + mapper = _state_mapper(state) - # prevent against last minute dereferences of the object - obj = state.obj() - if obj is not None: + # prevent against last minute dereferences of the object + obj = state.obj() + if obj is not None: - instance_key = mapper._identity_key_from_state(state) + instance_key = mapper._identity_key_from_state(state) - if _none_set.issubset(instance_key[1]) and \ - not mapper.allow_partial_pks or \ - _none_set.issuperset(instance_key[1]): - raise exc.FlushError( - "Instance %s has a NULL identity key. If this is an " - "auto-generated value, check that the database table " - "allows generation of new primary key values, and that " - "the mapped Column object is configured to expect these " - "generated values. Ensure also that this flush() is " - "not occurring at an inappropriate time, such as within " - "a load() event." % mapperutil.state_str(state) - ) + if _none_set.issubset(instance_key[1]) and \ + not mapper.allow_partial_pks or \ + _none_set.issuperset(instance_key[1]): + raise exc.FlushError( + "Instance %s has a NULL identity key. If this is an " + "auto-generated value, check that the database table " + "allows generation of new primary key values, and " + "that the mapped Column object is configured to " + "expect these generated values. Ensure also that " + "this flush() is not occurring at an inappropriate " + "time, such aswithin a load() event." + % state_str(state) + ) - if state.key is None: - state.key = instance_key - elif state.key != instance_key: - # primary key switch. use discard() in case another - # state has already replaced this one in the identity - # map (see test/orm/test_naturalpks.py ReversePKsTest) - self.identity_map.discard(state) - if state in self.transaction._key_switches: - orig_key = self.transaction._key_switches[state][0] - else: - orig_key = state.key - self.transaction._key_switches[state] = (orig_key, instance_key) - state.key = instance_key + if state.key is None: + state.key = instance_key + elif state.key != instance_key: + # primary key switch. use discard() in case another + # state has already replaced this one in the identity + # map (see test/orm/test_naturalpks.py ReversePKsTest) + self.identity_map.discard(state) + if state in self.transaction._key_switches: + orig_key = self.transaction._key_switches[state][0] + else: + orig_key = state.key + self.transaction._key_switches[state] = ( + orig_key, instance_key) + state.key = instance_key - self.identity_map.replace(state) - state.commit_all(state.dict, self.identity_map) + self.identity_map.replace(state) + statelib.InstanceState._commit_all_states( + ((state, state.dict) for state in states), + self.identity_map + ) + + self._register_altered(states) # remove from new last, might be the last strong ref - if state in self._new: - if self._enable_transaction_accounting and self.transaction: - self.transaction._new[state] = True + for state in set(states).intersection(self._new): self._new.pop(state) - def _remove_newly_deleted(self, state): + def _register_altered(self, states): if self._enable_transaction_accounting and self.transaction: - self.transaction._deleted[state] = True + for state in states: + if state in self._new: + self.transaction._new[state] = True + else: + self.transaction._dirty[state] = True - self.identity_map.discard(state) - self._deleted.pop(state, None) - state.deleted = True + def _remove_newly_deleted(self, states): + for state in states: + if self._enable_transaction_accounting and self.transaction: + self.transaction._deleted[state] = True - def add(self, instance): + self.identity_map.discard(state) + self._deleted.pop(state, None) + state.deleted = True + + def add(self, instance, _warn=True): """Place an object in the ``Session``. Its state will be persisted to the database on the next flush @@ -1364,6 +1443,9 @@ class Session(object): is ``expunge()``. """ + if _warn and self._warn_on_events: + self._flush_warning("Session.add()") + try: state = attributes.instance_state(instance) except exc.NO_STATE: @@ -1374,8 +1456,11 @@ class Session(object): def add_all(self, instances): """Add the given collection of instances to this ``Session``.""" + if self._warn_on_events: + self._flush_warning("Session.add_all()") + for instance in instances: - self.add(instance) + self.add(instance, _warn=False) def _save_or_update_state(self, state): self._save_or_update_impl(state) @@ -1393,6 +1478,9 @@ class Session(object): The database delete operation occurs upon ``flush()``. """ + if self._warn_on_events: + self._flush_warning("Session.delete()") + try: state = attributes.instance_state(instance) except exc.NO_STATE: @@ -1401,7 +1489,7 @@ class Session(object): if state.key is None: raise sa_exc.InvalidRequestError( "Instance '%s' is not persisted" % - mapperutil.state_str(state)) + state_str(state)) if state in self._deleted: return @@ -1409,7 +1497,7 @@ class Session(object): # ensure object is attached to allow the # cascade operation to load deferred attributes # and collections - self._attach(state) + self._attach(state, include_before=True) # grab the cascades before adding the item to the deleted list # so that autoflush does not delete the item @@ -1423,7 +1511,7 @@ class Session(object): for o, m, st_, dct_ in cascade_states: self._delete_impl(st_) - def merge(self, instance, load=True, **kw): + def merge(self, instance, load=True): """Copy the state of a given instance into a corresponding instance within this :class:`.Session`. @@ -1431,8 +1519,8 @@ class Session(object): source instance, and attempts to reconcile it with an instance of the same primary key in the session. If not found locally, it attempts to load the object from the database based on primary key, and if - none can be located, creates a new instance. The state of each attribute - on the source instance is then copied to the target instance. + none can be located, creates a new instance. The state of each + attribute on the source instance is then copied to the target instance. The resulting target instance is then returned by the method; the original source instance is left unmodified, and un-associated with the :class:`.Session` if not already. @@ -1466,10 +1554,9 @@ class Session(object): should be "clean" as well, else this suggests a mis-use of the method. """ - if 'dont_load' in kw: - load = not kw['dont_load'] - util.warn_deprecated('dont_load=True has been renamed to ' - 'load=False.') + + if self._warn_on_events: + self._flush_warning("Session.merge()") _recursive = {} @@ -1477,7 +1564,7 @@ class Session(object): # flush current contents if we expect to load data self._autoflush() - _object_mapper(instance) # verify mapped + object_mapper(instance) # verify mapped autoflush = self.autoflush try: self.autoflush = False @@ -1566,7 +1653,7 @@ class Session(object): "merging to update the most recent version." % ( existing_version, - mapperutil.state_str(merged_state), + state_str(merged_state), merged_version )) @@ -1580,38 +1667,29 @@ class Session(object): if not load: # remove any history - merged_state.commit_all(merged_dict, self.identity_map) + merged_state._commit_all(merged_dict, self.identity_map) if new_instance: merged_state.manager.dispatch.load(merged_state, None) return merged - @classmethod - def identity_key(cls, *args, **kwargs): - return mapperutil.identity_key(*args, **kwargs) - - @classmethod - def object_session(cls, instance): - """Return the ``Session`` to which an object belongs.""" - - return object_session(instance) - def _validate_persistent(self, state): if not self.identity_map.contains_state(state): raise sa_exc.InvalidRequestError( "Instance '%s' is not persistent within this Session" % - mapperutil.state_str(state)) + state_str(state)) def _save_impl(self, state): if state.key is not None: raise sa_exc.InvalidRequestError( "Object '%s' already has an identity - it can't be registered " - "as pending" % mapperutil.state_str(state)) + "as pending" % state_str(state)) - self._attach(state) + self._before_attach(state) if state not in self._new: self._new[state] = state.obj() state.insert_order = len(self._new) + self._attach(state) def _update_impl(self, state, discard_existing=False): if (self.identity_map.contains_state(state) and @@ -1621,21 +1699,21 @@ class Session(object): if state.key is None: raise sa_exc.InvalidRequestError( "Instance '%s' is not persisted" % - mapperutil.state_str(state)) + state_str(state)) if state.deleted: raise sa_exc.InvalidRequestError( "Instance '%s' has been deleted. Use the make_transient() " "function to send this object back to the transient state." % - mapperutil.state_str(state) + state_str(state) ) - if discard_existing: - existing = self.identity_map.get(state.key) - if existing is not None: - self.identity_map.discard(attributes.instance_state(existing)) - self._attach(state) + self._before_attach(state) self._deleted.pop(state, None) - self.identity_map.add(state) + if discard_existing: + self.identity_map.replace(state) + else: + self.identity_map.add(state) + self._attach(state) def _save_or_update_impl(self, state): if state.key is None: @@ -1650,29 +1728,94 @@ class Session(object): if state.key is None: return - self._attach(state) + self._attach(state, include_before=True) self._deleted[state] = state.obj() self.identity_map.add(state) - def _attach(self, state): + def enable_relationship_loading(self, obj): + """Associate an object with this :class:`.Session` for related + object loading. + + .. warning:: + + :meth:`.enable_relationship_loading` exists to serve special + use cases and is not recommended for general use. + + Accesses of attributes mapped with :func:`.relationship` + will attempt to load a value from the database using this + :class:`.Session` as the source of connectivity. The values + will be loaded based on foreign key values present on this + object - it follows that this functionality + generally only works for many-to-one-relationships. + + The object will be attached to this session, but will + **not** participate in any persistence operations; its state + for almost all purposes will remain either "transient" or + "detached", except for the case of relationship loading. + + Also note that backrefs will often not work as expected. + Altering a relationship-bound attribute on the target object + may not fire off a backref event, if the effective value + is what was already loaded from a foreign-key-holding value. + + The :meth:`.Session.enable_relationship_loading` method is + similar to the ``load_on_pending`` flag on :func:`.relationship`. Unlike + that flag, :meth:`.Session.enable_relationship_loading` allows + an object to remain transient while still being able to load + related items. + + To make a transient object associated with a :class:`.Session` + via :meth:`.Session.enable_relationship_loading` pending, add + it to the :class:`.Session` using :meth:`.Session.add` normally. + + :meth:`.Session.enable_relationship_loading` does not improve + behavior when the ORM is used normally - object references should be + constructed at the object level, not at the foreign key level, so + that they are present in an ordinary way before flush() + proceeds. This method is not intended for general use. + + .. versionadded:: 0.8 + + .. seealso:: + + ``load_on_pending`` at :func:`.relationship` - this flag + allows per-relationship loading of many-to-ones on items that + are pending. + + """ + state = attributes.instance_state(obj) + self._attach(state, include_before=True) + state._load_pending = True + + def _before_attach(self, state): + if state.session_id != self.hash_key and \ + self.dispatch.before_attach: + self.dispatch.before_attach(self, state.obj()) + + def _attach(self, state, include_before=False): if state.key and \ state.key in self.identity_map and \ - not self.identity_map.contains_state(state): + not self.identity_map.contains_state(state): raise sa_exc.InvalidRequestError("Can't attach instance " "%s; another instance with key %s is already " "present in this session." - % (mapperutil.state_str(state), state.key)) + % (state_str(state), state.key)) if state.session_id and \ state.session_id is not self.hash_key and \ state.session_id in _sessions: raise sa_exc.InvalidRequestError( "Object '%s' is already attached to session '%s' " - "(this is '%s')" % (mapperutil.state_str(state), + "(this is '%s')" % (state_str(state), state.session_id, self.hash_key)) if state.session_id != self.hash_key: + if include_before and \ + self.dispatch.before_attach: + self.dispatch.before_attach(self, state.obj()) state.session_id = self.hash_key + if state.modified and state._strong_obj is None: + state._strong_obj = state.obj() if self.dispatch.after_attach: self.dispatch.after_attach(self, state.obj()) @@ -1690,9 +1833,11 @@ class Session(object): return self._contains_state(state) def __iter__(self): - """Iterate over all pending or persistent instances within this Session.""" + """Iterate over all pending or persistent instances within this + Session. - return iter(list(self._new.values()) + self.identity_map.values()) + """ + return iter(list(self._new.values()) + list(self.identity_map.values())) def _contains_state(self, state): return state in self._new or self.identity_map.contains_state(state) @@ -1735,6 +1880,14 @@ class Session(object): finally: self._flushing = False + def _flush_warning(self, method): + util.warn( + "Usage of the '%s' operation is not currently supported " + "within the execution stage of the flush process. " + "Results may not be consistent. Consider using alternative " + "event listeners or connection-level operations instead." + % method) + def _is_clean(self): return not self.identity_map.check_modified() and \ not self._deleted and \ @@ -1784,7 +1937,8 @@ class Session(object): proc = new.union(dirty).difference(deleted) for state in proc: - is_orphan = _state_mapper(state)._is_orphan(state) and state.has_identity + is_orphan = ( + _state_mapper(state)._is_orphan(state) and state.has_identity) flush_context.register_object(state, isdelete=is_orphan) processed.add(state) @@ -1802,12 +1956,31 @@ class Session(object): flush_context.transaction = transaction = self.begin( subtransactions=True) try: - flush_context.execute() + self._warn_on_events = True + try: + flush_context.execute() + finally: + self._warn_on_events = False self.dispatch.after_flush(self, flush_context) flush_context.finalize_flush_changes() + if not objects and self.identity_map._modified: + len_ = len(self.identity_map._modified) + + statelib.InstanceState._commit_all_states( + [(state, state.dict) for state in + self.identity_map._modified], + instance_dict=self.identity_map) + util.warn("Attribute history events accumulated on %d " + "previously clean instances " + "within inner-flush event handlers have been reset, " + "and will not result in database updates. " + "Consider using set_committed_value() within " + "inner-flush event handlers to avoid this warning." + % len_) + # useful assertions: #if not objects: # assert not self.identity_map._modified @@ -1820,12 +1993,11 @@ class Session(object): transaction.commit() except: - transaction.rollback(_capture_exception=True) - raise - + with util.safe_reraise(): + transaction.rollback(_capture_exception=True) def is_modified(self, instance, include_collections=True, - passive=attributes.PASSIVE_OFF): + passive=True): """Return ``True`` if the given instance has locally modified attributes. @@ -1840,18 +2012,16 @@ class Session(object): E.g.:: - return session.is_modified(someobject, passive=True) + return session.is_modified(someobject) .. versionchanged:: 0.8 - In SQLAlchemy 0.7 and earlier, the ``passive`` - flag should **always** be explicitly set to ``True``. - The current default value of :data:`.attributes.PASSIVE_OFF` - for this flag is incorrect, in that it loads unloaded - collections and attributes which by definition - have no modified state, and furthermore trips off - autoflush which then causes all subsequent, possibly - modified attributes to lose their modified state. - The default value of the flag will be changed in 0.8. + When using SQLAlchemy 0.7 and earlier, the ``passive`` + flag should **always** be explicitly set to ``True``, + else SQL loads/autoflushes may proceed which can affect + the modified state itself: + ``session.is_modified(someobject, passive=True)``\ . + In 0.8 and above, the behavior is corrected and + this flag is ignored. A few caveats to this method apply: @@ -1872,47 +2042,32 @@ class Session(object): usually needed, and in those few cases where it isn't, is less expensive on average than issuing a defensive SELECT. - The "old" value is fetched unconditionally only if the attribute - container has the ``active_history`` flag set to ``True``. This flag - is set typically for primary key attributes and scalar object references - that are not a simple many-to-one. To set this flag for - any arbitrary mapped column, use the ``active_history`` argument - with :func:`.column_property`. + The "old" value is fetched unconditionally upon set only if the + attribute container has the ``active_history`` flag set to ``True``. + This flag is set typically for primary key attributes and scalar + object references that are not a simple many-to-one. To set this + flag for any arbitrary mapped column, use the ``active_history`` + argument with :func:`.column_property`. :param instance: mapped instance to be tested for pending changes. - :param include_collections: Indicates if multivalued collections should be - included in the operation. Setting this to ``False`` is a way to detect - only local-column based properties (i.e. scalar columns or many-to-one - foreign keys) that would result in an UPDATE for this instance upon - flush. - :param passive: Indicates if unloaded attributes and - collections should be loaded in the course of performing - this test. If set to ``False``, or left at its default - value of :data:`.PASSIVE_OFF`, unloaded attributes - will be loaded. If set to ``True`` or - :data:`.PASSIVE_NO_INITIALIZE`, unloaded - collections and attributes will remain unloaded. As - noted previously, the existence of this flag here - is a bug, as unloaded attributes by definition have - no changes, and the load operation also triggers an - autoflush which then cancels out subsequent changes. - This flag should **always be set to True**. - + :param include_collections: Indicates if multivalued collections + should be included in the operation. Setting this to ``False`` is a + way to detect only local-column based properties (i.e. scalar columns + or many-to-one foreign keys) that would result in an UPDATE for this + instance upon flush. + :param passive: .. versionchanged:: 0.8 - The flag will be deprecated and the default - set to ``True``. + Ignored for backwards compatibility. + When using SQLAlchemy 0.7 and earlier, this flag should always + be set to ``True``. """ - try: - state = attributes.instance_state(instance) - except exc.NO_STATE: - raise exc.UnmappedInstanceError(instance) - dict_ = state.dict + state = object_state(instance) - if passive is True: - passive = attributes.PASSIVE_NO_INITIALIZE - elif passive is False: - passive = attributes.PASSIVE_OFF + if not state.modified: + return False + + dict_ = state.dict for attr in state.manager.attributes: if \ @@ -1923,11 +2078,13 @@ class Session(object): continue (added, unchanged, deleted) = \ - attr.impl.get_history(state, dict_, passive=passive) + attr.impl.get_history(state, dict_, + passive=attributes.NO_CHANGE) if added or deleted: return True - return False + else: + return False @property def is_active(self): @@ -1953,34 +2110,36 @@ class Session(object): target :class:`.Connection` to a user-defined event listener. The "partial rollback" state refers to when an "inner" transaction, - typically used during a flush, encounters an error and emits - a rollback of the DBAPI connection. At this point, the :class:`.Session` - is in "partial rollback" and awaits for the user to call :meth:`.rollback`, - in order to close out the transaction stack. It is in this "partial - rollback" period that the :attr:`.is_active` flag returns False. After - the call to :meth:`.rollback`, the :class:`.SessionTransaction` is replaced + typically used during a flush, encounters an error and emits a + rollback of the DBAPI connection. At this point, the + :class:`.Session` is in "partial rollback" and awaits for the user to + call :meth:`.Session.rollback`, in order to close out the + transaction stack. It is in this "partial rollback" period that the + :attr:`.is_active` flag returns False. After the call to + :meth:`.Session.rollback`, the :class:`.SessionTransaction` is replaced with a new one and :attr:`.is_active` returns ``True`` again. When a :class:`.Session` is used in ``autocommit=True`` mode, the :class:`.SessionTransaction` is only instantiated within the scope of a flush call, or when :meth:`.Session.begin` is called. So :attr:`.is_active` will always be ``False`` outside of a flush or - :meth:`.begin` block in this mode, and will be ``True`` within the - :meth:`.begin` block as long as it doesn't enter "partial rollback" - state. + :meth:`.Session.begin` block in this mode, and will be ``True`` + within the :meth:`.Session.begin` block as long as it doesn't enter + "partial rollback" state. From all the above, it follows that the only purpose to this flag is for application frameworks that wish to detect is a "rollback" is - necessary within a generic error handling routine, for :class:`.Session` - objects that would otherwise be in "partial rollback" mode. In - a typical integration case, this is also not necessary as it is standard - practice to emit :meth:`.Session.rollback` unconditionally within the - outermost exception catch. + necessary within a generic error handling routine, for + :class:`.Session` objects that would otherwise be in + "partial rollback" mode. In a typical integration case, this is also + not necessary as it is standard practice to emit + :meth:`.Session.rollback` unconditionally within the outermost + exception catch. To track the transactional state of a :class:`.Session` fully, use event listeners, primarily the :meth:`.SessionEvents.after_begin`, - :meth:`.SessionEvents.after_commit`, :meth:`.SessionEvents.after_rollback` - and related events. + :meth:`.SessionEvents.after_commit`, + :meth:`.SessionEvents.after_rollback` and related events. """ return self.transaction and self.transaction.is_active @@ -1992,9 +2151,10 @@ class Session(object): access to the full set of persistent objects (i.e., those that have row identity) currently in the session. - See also: + .. seealso:: - :func:`.identity_key` - operations involving identity keys. + :func:`.identity_key` - helper function to produce the keys used + in this dictionary. """ @@ -2041,15 +2201,142 @@ class Session(object): def deleted(self): "The set of all instances marked as 'deleted' within this ``Session``" - return util.IdentitySet(self._deleted.values()) + return util.IdentitySet(list(self._deleted.values())) @property def new(self): "The set of all instances marked as 'new' within this ``Session``." - return util.IdentitySet(self._new.values()) + return util.IdentitySet(list(self._new.values())) + + +class sessionmaker(_SessionClassMethods): + """A configurable :class:`.Session` factory. + + The :class:`.sessionmaker` factory generates new + :class:`.Session` objects when called, creating them given + the configurational arguments established here. + + e.g.:: + + # global scope + Session = sessionmaker(autoflush=False) + + # later, in a local scope, create and use a session: + sess = Session() + + Any keyword arguments sent to the constructor itself will override the + "configured" keywords:: + + Session = sessionmaker() + + # bind an individual session to a connection + sess = Session(bind=connection) + + The class also includes a method :meth:`.configure`, which can + be used to specify additional keyword arguments to the factory, which + will take effect for subsequent :class:`.Session` objects generated. + This is usually used to associate one or more :class:`.Engine` objects + with an existing :class:`.sessionmaker` factory before it is first + used:: + + # application starts + Session = sessionmaker() + + # ... later + engine = create_engine('sqlite:///foo.db') + Session.configure(bind=engine) + + sess = Session() + + .. seealso: + + :ref:`session_getting` - introductory text on creating + sessions using :class:`.sessionmaker`. + + """ + + def __init__(self, bind=None, class_=Session, autoflush=True, + autocommit=False, + expire_on_commit=True, + info=None, **kw): + """Construct a new :class:`.sessionmaker`. + + All arguments here except for ``class_`` correspond to arguments + accepted by :class:`.Session` directly. See the + :meth:`.Session.__init__` docstring for more details on parameters. + + :param bind: a :class:`.Engine` or other :class:`.Connectable` with + which newly created :class:`.Session` objects will be associated. + :param class_: class to use in order to create new :class:`.Session` + objects. Defaults to :class:`.Session`. + :param autoflush: The autoflush setting to use with newly created + :class:`.Session` objects. + :param autocommit: The autocommit setting to use with newly created + :class:`.Session` objects. + :param expire_on_commit=True: the expire_on_commit setting to use + with newly created :class:`.Session` objects. + :param info: optional dictionary of information that will be available + via :attr:`.Session.info`. Note this dictionary is *updated*, not + replaced, when the ``info`` parameter is specified to the specific + :class:`.Session` construction operation. + + .. versionadded:: 0.9.0 + + :param \**kw: all other keyword arguments are passed to the constructor + of newly created :class:`.Session` objects. + + """ + kw['bind'] = bind + kw['autoflush'] = autoflush + kw['autocommit'] = autocommit + kw['expire_on_commit'] = expire_on_commit + if info is not None: + kw['info'] = info + self.kw = kw + # make our own subclass of the given class, so that + # events can be associated with it specifically. + self.class_ = type(class_.__name__, (class_,), {}) + + def __call__(self, **local_kw): + """Produce a new :class:`.Session` object using the configuration + established in this :class:`.sessionmaker`. + + In Python, the ``__call__`` method is invoked on an object when + it is "called" in the same way as a function:: + + Session = sessionmaker() + session = Session() # invokes sessionmaker.__call__() + + """ + for k, v in self.kw.items(): + if k == 'info' and 'info' in local_kw: + d = v.copy() + d.update(local_kw['info']) + local_kw['info'] = d + else: + local_kw.setdefault(k, v) + return self.class_(**local_kw) + + def configure(self, **new_kw): + """(Re)configure the arguments for this sessionmaker. + + e.g.:: + + Session = sessionmaker() + + Session.configure(bind=create_engine('sqlite://')) + """ + self.kw.update(new_kw) + + def __repr__(self): + return "%s(class_=%r,%s)" % ( + self.__class__.__name__, + self.class_.__name__, + ", ".join("%s=%r" % (k, v) for k, v in self.kw.items()) + ) + -_sessions = weakref.WeakValueDictionary() def make_transient(instance): """Make the given instance 'transient'. @@ -2079,6 +2366,7 @@ def make_transient(instance): if state.deleted: del state.deleted + def object_session(instance): """Return the ``Session`` to which instance belongs. @@ -2092,12 +2380,4 @@ def object_session(instance): raise exc.UnmappedInstanceError(instance) -def _state_session(state): - if state.session_id: - try: - return _sessions[state.session_id] - except KeyError: - pass - return None - _new_sessionid = util.counter() diff --git a/libs/sqlalchemy/orm/shard.py b/libs/sqlalchemy/orm/shard.py deleted file mode 100644 index 93bc7a6b..00000000 --- a/libs/sqlalchemy/orm/shard.py +++ /dev/null @@ -1,15 +0,0 @@ -# orm/shard.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php - -from sqlalchemy import util - -util.warn_deprecated( - "Horizontal sharding is now importable via " - "'import sqlalchemy.ext.horizontal_shard" -) - -from sqlalchemy.ext.horizontal_shard import * - diff --git a/libs/sqlalchemy/orm/state.py b/libs/sqlalchemy/orm/state.py index b9a9c463..9712dd05 100644 --- a/libs/sqlalchemy/orm/state.py +++ b/libs/sqlalchemy/orm/state.py @@ -1,5 +1,5 @@ # orm/state.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -11,33 +11,30 @@ defines a large part of the ORM's interactivity. """ -from sqlalchemy.util import EMPTY_SET import weakref -from sqlalchemy import util +from .. import util +from . import exc as orm_exc, interfaces +from .path_registry import PathRegistry +from .base import PASSIVE_NO_RESULT, SQL_OK, NEVER_SET, ATTR_WAS_SET, \ + NO_VALUE, PASSIVE_NO_INITIALIZE, INIT_OK, PASSIVE_OFF +from . import base -from sqlalchemy.orm import exc as orm_exc, attributes, interfaces,\ - util as orm_util -from sqlalchemy.orm.attributes import PASSIVE_OFF, PASSIVE_NO_RESULT, \ - PASSIVE_NO_FETCH, NEVER_SET, ATTR_WAS_SET, NO_VALUE - -mapperlib = util.importlater("sqlalchemy.orm", "mapperlib") - -import sys - -class InstanceState(object): +class InstanceState(interfaces._InspectionAttr): """tracks state information at the instance level.""" session_id = None key = None runid = None - load_options = EMPTY_SET + load_options = util.EMPTY_SET load_path = () insert_order = None - mutable_dict = None _strong_obj = None modified = False expired = False deleted = False + _load_pending = False + + is_instance = True def __init__(self, obj, manager): self.class_ = obj.__class__ @@ -46,23 +43,127 @@ class InstanceState(object): self.callables = {} self.committed_state = {} + @util.memoized_property + def attrs(self): + """Return a namespace representing each attribute on + the mapped object, including its current value + and history. + + The returned object is an instance of :class:`.AttributeState`. + + """ + return util.ImmutableProperties( + dict( + (key, AttributeState(self, key)) + for key in self.manager + ) + ) + + @property + def transient(self): + """Return true if the object is transient.""" + return self.key is None and \ + not self._attached + + @property + def pending(self): + """Return true if the object is pending.""" + return self.key is None and \ + self._attached + + @property + def persistent(self): + """Return true if the object is persistent.""" + return self.key is not None and \ + self._attached + + @property + def detached(self): + """Return true if the object is detached.""" + return self.key is not None and \ + not self._attached + + @property + @util.dependencies("sqlalchemy.orm.session") + def _attached(self, sessionlib): + return self.session_id is not None and \ + self.session_id in sessionlib._sessions + + @property + @util.dependencies("sqlalchemy.orm.session") + def session(self, sessionlib): + """Return the owning :class:`.Session` for this instance, + or ``None`` if none available.""" + return sessionlib._state_session(self) + + @property + def object(self): + """Return the mapped object represented by this + :class:`.InstanceState`.""" + return self.obj() + + @property + def identity(self): + """Return the mapped identity of the mapped object. + This is the primary key identity as persisted by the ORM + which can always be passed directly to + :meth:`.Query.get`. + + Returns ``None`` if the object has no primary key identity. + + .. note:: + An object which is transient or pending + does **not** have a mapped identity until it is flushed, + even if its attributes include primary key values. + + """ + if self.key is None: + return None + else: + return self.key[1] + + @property + def identity_key(self): + """Return the identity key for the mapped object. + + This is the key used to locate the object within + the :attr:`.Session.identity_map` mapping. It contains + the identity as returned by :attr:`.identity` within it. + + + """ + # TODO: just change .key to .identity_key across + # the board ? probably + return self.key + @util.memoized_property def parents(self): return {} @util.memoized_property - def pending(self): + def _pending_mutations(self): return {} + @util.memoized_property + def mapper(self): + """Return the :class:`.Mapper` used for this mapepd object.""" + return self.manager.mapper + @property def has_identity(self): + """Return ``True`` if this object has an identity key. + + This should always have the same value as the + expression ``state.persistent or state.detached``. + + """ return bool(self.key) - def detach(self): - self.session_id = None + def _detach(self): + self.session_id = self._strong_obj = None - def dispose(self): - self.detach() + def _dispose(self): + self._detach() del self.obj def _cleanup(self, ref): @@ -71,7 +172,7 @@ class InstanceState(object): instance_dict.discard(self) self.callables = {} - self.session_id = None + self.session_id = self._strong_obj = None del self.obj def obj(self): @@ -81,19 +182,16 @@ class InstanceState(object): def dict(self): o = self.obj() if o is not None: - return attributes.instance_dict(o) + return base.instance_dict(o) else: return {} - def initialize_instance(*mixed, **kwargs): + def _initialize_instance(*mixed, **kwargs): self, instance, args = mixed[0], mixed[1], mixed[2:] manager = self.manager manager.dispatch.init(self, args, kwargs) - #if manager.mutable_attributes: - # assert self.__class__ is MutableAttrInstanceState - try: return manager.original_init(*mixed[1:], **kwargs) except: @@ -106,49 +204,29 @@ class InstanceState(object): def get_impl(self, key): return self.manager[key].impl - def get_pending(self, key): - if key not in self.pending: - self.pending[key] = PendingCollection() - return self.pending[key] - - def value_as_iterable(self, dict_, key, passive=PASSIVE_OFF): - """Return a list of tuples (state, obj) for the given - key. - - returns an empty list if the value is None/empty/PASSIVE_NO_RESULT - """ - - impl = self.manager[key].impl - x = impl.get(self, dict_, passive=passive) - if x is PASSIVE_NO_RESULT or x is None: - return [] - elif hasattr(impl, 'get_collection'): - return [ - (attributes.instance_state(o), o) for o in - impl.get_collection(self, dict_, x, passive=passive) - ] - else: - return [(attributes.instance_state(x), x)] + def _get_pending_mutation(self, key): + if key not in self._pending_mutations: + self._pending_mutations[key] = PendingCollection() + return self._pending_mutations[key] def __getstate__(self): - d = {'instance':self.obj()} - d.update( + state_dict = {'instance': self.obj()} + state_dict.update( (k, self.__dict__[k]) for k in ( - 'committed_state', 'pending', 'modified', 'expired', - 'callables', 'key', 'parents', 'load_options', 'mutable_dict', + 'committed_state', '_pending_mutations', 'modified', 'expired', + 'callables', 'key', 'parents', 'load_options', 'class_', ) if k in self.__dict__ ) if self.load_path: - d['load_path'] = interfaces.serialize_path(self.load_path) + state_dict['load_path'] = self.load_path.serialize() - self.manager.dispatch.pickle(self, d) + state_dict['manager'] = self.manager._serialize(self, state_dict) - return d + return state_dict - def __setstate__(self, state): - from sqlalchemy.orm import instrumentation - inst = state['instance'] + def __setstate__(self, state_dict): + inst = state_dict['instance'] if inst is not None: self.obj = weakref.ref(inst, self._cleanup) self.class_ = inst.__class__ @@ -157,60 +235,49 @@ class InstanceState(object): # due to storage of state in "parents". "class_" # also new. self.obj = None - self.class_ = state['class_'] - self.manager = manager = instrumentation.manager_of_class(self.class_) - if manager is None: - raise orm_exc.UnmappedInstanceError( - inst, - "Cannot deserialize object of type %r - no mapper() has" - " been configured for this class within the current Python process!" % - self.class_) - elif manager.is_mapped and not manager.mapper.configured: - mapperlib.configure_mappers() + self.class_ = state_dict['class_'] - self.committed_state = state.get('committed_state', {}) - self.pending = state.get('pending', {}) - self.parents = state.get('parents', {}) - self.modified = state.get('modified', False) - self.expired = state.get('expired', False) - self.callables = state.get('callables', {}) - - if self.modified: - self._strong_obj = inst + self.committed_state = state_dict.get('committed_state', {}) + self._pending_mutations = state_dict.get('_pending_mutations', {}) + self.parents = state_dict.get('parents', {}) + self.modified = state_dict.get('modified', False) + self.expired = state_dict.get('expired', False) + self.callables = state_dict.get('callables', {}) self.__dict__.update([ - (k, state[k]) for k in ( - 'key', 'load_options', 'mutable_dict' - ) if k in state + (k, state_dict[k]) for k in ( + 'key', 'load_options', + ) if k in state_dict ]) - if 'load_path' in state: - self.load_path = interfaces.deserialize_path(state['load_path']) + if 'load_path' in state_dict: + self.load_path = PathRegistry.\ + deserialize(state_dict['load_path']) - # setup _sa_instance_state ahead of time so that - # unpickle events can access the object normally. - # see [ticket:2362] - manager.setup_instance(inst, self) - manager.dispatch.unpickle(self, state) + state_dict['manager'](self, inst, state_dict) - def initialize(self, key): + def _initialize(self, key): """Set this attribute to an empty value or collection, based on the AttributeImpl in use.""" self.manager.get_impl(key).initialize(self, self.dict) - def reset(self, dict_, key): + def _reset(self, dict_, key): """Remove the given attribute and any callables associated with it.""" - dict_.pop(key, None) + old = dict_.pop(key, None) + if old is not None and self.manager[key].impl.collection: + self.manager[key].impl._invalidate_collection(old) self.callables.pop(key, None) - def expire_attribute_pre_commit(self, dict_, key): + def _expire_attribute_pre_commit(self, dict_, key): """a fast expire that can be called by column loaders during a load. The additional bookkeeping is finished up in commit_all(). + Should only be called for scalar attributes. + This method is actually called a lot with joined-table loading, when the second table isn't present in the result. @@ -218,65 +285,73 @@ class InstanceState(object): dict_.pop(key, None) self.callables[key] = self - def set_callable(self, dict_, key, callable_): - """Remove the given attribute and set the given callable - as a loader.""" + @classmethod + def _row_processor(cls, manager, fn, key): + impl = manager[key].impl + if impl.collection: + def _set_callable(state, dict_, row): + old = dict_.pop(key, None) + if old is not None: + impl._invalidate_collection(old) + state.callables[key] = fn + else: + def _set_callable(state, dict_, row): + state.callables[key] = fn + return _set_callable - dict_.pop(key, None) - self.callables[key] = callable_ - - def expire(self, dict_, modified_set): + def _expire(self, dict_, modified_set): self.expired = True if self.modified: modified_set.discard(self) self.modified = False + self._strong_obj = None self.committed_state.clear() - self.__dict__.pop('pending', None) - self.__dict__.pop('mutable_dict', None) + InstanceState._pending_mutations._reset(self) # clear out 'parents' collection. not # entirely clear how we can best determine # which to remove, or not. - self.__dict__.pop('parents', None) + InstanceState.parents._reset(self) for key in self.manager: impl = self.manager[key].impl if impl.accepts_scalar_loader and \ - (impl.expire_missing or key in dict_): + (impl.expire_missing or key in dict_): self.callables[key] = self - dict_.pop(key, None) + old = dict_.pop(key, None) + if impl.collection and old is not None: + impl._invalidate_collection(old) self.manager.dispatch.expire(self, None) - def expire_attributes(self, dict_, attribute_names): - pending = self.__dict__.get('pending', None) - mutable_dict = self.mutable_dict + def _expire_attributes(self, dict_, attribute_names): + pending = self.__dict__.get('_pending_mutations', None) for key in attribute_names: impl = self.manager[key].impl if impl.accepts_scalar_loader: self.callables[key] = self - dict_.pop(key, None) + old = dict_.pop(key, None) + if impl.collection and old is not None: + impl._invalidate_collection(old) self.committed_state.pop(key, None) - if mutable_dict: - mutable_dict.pop(key, None) if pending: pending.pop(key, None) self.manager.dispatch.expire(self, attribute_names) - def __call__(self, passive): + def __call__(self, state, passive): """__call__ allows the InstanceState to act as a deferred callable for loading expired attributes, which is also serializable (picklable). """ - if passive is PASSIVE_NO_FETCH: + if not passive & SQL_OK: return PASSIVE_NO_RESULT toload = self.expired_attributes.\ @@ -305,7 +380,6 @@ class InstanceState(object): return set(keys).intersection(self.manager).\ difference(self.committed_state) - @property def unloaded(self): """Return the set of keys which do not have a loaded value. @@ -318,6 +392,13 @@ class InstanceState(object): difference(self.committed_state).\ difference(self.dict) + @property + def _unloaded_non_object(self): + return self.unloaded.intersection( + attr for attr in self.manager + if self.manager[attr].impl.accepts_scalar_loader + ) + @property def expired_attributes(self): """Return the set of keys which are 'expired' to be loaded by @@ -333,10 +414,9 @@ class InstanceState(object): def _instance_dict(self): return None - def _is_really_none(self): - return self.obj() - - def modified_event(self, dict_, attr, previous, collection=False): + def _modified_event(self, dict_, attr, previous, collection=False): + if not attr.send_modified_events: + return if attr.key not in self.committed_state: if collection: if previous is NEVER_SET: @@ -348,29 +428,33 @@ class InstanceState(object): self.committed_state[attr.key] = previous - # the "or not self.modified" is defensive at - # this point. The assertion below is expected - # to be True: # assert self._strong_obj is None or self.modified - if self._strong_obj is None or not self.modified: + if (self.session_id and self._strong_obj is None) \ + or not self.modified: instance_dict = self._instance_dict() if instance_dict: instance_dict._modified.add(self) - self._strong_obj = self.obj() - if self._strong_obj is None: + # only create _strong_obj link if attached + # to a session + + inst = self.obj() + if self.session_id: + self._strong_obj = inst + + if inst is None: raise orm_exc.ObjectDereferencedError( "Can't emit change event for attribute '%s' - " "parent object of type %s has been garbage " "collected." % ( self.manager[attr.key], - orm_util.state_class_str(self) + base.state_class_str(self) )) self.modified = True - def commit(self, dict_, keys): + def _commit(self, dict_, keys): """Commit attributes. This is used by a partial-attribute load operation to mark committed @@ -380,16 +464,8 @@ class InstanceState(object): this step if a value was not populated in state.dict. """ - class_manager = self.manager - if class_manager.mutable_attributes: - for key in keys: - if key in dict_ and key in class_manager.mutable_attributes: - self.committed_state[key] = self.manager[key].impl.copy(dict_[key]) - else: - self.committed_state.pop(key, None) - else: - for key in keys: - self.committed_state.pop(key, None) + for key in keys: + self.committed_state.pop(key, None) self.expired = False @@ -398,7 +474,7 @@ class InstanceState(object): intersection(dict_): del self.callables[key] - def commit_all(self, dict_, instance_dict=None): + def _commit_all(self, dict_, instance_dict=None): """commit all attributes unconditionally. This is used after a flush() or a full load/refresh @@ -409,137 +485,112 @@ class InstanceState(object): - the "modified" flag is set to False - any "expired" markers/callables for attributes loaded are removed. - Attributes marked as "expired" can potentially remain "expired" after this step - if a value was not populated in state.dict. + Attributes marked as "expired" can potentially remain + "expired" after this step if a value was not populated in state.dict. """ + self._commit_all_states([(self, dict_)], instance_dict) - self.committed_state.clear() - self.__dict__.pop('pending', None) + @classmethod + def _commit_all_states(self, iter, instance_dict=None): + """Mass version of commit_all().""" - callables = self.callables - for key in list(callables): - if key in dict_ and callables[key] is self: - del callables[key] + for state, dict_ in iter: + state.committed_state.clear() + InstanceState._pending_mutations._reset(state) - for key in self.manager.mutable_attributes: - if key in dict_: - self.committed_state[key] = self.manager[key].impl.copy(dict_[key]) + callables = state.callables + for key in list(callables): + if key in dict_ and callables[key] is state: + del callables[key] - if instance_dict and self.modified: - instance_dict._modified.discard(self) + if instance_dict and state.modified: + instance_dict._modified.discard(state) - self.modified = self.expired = False - self._strong_obj = None + state.modified = state.expired = False + state._strong_obj = None -class MutableAttrInstanceState(InstanceState): - """InstanceState implementation for objects that reference 'mutable' - attributes. - Has a more involved "cleanup" handler that checks mutable attributes - for changes upon dereference, resurrecting if needed. +class AttributeState(object): + """Provide an inspection interface corresponding + to a particular attribute on a particular mapped object. + + The :class:`.AttributeState` object is accessed + via the :attr:`.InstanceState.attrs` collection + of a particular :class:`.InstanceState`:: + + from sqlalchemy import inspect + + insp = inspect(some_mapped_object) + attr_state = insp.attrs.some_attribute """ - @util.memoized_property - def mutable_dict(self): - return {} - - def _get_modified(self, dict_=None): - if self.__dict__.get('modified', False): - return True - else: - if dict_ is None: - dict_ = self.dict - for key in self.manager.mutable_attributes: - if self.manager[key].impl.check_mutable_modified(self, dict_): - return True - else: - return False - - def _set_modified(self, value): - self.__dict__['modified'] = value - - modified = property(_get_modified, _set_modified) + def __init__(self, state, key): + self.state = state + self.key = key @property - def unmodified(self): - """a set of keys which have no uncommitted changes""" + def loaded_value(self): + """The current value of this attribute as loaded from the database. - dict_ = self.dict - - return set([ - key for key in self.manager - if (key not in self.committed_state or - (key in self.manager.mutable_attributes and - not self.manager[key].impl.check_mutable_modified(self, dict_)))]) - - def unmodified_intersection(self, keys): - """Return self.unmodified.intersection(keys).""" - - dict_ = self.dict - - return set([ - key for key in keys - if (key not in self.committed_state or - (key in self.manager.mutable_attributes and - not self.manager[key].impl.check_mutable_modified(self, dict_)))]) - - - def _is_really_none(self): - """do a check modified/resurrect. - - This would be called in the extremely rare - race condition that the weakref returned None but - the cleanup handler had not yet established the - __resurrect callable as its replacement. + If the value has not been loaded, or is otherwise not present + in the object's dictionary, returns NO_VALUE. """ - if self.modified: - self.obj = self.__resurrect - return self.obj() - else: - return None + return self.state.dict.get(self.key, NO_VALUE) - def reset(self, dict_, key): - self.mutable_dict.pop(key, None) - InstanceState.reset(self, dict_, key) + @property + def value(self): + """Return the value of this attribute. - def _cleanup(self, ref): - """weakref callback. - - This method may be called by an asynchronous - gc. - - If the state shows pending changes, the weakref - is replaced by the __resurrect callable which will - re-establish an object reference on next access, - else removes this InstanceState from the owning - identity map, if any. + This operation is equivalent to accessing the object's + attribute directly or via ``getattr()``, and will fire + off any pending loader callables if needed. """ - if self._get_modified(self.mutable_dict): - self.obj = self.__resurrect - else: - instance_dict = self._instance_dict() - if instance_dict: - instance_dict.discard(self) - self.dispose() + return self.state.manager[self.key].__get__( + self.state.obj(), self.state.class_) - def __resurrect(self): - """A substitute for the obj() weakref function which resurrects.""" + @property + def history(self): + """Return the current pre-flush change history for + this attribute, via the :class:`.History` interface. - # store strong ref'ed version of the object; will revert - # to weakref when changes are persisted - obj = self.manager.new_instance(state=self) - self.obj = weakref.ref(obj, self._cleanup) - self._strong_obj = obj - obj.__dict__.update(self.mutable_dict) + This method will **not** emit loader callables if the value of the + attribute is unloaded. + + .. seealso:: + + :meth:`.AttributeState.load_history` - retrieve history + using loader callables if the value is not locally present. + + :func:`.attributes.get_history` - underlying function + + """ + return self.state.get_history(self.key, + PASSIVE_NO_INITIALIZE) + + def load_history(self): + """Return the current pre-flush change history for + this attribute, via the :class:`.History` interface. + + This method **will** emit loader callables if the value of the + attribute is unloaded. + + .. seealso:: + + :attr:`.AttributeState.history` + + :func:`.attributes.get_history` - underlying function + + .. versionadded:: 0.9.0 + + """ + return self.state.get_history(self.key, + PASSIVE_OFF ^ INIT_OK) - # re-establishes identity attributes from the key - self.manager.dispatch.resurrect(self) - return obj class PendingCollection(object): """A writable placeholder for an unloaded collection. @@ -564,4 +615,3 @@ class PendingCollection(object): self.added_items.remove(value) else: self.deleted_items.add(value) - diff --git a/libs/sqlalchemy/orm/strategies.py b/libs/sqlalchemy/orm/strategies.py index 2cde3f67..033e3d06 100644 --- a/libs/sqlalchemy/orm/strategies.py +++ b/libs/sqlalchemy/orm/strategies.py @@ -1,5 +1,5 @@ # orm/strategies.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,26 +7,25 @@ """sqlalchemy.orm.interfaces.LoaderStrategy implementations, and related MapperOptions.""" -from sqlalchemy import exc as sa_exc -from sqlalchemy import sql, util, log, event -from sqlalchemy.sql import util as sql_util -from sqlalchemy.sql import visitors -from sqlalchemy.orm import attributes, interfaces, exc as orm_exc -from sqlalchemy.orm.mapper import _none_set -from sqlalchemy.orm.interfaces import ( - LoaderStrategy, StrategizedOption, MapperOption, PropertyOption, - StrategizedProperty +from .. import exc as sa_exc, inspect +from .. import util, log, event +from ..sql import util as sql_util, visitors +from . import ( + attributes, interfaces, exc as orm_exc, loading, + unitofwork, util as orm_util ) -from sqlalchemy.orm import session as sessionlib, unitofwork -from sqlalchemy.orm import util as mapperutil -from sqlalchemy.orm.query import Query +from .state import InstanceState +from .util import _none_set +from . import properties +from .interfaces import ( + LoaderStrategy, StrategizedProperty + ) +from .session import _state_session import itertools def _register_attribute(strategy, mapper, useobject, compare_function=None, typecallable=None, - copy_function=None, - mutable_scalars=False, uselist=False, callable_=None, proxy_property=None, @@ -45,10 +44,10 @@ def _register_attribute(strategy, mapper, useobject, listen_hooks.append(single_parent_validator) if prop.key in prop.parent.validators: - fn, include_removes = prop.parent.validators[prop.key] + fn, opts = prop.parent.validators[prop.key] listen_hooks.append( - lambda desc, prop: mapperutil._validator_events(desc, - prop.key, fn, include_removes) + lambda desc, prop: orm_util._validator_events(desc, + prop.key, fn, **opts) ) if useobject: @@ -71,17 +70,17 @@ def _register_attribute(strategy, mapper, useobject, m.class_, prop.key, parent_token=prop, - mutable_scalars=mutable_scalars, uselist=uselist, - copy_function=copy_function, compare_function=compare_function, useobject=useobject, extension=attribute_ext, - trackparent=useobject and (prop.single_parent or prop.direction is interfaces.ONETOMANY), + trackparent=useobject and (prop.single_parent + or prop.direction is interfaces.ONETOMANY), typecallable=typecallable, callable_=callable_, active_history=active_history, impl_class=impl_class, + send_modified_events=not useobject or not prop.viewonly, doc=prop.doc, **kw ) @@ -89,6 +88,7 @@ def _register_attribute(strategy, mapper, useobject, for hook in listen_hooks: hook(desc, prop) +@properties.ColumnProperty.strategy_for(instrument=False, deferred=False) class UninstrumentedColumnLoader(LoaderStrategy): """Represent the a non-instrumented MapperProperty. @@ -96,27 +96,32 @@ class UninstrumentedColumnLoader(LoaderStrategy): if the argument is against the with_polymorphic selectable. """ - def init(self): + def __init__(self, parent): + super(UninstrumentedColumnLoader, self).__init__(parent) self.columns = self.parent_property.columns - def setup_query(self, context, entity, path, reduced_path, adapter, + def setup_query(self, context, entity, path, loadopt, adapter, column_collection=None, **kwargs): for c in self.columns: if adapter: c = adapter.columns[c] column_collection.append(c) - def create_row_processor(self, context, path, reduced_path, mapper, row, adapter): + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): return None, None, None + +@log.class_logger +@properties.ColumnProperty.strategy_for(instrument=True, deferred=False) class ColumnLoader(LoaderStrategy): """Provide loading behavior for a :class:`.ColumnProperty`.""" - def init(self): + def __init__(self, parent): + super(ColumnLoader, self).__init__(parent) self.columns = self.parent_property.columns self.is_composite = hasattr(self.parent_property, 'composite_class') - def setup_query(self, context, entity, path, reduced_path, + def setup_query(self, context, entity, path, loadopt, adapter, column_collection, **kwargs): for c in self.columns: if adapter: @@ -128,17 +133,16 @@ class ColumnLoader(LoaderStrategy): coltype = self.columns[0].type # TODO: check all columns ? check for foreign key as well? active_history = self.parent_property.active_history or \ - self.columns[0].primary_key + self.columns[0].primary_key or \ + mapper.version_id_col in set(self.columns) _register_attribute(self, mapper, useobject=False, compare_function=coltype.compare_values, - copy_function=coltype.copy_value, - mutable_scalars=self.columns[0].type.is_mutable(), - active_history = active_history + active_history=active_history ) - def create_row_processor(self, context, path, reduced_path, - mapper, row, adapter): + def create_row_processor(self, context, path, + loadopt, mapper, row, adapter): key = self.key # look through list of columns represented here # to see which, if any, is present in the row. @@ -151,69 +155,71 @@ class ColumnLoader(LoaderStrategy): return fetch_col, None, None else: def expire_for_non_present_col(state, dict_, row): - state.expire_attribute_pre_commit(dict_, key) + state._expire_attribute_pre_commit(dict_, key) return expire_for_non_present_col, None, None -log.class_logger(ColumnLoader) + +@log.class_logger +@properties.ColumnProperty.strategy_for(deferred=True, instrument=True) class DeferredColumnLoader(LoaderStrategy): """Provide loading behavior for a deferred :class:`.ColumnProperty`.""" - def create_row_processor(self, context, path, reduced_path, mapper, row, adapter): - col = self.columns[0] - if adapter: - col = adapter.columns[col] - - key = self.key - if col in row: - return self.parent_property._get_strategy(ColumnLoader).\ - create_row_processor( - context, path, reduced_path, mapper, row, adapter) - - elif not self.is_class_level: - def set_deferred_for_local_state(state, dict_, row): - state.set_callable(dict_, key, LoadDeferredColumns(state, key)) - return set_deferred_for_local_state, None, None - else: - def reset_col_for_deferred(state, dict_, row): - # reset state on the key so that deferred callables - # fire off on next access. - state.reset(dict_, key) - return reset_col_for_deferred, None, None - - def init(self): + def __init__(self, parent): + super(DeferredColumnLoader, self).__init__(parent) if hasattr(self.parent_property, 'composite_class'): raise NotImplementedError("Deferred loading for composite " "types not implemented yet") self.columns = self.parent_property.columns self.group = self.parent_property.group + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): + col = self.columns[0] + if adapter: + col = adapter.columns[col] + + key = self.key + if col in row: + return self.parent_property._get_strategy_by_cls(ColumnLoader).\ + create_row_processor( + context, path, loadopt, mapper, row, adapter) + + elif not self.is_class_level: + set_deferred_for_local_state = InstanceState._row_processor( + mapper.class_manager, + LoadDeferredColumns(key), key) + return set_deferred_for_local_state, None, None + else: + def reset_col_for_deferred(state, dict_, row): + # reset state on the key so that deferred callables + # fire off on next access. + state._reset(dict_, key) + return reset_col_for_deferred, None, None + def init_class_attribute(self, mapper): self.is_class_level = True _register_attribute(self, mapper, useobject=False, compare_function=self.columns[0].type.compare_values, - copy_function=self.columns[0].type.copy_value, - mutable_scalars=self.columns[0].type.is_mutable(), callable_=self._load_for_state, expire_missing=False ) - def setup_query(self, context, entity, path, reduced_path, adapter, + def setup_query(self, context, entity, path, loadopt, adapter, only_load_props=None, **kwargs): if ( - self.group is not None and - context.attributes.get(('undefer', self.group), False) + loadopt and self.group and + loadopt.local_opts.get('undefer_group', False) == self.group ) or (only_load_props and self.key in only_load_props): - self.parent_property._get_strategy(ColumnLoader).\ + self.parent_property._get_strategy_by_cls(ColumnLoader).\ setup_query(context, entity, - path, reduced_path, adapter, **kwargs) + path, loadopt, adapter, **kwargs) def _load_for_state(self, state, passive): if not state.key: return attributes.ATTR_EMPTY - if passive is attributes.PASSIVE_NO_FETCH: + if not passive & attributes.SQL_OK: return attributes.PASSIVE_NO_RESULT localparent = state.manager.mapper @@ -224,7 +230,7 @@ class DeferredColumnLoader(LoaderStrategy): localparent.iterate_properties if isinstance(p, StrategizedProperty) and isinstance(p.strategy, DeferredColumnLoader) and - p.group==self.group + p.group == self.group ] else: toload = [self.key] @@ -232,68 +238,53 @@ class DeferredColumnLoader(LoaderStrategy): # narrow the keys down to just those which have no history group = [k for k in toload if k in state.unmodified] - session = sessionlib._state_session(state) + session = _state_session(state) if session is None: raise orm_exc.DetachedInstanceError( "Parent instance %s is not bound to a Session; " "deferred load operation of attribute '%s' cannot proceed" % - (mapperutil.state_str(state), self.key) + (orm_util.state_str(state), self.key) ) query = session.query(localparent) - if query._load_on_ident(state.key, + if loading.load_on_ident(query, state.key, only_load_props=group, refresh_state=state) is None: raise orm_exc.ObjectDeletedError(state) return attributes.ATTR_WAS_SET -log.class_logger(DeferredColumnLoader) + class LoadDeferredColumns(object): """serializable loader object used by DeferredColumnLoader""" - def __init__(self, state, key): - self.state = state + def __init__(self, key): self.key = key - def __call__(self, passive=attributes.PASSIVE_OFF): - state, key = self.state, self.key + def __call__(self, state, passive=attributes.PASSIVE_OFF): + key = self.key localparent = state.manager.mapper prop = localparent._props[key] strategy = prop._strategies[DeferredColumnLoader] return strategy._load_for_state(state, passive) -class DeferredOption(StrategizedOption): - propagate_to_loaders = True - def __init__(self, key, defer=False): - super(DeferredOption, self).__init__(key) - self.defer = defer - - def get_strategy_class(self): - if self.defer: - return DeferredColumnLoader - else: - return ColumnLoader - -class UndeferGroupOption(MapperOption): - propagate_to_loaders = True - - def __init__(self, group): - self.group = group - - def process_query(self, query): - query._attributes[('undefer', self.group)] = True class AbstractRelationshipLoader(LoaderStrategy): """LoaderStratgies which deal with related objects.""" - def init(self): + def __init__(self, parent): + super(AbstractRelationshipLoader, self).__init__(parent) self.mapper = self.parent_property.mapper self.target = self.parent_property.target self.uselist = self.parent_property.uselist + + +@log.class_logger +@properties.RelationshipProperty.strategy_for(lazy="noload") +@properties.RelationshipProperty.strategy_for(lazy=None) class NoLoader(AbstractRelationshipLoader): """Provide loading behavior for a :class:`.RelationshipProperty` with "lazy=None". @@ -306,39 +297,41 @@ class NoLoader(AbstractRelationshipLoader): _register_attribute(self, mapper, useobject=True, uselist=self.parent_property.uselist, - typecallable = self.parent_property.collection_class, + typecallable=self.parent_property.collection_class, ) - def create_row_processor(self, context, path, reduced_path, mapper, row, adapter): + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): def invoke_no_load(state, dict_, row): - state.initialize(self.key) + state._initialize(self.key) return invoke_no_load, None, None -log.class_logger(NoLoader) + +@log.class_logger +@properties.RelationshipProperty.strategy_for(lazy=True) +@properties.RelationshipProperty.strategy_for(lazy="select") class LazyLoader(AbstractRelationshipLoader): """Provide loading behavior for a :class:`.RelationshipProperty` with "lazy=True", that is loads when first accessed. """ - def init(self): - super(LazyLoader, self).init() + def __init__(self, parent): + super(LazyLoader, self).__init__(parent) + join_condition = self.parent_property._join_condition self._lazywhere, \ self._bind_to_col, \ - self._equated_columns = self._create_lazy_clause(self.parent_property) + self._equated_columns = join_condition.create_lazy_clause() self._rev_lazywhere, \ self._rev_bind_to_col, \ - self._rev_equated_columns = self._create_lazy_clause( - self.parent_property, + self._rev_equated_columns = join_condition.create_lazy_clause( reverse_direction=True) self.logger.info("%s lazy loading clause %s", self, self._lazywhere) # determine if our "lazywhere" clause is the same as the mapper's # get() clause. then we can just use mapper.get() - #from sqlalchemy.orm import query self.use_get = not self.uselist and \ self.mapper._get_clause[0].compare( self._lazywhere, @@ -347,7 +340,7 @@ class LazyLoader(AbstractRelationshipLoader): ) if self.use_get: - for col in self._equated_columns.keys(): + for col in list(self._equated_columns): if col in self.mapper._equivalent_columns: for c in self.mapper._equivalent_columns[col]: self._equated_columns[c] = self._equated_columns[col] @@ -358,6 +351,12 @@ class LazyLoader(AbstractRelationshipLoader): def init_class_attribute(self, mapper): self.is_class_level = True + active_history = ( + self.parent_property.active_history or + self.parent_property.direction is not interfaces.MANYTOONE or + not self.use_get + ) + # MANYTOONE currently only needs the # "old" value for delete-orphan # cascades. the required _SingleParentValidator @@ -365,22 +364,19 @@ class LazyLoader(AbstractRelationshipLoader): # in that case. otherwise we don't need the # "old" value during backref operations. _register_attribute(self, - mapper, - useobject=True, - callable_=self._load_for_state, - uselist = self.parent_property.uselist, - backref = self.parent_property.back_populates, - typecallable = self.parent_property.collection_class, - active_history = \ - self.parent_property.active_history or \ - self.parent_property.direction is not \ - interfaces.MANYTOONE or \ - not self.use_get, - ) + mapper, + useobject=True, + callable_=self._load_for_state, + uselist=self.parent_property.uselist, + backref=self.parent_property.back_populates, + typecallable=self.parent_property.collection_class, + active_history=active_history + ) def lazy_clause(self, state, reverse_direction=False, alias_secondary=False, - adapt_source=None): + adapt_source=None, + passive=None): if state is None: return self._lazy_none_clause( reverse_direction, @@ -402,14 +398,13 @@ class LazyLoader(AbstractRelationshipLoader): else: mapper = self.parent_property.parent - o = state.obj() # strong ref + o = state.obj() # strong ref dict_ = attributes.instance_dict(o) # use the "committed state" only if we're in a flush # for this state. - sess = sessionlib._state_session(state) - if sess is not None and sess._flushing: + if passive and passive & attributes.LOAD_AGAINST_COMMITTED: def visit_bindparam(bindparam): if bindparam._identifying_key in bind_to_col: bindparam.callable = \ @@ -424,14 +419,13 @@ class LazyLoader(AbstractRelationshipLoader): state, dict_, bind_to_col[bindparam._identifying_key]) - if self.parent_property.secondary is not None and alias_secondary: criterion = sql_util.ClauseAdapter( self.parent_property.secondary.alias()).\ traverse(criterion) criterion = visitors.cloned_traverse( - criterion, {}, {'bindparam':visit_bindparam}) + criterion, {}, {'bindparam': visit_bindparam}) if adapt_source: criterion = adapt_source(criterion) @@ -457,28 +451,31 @@ class LazyLoader(AbstractRelationshipLoader): def _load_for_state(self, state, passive): if not state.key and \ - (not self.parent_property.load_on_pending or not state.session_id): + ( + ( + not self.parent_property.load_on_pending + and not state._load_pending + ) + or not state.session_id + ): return attributes.ATTR_EMPTY pending = not state.key ident_key = None if ( - (passive is attributes.PASSIVE_NO_FETCH or \ - passive is attributes.PASSIVE_NO_FETCH_RELATED) and - not self.use_get - ) or ( - passive is attributes.PASSIVE_ONLY_PERSISTENT and - pending - ): + (not passive & attributes.SQL_OK and not self.use_get) + or + (not passive & attributes.NON_PERSISTENT_OK and pending) + ): return attributes.PASSIVE_NO_RESULT - session = sessionlib._state_session(state) + session = _state_session(state) if not session: raise orm_exc.DetachedInstanceError( "Parent instance %s is not bound to a Session; " "lazy load operation of attribute '%s' cannot proceed" % - (mapperutil.state_str(state), self.key) + (orm_util.state_str(state), self.key) ) # if we have a simple primary key load, check the @@ -498,40 +495,36 @@ class LazyLoader(AbstractRelationshipLoader): return None ident_key = self.mapper.identity_key_from_primary_key(ident) - instance = Query._get_from_identity(session, ident_key, passive) + instance = loading.get_from_identity(session, ident_key, passive) if instance is not None: return instance - elif passive is attributes.PASSIVE_NO_FETCH or \ - passive is attributes.PASSIVE_NO_FETCH_RELATED: + elif not passive & attributes.SQL_OK or \ + not passive & attributes.RELATED_OBJECT_OK: return attributes.PASSIVE_NO_RESULT - return self._emit_lazyload(session, state, ident_key) + return self._emit_lazyload(session, state, ident_key, passive) def _get_ident_for_use_get(self, session, state, passive): instance_mapper = state.manager.mapper - if session._flushing: + if passive & attributes.LOAD_AGAINST_COMMITTED: get_attr = instance_mapper._get_committed_state_attr_by_column else: get_attr = instance_mapper._get_state_attr_by_column dict_ = state.dict - if passive is attributes.PASSIVE_NO_FETCH_RELATED: - attr_passive = attributes.PASSIVE_OFF - else: - attr_passive = passive - return [ get_attr( state, dict_, self._equated_columns[pk], - passive=attr_passive) + passive=passive) for pk in self.mapper.primary_key ] - def _emit_lazyload(self, session, state, ident_key): + @util.dependencies("sqlalchemy.orm.strategy_options") + def _emit_lazyload(self, strategy_options, session, state, ident_key, passive): q = session.query(self.mapper)._adapt_all_clauses() q = q._with_invoke_all_eagers(False) @@ -543,13 +536,13 @@ class LazyLoader(AbstractRelationshipLoader): q = q.autoflush(False) if state.load_path: - q = q._with_current_path(state.load_path + (self.key,)) + q = q._with_current_path(state.load_path[self.parent_property]) if state.load_options: q = q._conditional_options(*state.load_options) if self.use_get: - return q._load_on_ident(ident_key) + return loading.load_on_ident(q, ident_key) if self.parent_property.order_by: q = q.order_by(*util.to_list(self.parent_property.order_by)) @@ -560,9 +553,9 @@ class LazyLoader(AbstractRelationshipLoader): if rev.direction is interfaces.MANYTOONE and \ rev._use_get and \ not isinstance(rev.strategy, LazyLoader): - q = q.options(EagerLazyOption((rev.key,), lazy='select')) + q = q.options(strategy_options.Load(rev.parent).lazyload(rev.key)) - lazy_clause = self.lazy_clause(state) + lazy_clause = self.lazy_clause(state, passive=passive) if pending: bind_values = sql_util.bind_values(lazy_clause) @@ -587,21 +580,22 @@ class LazyLoader(AbstractRelationshipLoader): else: return None - - def create_row_processor(self, context, path, reduced_path, + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): key = self.key if not self.is_class_level: - def set_lazy_callable(state, dict_, row): - # we are not the primary manager for this attribute - # on this class - set up a - # per-instance lazyloader, which will override the - # class-level behavior. - # this currently only happens when using a - # "lazyload" option on a "no load" - # attribute - "eager" attributes always have a - # class-level lazyloader installed. - state.set_callable(dict_, key, LoadLazyAttribute(state, key)) + # we are not the primary manager for this attribute + # on this class - set up a + # per-instance lazyloader, which will override the + # class-level behavior. + # this currently only happens when using a + # "lazyload" option on a "no load" + # attribute - "eager" attributes always have a + # class-level lazyloader installed. + set_lazy_callable = InstanceState._row_processor( + mapper.class_manager, + LoadLazyAttribute(key), key) + return set_lazy_callable, None, None else: def reset_for_lazy_callable(state, dict_, row): @@ -613,65 +607,20 @@ class LazyLoader(AbstractRelationshipLoader): # this is needed in # populate_existing() types of scenarios to reset # any existing state. - state.reset(dict_, key) + state._reset(dict_, key) return reset_for_lazy_callable, None, None - @classmethod - def _create_lazy_clause(cls, prop, reverse_direction=False): - binds = util.column_dict() - lookup = util.column_dict() - equated_columns = util.column_dict() - if reverse_direction and prop.secondaryjoin is None: - for l, r in prop.local_remote_pairs: - _list = lookup.setdefault(r, []) - _list.append((r, l)) - equated_columns[l] = r - else: - for l, r in prop.local_remote_pairs: - _list = lookup.setdefault(l, []) - _list.append((l, r)) - equated_columns[r] = l - - def col_to_bind(col): - if col in lookup: - for tobind, equated in lookup[col]: - if equated in binds: - return None - if col not in binds: - binds[col] = sql.bindparam(None, None, type_=col.type, unique=True) - return binds[col] - return None - - lazywhere = prop.primaryjoin - - if prop.secondaryjoin is None or not reverse_direction: - lazywhere = visitors.replacement_traverse( - lazywhere, {}, col_to_bind) - - if prop.secondaryjoin is not None: - secondaryjoin = prop.secondaryjoin - if reverse_direction: - secondaryjoin = visitors.replacement_traverse( - secondaryjoin, {}, col_to_bind) - lazywhere = sql.and_(lazywhere, secondaryjoin) - - bind_to_col = dict((binds[col].key, col) for col in binds) - - return lazywhere, bind_to_col, equated_columns - -log.class_logger(LazyLoader) class LoadLazyAttribute(object): """serializable loader object used by LazyLoader""" - def __init__(self, state, key): - self.state = state + def __init__(self, key): self.key = key - def __call__(self, passive=attributes.PASSIVE_OFF): - state, key = self.state, self.key + def __call__(self, state, passive=attributes.PASSIVE_OFF): + key = self.key instance_mapper = state.manager.mapper prop = instance_mapper._props[key] strategy = prop._strategies[LazyLoader] @@ -679,62 +628,72 @@ class LoadLazyAttribute(object): return strategy._load_for_state(state, passive) +@properties.RelationshipProperty.strategy_for(lazy="immediate") class ImmediateLoader(AbstractRelationshipLoader): def init_class_attribute(self, mapper): self.parent_property.\ - _get_strategy(LazyLoader).\ + _get_strategy_by_cls(LazyLoader).\ init_class_attribute(mapper) def setup_query(self, context, entity, - path, reduced_path, adapter, column_collection=None, + path, loadopt, adapter, column_collection=None, parentmapper=None, **kwargs): pass - def create_row_processor(self, context, path, reduced_path, + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): def load_immediate(state, dict_, row): state.get_impl(self.key).get(state, dict_) return None, None, load_immediate + +@log.class_logger +@properties.RelationshipProperty.strategy_for(lazy="subquery") class SubqueryLoader(AbstractRelationshipLoader): - def init(self): - super(SubqueryLoader, self).init() + def __init__(self, parent): + super(SubqueryLoader, self).__init__(parent) self.join_depth = self.parent_property.join_depth def init_class_attribute(self, mapper): self.parent_property.\ - _get_strategy(LazyLoader).\ + _get_strategy_by_cls(LazyLoader).\ init_class_attribute(mapper) def setup_query(self, context, entity, - path, reduced_path, adapter, + path, loadopt, adapter, column_collection=None, parentmapper=None, **kwargs): if not context.query._enable_eagerloads: return - path = path + (self.key, ) - reduced_path = reduced_path + (self.key, ) + path = path[self.parent_property] # build up a path indicating the path from the leftmost # entity to the thing we're subquery loading. - subq_path = context.attributes.get(('subquery_path', None), ()) + with_poly_info = path.get(context.attributes, + "path_with_polymorphic", None) + if with_poly_info is not None: + effective_entity = with_poly_info.entity + else: + effective_entity = self.mapper + + subq_path = context.attributes.get(('subquery_path', None), + orm_util.PathRegistry.root) subq_path = subq_path + path - # join-depth / recursion check - if ("loaderstrategy", reduced_path) not in context.attributes: + # if not via query option, check for + # a cycle + if not path.contains(context.attributes, "loader"): if self.join_depth: - if len(path) / 2 > self.join_depth: - return - else: - if self.mapper.base_mapper in \ - interfaces._reduce_path(subq_path): + if path.length / 2 > self.join_depth: return + elif subq_path.contains_mapper(self.mapper): + return - subq_mapper, leftmost_mapper, leftmost_attr = \ + subq_mapper, leftmost_mapper, leftmost_attr, leftmost_relationship = \ self._get_leftmost(subq_path) orig_query = context.attributes.get( @@ -745,7 +704,8 @@ class SubqueryLoader(AbstractRelationshipLoader): # produce a subquery from it. left_alias = self._generate_from_original_query( orig_query, leftmost_mapper, - leftmost_attr, subq_path + leftmost_attr, leftmost_relationship, + entity.mapper ) # generate another Query that will join the @@ -753,10 +713,10 @@ class SubqueryLoader(AbstractRelationshipLoader): # basically doing a longhand # "from_self()". (from_self() itself not quite industrial # strength enough for all contingencies...but very close) - q = orig_query.session.query(self.mapper) + q = orig_query.session.query(effective_entity) q._attributes = { ("orig_query", SubqueryLoader): orig_query, - ('subquery_path', None) : subq_path + ('subquery_path', None): subq_path } q = q._enable_single_crit(False) @@ -765,46 +725,67 @@ class SubqueryLoader(AbstractRelationshipLoader): q = q.order_by(*local_attr) q = q.add_columns(*local_attr) - q = self._apply_joins(q, to_join, left_alias, parent_alias) + q = self._apply_joins(q, to_join, left_alias, + parent_alias, effective_entity) - q = self._setup_options(q, subq_path, orig_query) + q = self._setup_options(q, subq_path, orig_query, effective_entity) q = self._setup_outermost_orderby(q) # add new query to attributes to be picked up # by create_row_processor - context.attributes[('subquery', reduced_path)] = q + path.set(context.attributes, "subquery", q) def _get_leftmost(self, subq_path): - subq_mapper = mapperutil._class_to_mapper(subq_path[0]) + subq_path = subq_path.path + subq_mapper = orm_util._class_to_mapper(subq_path[0]) # determine attributes of the leftmost mapper - if self.parent.isa(subq_mapper) and self.key==subq_path[1]: + if self.parent.isa(subq_mapper) and self.parent_property is subq_path[1]: leftmost_mapper, leftmost_prop = \ self.parent, self.parent_property else: leftmost_mapper, leftmost_prop = \ subq_mapper, \ - subq_mapper._props[subq_path[1]] - leftmost_cols, remote_cols = self._local_remote_columns(leftmost_prop) + subq_path[1] + + leftmost_cols = leftmost_prop.local_columns leftmost_attr = [ leftmost_mapper._columntoproperty[c].class_attribute for c in leftmost_cols ] - return subq_mapper, leftmost_mapper, leftmost_attr + return subq_mapper, leftmost_mapper, leftmost_attr, leftmost_prop def _generate_from_original_query(self, orig_query, leftmost_mapper, - leftmost_attr, subq_path + leftmost_attr, leftmost_relationship, + entity_mapper ): # reformat the original query # to look only for significant columns - q = orig_query._clone() + q = orig_query._clone().correlate(None) - # TODO: why does polymporphic etc. require hardcoding - # into _adapt_col_list ? Does query.add_columns(...) work - # with polymorphic loading ? - q._set_entities(q._adapt_col_list(leftmost_attr)) + # set a real "from" if not present, as this is more + # accurate than just going off of the column expression + if not q._from_obj and entity_mapper.isa(leftmost_mapper): + q._set_select_from([entity_mapper], False) + + target_cols = q._adapt_col_list(leftmost_attr) + + # select from the identity columns of the outer + q._set_entities(target_cols) + + distinct_target_key = leftmost_relationship.distinct_target_key + + if distinct_target_key is True: + q._distinct = True + elif distinct_target_key is None: + # if target_cols refer to a non-primary key or only + # part of a composite primary key, set the q as distinct + for t in set(c.table for c in target_cols): + if not set(target_cols).issuperset(t.primary_key): + q._distinct = True + break if q._order_by is False: q._order_by = leftmost_mapper.order_by @@ -815,48 +796,66 @@ class SubqueryLoader(AbstractRelationshipLoader): # the original query now becomes a subquery # which we'll join onto. - embed_q = q.with_labels().subquery() - left_alias = mapperutil.AliasedClass(leftmost_mapper, embed_q) - return left_alias + embed_q = q.with_labels().subquery() + left_alias = orm_util.AliasedClass(leftmost_mapper, embed_q, + use_mapper_path=True) + return left_alias def _prep_for_joins(self, left_alias, subq_path): # figure out what's being joined. a.k.a. the fun part - to_join = [ - (subq_path[i], subq_path[i+1]) - for i in xrange(0, len(subq_path), 2) - ] + to_join = [] + pairs = list(subq_path.pairs()) + + for i, (mapper, prop) in enumerate(pairs): + if i > 0: + # look at the previous mapper in the chain - + # if it is as or more specific than this prop's + # mapper, use that instead. + # note we have an assumption here that + # the non-first element is always going to be a mapper, + # not an AliasedClass + + prev_mapper = pairs[i - 1][1].mapper + to_append = prev_mapper if prev_mapper.isa(mapper) else mapper + else: + to_append = mapper + + to_join.append((to_append, prop.key)) # determine the immediate parent class we are joining from, # which needs to be aliased. + if len(to_join) > 1: + info = inspect(to_join[-1][0]) if len(to_join) < 2: # in the case of a one level eager load, this is the # leftmost "left_alias". parent_alias = left_alias - elif subq_path[-2].isa(self.parent): + elif info.mapper.isa(self.parent): # In the case of multiple levels, retrieve # it from subq_path[-2]. This is the same as self.parent # in the vast majority of cases, and [ticket:2014] # illustrates a case where sub_path[-2] is a subclass # of self.parent - parent_alias = mapperutil.AliasedClass(subq_path[-2]) + parent_alias = orm_util.AliasedClass(to_join[-1][0], + use_mapper_path=True) else: # if of_type() were used leading to this relationship, # self.parent is more specific than subq_path[-2] - parent_alias = mapperutil.AliasedClass(self.parent) + parent_alias = orm_util.AliasedClass(self.parent, + use_mapper_path=True) - local_cols, remote_cols = \ - self._local_remote_columns(self.parent_property) + local_cols = self.parent_property.local_columns local_attr = [ getattr(parent_alias, self.parent._columntoproperty[c].key) for c in local_cols ] - return to_join, local_attr, parent_alias - def _apply_joins(self, q, to_join, left_alias, parent_alias): + def _apply_joins(self, q, to_join, left_alias, parent_alias, + effective_entity): for i, (mapper, key) in enumerate(to_join): # we need to use query.join() as opposed to @@ -869,11 +868,18 @@ class SubqueryLoader(AbstractRelationshipLoader): first = i == 0 middle = i < len(to_join) - 1 second_to_last = i == len(to_join) - 2 + last = i == len(to_join) - 1 if first: attr = getattr(left_alias, key) + if last and effective_entity is not self.mapper: + attr = attr.of_type(effective_entity) else: - attr = key + if last and effective_entity is not self.mapper: + attr = getattr(parent_alias, key).\ + of_type(effective_entity) + else: + attr = key if second_to_last: q = q.join(parent_alias, attr, from_joinpoint=True) @@ -881,24 +887,14 @@ class SubqueryLoader(AbstractRelationshipLoader): q = q.join(attr, aliased=middle, from_joinpoint=True) return q - def _local_remote_columns(self, prop): - if prop.secondary is None: - return zip(*prop.local_remote_pairs) - else: - return \ - [p[0] for p in prop.synchronize_pairs],\ - [ - p[0] for p in prop. - secondary_synchronize_pairs - ] - - def _setup_options(self, q, subq_path, orig_query): + def _setup_options(self, q, subq_path, orig_query, effective_entity): # propagate loader options etc. to the new query. # these will fire relative to subq_path. q = q._with_current_path(subq_path) q = q._conditional_options(*orig_query._with_options) if orig_query._populate_existing: q._populate_existing = orig_query._populate_existing + return q def _setup_outermost_orderby(self, q): @@ -919,7 +915,36 @@ class SubqueryLoader(AbstractRelationshipLoader): q = q.order_by(*eager_order_by) return q - def create_row_processor(self, context, path, reduced_path, + class _SubqCollections(object): + """Given a :class:`.Query` used to emit the "subquery load", + provide a load interface that executes the query at the + first moment a value is needed. + + """ + _data = None + + def __init__(self, subq): + self.subq = subq + + def get(self, key, default): + if self._data is None: + self._load() + return self._data.get(key, default) + + def _load(self): + self._data = dict( + (k, [vv[0] for vv in v]) + for k, v in itertools.groupby( + self.subq, + lambda x: x[1:] + ) + ) + + def loader(self, state, dict_, row): + if self._data is None: + self._load() + + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): if not self.parent.class_manager[self.key].impl.supports_population: raise sa_exc.InvalidRequestError( @@ -927,27 +952,22 @@ class SubqueryLoader(AbstractRelationshipLoader): "population - eager loading cannot be applied." % self) - reduced_path = reduced_path + (self.key,) + path = path[self.parent_property] - if ('subquery', reduced_path) not in context.attributes: + subq = path.get(context.attributes, 'subquery') + + if subq is None: return None, None, None - local_cols, remote_cols = self._local_remote_columns(self.parent_property) - - q = context.attributes[('subquery', reduced_path)] + local_cols = self.parent_property.local_columns # cache the loaded collections in the context # so that inheriting mappers don't re-load when they # call upon create_row_processor again - if ('collections', reduced_path) in context.attributes: - collections = context.attributes[('collections', reduced_path)] - else: - collections = context.attributes[('collections', reduced_path)] = dict( - (k, [v[0] for v in v]) - for k, v in itertools.groupby( - q, - lambda x:x[1:] - )) + collections = path.get(context.attributes, "collections") + if collections is None: + collections = self._SubqCollections(subq) + path.set(context.attributes, 'collections', collections) if adapter: local_cols = [adapter.columns[c] for c in local_cols] @@ -966,7 +986,7 @@ class SubqueryLoader(AbstractRelationshipLoader): state.get_impl(self.key).\ set_committed_value(state, dict_, collection) - return load_collection_from_subq, None, None + return load_collection_from_subq, None, None, collections.loader def _create_scalar_loader(self, collections, local_cols): def load_scalar_from_subq(state, dict_, row): @@ -984,108 +1004,191 @@ class SubqueryLoader(AbstractRelationshipLoader): state.get_impl(self.key).\ set_committed_value(state, dict_, scalar) - return load_scalar_from_subq, None, None + return load_scalar_from_subq, None, None, collections.loader -log.class_logger(SubqueryLoader) + +@log.class_logger +@properties.RelationshipProperty.strategy_for(lazy="joined") +@properties.RelationshipProperty.strategy_for(lazy=False) class JoinedLoader(AbstractRelationshipLoader): """Provide loading behavior for a :class:`.RelationshipProperty` using joined eager loading. """ - def init(self): - super(JoinedLoader, self).init() + def __init__(self, parent): + super(JoinedLoader, self).__init__(parent) self.join_depth = self.parent_property.join_depth def init_class_attribute(self, mapper): self.parent_property.\ - _get_strategy(LazyLoader).init_class_attribute(mapper) + _get_strategy_by_cls(LazyLoader).init_class_attribute(mapper) - def setup_query(self, context, entity, path, reduced_path, adapter, \ + def setup_query(self, context, entity, path, loadopt, adapter, \ column_collection=None, parentmapper=None, allow_innerjoin=True, **kwargs): """Add a left outer join to the statement thats being constructed.""" - if not context.query._enable_eagerloads: return - path = path + (self.key,) - reduced_path = reduced_path + (self.key,) + path = path[self.parent_property] - if ("user_defined_eager_row_processor", reduced_path) in\ - context.attributes: + with_polymorphic = None + + user_defined_adapter = self._init_user_defined_eager_proc( + loadopt, context) if loadopt else False + + if user_defined_adapter is not False: clauses, adapter, add_to_collection = \ - self._get_user_defined_adapter( - context, entity, reduced_path, adapter + self._setup_query_on_user_defined_adapter( + context, entity, path, adapter, + user_defined_adapter ) else: - # check for join_depth or basic recursion, - # if the current path was not explicitly stated as - # a desired "loaderstrategy" (i.e. via query.options()) - if ("loaderstrategy", reduced_path) not in context.attributes: + # if not via query option, check for + # a cycle + if not path.contains(context.attributes, "loader"): if self.join_depth: - if len(path) / 2 > self.join_depth: - return - else: - if self.mapper.base_mapper in reduced_path: + if path.length / 2 > self.join_depth: return + elif path.contains_mapper(self.mapper): + return clauses, adapter, add_to_collection, \ allow_innerjoin = self._generate_row_adapter( - context, entity, path, reduced_path, adapter, + context, entity, path, loadopt, adapter, column_collection, parentmapper, allow_innerjoin ) - path += (self.mapper,) - reduced_path += (self.mapper.base_mapper,) + with_poly_info = path.get( + context.attributes, + "path_with_polymorphic", + None + ) + if with_poly_info is not None: + with_polymorphic = with_poly_info.with_polymorphic_mappers + else: + with_polymorphic = None - for value in self.mapper._polymorphic_properties: + path = path[self.mapper] + + for value in self.mapper._iterate_polymorphic_properties( + mappers=with_polymorphic): value.setup( context, entity, path, - reduced_path, clauses, parentmapper=self.mapper, column_collection=add_to_collection, allow_innerjoin=allow_innerjoin) - def _get_user_defined_adapter(self, context, entity, - reduced_path, adapter): - clauses = context.attributes[ - ("user_defined_eager_row_processor", - reduced_path)] + if with_poly_info is not None and \ + None in set(context.secondary_columns): + raise sa_exc.InvalidRequestError( + "Detected unaliased columns when generating joined " + "load. Make sure to use aliased=True or flat=True " + "when using joined loading with with_polymorphic()." + ) - adapter = entity._get_entity_clauses(context.query, context) - if adapter and clauses: - context.attributes[ - ("user_defined_eager_row_processor", - reduced_path)] = clauses = clauses.wrap(adapter) - elif adapter: - context.attributes[ - ("user_defined_eager_row_processor", - reduced_path)] = clauses = adapter + def _init_user_defined_eager_proc(self, loadopt, context): - add_to_collection = context.primary_columns - return clauses, adapter, add_to_collection + # check if the opt applies at all + if "eager_from_alias" not in loadopt.local_opts: + # nope + return False + + path = loadopt.path.parent + + # the option applies. check if the "user_defined_eager_row_processor" + # has been built up. + adapter = path.get(context.attributes, + "user_defined_eager_row_processor", False) + if adapter is not False: + # just return it + return adapter + + # otherwise figure it out. + alias = loadopt.local_opts["eager_from_alias"] + + root_mapper, prop = path[-2:] + + #from .mapper import Mapper + #from .interfaces import MapperProperty + #assert isinstance(root_mapper, Mapper) + #assert isinstance(prop, MapperProperty) + + if alias is not None: + if isinstance(alias, str): + alias = prop.target.alias(alias) + adapter = sql_util.ColumnAdapter(alias, + equivalents=prop.mapper._equivalent_columns) + else: + if path.contains(context.attributes, "path_with_polymorphic"): + with_poly_info = path.get(context.attributes, + "path_with_polymorphic") + adapter = orm_util.ORMAdapter( + with_poly_info.entity, + equivalents=prop.mapper._equivalent_columns) + else: + adapter = context.query._polymorphic_adapters.get(prop.mapper, None) + path.set(context.attributes, + "user_defined_eager_row_processor", + adapter) + + return adapter + + def _setup_query_on_user_defined_adapter(self, context, entity, + path, adapter, user_defined_adapter): + + # apply some more wrapping to the "user defined adapter" + # if we are setting up the query for SQL render. + adapter = entity._get_entity_clauses(context.query, context) + + if adapter and user_defined_adapter: + user_defined_adapter = user_defined_adapter.wrap(adapter) + path.set(context.attributes, "user_defined_eager_row_processor", + user_defined_adapter) + elif adapter: + user_defined_adapter = adapter + path.set(context.attributes, "user_defined_eager_row_processor", + user_defined_adapter) + + add_to_collection = context.primary_columns + return user_defined_adapter, adapter, add_to_collection def _generate_row_adapter(self, - context, entity, path, reduced_path, adapter, + context, entity, path, loadopt, adapter, column_collection, parentmapper, allow_innerjoin ): - clauses = mapperutil.ORMAdapter( - mapperutil.AliasedClass(self.mapper), + with_poly_info = path.get( + context.attributes, + "path_with_polymorphic", + None + ) + if with_poly_info: + to_adapt = with_poly_info.entity + else: + to_adapt = orm_util.AliasedClass(self.mapper, + flat=True, + use_mapper_path=True) + clauses = orm_util.ORMAdapter( + to_adapt, equivalents=self.mapper._equivalent_columns, adapt_required=True) + assert clauses.aliased_class is not None if self.parent_property.direction != interfaces.MANYTOONE: context.multi_row_eager_loaders = True - innerjoin = allow_innerjoin and context.attributes.get( - ("eager_join_type", path), - self.parent_property.innerjoin) + innerjoin = allow_innerjoin and ( + loadopt.local_opts.get( + 'innerjoin', self.parent_property.innerjoin) + if loadopt is not None + else self.parent_property.innerjoin + ) if not innerjoin: # if this is an outer join, all eager joins from # here must also be outer joins @@ -1098,9 +1201,8 @@ class JoinedLoader(AbstractRelationshipLoader): ) add_to_collection = context.secondary_columns - context.attributes[ - ("eager_row_processor", reduced_path) - ] = clauses + path.set(context.attributes, "eager_row_processor", clauses) + return clauses, adapter, add_to_collection, allow_innerjoin def _create_eager_join(self, context, entity, @@ -1119,6 +1221,7 @@ class JoinedLoader(AbstractRelationshipLoader): context.query._should_nest_selectable entity_key = None + if entity not in context.eager_joins and \ not should_nest_selectable and \ context.from_clause: @@ -1137,7 +1240,6 @@ class JoinedLoader(AbstractRelationshipLoader): towrap = context.eager_joins.setdefault(entity_key, default_towrap) - join_to_left = False if adapter: if getattr(adapter, 'aliased_class', None): onclause = getattr( @@ -1145,27 +1247,23 @@ class JoinedLoader(AbstractRelationshipLoader): self.parent_property) else: onclause = getattr( - mapperutil.AliasedClass( + orm_util.AliasedClass( self.parent, - adapter.selectable + adapter.selectable, + use_mapper_path=True ), self.key, self.parent_property ) - if onclause is self.parent_property: - # TODO: this is a temporary hack to - # account for polymorphic eager loads where - # the eagerload is referencing via of_type(). - join_to_left = True else: onclause = self.parent_property + assert clauses.aliased_class is not None context.eager_joins[entity_key] = eagerjoin = \ - mapperutil.join( + orm_util.join( towrap, clauses.aliased_class, onclause, - join_to_left=join_to_left, isouter=not innerjoin ) @@ -1181,7 +1279,7 @@ class JoinedLoader(AbstractRelationshipLoader): # by the Query propagates those columns outward. # This has the effect # of "undefering" those columns. - for col in sql_util.find_columns( + for col in sql_util._find_columns( self.parent_property.primaryjoin): if localparent.mapped_table.c.contains_column(col): if adapter: @@ -1197,13 +1295,12 @@ class JoinedLoader(AbstractRelationshipLoader): ) ) + def _create_eager_adapter(self, context, row, adapter, path, loadopt): + user_defined_adapter = self._init_user_defined_eager_proc( + loadopt, context) if loadopt else False - def _create_eager_adapter(self, context, row, adapter, path, reduced_path): - if ("user_defined_eager_row_processor", reduced_path) in \ - context.attributes: - decorator = context.attributes[ - ("user_defined_eager_row_processor", - reduced_path)] + if user_defined_adapter is not False: + decorator = user_defined_adapter # user defined eagerloads are part of the "primary" # portion of the load. # the adapters applied to the Query should be honored. @@ -1211,11 +1308,10 @@ class JoinedLoader(AbstractRelationshipLoader): decorator = decorator.wrap(context.adapter) elif context.adapter: decorator = context.adapter - elif ("eager_row_processor", reduced_path) in context.attributes: - decorator = context.attributes[ - ("eager_row_processor", reduced_path)] else: - return False + decorator = path.get(context.attributes, "eager_row_processor") + if decorator is None: + return False try: self.mapper.identity_key_from_row(row, decorator) @@ -1225,28 +1321,27 @@ class JoinedLoader(AbstractRelationshipLoader): # processor, will cause a degrade to lazy return False - def create_row_processor(self, context, path, reduced_path, mapper, row, adapter): + def create_row_processor(self, context, path, loadopt, mapper, row, adapter): if not self.parent.class_manager[self.key].impl.supports_population: raise sa_exc.InvalidRequestError( "'%s' does not support object " "population - eager loading cannot be applied." % self) - our_path = path + (self.key,) - our_reduced_path = reduced_path + (self.key,) + our_path = path[self.parent_property] eager_adapter = self._create_eager_adapter( context, row, - adapter, our_path, - our_reduced_path) + adapter, our_path, loadopt) if eager_adapter is not False: key = self.key - _instance = self.mapper._instance_processor( + + _instance = loading.instance_processor( + self.mapper, context, - our_path + (self.mapper,), - our_reduced_path + (self.mapper.base_mapper,), + our_path[self.mapper], eager_adapter) if not self.uselist: @@ -1255,10 +1350,9 @@ class JoinedLoader(AbstractRelationshipLoader): return self._create_collection_loader(context, key, _instance) else: return self.parent_property.\ - _get_strategy(LazyLoader).\ + _get_strategy_by_cls(LazyLoader).\ create_row_processor( - context, path, - reduced_path, + context, path, loadopt, mapper, row, adapter) def _create_collection_loader(self, context, key, _instance): @@ -1318,95 +1412,7 @@ class JoinedLoader(AbstractRelationshipLoader): load_scalar_from_joined_existing_row, \ None, load_scalar_from_joined_exec -EagerLoader = JoinedLoader -"""Deprecated, use JoinedLoader""" -log.class_logger(JoinedLoader) - -class EagerLazyOption(StrategizedOption): - def __init__(self, key, lazy=True, chained=False, - propagate_to_loaders=True - ): - if isinstance(key[0], basestring) and key[0] == '*': - if len(key) != 1: - raise sa_exc.ArgumentError( - "Wildcard identifier '*' must " - "be specified alone.") - key = ("relationship:*",) - propagate_to_loaders = False - super(EagerLazyOption, self).__init__(key) - self.lazy = lazy - self.chained = self.lazy in (False, 'joined', 'subquery') and chained - self.propagate_to_loaders = propagate_to_loaders - self.strategy_cls = factory(lazy) - - def get_strategy_class(self): - return self.strategy_cls - -def factory(identifier): - if identifier is False or identifier == 'joined': - return JoinedLoader - elif identifier is None or identifier == 'noload': - return NoLoader - elif identifier is False or identifier == 'select': - return LazyLoader - elif identifier == 'subquery': - return SubqueryLoader - elif identifier == 'immediate': - return ImmediateLoader - else: - return LazyLoader - -class EagerJoinOption(PropertyOption): - - def __init__(self, key, innerjoin, chained=False): - super(EagerJoinOption, self).__init__(key) - self.innerjoin = innerjoin - self.chained = chained - - def process_query_property(self, query, paths, mappers): - if self.chained: - for path in paths: - query._attributes[("eager_join_type", path)] = self.innerjoin - else: - query._attributes[("eager_join_type", paths[-1])] = self.innerjoin - -class LoadEagerFromAliasOption(PropertyOption): - - def __init__(self, key, alias=None, chained=False): - super(LoadEagerFromAliasOption, self).__init__(key) - if alias is not None: - if not isinstance(alias, basestring): - m, alias, is_aliased_class = mapperutil._entity_info(alias) - self.alias = alias - self.chained = chained - - def process_query_property(self, query, paths, mappers): - if self.chained: - for path in paths[0:-1]: - (root_mapper, propname) = path[-2:] - prop = root_mapper._props[propname] - adapter = query._polymorphic_adapters.get(prop.mapper, None) - query._attributes.setdefault( - ("user_defined_eager_row_processor", - interfaces._reduce_path(path)), adapter) - - if self.alias is not None: - if isinstance(self.alias, basestring): - (root_mapper, propname) = paths[-1][-2:] - prop = root_mapper._props[propname] - self.alias = prop.target.alias(self.alias) - query._attributes[ - ("user_defined_eager_row_processor", - interfaces._reduce_path(paths[-1])) - ] = sql_util.ColumnAdapter(self.alias) - else: - (root_mapper, propname) = paths[-1][-2:] - prop = root_mapper._props[propname] - adapter = query._polymorphic_adapters.get(prop.mapper, None) - query._attributes[ - ("user_defined_eager_row_processor", - interfaces._reduce_path(paths[-1]))] = adapter def single_parent_validator(desc, prop): def _do_check(state, value, oldvalue, initiator): @@ -1417,7 +1423,7 @@ def single_parent_validator(desc, prop): "Instance %s is already associated with an instance " "of %s via its %s attribute, and is only allowed a " "single parent." % - (mapperutil.instance_str(value), state.class_, prop) + (orm_util.instance_str(value), state.class_, prop) ) return value @@ -1427,6 +1433,7 @@ def single_parent_validator(desc, prop): def set_(state, value, oldvalue, initiator): return _do_check(state, value, oldvalue, initiator) - event.listen(desc, 'append', append, raw=True, retval=True, active_history=True) - event.listen(desc, 'set', set_, raw=True, retval=True, active_history=True) - + event.listen(desc, 'append', append, raw=True, retval=True, + active_history=True) + event.listen(desc, 'set', set_, raw=True, retval=True, + active_history=True) diff --git a/libs/sqlalchemy/orm/strategy_options.py b/libs/sqlalchemy/orm/strategy_options.py new file mode 100644 index 00000000..6e838ccb --- /dev/null +++ b/libs/sqlalchemy/orm/strategy_options.py @@ -0,0 +1,924 @@ +# orm/strategy_options.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" + +""" + +from .interfaces import MapperOption, PropComparator +from .. import util +from ..sql.base import _generative, Generative +from .. import exc as sa_exc, inspect +from .base import _is_aliased_class, _class_to_mapper +from . import util as orm_util +from .path_registry import PathRegistry, TokenRegistry, \ + _WILDCARD_TOKEN, _DEFAULT_TOKEN + +class Load(Generative, MapperOption): + """Represents loader options which modify the state of a + :class:`.Query` in order to affect how various mapped attributes are loaded. + + .. versionadded:: 0.9.0 The :meth:`.Load` system is a new foundation for + the existing system of loader options, including options such as + :func:`.orm.joinedload`, :func:`.orm.defer`, and others. In particular, + it introduces a new method-chained system that replaces the need for + dot-separated paths as well as "_all()" options such as :func:`.orm.joinedload_all`. + + A :class:`.Load` object can be used directly or indirectly. To use one + directly, instantiate given the parent class. This style of usage is + useful when dealing with a :class:`.Query` that has multiple entities, + or when producing a loader option that can be applied generically to + any style of query:: + + myopt = Load(MyClass).joinedload("widgets") + + The above ``myopt`` can now be used with :meth:`.Query.options`:: + + session.query(MyClass).options(myopt) + + The :class:`.Load` construct is invoked indirectly whenever one makes use + of the various loader options that are present in ``sqlalchemy.orm``, including + options such as :func:`.orm.joinedload`, :func:`.orm.defer`, :func:`.orm.subqueryload`, + and all the rest. These constructs produce an "anonymous" form of the + :class:`.Load` object which tracks attributes and options, but is not linked + to a parent class until it is associated with a parent :class:`.Query`:: + + # produce "unbound" Load object + myopt = joinedload("widgets") + + # when applied using options(), the option is "bound" to the + # class observed in the given query, e.g. MyClass + session.query(MyClass).options(myopt) + + Whether the direct or indirect style is used, the :class:`.Load` object + returned now represents a specific "path" along the entities of a :class:`.Query`. + This path can be traversed using a standard method-chaining approach. + Supposing a class hierarchy such as ``User``, ``User.addresses -> Address``, + ``User.orders -> Order`` and ``Order.items -> Item``, we can specify a variety + of loader options along each element in the "path":: + + session.query(User).options( + joinedload("addresses"), + subqueryload("orders").joinedload("items") + ) + + Where above, the ``addresses`` collection will be joined-loaded, the + ``orders`` collection will be subquery-loaded, and within that subquery load + the ``items`` collection will be joined-loaded. + + + """ + def __init__(self, entity): + insp = inspect(entity) + self.path = insp._path_registry + self.context = {} + self.local_opts = {} + + def _generate(self): + cloned = super(Load, self)._generate() + cloned.local_opts = {} + return cloned + + strategy = None + propagate_to_loaders = False + + def process_query(self, query): + self._process(query, True) + + def process_query_conditionally(self, query): + self._process(query, False) + + def _process(self, query, raiseerr): + current_path = query._current_path + if current_path: + for (token, start_path), loader in self.context.items(): + chopped_start_path = self._chop_path(start_path, current_path) + if chopped_start_path is not None: + query._attributes[(token, chopped_start_path)] = loader + else: + query._attributes.update(self.context) + + def _generate_path(self, path, attr, wildcard_key, raiseerr=True): + if raiseerr and not path.has_entity: + if isinstance(path, TokenRegistry): + raise sa_exc.ArgumentError( + "Wildcard token cannot be followed by another entity") + else: + raise sa_exc.ArgumentError( + "Attribute '%s' of entity '%s' does not " + "refer to a mapped entity" % + (path.prop.key, path.parent.entity) + ) + + if isinstance(attr, util.string_types): + default_token = attr.endswith(_DEFAULT_TOKEN) + if attr.endswith(_WILDCARD_TOKEN) or default_token: + if default_token: + self.propagate_to_loaders = False + if wildcard_key: + attr = "%s:%s" % (wildcard_key, attr) + return path.token(attr) + + try: + # use getattr on the class to work around + # synonyms, hybrids, etc. + attr = getattr(path.entity.class_, attr) + except AttributeError: + if raiseerr: + raise sa_exc.ArgumentError( + "Can't find property named '%s' on the " + "mapped entity %s in this Query. " % ( + attr, path.entity) + ) + else: + return None + else: + attr = attr.property + + path = path[attr] + else: + prop = attr.property + + if not prop.parent.common_parent(path.mapper): + if raiseerr: + raise sa_exc.ArgumentError("Attribute '%s' does not " + "link from element '%s'" % (attr, path.entity)) + else: + return None + + if getattr(attr, '_of_type', None): + ac = attr._of_type + ext_info = inspect(ac) + + path_element = ext_info.mapper + if not ext_info.is_aliased_class: + ac = orm_util.with_polymorphic( + ext_info.mapper.base_mapper, + ext_info.mapper, aliased=True, + _use_mapper_path=True) + path.entity_path[prop].set(self.context, + "path_with_polymorphic", inspect(ac)) + path = path[prop][path_element] + else: + path = path[prop] + + if path.has_entity: + path = path.entity_path + return path + + def _coerce_strat(self, strategy): + if strategy is not None: + strategy = tuple(sorted(strategy.items())) + return strategy + + @_generative + def set_relationship_strategy(self, attr, strategy, propagate_to_loaders=True): + strategy = self._coerce_strat(strategy) + + self.propagate_to_loaders = propagate_to_loaders + # if the path is a wildcard, this will set propagate_to_loaders=False + self.path = self._generate_path(self.path, attr, "relationship") + self.strategy = strategy + if strategy is not None: + self._set_path_strategy() + + @_generative + def set_column_strategy(self, attrs, strategy, opts=None): + strategy = self._coerce_strat(strategy) + + for attr in attrs: + path = self._generate_path(self.path, attr, "column") + cloned = self._generate() + cloned.strategy = strategy + cloned.path = path + cloned.propagate_to_loaders = True + if opts: + cloned.local_opts.update(opts) + cloned._set_path_strategy() + + def _set_path_strategy(self): + if self.path.has_entity: + self.path.parent.set(self.context, "loader", self) + else: + self.path.set(self.context, "loader", self) + + def __getstate__(self): + d = self.__dict__.copy() + d["path"] = self.path.serialize() + return d + + def __setstate__(self, state): + self.__dict__.update(state) + self.path = PathRegistry.deserialize(self.path) + + def _chop_path(self, to_chop, path): + i = -1 + + for i, (c_token, p_token) in enumerate(zip(to_chop, path.path)): + if isinstance(c_token, util.string_types): + # TODO: this is approximated from the _UnboundLoad + # version and probably has issues, not fully covered. + + if i == 0 and c_token.endswith(':' + _DEFAULT_TOKEN): + return to_chop + elif c_token != 'relationship:%s' % (_WILDCARD_TOKEN,) and c_token != p_token.key: + return None + + if c_token is p_token: + continue + else: + return None + return to_chop[i+1:] + + +class _UnboundLoad(Load): + """Represent a loader option that isn't tied to a root entity. + + The loader option will produce an entity-linked :class:`.Load` + object when it is passed :meth:`.Query.options`. + + This provides compatibility with the traditional system + of freestanding options, e.g. ``joinedload('x.y.z')``. + + """ + def __init__(self): + self.path = () + self._to_bind = set() + self.local_opts = {} + + _is_chain_link = False + + def _set_path_strategy(self): + self._to_bind.add(self) + + def _generate_path(self, path, attr, wildcard_key): + if wildcard_key and isinstance(attr, util.string_types) and \ + attr in (_WILDCARD_TOKEN, _DEFAULT_TOKEN): + if attr == _DEFAULT_TOKEN: + self.propagate_to_loaders = False + attr = "%s:%s" % (wildcard_key, attr) + + return path + (attr, ) + + def __getstate__(self): + d = self.__dict__.copy() + d['path'] = ret = [] + for token in util.to_list(self.path): + if isinstance(token, PropComparator): + ret.append((token._parentmapper.class_, token.key)) + else: + ret.append(token) + return d + + def __setstate__(self, state): + ret = [] + for key in state['path']: + if isinstance(key, tuple): + cls, propkey = key + ret.append(getattr(cls, propkey)) + else: + ret.append(key) + state['path'] = tuple(ret) + self.__dict__ = state + + def _process(self, query, raiseerr): + for val in self._to_bind: + val._bind_loader(query, query._attributes, raiseerr) + + @classmethod + def _from_keys(self, meth, keys, chained, kw): + opt = _UnboundLoad() + + def _split_key(key): + if isinstance(key, util.string_types): + # coerce fooload('*') into "default loader strategy" + if key == _WILDCARD_TOKEN: + return (_DEFAULT_TOKEN, ) + # coerce fooload(".*") into "wildcard on default entity" + elif key.startswith("." + _WILDCARD_TOKEN): + key = key[1:] + return key.split(".") + else: + return (key,) + all_tokens = [token for key in keys for token in _split_key(key)] + + for token in all_tokens[0:-1]: + if chained: + opt = meth(opt, token, **kw) + else: + opt = opt.defaultload(token) + opt._is_chain_link = True + + opt = meth(opt, all_tokens[-1], **kw) + opt._is_chain_link = False + + return opt + + + def _chop_path(self, to_chop, path): + i = -1 + for i, (c_token, (p_mapper, p_prop)) in enumerate(zip(to_chop, path.pairs())): + if isinstance(c_token, util.string_types): + if i == 0 and c_token.endswith(':' + _DEFAULT_TOKEN): + return to_chop + elif c_token != 'relationship:%s' % (_WILDCARD_TOKEN,) and c_token != p_prop.key: + return None + elif isinstance(c_token, PropComparator): + if c_token.property is not p_prop: + return None + else: + i += 1 + + return to_chop[i:] + + + def _bind_loader(self, query, context, raiseerr): + start_path = self.path + # _current_path implies we're in a + # secondary load with an existing path + + current_path = query._current_path + if current_path: + start_path = self._chop_path(start_path, current_path) + + if not start_path: + return None + + token = start_path[0] + if isinstance(token, util.string_types): + entity = self._find_entity_basestring(query, token, raiseerr) + elif isinstance(token, PropComparator): + prop = token.property + entity = self._find_entity_prop_comparator( + query, + prop.key, + token._parententity, + raiseerr) + + else: + raise sa_exc.ArgumentError( + "mapper option expects " + "string key or list of attributes") + + if not entity: + return + + path_element = entity.entity_zero + + # transfer our entity-less state into a Load() object + # with a real entity path. + loader = Load(path_element) + loader.context = context + loader.strategy = self.strategy + + path = loader.path + for token in start_path: + loader.path = path = loader._generate_path( + loader.path, token, None, raiseerr) + if path is None: + return + + loader.local_opts.update(self.local_opts) + + if loader.path.has_entity: + effective_path = loader.path.parent + else: + effective_path = loader.path + + # prioritize "first class" options over those + # that were "links in the chain", e.g. "x" and "y" in someload("x.y.z") + # versus someload("x") / someload("x.y") + if self._is_chain_link: + effective_path.setdefault(context, "loader", loader) + else: + effective_path.set(context, "loader", loader) + + def _find_entity_prop_comparator(self, query, token, mapper, raiseerr): + if _is_aliased_class(mapper): + searchfor = mapper + else: + searchfor = _class_to_mapper(mapper) + for ent in query._mapper_entities: + if ent.corresponds_to(searchfor): + return ent + else: + if raiseerr: + if not list(query._mapper_entities): + raise sa_exc.ArgumentError( + "Query has only expression-based entities - " + "can't find property named '%s'." + % (token, ) + ) + else: + raise sa_exc.ArgumentError( + "Can't find property '%s' on any entity " + "specified in this Query. Note the full path " + "from root (%s) to target entity must be specified." + % (token, ",".join(str(x) for + x in query._mapper_entities)) + ) + else: + return None + + def _find_entity_basestring(self, query, token, raiseerr): + if token.endswith(':' + _WILDCARD_TOKEN): + if len(list(query._mapper_entities)) != 1: + if raiseerr: + raise sa_exc.ArgumentError( + "Wildcard loader can only be used with exactly " + "one entity. Use Load(ent) to specify " + "specific entities.") + + for ent in query._mapper_entities: + # return only the first _MapperEntity when searching + # based on string prop name. Ideally object + # attributes are used to specify more exactly. + return ent + else: + if raiseerr: + raise sa_exc.ArgumentError( + "Query has only expression-based entities - " + "can't find property named '%s'." + % (token, ) + ) + else: + return None + + + +class loader_option(object): + def __init__(self): + pass + + def __call__(self, fn): + self.name = name = fn.__name__ + self.fn = fn + if hasattr(Load, name): + raise TypeError("Load class already has a %s method." % (name)) + setattr(Load, name, fn) + + return self + + def _add_unbound_fn(self, fn): + self._unbound_fn = fn + fn_doc = self.fn.__doc__ + self.fn.__doc__ = """Produce a new :class:`.Load` object with the +:func:`.orm.%(name)s` option applied. + +See :func:`.orm.%(name)s` for usage examples. + +""" % {"name": self.name} + + fn.__doc__ = fn_doc + return self + + def _add_unbound_all_fn(self, fn): + self._unbound_all_fn = fn + fn.__doc__ = """Produce a standalone "all" option for :func:`.orm.%(name)s`. + +.. deprecated:: 0.9.0 + + The "_all()" style is replaced by method chaining, e.g.:: + + session.query(MyClass).options( + %(name)s("someattribute").%(name)s("anotherattribute") + ) + +""" % {"name": self.name} + return self + +@loader_option() +def contains_eager(loadopt, attr, alias=None): + """Indicate that the given attribute should be eagerly loaded from + columns stated manually in the query. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + The option is used in conjunction with an explicit join that loads + the desired rows, i.e.:: + + sess.query(Order).\\ + join(Order.user).\\ + options(contains_eager(Order.user)) + + The above query would join from the ``Order`` entity to its related + ``User`` entity, and the returned ``Order`` objects would have the + ``Order.user`` attribute pre-populated. + + :func:`contains_eager` also accepts an `alias` argument, which is the + string name of an alias, an :func:`~sqlalchemy.sql.expression.alias` + construct, or an :func:`~sqlalchemy.orm.aliased` construct. Use this when + the eagerly-loaded rows are to come from an aliased table:: + + user_alias = aliased(User) + sess.query(Order).\\ + join((user_alias, Order.user)).\\ + options(contains_eager(Order.user, alias=user_alias)) + + .. seealso:: + + :ref:`contains_eager` + + """ + if alias is not None: + if not isinstance(alias, str): + info = inspect(alias) + alias = info.selectable + + cloned = loadopt.set_relationship_strategy( + attr, + {"lazy": "joined"}, + propagate_to_loaders=False + ) + cloned.local_opts['eager_from_alias'] = alias + return cloned + +@contains_eager._add_unbound_fn +def contains_eager(*keys, **kw): + return _UnboundLoad()._from_keys(_UnboundLoad.contains_eager, keys, True, kw) + +@loader_option() +def load_only(loadopt, *attrs): + """Indicate that for a particular entity, only the given list + of column-based attribute names should be loaded; all others will be + deferred. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + Example - given a class ``User``, load only the ``name`` and ``fullname`` + attributes:: + + session.query(User).options(load_only("name", "fullname")) + + Example - given a relationship ``User.addresses -> Address``, specify + subquery loading for the ``User.addresses`` collection, but on each ``Address`` + object load only the ``email_address`` attribute:: + + session.query(User).options( + subqueryload("addreses").load_only("email_address") + ) + + For a :class:`.Query` that has multiple entities, the lead entity can be + specifically referred to using the :class:`.Load` constructor:: + + session.query(User, Address).join(User.addresses).options( + Load(User).load_only("name", "fullname"), + Load(Address).load_only("email_addres") + ) + + + .. versionadded:: 0.9.0 + + """ + cloned = loadopt.set_column_strategy( + attrs, + {"deferred": False, "instrument": True} + ) + cloned.set_column_strategy("*", + {"deferred": True, "instrument": True}) + return cloned + +@load_only._add_unbound_fn +def load_only(*attrs): + return _UnboundLoad().load_only(*attrs) + +@loader_option() +def joinedload(loadopt, attr, innerjoin=None): + """Indicate that the given attribute should be loaded using joined + eager loading. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + examples:: + + # joined-load the "orders" collection on "User" + query(User).options(joinedload(User.orders)) + + # joined-load Order.items and then Item.keywords + query(Order).options(joinedload(Order.items).joinedload(Item.keywords)) + + # lazily load Order.items, but when Items are loaded, + # joined-load the keywords collection + query(Order).options(lazyload(Order.items).joinedload(Item.keywords)) + + :func:`.orm.joinedload` also accepts a keyword argument `innerjoin=True` which + indicates using an inner join instead of an outer:: + + query(Order).options(joinedload(Order.user, innerjoin=True)) + + .. note:: + + The joins produced by :func:`.orm.joinedload` are **anonymously aliased**. + The criteria by which the join proceeds cannot be modified, nor can the + :class:`.Query` refer to these joins in any way, including ordering. + + To produce a specific SQL JOIN which is explicitly available, use + :meth:`.Query.join`. To combine explicit JOINs with eager loading + of collections, use :func:`.orm.contains_eager`; see :ref:`contains_eager`. + + .. seealso:: + + :ref:`loading_toplevel` + + :ref:`contains_eager` + + :func:`.orm.subqueryload` + + :func:`.orm.lazyload` + + """ + loader = loadopt.set_relationship_strategy(attr, {"lazy": "joined"}) + if innerjoin is not None: + loader.local_opts['innerjoin'] = innerjoin + return loader + +@joinedload._add_unbound_fn +def joinedload(*keys, **kw): + return _UnboundLoad._from_keys( + _UnboundLoad.joinedload, keys, False, kw) + +@joinedload._add_unbound_all_fn +def joinedload_all(*keys, **kw): + return _UnboundLoad._from_keys( + _UnboundLoad.joinedload, keys, True, kw) + + +@loader_option() +def subqueryload(loadopt, attr): + """Indicate that the given attribute should be loaded using + subquery eager loading. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + examples:: + + # subquery-load the "orders" collection on "User" + query(User).options(subqueryload(User.orders)) + + # subquery-load Order.items and then Item.keywords + query(Order).options(subqueryload(Order.items).subqueryload(Item.keywords)) + + # lazily load Order.items, but when Items are loaded, + # subquery-load the keywords collection + query(Order).options(lazyload(Order.items).subqueryload(Item.keywords)) + + + .. seealso:: + + :ref:`loading_toplevel` + + :func:`.orm.joinedload` + + :func:`.orm.lazyload` + + """ + return loadopt.set_relationship_strategy(attr, {"lazy": "subquery"}) + +@subqueryload._add_unbound_fn +def subqueryload(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.subqueryload, keys, False, {}) + +@subqueryload._add_unbound_all_fn +def subqueryload_all(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.subqueryload, keys, True, {}) + +@loader_option() +def lazyload(loadopt, attr): + """Indicate that the given attribute should be loaded using "lazy" + loading. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + """ + return loadopt.set_relationship_strategy(attr, {"lazy": "select"}) + +@lazyload._add_unbound_fn +def lazyload(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.lazyload, keys, False, {}) + +@lazyload._add_unbound_all_fn +def lazyload_all(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.lazyload, keys, True, {}) + +@loader_option() +def immediateload(loadopt, attr): + """Indicate that the given attribute should be loaded using + an immediate load with a per-attribute SELECT statement. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + .. seealso:: + + :ref:`loading_toplevel` + + :func:`.orm.joinedload` + + :func:`.orm.lazyload` + + """ + loader = loadopt.set_relationship_strategy(attr, {"lazy": "immediate"}) + return loader + +@immediateload._add_unbound_fn +def immediateload(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.immediateload, keys, False, {}) + + +@loader_option() +def noload(loadopt, attr): + """Indicate that the given relationship attribute should remain unloaded. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + :func:`.orm.noload` applies to :func:`.relationship` attributes; for + column-based attributes, see :func:`.orm.defer`. + + """ + + return loadopt.set_relationship_strategy(attr, {"lazy": "noload"}) + +@noload._add_unbound_fn +def noload(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.noload, keys, False, {}) + +@loader_option() +def defaultload(loadopt, attr): + """Indicate an attribute should load using its default loader style. + + This method is used to link to other loader options, such as + to set the :func:`.orm.defer` option on a class that is linked to + a relationship of the parent class being loaded, :func:`.orm.defaultload` + can be used to navigate this path without changing the loading style + of the relationship:: + + session.query(MyClass).options(defaultload("someattr").defer("some_column")) + + .. seealso:: + + :func:`.orm.defer` + + :func:`.orm.undefer` + + """ + return loadopt.set_relationship_strategy( + attr, + None + ) + +@defaultload._add_unbound_fn +def defaultload(*keys): + return _UnboundLoad._from_keys(_UnboundLoad.defaultload, keys, False, {}) + +@loader_option() +def defer(loadopt, key): + """Indicate that the given column-oriented attribute should be deferred, e.g. + not loaded until accessed. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + e.g.:: + + from sqlalchemy.orm import defer + + session.query(MyClass).options( + defer("attribute_one"), + defer("attribute_two")) + + session.query(MyClass).options( + defer(MyClass.attribute_one), + defer(MyClass.attribute_two)) + + To specify a deferred load of an attribute on a related class, + the path can be specified one token at a time, specifying the loading + style for each link along the chain. To leave the loading style + for a link unchanged, use :func:`.orm.defaultload`:: + + session.query(MyClass).options(defaultload("someattr").defer("some_column")) + + A :class:`.Load` object that is present on a certain path can have + :meth:`.Load.defer` called multiple times, each will operate on the same + parent entity:: + + + session.query(MyClass).options( + defaultload("someattr"). + defer("some_column"). + defer("some_other_column"). + defer("another_column") + ) + + :param key: Attribute to be deferred. + + :param \*addl_attrs: Deprecated; this option supports the old 0.8 style + of specifying a path as a series of attributes, which is now superseded + by the method-chained style. + + .. seealso:: + + :ref:`deferred` + + :func:`.orm.undefer` + + """ + return loadopt.set_column_strategy( + (key, ), + {"deferred": True, "instrument": True} + ) + + +@defer._add_unbound_fn +def defer(key, *addl_attrs): + return _UnboundLoad._from_keys(_UnboundLoad.defer, (key, ) + addl_attrs, False, {}) + +@loader_option() +def undefer(loadopt, key): + """Indicate that the given column-oriented attribute should be undeferred, e.g. + specified within the SELECT statement of the entity as a whole. + + The column being undeferred is typically set up on the mapping as a + :func:`.deferred` attribute. + + This function is part of the :class:`.Load` interface and supports + both method-chained and standalone operation. + + Examples:: + + # undefer two columns + session.query(MyClass).options(undefer("col1"), undefer("col2")) + + # undefer all columns specific to a single class using Load + * + session.query(MyClass, MyOtherClass).options(Load(MyClass).undefer("*")) + + :param key: Attribute to be undeferred. + + :param \*addl_attrs: Deprecated; this option supports the old 0.8 style + of specifying a path as a series of attributes, which is now superseded + by the method-chained style. + + .. seealso:: + + :ref:`deferred` + + :func:`.orm.defer` + + :func:`.orm.undefer_group` + + """ + return loadopt.set_column_strategy( + (key, ), + {"deferred": False, "instrument": True} + ) + +@undefer._add_unbound_fn +def undefer(key, *addl_attrs): + return _UnboundLoad._from_keys(_UnboundLoad.undefer, (key, ) + addl_attrs, False, {}) + +@loader_option() +def undefer_group(loadopt, name): + """Indicate that columns within the given deferred group name should be undeferred. + + The columns being undeferred are set up on the mapping as + :func:`.deferred` attributes and include a "group" name. + + E.g:: + + session.query(MyClass).options(undefer_group("large_attrs")) + + To undefer a group of attributes on a related entity, the path can be + spelled out using relationship loader options, such as :func:`.orm.defaultload`:: + + session.query(MyClass).options(defaultload("someattr").undefer_group("large_attrs")) + + .. versionchanged:: 0.9.0 :func:`.orm.undefer_group` is now specific to a + particiular entity load path. + + .. seealso:: + + :ref:`deferred` + + :func:`.orm.defer` + + :func:`.orm.undefer` + + """ + return loadopt.set_column_strategy( + "*", + None, + {"undefer_group": name} + ) + +@undefer_group._add_unbound_fn +def undefer_group(name): + return _UnboundLoad().undefer_group(name) + diff --git a/libs/sqlalchemy/orm/sync.py b/libs/sqlalchemy/orm/sync.py index 3094386b..cf735fc5 100644 --- a/libs/sqlalchemy/orm/sync.py +++ b/libs/sqlalchemy/orm/sync.py @@ -1,5 +1,5 @@ # orm/sync.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -9,7 +9,8 @@ between instances based on join conditions. """ -from sqlalchemy.orm import exc, util as mapperutil, attributes +from . import exc, util as orm_util, attributes + def populate(source, source_mapper, dest, dest_mapper, synchronize_pairs, uowcommit, flag_cascaded_pks): @@ -42,38 +43,45 @@ def populate(source, source_mapper, dest, dest_mapper, r.references(l): uowcommit.attributes[("pk_cascaded", dest, r)] = True + def clear(dest, dest_mapper, synchronize_pairs): for l, r in synchronize_pairs: if r.primary_key: raise AssertionError( - "Dependency rule tried to blank-out primary key " - "column '%s' on instance '%s'" % - (r, mapperutil.state_str(dest)) - ) + "Dependency rule tried to blank-out primary key " + "column '%s' on instance '%s'" % + (r, orm_util.state_str(dest)) + ) try: dest_mapper._set_state_attr_by_column(dest, dest.dict, r, None) except exc.UnmappedColumnError: _raise_col_to_prop(True, None, l, dest_mapper, r) + def update(source, source_mapper, dest, old_prefix, synchronize_pairs): for l, r in synchronize_pairs: try: - oldvalue = source_mapper._get_committed_attr_by_column(source.obj(), l) - value = source_mapper._get_state_attr_by_column(source, source.dict, l) + oldvalue = source_mapper._get_committed_attr_by_column( + source.obj(), l) + value = source_mapper._get_state_attr_by_column( + source, source.dict, l) except exc.UnmappedColumnError: _raise_col_to_prop(False, source_mapper, l, None, r) dest[r.key] = value dest[old_prefix + r.key] = oldvalue + def populate_dict(source, source_mapper, dict_, synchronize_pairs): for l, r in synchronize_pairs: try: - value = source_mapper._get_state_attr_by_column(source, source.dict, l) + value = source_mapper._get_state_attr_by_column( + source, source.dict, l) except exc.UnmappedColumnError: _raise_col_to_prop(False, source_mapper, l, None, r) dict_[r.key] = value + def source_modified(uowcommit, source, source_mapper, synchronize_pairs): """return true if the source object has changes from an old to a new value on the given synchronize pairs @@ -86,23 +94,25 @@ def source_modified(uowcommit, source, source_mapper, synchronize_pairs): _raise_col_to_prop(False, source_mapper, l, None, r) history = uowcommit.get_attribute_history(source, prop.key, attributes.PASSIVE_NO_INITIALIZE) - return bool(history.deleted) + if bool(history.deleted): + return True else: return False -def _raise_col_to_prop(isdest, source_mapper, source_column, dest_mapper, dest_column): + +def _raise_col_to_prop(isdest, source_mapper, source_column, + dest_mapper, dest_column): if isdest: - raise exc.UnmappedColumnError( - "Can't execute sync rule for destination column '%s'; " - "mapper '%s' does not map this column. Try using an explicit" - " `foreign_keys` collection which does not include this column " - "(or use a viewonly=True relation)." % (dest_column, dest_mapper) - ) + raise exc.UnmappedColumnError("Can't execute sync rule for " + "destination column '%s'; mapper '%s' does not map " + "this column. Try using an explicit `foreign_keys` " + "collection which does not include this column (or use " + "a viewonly=True relation)." % (dest_column, + dest_mapper)) else: - raise exc.UnmappedColumnError( - "Can't execute sync rule for source column '%s'; mapper '%s' " - "does not map this column. Try using an explicit `foreign_keys`" - " collection which does not include destination column '%s' (or " - "use a viewonly=True relation)." % - (source_column, source_mapper, dest_column) - ) + raise exc.UnmappedColumnError("Can't execute sync rule for " + "source column '%s'; mapper '%s' does not map this " + "column. Try using an explicit `foreign_keys` " + "collection which does not include destination column " + "'%s' (or use a viewonly=True relation)." + % (source_column, source_mapper, dest_column)) diff --git a/libs/sqlalchemy/orm/unitofwork.py b/libs/sqlalchemy/orm/unitofwork.py index 003d7ae7..2964705a 100644 --- a/libs/sqlalchemy/orm/unitofwork.py +++ b/libs/sqlalchemy/orm/unitofwork.py @@ -1,5 +1,5 @@ # orm/unitofwork.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -12,11 +12,10 @@ organizes them in order of dependency, and executes. """ -from sqlalchemy import util, event -from sqlalchemy.util import topological -from sqlalchemy.orm import attributes, interfaces, persistence -from sqlalchemy.orm import util as mapperutil -session = util.importlater("sqlalchemy.orm", "session") +from .. import util, event +from ..util import topological +from . import attributes, persistence, util as orm_util + def track_cascade_events(descriptor, prop): """Establish event listeners on object attributes which handle @@ -29,25 +28,42 @@ def track_cascade_events(descriptor, prop): # process "save_update" cascade rules for when # an instance is appended to the list of another instance - sess = session._state_session(state) + if item is None: + return + + sess = state.session if sess: + if sess._warn_on_events: + sess._flush_warning("collection append") + prop = state.manager.mapper._props[key] item_state = attributes.instance_state(item) - if prop.cascade.save_update and \ + if prop._cascade.save_update and \ (prop.cascade_backrefs or key == initiator.key) and \ - not sess._contains_state(item_state): + not sess._contains_state(item_state): sess._save_or_update_state(item_state) return item def remove(state, item, initiator): - sess = session._state_session(state) + if item is None: + return + + sess = state.session if sess: + prop = state.manager.mapper._props[key] + + if sess._warn_on_events: + sess._flush_warning( + "collection remove" + if prop.uselist + else "related attribute delete") + # expunge pending orphans item_state = attributes.instance_state(item) - if prop.cascade.delete_orphan and \ + if prop._cascade.delete_orphan and \ item_state in sess._new and \ - prop.mapper._is_orphan(item_state): + prop.mapper._is_orphan(item_state): sess.expunge(item) def set_(state, newvalue, oldvalue, initiator): @@ -56,19 +72,23 @@ def track_cascade_events(descriptor, prop): if oldvalue is newvalue: return newvalue - sess = session._state_session(state) + sess = state.session if sess: + + if sess._warn_on_events: + sess._flush_warning("related attribute set") + prop = state.manager.mapper._props[key] if newvalue is not None: newvalue_state = attributes.instance_state(newvalue) - if prop.cascade.save_update and \ + if prop._cascade.save_update and \ (prop.cascade_backrefs or key == initiator.key) and \ not sess._contains_state(newvalue_state): sess._save_or_update_state(newvalue_state) if oldvalue is not None and \ oldvalue is not attributes.PASSIVE_NO_RESULT and \ - prop.cascade.delete_orphan: + prop._cascade.delete_orphan: # possible to reach here with attributes.NEVER_SET ? oldvalue_state = attributes.instance_state(oldvalue) @@ -154,7 +174,8 @@ class UOWTransaction(object): def get_attribute_history(self, state, key, passive=attributes.PASSIVE_NO_INITIALIZE): - """facade to attributes.get_state_history(), including caching of results.""" + """facade to attributes.get_state_history(), including + caching of results.""" hashkey = ("history", state, key) @@ -166,11 +187,13 @@ class UOWTransaction(object): history, state_history, cached_passive = self.attributes[hashkey] # if the cached lookup was "passive" and now # we want non-passive, do a non-passive lookup and re-cache - if cached_passive is not attributes.PASSIVE_OFF \ - and passive is attributes.PASSIVE_OFF: + + if not cached_passive & attributes.SQL_OK \ + and passive & attributes.SQL_OK: impl = state.manager[key].impl history = impl.get_history(state, state.dict, - attributes.PASSIVE_OFF) + attributes.PASSIVE_OFF | + attributes.LOAD_AGAINST_COMMITTED) if history and impl.uses_objects: state_history = history.as_state() else: @@ -180,12 +203,14 @@ class UOWTransaction(object): impl = state.manager[key].impl # TODO: store the history as (state, object) tuples # so we don't have to keep converting here - history = impl.get_history(state, state.dict, passive) + history = impl.get_history(state, state.dict, passive | + attributes.LOAD_AGAINST_COMMITTED) if history and impl.uses_objects: state_history = history.as_state() else: state_history = history - self.attributes[hashkey] = (history, state_history, passive) + self.attributes[hashkey] = (history, state_history, + passive) return state_history @@ -204,14 +229,14 @@ class UOWTransaction(object): if not state.deleted and operation is not None: util.warn("Object of type %s not in session, %s operation " "along '%s' will not proceed" % - (mapperutil.state_class_str(state), operation, prop)) + (orm_util.state_class_str(state), operation, prop)) return False if state not in self.states: mapper = state.manager.mapper if mapper not in self.mappers: - mapper._per_mapper_flush_actions(self) + self._per_mapper_flush_actions(mapper) self.mappers[mapper].add(state) self.states[state] = (isdelete, listonly) @@ -226,6 +251,20 @@ class UOWTransaction(object): states.add(state) cols.update(post_update_cols) + def _per_mapper_flush_actions(self, mapper): + saves = SaveUpdateAll(self, mapper.base_mapper) + deletes = DeleteAll(self, mapper.base_mapper) + self.dependencies.add((saves, deletes)) + + for dep in mapper._dependency_processors: + dep.per_property_preprocessors(self) + + for prop in mapper.relationships: + if prop.viewonly: + continue + dep = prop._dependency_processor + dep.per_property_preprocessors(self) + @util.memoized_property def _mapper_for_dep(self): """return a dynamic mapping of (Mapper, DependencyProcessor) to @@ -237,7 +276,7 @@ class UOWTransaction(object): """ return util.PopulateDict( - lambda tup:tup[0]._props.get(tup[1].key) is tup[1].prop + lambda tup: tup[0]._props.get(tup[1].key) is tup[1].prop ) def filter_states_for_dep(self, dep, states): @@ -274,7 +313,7 @@ class UOWTransaction(object): # see if the graph of mapper dependencies has cycles. self.cycles = cycles = topological.find_cycles( self.dependencies, - self.postsort_actions.values()) + list(self.postsort_actions.values())) if cycles: # if yes, break the per-mapper actions into @@ -330,32 +369,36 @@ class UOWTransaction(object): postsort_actions): rec.execute(self) - def finalize_flush_changes(self): - """mark processed objects as clean / deleted after a successful flush(). + """mark processed objects as clean / deleted after a successful + flush(). this method is called within the flush() method after the execute() method has succeeded and the transaction has been committed. """ - for state, (isdelete, listonly) in self.states.iteritems(): - if isdelete: - self.session._remove_newly_deleted(state) - else: - # if listonly: - # debug... would like to see how many do this - self.session._register_newly_persistent(state) + states = set(self.states) + isdel = set( + s for (s, (isdelete, listonly)) in self.states.items() + if isdelete + ) + other = states.difference(isdel) + self.session._remove_newly_deleted(isdel) + self.session._register_newly_persistent(other) + class IterateMappersMixin(object): def _mappers(self, uow): if self.fromparent: return iter( - m for m in self.dependency_processor.parent.self_and_descendants + m for m in + self.dependency_processor.parent.self_and_descendants if uow._mapper_for_dep[(m, self.dependency_processor)] ) else: return self.dependency_processor.mapper.self_and_descendants + class Preprocess(IterateMappersMixin): def __init__(self, dependency_processor, fromparent): self.dependency_processor = dependency_processor @@ -396,6 +439,7 @@ class Preprocess(IterateMappersMixin): else: return False + class PostSortRec(object): disabled = False @@ -418,12 +462,14 @@ class PostSortRec(object): ",".join(str(x) for x in self.__dict__.values()) ) + class ProcessAll(IterateMappersMixin, PostSortRec): def __init__(self, uow, dependency_processor, delete, fromparent): self.dependency_processor = dependency_processor self.delete = delete self.fromparent = fromparent - uow.deps[dependency_processor.parent.base_mapper].add(dependency_processor) + uow.deps[dependency_processor.parent.base_mapper].\ + add(dependency_processor) def execute(self, uow): states = self._elements(uow) @@ -453,6 +499,7 @@ class ProcessAll(IterateMappersMixin, PostSortRec): if isdelete == self.delete and not listonly: yield state + class IssuePostUpdate(PostSortRec): def __init__(self, uow, mapper, isdelete): self.mapper = mapper @@ -464,6 +511,7 @@ class IssuePostUpdate(PostSortRec): persistence.post_update(self.mapper, states, uow, cols) + class SaveUpdateAll(PostSortRec): def __init__(self, uow, mapper): self.mapper = mapper @@ -476,17 +524,22 @@ class SaveUpdateAll(PostSortRec): ) def per_state_flush_actions(self, uow): - states = list(uow.states_for_mapper_hierarchy(self.mapper, False, False)) - for rec in self.mapper._per_state_flush_actions( - uow, - states, - False): - yield rec + states = list(uow.states_for_mapper_hierarchy( + self.mapper, False, False)) + base_mapper = self.mapper.base_mapper + delete_all = DeleteAll(uow, base_mapper) + for state in states: + # keep saves before deletes - + # this ensures 'row switch' operations work + action = SaveUpdateState(uow, state, base_mapper) + uow.dependencies.add((action, delete_all)) + yield action for dep in uow.deps[self.mapper]: states_for_prop = uow.filter_states_for_dep(dep, states) dep.per_state_flush_actions(uow, states_for_prop, False) + class DeleteAll(PostSortRec): def __init__(self, uow, mapper): self.mapper = mapper @@ -499,17 +552,22 @@ class DeleteAll(PostSortRec): ) def per_state_flush_actions(self, uow): - states = list(uow.states_for_mapper_hierarchy(self.mapper, True, False)) - for rec in self.mapper._per_state_flush_actions( - uow, - states, - True): - yield rec + states = list(uow.states_for_mapper_hierarchy( + self.mapper, True, False)) + base_mapper = self.mapper.base_mapper + save_all = SaveUpdateAll(uow, base_mapper) + for state in states: + # keep saves before deletes - + # this ensures 'row switch' operations work + action = DeleteState(uow, state, base_mapper) + uow.dependencies.add((save_all, action)) + yield action for dep in uow.deps[self.mapper]: states_for_prop = uow.filter_states_for_dep(dep, states) dep.per_state_flush_actions(uow, states_for_prop, True) + class ProcessState(PostSortRec): def __init__(self, uow, dependency_processor, delete, state): self.dependency_processor = dependency_processor @@ -535,10 +593,11 @@ class ProcessState(PostSortRec): return "%s(%s, %s, delete=%s)" % ( self.__class__.__name__, self.dependency_processor, - mapperutil.state_str(self.state), + orm_util.state_str(self.state), self.delete ) + class SaveUpdateState(PostSortRec): def __init__(self, uow, state, mapper): self.state = state @@ -559,9 +618,10 @@ class SaveUpdateState(PostSortRec): def __repr__(self): return "%s(%s)" % ( self.__class__.__name__, - mapperutil.state_str(self.state) + orm_util.state_str(self.state) ) + class DeleteState(PostSortRec): def __init__(self, uow, state, mapper): self.state = state @@ -582,6 +642,5 @@ class DeleteState(PostSortRec): def __repr__(self): return "%s(%s)" % ( self.__class__.__name__, - mapperutil.state_str(self.state) + orm_util.state_str(self.state) ) - diff --git a/libs/sqlalchemy/orm/util.py b/libs/sqlalchemy/orm/util.py index a8cc80ce..dd85f2ef 100644 --- a/libs/sqlalchemy/orm/util.py +++ b/libs/sqlalchemy/orm/util.py @@ -1,25 +1,26 @@ # orm/util.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy import sql, util, event, exc as sa_exc -from sqlalchemy.sql import expression, util as sql_util, operators -from sqlalchemy.orm.interfaces import MapperExtension, EXT_CONTINUE,\ - PropComparator, MapperProperty -from sqlalchemy.orm import attributes, exc -import operator +from .. import sql, util, event, exc as sa_exc, inspection +from ..sql import expression, util as sql_util, operators +from .interfaces import PropComparator, MapperProperty +from . import attributes import re -mapperlib = util.importlater("sqlalchemy.orm", "mapperlib") +from .base import instance_str, state_str, state_class_str, attribute_str, \ + state_attribute_str, object_mapper, object_state, _none_set +from .base import class_mapper, _class_to_mapper +from .base import _InspectionAttr +from .path_registry import PathRegistry all_cascades = frozenset(("delete", "delete-orphan", "all", "merge", "expunge", "save-update", "refresh-expire", "none")) -_INSTRUMENTOR = ('mapper', 'instrumentor') class CascadeOptions(frozenset): """Keeps track of the options sent to relationship().cascade""" @@ -68,31 +69,53 @@ class CascadeOptions(frozenset): ",".join([x for x in sorted(self)]) ) -def _validator_events(desc, key, validator, include_removes): + +def _validator_events(desc, key, validator, include_removes, include_backrefs): """Runs a validation method on an attribute value to be set or appended.""" + if not include_backrefs: + def detect_is_backref(state, initiator): + impl = state.manager[key].impl + return initiator.impl is not impl + if include_removes: def append(state, value, initiator): - return validator(state.obj(), key, value, False) + if include_backrefs or not detect_is_backref(state, initiator): + return validator(state.obj(), key, value, False) + else: + return value def set_(state, value, oldvalue, initiator): - return validator(state.obj(), key, value, False) + if include_backrefs or not detect_is_backref(state, initiator): + return validator(state.obj(), key, value, False) + else: + return value def remove(state, value, initiator): - validator(state.obj(), key, value, True) + if include_backrefs or not detect_is_backref(state, initiator): + validator(state.obj(), key, value, True) + else: def append(state, value, initiator): - return validator(state.obj(), key, value) + if include_backrefs or not detect_is_backref(state, initiator): + return validator(state.obj(), key, value) + else: + return value def set_(state, value, oldvalue, initiator): - return validator(state.obj(), key, value) + if include_backrefs or not detect_is_backref(state, initiator): + return validator(state.obj(), key, value) + else: + return value event.listen(desc, 'append', append, raw=True, retval=True) event.listen(desc, 'set', set_, raw=True, retval=True) if include_removes: event.listen(desc, "remove", remove, raw=True, retval=True) -def polymorphic_union(table_map, typecolname, aliasname='p_union', cast_nulls=True): + +def polymorphic_union(table_map, typecolname, + aliasname='p_union', cast_nulls=True): """Create a ``UNION`` statement used by a polymorphic mapper. See :ref:`concrete_inheritance` for an example of how @@ -101,20 +124,21 @@ def polymorphic_union(table_map, typecolname, aliasname='p_union', cast_nulls=Tr :param table_map: mapping of polymorphic identities to :class:`.Table` objects. :param typecolname: string name of a "discriminator" column, which will be - derived from the query, producing the polymorphic identity for each row. If - ``None``, no polymorphic discriminator is generated. + derived from the query, producing the polymorphic identity for + each row. If ``None``, no polymorphic discriminator is generated. :param aliasname: name of the :func:`~sqlalchemy.sql.expression.alias()` construct generated. - :param cast_nulls: if True, non-existent columns, which are represented as labeled - NULLs, will be passed into CAST. This is a legacy behavior that is problematic - on some backends such as Oracle - in which case it can be set to False. + :param cast_nulls: if True, non-existent columns, which are represented + as labeled NULLs, will be passed into CAST. This is a legacy behavior + that is problematic on some backends such as Oracle - in which case it + can be set to False. """ colnames = util.OrderedSet() colnamemaps = {} types = {} - for key in table_map.keys(): + for key in table_map: table = table_map[key] # mysql doesnt like selecting from a select; @@ -140,7 +164,7 @@ def polymorphic_union(table_map, typecolname, aliasname='p_union', cast_nulls=Tr return sql.type_coerce(sql.null(), types[name]).label(name) result = [] - for type, table in table_map.iteritems(): + for type, table in table_map.items(): if typecolname is not None: result.append( sql.select([col(name, table) for name in colnames] + @@ -152,32 +176,61 @@ def polymorphic_union(table_map, typecolname, aliasname='p_union', cast_nulls=Tr from_obj=[table])) return sql.union_all(*result).alias(aliasname) -def identity_key(*args, **kwargs): - """Get an identity key. - Valid call signatures: +def identity_key(*args, **kwargs): + """Generate "identity key" tuples, as are used as keys in the + :attr:`.Session.identity_map` dictionary. + + This function has several call styles: * ``identity_key(class, ident)`` - class - mapped class (must be a positional argument) + This form receives a mapped class and a primary key scalar or + tuple as an argument. - ident - primary key, if the key is composite this is a tuple + E.g.:: + + >>> identity_key(MyClass, (1, 2)) + (, (1, 2)) + + :param class: mapped class (must be a positional argument) + :param ident: primary key, may be a scalar or tuple argument. * ``identity_key(instance=instance)`` - instance - object instance (must be given as a keyword arg) + This form will produce the identity key for a given instance. The + instance need not be persistent, only that its primary key attributes + are populated (else the key will contain ``None`` for those missing + values). + + E.g.:: + + >>> instance = MyClass(1, 2) + >>> identity_key(instance=instance) + (, (1, 2)) + + In this form, the given instance is ultimately run though + :meth:`.Mapper.identity_key_from_instance`, which will have the + effect of performing a database check for the corresponding row + if the object is expired. + + :param instance: object instance (must be given as a keyword arg) * ``identity_key(class, row=row)`` - class - mapped class (must be a positional argument) + This form is similar to the class/tuple form, except is passed a + database result row as a :class:`.RowProxy` object. - row - result proxy row (must be given as a keyword arg) + E.g.:: + + >>> row = engine.execute("select * from table where a=1 and b=2").first() + >>> identity_key(MyClass, row=row) + (, (1, 2)) + + :param class: mapped class (must be a positional argument) + :param row: :class:`.RowProxy` row returned by a :class:`.ResultProxy` + (must be given as a keyword arg) """ if args: @@ -196,7 +249,7 @@ def identity_key(*args, **kwargs): "positional arguments, got %s" % len(args)) if kwargs: raise sa_exc.ArgumentError("unknown keyword arguments: %s" - % ", ".join(kwargs.keys())) + % ", ".join(kwargs)) mapper = class_mapper(class_) if "ident" in locals(): return mapper.identity_key_from_primary_key(util.to_list(ident)) @@ -204,10 +257,11 @@ def identity_key(*args, **kwargs): instance = kwargs.pop("instance") if kwargs: raise sa_exc.ArgumentError("unknown keyword arguments: %s" - % ", ".join(kwargs.keys())) + % ", ".join(kwargs.keys)) mapper = object_mapper(instance) return mapper.identity_key_from_instance(instance) + class ORMAdapter(sql_util.ColumnAdapter): """Extends ColumnAdapter to accept ORM entities. @@ -215,9 +269,13 @@ class ORMAdapter(sql_util.ColumnAdapter): and the AliasedClass if any is referenced. """ - def __init__(self, entity, equivalents=None, - chain_to=None, adapt_required=False): - self.mapper, selectable, is_aliased_class = _entity_info(entity) + def __init__(self, entity, equivalents=None, adapt_required=False, + chain_to=None): + info = inspection.inspect(entity) + + self.mapper = info.mapper + selectable = info.selectable + is_aliased_class = info.is_aliased_class if is_aliased_class: self.aliased_class = entity else: @@ -241,7 +299,10 @@ class AliasedClass(object): __getattr__ scheme and maintains a reference to a real :class:`~sqlalchemy.sql.expression.Alias` object. - Usage is via the :class:`~sqlalchemy.orm.aliased()` synonym:: + Usage is via the :func:`.orm.aliased` function, or alternatively + via the :func:`.orm.with_polymorphic` function. + + Usage example:: # find all pairs of users with the same name user_alias = aliased(User) @@ -249,28 +310,286 @@ class AliasedClass(object): join((user_alias, User.id > user_alias.id)).\\ filter(User.name==user_alias.name) - The resulting object is an instance of :class:`.AliasedClass`, however - it implements a ``__getattribute__()`` scheme which will proxy attribute - access to that of the ORM class being aliased. All classmethods - on the mapped entity should also be available here, including - hybrids created with the :ref:`hybrids_toplevel` extension, - which will receive the :class:`.AliasedClass` as the "class" argument - when classmethods are called. + The resulting object is an instance of :class:`.AliasedClass`. + This object implements an attribute scheme which produces the + same attribute and method interface as the original mapped + class, allowing :class:`.AliasedClass` to be compatible + with any attribute technique which works on the original class, + including hybrid attributes (see :ref:`hybrids_toplevel`). + + The :class:`.AliasedClass` can be inspected for its underlying + :class:`.Mapper`, aliased selectable, and other information + using :func:`.inspect`:: + + from sqlalchemy import inspect + my_alias = aliased(MyClass) + insp = inspect(my_alias) + + The resulting inspection object is an instance of :class:`.AliasedInsp`. + + See :func:`.aliased` and :func:`.with_polymorphic` for construction + argument descriptions. + + """ + def __init__(self, cls, alias=None, + name=None, + flat=False, + adapt_on_names=False, + # TODO: None for default here? + with_polymorphic_mappers=(), + with_polymorphic_discriminator=None, + base_alias=None, + use_mapper_path=False): + mapper = _class_to_mapper(cls) + if alias is None: + alias = mapper._with_polymorphic_selectable.alias( + name=name, flat=flat) + self._aliased_insp = AliasedInsp( + self, + mapper, + alias, + name, + with_polymorphic_mappers + if with_polymorphic_mappers + else mapper.with_polymorphic_mappers, + with_polymorphic_discriminator + if with_polymorphic_discriminator is not None + else mapper.polymorphic_on, + base_alias, + use_mapper_path, + adapt_on_names + ) + + self.__name__ = 'AliasedClass_%s' % mapper.class_.__name__ + + def __getattr__(self, key): + try: + _aliased_insp = self.__dict__['_aliased_insp'] + except KeyError: + raise AttributeError() + else: + for base in _aliased_insp._target.__mro__: + try: + attr = object.__getattribute__(base, key) + except AttributeError: + continue + else: + break + else: + raise AttributeError(key) + + if isinstance(attr, PropComparator): + ret = attr.adapt_to_entity(_aliased_insp) + setattr(self, key, ret) + return ret + elif hasattr(attr, 'func_code'): + is_method = getattr(_aliased_insp._target, key, None) + if is_method and is_method.__self__ is not None: + return util.types.MethodType(attr.__func__, self, self) + else: + return None + elif hasattr(attr, '__get__'): + ret = attr.__get__(None, self) + if isinstance(ret, PropComparator): + return ret.adapt_to_entity(_aliased_insp) + else: + return ret + else: + return attr + + def __repr__(self): + return '' % ( + id(self), self._aliased_insp._target.__name__) + + +class AliasedInsp(_InspectionAttr): + """Provide an inspection interface for an + :class:`.AliasedClass` object. + + The :class:`.AliasedInsp` object is returned + given an :class:`.AliasedClass` using the + :func:`.inspect` function:: + + from sqlalchemy import inspect + from sqlalchemy.orm import aliased + + my_alias = aliased(MyMappedClass) + insp = inspect(my_alias) + + Attributes on :class:`.AliasedInsp` + include: + + * ``entity`` - the :class:`.AliasedClass` represented. + * ``mapper`` - the :class:`.Mapper` mapping the underlying class. + * ``selectable`` - the :class:`.Alias` construct which ultimately + represents an aliased :class:`.Table` or :class:`.Select` + construct. + * ``name`` - the name of the alias. Also is used as the attribute + name when returned in a result tuple from :class:`.Query`. + * ``with_polymorphic_mappers`` - collection of :class:`.Mapper` objects + indicating all those mappers expressed in the select construct + for the :class:`.AliasedClass`. + * ``polymorphic_on`` - an alternate column or SQL expression which + will be used as the "discriminator" for a polymorphic load. + + .. seealso:: + + :ref:`inspection_toplevel` + + """ + + def __init__(self, entity, mapper, selectable, name, + with_polymorphic_mappers, polymorphic_on, + _base_alias, _use_mapper_path, adapt_on_names): + self.entity = entity + self.mapper = mapper + self.selectable = selectable + self.name = name + self.with_polymorphic_mappers = with_polymorphic_mappers + self.polymorphic_on = polymorphic_on + self._base_alias = _base_alias or self + self._use_mapper_path = _use_mapper_path + + self._adapter = sql_util.ClauseAdapter(selectable, + equivalents=mapper._equivalent_columns, + adapt_on_names=adapt_on_names) + + self._adapt_on_names = adapt_on_names + self._target = mapper.class_ + + for poly in self.with_polymorphic_mappers: + if poly is not mapper: + setattr(self.entity, poly.class_.__name__, + AliasedClass(poly.class_, selectable, base_alias=self, + adapt_on_names=adapt_on_names, + use_mapper_path=_use_mapper_path)) + + is_aliased_class = True + "always returns True" + + @property + def class_(self): + """Return the mapped class ultimately represented by this + :class:`.AliasedInsp`.""" + return self.mapper.class_ + + @util.memoized_property + def _path_registry(self): + if self._use_mapper_path: + return self.mapper._path_registry + else: + return PathRegistry.per_mapper(self) + + def __getstate__(self): + return { + 'entity': self.entity, + 'mapper': self.mapper, + 'alias': self.selectable, + 'name': self.name, + 'adapt_on_names': self._adapt_on_names, + 'with_polymorphic_mappers': + self.with_polymorphic_mappers, + 'with_polymorphic_discriminator': + self.polymorphic_on, + 'base_alias': self._base_alias, + 'use_mapper_path': self._use_mapper_path + } + + def __setstate__(self, state): + self.__init__( + state['entity'], + state['mapper'], + state['alias'], + state['name'], + state['with_polymorphic_mappers'], + state['with_polymorphic_discriminator'], + state['base_alias'], + state['use_mapper_path'], + state['adapt_on_names'] + ) + + def _adapt_element(self, elem): + return self._adapter.traverse(elem).\ + _annotate({ + 'parententity': self.entity, + 'parentmapper': self.mapper} + ) + + def _entity_for_mapper(self, mapper): + self_poly = self.with_polymorphic_mappers + if mapper in self_poly: + return getattr(self.entity, mapper.class_.__name__)._aliased_insp + elif mapper.isa(self.mapper): + return self + else: + assert False, "mapper %s doesn't correspond to %s" % (mapper, self) + + def __repr__(self): + return '' % ( + id(self), self.class_.__name__) + + +inspection._inspects(AliasedClass)(lambda target: target._aliased_insp) +inspection._inspects(AliasedInsp)(lambda target: target) + + +def aliased(element, alias=None, name=None, flat=False, adapt_on_names=False): + """Produce an alias of the given element, usually an :class:`.AliasedClass` + instance. + + E.g.:: + + my_alias = aliased(MyClass) + + session.query(MyClass, my_alias).filter(MyClass.id > my_alias.id) + + The :func:`.aliased` function is used to create an ad-hoc mapping + of a mapped class to a new selectable. By default, a selectable + is generated from the normally mapped selectable (typically a + :class:`.Table`) using the :meth:`.FromClause.alias` method. + However, :func:`.aliased` can also be used to link the class to + a new :func:`.select` statement. Also, the :func:`.with_polymorphic` + function is a variant of :func:`.aliased` that is intended to specify + a so-called "polymorphic selectable", that corresponds to the union + of several joined-inheritance subclasses at once. + + For convenience, the :func:`.aliased` function also accepts plain + :class:`.FromClause` constructs, such as a :class:`.Table` or + :func:`.select` construct. In those cases, the :meth:`.FromClause.alias` + method is called on the object and the new :class:`.Alias` object + returned. The returned :class:`.Alias` is not ORM-mapped in this case. + + :param element: element to be aliased. Is normally a mapped class, + but for convenience can also be a :class:`.FromClause` element. + + :param alias: Optional selectable unit to map the element to. This should + normally be a :class:`.Alias` object corresponding to the :class:`.Table` + to which the class is mapped, or to a :func:`.select` construct that + is compatible with the mapping. By default, a simple anonymous + alias of the mapped table is generated. + + :param name: optional string name to use for the alias, if not specified + by the ``alias`` parameter. The name, among other things, forms the + attribute name that will be accessible via tuples returned by a + :class:`.Query` object. + + :param flat: Boolean, will be passed through to the :meth:`.FromClause.alias` + call so that aliases of :class:`.Join` objects don't include an enclosing + SELECT. This can lead to more efficient queries in many circumstances. + A JOIN against a nested JOIN will be rewritten as a JOIN against an aliased + SELECT subquery on backends that don't support this syntax. + + .. versionadded:: 0.9.0 + + .. seealso:: :meth:`.Join.alias` - :param cls: ORM mapped entity which will be "wrapped" around an alias. - :param alias: a selectable, such as an :func:`.alias` or :func:`.select` - construct, which will be rendered in place of the mapped table of the - ORM entity. If left as ``None``, an ordinary :class:`.Alias` of the - ORM entity's mapped table will be generated. - :param name: A name which will be applied both to the :class:`.Alias` - if one is generated, as well as the name present in the "named tuple" - returned by the :class:`.Query` object when results are returned. :param adapt_on_names: if True, more liberal "matching" will be used when - mapping the mapped columns of the ORM entity to those of the given selectable - - a name-based match will be performed if the given selectable doesn't - otherwise have a column that corresponds to one on the entity. The - use case for this is when associating an entity with some derived - selectable such as one that uses aggregate functions:: + mapping the mapped columns of the ORM entity to those of the + given selectable - a name-based match will be performed if the + given selectable doesn't otherwise have a column that corresponds + to one on the entity. The use case for this is when associating + an entity with some derived selectable such as one that uses + aggregate functions:: class UnitPrice(Base): __tablename__ = 'unit_price' @@ -282,107 +601,109 @@ class AliasedClass(object): func.sum(UnitPrice.price).label('price') ).group_by(UnitPrice.unit_id).subquery() - aggregated_unit_price = aliased(UnitPrice, alias=aggregated_unit_price, adapt_on_names=True) + aggregated_unit_price = aliased(UnitPrice, + alias=aggregated_unit_price, adapt_on_names=True) - Above, functions on ``aggregated_unit_price`` which - refer to ``.price`` will return the - ``fund.sum(UnitPrice.price).label('price')`` column, - as it is matched on the name "price". Ordinarily, the "price" function wouldn't - have any "column correspondence" to the actual ``UnitPrice.price`` column - as it is not a proxy of the original. + Above, functions on ``aggregated_unit_price`` which refer to + ``.price`` will return the + ``fund.sum(UnitPrice.price).label('price')`` column, as it is + matched on the name "price". Ordinarily, the "price" function + wouldn't have any "column correspondence" to the actual + ``UnitPrice.price`` column as it is not a proxy of the original. .. versionadded:: 0.7.3 + """ - def __init__(self, cls, alias=None, name=None, adapt_on_names=False): - self.__mapper = _class_to_mapper(cls) - self.__target = self.__mapper.class_ - self.__adapt_on_names = adapt_on_names - if alias is None: - alias = self.__mapper._with_polymorphic_selectable.alias(name=name) - self.__adapter = sql_util.ClauseAdapter(alias, - equivalents=self.__mapper._equivalent_columns, - adapt_on_names=self.__adapt_on_names) - self.__alias = alias - # used to assign a name to the RowTuple object - # returned by Query. - self._sa_label_name = name - self.__name__ = 'AliasedClass_' + str(self.__target) - - def __getstate__(self): - return { - 'mapper':self.__mapper, - 'alias':self.__alias, - 'name':self._sa_label_name, - 'adapt_on_names':self.__adapt_on_names, - } - - def __setstate__(self, state): - self.__mapper = state['mapper'] - self.__target = self.__mapper.class_ - self.__adapt_on_names = state['adapt_on_names'] - alias = state['alias'] - self.__adapter = sql_util.ClauseAdapter(alias, - equivalents=self.__mapper._equivalent_columns, - adapt_on_names=self.__adapt_on_names) - self.__alias = alias - name = state['name'] - self._sa_label_name = name - self.__name__ = 'AliasedClass_' + str(self.__target) - - def __adapt_element(self, elem): - return self.__adapter.traverse(elem).\ - _annotate({ - 'parententity': self, - 'parentmapper':self.__mapper} - ) - - def __adapt_prop(self, existing, key): - comparator = existing.comparator.adapted(self.__adapt_element) - - queryattr = attributes.QueryableAttribute(self, key, - impl=existing.impl, parententity=self, comparator=comparator) - setattr(self, key, queryattr) - return queryattr - - def __getattr__(self, key): - for base in self.__target.__mro__: - try: - attr = object.__getattribute__(base, key) - except AttributeError: - continue - else: - break - else: - raise AttributeError(key) - - if isinstance(attr, attributes.QueryableAttribute): - return self.__adapt_prop(attr, key) - elif hasattr(attr, 'func_code'): - is_method = getattr(self.__target, key, None) - if is_method and is_method.im_self is not None: - return util.types.MethodType(attr.im_func, self, self) - else: - return None - elif hasattr(attr, '__get__'): - ret = attr.__get__(None, self) - if isinstance(ret, PropComparator): - return ret.adapted(self.__adapt_element) - return ret - else: - return attr - - def __repr__(self): - return '' % ( - id(self), self.__target.__name__) - -def aliased(element, alias=None, name=None, adapt_on_names=False): if isinstance(element, expression.FromClause): if adapt_on_names: - raise sa_exc.ArgumentError("adapt_on_names only applies to ORM elements") - return element.alias(name) + raise sa_exc.ArgumentError( + "adapt_on_names only applies to ORM elements" + ) + return element.alias(name, flat=flat) else: - return AliasedClass(element, alias=alias, name=name, adapt_on_names=adapt_on_names) + return AliasedClass(element, alias=alias, flat=flat, + name=name, adapt_on_names=adapt_on_names) + + +def with_polymorphic(base, classes, selectable=False, + flat=False, + polymorphic_on=None, aliased=False, + innerjoin=False, _use_mapper_path=False): + """Produce an :class:`.AliasedClass` construct which specifies + columns for descendant mappers of the given base. + + .. versionadded:: 0.8 + :func:`.orm.with_polymorphic` is in addition to the existing + :class:`.Query` method :meth:`.Query.with_polymorphic`, + which has the same purpose but is not as flexible in its usage. + + Using this method will ensure that each descendant mapper's + tables are included in the FROM clause, and will allow filter() + criterion to be used against those tables. The resulting + instances will also have those columns already loaded so that + no "post fetch" of those columns will be required. + + See the examples at :ref:`with_polymorphic`. + + :param base: Base class to be aliased. + + :param classes: a single class or mapper, or list of + class/mappers, which inherit from the base class. + Alternatively, it may also be the string ``'*'``, in which case + all descending mapped classes will be added to the FROM clause. + + :param aliased: when True, the selectable will be wrapped in an + alias, that is ``(SELECT * FROM ) AS anon_1``. + This can be important when using the with_polymorphic() + to create the target of a JOIN on a backend that does not + support parenthesized joins, such as SQLite and older + versions of MySQL. + + :param flat: Boolean, will be passed through to the :meth:`.FromClause.alias` + call so that aliases of :class:`.Join` objects don't include an enclosing + SELECT. This can lead to more efficient queries in many circumstances. + A JOIN against a nested JOIN will be rewritten as a JOIN against an aliased + SELECT subquery on backends that don't support this syntax. + + Setting ``flat`` to ``True`` implies the ``aliased`` flag is + also ``True``. + + .. versionadded:: 0.9.0 + + .. seealso:: :meth:`.Join.alias` + + :param selectable: a table or select() statement that will + be used in place of the generated FROM clause. This argument is + required if any of the desired classes use concrete table + inheritance, since SQLAlchemy currently cannot generate UNIONs + among tables automatically. If used, the ``selectable`` argument + must represent the full set of tables and columns mapped by every + mapped class. Otherwise, the unaccounted mapped columns will + result in their table being appended directly to the FROM clause + which will usually lead to incorrect results. + + :param polymorphic_on: a column to be used as the "discriminator" + column for the given selectable. If not given, the polymorphic_on + attribute of the base classes' mapper will be used, if any. This + is useful for mappings that don't have polymorphic loading + behavior by default. + + :param innerjoin: if True, an INNER JOIN will be used. This should + only be specified if querying for one specific subtype only + """ + primary_mapper = _class_to_mapper(base) + mappers, selectable = primary_mapper.\ + _with_polymorphic_args(classes, selectable, + innerjoin=innerjoin) + if aliased or flat: + selectable = selectable.alias(flat=flat) + return AliasedClass(base, + selectable, + with_polymorphic_mappers=mappers, + with_polymorphic_discriminator=polymorphic_on, + use_mapper_path=_use_mapper_path) + def _orm_annotate(element, exclude=None): """Deep copy the given ClauseElement, annotating each element with the @@ -391,73 +712,90 @@ def _orm_annotate(element, exclude=None): Elements within the exclude collection will be cloned but not annotated. """ - return sql_util._deep_annotate(element, {'_orm_adapt':True}, exclude) + return sql_util._deep_annotate(element, {'_orm_adapt': True}, exclude) + + +def _orm_deannotate(element): + """Remove annotations that link a column to a particular mapping. + + Note this doesn't affect "remote" and "foreign" annotations + passed by the :func:`.orm.foreign` and :func:`.orm.remote` + annotators. + + """ + + return sql_util._deep_deannotate(element, + values=("_orm_adapt", "parententity") + ) + + +def _orm_full_deannotate(element): + return sql_util._deep_deannotate(element) -_orm_deannotate = sql_util._deep_deannotate class _ORMJoin(expression.Join): """Extend Join to support ORM constructs as input.""" __visit_name__ = expression.Join.__visit_name__ - def __init__(self, left, right, onclause=None, - isouter=False, join_to_left=True): - adapt_from = None + def __init__(self, left, right, onclause=None, isouter=False): - if hasattr(left, '_orm_mappers'): - left_mapper = left._orm_mappers[1] - if join_to_left: - adapt_from = left.right + left_info = inspection.inspect(left) + left_orm_info = getattr(left, '_joined_from_info', left_info) + + right_info = inspection.inspect(right) + adapt_to = right_info.selectable + + self._joined_from_info = right_info + + if isinstance(onclause, util.string_types): + onclause = getattr(left_orm_info.entity, onclause) + + if isinstance(onclause, attributes.QueryableAttribute): + on_selectable = onclause.comparator._source_selectable() + prop = onclause.property + elif isinstance(onclause, MapperProperty): + prop = onclause + on_selectable = prop.parent.selectable else: - left_mapper, left, left_is_aliased = _entity_info(left) - if join_to_left and (left_is_aliased or not left_mapper): - adapt_from = left + prop = None - right_mapper, right, right_is_aliased = _entity_info(right) - if right_is_aliased: - adapt_to = right - else: - adapt_to = None - - if left_mapper or right_mapper: - self._orm_mappers = (left_mapper, right_mapper) - - if isinstance(onclause, basestring): - prop = left_mapper.get_property(onclause) - elif isinstance(onclause, attributes.QueryableAttribute): - if adapt_from is None: - adapt_from = onclause.__clause_element__() - prop = onclause.property - elif isinstance(onclause, MapperProperty): - prop = onclause + if prop: + if sql_util.clause_is_present(on_selectable, left_info.selectable): + adapt_from = on_selectable else: - prop = None + adapt_from = left_info.selectable - if prop: - pj, sj, source, dest, \ + pj, sj, source, dest, \ secondary, target_adapter = prop._create_joins( - source_selectable=adapt_from, - dest_selectable=adapt_to, - source_polymorphic=True, - dest_polymorphic=True, - of_type=right_mapper) + source_selectable=adapt_from, + dest_selectable=adapt_to, + source_polymorphic=True, + dest_polymorphic=True, + of_type=right_info.mapper) - if sj is not None: + if sj is not None: + if isouter: + # note this is an inner join from secondary->right + right = sql.join(secondary, right, sj) + onclause = pj + else: left = sql.join(left, secondary, pj, isouter) onclause = sj - else: - onclause = pj - self._target_adapter = target_adapter + else: + onclause = pj + self._target_adapter = target_adapter expression.Join.__init__(self, left, right, onclause, isouter) - def join(self, right, onclause=None, isouter=False, join_to_left=True): - return _ORMJoin(self, right, onclause, isouter, join_to_left) + def join(self, right, onclause=None, isouter=False, join_to_left=None): + return _ORMJoin(self, right, onclause, isouter) - def outerjoin(self, right, onclause=None, join_to_left=True): - return _ORMJoin(self, right, onclause, True, join_to_left) + def outerjoin(self, right, onclause=None, join_to_left=None): + return _ORMJoin(self, right, onclause, True) -def join(left, right, onclause=None, isouter=False, join_to_left=True): + +def join(left, right, onclause=None, isouter=False, join_to_left=None): """Produce an inner join between left and right clauses. :func:`.orm.join` is an extension to the core join interface @@ -468,11 +806,6 @@ def join(left, right, onclause=None, isouter=False, join_to_left=True): be a SQL expression, or an attribute or string name referencing a configured :func:`.relationship`. - ``join_to_left`` indicates to attempt aliasing the ON clause, - in whatever form it is passed, to the selectable - passed as the left side. If False, the onclause - is used as is. - :func:`.orm.join` is not commonly needed in modern usage, as its functionality is encapsulated within that of the :meth:`.Query.join` method, which features a @@ -496,10 +829,14 @@ def join(left, right, onclause=None, isouter=False, join_to_left=True): See :meth:`.Query.join` for information on modern usage of ORM level joins. - """ - return _ORMJoin(left, right, onclause, isouter, join_to_left) + .. versionchanged:: 0.8.1 - the ``join_to_left`` parameter + is no longer used, and is deprecated. -def outerjoin(left, right, onclause=None, join_to_left=True): + """ + return _ORMJoin(left, right, onclause, isouter) + + +def outerjoin(left, right, onclause=None, join_to_left=None): """Produce a left outer join between left and right clauses. This is the "outer join" version of the :func:`.orm.join` function, @@ -507,7 +844,8 @@ def outerjoin(left, right, onclause=None, join_to_left=True): See that function's documentation for other usage details. """ - return _ORMJoin(left, right, onclause, True, join_to_left) + return _ORMJoin(left, right, onclause, True) + def with_parent(instance, prop): """Create filtering criterion that relates this query's primary entity @@ -536,7 +874,7 @@ def with_parent(instance, prop): parent/child relationship. """ - if isinstance(prop, basestring): + if isinstance(prop, util.string_types): mapper = object_mapper(instance) prop = getattr(mapper.class_, prop).property elif isinstance(prop, attributes.QueryableAttribute): @@ -547,204 +885,69 @@ def with_parent(instance, prop): value_is_parent=True) -def _entity_info(entity, compile=True): - """Return mapping information given a class, mapper, or AliasedClass. - - Returns 3-tuple of: mapper, mapped selectable, boolean indicating if this - is an aliased() construct. - - If the given entity is not a mapper, mapped class, or aliased construct, - returns None, the entity, False. This is typically used to allow - unmapped selectables through. - - """ - if isinstance(entity, AliasedClass): - return entity._AliasedClass__mapper, entity._AliasedClass__alias, True - - if isinstance(entity, mapperlib.Mapper): - mapper = entity - - elif isinstance(entity, type): - class_manager = attributes.manager_of_class(entity) - - if class_manager is None: - return None, entity, False - - mapper = class_manager.mapper - else: - return None, entity, False - - if compile and mapperlib.module._new_mappers: - mapperlib.configure_mappers() - return mapper, mapper._with_polymorphic_selectable, False - -def _entity_descriptor(entity, key): - """Return a class attribute given an entity and string name. - - May return :class:`.InstrumentedAttribute` or user-defined - attribute. - - """ - if isinstance(entity, expression.FromClause): - description = entity - entity = entity.c - elif not isinstance(entity, (AliasedClass, type)): - description = entity = entity.class_ - else: - description = entity - - try: - return getattr(entity, key) - except AttributeError: - raise sa_exc.InvalidRequestError( - "Entity '%s' has no property '%s'" % - (description, key) - ) - -def _orm_columns(entity): - mapper, selectable, is_aliased_class = _entity_info(entity) - if isinstance(selectable, expression.Selectable): - return [c for c in selectable.c] - else: - return [selectable] - -def _orm_selectable(entity): - mapper, selectable, is_aliased_class = _entity_info(entity) - return selectable - -def _attr_as_key(attr): - if hasattr(attr, 'key'): - return attr.key - else: - return expression._column_as_key(attr) - -def _is_aliased_class(entity): - return isinstance(entity, AliasedClass) - -_state_mapper = util.dottedgetter('manager.mapper') - -def object_mapper(instance): - """Given an object, return the primary Mapper associated with the object - instance. - - Raises UnmappedInstanceError if no mapping is configured. - - """ - try: - state = attributes.instance_state(instance) - return state.manager.mapper - except exc.UnmappedClassError: - raise exc.UnmappedInstanceError(instance) - except exc.NO_STATE: - raise exc.UnmappedInstanceError(instance) - -def class_mapper(class_, compile=True): - """Given a class, return the primary :class:`.Mapper` associated - with the key. - - Raises :class:`.UnmappedClassError` if no mapping is configured - on the given class, or :class:`.ArgumentError` if a non-class - object is passed. - - """ - - try: - class_manager = attributes.manager_of_class(class_) - mapper = class_manager.mapper - - except exc.NO_STATE: - if not isinstance(class_, type): - raise sa_exc.ArgumentError("Class object expected, got '%r'." % class_) - raise exc.UnmappedClassError(class_) - - if compile and mapperlib.module._new_mappers: - mapperlib.configure_mappers() - return mapper - -def _class_to_mapper(class_or_mapper, compile=True): - if _is_aliased_class(class_or_mapper): - return class_or_mapper._AliasedClass__mapper - - elif isinstance(class_or_mapper, type): - try: - class_manager = attributes.manager_of_class(class_or_mapper) - mapper = class_manager.mapper - except exc.NO_STATE: - raise exc.UnmappedClassError(class_or_mapper) - elif isinstance(class_or_mapper, mapperlib.Mapper): - mapper = class_or_mapper - else: - raise exc.UnmappedClassError(class_or_mapper) - - if compile and mapperlib.module._new_mappers: - mapperlib.configure_mappers() - return mapper def has_identity(object): + """Return True if the given object has a database + identity. + + This typically corresponds to the object being + in either the persistent or detached state. + + .. seealso:: + + :func:`.was_deleted` + + """ state = attributes.instance_state(object) return state.has_identity -def _is_mapped_class(cls): - """Return True if the given object is a mapped class, - :class:`.Mapper`, or :class:`.AliasedClass`.""" +def was_deleted(object): + """Return True if the given object was deleted + within a session flush. - if isinstance(cls, (AliasedClass, mapperlib.Mapper)): - return True - if isinstance(cls, expression.ClauseElement): - return False - if isinstance(cls, type): - manager = attributes.manager_of_class(cls) - return manager and _INSTRUMENTOR in manager.info - return False + .. versionadded:: 0.8.0 -def _mapper_or_none(cls): - """Return the :class:`.Mapper` for the given class or None if the - class is not mapped.""" + """ - manager = attributes.manager_of_class(cls) - if manager is not None and _INSTRUMENTOR in manager.info: - return manager.info[_INSTRUMENTOR] - else: - return None + state = attributes.instance_state(object) + return state.deleted -def instance_str(instance): - """Return a string describing an instance.""" - return state_str(attributes.instance_state(instance)) -def state_str(state): - """Return a string describing an instance via its InstanceState.""" - if state is None: - return "None" - else: - return '<%s at 0x%x>' % (state.class_.__name__, id(state.obj())) +def randomize_unitofwork(): + """Use random-ordering sets within the unit of work in order + to detect unit of work sorting issues. -def state_class_str(state): - """Return a string describing an instance's class via its InstanceState.""" + This is a utility function that can be used to help reproduce + inconsistent unit of work sorting issues. For example, + if two kinds of objects A and B are being inserted, and + B has a foreign key reference to A - the A must be inserted first. + However, if there is no relationship between A and B, the unit of work + won't know to perform this sorting, and an operation may or may not + fail, depending on how the ordering works out. Since Python sets + and dictionaries have non-deterministic ordering, such an issue may + occur on some runs and not on others, and in practice it tends to + have a great dependence on the state of the interpreter. This leads + to so-called "heisenbugs" where changing entirely irrelevant aspects + of the test program still cause the failure behavior to change. - if state is None: - return "None" - else: - return '<%s>' % (state.class_.__name__, ) + By calling ``randomize_unitofwork()`` when a script first runs, the + ordering of a key series of sets within the unit of work implementation + are randomized, so that the script can be minimized down to the fundamental + mapping and operation that's failing, while still reproducing the issue + on at least some runs. -def attribute_str(instance, attribute): - return instance_str(instance) + "." + attribute + This utility is also available when running the test suite via the + ``--reversetop`` flag. -def state_attribute_str(state, attribute): - return state_str(state) + "." + attribute + .. versionadded:: 0.8.1 created a standalone version of the + ``--reversetop`` feature. -def identity_equal(a, b): - if a is b: - return True - if a is None or b is None: - return False - try: - state_a = attributes.instance_state(a) - state_b = attributes.instance_state(b) - except exc.NO_STATE: - return False - if state_a.key is None or state_b.key is None: - return False - return state_a.key == state_b.key + """ + from sqlalchemy.orm import unitofwork, session, mapper, dependency + from sqlalchemy.util import topological + from sqlalchemy.testing.util import RandomSet + topological.set = unitofwork.set = session.set = mapper.set = \ + dependency.set = RandomSet diff --git a/libs/sqlalchemy/pool.py b/libs/sqlalchemy/pool.py index 0d04998c..3adfb320 100644 --- a/libs/sqlalchemy/pool.py +++ b/libs/sqlalchemy/pool.py @@ -1,5 +1,5 @@ # sqlalchemy/pool.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -16,16 +16,19 @@ regular DB-API connect() methods to be transparently managed by a SQLAlchemy connection pool. """ -import weakref import time import traceback +import weakref -from sqlalchemy import exc, log, event, events, interfaces, util -from sqlalchemy.util import queue as sqla_queue -from sqlalchemy.util import threading, memoized_property, \ +from . import exc, log, event, interfaces, util +from .util import queue as sqla_queue +from .util import threading, memoized_property, \ chop_traceback + +from collections import deque proxies = {} + def manage(module, **params): """Return a proxy for a DB-API module that automatically pools connections. @@ -48,13 +51,14 @@ def manage(module, **params): except KeyError: return proxies.setdefault(module, _DBProxy(module, **params)) + def clear_managers(): """Remove all current DB-API 2.0 managers. All pools and connections are disposed. """ - for manager in proxies.itervalues(): + for manager in proxies.values(): manager.close() proxies.clear() @@ -62,10 +66,29 @@ reset_rollback = util.symbol('reset_rollback') reset_commit = util.symbol('reset_commit') reset_none = util.symbol('reset_none') +class _ConnDialect(object): + """partial implementation of :class:`.Dialect` + which provides DBAPI connection methods. + + When a :class:`.Pool` is combined with an :class:`.Engine`, + the :class:`.Engine` replaces this with its own + :class:`.Dialect`. + + """ + def do_rollback(self, dbapi_connection): + dbapi_connection.rollback() + + def do_commit(self, dbapi_connection): + dbapi_connection.commit() + + def do_close(self, dbapi_connection): + dbapi_connection.close() class Pool(log.Identified): """Abstract base class for connection pools.""" + _dialect = _ConnDialect() + def __init__(self, creator, recycle=-1, echo=None, use_threadlocal=False, @@ -73,7 +96,8 @@ class Pool(log.Identified): reset_on_return=True, listeners=None, events=None, - _dispatch=None): + _dispatch=None, + _dialect=None): """ Construct a Pool. @@ -106,10 +130,10 @@ class Pool(log.Identified): :meth:`unique_connection` method is provided to bypass the threadlocal behavior installed into :meth:`connect`. - :param reset_on_return: If true, reset the database state of - connections returned to the pool. This is typically a - ROLLBACK to release locks and transaction resources. - Disable at your own peril. Defaults to True. + :param reset_on_return: Configures the action to take + on connections as they are returned to the pool. + See the argument description in :class:`.QueuePool` for + more detail. :param events: a list of 2-tuples, each of the form ``(callable, target)`` which will be passed to event.listen() @@ -149,6 +173,8 @@ class Pool(log.Identified): self.echo = echo if _dispatch: self.dispatch._update(_dispatch, only_propagate=False) + if _dialect: + self._dialect = _dialect if events: for fn, target in events: event.listen(self, target, fn) @@ -159,9 +185,18 @@ class Pool(log.Identified): for l in listeners: self.add_listener(l) - dispatch = event.dispatcher(events.PoolEvents) + def _close_connection(self, connection): + self.logger.debug("Closing connection %r", connection) + try: + self._dialect.do_close(connection) + except (SystemExit, KeyboardInterrupt): + raise + except: + self.logger.error("Exception closing connection %r", + connection, exc_info=True) - @util.deprecated(2.7, "Pool.add_listener is deprecated. Use event.listen()") + @util.deprecated( + 2.7, "Pool.add_listener is deprecated. Use event.listen()") def add_listener(self, listener): """Add a :class:`.PoolListener`-like object to this pool. @@ -181,7 +216,7 @@ class Pool(log.Identified): """ - return _ConnectionFairy(self).checkout() + return _ConnectionFairy.checkout(self) def _create_connection(self): """Called by subclasses to create a new ConnectionRecord.""" @@ -233,18 +268,17 @@ class Pool(log.Identified): """ if not self._use_threadlocal: - return _ConnectionFairy(self).checkout() + return _ConnectionFairy.checkout(self) try: rec = self._threadconns.current() - if rec: - return rec.checkout() except AttributeError: pass + else: + if rec is not None: + return rec.checkout_existing() - agent = _ConnectionFairy(self) - self._threadconns.current = weakref.ref(agent) - return agent.checkout() + return _ConnectionFairy.checkout(self, self._threadconns) def _return_conn(self, record): """Given a _ConnectionRecord, return it to the :class:`.Pool`. @@ -275,28 +309,58 @@ class Pool(log.Identified): class _ConnectionRecord(object): - finalize_callback = None def __init__(self, pool): self.__pool = pool self.connection = self.__connect() - self.info = {} + self.finalize_callback = deque() pool.dispatch.first_connect.\ for_modify(pool.dispatch).\ exec_once(self.connection, self) pool.dispatch.connect(self.connection, self) + @util.memoized_property + def info(self): + return {} + + @classmethod + def checkout(cls, pool): + rec = pool._do_get() + try: + dbapi_connection = rec.get_connection() + except: + rec.checkin() + raise + fairy = _ConnectionFairy(dbapi_connection, rec) + rec.fairy_ref = weakref.ref( + fairy, + lambda ref: _finalize_fairy and \ + _finalize_fairy( + dbapi_connection, + rec, pool, ref, pool._echo) + ) + _refs.add(rec) + if pool._echo: + pool.logger.debug("Connection %r checked out from pool", + dbapi_connection) + return fairy + + def checkin(self): + self.fairy_ref = None + connection = self.connection + pool = self.__pool + while self.finalize_callback: + finalizer = self.finalize_callback.pop() + finalizer(connection) + if pool.dispatch.checkin: + pool.dispatch.checkin(connection, self) + pool._return_conn(self) + + def close(self): if self.connection is not None: - self.__pool.logger.debug("Closing connection %r", self.connection) - try: - self.connection.close() - except (SystemExit, KeyboardInterrupt): - raise - except: - self.__pool.logger.debug("Exception closing connection %r", - self.connection) + self.__pool._close_connection(self.connection) def invalidate(self, e=None): if e is not None: @@ -328,15 +392,7 @@ class _ConnectionRecord(object): return self.connection def __close(self): - try: - self.__pool.logger.debug("Closing connection %r", self.connection) - self.connection.close() - except (SystemExit, KeyboardInterrupt): - raise - except Exception, e: - self.__pool.logger.debug( - "Connection %r threw an error on close: %s", - self.connection, e) + self.__pool._close_connection(self.connection) def __connect(self): try: @@ -344,75 +400,114 @@ class _ConnectionRecord(object): connection = self.__pool._creator() self.__pool.logger.debug("Created new connection %r", connection) return connection - except Exception, e: + except Exception as e: self.__pool.logger.debug("Error on connect(): %s", e) raise -def _finalize_fairy(connection, connection_record, pool, ref, echo): +def _finalize_fairy(connection, connection_record, pool, ref, echo, fairy=None): + """Cleanup for a :class:`._ConnectionFairy` whether or not it's already + been garbage collected. + + """ _refs.discard(connection_record) if ref is not None and \ - connection_record.fairy is not ref: + connection_record.fairy_ref is not ref: return if connection is not None: + if connection_record and echo: + pool.logger.debug("Connection %r being returned to pool", + connection) + try: + fairy = fairy or _ConnectionFairy(connection, connection_record) + if pool.dispatch.reset: + pool.dispatch.reset(fairy, connection_record) if pool._reset_on_return is reset_rollback: - connection.rollback() + if echo: + pool.logger.debug("Connection %s rollback-on-return", + connection) + pool._dialect.do_rollback(fairy) elif pool._reset_on_return is reset_commit: - connection.commit() + if echo: + pool.logger.debug("Connection %s commit-on-return", + connection) + pool._dialect.do_commit(fairy) + # Immediately close detached instances - if connection_record is None: - connection.close() - except Exception, e: - if connection_record is not None: + if not connection_record: + pool._close_connection(connection) + except Exception as e: + if connection_record: connection_record.invalidate(e=e) if isinstance(e, (SystemExit, KeyboardInterrupt)): raise - if connection_record is not None: - connection_record.fairy = None - if echo: - pool.logger.debug("Connection %r being returned to pool", - connection) - if connection_record.finalize_callback: - connection_record.finalize_callback(connection) - del connection_record.finalize_callback - if pool.dispatch.checkin: - pool.dispatch.checkin(connection, connection_record) - pool._return_conn(connection_record) + if connection_record: + connection_record.checkin() + _refs = set() + class _ConnectionFairy(object): """Proxies a DB-API connection and provides return-on-dereference support.""" - __slots__ = '_pool', '__counter', 'connection', \ - '_connection_record', '__weakref__', \ - '_detached_info', '_echo' + def __init__(self, dbapi_connection, connection_record): + self.connection = dbapi_connection + self._connection_record = connection_record - def __init__(self, pool): - self._pool = pool - self.__counter = 0 - self._echo = _echo = pool._should_log_debug() - try: - rec = self._connection_record = pool._do_get() - conn = self.connection = self._connection_record.get_connection() - rec.fairy = weakref.ref( - self, - lambda ref:_finalize_fairy and _finalize_fairy(conn, rec, pool, ref, _echo) - ) - _refs.add(rec) - except: - # helps with endless __getattr__ loops later on - self.connection = None - self._connection_record = None - raise - if self._echo: - self._pool.logger.debug("Connection %r checked out from pool" % - self.connection) + @classmethod + def checkout(cls, pool, threadconns=None, fairy=None): + if not fairy: + fairy = _ConnectionRecord.checkout(pool) + + fairy._pool = pool + fairy._counter = 0 + fairy._echo = pool._should_log_debug() + + if threadconns is not None: + threadconns.current = weakref.ref(fairy) + + if fairy.connection is None: + raise exc.InvalidRequestError("This connection is closed") + fairy._counter += 1 + + if not pool.dispatch.checkout or fairy._counter != 1: + return fairy + + # Pool listeners can trigger a reconnection on checkout + attempts = 2 + while attempts > 0: + try: + pool.dispatch.checkout(fairy.connection, + fairy._connection_record, + fairy) + return fairy + except exc.DisconnectionError as e: + pool.logger.info( + "Disconnection detected on checkout: %s", e) + fairy._connection_record.invalidate(e) + fairy.connection = fairy._connection_record.get_connection() + attempts -= 1 + + pool.logger.info("Reconnection attempts exhausted on checkout") + fairy.invalidate() + raise exc.InvalidRequestError("This connection is closed") + + def checkout_existing(self): + return _ConnectionFairy.checkout(self._pool, fairy=self) + + def checkin(self): + _finalize_fairy(self.connection, self._connection_record, + self._pool, None, self._echo, fairy=self) + self.connection = None + self._connection_record = None + + _close = checkin @property def _logger(self): @@ -422,20 +517,18 @@ class _ConnectionFairy(object): def is_valid(self): return self.connection is not None - @property + @util.memoized_property def info(self): - """An info collection unique to this DB-API connection.""" + """Info dictionary associated with the underlying DBAPI connection + referred to by this :class:`.ConnectionFairy`, allowing user-defined + data to be associated with the connection. - try: - return self._connection_record.info - except AttributeError: - if self.connection is None: - raise exc.InvalidRequestError("This connection is closed") - try: - return self._detached_info - except AttributeError: - self._detached_info = value = {} - return value + The data here will follow along with the DBAPI connection including + after it is returned to the connection pool and used again + in subsequent instances of :class:`.ConnectionFairy`. + + """ + return self._connection_record.info def invalidate(self, e=None): """Mark this connection as invalidated. @@ -446,10 +539,10 @@ class _ConnectionFairy(object): if self.connection is None: raise exc.InvalidRequestError("This connection is closed") - if self._connection_record is not None: + if self._connection_record: self._connection_record.invalidate(e=e) self.connection = None - self._close() + self.checkin() def cursor(self, *args, **kwargs): return self.connection.cursor(*args, **kwargs) @@ -457,32 +550,6 @@ class _ConnectionFairy(object): def __getattr__(self, key): return getattr(self.connection, key) - def checkout(self): - if self.connection is None: - raise exc.InvalidRequestError("This connection is closed") - self.__counter += 1 - - if not self._pool.dispatch.checkout or self.__counter != 1: - return self - - # Pool listeners can trigger a reconnection on checkout - attempts = 2 - while attempts > 0: - try: - self._pool.dispatch.checkout(self.connection, - self._connection_record, - self) - return self - except exc.DisconnectionError, e: - self._pool.logger.info( - "Disconnection detected on checkout: %s", e) - self._connection_record.invalidate(e) - self.connection = self._connection_record.get_connection() - attempts -= 1 - - self._pool.logger.info("Reconnection attempts exhausted on checkout") - self.invalidate() - raise exc.InvalidRequestError("This connection is closed") def detach(self): """Separate this connection from its Pool. @@ -499,23 +566,19 @@ class _ConnectionFairy(object): if self._connection_record is not None: _refs.remove(self._connection_record) - self._connection_record.fairy = None + self._connection_record.fairy_ref = None self._connection_record.connection = None + # TODO: should this be _return_conn? self._pool._do_return_conn(self._connection_record) - self._detached_info = \ - self._connection_record.info.copy() + self.info = self.info.copy() self._connection_record = None def close(self): - self.__counter -= 1 - if self.__counter == 0: - self._close() + self._counter -= 1 + if self._counter == 0: + self.checkin() + - def _close(self): - _finalize_fairy(self.connection, self._connection_record, - self._pool, None, self._echo) - self.connection = None - self._connection_record = None class SingletonThreadPool(Pool): """A Pool that maintains one connection per thread. @@ -549,7 +612,9 @@ class SingletonThreadPool(Pool): echo=self.echo, logging_name=self._orig_logging_name, use_threadlocal=self._use_threadlocal, - _dispatch=self.dispatch) + reset_on_return=self._reset_on_return, + _dispatch=self.dispatch, + _dialect=self._dialect) def dispose(self): """Dispose of this pool.""" @@ -592,11 +657,6 @@ class SingletonThreadPool(Pool): self._cleanup() return c -class DummyLock(object): - def acquire(self, wait=True): - return True - def release(self): - pass class QueuePool(Pool): """A :class:`.Pool` that imposes a limit on the number of open connections. @@ -705,30 +765,27 @@ class QueuePool(Pool): self._overflow = 0 - pool_size self._max_overflow = max_overflow self._timeout = timeout - self._overflow_lock = self._max_overflow > -1 and \ - threading.Lock() or DummyLock() + self._overflow_lock = threading.Lock() def _do_return_conn(self, conn): try: self._pool.put(conn, False) except sqla_queue.Full: - conn.close() - self._overflow_lock.acquire() try: - self._overflow -= 1 + conn.close() finally: - self._overflow_lock.release() + self._dec_overflow() def _do_get(self): + use_overflow = self._max_overflow > -1 + try: - wait = self._max_overflow > -1 and \ - self._overflow >= self._max_overflow + wait = use_overflow and self._overflow >= self._max_overflow return self._pool.get(wait, self._timeout) - except sqla_queue.SAAbort, aborted: + except sqla_queue.SAAbort as aborted: return aborted.context._do_get() except sqla_queue.Empty: - if self._max_overflow > -1 and \ - self._overflow >= self._max_overflow: + if use_overflow and self._overflow >= self._max_overflow: if not wait: return self._do_get() else: @@ -737,17 +794,33 @@ class QueuePool(Pool): "connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) - self._overflow_lock.acquire() - try: - if self._max_overflow > -1 and \ - self._overflow >= self._max_overflow: - return self._do_get() - else: - con = self._create_connection() - self._overflow += 1 - return con - finally: - self._overflow_lock.release() + if self._inc_overflow(): + try: + return self._create_connection() + except: + self._dec_overflow() + raise + else: + return self._do_get() + + def _inc_overflow(self): + if self._max_overflow == -1: + self._overflow += 1 + return True + with self._overflow_lock: + if self._overflow < self._max_overflow: + self._overflow += 1 + return True + else: + return False + + def _dec_overflow(self): + if self._max_overflow == -1: + self._overflow -= 1 + return True + with self._overflow_lock: + self._overflow -= 1 + return True def recreate(self): self.logger.info("Pool recreating") @@ -757,7 +830,9 @@ class QueuePool(Pool): recycle=self._recycle, echo=self.echo, logging_name=self._orig_logging_name, use_threadlocal=self._use_threadlocal, - _dispatch=self.dispatch) + reset_on_return=self._reset_on_return, + _dispatch=self.dispatch, + _dialect=self._dialect) def dispose(self): while True: @@ -796,6 +871,7 @@ class QueuePool(Pool): def checkedout(self): return self._pool.maxsize - self._pool.qsize() + self._overflow + class NullPool(Pool): """A Pool which does not pool connections. @@ -829,7 +905,9 @@ class NullPool(Pool): echo=self.echo, logging_name=self._orig_logging_name, use_threadlocal=self._use_threadlocal, - _dispatch=self.dispatch) + reset_on_return=self._reset_on_return, + _dispatch=self.dispatch, + _dialect=self._dialect) def dispose(self): pass @@ -869,7 +947,8 @@ class StaticPool(Pool): reset_on_return=self._reset_on_return, echo=self.echo, logging_name=self._orig_logging_name, - _dispatch=self.dispatch) + _dispatch=self.dispatch, + _dialect=self._dialect) def _create_connection(self): return self._conn @@ -880,9 +959,10 @@ class StaticPool(Pool): def _do_get(self): return self.connection + class AssertionPool(Pool): - """A :class:`.Pool` that allows at most one checked out connection at any given - time. + """A :class:`.Pool` that allows at most one checked out connection at + any given time. This will raise an exception if more than one connection is checked out at a time. Useful for debugging code that is using more connections @@ -919,7 +999,8 @@ class AssertionPool(Pool): self.logger.info("Pool recreating") return self.__class__(self._creator, echo=self.echo, logging_name=self._orig_logging_name, - _dispatch=self.dispatch) + _dispatch=self.dispatch, + _dialect=self._dialect) def _do_get(self): if self._checked_out: @@ -938,6 +1019,7 @@ class AssertionPool(Pool): self._checkout_traceback = traceback.format_stack() return self._conn + class _DBProxy(object): """Layers connection pooling behavior on top of a standard DB-API module. @@ -966,7 +1048,7 @@ class _DBProxy(object): self._create_pool_mutex = threading.Lock() def close(self): - for key in self.pools.keys(): + for key in list(self.pools): del self.pools[key] def __del__(self): diff --git a/libs/sqlalchemy/processors.py b/libs/sqlalchemy/processors.py index bc5c3909..0abf063b 100644 --- a/libs/sqlalchemy/processors.py +++ b/libs/sqlalchemy/processors.py @@ -1,5 +1,5 @@ # sqlalchemy/processors.py -# Copyright (C) 2010-2011 the SQLAlchemy authors and contributors +# Copyright (C) 2010-2014 the SQLAlchemy authors and contributors # Copyright (C) 2010 Gaetan de Menten gdementen@gmail.com # # This module is part of SQLAlchemy and is released under @@ -16,10 +16,13 @@ import codecs import re import datetime + def str_to_datetime_processor_factory(regexp, type_): rmatch = regexp.match # Even on python2.6 datetime.strptime is both slower than this code # and it does not support microseconds. + has_named_groups = bool(regexp.groupindex) + def process(value): if value is None: return None @@ -28,19 +31,27 @@ def str_to_datetime_processor_factory(regexp, type_): m = rmatch(value) except TypeError: raise ValueError("Couldn't parse %s string '%r' " - "- value is not a string." % (type_.__name__ , value)) + "- value is not a string." % + (type_.__name__, value)) if m is None: raise ValueError("Couldn't parse %s string: " - "'%s'" % (type_.__name__ , value)) - return type_(*map(int, m.groups(0))) + "'%s'" % (type_.__name__, value)) + if has_named_groups: + groups = m.groupdict(0) + return type_(**dict(list(zip(iter(groups.keys()), + list(map(int, iter(groups.values()))))))) + else: + return type_(*list(map(int, m.groups(0)))) return process + def boolean_to_int(value): if value is None: return None else: return int(value) + def py_fallback(): def to_unicode_processor_factory(encoding, errors=None): decoder = codecs.getdecoder(encoding) @@ -55,7 +66,7 @@ def py_fallback(): return decoder(value, errors)[0] return process - def to_decimal_processor_factory(target_class, scale=10): + def to_decimal_processor_factory(target_class, scale): fstring = "%%.%df" % scale def process(value): @@ -108,7 +119,7 @@ try: else: return UnicodeResultProcessor(encoding).process - def to_decimal_processor_factory(target_class, scale=10): + def to_decimal_processor_factory(target_class, scale): # Note that the scale argument is not taken into account for integer # values in the C implementation while it is in the Python one. # For example, the Python implementation might return @@ -118,4 +129,3 @@ try: except ImportError: globals().update(py_fallback()) - diff --git a/libs/sqlalchemy/schema.py b/libs/sqlalchemy/schema.py index 154fb5f7..17214153 100644 --- a/libs/sqlalchemy/schema.py +++ b/libs/sqlalchemy/schema.py @@ -1,3204 +1,54 @@ -# sqlalchemy/schema.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# schema.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""The schema module provides the building blocks for database metadata. - -Each element within this module describes a database entity which can be -created and dropped, or is otherwise part of such an entity. Examples include -tables, columns, sequences, and indexes. - -All entities are subclasses of :class:`~sqlalchemy.schema.SchemaItem`, and as -defined in this module they are intended to be agnostic of any vendor-specific -constructs. - -A collection of entities are grouped into a unit called -:class:`~sqlalchemy.schema.MetaData`. MetaData serves as a logical grouping of -schema elements, and can also be associated with an actual database connection -such that operations involving the contained elements can contact the database -as needed. - -Two of the elements here also build upon their "syntactic" counterparts, which -are defined in :class:`~sqlalchemy.sql.expression.`, specifically -:class:`~sqlalchemy.schema.Table` and :class:`~sqlalchemy.schema.Column`. -Since these objects are part of the SQL expression language, they are usable -as components in SQL expressions. +"""Compatiblity namespace for sqlalchemy.sql.schema and related. """ -import re, inspect -from sqlalchemy import exc, util, dialects -from sqlalchemy.sql import expression, visitors -from sqlalchemy import event, events -ddl = util.importlater("sqlalchemy.engine", "ddl") -sqlutil = util.importlater("sqlalchemy.sql", "util") -url = util.importlater("sqlalchemy.engine", "url") -sqltypes = util.importlater("sqlalchemy", "types") - -__all__ = ['SchemaItem', 'Table', 'Column', 'ForeignKey', 'Sequence', 'Index', - 'ForeignKeyConstraint', 'PrimaryKeyConstraint', 'CheckConstraint', - 'UniqueConstraint', 'DefaultGenerator', 'Constraint', 'MetaData', - 'ThreadLocalMetaData', 'SchemaVisitor', 'PassiveDefault', - 'DefaultClause', 'FetchedValue', 'ColumnDefault', 'DDL', - 'CreateTable', 'DropTable', 'CreateSequence', 'DropSequence', - 'AddConstraint', 'DropConstraint', - ] -__all__.sort() - -RETAIN_SCHEMA = util.symbol('retain_schema') - -class SchemaItem(events.SchemaEventTarget, visitors.Visitable): - """Base class for items that define a database schema.""" - - __visit_name__ = 'schema_item' - quote = None - - def _init_items(self, *args): - """Initialize the list of child items for this SchemaItem.""" - - for item in args: - if item is not None: - item._set_parent_with_dispatch(self) - - def get_children(self, **kwargs): - """used to allow SchemaVisitor access""" - return [] - - def __repr__(self): - return util.generic_repr(self) - - @util.memoized_property - def info(self): - return {} - -def _get_table_key(name, schema): - if schema is None: - return name - else: - return schema + "." + name - -def _validate_dialect_kwargs(kwargs, name): - # validate remaining kwargs that they all specify DB prefixes - if len([k for k in kwargs - if not re.match( - r'^(?:%s)_' % - '|'.join(dialects.__all__), k - ) - ]): - raise TypeError( - "Invalid argument(s) for %s: %r" % (name, kwargs.keys())) - - -class Table(SchemaItem, expression.TableClause): - """Represent a table in a database. - - e.g.:: - - mytable = Table("mytable", metadata, - Column('mytable_id', Integer, primary_key=True), - Column('value', String(50)) - ) - - The :class:`.Table` object constructs a unique instance of itself based on its - name and optional schema name within the given :class:`.MetaData` object. - Calling the :class:`.Table` - constructor with the same name and same :class:`.MetaData` argument - a second time will return the *same* :class:`.Table` object - in this way - the :class:`.Table` constructor acts as a registry function. - - See also: - - :ref:`metadata_describing` - Introduction to database metadata - - Constructor arguments are as follows: - - :param name: The name of this table as represented in the database. - - This property, along with the *schema*, indicates the *singleton - identity* of this table in relation to its parent :class:`.MetaData`. - Additional calls to :class:`.Table` with the same name, metadata, - and schema name will return the same :class:`.Table` object. - - Names which contain no upper case characters - will be treated as case insensitive names, and will not be quoted - unless they are a reserved word. Names with any number of upper - case characters will be quoted and sent exactly. Note that this - behavior applies even for databases which standardize upper - case names as case insensitive such as Oracle. - - :param metadata: a :class:`.MetaData` object which will contain this - table. The metadata is used as a point of association of this table - with other tables which are referenced via foreign key. It also - may be used to associate this table with a particular - :class:`~sqlalchemy.engine.base.Connectable`. - - :param \*args: Additional positional arguments are used primarily - to add the list of :class:`.Column` objects contained within this - table. Similar to the style of a CREATE TABLE statement, other - :class:`.SchemaItem` constructs may be added here, including - :class:`.PrimaryKeyConstraint`, and :class:`.ForeignKeyConstraint`. - - :param autoload: Defaults to False: the Columns for this table should - be reflected from the database. Usually there will be no Column - objects in the constructor if this property is set. - - :param autoload_replace: If ``True``, when using ``autoload=True`` - and ``extend_existing=True``, - replace ``Column`` objects already present in the ``Table`` that's - in the ``MetaData`` registry with - what's reflected. Otherwise, all existing columns will be - excluded from the reflection process. Note that this does - not impact ``Column`` objects specified in the same call to ``Table`` - which includes ``autoload``, those always take precedence. - Defaults to ``True``. - - .. versionadded:: 0.7.5 - - :param autoload_with: If autoload==True, this is an optional Engine - or Connection instance to be used for the table reflection. If - ``None``, the underlying MetaData's bound connectable will be used. - - :param extend_existing: When ``True``, indicates that if this :class:`.Table` is already - present in the given :class:`.MetaData`, apply further arguments within - the constructor to the existing :class:`.Table`. - - If ``extend_existing`` or ``keep_existing`` are not set, an error is - raised if additional table modifiers are specified when - the given :class:`.Table` is already present in the :class:`.MetaData`. - - .. versionchanged:: 0.7.4 - ``extend_existing`` will work in conjunction - with ``autoload=True`` to run a new reflection operation against - the database; new :class:`.Column` objects will be produced - from database metadata to replace those existing with the same - name, and additional :class:`.Column` objects not present - in the :class:`.Table` will be added. - - As is always the case with ``autoload=True``, :class:`.Column` - objects can be specified in the same :class:`.Table` constructor, - which will take precedence. I.e.:: - - Table("mytable", metadata, - Column('y', Integer), - extend_existing=True, - autoload=True, - autoload_with=engine - ) - - The above will overwrite all columns within ``mytable`` which - are present in the database, except for ``y`` which will be used as is - from the above definition. If the ``autoload_replace`` flag - is set to False, no existing columns will be replaced. - - :param implicit_returning: True by default - indicates that - RETURNING can be used by default to fetch newly inserted primary key - values, for backends which support this. Note that - create_engine() also provides an implicit_returning flag. - - :param include_columns: A list of strings indicating a subset of - columns to be loaded via the ``autoload`` operation; table columns who - aren't present in this list will not be represented on the resulting - ``Table`` object. Defaults to ``None`` which indicates all columns - should be reflected. - - :param info: A dictionary which defaults to ``{}``. A space to store - application specific data. This must be a dictionary. - - :param keep_existing: When ``True``, indicates that if this Table - is already present in the given :class:`.MetaData`, ignore - further arguments within the constructor to the existing - :class:`.Table`, and return the :class:`.Table` object as - originally created. This is to allow a function that wishes - to define a new :class:`.Table` on first call, but on - subsequent calls will return the same :class:`.Table`, - without any of the declarations (particularly constraints) - being applied a second time. Also see extend_existing. - - If extend_existing or keep_existing are not set, an error is - raised if additional table modifiers are specified when - the given :class:`.Table` is already present in the :class:`.MetaData`. - - :param listeners: A list of tuples of the form ``(, )`` - which will be passed to :func:`.event.listen` upon construction. - This alternate hook to :func:`.event.listen` allows the establishment - of a listener function specific to this :class:`.Table` before - the "autoload" process begins. Particularly useful for - the :meth:`.events.column_reflect` event:: - - def listen_for_reflect(table, column_info): - "handle the column reflection event" - # ... - - t = Table( - 'sometable', - autoload=True, - listeners=[ - ('column_reflect', listen_for_reflect) - ]) - - :param mustexist: When ``True``, indicates that this Table must already - be present in the given :class:`.MetaData`` collection, else - an exception is raised. - - :param prefixes: - A list of strings to insert after CREATE in the CREATE TABLE - statement. They will be separated by spaces. - - :param quote: Force quoting of this table's name on or off, corresponding - to ``True`` or ``False``. When left at its default of ``None``, - the column identifier will be quoted according to whether the name is - case sensitive (identifiers with at least one upper case character are - treated as case sensitive), or if it's a reserved word. This flag - is only needed to force quoting of a reserved word which is not known - by the SQLAlchemy dialect. - - :param quote_schema: same as 'quote' but applies to the schema identifier. - - :param schema: The *schema name* for this table, which is required if - the table resides in a schema other than the default selected schema - for the engine's database connection. Defaults to ``None``. - - :param useexisting: Deprecated. Use extend_existing. - - """ - - __visit_name__ = 'table' - - def __new__(cls, *args, **kw): - if not args: - # python3k pickle seems to call this - return object.__new__(cls) - - try: - name, metadata, args = args[0], args[1], args[2:] - except IndexError: - raise TypeError("Table() takes at least two arguments") - - schema = kw.get('schema', None) - if schema is None: - schema = metadata.schema - keep_existing = kw.pop('keep_existing', False) - extend_existing = kw.pop('extend_existing', False) - if 'useexisting' in kw: - util.warn_deprecated("useexisting is deprecated. Use extend_existing.") - if extend_existing: - raise exc.ArgumentError("useexisting is synonymous " - "with extend_existing.") - extend_existing = kw.pop('useexisting', False) - - if keep_existing and extend_existing: - raise exc.ArgumentError("keep_existing and extend_existing " - "are mutually exclusive.") - - mustexist = kw.pop('mustexist', False) - key = _get_table_key(name, schema) - if key in metadata.tables: - if not keep_existing and not extend_existing and bool(args): - raise exc.InvalidRequestError( - "Table '%s' is already defined for this MetaData " - "instance. Specify 'extend_existing=True' " - "to redefine " - "options and columns on an " - "existing Table object." % key) - table = metadata.tables[key] - if extend_existing: - table._init_existing(*args, **kw) - return table - else: - if mustexist: - raise exc.InvalidRequestError( - "Table '%s' not defined" % (key)) - table = object.__new__(cls) - table.dispatch.before_parent_attach(table, metadata) - metadata._add_table(name, schema, table) - try: - table._init(name, metadata, *args, **kw) - table.dispatch.after_parent_attach(table, metadata) - return table - except: - metadata._remove_table(name, schema) - raise - - def __init__(self, *args, **kw): - """Constructor for :class:`~.schema.Table`. - - This method is a no-op. See the top-level - documentation for :class:`~.schema.Table` - for constructor arguments. - - """ - # __init__ is overridden to prevent __new__ from - # calling the superclass constructor. - - def _init(self, name, metadata, *args, **kwargs): - super(Table, self).__init__(name) - self.metadata = metadata - self.schema = kwargs.pop('schema', None) - if self.schema is None: - self.schema = metadata.schema - self.quote_schema = kwargs.pop('quote_schema', metadata.quote_schema) - else: - self.quote_schema = kwargs.pop('quote_schema', None) - - self.indexes = set() - self.constraints = set() - self._columns = expression.ColumnCollection() - PrimaryKeyConstraint()._set_parent_with_dispatch(self) - self.foreign_keys = set() - self._extra_dependencies = set() - self.kwargs = {} - if self.schema is not None: - self.fullname = "%s.%s" % (self.schema, self.name) - else: - self.fullname = self.name - - autoload = kwargs.pop('autoload', False) - autoload_with = kwargs.pop('autoload_with', None) - # this argument is only used with _init_existing() - kwargs.pop('autoload_replace', True) - include_columns = kwargs.pop('include_columns', None) - - self.implicit_returning = kwargs.pop('implicit_returning', True) - self.quote = kwargs.pop('quote', None) - if 'info' in kwargs: - self.info = kwargs.pop('info') - if 'listeners' in kwargs: - listeners = kwargs.pop('listeners') - for evt, fn in listeners: - event.listen(self, evt, fn) - - self._prefixes = kwargs.pop('prefixes', []) - - self._extra_kwargs(**kwargs) - - # load column definitions from the database if 'autoload' is defined - # we do it after the table is in the singleton dictionary to support - # circular foreign keys - if autoload: - self._autoload(metadata, autoload_with, include_columns) - - # initialize all the column, etc. objects. done after reflection to - # allow user-overrides - self._init_items(*args) - - def _autoload(self, metadata, autoload_with, include_columns, exclude_columns=()): - if self.primary_key.columns: - PrimaryKeyConstraint(*[ - c for c in self.primary_key.columns - if c.key in exclude_columns - ])._set_parent_with_dispatch(self) - - if autoload_with: - autoload_with.run_callable( - autoload_with.dialect.reflecttable, - self, include_columns, exclude_columns - ) - else: - bind = _bind_or_error(metadata, - msg="No engine is bound to this Table's MetaData. " - "Pass an engine to the Table via " - "autoload_with=, " - "or associate the MetaData with an engine via " - "metadata.bind=") - bind.run_callable( - bind.dialect.reflecttable, - self, include_columns, exclude_columns - ) - - @property - def _sorted_constraints(self): - """Return the set of constraints as a list, sorted by creation order.""" - - return sorted(self.constraints, key=lambda c:c._creation_order) - - def _init_existing(self, *args, **kwargs): - autoload = kwargs.pop('autoload', False) - autoload_with = kwargs.pop('autoload_with', None) - autoload_replace = kwargs.pop('autoload_replace', True) - schema = kwargs.pop('schema', None) - if schema and schema != self.schema: - raise exc.ArgumentError( - "Can't change schema of existing table from '%s' to '%s'", - (self.schema, schema)) - - include_columns = kwargs.pop('include_columns', None) - - if include_columns is not None: - for c in self.c: - if c.name not in include_columns: - self._columns.remove(c) - - for key in ('quote', 'quote_schema'): - if key in kwargs: - setattr(self, key, kwargs.pop(key)) - - if 'info' in kwargs: - self.info = kwargs.pop('info') - - if autoload: - if not autoload_replace: - exclude_columns = [c.name for c in self.c] - else: - exclude_columns = () - self._autoload(self.metadata, autoload_with, include_columns, exclude_columns) - - self._extra_kwargs(**kwargs) - self._init_items(*args) - - def _extra_kwargs(self, **kwargs): - # validate remaining kwargs that they all specify DB prefixes - _validate_dialect_kwargs(kwargs, "Table") - self.kwargs.update(kwargs) - - def _init_collections(self): - pass - - @util.memoized_property - def _autoincrement_column(self): - for col in self.primary_key: - if col.autoincrement and \ - col.type._type_affinity is not None and \ - issubclass(col.type._type_affinity, sqltypes.Integer) and \ - (not col.foreign_keys or col.autoincrement == 'ignore_fk') and \ - isinstance(col.default, (type(None), Sequence)) and \ - (col.server_default is None or col.server_default.reflected): - return col - - @property - def key(self): - return _get_table_key(self.name, self.schema) - - def __repr__(self): - return "Table(%s)" % ', '.join( - [repr(self.name)] + [repr(self.metadata)] + - [repr(x) for x in self.columns] + - ["%s=%s" % (k, repr(getattr(self, k))) for k in ['schema']]) - - def __str__(self): - return _get_table_key(self.description, self.schema) - - @property - def bind(self): - """Return the connectable associated with this Table.""" - - return self.metadata and self.metadata.bind or None - - def add_is_dependent_on(self, table): - """Add a 'dependency' for this Table. - - This is another Table object which must be created - first before this one can, or dropped after this one. - - Usually, dependencies between tables are determined via - ForeignKey objects. However, for other situations that - create dependencies outside of foreign keys (rules, inheriting), - this method can manually establish such a link. - - """ - self._extra_dependencies.add(table) - - def append_column(self, column): - """Append a :class:`~.schema.Column` to this :class:`~.schema.Table`. - - The "key" of the newly added :class:`~.schema.Column`, i.e. the - value of its ``.key`` attribute, will then be available - in the ``.c`` collection of this :class:`~.schema.Table`, and the - column definition will be included in any CREATE TABLE, SELECT, - UPDATE, etc. statements generated from this :class:`~.schema.Table` - construct. - - Note that this does **not** change the definition of the table - as it exists within any underlying database, assuming that - table has already been created in the database. Relational - databases support the addition of columns to existing tables - using the SQL ALTER command, which would need to be - emitted for an already-existing table that doesn't contain - the newly added column. - - """ - - column._set_parent_with_dispatch(self) - - def append_constraint(self, constraint): - """Append a :class:`~.schema.Constraint` to this :class:`~.schema.Table`. - - This has the effect of the constraint being included in any - future CREATE TABLE statement, assuming specific DDL creation - events have not been associated with the given :class:`~.schema.Constraint` - object. - - Note that this does **not** produce the constraint within the - relational database automatically, for a table that already exists - in the database. To add a constraint to an - existing relational database table, the SQL ALTER command must - be used. SQLAlchemy also provides the :class:`.AddConstraint` construct - which can produce this SQL when invoked as an executable clause. - - """ - - constraint._set_parent_with_dispatch(self) - - def append_ddl_listener(self, event_name, listener): - """Append a DDL event listener to this ``Table``. - - Deprecated. See :class:`.DDLEvents`. - - """ - - def adapt_listener(target, connection, **kw): - listener(event_name, target, connection) - - event.listen(self, "" + event_name.replace('-', '_'), adapt_listener) - - def _set_parent(self, metadata): - metadata._add_table(self.name, self.schema, self) - self.metadata = metadata - - def get_children(self, column_collections=True, - schema_visitor=False, **kw): - if not schema_visitor: - return expression.TableClause.get_children( - self, column_collections=column_collections, **kw) - else: - if column_collections: - return list(self.columns) - else: - return [] - - def exists(self, bind=None): - """Return True if this table exists.""" - - if bind is None: - bind = _bind_or_error(self) - - return bind.run_callable(bind.dialect.has_table, - self.name, schema=self.schema) - - def create(self, bind=None, checkfirst=False): - """Issue a ``CREATE`` statement for this - :class:`.Table`, using the given :class:`.Connectable` - for connectivity. - - See also :meth:`.MetaData.create_all`. - - """ - - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaGenerator, - self, - checkfirst=checkfirst) - - - def drop(self, bind=None, checkfirst=False): - """Issue a ``DROP`` statement for this - :class:`.Table`, using the given :class:`.Connectable` - for connectivity. - - See also :meth:`.MetaData.drop_all`. - - """ - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaDropper, - self, - checkfirst=checkfirst) - - - def tometadata(self, metadata, schema=RETAIN_SCHEMA): - """Return a copy of this :class:`.Table` associated with a different - :class:`.MetaData`. - - E.g.:: - - # create two metadata - meta1 = MetaData('sqlite:///querytest.db') - meta2 = MetaData() - - # load 'users' from the sqlite engine - users_table = Table('users', meta1, autoload=True) - - # create the same Table object for the plain metadata - users_table_2 = users_table.tometadata(meta2) - - """ - - if schema is RETAIN_SCHEMA: - schema = self.schema - elif schema is None: - schema = metadata.schema - key = _get_table_key(self.name, schema) - if key in metadata.tables: - util.warn("Table '%s' already exists within the given " - "MetaData - not copying." % self.description) - return metadata.tables[key] - - args = [] - for c in self.columns: - args.append(c.copy(schema=schema)) - for c in self.constraints: - args.append(c.copy(schema=schema)) - table = Table( - self.name, metadata, schema=schema, - *args, **self.kwargs - ) - for index in self.indexes: - # skip indexes that would be generated - # by the 'index' flag on Column - if len(index.columns) == 1 and \ - list(index.columns)[0].index: - continue - Index(index.name, - unique=index.unique, - *[table.c[col] for col in index.columns.keys()], - **index.kwargs) - table.dispatch._update(self.dispatch) - return table - -class Column(SchemaItem, expression.ColumnClause): - """Represents a column in a database table.""" - - __visit_name__ = 'column' - - def __init__(self, *args, **kwargs): - """ - Construct a new ``Column`` object. - - :param name: The name of this column as represented in the database. - This argument may be the first positional argument, or specified - via keyword. - - Names which contain no upper case characters - will be treated as case insensitive names, and will not be quoted - unless they are a reserved word. Names with any number of upper - case characters will be quoted and sent exactly. Note that this - behavior applies even for databases which standardize upper - case names as case insensitive such as Oracle. - - The name field may be omitted at construction time and applied - later, at any time before the Column is associated with a - :class:`.Table`. This is to support convenient - usage within the :mod:`~sqlalchemy.ext.declarative` extension. - - :param type\_: The column's type, indicated using an instance which - subclasses :class:`~sqlalchemy.types.TypeEngine`. If no arguments - are required for the type, the class of the type can be sent - as well, e.g.:: - - # use a type with arguments - Column('data', String(50)) - - # use no arguments - Column('level', Integer) - - The ``type`` argument may be the second positional argument - or specified by keyword. - - There is partial support for automatic detection of the - type based on that of a :class:`.ForeignKey` associated - with this column, if the type is specified as ``None``. - However, this feature is not fully implemented and - may not function in all cases. - - :param \*args: Additional positional arguments include various - :class:`.SchemaItem` derived constructs which will be applied - as options to the column. These include instances of - :class:`.Constraint`, :class:`.ForeignKey`, :class:`.ColumnDefault`, - and :class:`.Sequence`. In some cases an equivalent keyword - argument is available such as ``server_default``, ``default`` - and ``unique``. - - :param autoincrement: This flag may be set to ``False`` to - indicate an integer primary key column that should not be - considered to be the "autoincrement" column, that is - the integer primary key column which generates values - implicitly upon INSERT and whose value is usually returned - via the DBAPI cursor.lastrowid attribute. It defaults - to ``True`` to satisfy the common use case of a table - with a single integer primary key column. If the table - has a composite primary key consisting of more than one - integer column, set this flag to True only on the - column that should be considered "autoincrement". - - The setting *only* has an effect for columns which are: - - * Integer derived (i.e. INT, SMALLINT, BIGINT). - - * Part of the primary key - - * Are not referenced by any foreign keys, unless - the value is specified as ``'ignore_fk'`` - - .. versionadded:: 0.7.4 - - * have no server side or client side defaults (with the exception - of Postgresql SERIAL). - - The setting has these two effects on columns that meet the - above criteria: - - * DDL issued for the column will include database-specific - keywords intended to signify this column as an - "autoincrement" column, such as AUTO INCREMENT on MySQL, - SERIAL on Postgresql, and IDENTITY on MS-SQL. It does - *not* issue AUTOINCREMENT for SQLite since this is a - special SQLite flag that is not required for autoincrementing - behavior. See the SQLite dialect documentation for - information on SQLite's AUTOINCREMENT. - - * The column will be considered to be available as - cursor.lastrowid or equivalent, for those dialects which - "post fetch" newly inserted identifiers after a row has - been inserted (SQLite, MySQL, MS-SQL). It does not have - any effect in this regard for databases that use sequences - to generate primary key identifiers (i.e. Firebird, Postgresql, - Oracle). - - .. versionchanged:: 0.7.4 - ``autoincrement`` accepts a special value ``'ignore_fk'`` - to indicate that autoincrementing status regardless of foreign key - references. This applies to certain composite foreign key - setups, such as the one demonstrated in the ORM documentation - at :ref:`post_update`. - - :param default: A scalar, Python callable, or - :class:`~sqlalchemy.sql.expression.ClauseElement` representing the - *default value* for this column, which will be invoked upon insert - if this column is otherwise not specified in the VALUES clause of - the insert. This is a shortcut to using :class:`.ColumnDefault` as - a positional argument. - - Contrast this argument to ``server_default`` which creates a - default generator on the database side. - - :param doc: optional String that can be used by the ORM or similar - to document attributes. This attribute does not render SQL - comments (a future attribute 'comment' will achieve that). - - :param key: An optional string identifier which will identify this - ``Column`` object on the :class:`.Table`. When a key is provided, - this is the only identifier referencing the ``Column`` within the - application, including ORM attribute mapping; the ``name`` field - is used only when rendering SQL. - - :param index: When ``True``, indicates that the column is indexed. - This is a shortcut for using a :class:`.Index` construct on the - table. To specify indexes with explicit names or indexes that - contain multiple columns, use the :class:`.Index` construct - instead. - - :param info: A dictionary which defaults to ``{}``. A space to store - application specific data. This must be a dictionary. - - :param nullable: If set to the default of ``True``, indicates the - column will be rendered as allowing NULL, else it's rendered as - NOT NULL. This parameter is only used when issuing CREATE TABLE - statements. - - :param onupdate: A scalar, Python callable, or - :class:`~sqlalchemy.sql.expression.ClauseElement` representing a - default value to be applied to the column within UPDATE - statements, which wil be invoked upon update if this column is not - present in the SET clause of the update. This is a shortcut to - using :class:`.ColumnDefault` as a positional argument with - ``for_update=True``. - - :param primary_key: If ``True``, marks this column as a primary key - column. Multiple columns can have this flag set to specify - composite primary keys. As an alternative, the primary key of a - :class:`.Table` can be specified via an explicit - :class:`.PrimaryKeyConstraint` object. - - :param server_default: A :class:`.FetchedValue` instance, str, Unicode - or :func:`~sqlalchemy.sql.expression.text` construct representing - the DDL DEFAULT value for the column. - - String types will be emitted as-is, surrounded by single quotes:: - - Column('x', Text, server_default="val") - - x TEXT DEFAULT 'val' - - A :func:`~sqlalchemy.sql.expression.text` expression will be - rendered as-is, without quotes:: - - Column('y', DateTime, server_default=text('NOW()'))0 - - y DATETIME DEFAULT NOW() - - Strings and text() will be converted into a :class:`.DefaultClause` - object upon initialization. - - Use :class:`.FetchedValue` to indicate that an already-existing - column will generate a default value on the database side which - will be available to SQLAlchemy for post-fetch after inserts. This - construct does not specify any DDL and the implementation is left - to the database, such as via a trigger. - - :param server_onupdate: A :class:`.FetchedValue` instance - representing a database-side default generation function. This - indicates to SQLAlchemy that a newly generated value will be - available after updates. This construct does not specify any DDL - and the implementation is left to the database, such as via a - trigger. - - :param quote: Force quoting of this column's name on or off, - corresponding to ``True`` or ``False``. When left at its default - of ``None``, the column identifier will be quoted according to - whether the name is case sensitive (identifiers with at least one - upper case character are treated as case sensitive), or if it's a - reserved word. This flag is only needed to force quoting of a - reserved word which is not known by the SQLAlchemy dialect. - - :param unique: When ``True``, indicates that this column contains a - unique constraint, or if ``index`` is ``True`` as well, indicates - that the :class:`.Index` should be created with the unique flag. - To specify multiple columns in the constraint/index or to specify - an explicit name, use the :class:`.UniqueConstraint` or - :class:`.Index` constructs explicitly. - - """ - - name = kwargs.pop('name', None) - type_ = kwargs.pop('type_', None) - args = list(args) - if args: - if isinstance(args[0], basestring): - if name is not None: - raise exc.ArgumentError( - "May not pass name positionally and as a keyword.") - name = args.pop(0) - if args: - coltype = args[0] - - if (isinstance(coltype, sqltypes.TypeEngine) or - (isinstance(coltype, type) and - issubclass(coltype, sqltypes.TypeEngine))): - if type_ is not None: - raise exc.ArgumentError( - "May not pass type_ positionally and as a keyword.") - type_ = args.pop(0) - - no_type = type_ is None - - super(Column, self).__init__(name, None, type_) - self.key = kwargs.pop('key', name) - self.primary_key = kwargs.pop('primary_key', False) - self.nullable = kwargs.pop('nullable', not self.primary_key) - self.default = kwargs.pop('default', None) - self.server_default = kwargs.pop('server_default', None) - self.server_onupdate = kwargs.pop('server_onupdate', None) - self.index = kwargs.pop('index', None) - self.unique = kwargs.pop('unique', None) - self.quote = kwargs.pop('quote', None) - self.doc = kwargs.pop('doc', None) - self.onupdate = kwargs.pop('onupdate', None) - self.autoincrement = kwargs.pop('autoincrement', True) - self.constraints = set() - self.foreign_keys = set() - - # check if this Column is proxying another column - if '_proxies' in kwargs: - self.proxies = kwargs.pop('_proxies') - # otherwise, add DDL-related events - elif isinstance(self.type, sqltypes.SchemaType): - self.type._set_parent_with_dispatch(self) - - if self.default is not None: - if isinstance(self.default, (ColumnDefault, Sequence)): - args.append(self.default) - else: - if getattr(self.type, '_warn_on_bytestring', False): - # Py3K - #if isinstance(self.default, bytes): - # Py2K - if isinstance(self.default, str): - # end Py2K - util.warn("Unicode column received non-unicode " - "default value.") - args.append(ColumnDefault(self.default)) - - if self.server_default is not None: - if isinstance(self.server_default, FetchedValue): - args.append(self.server_default._as_for_update(False)) - else: - args.append(DefaultClause(self.server_default)) - - if self.onupdate is not None: - if isinstance(self.onupdate, (ColumnDefault, Sequence)): - args.append(self.onupdate) - else: - args.append(ColumnDefault(self.onupdate, for_update=True)) - - if self.server_onupdate is not None: - if isinstance(self.server_onupdate, FetchedValue): - args.append(self.server_onupdate._as_for_update(True)) - else: - args.append(DefaultClause(self.server_onupdate, - for_update=True)) - self._init_items(*args) - - if not self.foreign_keys and no_type: - raise exc.ArgumentError("'type' is required on Column objects " - "which have no foreign keys.") - util.set_creation_order(self) - - if 'info' in kwargs: - self.info = kwargs.pop('info') - - if kwargs: - raise exc.ArgumentError( - "Unknown arguments passed to Column: " + repr(kwargs.keys())) - - def __str__(self): - if self.name is None: - return "(no name)" - elif self.table is not None: - if self.table.named_with_column: - return (self.table.description + "." + self.description) - else: - return self.description - else: - return self.description - - def references(self, column): - """Return True if this Column references the given column via foreign - key.""" - - for fk in self.foreign_keys: - if fk.column.proxy_set.intersection(column.proxy_set): - return True - else: - return False - - def append_foreign_key(self, fk): - fk._set_parent_with_dispatch(self) - - def __repr__(self): - kwarg = [] - if self.key != self.name: - kwarg.append('key') - if self.primary_key: - kwarg.append('primary_key') - if not self.nullable: - kwarg.append('nullable') - if self.onupdate: - kwarg.append('onupdate') - if self.default: - kwarg.append('default') - if self.server_default: - kwarg.append('server_default') - return "Column(%s)" % ', '.join( - [repr(self.name)] + [repr(self.type)] + - [repr(x) for x in self.foreign_keys if x is not None] + - [repr(x) for x in self.constraints] + - [(self.table is not None and "table=<%s>" % - self.table.description or "table=None")] + - ["%s=%s" % (k, repr(getattr(self, k))) for k in kwarg]) - - def _set_parent(self, table): - if not self.name: - raise exc.ArgumentError( - "Column must be constructed with a non-blank name or " - "assign a non-blank .name before adding to a Table.") - if self.key is None: - self.key = self.name - - existing = getattr(self, 'table', None) - if existing is not None and existing is not table: - raise exc.ArgumentError( - "Column object already assigned to Table '%s'" % - existing.description) - - if self.key in table._columns: - col = table._columns.get(self.key) - if col is not self: - for fk in list(col.foreign_keys): - table.foreign_keys.remove(fk) - if fk.constraint in table.constraints: - # this might have been removed - # already, if it's a composite constraint - # and more than one col being replaced - table.constraints.remove(fk.constraint) - - table._columns.replace(self) - - if self.primary_key: - table.primary_key._replace(self) - Table._autoincrement_column._reset(table) - elif self.key in table.primary_key: - raise exc.ArgumentError( - "Trying to redefine primary-key column '%s' as a " - "non-primary-key column on table '%s'" % ( - self.key, table.fullname)) - self.table = table - - if self.index: - if isinstance(self.index, basestring): - raise exc.ArgumentError( - "The 'index' keyword argument on Column is boolean only. " - "To create indexes with a specific name, create an " - "explicit Index object external to the Table.") - Index(expression._truncated_label('ix_%s' % self._label), self, unique=self.unique) - elif self.unique: - if isinstance(self.unique, basestring): - raise exc.ArgumentError( - "The 'unique' keyword argument on Column is boolean " - "only. To create unique constraints or indexes with a " - "specific name, append an explicit UniqueConstraint to " - "the Table's list of elements, or create an explicit " - "Index object external to the Table.") - table.append_constraint(UniqueConstraint(self.key)) - - def _on_table_attach(self, fn): - if self.table is not None: - fn(self, self.table) - event.listen(self, 'after_parent_attach', fn) - - def copy(self, **kw): - """Create a copy of this ``Column``, unitialized. - - This is used in ``Table.tometadata``. - - """ - - # Constraint objects plus non-constraint-bound ForeignKey objects - args = \ - [c.copy(**kw) for c in self.constraints] + \ - [c.copy(**kw) for c in self.foreign_keys if not c.constraint] - - c = self._constructor( - name=self.name, - type_=self.type, - key = self.key, - primary_key = self.primary_key, - nullable = self.nullable, - unique = self.unique, - quote=self.quote, - index=self.index, - autoincrement=self.autoincrement, - default=self.default, - server_default=self.server_default, - onupdate=self.onupdate, - server_onupdate=self.server_onupdate, - info=self.info, - doc=self.doc, - *args - ) - c.dispatch._update(self.dispatch) - return c - - def _make_proxy(self, selectable, name=None): - """Create a *proxy* for this column. - - This is a copy of this ``Column`` referenced by a different parent - (such as an alias or select statement). The column should - be used only in select scenarios, as its full DDL/default - information is not transferred. - - """ - fk = [ForeignKey(f.column) for f in self.foreign_keys] - if name is None and self.name is None: - raise exc.InvalidRequestError("Cannot initialize a sub-selectable" - " with this Column object until it's 'name' has " - "been assigned.") - try: - c = self._constructor( - expression._as_truncated(name or self.name), - self.type, - key = name or self.key, - primary_key = self.primary_key, - nullable = self.nullable, - quote=self.quote, _proxies=[self], *fk) - except TypeError, e: - # Py3K - #raise TypeError( - # "Could not create a copy of this %r object. " - # "Ensure the class includes a _constructor() " - # "attribute or method which accepts the " - # "standard Column constructor arguments, or " - # "references the Column class itself." % self.__class__) from e - # Py2K - raise TypeError( - "Could not create a copy of this %r object. " - "Ensure the class includes a _constructor() " - "attribute or method which accepts the " - "standard Column constructor arguments, or " - "references the Column class itself. " - "Original error: %s" % (self.__class__, e)) - # end Py2K - - c.table = selectable - selectable._columns.add(c) - if selectable._is_clone_of is not None: - c._is_clone_of = selectable._is_clone_of.columns[c.name] - if self.primary_key: - selectable.primary_key.add(c) - c.dispatch.after_parent_attach(c, selectable) - return c - - def get_children(self, schema_visitor=False, **kwargs): - if schema_visitor: - return [x for x in (self.default, self.onupdate) - if x is not None] + \ - list(self.foreign_keys) + list(self.constraints) - else: - return expression.ColumnClause.get_children(self, **kwargs) - - -class ForeignKey(SchemaItem): - """Defines a dependency between two columns. - - ``ForeignKey`` is specified as an argument to a :class:`.Column` object, - e.g.:: - - t = Table("remote_table", metadata, - Column("remote_id", ForeignKey("main_table.id")) - ) - - Note that ``ForeignKey`` is only a marker object that defines - a dependency between two columns. The actual constraint - is in all cases represented by the :class:`.ForeignKeyConstraint` - object. This object will be generated automatically when - a ``ForeignKey`` is associated with a :class:`.Column` which - in turn is associated with a :class:`.Table`. Conversely, - when :class:`.ForeignKeyConstraint` is applied to a :class:`.Table`, - ``ForeignKey`` markers are automatically generated to be - present on each associated :class:`.Column`, which are also - associated with the constraint object. - - Note that you cannot define a "composite" foreign key constraint, - that is a constraint between a grouping of multiple parent/child - columns, using ``ForeignKey`` objects. To define this grouping, - the :class:`.ForeignKeyConstraint` object must be used, and applied - to the :class:`.Table`. The associated ``ForeignKey`` objects - are created automatically. - - The ``ForeignKey`` objects associated with an individual - :class:`.Column` object are available in the `foreign_keys` collection - of that column. - - Further examples of foreign key configuration are in - :ref:`metadata_foreignkeys`. - - """ - - __visit_name__ = 'foreign_key' - - def __init__(self, column, _constraint=None, use_alter=False, name=None, - onupdate=None, ondelete=None, deferrable=None, - schema=None, - initially=None, link_to_name=False): - """ - Construct a column-level FOREIGN KEY. - - The :class:`.ForeignKey` object when constructed generates a - :class:`.ForeignKeyConstraint` which is associated with the parent - :class:`.Table` object's collection of constraints. - - :param column: A single target column for the key relationship. A - :class:`.Column` object or a column name as a string: - ``tablename.columnkey`` or ``schema.tablename.columnkey``. - ``columnkey`` is the ``key`` which has been assigned to the column - (defaults to the column name itself), unless ``link_to_name`` is - ``True`` in which case the rendered name of the column is used. - - .. versionadded:: 0.7.4 - Note that if the schema name is not included, and the underlying - :class:`.MetaData` has a "schema", that value will be used. - - :param name: Optional string. An in-database name for the key if - `constraint` is not provided. - - :param onupdate: Optional string. If set, emit ON UPDATE when - issuing DDL for this constraint. Typical values include CASCADE, - DELETE and RESTRICT. - - :param ondelete: Optional string. If set, emit ON DELETE when - issuing DDL for this constraint. Typical values include CASCADE, - DELETE and RESTRICT. - - :param deferrable: Optional bool. If set, emit DEFERRABLE or NOT - DEFERRABLE when issuing DDL for this constraint. - - :param initially: Optional string. If set, emit INITIALLY when - issuing DDL for this constraint. - - :param link_to_name: if True, the string name given in ``column`` is - the rendered name of the referenced column, not its locally - assigned ``key``. - - :param use_alter: passed to the underlying - :class:`.ForeignKeyConstraint` to indicate the constraint should be - generated/dropped externally from the CREATE TABLE/ DROP TABLE - statement. See that classes' constructor for details. - - """ - - self._colspec = column - - # the linked ForeignKeyConstraint. - # ForeignKey will create this when parent Column - # is attached to a Table, *or* ForeignKeyConstraint - # object passes itself in when creating ForeignKey - # markers. - self.constraint = _constraint - - - self.use_alter = use_alter - self.name = name - self.onupdate = onupdate - self.ondelete = ondelete - self.deferrable = deferrable - self.initially = initially - self.link_to_name = link_to_name - - def __repr__(self): - return "ForeignKey(%r)" % self._get_colspec() - - def copy(self, schema=None): - """Produce a copy of this :class:`.ForeignKey` object. - - The new :class:`.ForeignKey` will not be bound - to any :class:`.Column`. - - This method is usually used by the internal - copy procedures of :class:`.Column`, :class:`.Table`, - and :class:`.MetaData`. - - :param schema: The returned :class:`.ForeignKey` will - reference the original table and column name, qualified - by the given string schema name. - - """ - - fk = ForeignKey( - self._get_colspec(schema=schema), - use_alter=self.use_alter, - name=self.name, - onupdate=self.onupdate, - ondelete=self.ondelete, - deferrable=self.deferrable, - initially=self.initially, - link_to_name=self.link_to_name - ) - fk.dispatch._update(self.dispatch) - return fk - - def _get_colspec(self, schema=None): - """Return a string based 'column specification' for this :class:`.ForeignKey`. - - This is usually the equivalent of the string-based "tablename.colname" - argument first passed to the object's constructor. - - """ - if schema: - return schema + "." + self.column.table.name + \ - "." + self.column.key - elif isinstance(self._colspec, basestring): - return self._colspec - elif hasattr(self._colspec, '__clause_element__'): - _column = self._colspec.__clause_element__() - else: - _column = self._colspec - - return "%s.%s" % (_column.table.fullname, _column.key) - - target_fullname = property(_get_colspec) - - def references(self, table): - """Return True if the given :class:`.Table` is referenced by this :class:`.ForeignKey`.""" - - return table.corresponding_column(self.column) is not None - - def get_referent(self, table): - """Return the :class:`.Column` in the given :class:`.Table` - referenced by this :class:`.ForeignKey`. - - Returns None if this :class:`.ForeignKey` does not reference the given - :class:`.Table`. - - """ - - return table.corresponding_column(self.column) - - @util.memoized_property - def column(self): - """Return the target :class:`.Column` referenced by this :class:`.ForeignKey`. - - If this :class:`.ForeignKey` was created using a - string-based target column specification, this - attribute will on first access initiate a resolution - process to locate the referenced remote - :class:`.Column`. The resolution process traverses - to the parent :class:`.Column`, :class:`.Table`, and - :class:`.MetaData` to proceed - if any of these aren't - yet present, an error is raised. - - """ - # ForeignKey inits its remote column as late as possible, so tables - # can be defined without dependencies - if isinstance(self._colspec, basestring): - # locate the parent table this foreign key is attached to. we - # use the "original" column which our parent column represents - # (its a list of columns/other ColumnElements if the parent - # table is a UNION) - for c in self.parent.base_columns: - if isinstance(c, Column): - parenttable = c.table - break - else: - raise exc.ArgumentError( - "Parent column '%s' does not descend from a " - "table-attached Column" % str(self.parent)) - - m = self._colspec.split('.') - - if m is None: - raise exc.ArgumentError( - "Invalid foreign key column specification: %s" % - self._colspec) - - # A FK between column 'bar' and table 'foo' can be - # specified as 'foo', 'foo.bar', 'dbo.foo.bar', - # 'otherdb.dbo.foo.bar'. Once we have the column name and - # the table name, treat everything else as the schema - # name. Some databases (e.g. Sybase) support - # inter-database foreign keys. See tickets#1341 and -- - # indirectly related -- Ticket #594. This assumes that '.' - # will never appear *within* any component of the FK. - - (schema, tname, colname) = (None, None, None) - if schema is None and parenttable.metadata.schema is not None: - schema = parenttable.metadata.schema - - if (len(m) == 1): - tname = m.pop() - else: - colname = m.pop() - tname = m.pop() - - if (len(m) > 0): - schema = '.'.join(m) - - if _get_table_key(tname, schema) not in parenttable.metadata: - raise exc.NoReferencedTableError( - "Foreign key associated with column '%s' could not find " - "table '%s' with which to generate a " - "foreign key to target column '%s'" % (self.parent, tname, colname), - tname) - table = Table(tname, parenttable.metadata, - mustexist=True, schema=schema) - - _column = None - if colname is None: - # colname is None in the case that ForeignKey argument - # was specified as table name only, in which case we - # match the column name to the same column on the - # parent. - key = self.parent - _column = table.c.get(self.parent.key, None) - elif self.link_to_name: - key = colname - for c in table.c: - if c.name == colname: - _column = c - else: - key = colname - _column = table.c.get(colname, None) - - if _column is None: - raise exc.NoReferencedColumnError( - "Could not create ForeignKey '%s' on table '%s': " - "table '%s' has no column named '%s'" % ( - self._colspec, parenttable.name, table.name, key), - table.name, key) - - elif hasattr(self._colspec, '__clause_element__'): - _column = self._colspec.__clause_element__() - else: - _column = self._colspec - - # propagate TypeEngine to parent if it didn't have one - if isinstance(self.parent.type, sqltypes.NullType): - self.parent.type = _column.type - return _column - - def _set_parent(self, column): - if hasattr(self, 'parent'): - if self.parent is column: - return - raise exc.InvalidRequestError( - "This ForeignKey already has a parent !") - self.parent = column - self.parent.foreign_keys.add(self) - self.parent._on_table_attach(self._set_table) - - def _set_table(self, column, table): - # standalone ForeignKey - create ForeignKeyConstraint - # on the hosting Table when attached to the Table. - if self.constraint is None and isinstance(table, Table): - self.constraint = ForeignKeyConstraint( - [], [], use_alter=self.use_alter, name=self.name, - onupdate=self.onupdate, ondelete=self.ondelete, - deferrable=self.deferrable, initially=self.initially, - ) - self.constraint._elements[self.parent] = self - self.constraint._set_parent_with_dispatch(table) - table.foreign_keys.add(self) - -class _NotAColumnExpr(object): - def _not_a_column_expr(self): - raise exc.InvalidRequestError( - "This %s cannot be used directly " - "as a column expression." % self.__class__.__name__) - - __clause_element__ = self_group = lambda self: self._not_a_column_expr() - _from_objects = property(lambda self: self._not_a_column_expr()) - -class DefaultGenerator(_NotAColumnExpr, SchemaItem): - """Base class for column *default* values.""" - - __visit_name__ = 'default_generator' - - is_sequence = False - is_server_default = False - column = None - - def __init__(self, for_update=False): - self.for_update = for_update - - def _set_parent(self, column): - self.column = column - if self.for_update: - self.column.onupdate = self - else: - self.column.default = self - - def execute(self, bind=None, **kwargs): - if bind is None: - bind = _bind_or_error(self) - return bind._execute_default(self, **kwargs) - - @property - def bind(self): - """Return the connectable associated with this default.""" - if getattr(self, 'column', None) is not None: - return self.column.table.bind - else: - return None - - -class ColumnDefault(DefaultGenerator): - """A plain default value on a column. - - This could correspond to a constant, a callable function, - or a SQL clause. - - :class:`.ColumnDefault` is generated automatically - whenever the ``default``, ``onupdate`` arguments of - :class:`.Column` are used. A :class:`.ColumnDefault` - can be passed positionally as well. - - For example, the following:: - - Column('foo', Integer, default=50) - - Is equivalent to:: - - Column('foo', Integer, ColumnDefault(50)) - - - """ - - def __init__(self, arg, **kwargs): - super(ColumnDefault, self).__init__(**kwargs) - if isinstance(arg, FetchedValue): - raise exc.ArgumentError( - "ColumnDefault may not be a server-side default type.") - if util.callable(arg): - arg = self._maybe_wrap_callable(arg) - self.arg = arg - - @util.memoized_property - def is_callable(self): - return util.callable(self.arg) - - @util.memoized_property - def is_clause_element(self): - return isinstance(self.arg, expression.ClauseElement) - - @util.memoized_property - def is_scalar(self): - return not self.is_callable and \ - not self.is_clause_element and \ - not self.is_sequence - - def _maybe_wrap_callable(self, fn): - """Backward compat: Wrap callables that don't accept a context.""" - - if inspect.isfunction(fn): - inspectable = fn - elif inspect.isclass(fn): - inspectable = fn.__init__ - elif hasattr(fn, '__call__'): - inspectable = fn.__call__ - else: - # probably not inspectable, try anyways. - inspectable = fn - try: - argspec = inspect.getargspec(inspectable) - except TypeError: - return lambda ctx: fn() - - positionals = len(argspec[0]) - - # Py3K compat - no unbound methods - if inspect.ismethod(inspectable) or inspect.isclass(fn): - positionals -= 1 - - if positionals == 0: - return lambda ctx: fn() - - defaulted = argspec[3] is not None and len(argspec[3]) or 0 - if positionals - defaulted > 1: - raise exc.ArgumentError( - "ColumnDefault Python function takes zero or one " - "positional arguments") - return fn - - def _visit_name(self): - if self.for_update: - return "column_onupdate" - else: - return "column_default" - __visit_name__ = property(_visit_name) - - def __repr__(self): - return "ColumnDefault(%r)" % self.arg - -class Sequence(DefaultGenerator): - """Represents a named database sequence. - - The :class:`.Sequence` object represents the name and configurational - parameters of a database sequence. It also represents - a construct that can be "executed" by a SQLAlchemy :class:`.Engine` - or :class:`.Connection`, rendering the appropriate "next value" function - for the target database and returning a result. - - The :class:`.Sequence` is typically associated with a primary key column:: - - some_table = Table('some_table', metadata, - Column('id', Integer, Sequence('some_table_seq'), primary_key=True) - ) - - When CREATE TABLE is emitted for the above :class:`.Table`, if the - target platform supports sequences, a CREATE SEQUENCE statement will - be emitted as well. For platforms that don't support sequences, - the :class:`.Sequence` construct is ignored. - - See also: :class:`.CreateSequence` :class:`.DropSequence` - - """ - - __visit_name__ = 'sequence' - - is_sequence = True - - def __init__(self, name, start=None, increment=None, schema=None, - optional=False, quote=None, metadata=None, - quote_schema=None, - for_update=False): - """Construct a :class:`.Sequence` object. - - :param name: The name of the sequence. - :param start: the starting index of the sequence. This value is - used when the CREATE SEQUENCE command is emitted to the database - as the value of the "START WITH" clause. If ``None``, the - clause is omitted, which on most platforms indicates a starting - value of 1. - :param increment: the increment value of the sequence. This - value is used when the CREATE SEQUENCE command is emitted to - the database as the value of the "INCREMENT BY" clause. If ``None``, - the clause is omitted, which on most platforms indicates an - increment of 1. - :param schema: Optional schema name for the sequence, if located - in a schema other than the default. - :param optional: boolean value, when ``True``, indicates that this - :class:`.Sequence` object only needs to be explicitly generated - on backends that don't provide another way to generate primary - key identifiers. Currently, it essentially means, "don't create - this sequence on the Postgresql backend, where the SERIAL keyword - creates a sequence for us automatically". - :param quote: boolean value, when ``True`` or ``False``, explicitly - forces quoting of the schema name on or off. When left at its - default of ``None``, normal quoting rules based on casing and reserved - words take place. - :param metadata: optional :class:`.MetaData` object which will be - associated with this :class:`.Sequence`. A :class:`.Sequence` - that is associated with a :class:`.MetaData` gains access to the - ``bind`` of that :class:`.MetaData`, meaning the :meth:`.Sequence.create` - and :meth:`.Sequence.drop` methods will make usage of that engine - automatically. - - .. versionchanged:: 0.7 - Additionally, the appropriate CREATE SEQUENCE/ - DROP SEQUENCE DDL commands will be emitted corresponding to this - :class:`.Sequence` when :meth:`.MetaData.create_all` and - :meth:`.MetaData.drop_all` are invoked. - - Note that when a :class:`.Sequence` is applied to a :class:`.Column`, - the :class:`.Sequence` is automatically associated with the - :class:`.MetaData` object of that column's parent :class:`.Table`, - when that association is made. The :class:`.Sequence` will then - be subject to automatic CREATE SEQUENCE/DROP SEQUENCE corresponding - to when the :class:`.Table` object itself is created or dropped, - rather than that of the :class:`.MetaData` object overall. - :param for_update: Indicates this :class:`.Sequence`, when associated - with a :class:`.Column`, should be invoked for UPDATE statements - on that column's table, rather than for INSERT statements, when - no value is otherwise present for that column in the statement. - - """ - super(Sequence, self).__init__(for_update=for_update) - self.name = name - self.start = start - self.increment = increment - self.optional = optional - self.quote = quote - if metadata is not None and schema is None and metadata.schema: - self.schema = schema = metadata.schema - self.quote_schema = metadata.quote_schema - else: - self.schema = schema - self.quote_schema = quote_schema - self.metadata = metadata - self._key = _get_table_key(name, schema) - if metadata: - self._set_metadata(metadata) - - @util.memoized_property - def is_callable(self): - return False - - @util.memoized_property - def is_clause_element(self): - return False - - def next_value(self): - """Return a :class:`.next_value` function element - which will render the appropriate increment function - for this :class:`.Sequence` within any SQL expression. - - """ - return expression.func.next_value(self, bind=self.bind) - - def _set_parent(self, column): - super(Sequence, self)._set_parent(column) - column._on_table_attach(self._set_table) - - def _set_table(self, column, table): - self._set_metadata(table.metadata) - - def _set_metadata(self, metadata): - self.metadata = metadata - self.metadata._sequences[self._key] = self - - @property - def bind(self): - if self.metadata: - return self.metadata.bind - else: - return None - - def create(self, bind=None, checkfirst=True): - """Creates this sequence in the database.""" - - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaGenerator, - self, - checkfirst=checkfirst) - - def drop(self, bind=None, checkfirst=True): - """Drops this sequence from the database.""" - - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaDropper, - self, - checkfirst=checkfirst) - - def _not_a_column_expr(self): - raise exc.InvalidRequestError( - "This %s cannot be used directly " - "as a column expression. Use func.next_value(sequence) " - "to produce a 'next value' function that's usable " - "as a column element." - % self.__class__.__name__) - - -class FetchedValue(_NotAColumnExpr, events.SchemaEventTarget): - """A marker for a transparent database-side default. - - Use :class:`.FetchedValue` when the database is configured - to provide some automatic default for a column. - - E.g.:: - - Column('foo', Integer, FetchedValue()) - - Would indicate that some trigger or default generator - will create a new value for the ``foo`` column during an - INSERT. - - """ - is_server_default = True - reflected = False - has_argument = False - - def __init__(self, for_update=False): - self.for_update = for_update - - def _as_for_update(self, for_update): - if for_update == self.for_update: - return self - else: - return self._clone(for_update) - - def _clone(self, for_update): - n = self.__class__.__new__(self.__class__) - n.__dict__.update(self.__dict__) - n.__dict__.pop('column', None) - n.for_update = for_update - return n - - def _set_parent(self, column): - self.column = column - if self.for_update: - self.column.server_onupdate = self - else: - self.column.server_default = self - - def __repr__(self): - return util.generic_repr(self) - -class DefaultClause(FetchedValue): - """A DDL-specified DEFAULT column value. - - :class:`.DefaultClause` is a :class:`.FetchedValue` - that also generates a "DEFAULT" clause when - "CREATE TABLE" is emitted. - - :class:`.DefaultClause` is generated automatically - whenever the ``server_default``, ``server_onupdate`` arguments of - :class:`.Column` are used. A :class:`.DefaultClause` - can be passed positionally as well. - - For example, the following:: - - Column('foo', Integer, server_default="50") - - Is equivalent to:: - - Column('foo', Integer, DefaultClause("50")) - - """ - - has_argument = True - - def __init__(self, arg, for_update=False, _reflected=False): - util.assert_arg_type(arg, (basestring, - expression.ClauseElement, - expression._TextClause), 'arg') - super(DefaultClause, self).__init__(for_update) - self.arg = arg - self.reflected = _reflected - - def __repr__(self): - return "DefaultClause(%r, for_update=%r)" % \ - (self.arg, self.for_update) - -class PassiveDefault(DefaultClause): - """A DDL-specified DEFAULT column value. - - .. deprecated:: 0.6 - :class:`.PassiveDefault` is deprecated. - Use :class:`.DefaultClause`. - """ - @util.deprecated("0.6", - ":class:`.PassiveDefault` is deprecated. " - "Use :class:`.DefaultClause`.", - False) - def __init__(self, *arg, **kw): - DefaultClause.__init__(self, *arg, **kw) - -class Constraint(SchemaItem): - """A table-level SQL constraint.""" - - __visit_name__ = 'constraint' - - def __init__(self, name=None, deferrable=None, initially=None, - _create_rule=None, - **kw): - """Create a SQL constraint. - - :param name: - Optional, the in-database name of this ``Constraint``. - - :param deferrable: - Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when - issuing DDL for this constraint. - - :param initially: - Optional string. If set, emit INITIALLY when issuing DDL - for this constraint. - - :param _create_rule: - a callable which is passed the DDLCompiler object during - compilation. Returns True or False to signal inline generation of - this Constraint. - - The AddConstraint and DropConstraint DDL constructs provide - DDLElement's more comprehensive "conditional DDL" approach that is - passed a database connection when DDL is being issued. _create_rule - is instead called during any CREATE TABLE compilation, where there - may not be any transaction/connection in progress. However, it - allows conditional compilation of the constraint even for backends - which do not support addition of constraints through ALTER TABLE, - which currently includes SQLite. - - _create_rule is used by some types to create constraints. - Currently, its call signature is subject to change at any time. - - :param \**kwargs: - Dialect-specific keyword parameters, see the documentation - for various dialects and constraints regarding options here. - - """ - - self.name = name - self.deferrable = deferrable - self.initially = initially - self._create_rule = _create_rule - util.set_creation_order(self) - _validate_dialect_kwargs(kw, self.__class__.__name__) - self.kwargs = kw - - @property - def table(self): - try: - if isinstance(self.parent, Table): - return self.parent - except AttributeError: - pass - raise exc.InvalidRequestError( - "This constraint is not bound to a table. Did you " - "mean to call table.add_constraint(constraint) ?") - - def _set_parent(self, parent): - self.parent = parent - parent.constraints.add(self) - - def copy(self, **kw): - raise NotImplementedError() - -class ColumnCollectionMixin(object): - def __init__(self, *columns): - self.columns = expression.ColumnCollection() - self._pending_colargs = [_to_schema_column_or_string(c) - for c in columns] - if self._pending_colargs and \ - isinstance(self._pending_colargs[0], Column) and \ - self._pending_colargs[0].table is not None: - self._set_parent_with_dispatch(self._pending_colargs[0].table) - - def _set_parent(self, table): - for col in self._pending_colargs: - if isinstance(col, basestring): - col = table.c[col] - self.columns.add(col) - -class ColumnCollectionConstraint(ColumnCollectionMixin, Constraint): - """A constraint that proxies a ColumnCollection.""" - - def __init__(self, *columns, **kw): - """ - :param \*columns: - A sequence of column names or Column objects. - - :param name: - Optional, the in-database name of this constraint. - - :param deferrable: - Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when - issuing DDL for this constraint. - - :param initially: - Optional string. If set, emit INITIALLY when issuing DDL - for this constraint. - - """ - ColumnCollectionMixin.__init__(self, *columns) - Constraint.__init__(self, **kw) - - def _set_parent(self, table): - ColumnCollectionMixin._set_parent(self, table) - Constraint._set_parent(self, table) - - def __contains__(self, x): - return x in self.columns - - def copy(self, **kw): - c = self.__class__(name=self.name, deferrable=self.deferrable, - initially=self.initially, *self.columns.keys()) - c.dispatch._update(self.dispatch) - return c - - def contains_column(self, col): - return self.columns.contains_column(col) - - def __iter__(self): - # inlining of - # return iter(self.columns) - # ColumnCollection->OrderedProperties->OrderedDict - ordered_dict = self.columns._data - return (ordered_dict[key] for key in ordered_dict._list) - - def __len__(self): - return len(self.columns._data) - - -class CheckConstraint(Constraint): - """A table- or column-level CHECK constraint. - - Can be included in the definition of a Table or Column. - """ - - def __init__(self, sqltext, name=None, deferrable=None, - initially=None, table=None, _create_rule=None): - """Construct a CHECK constraint. - - :param sqltext: - A string containing the constraint definition, which will be used - verbatim, or a SQL expression construct. - - :param name: - Optional, the in-database name of the constraint. - - :param deferrable: - Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when - issuing DDL for this constraint. - - :param initially: - Optional string. If set, emit INITIALLY when issuing DDL - for this constraint. - - """ - - super(CheckConstraint, self).\ - __init__(name, deferrable, initially, _create_rule) - self.sqltext = expression._literal_as_text(sqltext) - if table is not None: - self._set_parent_with_dispatch(table) - - def __visit_name__(self): - if isinstance(self.parent, Table): - return "check_constraint" - else: - return "column_check_constraint" - __visit_name__ = property(__visit_name__) - - def copy(self, **kw): - c = CheckConstraint(self.sqltext, - name=self.name, - initially=self.initially, - deferrable=self.deferrable, - _create_rule=self._create_rule) - c.dispatch._update(self.dispatch) - return c - -class ForeignKeyConstraint(Constraint): - """A table-level FOREIGN KEY constraint. - - Defines a single column or composite FOREIGN KEY ... REFERENCES - constraint. For a no-frills, single column foreign key, adding a - :class:`.ForeignKey` to the definition of a :class:`.Column` is a shorthand - equivalent for an unnamed, single column :class:`.ForeignKeyConstraint`. - - Examples of foreign key configuration are in :ref:`metadata_foreignkeys`. - - """ - __visit_name__ = 'foreign_key_constraint' - - def __init__(self, columns, refcolumns, name=None, onupdate=None, - ondelete=None, deferrable=None, initially=None, use_alter=False, - link_to_name=False, table=None): - """Construct a composite-capable FOREIGN KEY. - - :param columns: A sequence of local column names. The named columns - must be defined and present in the parent Table. The names should - match the ``key`` given to each column (defaults to the name) unless - ``link_to_name`` is True. - - :param refcolumns: A sequence of foreign column names or Column - objects. The columns must all be located within the same Table. - - :param name: Optional, the in-database name of the key. - - :param onupdate: Optional string. If set, emit ON UPDATE when - issuing DDL for this constraint. Typical values include CASCADE, - DELETE and RESTRICT. - - :param ondelete: Optional string. If set, emit ON DELETE when - issuing DDL for this constraint. Typical values include CASCADE, - DELETE and RESTRICT. - - :param deferrable: Optional bool. If set, emit DEFERRABLE or NOT - DEFERRABLE when issuing DDL for this constraint. - - :param initially: Optional string. If set, emit INITIALLY when - issuing DDL for this constraint. - - :param link_to_name: if True, the string name given in ``column`` is - the rendered name of the referenced column, not its locally assigned - ``key``. - - :param use_alter: If True, do not emit the DDL for this constraint as - part of the CREATE TABLE definition. Instead, generate it via an - ALTER TABLE statement issued after the full collection of tables - have been created, and drop it via an ALTER TABLE statement before - the full collection of tables are dropped. This is shorthand for the - usage of :class:`.AddConstraint` and :class:`.DropConstraint` applied - as "after-create" and "before-drop" events on the MetaData object. - This is normally used to generate/drop constraints on objects that - are mutually dependent on each other. - - """ - super(ForeignKeyConstraint, self).\ - __init__(name, deferrable, initially) - - self.onupdate = onupdate - self.ondelete = ondelete - self.link_to_name = link_to_name - if self.name is None and use_alter: - raise exc.ArgumentError("Alterable Constraint requires a name") - self.use_alter = use_alter - - self._elements = util.OrderedDict() - - # standalone ForeignKeyConstraint - create - # associated ForeignKey objects which will be applied to hosted - # Column objects (in col.foreign_keys), either now or when attached - # to the Table for string-specified names - for col, refcol in zip(columns, refcolumns): - self._elements[col] = ForeignKey( - refcol, - _constraint=self, - name=self.name, - onupdate=self.onupdate, - ondelete=self.ondelete, - use_alter=self.use_alter, - link_to_name=self.link_to_name - ) - - if table is not None: - self._set_parent_with_dispatch(table) - - @property - def columns(self): - return self._elements.keys() - - @property - def elements(self): - return self._elements.values() - - def _set_parent(self, table): - super(ForeignKeyConstraint, self)._set_parent(table) - for col, fk in self._elements.iteritems(): - # string-specified column names now get - # resolved to Column objects - if isinstance(col, basestring): - try: - col = table.c[col] - except KeyError: - raise exc.ArgumentError( - "Can't create ForeignKeyConstraint " - "on table '%s': no column " - "named '%s' is present." % (table.description, col)) - - if not hasattr(fk, 'parent') or \ - fk.parent is not col: - fk._set_parent_with_dispatch(col) - - if self.use_alter: - def supports_alter(ddl, event, schema_item, bind, **kw): - return table in set(kw['tables']) and \ - bind.dialect.supports_alter - - event.listen(table.metadata, "after_create", AddConstraint(self, on=supports_alter)) - event.listen(table.metadata, "before_drop", DropConstraint(self, on=supports_alter)) - - - def copy(self, **kw): - fkc = ForeignKeyConstraint( - [x.parent.key for x in self._elements.values()], - [x._get_colspec(**kw) for x in self._elements.values()], - name=self.name, - onupdate=self.onupdate, - ondelete=self.ondelete, - use_alter=self.use_alter, - deferrable=self.deferrable, - initially=self.initially, - link_to_name=self.link_to_name - ) - fkc.dispatch._update(self.dispatch) - return fkc - -class PrimaryKeyConstraint(ColumnCollectionConstraint): - """A table-level PRIMARY KEY constraint. - - Defines a single column or composite PRIMARY KEY constraint. For a - no-frills primary key, adding ``primary_key=True`` to one or more - ``Column`` definitions is a shorthand equivalent for an unnamed single- or - multiple-column PrimaryKeyConstraint. - """ - - __visit_name__ = 'primary_key_constraint' - - def _set_parent(self, table): - super(PrimaryKeyConstraint, self)._set_parent(table) - - if table.primary_key in table.constraints: - table.constraints.remove(table.primary_key) - table.primary_key = self - table.constraints.add(self) - - for c in self.columns: - c.primary_key = True - - def _replace(self, col): - self.columns.replace(col) - -class UniqueConstraint(ColumnCollectionConstraint): - """A table-level UNIQUE constraint. - - Defines a single column or composite UNIQUE constraint. For a no-frills, - single column constraint, adding ``unique=True`` to the ``Column`` - definition is a shorthand equivalent for an unnamed, single column - UniqueConstraint. - """ - - __visit_name__ = 'unique_constraint' - -class Index(ColumnCollectionMixin, SchemaItem): - """A table-level INDEX. - - Defines a composite (one or more column) INDEX. For a no-frills, single - column index, adding ``index=True`` to the ``Column`` definition is - a shorthand equivalent for an unnamed, single column :class:`.Index`. - - See also: - - :ref:`schema_indexes` - General information on :class:`.Index`. - - :ref:`postgresql_indexes` - PostgreSQL-specific options available for the :class:`.Index` construct. - - :ref:`mysql_indexes` - MySQL-specific options available for the :class:`.Index` construct. - """ - - __visit_name__ = 'index' - - def __init__(self, name, *columns, **kw): - """Construct an index object. - - :param name: - The name of the index - - :param \*columns: - Columns to include in the index. All columns must belong to the same - table. - - :param unique: - Defaults to False: create a unique index. - - :param \**kw: - Other keyword arguments may be interpreted by specific dialects. - - """ - self.table = None - # will call _set_parent() if table-bound column - # objects are present - ColumnCollectionMixin.__init__(self, *columns) - self.name = name - self.unique = kw.pop('unique', False) - self.kwargs = kw - - def _set_parent(self, table): - ColumnCollectionMixin._set_parent(self, table) - - if self.table is not None and table is not self.table: - raise exc.ArgumentError( - "Index '%s' is against table '%s', and " - "cannot be associated with table '%s'." % ( - self.name, - self.table.description, - table.description - ) - ) - self.table = table - for c in self.columns: - if c.table != self.table: - raise exc.ArgumentError( - "Column '%s' is not part of table '%s'." % - (c, self.table.description) - ) - table.indexes.add(self) - - @property - def bind(self): - """Return the connectable associated with this Index.""" - - return self.table.bind - - def create(self, bind=None): - """Issue a ``CREATE`` statement for this - :class:`.Index`, using the given :class:`.Connectable` - for connectivity. - - See also :meth:`.MetaData.create_all`. - - """ - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaGenerator, self) - return self - - def drop(self, bind=None): - """Issue a ``DROP`` statement for this - :class:`.Index`, using the given :class:`.Connectable` - for connectivity. - - See also :meth:`.MetaData.drop_all`. - - """ - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaDropper, self) - - def __repr__(self): - return 'Index(%s)' % ( - ", ".join( - [repr(self.name)] + - [repr(c) for c in self.columns] + - (self.unique and ["unique=True"] or []) - )) - -class MetaData(SchemaItem): - """A collection of :class:`.Table` objects and their associated schema constructs. - - Holds a collection of :class:`.Table` objects as well as - an optional binding to an :class:`.Engine` or - :class:`.Connection`. If bound, the :class:`.Table` objects - in the collection and their columns may participate in implicit SQL - execution. - - The :class:`.Table` objects themselves are stored in the ``metadata.tables`` - dictionary. - - The ``bind`` property may be assigned to dynamically. A common pattern is - to start unbound and then bind later when an engine is available:: - - metadata = MetaData() - # define tables - Table('mytable', metadata, ...) - # connect to an engine later, perhaps after loading a URL from a - # configuration file - metadata.bind = an_engine - - MetaData is a thread-safe object after tables have been explicitly defined - or loaded via reflection. - - See also: - - :ref:`metadata_describing` - Introduction to database metadata - - .. index:: - single: thread safety; MetaData - - """ - - __visit_name__ = 'metadata' - - def __init__(self, bind=None, reflect=False, schema=None, quote_schema=None): - """Create a new MetaData object. - - :param bind: - An Engine or Connection to bind to. May also be a string or URL - instance, these are passed to create_engine() and this MetaData will - be bound to the resulting engine. - - :param reflect: - Optional, automatically load all tables from the bound database. - Defaults to False. ``bind`` is required when this option is set. - For finer control over loaded tables, use the ``reflect`` method of - ``MetaData``. - - :param schema: - The default schema to use for the :class:`.Table`, :class:`.Sequence`, and other - objects associated with this :class:`.MetaData`. - Defaults to ``None``. - - :param quote_schema: - Sets the ``quote_schema`` flag for those :class:`.Table`, :class:`.Sequence`, - and other objects which make usage of the local ``schema`` name. - - .. versionadded:: 0.7.4 - ``schema`` and ``quote_schema`` parameters. - - """ - self.tables = util.immutabledict() - self.schema = schema - self.quote_schema = quote_schema - self._schemas = set() - self._sequences = {} - self.bind = bind - if reflect: - if not bind: - raise exc.ArgumentError( - "A bind must be supplied in conjunction " - "with reflect=True") - self.reflect() - - def __repr__(self): - return 'MetaData(bind=%r)' % self.bind - - def __contains__(self, table_or_key): - if not isinstance(table_or_key, basestring): - table_or_key = table_or_key.key - return table_or_key in self.tables - - def _add_table(self, name, schema, table): - key = _get_table_key(name, schema) - dict.__setitem__(self.tables, key, table) - if schema: - self._schemas.add(schema) - - def _remove_table(self, name, schema): - key = _get_table_key(name, schema) - dict.pop(self.tables, key, None) - if self._schemas: - self._schemas = set([t.schema - for t in self.tables.values() - if t.schema is not None]) - - def __getstate__(self): - return {'tables': self.tables, 'schema':self.schema, - 'quote_schema':self.quote_schema, - 'schemas':self._schemas, - 'sequences':self._sequences} - - def __setstate__(self, state): - self.tables = state['tables'] - self.schema = state['schema'] - self.quote_schema = state['quote_schema'] - self._bind = None - self._sequences = state['sequences'] - self._schemas = state['schemas'] - - def is_bound(self): - """True if this MetaData is bound to an Engine or Connection.""" - - return self._bind is not None - - def bind(self): - """An :class:`.Engine` or :class:`.Connection` to which this - :class:`.MetaData` is bound. - - Typically, a :class:`.Engine` is assigned to this attribute - so that "implicit execution" may be used, or alternatively - as a means of providing engine binding information to an - ORM :class:`.Session` object:: - - engine = create_engine("someurl://") - metadata.bind = engine - - .. seealso:: - - :ref:`dbengine_implicit` - background on "bound metadata" - - """ - return self._bind - - def _bind_to(self, bind): - """Bind this MetaData to an Engine, Connection, string or URL.""" - - if isinstance(bind, (basestring, url.URL)): - from sqlalchemy import create_engine - self._bind = create_engine(bind) - else: - self._bind = bind - bind = property(bind, _bind_to) - - def clear(self): - """Clear all Table objects from this MetaData.""" - - dict.clear(self.tables) - self._schemas.clear() - - def remove(self, table): - """Remove the given Table object from this MetaData.""" - - self._remove_table(table.name, table.schema) - - @property - def sorted_tables(self): - """Returns a list of ``Table`` objects sorted in order of - dependency. - """ - return sqlutil.sort_tables(self.tables.itervalues()) - - def reflect(self, bind=None, schema=None, views=False, only=None): - """Load all available table definitions from the database. - - Automatically creates ``Table`` entries in this ``MetaData`` for any - table available in the database but not yet present in the - ``MetaData``. May be called multiple times to pick up tables recently - added to the database, however no special action is taken if a table - in this ``MetaData`` no longer exists in the database. - - :param bind: - A :class:`~sqlalchemy.engine.base.Connectable` used to access the - database; if None, uses the existing bind on this ``MetaData``, if - any. - - :param schema: - Optional, query and reflect tables from an alterate schema. - If None, the schema associated with this :class:`.MetaData` - is used, if any. - - :param views: - If True, also reflect views. - - :param only: - Optional. Load only a sub-set of available named tables. May be - specified as a sequence of names or a callable. - - If a sequence of names is provided, only those tables will be - reflected. An error is raised if a table is requested but not - available. Named tables already present in this ``MetaData`` are - ignored. - - If a callable is provided, it will be used as a boolean predicate to - filter the list of potential table names. The callable is called - with a table name and this ``MetaData`` instance as positional - arguments and should return a true value for any table to reflect. - - """ - if bind is None: - bind = _bind_or_error(self) - - if bind.engine is not bind: - conn = bind - close = False - else: - conn = bind.contextual_connect() - close = True - - reflect_opts = { - 'autoload': True, - 'autoload_with': bind - } - - if schema is None: - schema = self.schema - - if schema is not None: - reflect_opts['schema'] = schema - - try: - available = util.OrderedSet(bind.engine.table_names(schema, - connection=conn)) - if views: - available.update( - bind.dialect.get_view_names(conn, schema) - ) - - current = set(self.tables.iterkeys()) - - if only is None: - load = [name for name in available if name not in current] - elif util.callable(only): - load = [name for name in available - if name not in current and only(name, self)] - else: - missing = [name for name in only if name not in available] - if missing: - s = schema and (" schema '%s'" % schema) or '' - raise exc.InvalidRequestError( - 'Could not reflect: requested table(s) not available ' - 'in %s%s: (%s)' % - (bind.engine.url, s, ', '.join(missing))) - load = [name for name in only if name not in current] - - for name in load: - Table(name, self, **reflect_opts) - finally: - if close: - conn.close() - - def append_ddl_listener(self, event_name, listener): - """Append a DDL event listener to this ``MetaData``. - - Deprecated. See :class:`.DDLEvents`. - - """ - def adapt_listener(target, connection, **kw): - tables = kw['tables'] - listener(event, target, connection, tables=tables) - - event.listen(self, "" + event_name.replace('-', '_'), adapt_listener) - - def create_all(self, bind=None, tables=None, checkfirst=True): - """Create all tables stored in this metadata. - - Conditional by default, will not attempt to recreate tables already - present in the target database. - - :param bind: - A :class:`~sqlalchemy.engine.base.Connectable` used to access the - database; if None, uses the existing bind on this ``MetaData``, if - any. - - :param tables: - Optional list of ``Table`` objects, which is a subset of the total - tables in the ``MetaData`` (others are ignored). - - :param checkfirst: - Defaults to True, don't issue CREATEs for tables already present - in the target database. - - """ - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaGenerator, - self, - checkfirst=checkfirst, - tables=tables) - - def drop_all(self, bind=None, tables=None, checkfirst=True): - """Drop all tables stored in this metadata. - - Conditional by default, will not attempt to drop tables not present in - the target database. - - :param bind: - A :class:`~sqlalchemy.engine.base.Connectable` used to access the - database; if None, uses the existing bind on this ``MetaData``, if - any. - - :param tables: - Optional list of ``Table`` objects, which is a subset of the - total tables in the ``MetaData`` (others are ignored). - - :param checkfirst: - Defaults to True, only issue DROPs for tables confirmed to be - present in the target database. - - """ - if bind is None: - bind = _bind_or_error(self) - bind._run_visitor(ddl.SchemaDropper, - self, - checkfirst=checkfirst, - tables=tables) - -class ThreadLocalMetaData(MetaData): - """A MetaData variant that presents a different ``bind`` in every thread. - - Makes the ``bind`` property of the MetaData a thread-local value, allowing - this collection of tables to be bound to different ``Engine`` - implementations or connections in each thread. - - The ThreadLocalMetaData starts off bound to None in each thread. Binds - must be made explicitly by assigning to the ``bind`` property or using - ``connect()``. You can also re-bind dynamically multiple times per - thread, just like a regular ``MetaData``. - - """ - - __visit_name__ = 'metadata' - - def __init__(self): - """Construct a ThreadLocalMetaData.""" - - self.context = util.threading.local() - self.__engines = {} - super(ThreadLocalMetaData, self).__init__() - - def bind(self): - """The bound Engine or Connection for this thread. - - This property may be assigned an Engine or Connection, or assigned a - string or URL to automatically create a basic Engine for this bind - with ``create_engine()``.""" - - return getattr(self.context, '_engine', None) - - def _bind_to(self, bind): - """Bind to a Connectable in the caller's thread.""" - - if isinstance(bind, (basestring, url.URL)): - try: - self.context._engine = self.__engines[bind] - except KeyError: - from sqlalchemy import create_engine - e = create_engine(bind) - self.__engines[bind] = e - self.context._engine = e - else: - # TODO: this is squirrely. we shouldnt have to hold onto engines - # in a case like this - if bind not in self.__engines: - self.__engines[bind] = bind - self.context._engine = bind - - bind = property(bind, _bind_to) - - def is_bound(self): - """True if there is a bind for this thread.""" - return (hasattr(self.context, '_engine') and - self.context._engine is not None) - - def dispose(self): - """Dispose all bound engines, in all thread contexts.""" - - for e in self.__engines.itervalues(): - if hasattr(e, 'dispose'): - e.dispose() - -class SchemaVisitor(visitors.ClauseVisitor): - """Define the visiting for ``SchemaItem`` objects.""" - - __traverse_options__ = {'schema_visitor':True} - - -class DDLElement(expression.Executable, expression.ClauseElement): - """Base class for DDL expression constructs. - - This class is the base for the general purpose :class:`.DDL` class, - as well as the various create/drop clause constructs such as - :class:`.CreateTable`, :class:`.DropTable`, :class:`.AddConstraint`, - etc. - - :class:`.DDLElement` integrates closely with SQLAlchemy events, - introduced in :ref:`event_toplevel`. An instance of one is - itself an event receiving callable:: - - event.listen( - users, - 'after_create', - AddConstraint(constraint).execute_if(dialect='postgresql') - ) - - See also: - - :class:`.DDL` - - :class:`.DDLEvents` - - :ref:`event_toplevel` - - :ref:`schema_ddl_sequences` - - """ - - _execution_options = expression.Executable.\ - _execution_options.union({'autocommit':True}) - - target = None - on = None - dialect = None - callable_ = None - - def execute(self, bind=None, target=None): - """Execute this DDL immediately. - - Executes the DDL statement in isolation using the supplied - :class:`~sqlalchemy.engine.base.Connectable` or - :class:`~sqlalchemy.engine.base.Connectable` assigned to the ``.bind`` - property, if not supplied. If the DDL has a conditional ``on`` - criteria, it will be invoked with None as the event. - - :param bind: - Optional, an ``Engine`` or ``Connection``. If not supplied, a valid - :class:`~sqlalchemy.engine.base.Connectable` must be present in the - ``.bind`` property. - - :param target: - Optional, defaults to None. The target SchemaItem for the - execute call. Will be passed to the ``on`` callable if any, - and may also provide string expansion data for the - statement. See ``execute_at`` for more information. - - """ - - if bind is None: - bind = _bind_or_error(self) - - if self._should_execute(target, bind): - return bind.execute(self.against(target)) - else: - bind.engine.logger.info( - "DDL execution skipped, criteria not met.") - - @util.deprecated("0.7", "See :class:`.DDLEvents`, as well as " - ":meth:`.DDLElement.execute_if`.") - def execute_at(self, event_name, target): - """Link execution of this DDL to the DDL lifecycle of a SchemaItem. - - Links this ``DDLElement`` to a ``Table`` or ``MetaData`` instance, - executing it when that schema item is created or dropped. The DDL - statement will be executed using the same Connection and transactional - context as the Table create/drop itself. The ``.bind`` property of - this statement is ignored. - - :param event: - One of the events defined in the schema item's ``.ddl_events``; - e.g. 'before-create', 'after-create', 'before-drop' or 'after-drop' - - :param target: - The Table or MetaData instance for which this DDLElement will - be associated with. - - A DDLElement instance can be linked to any number of schema items. - - ``execute_at`` builds on the ``append_ddl_listener`` interface of - :class:`.MetaData` and :class:`.Table` objects. - - Caveat: Creating or dropping a Table in isolation will also trigger - any DDL set to ``execute_at`` that Table's MetaData. This may change - in a future release. - - """ - - def call_event(target, connection, **kw): - if self._should_execute_deprecated(event_name, - target, connection, **kw): - return connection.execute(self.against(target)) - - event.listen(target, "" + event_name.replace('-', '_'), call_event) - - @expression._generative - def against(self, target): - """Return a copy of this DDL against a specific schema item.""" - - self.target = target - - @expression._generative - def execute_if(self, dialect=None, callable_=None, state=None): - """Return a callable that will execute this - DDLElement conditionally. - - Used to provide a wrapper for event listening:: - - event.listen( - metadata, - 'before_create', - DDL("my_ddl").execute_if(dialect='postgresql') - ) - - :param dialect: May be a string, tuple or a callable - predicate. If a string, it will be compared to the name of the - executing database dialect:: - - DDL('something').execute_if(dialect='postgresql') - - If a tuple, specifies multiple dialect names:: - - DDL('something').execute_if(dialect=('postgresql', 'mysql')) - - :param callable_: A callable, which will be invoked with - four positional arguments as well as optional keyword - arguments: - - :ddl: - This DDL element. - - :target: - The :class:`.Table` or :class:`.MetaData` object which is the target of - this event. May be None if the DDL is executed explicitly. - - :bind: - The :class:`.Connection` being used for DDL execution - - :tables: - Optional keyword argument - a list of Table objects which are to - be created/ dropped within a MetaData.create_all() or drop_all() - method call. - - :state: - Optional keyword argument - will be the ``state`` argument - passed to this function. - - :checkfirst: - Keyword argument, will be True if the 'checkfirst' flag was - set during the call to ``create()``, ``create_all()``, - ``drop()``, ``drop_all()``. - - If the callable returns a true value, the DDL statement will be - executed. - - :param state: any value which will be passed to the callable_ - as the ``state`` keyword argument. - - See also: - - :class:`.DDLEvents` - - :ref:`event_toplevel` - - """ - self.dialect = dialect - self.callable_ = callable_ - self.state = state - - def _should_execute(self, target, bind, **kw): - if self.on is not None and \ - not self._should_execute_deprecated(None, target, bind, **kw): - return False - - if isinstance(self.dialect, basestring): - if self.dialect != bind.engine.name: - return False - elif isinstance(self.dialect, (tuple, list, set)): - if bind.engine.name not in self.dialect: - return False - if self.callable_ is not None and \ - not self.callable_(self, target, bind, state=self.state, **kw): - return False - - return True - - def _should_execute_deprecated(self, event, target, bind, **kw): - if self.on is None: - return True - elif isinstance(self.on, basestring): - return self.on == bind.engine.name - elif isinstance(self.on, (tuple, list, set)): - return bind.engine.name in self.on - else: - return self.on(self, event, target, bind, **kw) - - def __call__(self, target, bind, **kw): - """Execute the DDL as a ddl_listener.""" - - if self._should_execute(target, bind, **kw): - return bind.execute(self.against(target)) - - def _check_ddl_on(self, on): - if (on is not None and - (not isinstance(on, (basestring, tuple, list, set)) and - not util.callable(on))): - raise exc.ArgumentError( - "Expected the name of a database dialect, a tuple " - "of names, or a callable for " - "'on' criteria, got type '%s'." % type(on).__name__) - - def bind(self): - if self._bind: - return self._bind - def _set_bind(self, bind): - self._bind = bind - bind = property(bind, _set_bind) - - def _generate(self): - s = self.__class__.__new__(self.__class__) - s.__dict__ = self.__dict__.copy() - return s - - def _compiler(self, dialect, **kw): - """Return a compiler appropriate for this ClauseElement, given a - Dialect.""" - - return dialect.ddl_compiler(dialect, self, **kw) - -class DDL(DDLElement): - """A literal DDL statement. - - Specifies literal SQL DDL to be executed by the database. DDL objects - function as DDL event listeners, and can be subscribed to those events - listed in :class:`.DDLEvents`, using either :class:`.Table` or :class:`.MetaData` - objects as targets. Basic templating support allows a single DDL instance - to handle repetitive tasks for multiple tables. - - Examples:: - - from sqlalchemy import event, DDL - - tbl = Table('users', metadata, Column('uid', Integer)) - event.listen(tbl, 'before_create', DDL('DROP TRIGGER users_trigger')) - - spow = DDL('ALTER TABLE %(table)s SET secretpowers TRUE') - event.listen(tbl, 'after_create', spow.execute_if(dialect='somedb')) - - drop_spow = DDL('ALTER TABLE users SET secretpowers FALSE') - connection.execute(drop_spow) - - When operating on Table events, the following ``statement`` - string substitions are available:: - - %(table)s - the Table name, with any required quoting applied - %(schema)s - the schema name, with any required quoting applied - %(fullname)s - the Table name including schema, quoted if needed - - The DDL's "context", if any, will be combined with the standard - substutions noted above. Keys present in the context will override - the standard substitutions. - - """ - - __visit_name__ = "ddl" - - def __init__(self, statement, on=None, context=None, bind=None): - """Create a DDL statement. - - :param statement: - A string or unicode string to be executed. Statements will be - processed with Python's string formatting operator. See the - ``context`` argument and the ``execute_at`` method. - - A literal '%' in a statement must be escaped as '%%'. - - SQL bind parameters are not available in DDL statements. - - :param on: - Deprecated. See :meth:`.DDLElement.execute_if`. - - Optional filtering criteria. May be a string, tuple or a callable - predicate. If a string, it will be compared to the name of the - executing database dialect:: - - DDL('something', on='postgresql') - - If a tuple, specifies multiple dialect names:: - - DDL('something', on=('postgresql', 'mysql')) - - If a callable, it will be invoked with four positional arguments - as well as optional keyword arguments: - - :ddl: - This DDL element. - - :event: - The name of the event that has triggered this DDL, such as - 'after-create' Will be None if the DDL is executed explicitly. - - :target: - The ``Table`` or ``MetaData`` object which is the target of - this event. May be None if the DDL is executed explicitly. - - :connection: - The ``Connection`` being used for DDL execution - - :tables: - Optional keyword argument - a list of Table objects which are to - be created/ dropped within a MetaData.create_all() or drop_all() - method call. - - - If the callable returns a true value, the DDL statement will be - executed. - - :param context: - Optional dictionary, defaults to None. These values will be - available for use in string substitutions on the DDL statement. - - :param bind: - Optional. A :class:`~sqlalchemy.engine.base.Connectable`, used by - default when ``execute()`` is invoked without a bind argument. - - - See also: - - :class:`.DDLEvents` - :mod:`sqlalchemy.event` - - """ - - if not isinstance(statement, basestring): - raise exc.ArgumentError( - "Expected a string or unicode SQL statement, got '%r'" % - statement) - - self.statement = statement - self.context = context or {} - - self._check_ddl_on(on) - self.on = on - self._bind = bind - - - def __repr__(self): - return '<%s@%s; %s>' % ( - type(self).__name__, id(self), - ', '.join([repr(self.statement)] + - ['%s=%r' % (key, getattr(self, key)) - for key in ('on', 'context') - if getattr(self, key)])) - -def _to_schema_column(element): - if hasattr(element, '__clause_element__'): - element = element.__clause_element__() - if not isinstance(element, Column): - raise exc.ArgumentError("schema.Column object expected") - return element - -def _to_schema_column_or_string(element): - if hasattr(element, '__clause_element__'): - element = element.__clause_element__() - if not isinstance(element, (basestring, expression.ColumnElement)): - raise exc.ArgumentError("Element %r is not a string name or column element" % element) - return element - -class _CreateDropBase(DDLElement): - """Base class for DDL constucts that represent CREATE and DROP or - equivalents. - - The common theme of _CreateDropBase is a single - ``element`` attribute which refers to the element - to be created or dropped. - - """ - - def __init__(self, element, on=None, bind=None): - self.element = element - self._check_ddl_on(on) - self.on = on - self.bind = bind - - def _create_rule_disable(self, compiler): - """Allow disable of _create_rule using a callable. - - Pass to _create_rule using - util.portable_instancemethod(self._create_rule_disable) - to retain serializability. - - """ - return False - -class CreateSchema(_CreateDropBase): - """Represent a CREATE SCHEMA statement. - - .. versionadded:: 0.7.4 - - The argument here is the string name of the schema. - - """ - - __visit_name__ = "create_schema" - - def __init__(self, name, quote=None, **kw): - """Create a new :class:`.CreateSchema` construct.""" - - self.quote = quote - super(CreateSchema, self).__init__(name, **kw) - -class DropSchema(_CreateDropBase): - """Represent a DROP SCHEMA statement. - - The argument here is the string name of the schema. - - .. versionadded:: 0.7.4 - - """ - - __visit_name__ = "drop_schema" - - def __init__(self, name, quote=None, cascade=False, **kw): - """Create a new :class:`.DropSchema` construct.""" - - self.quote = quote - self.cascade=cascade - super(DropSchema, self).__init__(name, **kw) - - -class CreateTable(_CreateDropBase): - """Represent a CREATE TABLE statement.""" - - __visit_name__ = "create_table" - -class DropTable(_CreateDropBase): - """Represent a DROP TABLE statement.""" - - __visit_name__ = "drop_table" - -class CreateSequence(_CreateDropBase): - """Represent a CREATE SEQUENCE statement.""" - - __visit_name__ = "create_sequence" - -class DropSequence(_CreateDropBase): - """Represent a DROP SEQUENCE statement.""" - - __visit_name__ = "drop_sequence" - -class CreateIndex(_CreateDropBase): - """Represent a CREATE INDEX statement.""" - - __visit_name__ = "create_index" - -class DropIndex(_CreateDropBase): - """Represent a DROP INDEX statement.""" - - __visit_name__ = "drop_index" - -class AddConstraint(_CreateDropBase): - """Represent an ALTER TABLE ADD CONSTRAINT statement.""" - - __visit_name__ = "add_constraint" - - def __init__(self, element, *args, **kw): - super(AddConstraint, self).__init__(element, *args, **kw) - element._create_rule = util.portable_instancemethod( - self._create_rule_disable) - -class DropConstraint(_CreateDropBase): - """Represent an ALTER TABLE DROP CONSTRAINT statement.""" - - __visit_name__ = "drop_constraint" - - def __init__(self, element, cascade=False, **kw): - self.cascade = cascade - super(DropConstraint, self).__init__(element, **kw) - element._create_rule = util.portable_instancemethod( - self._create_rule_disable) - -def _bind_or_error(schemaitem, msg=None): - bind = schemaitem.bind - if not bind: - name = schemaitem.__class__.__name__ - label = getattr(schemaitem, 'fullname', - getattr(schemaitem, 'name', None)) - if label: - item = '%s %r' % (name, label) - else: - item = name - if isinstance(schemaitem, (MetaData, DDL)): - bindable = "the %s's .bind" % name - else: - bindable = "this %s's .metadata.bind" % name - - if msg is None: - msg = "The %s is not bound to an Engine or Connection. "\ - "Execution can not proceed without a database to execute "\ - "against. Either execute with an explicit connection or "\ - "assign %s to enable implicit execution." % \ - (item, bindable) - raise exc.UnboundExecutionError(msg) - return bind - +from .sql.schema import ( + CheckConstraint, + Column, + ColumnDefault, + Constraint, + DefaultClause, + DefaultGenerator, + FetchedValue, + ForeignKey, + ForeignKeyConstraint, + Index, + MetaData, + PassiveDefault, + PrimaryKeyConstraint, + SchemaItem, + Sequence, + Table, + ThreadLocalMetaData, + UniqueConstraint, + _get_table_key, + ColumnCollectionConstraint, + ) + + +from .sql.ddl import ( + DDL, + CreateTable, + DropTable, + CreateSequence, + DropSequence, + CreateIndex, + DropIndex, + CreateSchema, + DropSchema, + _DropView, + CreateColumn, + AddConstraint, + DropConstraint, + DDLBase, + DDLElement, + _CreateDropBase, + _DDLCompiles +) diff --git a/libs/sqlalchemy/sql/__init__.py b/libs/sqlalchemy/sql/__init__.py index 77fbfc84..9ed6049a 100644 --- a/libs/sqlalchemy/sql/__init__.py +++ b/libs/sqlalchemy/sql/__init__.py @@ -1,10 +1,10 @@ # sql/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy.sql.expression import ( +from .expression import ( Alias, ClauseElement, ColumnCollection, @@ -35,6 +35,7 @@ from sqlalchemy.sql.expression import ( exists, extract, false, + False_, func, insert, intersect, @@ -55,6 +56,7 @@ from sqlalchemy.sql.expression import ( table, text, true, + True_, tuple_, type_coerce, union, @@ -62,8 +64,26 @@ from sqlalchemy.sql.expression import ( update, ) -from sqlalchemy.sql.visitors import ClauseVisitor +from .visitors import ClauseVisitor -__tmp = locals().keys() -__all__ = sorted([i for i in __tmp if not i.startswith('__')]) + +def __go(lcls): + global __all__ + from .. import util as _sa_util + + import inspect as _inspect + + __all__ = sorted(name for name, obj in lcls.items() + if not (name.startswith('_') or _inspect.ismodule(obj))) + + from .annotation import _prepare_annotations, Annotated + from .elements import AnnotatedColumnElement, ClauseList + from .selectable import AnnotatedFromClause + _prepare_annotations(ColumnElement, AnnotatedColumnElement) + _prepare_annotations(FromClause, AnnotatedFromClause) + _prepare_annotations(ClauseList, Annotated) + + _sa_util.dependencies.resolve_all("sqlalchemy.sql") + +__go(locals()) diff --git a/libs/sqlalchemy/sql/annotation.py b/libs/sqlalchemy/sql/annotation.py new file mode 100644 index 00000000..b5b7849d --- /dev/null +++ b/libs/sqlalchemy/sql/annotation.py @@ -0,0 +1,182 @@ +# sql/annotation.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""The :class:`.Annotated` class and related routines; creates hash-equivalent +copies of SQL constructs which contain context-specific markers and associations. + +""" + +from .. import util +from . import operators + +class Annotated(object): + """clones a ClauseElement and applies an 'annotations' dictionary. + + Unlike regular clones, this clone also mimics __hash__() and + __cmp__() of the original element so that it takes its place + in hashed collections. + + A reference to the original element is maintained, for the important + reason of keeping its hash value current. When GC'ed, the + hash value may be reused, causing conflicts. + + """ + + def __new__(cls, *args): + if not args: + # clone constructor + return object.__new__(cls) + else: + element, values = args + # pull appropriate subclass from registry of annotated + # classes + try: + cls = annotated_classes[element.__class__] + except KeyError: + cls = _new_annotation_type(element.__class__, cls) + return object.__new__(cls) + + def __init__(self, element, values): + self.__dict__ = element.__dict__.copy() + self.__element = element + self._annotations = values + + def _annotate(self, values): + _values = self._annotations.copy() + _values.update(values) + return self._with_annotations(_values) + + def _with_annotations(self, values): + clone = self.__class__.__new__(self.__class__) + clone.__dict__ = self.__dict__.copy() + clone._annotations = values + return clone + + def _deannotate(self, values=None, clone=True): + if values is None: + return self.__element + else: + _values = self._annotations.copy() + for v in values: + _values.pop(v, None) + return self._with_annotations(_values) + + def _compiler_dispatch(self, visitor, **kw): + return self.__element.__class__._compiler_dispatch(self, visitor, **kw) + + @property + def _constructor(self): + return self.__element._constructor + + def _clone(self): + clone = self.__element._clone() + if clone is self.__element: + # detect immutable, don't change anything + return self + else: + # update the clone with any changes that have occurred + # to this object's __dict__. + clone.__dict__.update(self.__dict__) + return self.__class__(clone, self._annotations) + + def __hash__(self): + return hash(self.__element) + + def __eq__(self, other): + if isinstance(self.__element, operators.ColumnOperators): + return self.__element.__class__.__eq__(self, other) + else: + return hash(other) == hash(self) + + + +# hard-generate Annotated subclasses. this technique +# is used instead of on-the-fly types (i.e. type.__new__()) +# so that the resulting objects are pickleable. +annotated_classes = {} + + + +def _deep_annotate(element, annotations, exclude=None): + """Deep copy the given ClauseElement, annotating each element + with the given annotations dictionary. + + Elements within the exclude collection will be cloned but not annotated. + + """ + def clone(elem): + if exclude and \ + hasattr(elem, 'proxy_set') and \ + elem.proxy_set.intersection(exclude): + newelem = elem._clone() + elif annotations != elem._annotations: + newelem = elem._annotate(annotations) + else: + newelem = elem + newelem._copy_internals(clone=clone) + return newelem + + if element is not None: + element = clone(element) + return element + + +def _deep_deannotate(element, values=None): + """Deep copy the given element, removing annotations.""" + + cloned = util.column_dict() + + def clone(elem): + # if a values dict is given, + # the elem must be cloned each time it appears, + # as there may be different annotations in source + # elements that are remaining. if totally + # removing all annotations, can assume the same + # slate... + if values or elem not in cloned: + newelem = elem._deannotate(values=values, clone=True) + newelem._copy_internals(clone=clone) + if not values: + cloned[elem] = newelem + return newelem + else: + return cloned[elem] + + if element is not None: + element = clone(element) + return element + + +def _shallow_annotate(element, annotations): + """Annotate the given ClauseElement and copy its internals so that + internal objects refer to the new annotated object. + + Basically used to apply a "dont traverse" annotation to a + selectable, without digging throughout the whole + structure wasting time. + """ + element = element._annotate(annotations) + element._copy_internals() + return element + +def _new_annotation_type(cls, base_cls): + if issubclass(cls, Annotated): + return cls + elif cls in annotated_classes: + return annotated_classes[cls] + annotated_classes[cls] = anno_cls = type( + "Annotated%s" % cls.__name__, + (base_cls, cls), {}) + globals()["Annotated%s" % cls.__name__] = anno_cls + return anno_cls + +def _prepare_annotations(target_hierarchy, base_cls): + stack = [target_hierarchy] + while stack: + cls = stack.pop() + stack.extend(cls.__subclasses__()) + + _new_annotation_type(cls, base_cls) diff --git a/libs/sqlalchemy/sql/base.py b/libs/sqlalchemy/sql/base.py new file mode 100644 index 00000000..e6c5d6ed --- /dev/null +++ b/libs/sqlalchemy/sql/base.py @@ -0,0 +1,348 @@ +# sql/base.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Foundational utilities common to many sql modules. + +""" + + +from .. import util, exc +import itertools +from .visitors import ClauseVisitor + + +PARSE_AUTOCOMMIT = util.symbol('PARSE_AUTOCOMMIT') +NO_ARG = util.symbol('NO_ARG') + +class Immutable(object): + """mark a ClauseElement as 'immutable' when expressions are cloned.""" + + def unique_params(self, *optionaldict, **kwargs): + raise NotImplementedError("Immutable objects do not support copying") + + def params(self, *optionaldict, **kwargs): + raise NotImplementedError("Immutable objects do not support copying") + + def _clone(self): + return self + + + +def _from_objects(*elements): + return itertools.chain(*[element._from_objects for element in elements]) + +@util.decorator +def _generative(fn, *args, **kw): + """Mark a method as generative.""" + + self = args[0]._generate() + fn(self, *args[1:], **kw) + return self + + +class Generative(object): + """Allow a ClauseElement to generate itself via the + @_generative decorator. + + """ + + def _generate(self): + s = self.__class__.__new__(self.__class__) + s.__dict__ = self.__dict__.copy() + return s + + +class Executable(Generative): + """Mark a ClauseElement as supporting execution. + + :class:`.Executable` is a superclass for all "statement" types + of objects, including :func:`select`, :func:`delete`, :func:`update`, + :func:`insert`, :func:`text`. + + """ + + supports_execution = True + _execution_options = util.immutabledict() + _bind = None + + @_generative + def execution_options(self, **kw): + """ Set non-SQL options for the statement which take effect during + execution. + + Execution options can be set on a per-statement or + per :class:`.Connection` basis. Additionally, the + :class:`.Engine` and ORM :class:`~.orm.query.Query` objects provide + access to execution options which they in turn configure upon + connections. + + The :meth:`execution_options` method is generative. A new + instance of this statement is returned that contains the options:: + + statement = select([table.c.x, table.c.y]) + statement = statement.execution_options(autocommit=True) + + Note that only a subset of possible execution options can be applied + to a statement - these include "autocommit" and "stream_results", + but not "isolation_level" or "compiled_cache". + See :meth:`.Connection.execution_options` for a full list of + possible options. + + .. seealso:: + + :meth:`.Connection.execution_options()` + + :meth:`.Query.execution_options()` + + """ + if 'isolation_level' in kw: + raise exc.ArgumentError( + "'isolation_level' execution option may only be specified " + "on Connection.execution_options(), or " + "per-engine using the isolation_level " + "argument to create_engine()." + ) + if 'compiled_cache' in kw: + raise exc.ArgumentError( + "'compiled_cache' execution option may only be specified " + "on Connection.execution_options(), not per statement." + ) + self._execution_options = self._execution_options.union(kw) + + def execute(self, *multiparams, **params): + """Compile and execute this :class:`.Executable`.""" + e = self.bind + if e is None: + label = getattr(self, 'description', self.__class__.__name__) + msg = ('This %s is not directly bound to a Connection or Engine.' + 'Use the .execute() method of a Connection or Engine ' + 'to execute this construct.' % label) + raise exc.UnboundExecutionError(msg) + return e._execute_clauseelement(self, multiparams, params) + + def scalar(self, *multiparams, **params): + """Compile and execute this :class:`.Executable`, returning the + result's scalar representation. + + """ + return self.execute(*multiparams, **params).scalar() + + @property + def bind(self): + """Returns the :class:`.Engine` or :class:`.Connection` to + which this :class:`.Executable` is bound, or None if none found. + + This is a traversal which checks locally, then + checks among the "from" clauses of associated objects + until a bound engine or connection is found. + + """ + if self._bind is not None: + return self._bind + + for f in _from_objects(self): + if f is self: + continue + engine = f.bind + if engine is not None: + return engine + else: + return None + + +class SchemaEventTarget(object): + """Base class for elements that are the targets of :class:`.DDLEvents` + events. + + This includes :class:`.SchemaItem` as well as :class:`.SchemaType`. + + """ + + def _set_parent(self, parent): + """Associate with this SchemaEvent's parent object.""" + + raise NotImplementedError() + + def _set_parent_with_dispatch(self, parent): + self.dispatch.before_parent_attach(self, parent) + self._set_parent(parent) + self.dispatch.after_parent_attach(self, parent) + +class SchemaVisitor(ClauseVisitor): + """Define the visiting for ``SchemaItem`` objects.""" + + __traverse_options__ = {'schema_visitor': True} + +class ColumnCollection(util.OrderedProperties): + """An ordered dictionary that stores a list of ColumnElement + instances. + + Overrides the ``__eq__()`` method to produce SQL clauses between + sets of correlated columns. + + """ + + def __init__(self, *cols): + super(ColumnCollection, self).__init__() + self._data.update((c.key, c) for c in cols) + self.__dict__['_all_cols'] = util.column_set(self) + + def __str__(self): + return repr([str(c) for c in self]) + + def replace(self, column): + """add the given column to this collection, removing unaliased + versions of this column as well as existing columns with the + same key. + + e.g.:: + + t = Table('sometable', metadata, Column('col1', Integer)) + t.columns.replace(Column('col1', Integer, key='columnone')) + + will remove the original 'col1' from the collection, and add + the new column under the name 'columnname'. + + Used by schema.Column to override columns during table reflection. + + """ + if column.name in self and column.key != column.name: + other = self[column.name] + if other.name == other.key: + del self._data[other.name] + self._all_cols.remove(other) + if column.key in self._data: + self._all_cols.remove(self._data[column.key]) + self._all_cols.add(column) + self._data[column.key] = column + + def add(self, column): + """Add a column to this collection. + + The key attribute of the column will be used as the hash key + for this dictionary. + + """ + self[column.key] = column + + def __delitem__(self, key): + raise NotImplementedError() + + def __setattr__(self, key, object): + raise NotImplementedError() + + def __setitem__(self, key, value): + if key in self: + + # this warning is primarily to catch select() statements + # which have conflicting column names in their exported + # columns collection + + existing = self[key] + if not existing.shares_lineage(value): + util.warn('Column %r on table %r being replaced by ' + '%r, which has the same key. Consider ' + 'use_labels for select() statements.' % (key, + getattr(existing, 'table', None), value)) + self._all_cols.remove(existing) + # pop out memoized proxy_set as this + # operation may very well be occurring + # in a _make_proxy operation + util.memoized_property.reset(value, "proxy_set") + self._all_cols.add(value) + self._data[key] = value + + def clear(self): + self._data.clear() + self._all_cols.clear() + + def remove(self, column): + del self._data[column.key] + self._all_cols.remove(column) + + def update(self, value): + self._data.update(value) + self._all_cols.clear() + self._all_cols.update(self._data.values()) + + def extend(self, iter): + self.update((c.key, c) for c in iter) + + __hash__ = None + + @util.dependencies("sqlalchemy.sql.elements") + def __eq__(self, elements, other): + l = [] + for c in other: + for local in self: + if c.shares_lineage(local): + l.append(c == local) + return elements.and_(*l) + + def __contains__(self, other): + if not isinstance(other, util.string_types): + raise exc.ArgumentError("__contains__ requires a string argument") + return util.OrderedProperties.__contains__(self, other) + + def __setstate__(self, state): + self.__dict__['_data'] = state['_data'] + self.__dict__['_all_cols'] = util.column_set(self._data.values()) + + def contains_column(self, col): + # this has to be done via set() membership + return col in self._all_cols + + def as_immutable(self): + return ImmutableColumnCollection(self._data, self._all_cols) + + +class ImmutableColumnCollection(util.ImmutableProperties, ColumnCollection): + def __init__(self, data, colset): + util.ImmutableProperties.__init__(self, data) + self.__dict__['_all_cols'] = colset + + extend = remove = util.ImmutableProperties._immutable + + +class ColumnSet(util.ordered_column_set): + def contains_column(self, col): + return col in self + + def extend(self, cols): + for col in cols: + self.add(col) + + def __add__(self, other): + return list(self) + list(other) + + @util.dependencies("sqlalchemy.sql.elements") + def __eq__(self, elements, other): + l = [] + for c in other: + for local in self: + if c.shares_lineage(local): + l.append(c == local) + return elements.and_(*l) + + def __hash__(self): + return hash(tuple(x for x in self)) + +def _bind_or_error(schemaitem, msg=None): + bind = schemaitem.bind + if not bind: + name = schemaitem.__class__.__name__ + label = getattr(schemaitem, 'fullname', + getattr(schemaitem, 'name', None)) + if label: + item = '%s object %r' % (name, label) + else: + item = '%s object' % name + if msg is None: + msg = "%s is not bound to an Engine or Connection. "\ + "Execution can not proceed without a database to execute "\ + "against." % item + raise exc.UnboundExecutionError(msg) + return bind diff --git a/libs/sqlalchemy/sql/compiler.py b/libs/sqlalchemy/sql/compiler.py index 9dc56d1f..ade3c623 100644 --- a/libs/sqlalchemy/sql/compiler.py +++ b/libs/sqlalchemy/sql/compiler.py @@ -1,5 +1,5 @@ # sql/compiler.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,26 +8,24 @@ Classes provided include: -:class:`~sqlalchemy.sql.compiler.SQLCompiler` - renders SQL +:class:`.compiler.SQLCompiler` - renders SQL strings -:class:`~sqlalchemy.sql.compiler.DDLCompiler` - renders DDL +:class:`.compiler.DDLCompiler` - renders DDL (data definition language) strings -:class:`~sqlalchemy.sql.compiler.GenericTypeCompiler` - renders +:class:`.compiler.GenericTypeCompiler` - renders type specification strings. To generate user-defined SQL strings, see -:module:`~sqlalchemy.ext.compiler`. +:doc:`/ext/compiler`. """ import re -import sys -from sqlalchemy import schema, engine, util, exc -from sqlalchemy.sql import operators, functions, util as sql_util, \ - visitors -from sqlalchemy.sql import expression as sql +from . import schema, sqltypes, operators, functions, \ + util as sql_util, visitors, elements, selectable, base +from .. import util, exc import decimal import itertools @@ -51,65 +49,75 @@ RESERVED_WORDS = set([ 'using', 'verbose', 'when', 'where']) LEGAL_CHARACTERS = re.compile(r'^[A-Z0-9_$]+$', re.I) -ILLEGAL_INITIAL_CHARACTERS = set([str(x) for x in xrange(0, 10)]).union(['$']) +ILLEGAL_INITIAL_CHARACTERS = set([str(x) for x in range(0, 10)]).union(['$']) BIND_PARAMS = re.compile(r'(? ', - operators.ge : ' >= ', - operators.eq : ' = ', - operators.concat_op : ' || ', - operators.between_op : ' BETWEEN ', - operators.match_op : ' MATCH ', - operators.in_op : ' IN ', - operators.notin_op : ' NOT IN ', - operators.comma_op : ', ', - operators.from_ : ' FROM ', - operators.as_ : ' AS ', - operators.is_ : ' IS ', - operators.isnot : ' IS NOT ', - operators.collate : ' COLLATE ', + operators.and_: ' AND ', + operators.or_: ' OR ', + operators.add: ' + ', + operators.mul: ' * ', + operators.sub: ' - ', + operators.div: ' / ', + operators.mod: ' % ', + operators.truediv: ' / ', + operators.neg: '-', + operators.lt: ' < ', + operators.le: ' <= ', + operators.ne: ' != ', + operators.gt: ' > ', + operators.ge: ' >= ', + operators.eq: ' = ', + operators.concat_op: ' || ', + operators.between_op: ' BETWEEN ', + operators.match_op: ' MATCH ', + operators.in_op: ' IN ', + operators.notin_op: ' NOT IN ', + operators.comma_op: ', ', + operators.from_: ' FROM ', + operators.as_: ' AS ', + operators.is_: ' IS ', + operators.isnot: ' IS NOT ', + operators.collate: ' COLLATE ', # unary - operators.exists : 'EXISTS ', - operators.distinct_op : 'DISTINCT ', - operators.inv : 'NOT ', + operators.exists: 'EXISTS ', + operators.distinct_op: 'DISTINCT ', + operators.inv: 'NOT ', # modifiers - operators.desc_op : ' DESC', - operators.asc_op : ' ASC', - operators.nullsfirst_op : ' NULLS FIRST', - operators.nullslast_op : ' NULLS LAST', + operators.desc_op: ' DESC', + operators.asc_op: ' ASC', + operators.nullsfirst_op: ' NULLS FIRST', + operators.nullslast_op: ' NULLS LAST', + } FUNCTIONS = { - functions.coalesce : 'coalesce%(expr)s', + functions.coalesce: 'coalesce%(expr)s', functions.current_date: 'CURRENT_DATE', functions.current_time: 'CURRENT_TIME', functions.current_timestamp: 'CURRENT_TIMESTAMP', @@ -118,7 +126,7 @@ FUNCTIONS = { functions.localtimestamp: 'LOCALTIMESTAMP', functions.random: 'random%(expr)s', functions.sysdate: 'sysdate', - functions.session_user :'SESSION_USER', + functions.session_user: 'SESSION_USER', functions.user: 'USER' } @@ -141,16 +149,125 @@ EXTRACT_MAP = { } COMPOUND_KEYWORDS = { - sql.CompoundSelect.UNION : 'UNION', - sql.CompoundSelect.UNION_ALL : 'UNION ALL', - sql.CompoundSelect.EXCEPT : 'EXCEPT', - sql.CompoundSelect.EXCEPT_ALL : 'EXCEPT ALL', - sql.CompoundSelect.INTERSECT : 'INTERSECT', - sql.CompoundSelect.INTERSECT_ALL : 'INTERSECT ALL' + selectable.CompoundSelect.UNION: 'UNION', + selectable.CompoundSelect.UNION_ALL: 'UNION ALL', + selectable.CompoundSelect.EXCEPT: 'EXCEPT', + selectable.CompoundSelect.EXCEPT_ALL: 'EXCEPT ALL', + selectable.CompoundSelect.INTERSECT: 'INTERSECT', + selectable.CompoundSelect.INTERSECT_ALL: 'INTERSECT ALL' } +class Compiled(object): + """Represent a compiled SQL or DDL expression. + + The ``__str__`` method of the ``Compiled`` object should produce + the actual text of the statement. ``Compiled`` objects are + specific to their underlying database dialect, and also may + or may not be specific to the columns referenced within a + particular set of bind parameters. In no case should the + ``Compiled`` object be dependent on the actual values of those + bind parameters, even though it may reference those values as + defaults. + """ + + def __init__(self, dialect, statement, bind=None, + compile_kwargs=util.immutabledict()): + """Construct a new ``Compiled`` object. + + :param dialect: ``Dialect`` to compile against. + + :param statement: ``ClauseElement`` to be compiled. + + :param bind: Optional Engine or Connection to compile this + statement against. + + :param compile_kwargs: additional kwargs that will be + passed to the initial call to :meth:`.Compiled.process`. + + .. versionadded:: 0.8 + + """ + + self.dialect = dialect + self.bind = bind + if statement is not None: + self.statement = statement + self.can_execute = statement.supports_execution + self.string = self.process(self.statement, **compile_kwargs) + + @util.deprecated("0.7", ":class:`.Compiled` objects now compile " + "within the constructor.") + def compile(self): + """Produce the internal string representation of this element. + """ + pass + + def _execute_on_connection(self, connection, multiparams, params): + return connection._execute_compiled(self, multiparams, params) + + @property + def sql_compiler(self): + """Return a Compiled that is capable of processing SQL expressions. + + If this compiler is one, it would likely just return 'self'. + + """ + + raise NotImplementedError() + + def process(self, obj, **kwargs): + return obj._compiler_dispatch(self, **kwargs) + + def __str__(self): + """Return the string text of the generated SQL or DDL.""" + + return self.string or '' + + def construct_params(self, params=None): + """Return the bind params for this compiled object. + + :param params: a dict of string/object pairs whose values will + override bind values compiled in to the + statement. + """ + + raise NotImplementedError() + + @property + def params(self): + """Return the bind params for this compiled object.""" + return self.construct_params() + + def execute(self, *multiparams, **params): + """Execute this compiled object.""" + + e = self.bind + if e is None: + raise exc.UnboundExecutionError( + "This Compiled object is not bound to any Engine " + "or Connection.") + return e._execute_compiled(self, multiparams, params) + + def scalar(self, *multiparams, **params): + """Execute this compiled object and return the result's + scalar value.""" + + return self.execute(*multiparams, **params).scalar() + + +class TypeCompiler(object): + """Produces DDL specification for TypeEngine objects.""" + + def __init__(self, dialect): + self.dialect = dialect + + def process(self, type_): + return type_._compiler_dispatch(self) + + + class _CompileLabel(visitors.Visitable): - """lightweight label object which acts as an expression._Label.""" + """lightweight label object which acts as an expression.Label.""" __visit_name__ = 'label' __slots__ = 'element', 'name' @@ -158,7 +275,7 @@ class _CompileLabel(visitors.Visitable): def __init__(self, col, name, alt_names=()): self.element = col self.name = name - self._alt_names = alt_names + self._alt_names = (col,) + alt_names @property def proxy_set(self): @@ -168,11 +285,8 @@ class _CompileLabel(visitors.Visitable): def type(self): return self.element.type - @property - def quote(self): - return self.element.quote -class SQLCompiler(engine.Compiled): +class SQLCompiler(Compiled): """Default implementation of Compiled. Compiles ClauseElements into SQL strings. Uses a similar visit @@ -236,11 +350,11 @@ class SQLCompiler(engine.Compiled): # execute) self.inline = inline or getattr(statement, 'inline', False) - # a dictionary of bind parameter keys to _BindParamClause + # a dictionary of bind parameter keys to BindParameter # instances. self.binds = {} - # a dictionary of _BindParamClause instances to "compiled" names + # a dictionary of BindParameter instances to "compiled" names # that are actually present in the generated SQL self.bind_names = util.column_dict() @@ -273,7 +387,7 @@ class SQLCompiler(engine.Compiled): # a map which tracks "truncated" names based on # dialect.label_length or dialect.max_identifier_length self.truncated_names = {} - engine.Compiled.__init__(self, dialect, statement, **kwargs) + Compiled.__init__(self, dialect, statement, **kwargs) if self.positional and dialect.paramstyle == 'numeric': self._apply_numbered_params() @@ -296,16 +410,16 @@ class SQLCompiler(engine.Compiled): poscount = itertools.count(1) self.string = re.sub( r'\[_POSITION\]', - lambda m:str(util.next(poscount)), + lambda m: str(util.next(poscount)), self.string) @util.memoized_property def _bind_processors(self): return dict( (key, value) for key, value in - ( (self.bind_names[bindparam], + ((self.bind_names[bindparam], bindparam.type._cached_bind_processor(self.dialect)) - for bindparam in self.bind_names ) + for bindparam in self.bind_names) if value is not None ) @@ -321,7 +435,7 @@ class SQLCompiler(engine.Compiled): if params: pd = {} - for bindparam, name in self.bind_names.iteritems(): + for bindparam, name in self.bind_names.items(): if bindparam.key in params: pd[name] = params[bindparam.key] elif name in params: @@ -373,24 +487,32 @@ class SQLCompiler(engine.Compiled): def visit_grouping(self, grouping, asfrom=False, **kwargs): return "(" + grouping.element._compiler_dispatch(self, **kwargs) + ")" - def visit_label(self, label, result_map=None, + def visit_label(self, label, + add_to_result_map=None, within_label_clause=False, - within_columns_clause=False, **kw): + within_columns_clause=False, + render_label_as_label=None, + **kw): # only render labels within the columns clause # or ORDER BY clause of a select. dialect-specific compilers # can modify this behavior. - if within_columns_clause and not within_label_clause: - if isinstance(label.name, sql._truncated_label): + render_label_with_as = within_columns_clause and not within_label_clause + render_label_only = render_label_as_label is label + + if render_label_only or render_label_with_as: + if isinstance(label.name, elements._truncated_label): labelname = self._truncated_identifier("colident", label.name) else: labelname = label.name - if result_map is not None: - result_map[labelname.lower()] = ( + if render_label_with_as: + if add_to_result_map is not None: + add_to_result_map( + labelname, label.name, - (label, label.element, labelname, ) + - label._alt_names, - label.type) + (label, labelname, ) + label._alt_names, + label.type + ) return label.element._compiler_dispatch(self, within_columns_clause=True, @@ -398,47 +520,51 @@ class SQLCompiler(engine.Compiled): **kw) + \ OPERATORS[operators.as_] + \ self.preparer.format_label(label, labelname) + elif render_label_only: + return labelname else: return label.element._compiler_dispatch(self, within_columns_clause=False, **kw) - def visit_column(self, column, result_map=None, **kwargs): + def visit_column(self, column, add_to_result_map=None, + include_table=True, **kwargs): name = orig_name = column.name if name is None: raise exc.CompileError("Cannot compile Column object until " - "it's 'name' is assigned.") + "its 'name' is assigned.") is_literal = column.is_literal - if not is_literal and isinstance(name, sql._truncated_label): + if not is_literal and isinstance(name, elements._truncated_label): name = self._truncated_identifier("colident", name) - if result_map is not None: - result_map[name.lower()] = (orig_name, - (column, name, column.key), - column.type) + if add_to_result_map is not None: + add_to_result_map( + name, + orig_name, + (column, name, column.key), + column.type + ) if is_literal: name = self.escape_literal_column(name) else: - name = self.preparer.quote(name, column.quote) + name = self.preparer.quote(name) table = column.table - if table is None or not table.named_with_column: + if table is None or not include_table or not table.named_with_column: return name else: if table.schema: - schema_prefix = self.preparer.quote_schema( - table.schema, - table.quote_schema) + '.' + schema_prefix = self.preparer.quote_schema(table.schema) + '.' else: schema_prefix = '' tablename = table.name - if isinstance(tablename, sql._truncated_label): + if isinstance(tablename, elements._truncated_label): tablename = self._truncated_identifier("alias", tablename) return schema_prefix + \ - self.preparer.quote(tablename, table.quote) + \ + self.preparer.quote(tablename) + \ "." + name def escape_literal_column(self, text): @@ -459,17 +585,13 @@ class SQLCompiler(engine.Compiled): def post_process_text(self, text): return text - def visit_textclause(self, textclause, **kwargs): - if textclause.typemap is not None: - for colname, type_ in textclause.typemap.iteritems(): - self.result_map[colname.lower()] = (colname, None, type_) - + def visit_textclause(self, textclause, **kw): def do_bindparam(m): name = m.group(1) - if name in textclause.bindparams: - return self.process(textclause.bindparams[name]) + if name in textclause._bindparams: + return self.process(textclause._bindparams[name], **kw) else: - return self.bindparam_string(name, **kwargs) + return self.bindparam_string(name, **kw) # un-escape any \:params return BIND_PARAMS_ESC.sub(lambda m: m.group(1), @@ -477,16 +599,53 @@ class SQLCompiler(engine.Compiled): self.post_process_text(textclause.text)) ) + def visit_text_as_from(self, taf, iswrapper=False, + compound_index=0, force_result_map=False, + asfrom=False, + parens=True, **kw): + + toplevel = not self.stack + entry = self._default_stack_entry if toplevel else self.stack[-1] + + populate_result_map = force_result_map or ( + compound_index == 0 and ( + toplevel or \ + entry['iswrapper'] + ) + ) + + if populate_result_map: + for c in taf.c: + self._add_to_result_map( + c.key, c.key, (c,), c.type + ) + + text = self.process(taf.element, **kw) + if asfrom and parens: + text = "(%s)" % text + return text + + def visit_null(self, expr, **kw): return 'NULL' def visit_true(self, expr, **kw): - return 'true' + if self.dialect.supports_native_boolean: + return 'true' + else: + return "1" def visit_false(self, expr, **kw): - return 'false' + if self.dialect.supports_native_boolean: + return 'false' + else: + return "0" + + def visit_clauselist(self, clauselist, order_by_select=None, **kw): + if order_by_select is not None: + return self._order_by_clauselist( + clauselist, order_by_select, **kw) - def visit_clauselist(self, clauselist, **kwargs): sep = clauselist.operator if sep is None: sep = " " @@ -494,8 +653,32 @@ class SQLCompiler(engine.Compiled): sep = OPERATORS[clauselist.operator] return sep.join( s for s in - (c._compiler_dispatch(self, **kwargs) - for c in clauselist.clauses) + ( + c._compiler_dispatch(self, **kw) + for c in clauselist.clauses) + if s) + + def _order_by_clauselist(self, clauselist, order_by_select, **kw): + # look through raw columns collection for labels. + # note that its OK we aren't expanding tables and other selectables + # here; we can only add a label in the ORDER BY for an individual + # label expression in the columns clause. + + raw_col = set(l._order_by_label_element.name + for l in order_by_select._raw_columns + if l._order_by_label_element is not None) + + return ", ".join( + s for s in + ( + c._compiler_dispatch(self, + render_label_as_label= + c._order_by_label_element if + c._order_by_label_element is not None and + c._order_by_label_element.name in raw_col + else None, + **kw) + for c in clauselist.clauses) if s) def visit_case(self, clause, **kwargs): @@ -522,7 +705,7 @@ class SQLCompiler(engine.Compiled): def visit_over(self, over, **kwargs): return "%s OVER (%s)" % ( over.func._compiler_dispatch(self, **kwargs), - ' '.join( + ' '.join( '%s BY %s' % (word, clause._compiler_dispatch(self, **kwargs)) for word, clause in ( ('PARTITION', over.partition_by), @@ -537,9 +720,11 @@ class SQLCompiler(engine.Compiled): return "EXTRACT(%s FROM %s)" % (field, extract.expr._compiler_dispatch(self, **kwargs)) - def visit_function(self, func, result_map=None, **kwargs): - if result_map is not None: - result_map[func.name.lower()] = (func.name, None, func.type) + def visit_function(self, func, add_to_result_map=None, **kwargs): + if add_to_result_map is not None: + add_to_result_map( + func.name, func.name, (), func.type + ) disp = getattr(self, "visit_%s_func" % func.name.lower(), None) if disp: @@ -554,17 +739,25 @@ class SQLCompiler(engine.Compiled): def visit_sequence(self, sequence): raise NotImplementedError( - "Dialect '%s' does not support sequence increments." % self.dialect.name + "Dialect '%s' does not support sequence increments." % + self.dialect.name ) def function_argspec(self, func, **kwargs): return func.clause_expr._compiler_dispatch(self, **kwargs) + def visit_compound_select(self, cs, asfrom=False, parens=True, compound_index=0, **kwargs): - entry = self.stack and self.stack[-1] or {} - self.stack.append({'from': entry.get('from', None), - 'iswrapper': not entry}) + toplevel = not self.stack + entry = self._default_stack_entry if toplevel else self.stack[-1] + + self.stack.append( + { + 'correlate_froms': entry['correlate_froms'], + 'iswrapper': toplevel, + 'asfrom_froms': entry['asfrom_froms'] + }) keyword = self.compound_keywords.get(cs.keyword) @@ -585,7 +778,7 @@ class SQLCompiler(engine.Compiled): self.limit_clause(cs) or "" if self.ctes and \ - compound_index == 0 and not entry: + compound_index == 0 and toplevel: text = self._render_cte_clause() + text self.stack.pop(-1) @@ -595,84 +788,191 @@ class SQLCompiler(engine.Compiled): return text def visit_unary(self, unary, **kw): - s = unary.element._compiler_dispatch(self, **kw) if unary.operator: - s = OPERATORS[unary.operator] + s - if unary.modifier: - s = s + OPERATORS[unary.modifier] - return s + if unary.modifier: + raise exc.CompileError( + "Unary expression does not support operator " + "and modifier simultaneously") + disp = getattr(self, "visit_%s_unary_operator" % + unary.operator.__name__, None) + if disp: + return disp(unary, unary.operator, **kw) + else: + return self._generate_generic_unary_operator(unary, + OPERATORS[unary.operator], **kw) + elif unary.modifier: + disp = getattr(self, "visit_%s_unary_modifier" % + unary.modifier.__name__, None) + if disp: + return disp(unary, unary.modifier, **kw) + else: + return self._generate_generic_unary_modifier(unary, + OPERATORS[unary.modifier], **kw) + else: + raise exc.CompileError( + "Unary expression has no operator or modifier") + + def visit_istrue_unary_operator(self, element, operator, **kw): + if self.dialect.supports_native_boolean: + return self.process(element.element, **kw) + else: + return "%s = 1" % self.process(element.element, **kw) + + def visit_isfalse_unary_operator(self, element, operator, **kw): + if self.dialect.supports_native_boolean: + return "NOT %s" % self.process(element.element, **kw) + else: + return "%s = 0" % self.process(element.element, **kw) def visit_binary(self, binary, **kw): # don't allow "? = ?" to render if self.ansi_bind_rules and \ - isinstance(binary.left, sql._BindParamClause) and \ - isinstance(binary.right, sql._BindParamClause): + isinstance(binary.left, elements.BindParameter) and \ + isinstance(binary.right, elements.BindParameter): kw['literal_binds'] = True - return self._operator_dispatch(binary.operator, - binary, - lambda opstr: binary.left._compiler_dispatch(self, **kw) + - opstr + - binary.right._compiler_dispatch( - self, **kw), - **kw - ) + operator = binary.operator + disp = getattr(self, "visit_%s_binary" % operator.__name__, None) + if disp: + return disp(binary, operator, **kw) + else: + try: + opstring = OPERATORS[operator] + except KeyError: + raise exc.UnsupportedCompilationError(self, operator) + else: + return self._generate_generic_binary(binary, opstring, **kw) - def visit_like_op(self, binary, **kw): + def visit_custom_op_binary(self, element, operator, **kw): + return self._generate_generic_binary(element, + " " + operator.opstring + " ", **kw) + + def visit_custom_op_unary_operator(self, element, operator, **kw): + return self._generate_generic_unary_operator(element, + operator.opstring + " ", **kw) + + def visit_custom_op_unary_modifier(self, element, operator, **kw): + return self._generate_generic_unary_modifier(element, + " " + operator.opstring, **kw) + + def _generate_generic_binary(self, binary, opstring, **kw): + return binary.left._compiler_dispatch(self, **kw) + \ + opstring + \ + binary.right._compiler_dispatch(self, **kw) + + def _generate_generic_unary_operator(self, unary, opstring, **kw): + return opstring + unary.element._compiler_dispatch(self, **kw) + + def _generate_generic_unary_modifier(self, unary, opstring, **kw): + return unary.element._compiler_dispatch(self, **kw) + opstring + + @util.memoized_property + def _like_percent_literal(self): + return elements.literal_column("'%'", type_=sqltypes.STRINGTYPE) + + def visit_contains_op_binary(self, binary, operator, **kw): + binary = binary._clone() + percent = self._like_percent_literal + binary.right = percent.__add__(binary.right).__add__(percent) + return self.visit_like_op_binary(binary, operator, **kw) + + def visit_notcontains_op_binary(self, binary, operator, **kw): + binary = binary._clone() + percent = self._like_percent_literal + binary.right = percent.__add__(binary.right).__add__(percent) + return self.visit_notlike_op_binary(binary, operator, **kw) + + def visit_startswith_op_binary(self, binary, operator, **kw): + binary = binary._clone() + percent = self._like_percent_literal + binary.right = percent.__radd__( + binary.right + ) + return self.visit_like_op_binary(binary, operator, **kw) + + def visit_notstartswith_op_binary(self, binary, operator, **kw): + binary = binary._clone() + percent = self._like_percent_literal + binary.right = percent.__radd__( + binary.right + ) + return self.visit_notlike_op_binary(binary, operator, **kw) + + def visit_endswith_op_binary(self, binary, operator, **kw): + binary = binary._clone() + percent = self._like_percent_literal + binary.right = percent.__add__(binary.right) + return self.visit_like_op_binary(binary, operator, **kw) + + def visit_notendswith_op_binary(self, binary, operator, **kw): + binary = binary._clone() + percent = self._like_percent_literal + binary.right = percent.__add__(binary.right) + return self.visit_notlike_op_binary(binary, operator, **kw) + + def visit_like_op_binary(self, binary, operator, **kw): escape = binary.modifiers.get("escape", None) + + # TODO: use ternary here, not "and"/ "or" return '%s LIKE %s' % ( binary.left._compiler_dispatch(self, **kw), binary.right._compiler_dispatch(self, **kw)) \ - + (escape and - (' ESCAPE ' + self.render_literal_value(escape, None)) - or '') + + ( + ' ESCAPE ' + + self.render_literal_value(escape, sqltypes.STRINGTYPE) + if escape else '' + ) - def visit_notlike_op(self, binary, **kw): + def visit_notlike_op_binary(self, binary, operator, **kw): escape = binary.modifiers.get("escape", None) return '%s NOT LIKE %s' % ( binary.left._compiler_dispatch(self, **kw), binary.right._compiler_dispatch(self, **kw)) \ - + (escape and - (' ESCAPE ' + self.render_literal_value(escape, None)) - or '') + + ( + ' ESCAPE ' + + self.render_literal_value(escape, sqltypes.STRINGTYPE) + if escape else '' + ) - def visit_ilike_op(self, binary, **kw): + def visit_ilike_op_binary(self, binary, operator, **kw): escape = binary.modifiers.get("escape", None) return 'lower(%s) LIKE lower(%s)' % ( binary.left._compiler_dispatch(self, **kw), binary.right._compiler_dispatch(self, **kw)) \ - + (escape and - (' ESCAPE ' + self.render_literal_value(escape, None)) - or '') + + ( + ' ESCAPE ' + + self.render_literal_value(escape, sqltypes.STRINGTYPE) + if escape else '' + ) - def visit_notilike_op(self, binary, **kw): + def visit_notilike_op_binary(self, binary, operator, **kw): escape = binary.modifiers.get("escape", None) return 'lower(%s) NOT LIKE lower(%s)' % ( binary.left._compiler_dispatch(self, **kw), binary.right._compiler_dispatch(self, **kw)) \ - + (escape and - (' ESCAPE ' + self.render_literal_value(escape, None)) - or '') - - def _operator_dispatch(self, operator, element, fn, **kw): - if util.callable(operator): - disp = getattr(self, "visit_%s" % operator.__name__, None) - if disp: - return disp(element, **kw) - else: - return fn(OPERATORS[operator]) - else: - return fn(" " + operator + " ") + + ( + ' ESCAPE ' + + self.render_literal_value(escape, sqltypes.STRINGTYPE) + if escape else '' + ) def visit_bindparam(self, bindparam, within_columns_clause=False, - literal_binds=False, **kwargs): + literal_binds=False, + skip_bind_expression=False, + **kwargs): + + if not skip_bind_expression and bindparam.type._has_bind_expression: + bind_expression = bindparam.type.bind_expression(bindparam) + return self.process(bind_expression, + skip_bind_expression=True) if literal_binds or \ (within_columns_clause and \ self.ansi_bind_rules): if bindparam.value is None: - raise exc.CompileError("Bind parameter without a " - "renderable value not allowed here.") + raise exc.CompileError("Bind parameter '%s' without a " + "renderable value not allowed here." + % bindparam.key) return self.render_literal_bindparam(bindparam, within_columns_clause=True, **kwargs) @@ -682,14 +982,14 @@ class SQLCompiler(engine.Compiled): existing = self.binds[name] if existing is not bindparam: if (existing.unique or bindparam.unique) and \ - not existing.proxy_set.intersection(bindparam.proxy_set): + not existing.proxy_set.intersection( + bindparam.proxy_set): raise exc.CompileError( "Bind parameter '%s' conflicts with " "unique bind parameter of the same name" % bindparam.key ) - elif getattr(existing, '_is_crud', False) or \ - getattr(bindparam, '_is_crud', False): + elif existing._is_crud or bindparam._is_crud: raise exc.CompileError( "bindparam() name '%s' is reserved " "for automatic usage in the VALUES or SET " @@ -706,9 +1006,6 @@ class SQLCompiler(engine.Compiled): def render_literal_bindparam(self, bindparam, **kw): value = bindparam.value - processor = bindparam.type._cached_bind_processor(self.dialect) - if processor: - value = processor(value) return self.render_literal_value(value, bindparam.type) def render_literal_value(self, value, type_): @@ -721,15 +1018,10 @@ class SQLCompiler(engine.Compiled): of the DBAPI. """ - if isinstance(value, basestring): - value = value.replace("'", "''") - return "'%s'" % value - elif value is None: - return "NULL" - elif isinstance(value, (float, int, long)): - return repr(value) - elif isinstance(value, decimal.Decimal): - return str(value) + + processor = type_._cached_literal_processor(self.dialect) + if processor: + return processor(value) else: raise NotImplementedError( "Don't know how to literal-quote value %r" % value) @@ -739,7 +1031,7 @@ class SQLCompiler(engine.Compiled): return self.bind_names[bindparam] bind_name = bindparam.key - if isinstance(bind_name, sql._truncated_label): + if isinstance(bind_name, elements._truncated_label): bind_name = self._truncated_identifier("bindparam", bind_name) # add to bind_names for translation @@ -778,7 +1070,7 @@ class SQLCompiler(engine.Compiled): positional_names.append(name) else: self.positiontup.append(name) - return self.bindtemplate % {'name':name} + return self.bindtemplate % {'name': name} def visit_cte(self, cte, asfrom=False, ashint=False, fromhints=None, @@ -787,7 +1079,7 @@ class SQLCompiler(engine.Compiled): if self.positional: kwargs['positional_names'] = self.cte_positional - if isinstance(cte.name, sql._truncated_label): + if isinstance(cte.name, elements._truncated_label): cte_name = self._truncated_identifier("alias", cte.name) else: cte_name = cte.name @@ -797,7 +1089,7 @@ class SQLCompiler(engine.Compiled): # we've generated a same-named CTE that we are enclosed in, # or this is the same CTE. just return the name. if cte in existing_cte._restates or cte is existing_cte: - return cte_name + return self.preparer.format_alias(cte, cte_name) elif existing_cte in cte._restates: # we've generated a same-named CTE that is # enclosed in us - we take precedence, so @@ -811,19 +1103,24 @@ class SQLCompiler(engine.Compiled): self.ctes_by_name[cte_name] = cte - if cte.cte_alias: - if isinstance(cte.cte_alias, sql._truncated_label): - cte_alias = self._truncated_identifier("alias", cte.cte_alias) - else: - cte_alias = cte.cte_alias - if not cte.cte_alias and cte not in self.ctes: + if cte._cte_alias is not None: + orig_cte = cte._cte_alias + if orig_cte not in self.ctes: + self.visit_cte(orig_cte) + cte_alias_name = cte._cte_alias.name + if isinstance(cte_alias_name, elements._truncated_label): + cte_alias_name = self._truncated_identifier("alias", cte_alias_name) + else: + orig_cte = cte + cte_alias_name = None + if not cte_alias_name and cte not in self.ctes: if cte.recursive: self.ctes_recursive = True text = self.preparer.format_alias(cte, cte_name) if cte.recursive: - if isinstance(cte.original, sql.Select): + if isinstance(cte.original, selectable.Select): col_source = cte.original - elif isinstance(cte.original, sql.CompoundSelect): + elif isinstance(cte.original, selectable.CompoundSelect): col_source = cte.original.selects[0] else: assert False @@ -839,18 +1136,20 @@ class SQLCompiler(engine.Compiled): self, asfrom=True, **kwargs ) self.ctes[cte] = text + if asfrom: - if cte.cte_alias: - text = self.preparer.format_alias(cte, cte_alias) + if cte_alias_name: + text = self.preparer.format_alias(cte, cte_alias_name) text += " AS " + cte_name else: return self.preparer.format_alias(cte, cte_name) return text def visit_alias(self, alias, asfrom=False, ashint=False, + iscrud=False, fromhints=None, **kwargs): if asfrom or ashint: - if isinstance(alias.name, sql._truncated_label): + if isinstance(alias.name, elements._truncated_label): alias_name = self._truncated_identifier("alias", alias.name) else: alias_name = alias.name @@ -864,44 +1163,104 @@ class SQLCompiler(engine.Compiled): self.preparer.format_alias(alias, alias_name) if fromhints and alias in fromhints: - hinttext = self.get_from_hint_text(alias, fromhints[alias]) - if hinttext: - ret += " " + hinttext + ret = self.format_from_hint_text(ret, alias, + fromhints[alias], iscrud) return ret else: return alias.original._compiler_dispatch(self, **kwargs) - def label_select_column(self, select, column, asfrom): - """label columns present in a select().""" + def _add_to_result_map(self, keyname, name, objects, type_): + if not self.dialect.case_sensitive: + keyname = keyname.lower() - if isinstance(column, sql._Label): - return column + if keyname in self.result_map: + # conflicting keyname, just double up the list + # of objects. this will cause an "ambiguous name" + # error if an attempt is made by the result set to + # access. + e_name, e_obj, e_type = self.result_map[keyname] + self.result_map[keyname] = e_name, e_obj + objects, e_type + else: + self.result_map[keyname] = name, objects, type_ - elif select is not None and \ - select.use_labels and \ - column._label: - return _CompileLabel( - column, - column._label, - alt_names=(column._key_label, ) + def _label_select_column(self, select, column, + populate_result_map, + asfrom, column_clause_args, + name=None, + within_columns_clause=True): + """produce labeled columns present in a select().""" + + if column.type._has_column_expression and \ + populate_result_map: + col_expr = column.type.column_expression(column) + add_to_result_map = lambda keyname, name, objects, type_: \ + self._add_to_result_map( + keyname, name, + objects + (column,), type_) + else: + col_expr = column + if populate_result_map: + add_to_result_map = self._add_to_result_map + else: + add_to_result_map = None + + if not within_columns_clause: + result_expr = col_expr + elif isinstance(column, elements.Label): + if col_expr is not column: + result_expr = _CompileLabel( + col_expr, + column.name, + alt_names=(column.element,) + ) + else: + result_expr = col_expr + + elif select is not None and name: + result_expr = _CompileLabel( + col_expr, + name, + alt_names=(column._key_label,) ) elif \ asfrom and \ - isinstance(column, sql.ColumnClause) and \ + isinstance(column, elements.ColumnClause) and \ not column.is_literal and \ column.table is not None and \ - not isinstance(column.table, sql.Select): - return _CompileLabel(column, sql._as_truncated(column.name), - alt_names=(column.key,)) + not isinstance(column.table, selectable.Select): + result_expr = _CompileLabel(col_expr, + elements._as_truncated(column.name), + alt_names=(column.key,)) elif not isinstance(column, - (sql._UnaryExpression, sql._TextClause)) \ + (elements.UnaryExpression, elements.TextClause)) \ and (not hasattr(column, 'name') or \ - isinstance(column, sql.Function)): - return _CompileLabel(column, column.anon_label) + isinstance(column, functions.Function)): + result_expr = _CompileLabel(col_expr, column.anon_label) + elif col_expr is not column: + # TODO: are we sure "column" has a .name and .key here ? + # assert isinstance(column, elements.ColumnClause) + result_expr = _CompileLabel(col_expr, + elements._as_truncated(column.name), + alt_names=(column.key,)) else: - return column + result_expr = col_expr + + column_clause_args.update( + within_columns_clause=within_columns_clause, + add_to_result_map=add_to_result_map + ) + return result_expr._compiler_dispatch( + self, + **column_clause_args + ) + + def format_from_hint_text(self, sqltext, table, hint, iscrud): + hinttext = self.get_from_hint_text(table, hint) + if hinttext: + sqltext += " " + hinttext + return sqltext def get_select_hint_text(self, byfroms): return None @@ -912,46 +1271,206 @@ class SQLCompiler(engine.Compiled): def get_crud_hint_text(self, table, text): return None + def _transform_select_for_nested_joins(self, select): + """Rewrite any "a JOIN (b JOIN c)" expression as + "a JOIN (select * from b JOIN c) AS anon", to support + databases that can't parse a parenthesized join correctly + (i.e. sqlite the main one). + + """ + cloned = {} + column_translate = [{}] + + # TODO: should we be using isinstance() for this, + # as this whole system won't work for custom Join/Select + # subclasses where compilation routines + # call down to compiler.visit_join(), compiler.visit_select() + join_name = selectable.Join.__visit_name__ + select_name = selectable.Select.__visit_name__ + + def visit(element, **kw): + if element in column_translate[-1]: + return column_translate[-1][element] + + elif element in cloned: + return cloned[element] + + newelem = cloned[element] = element._clone() + + if newelem.__visit_name__ is join_name and \ + isinstance(newelem.right, selectable.FromGrouping): + + newelem._reset_exported() + newelem.left = visit(newelem.left, **kw) + + right = visit(newelem.right, **kw) + + selectable_ = selectable.Select( + [right.element], + use_labels=True).alias() + + for c in selectable_.c: + c._key_label = c.key + c._label = c.name + + translate_dict = dict( + zip(newelem.right.element.c, selectable_.c) + ) + + translate_dict[right.element.left] = selectable_ + translate_dict[right.element.right] = selectable_ + + # propagate translations that we've gained + # from nested visit(newelem.right) outwards + # to the enclosing select here. this happens + # only when we have more than one level of right + # join nesting, i.e. "a JOIN (b JOIN (c JOIN d))" + for k, v in list(column_translate[-1].items()): + if v in translate_dict: + # remarkably, no current ORM tests (May 2013) + # hit this condition, only test_join_rewriting + # does. + column_translate[-1][k] = translate_dict[v] + + column_translate[-1].update(translate_dict) + + newelem.right = selectable_ + + newelem.onclause = visit(newelem.onclause, **kw) + elif newelem.__visit_name__ is select_name: + column_translate.append({}) + newelem._copy_internals(clone=visit, **kw) + del column_translate[-1] + else: + newelem._copy_internals(clone=visit, **kw) + + return newelem + + return visit(select) + + def _transform_result_map_for_nested_joins(self, select, transformed_select): + inner_col = dict((c._key_label, c) for + c in transformed_select.inner_columns) + + d = dict( + (inner_col[c._key_label], c) + for c in select.inner_columns + ) + for key, (name, objs, typ) in list(self.result_map.items()): + objs = tuple([d.get(col, col) for col in objs]) + self.result_map[key] = (name, objs, typ) + + + _default_stack_entry = util.immutabledict([ + ('iswrapper', False), + ('correlate_froms', frozenset()), + ('asfrom_froms', frozenset()) + ]) + + def _display_froms_for_select(self, select, asfrom): + # utility method to help external dialects + # get the correct from list for a select. + # specifically the oracle dialect needs this feature + # right now. + toplevel = not self.stack + entry = self._default_stack_entry if toplevel else self.stack[-1] + + correlate_froms = entry['correlate_froms'] + asfrom_froms = entry['asfrom_froms'] + + if asfrom: + froms = select._get_display_froms( + explicit_correlate_froms=\ + correlate_froms.difference(asfrom_froms), + implicit_correlate_froms=()) + else: + froms = select._get_display_froms( + explicit_correlate_froms=correlate_froms, + implicit_correlate_froms=asfrom_froms) + return froms + def visit_select(self, select, asfrom=False, parens=True, iswrapper=False, fromhints=None, compound_index=0, - positional_names=None, **kwargs): + force_result_map=False, + positional_names=None, + nested_join_translation=False, + **kwargs): - entry = self.stack and self.stack[-1] or {} + needs_nested_translation = \ + select.use_labels and \ + not nested_join_translation and \ + not self.stack and \ + not self.dialect.supports_right_nested_joins - existingfroms = entry.get('from', None) + if needs_nested_translation: + transformed_select = self._transform_select_for_nested_joins(select) + text = self.visit_select( + transformed_select, asfrom=asfrom, parens=parens, + iswrapper=iswrapper, fromhints=fromhints, + compound_index=compound_index, + force_result_map=force_result_map, + positional_names=positional_names, + nested_join_translation=True, **kwargs + ) - froms = select._get_display_froms(existingfroms) + toplevel = not self.stack + entry = self._default_stack_entry if toplevel else self.stack[-1] - correlate_froms = set(sql._from_objects(*froms)) - # TODO: might want to propagate existing froms for - # select(select(select)) where innermost select should correlate - # to outermost if existingfroms: correlate_froms = - # correlate_froms.union(existingfroms) + populate_result_map = force_result_map or ( + compound_index == 0 and ( + toplevel or \ + entry['iswrapper'] + ) + ) - populate_result_map = compound_index == 0 and ( - not entry or \ - entry.get('iswrapper', False) - ) + if needs_nested_translation: + if populate_result_map: + self._transform_result_map_for_nested_joins( + select, transformed_select) + return text - self.stack.append({'from': correlate_froms, 'iswrapper': iswrapper}) + correlate_froms = entry['correlate_froms'] + asfrom_froms = entry['asfrom_froms'] - if populate_result_map: - column_clause_args = {'result_map': self.result_map, - 'positional_names': positional_names} + if asfrom: + froms = select._get_display_froms( + explicit_correlate_froms= + correlate_froms.difference(asfrom_froms), + implicit_correlate_froms=()) else: - column_clause_args = {'positional_names': positional_names} + froms = select._get_display_froms( + explicit_correlate_froms=correlate_froms, + implicit_correlate_froms=asfrom_froms) + + new_correlate_froms = set(selectable._from_objects(*froms)) + all_correlate_froms = new_correlate_froms.union(correlate_froms) + + new_entry = { + 'asfrom_froms': new_correlate_froms, + 'iswrapper': iswrapper, + 'correlate_froms': all_correlate_froms + } + self.stack.append(new_entry) + + column_clause_args = kwargs.copy() + column_clause_args.update({ + 'positional_names': positional_names, + 'within_label_clause': False, + 'within_columns_clause': False + }) # the actual list of columns to print in the SELECT column list. inner_columns = [ c for c in [ - self.label_select_column(select, co, asfrom=asfrom).\ - _compiler_dispatch(self, - within_columns_clause=True, - **column_clause_args) - for co in util.unique_list(select.inner_columns) - ] + self._label_select_column(select, + column, + populate_result_map, asfrom, + column_clause_args, + name=name) + for name, column in select._columns_plus_names + ] if c is not None ] @@ -964,7 +1483,7 @@ class SQLCompiler(engine.Compiled): self, ashint=True) }) for (from_, dialect), hinttext in - select._hints.iteritems() + select._hints.items() if dialect in ('*', self.dialect.name) ]) hint_text = self.get_select_hint_text(byfrom) @@ -972,9 +1491,8 @@ class SQLCompiler(engine.Compiled): text += hint_text + " " if select._prefixes: - text += " ".join( - x._compiler_dispatch(self, **kwargs) - for x in select._prefixes) + " " + text += self._generate_prefixes(select, select._prefixes, **kwargs) + text += self.get_select_precolumns(select) text += ', '.join(inner_columns) @@ -1010,15 +1528,23 @@ class SQLCompiler(engine.Compiled): text += " \nHAVING " + t if select._order_by_clause.clauses: - text += self.order_by_clause(select, **kwargs) + if self.dialect.supports_simple_order_by_label: + order_by_select = select + else: + order_by_select = None + + text += self.order_by_clause(select, + order_by_select=order_by_select, **kwargs) + if select._limit is not None or select._offset is not None: text += self.limit_clause(select) - if select.for_update: + + if select._for_update_arg is not None: text += self.for_update_clause(select) if self.ctes and \ - compound_index == 0 and not entry: - text = self._render_cte_clause() + text + compound_index == 0 and toplevel: + text = self._render_cte_clause() + text self.stack.pop(-1) @@ -1027,6 +1553,17 @@ class SQLCompiler(engine.Compiled): else: return text + def _generate_prefixes(self, stmt, prefixes, **kw): + clause = " ".join( + prefix._compiler_dispatch(self, **kw) + for prefix, dialect_name in prefixes + if dialect_name is None or + dialect_name == self.dialect.name + ) + if clause: + clause += " " + return clause + def _render_cte_clause(self): if self.positional: self.positiontup = self.cte_positional + self.positiontup @@ -1058,35 +1595,34 @@ class SQLCompiler(engine.Compiled): return "" def for_update_clause(self, select): - if select.for_update: - return " FOR UPDATE" - else: - return "" + return " FOR UPDATE" + + def returning_clause(self, stmt, returning_cols): + raise exc.CompileError( + "RETURNING is not supported by this " + "dialect's statement compiler.") def limit_clause(self, select): text = "" if select._limit is not None: - text += "\n LIMIT " + self.process(sql.literal(select._limit)) + text += "\n LIMIT " + self.process(elements.literal(select._limit)) if select._offset is not None: if select._limit is None: text += "\n LIMIT -1" - text += " OFFSET " + self.process(sql.literal(select._offset)) + text += " OFFSET " + self.process(elements.literal(select._offset)) return text - def visit_table(self, table, asfrom=False, ashint=False, + def visit_table(self, table, asfrom=False, iscrud=False, ashint=False, fromhints=None, **kwargs): if asfrom or ashint: if getattr(table, "schema", None): - ret = self.preparer.quote_schema(table.schema, - table.quote_schema) + \ - "." + self.preparer.quote(table.name, - table.quote) + ret = self.preparer.quote_schema(table.schema) + \ + "." + self.preparer.quote(table.name) else: - ret = self.preparer.quote(table.name, table.quote) + ret = self.preparer.quote(table.name) if fromhints and table in fromhints: - hinttext = self.get_from_hint_text(table, fromhints[table]) - if hinttext: - ret += " " + hinttext + ret = self.format_from_hint_text(ret, table, + fromhints[table], iscrud) return ret else: return "" @@ -1100,28 +1636,40 @@ class SQLCompiler(engine.Compiled): join.onclause._compiler_dispatch(self, **kwargs) ) - def visit_insert(self, insert_stmt): + def visit_insert(self, insert_stmt, **kw): self.isinsert = True - colparams = self._get_colparams(insert_stmt) + colparams = self._get_colparams(insert_stmt, **kw) if not colparams and \ not self.dialect.supports_default_values and \ not self.dialect.supports_empty_insert: - raise exc.CompileError("The version of %s you are using does " - "not support empty inserts." % + raise exc.CompileError("The '%s' dialect with current database " + "version settings does not support empty " + "inserts." % self.dialect.name) + if insert_stmt._has_multi_parameters: + if not self.dialect.supports_multivalues_insert: + raise exc.CompileError("The '%s' dialect with current database " + "version settings does not support " + "in-place multirow inserts." % + self.dialect.name) + colparams_single = colparams[0] + else: + colparams_single = colparams + + preparer = self.preparer supports_default_values = self.dialect.supports_default_values - text = "INSERT" + text = "INSERT " + if insert_stmt._prefixes: + text += self._generate_prefixes(insert_stmt, + insert_stmt._prefixes, **kw) - prefixes = [self.process(x) for x in insert_stmt._prefixes] - if prefixes: - text += " " + " ".join(prefixes) - - text += " INTO " + preparer.format_table(insert_stmt.table) + text += "INTO " + table_text = preparer.format_table(insert_stmt.table) if insert_stmt._hints: dialect_hints = dict([ @@ -1131,14 +1679,18 @@ class SQLCompiler(engine.Compiled): if dialect in ('*', self.dialect.name) ]) if insert_stmt.table in dialect_hints: - text += " " + self.get_crud_hint_text( + table_text = self.format_from_hint_text( + table_text, insert_stmt.table, - dialect_hints[insert_stmt.table] + dialect_hints[insert_stmt.table], + True ) - if colparams or not supports_default_values: + text += table_text + + if colparams_single or not supports_default_values: text += " (%s)" % ', '.join([preparer.format_column(c[0]) - for c in colparams]) + for c in colparams_single]) if self.returning or insert_stmt._returning: self.returning = self.returning or insert_stmt._returning @@ -1148,8 +1700,19 @@ class SQLCompiler(engine.Compiled): if self.returning_precedes_values: text += " " + returning_clause - if not colparams and supports_default_values: + if insert_stmt.select is not None: + text += " %s" % self.process(insert_stmt.select, **kw) + elif not colparams and supports_default_values: text += " DEFAULT VALUES" + elif insert_stmt._has_multi_parameters: + text += " VALUES %s" % ( + ", ".join( + "(%s)" % ( + ', '.join(c[1] for c in colparam_set) + ) + for colparam_set in colparams + ) + ) else: text += " VALUES (%s)" % \ ', '.join([c[1] for c in colparams]) @@ -1171,7 +1734,8 @@ class SQLCompiler(engine.Compiled): MySQL overrides this. """ - return self.preparer.format_table(from_table) + return from_table._compiler_dispatch(self, asfrom=True, + iscrud=True, **kw) def update_from_clause(self, update_stmt, from_table, extra_froms, @@ -1189,18 +1753,25 @@ class SQLCompiler(engine.Compiled): for t in extra_froms) def visit_update(self, update_stmt, **kw): - self.stack.append({'from': set([update_stmt.table])}) + self.stack.append( + {'correlate_froms': set([update_stmt.table]), + "iswrapper": False, + "asfrom_froms": set([update_stmt.table])}) self.isupdate = True extra_froms = update_stmt._extra_froms - colparams = self._get_colparams(update_stmt, extra_froms) + text = "UPDATE " - text = "UPDATE " + self.update_tables_clause( - update_stmt, - update_stmt.table, - extra_froms, **kw) + if update_stmt._prefixes: + text += self._generate_prefixes(update_stmt, + update_stmt._prefixes, **kw) + + table_text = self.update_tables_clause(update_stmt, update_stmt.table, + extra_froms, **kw) + + colparams = self._get_colparams(update_stmt, extra_froms, **kw) if update_stmt._hints: dialect_hints = dict([ @@ -1210,30 +1781,32 @@ class SQLCompiler(engine.Compiled): if dialect in ('*', self.dialect.name) ]) if update_stmt.table in dialect_hints: - text += " " + self.get_crud_hint_text( + table_text = self.format_from_hint_text( + table_text, update_stmt.table, - dialect_hints[update_stmt.table] + dialect_hints[update_stmt.table], + True ) else: dialect_hints = None - text += ' SET ' - if extra_froms and self.render_table_with_column_in_update_from: - text += ', '.join( - self.visit_column(c[0]) + - '=' + c[1] for c in colparams - ) - else: - text += ', '.join( - self.preparer.quote(c[0].name, c[0].quote) + - '=' + c[1] for c in colparams - ) + text += table_text - if update_stmt._returning: - self.returning = update_stmt._returning + text += ' SET ' + include_table = extra_froms and \ + self.render_table_with_column_in_update_from + text += ', '.join( + c[0]._compiler_dispatch(self, + include_table=include_table) + + '=' + c[1] for c in colparams + ) + + if self.returning or update_stmt._returning: + if not self.returning: + self.returning = update_stmt._returning if self.returning_precedes_values: text += " " + self.returning_clause( - update_stmt, update_stmt._returning) + update_stmt, self.returning) if extra_froms: extra_from_text = self.update_from_clause( @@ -1253,20 +1826,21 @@ class SQLCompiler(engine.Compiled): if self.returning and not self.returning_precedes_values: text += " " + self.returning_clause( - update_stmt, update_stmt._returning) + update_stmt, self.returning) self.stack.pop(-1) return text - def _create_crud_bind_param(self, col, value, required=False): - bindparam = sql.bindparam(col.key, value, + def _create_crud_bind_param(self, col, value, required=False, name=None): + if name is None: + name = col.key + bindparam = elements.BindParameter(name, value, type_=col.type, required=required) bindparam._is_crud = True return bindparam._compiler_dispatch(self) - - def _get_colparams(self, stmt, extra_tables=None): + def _get_colparams(self, stmt, extra_tables=None, **kw): """create a set of tuples representing column/string pairs for use in an INSERT or UPDATE statement. @@ -1290,25 +1864,40 @@ class SQLCompiler(engine.Compiled): for c in stmt.table.columns ] - required = object() + if stmt._has_multi_parameters: + stmt_parameters = stmt.parameters[0] + else: + stmt_parameters = stmt.parameters # if we have statement parameters - set defaults in the # compiled params if self.column_keys is None: parameters = {} else: - parameters = dict((sql._column_as_key(key), required) + parameters = dict((elements._column_as_key(key), REQUIRED) for key in self.column_keys - if not stmt.parameters or - key not in stmt.parameters) - - if stmt.parameters is not None: - for k, v in stmt.parameters.iteritems(): - parameters.setdefault(sql._column_as_key(k), v) + if not stmt_parameters or + key not in stmt_parameters) # create a list of column assignment clauses as tuples values = [] + if stmt_parameters is not None: + for k, v in stmt_parameters.items(): + colkey = elements._column_as_key(k) + if colkey is not None: + parameters.setdefault(colkey, v) + else: + # a non-Column expression on the left side; + # add it to values() in an "as-is" state, + # coercing right side to bound param + if elements._is_literal(v): + v = self.process(elements.BindParameter(None, v, type_=k.type), **kw) + else: + v = self.process(v.self_group(), **kw) + + values.append((k, v)) + need_pks = self.isinsert and \ not self.inline and \ not stmt._returning @@ -1317,38 +1906,55 @@ class SQLCompiler(engine.Compiled): self.dialect.implicit_returning and \ stmt.table.implicit_returning + if self.isinsert: + implicit_return_defaults = implicit_returning and stmt._return_defaults + elif self.isupdate: + implicit_return_defaults = self.dialect.implicit_returning and \ + stmt.table.implicit_returning and \ + stmt._return_defaults + + if implicit_return_defaults: + if stmt._return_defaults is True: + implicit_return_defaults = set(stmt.table.c) + else: + implicit_return_defaults = set(stmt._return_defaults) + postfetch_lastrowid = need_pks and self.dialect.postfetch_lastrowid check_columns = {} # special logic that only occurs for multi-table UPDATE # statements - if extra_tables and stmt.parameters: + if extra_tables and stmt_parameters: + normalized_params = dict( + (elements._clause_element_as_expr(c), param) + for c, param in stmt_parameters.items() + ) assert self.isupdate affected_tables = set() for t in extra_tables: for c in t.c: - if c in stmt.parameters: + if c in normalized_params: affected_tables.add(t) check_columns[c.key] = c - value = stmt.parameters[c] - if sql._is_literal(value): + value = normalized_params[c] + if elements._is_literal(value): value = self._create_crud_bind_param( - c, value, required=value is required) + c, value, required=value is REQUIRED) else: self.postfetch.append(c) - value = self.process(value.self_group()) + value = self.process(value.self_group(), **kw) values.append((c, value)) # determine tables which are actually # to be updated - process onupdate and # server_onupdate for these for t in affected_tables: for c in t.c: - if c in stmt.parameters: + if c in normalized_params: continue elif c.onupdate is not None and not c.onupdate.is_sequence: if c.onupdate.is_clause_element: values.append( - (c, self.process(c.onupdate.arg.self_group())) + (c, self.process(c.onupdate.arg.self_group(), **kw)) ) self.postfetch.append(c) else: @@ -1359,21 +1965,42 @@ class SQLCompiler(engine.Compiled): elif c.server_onupdate is not None: self.postfetch.append(c) - # iterating through columns at the top to maintain ordering. - # otherwise we might iterate through individual sets of - # "defaults", "primary key cols", etc. - for c in stmt.table.columns: + if self.isinsert and stmt.select_names: + # for an insert from select, we can only use names that + # are given, so only select for those names. + cols = (stmt.table.c[elements._column_as_key(name)] + for name in stmt.select_names) + else: + # iterate through all table columns to maintain + # ordering, even for those cols that aren't included + cols = stmt.table.columns + + for c in cols: if c.key in parameters and c.key not in check_columns: value = parameters.pop(c.key) - if sql._is_literal(value): + if elements._is_literal(value): value = self._create_crud_bind_param( - c, value, required=value is required) - elif c.primary_key and implicit_returning: - self.returning.append(c) - value = self.process(value.self_group()) + c, value, required=value is REQUIRED, + name=c.key + if not stmt._has_multi_parameters + else "%s_0" % c.key + ) else: - self.postfetch.append(c) - value = self.process(value.self_group()) + if isinstance(value, elements.BindParameter) and \ + value.type._isnull: + value = value._clone() + value.type = c.type + + if c.primary_key and implicit_returning: + self.returning.append(c) + value = self.process(value.self_group(), **kw) + elif implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) + value = self.process(value.self_group(), **kw) + else: + self.postfetch.append(c) + value = self.process(value.self_group(), **kw) values.append((c, value)) elif self.isinsert: @@ -1391,13 +2018,13 @@ class SQLCompiler(engine.Compiled): if self.dialect.supports_sequences and \ (not c.default.optional or \ not self.dialect.sequences_optional): - proc = self.process(c.default) + proc = self.process(c.default, **kw) values.append((c, proc)) self.returning.append(c) elif c.default.is_clause_element: values.append( (c, - self.process(c.default.arg.self_group())) + self.process(c.default.arg.self_group(), **kw)) ) self.returning.append(c) else: @@ -1408,7 +2035,13 @@ class SQLCompiler(engine.Compiled): else: self.returning.append(c) else: - if c.default is not None or \ + if ( + c.default is not None and + ( + not c.default.is_sequence or + self.dialect.supports_sequences + ) + ) or \ c is stmt.table._autoincrement_column and ( self.dialect.supports_sequences or self.dialect.preexecute_autoincrement_sequences @@ -1425,16 +2058,22 @@ class SQLCompiler(engine.Compiled): if self.dialect.supports_sequences and \ (not c.default.optional or \ not self.dialect.sequences_optional): - proc = self.process(c.default) + proc = self.process(c.default, **kw) values.append((c, proc)) - if not c.primary_key: + if implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) + elif not c.primary_key: self.postfetch.append(c) elif c.default.is_clause_element: values.append( - (c, self.process(c.default.arg.self_group())) + (c, self.process(c.default.arg.self_group(), **kw)) ) - if not c.primary_key: + if implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) + elif not c.primary_key: # dont add primary key column to postfetch self.postfetch.append(c) else: @@ -1443,41 +2082,87 @@ class SQLCompiler(engine.Compiled): ) self.prefetch.append(c) elif c.server_default is not None: - if not c.primary_key: + if implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) + elif not c.primary_key: self.postfetch.append(c) + elif implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) elif self.isupdate: if c.onupdate is not None and not c.onupdate.is_sequence: if c.onupdate.is_clause_element: values.append( - (c, self.process(c.onupdate.arg.self_group())) + (c, self.process(c.onupdate.arg.self_group(), **kw)) ) - self.postfetch.append(c) + if implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) + else: + self.postfetch.append(c) else: values.append( (c, self._create_crud_bind_param(c, None)) ) self.prefetch.append(c) elif c.server_onupdate is not None: - self.postfetch.append(c) + if implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) + else: + self.postfetch.append(c) + elif implicit_return_defaults and \ + c in implicit_return_defaults: + self.returning.append(c) - if parameters and stmt.parameters: + if parameters and stmt_parameters: check = set(parameters).intersection( - sql._column_as_key(k) for k in stmt.parameters + elements._column_as_key(k) for k in stmt.parameters ).difference(check_columns) if check: - util.warn( + raise exc.CompileError( "Unconsumed column names: %s" % (", ".join(check)) ) + if stmt._has_multi_parameters: + values_0 = values + values = [values] + + values.extend( + [ + ( + c, + self._create_crud_bind_param( + c, row[c.key], + name="%s_%d" % (c.key, i + 1) + ) + if c.key in row else param + ) + for (c, param) in values_0 + ] + for i, row in enumerate(stmt.parameters[1:]) + ) + return values - def visit_delete(self, delete_stmt): - self.stack.append({'from': set([delete_stmt.table])}) + def visit_delete(self, delete_stmt, **kw): + self.stack.append({'correlate_froms': set([delete_stmt.table]), + "iswrapper": False, + "asfrom_froms": set([delete_stmt.table])}) self.isdelete = True - text = "DELETE FROM " + self.preparer.format_table(delete_stmt.table) + text = "DELETE " + + if delete_stmt._prefixes: + text += self._generate_prefixes(delete_stmt, + delete_stmt._prefixes, **kw) + + text += "FROM " + table_text = delete_stmt.table._compiler_dispatch(self, + asfrom=True, iscrud=True) if delete_stmt._hints: dialect_hints = dict([ @@ -1487,13 +2172,18 @@ class SQLCompiler(engine.Compiled): if dialect in ('*', self.dialect.name) ]) if delete_stmt.table in dialect_hints: - text += " " + self.get_crud_hint_text( + table_text = self.format_from_hint_text( + table_text, delete_stmt.table, - dialect_hints[delete_stmt.table] + dialect_hints[delete_stmt.table], + True ) + else: dialect_hints = None + text += table_text + if delete_stmt._returning: self.returning = delete_stmt._returning if self.returning_precedes_values: @@ -1501,7 +2191,8 @@ class SQLCompiler(engine.Compiled): delete_stmt, delete_stmt._returning) if delete_stmt._whereclause is not None: - text += " WHERE " + self.process(delete_stmt._whereclause) + text += " WHERE " + text += delete_stmt._whereclause._compiler_dispatch(self) if self.returning and not self.returning_precedes_values: text += " " + self.returning_clause( @@ -1523,12 +2214,16 @@ class SQLCompiler(engine.Compiled): self.preparer.format_savepoint(savepoint_stmt) -class DDLCompiler(engine.Compiled): +class DDLCompiler(Compiled): @util.memoized_property def sql_compiler(self): return self.dialect.statement_compiler(self.dialect, None) + @util.memoized_property + def type_compiler(self): + return self.dialect.type_compiler + @property def preparer(self): return self.dialect.identifier_preparer @@ -1556,10 +2251,12 @@ class DDLCompiler(engine.Compiled): return self.sql_compiler.post_process_text(ddl.statement % context) def visit_create_schema(self, create): - return "CREATE SCHEMA " + self.preparer.format_schema(create.element, create.quote) + schema = self.preparer.format_schema(create.element) + return "CREATE SCHEMA " + schema def visit_drop_schema(self, drop): - text = "DROP SCHEMA " + self.preparer.format_schema(drop.element, drop.quote) + schema = self.preparer.format_schema(drop.element) + text = "DROP SCHEMA " + schema if drop.cascade: text += " CASCADE" return text @@ -1577,37 +2274,25 @@ class DDLCompiler(engine.Compiled): # if only one primary key, specify it along with the column first_pk = False - for column in table.columns: + for create_column in create.columns: + column = create_column.element try: - text += separator - separator = ", \n" - text += "\t" + self.get_column_specification( - column, - first_pk=column.primary_key and \ - not first_pk - ) + processed = self.process(create_column, + first_pk=column.primary_key + and not first_pk) + if processed is not None: + text += separator + separator = ", \n" + text += "\t" + processed if column.primary_key: first_pk = True - const = " ".join(self.process(constraint) \ - for constraint in column.constraints) - if const: - text += " " + const - except exc.CompileError, ce: - # Py3K - #raise exc.CompileError("(in table '%s', column '%s'): %s" - # % ( - # table.description, - # column.name, - # ce.args[0] - # )) from ce - # Py2K - raise exc.CompileError("(in table '%s', column '%s'): %s" - % ( + except exc.CompileError as ce: + util.raise_from_cause( + exc.CompileError(util.u("(in table '%s', column '%s'): %s") % ( table.description, column.name, ce.args[0] - )), None, sys.exc_info()[2] - # end Py2K + ))) const = self.create_table_constraints(table) if const: @@ -1616,6 +2301,23 @@ class DDLCompiler(engine.Compiled): text += "\n)%s\n\n" % self.post_create_table(table) return text + def visit_create_column(self, create, first_pk=False): + column = create.element + + if column.system: + return None + + text = self.get_column_specification( + column, + first_pk=first_pk + ) + const = " ".join(self.process(constraint) \ + for constraint in column.constraints) + if const: + text += " " + const + + return text + def create_table_constraints(self, table): # On some DB order is significant: visit PK first, then the @@ -1642,51 +2344,66 @@ class DDLCompiler(engine.Compiled): def visit_drop_table(self, drop): return "\nDROP TABLE " + self.preparer.format_table(drop.element) - def _index_identifier(self, ident): - if isinstance(ident, sql._truncated_label): - max = self.dialect.max_index_name_length or \ - self.dialect.max_identifier_length - if len(ident) > max: - ident = ident[0:max - 8] + \ - "_" + util.md5_hex(ident)[-4:] - else: - self.dialect.validate_identifier(ident) + def visit_drop_view(self, drop): + return "\nDROP VIEW " + self.preparer.format_table(drop.element) - return ident - def visit_create_index(self, create): + def _verify_index_table(self, index): + if index.table is None: + raise exc.CompileError("Index '%s' is not associated " + "with any table." % index.name) + + + def visit_create_index(self, create, include_schema=False, + include_table_schema=True): index = create.element + self._verify_index_table(index) preparer = self.preparer text = "CREATE " if index.unique: text += "UNIQUE " text += "INDEX %s ON %s (%s)" \ - % (preparer.quote(self._index_identifier(index.name), - index.quote), - preparer.format_table(index.table), - ', '.join(preparer.quote(c.name, c.quote) - for c in index.columns)) + % ( + self._prepared_index_name(index, + include_schema=include_schema), + preparer.format_table(index.table, + use_schema=include_table_schema), + ', '.join( + self.sql_compiler.process(expr, + include_table=False, literal_binds=True) for + expr in index.expressions) + ) return text def visit_drop_index(self, drop): index = drop.element - if index.table is not None and index.table.schema: + return "\nDROP INDEX " + self._prepared_index_name(index, + include_schema=True) + + def _prepared_index_name(self, index, include_schema=False): + if include_schema and index.table is not None and index.table.schema: schema = index.table.schema - schema_name = self.preparer.quote_schema(schema, - index.table.quote_schema) + schema_name = self.preparer.quote_schema(schema) else: schema_name = None - index_name = self.preparer.quote( - self._index_identifier(index.name), - index.quote) + ident = index.name + if isinstance(ident, elements._truncated_label): + max_ = self.dialect.max_index_name_length or \ + self.dialect.max_identifier_length + if len(ident) > max_: + ident = ident[0:max_ - 8] + \ + "_" + util.md5_hex(ident)[-4:] + else: + self.dialect.validate_identifier(ident) + + index_name = self.preparer.quote(ident) if schema_name: index_name = schema_name + "." + index_name - return "\nDROP INDEX " + index_name + return index_name def visit_add_constraint(self, create): - preparer = self.preparer return "ALTER TABLE %s ADD %s" % ( self.preparer.format_table(create.element.table), self.process(create.element) @@ -1706,7 +2423,6 @@ class DDLCompiler(engine.Compiled): self.preparer.format_sequence(drop.element) def visit_drop_constraint(self, drop): - preparer = self.preparer return "ALTER TABLE %s DROP CONSTRAINT %s%s" % ( self.preparer.format_table(drop.element.table), self.preparer.format_constraint(drop.element), @@ -1729,7 +2445,7 @@ class DDLCompiler(engine.Compiled): def get_column_default_string(self, column): if isinstance(column.server_default, schema.DefaultClause): - if isinstance(column.server_default.arg, basestring): + if isinstance(column.server_default.arg, util.string_types): return "'%s'" % column.server_default.arg else: return self.sql_compiler.process(column.server_default.arg) @@ -1741,8 +2457,9 @@ class DDLCompiler(engine.Compiled): if constraint.name is not None: text += "CONSTRAINT %s " % \ self.preparer.format_constraint(constraint) - sqltext = sql_util.expression_as_ddl(constraint.sqltext) - text += "CHECK (%s)" % self.sql_compiler.process(sqltext) + text += "CHECK (%s)" % self.sql_compiler.process(constraint.sqltext, + include_table=False, + literal_binds=True) text += self.define_constraint_deferrability(constraint) return text @@ -1763,7 +2480,7 @@ class DDLCompiler(engine.Compiled): text += "CONSTRAINT %s " % \ self.preparer.format_constraint(constraint) text += "PRIMARY KEY " - text += "(%s)" % ', '.join(self.preparer.quote(c.name, c.quote) + text += "(%s)" % ', '.join(self.preparer.quote(c.name) for c in constraint) text += self.define_constraint_deferrability(constraint) return text @@ -1776,13 +2493,14 @@ class DDLCompiler(engine.Compiled): preparer.format_constraint(constraint) remote_table = list(constraint._elements.values())[0].column.table text += "FOREIGN KEY(%s) REFERENCES %s (%s)" % ( - ', '.join(preparer.quote(f.parent.name, f.parent.quote) + ', '.join(preparer.quote(f.parent.name) for f in constraint._elements.values()), self.define_constraint_remote_table( constraint, remote_table, preparer), - ', '.join(preparer.quote(f.column.name, f.column.quote) + ', '.join(preparer.quote(f.column.name) for f in constraint._elements.values()) ) + text += self.define_constraint_match(constraint) text += self.define_constraint_cascades(constraint) text += self.define_constraint_deferrability(constraint) return text @@ -1798,7 +2516,7 @@ class DDLCompiler(engine.Compiled): text += "CONSTRAINT %s " % \ self.preparer.format_constraint(constraint) text += "UNIQUE (%s)" % ( - ', '.join(self.preparer.quote(c.name, c.quote) + ', '.join(self.preparer.quote(c.name) for c in constraint)) text += self.define_constraint_deferrability(constraint) return text @@ -1822,13 +2540,14 @@ class DDLCompiler(engine.Compiled): text += " INITIALLY %s" % constraint.initially return text + def define_constraint_match(self, constraint): + text = "" + if constraint.match is not None: + text += " MATCH %s" % constraint.match + return text -class GenericTypeCompiler(engine.TypeCompiler): - def visit_CHAR(self, type_): - return "CHAR" + (type_.length and "(%d)" % type_.length or "") - def visit_NCHAR(self, type_): - return "NCHAR" + (type_.length and "(%d)" % type_.length or "") +class GenericTypeCompiler(TypeCompiler): def visit_FLOAT(self, type_): return "FLOAT" @@ -1845,10 +2564,18 @@ class GenericTypeCompiler(engine.TypeCompiler): else: return "NUMERIC(%(precision)s, %(scale)s)" % \ {'precision': type_.precision, - 'scale' : type_.scale} + 'scale': type_.scale} def visit_DECIMAL(self, type_): - return "DECIMAL" + if type_.precision is None: + return "DECIMAL" + elif type_.scale is None: + return "DECIMAL(%(precision)s)" % \ + {'precision': type_.precision} + else: + return "DECIMAL(%(precision)s, %(scale)s)" % \ + {'precision': type_.precision, + 'scale': type_.scale} def visit_INTEGER(self, type_): return "INTEGER" @@ -1877,11 +2604,29 @@ class GenericTypeCompiler(engine.TypeCompiler): def visit_NCLOB(self, type_): return "NCLOB" + def _render_string_type(self, type_, name): + + text = name + if type_.length: + text += "(%d)" % type_.length + if type_.collation: + text += ' COLLATE "%s"' % type_.collation + return text + + def visit_CHAR(self, type_): + return self._render_string_type(type_, "CHAR") + + def visit_NCHAR(self, type_): + return self._render_string_type(type_, "NCHAR") + def visit_VARCHAR(self, type_): - return "VARCHAR" + (type_.length and "(%d)" % type_.length or "") + return self._render_string_type(type_, "VARCHAR") def visit_NVARCHAR(self, type_): - return "NVARCHAR" + (type_.length and "(%d)" % type_.length or "") + return self._render_string_type(type_, "NVARCHAR") + + def visit_TEXT(self, type_): + return self._render_string_type(type_, "TEXT") def visit_BLOB(self, type_): return "BLOB" @@ -1895,9 +2640,6 @@ class GenericTypeCompiler(engine.TypeCompiler): def visit_BOOLEAN(self, type_): return "BOOLEAN" - def visit_TEXT(self, type_): - return "TEXT" - def visit_large_binary(self, type_): return self.visit_BLOB(type_) @@ -1947,7 +2689,9 @@ class GenericTypeCompiler(engine.TypeCompiler): return self.visit_VARCHAR(type_) def visit_null(self, type_): - raise NotImplementedError("Can't generate DDL for the null type") + raise exc.CompileError("Can't generate DDL for %r; " + "did you forget to specify a " + "type on this Column?" % type_) def visit_type_decorator(self, type_): return self.process(type_.type_engine(self.dialect)) @@ -1955,6 +2699,7 @@ class GenericTypeCompiler(engine.TypeCompiler): def visit_user_defined(self, type_): return type_.get_col_spec() + class IdentifierPreparer(object): """Handle quoting and case-folding of identifiers based on options.""" @@ -2022,18 +2767,28 @@ class IdentifierPreparer(object): lc_value = value.lower() return (lc_value in self.reserved_words or value[0] in self.illegal_initial_characters - or not self.legal_characters.match(unicode(value)) + or not self.legal_characters.match(util.text_type(value)) or (lc_value != value)) - def quote_schema(self, schema, force): - """Quote a schema. + def quote_schema(self, schema, force=None): + """Conditionally quote a schema. + + Subclasses can override this to provide database-dependent + quoting behavior for schema names. + + the 'force' flag should be considered deprecated. - Subclasses should override this to provide database-dependent - quoting behavior. """ return self.quote(schema, force) - def quote(self, ident, force): + def quote(self, ident, force=None): + """Conditionally quote an identifier. + + the 'force' flag should be considered deprecated. + """ + + force = getattr(ident, "quote", None) + if force is None: if ident in self._strings: return self._strings[ident] @@ -2049,38 +2804,35 @@ class IdentifierPreparer(object): return ident def format_sequence(self, sequence, use_schema=True): - name = self.quote(sequence.name, sequence.quote) - if not self.omit_schema and use_schema and \ - sequence.schema is not None: - name = self.quote_schema(sequence.schema, sequence.quote) + \ - "." + name + name = self.quote(sequence.name) + if not self.omit_schema and use_schema and sequence.schema is not None: + name = self.quote_schema(sequence.schema) + "." + name return name def format_label(self, label, name=None): - return self.quote(name or label.name, label.quote) + return self.quote(name or label.name) def format_alias(self, alias, name=None): - return self.quote(name or alias.name, alias.quote) + return self.quote(name or alias.name) def format_savepoint(self, savepoint, name=None): - return self.quote(name or savepoint.ident, savepoint.quote) + return self.quote(name or savepoint.ident) def format_constraint(self, constraint): - return self.quote(constraint.name, constraint.quote) + return self.quote(constraint.name) def format_table(self, table, use_schema=True, name=None): """Prepare a quoted table and schema name.""" if name is None: name = table.name - result = self.quote(name, table.quote) + result = self.quote(name) if not self.omit_schema and use_schema \ and getattr(table, "schema", None): - result = self.quote_schema(table.schema, table.quote_schema) + \ - "." + result + result = self.quote_schema(table.schema) + "." + result return result - def format_schema(self, name, quote): + def format_schema(self, name, quote=None): """Prepare a quoted schema name.""" return self.quote(name, quote) @@ -2095,10 +2847,9 @@ class IdentifierPreparer(object): if use_table: return self.format_table( column.table, use_schema=False, - name=table_name) + "." + \ - self.quote(name, column.quote) + name=table_name) + "." + self.quote(name) else: - return self.quote(name, column.quote) + return self.quote(name) else: # literal textual elements get stuck into ColumnClause a lot, # which shouldn't get quoted @@ -2118,7 +2869,7 @@ class IdentifierPreparer(object): if not self.omit_schema and use_schema and \ getattr(table, 'schema', None): - return (self.quote_schema(table.schema, table.quote_schema), + return (self.quote_schema(table.schema), self.format_table(table, use_schema=False)) else: return (self.format_table(table, use_schema=False), ) @@ -2133,9 +2884,9 @@ class IdentifierPreparer(object): r'(?:' r'(?:%(initial)s((?:%(escaped)s|[^%(final)s])+)%(final)s' r'|([^\.]+))(?=\.|$))+' % - { 'initial': initial, - 'final': final, - 'escaped': escaped_final }) + {'initial': initial, + 'final': final, + 'escaped': escaped_final}) return r def unformat_identifiers(self, identifiers): diff --git a/libs/sqlalchemy/sql/ddl.py b/libs/sqlalchemy/sql/ddl.py new file mode 100644 index 00000000..bda87650 --- /dev/null +++ b/libs/sqlalchemy/sql/ddl.py @@ -0,0 +1,864 @@ +# sql/ddl.py +# Copyright (C) 2009-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +""" +Provides the hierarchy of DDL-defining schema items as well as routines +to invoke them for a create/drop call. + +""" + +from .. import util +from .elements import ClauseElement +from .visitors import traverse +from .base import Executable, _generative, SchemaVisitor, _bind_or_error +from ..util import topological +from .. import event +from .. import exc + +class _DDLCompiles(ClauseElement): + def _compiler(self, dialect, **kw): + """Return a compiler appropriate for this ClauseElement, given a + Dialect.""" + + return dialect.ddl_compiler(dialect, self, **kw) + + +class DDLElement(Executable, _DDLCompiles): + """Base class for DDL expression constructs. + + This class is the base for the general purpose :class:`.DDL` class, + as well as the various create/drop clause constructs such as + :class:`.CreateTable`, :class:`.DropTable`, :class:`.AddConstraint`, + etc. + + :class:`.DDLElement` integrates closely with SQLAlchemy events, + introduced in :ref:`event_toplevel`. An instance of one is + itself an event receiving callable:: + + event.listen( + users, + 'after_create', + AddConstraint(constraint).execute_if(dialect='postgresql') + ) + + .. seealso:: + + :class:`.DDL` + + :class:`.DDLEvents` + + :ref:`event_toplevel` + + :ref:`schema_ddl_sequences` + + """ + + _execution_options = Executable.\ + _execution_options.union({'autocommit': True}) + + target = None + on = None + dialect = None + callable_ = None + + def _execute_on_connection(self, connection, multiparams, params): + return connection._execute_ddl(self, multiparams, params) + + def execute(self, bind=None, target=None): + """Execute this DDL immediately. + + Executes the DDL statement in isolation using the supplied + :class:`.Connectable` or + :class:`.Connectable` assigned to the ``.bind`` + property, if not supplied. If the DDL has a conditional ``on`` + criteria, it will be invoked with None as the event. + + :param bind: + Optional, an ``Engine`` or ``Connection``. If not supplied, a valid + :class:`.Connectable` must be present in the + ``.bind`` property. + + :param target: + Optional, defaults to None. The target SchemaItem for the + execute call. Will be passed to the ``on`` callable if any, + and may also provide string expansion data for the + statement. See ``execute_at`` for more information. + + """ + + if bind is None: + bind = _bind_or_error(self) + + if self._should_execute(target, bind): + return bind.execute(self.against(target)) + else: + bind.engine.logger.info( + "DDL execution skipped, criteria not met.") + + @util.deprecated("0.7", "See :class:`.DDLEvents`, as well as " + ":meth:`.DDLElement.execute_if`.") + def execute_at(self, event_name, target): + """Link execution of this DDL to the DDL lifecycle of a SchemaItem. + + Links this ``DDLElement`` to a ``Table`` or ``MetaData`` instance, + executing it when that schema item is created or dropped. The DDL + statement will be executed using the same Connection and transactional + context as the Table create/drop itself. The ``.bind`` property of + this statement is ignored. + + :param event: + One of the events defined in the schema item's ``.ddl_events``; + e.g. 'before-create', 'after-create', 'before-drop' or 'after-drop' + + :param target: + The Table or MetaData instance for which this DDLElement will + be associated with. + + A DDLElement instance can be linked to any number of schema items. + + ``execute_at`` builds on the ``append_ddl_listener`` interface of + :class:`.MetaData` and :class:`.Table` objects. + + Caveat: Creating or dropping a Table in isolation will also trigger + any DDL set to ``execute_at`` that Table's MetaData. This may change + in a future release. + + """ + + def call_event(target, connection, **kw): + if self._should_execute_deprecated(event_name, + target, connection, **kw): + return connection.execute(self.against(target)) + + event.listen(target, "" + event_name.replace('-', '_'), call_event) + + @_generative + def against(self, target): + """Return a copy of this DDL against a specific schema item.""" + + self.target = target + + @_generative + def execute_if(self, dialect=None, callable_=None, state=None): + """Return a callable that will execute this + DDLElement conditionally. + + Used to provide a wrapper for event listening:: + + event.listen( + metadata, + 'before_create', + DDL("my_ddl").execute_if(dialect='postgresql') + ) + + :param dialect: May be a string, tuple or a callable + predicate. If a string, it will be compared to the name of the + executing database dialect:: + + DDL('something').execute_if(dialect='postgresql') + + If a tuple, specifies multiple dialect names:: + + DDL('something').execute_if(dialect=('postgresql', 'mysql')) + + :param callable_: A callable, which will be invoked with + four positional arguments as well as optional keyword + arguments: + + :ddl: + This DDL element. + + :target: + The :class:`.Table` or :class:`.MetaData` object which is the + target of this event. May be None if the DDL is executed + explicitly. + + :bind: + The :class:`.Connection` being used for DDL execution + + :tables: + Optional keyword argument - a list of Table objects which are to + be created/ dropped within a MetaData.create_all() or drop_all() + method call. + + :state: + Optional keyword argument - will be the ``state`` argument + passed to this function. + + :checkfirst: + Keyword argument, will be True if the 'checkfirst' flag was + set during the call to ``create()``, ``create_all()``, + ``drop()``, ``drop_all()``. + + If the callable returns a true value, the DDL statement will be + executed. + + :param state: any value which will be passed to the callable\_ + as the ``state`` keyword argument. + + .. seealso:: + + :class:`.DDLEvents` + + :ref:`event_toplevel` + + """ + self.dialect = dialect + self.callable_ = callable_ + self.state = state + + def _should_execute(self, target, bind, **kw): + if self.on is not None and \ + not self._should_execute_deprecated(None, target, bind, **kw): + return False + + if isinstance(self.dialect, util.string_types): + if self.dialect != bind.engine.name: + return False + elif isinstance(self.dialect, (tuple, list, set)): + if bind.engine.name not in self.dialect: + return False + if self.callable_ is not None and \ + not self.callable_(self, target, bind, state=self.state, **kw): + return False + + return True + + def _should_execute_deprecated(self, event, target, bind, **kw): + if self.on is None: + return True + elif isinstance(self.on, util.string_types): + return self.on == bind.engine.name + elif isinstance(self.on, (tuple, list, set)): + return bind.engine.name in self.on + else: + return self.on(self, event, target, bind, **kw) + + def __call__(self, target, bind, **kw): + """Execute the DDL as a ddl_listener.""" + + if self._should_execute(target, bind, **kw): + return bind.execute(self.against(target)) + + def _check_ddl_on(self, on): + if (on is not None and + (not isinstance(on, util.string_types + (tuple, list, set)) and + not util.callable(on))): + raise exc.ArgumentError( + "Expected the name of a database dialect, a tuple " + "of names, or a callable for " + "'on' criteria, got type '%s'." % type(on).__name__) + + def bind(self): + if self._bind: + return self._bind + + def _set_bind(self, bind): + self._bind = bind + bind = property(bind, _set_bind) + + def _generate(self): + s = self.__class__.__new__(self.__class__) + s.__dict__ = self.__dict__.copy() + return s + + +class DDL(DDLElement): + """A literal DDL statement. + + Specifies literal SQL DDL to be executed by the database. DDL objects + function as DDL event listeners, and can be subscribed to those events + listed in :class:`.DDLEvents`, using either :class:`.Table` or + :class:`.MetaData` objects as targets. Basic templating support allows + a single DDL instance to handle repetitive tasks for multiple tables. + + Examples:: + + from sqlalchemy import event, DDL + + tbl = Table('users', metadata, Column('uid', Integer)) + event.listen(tbl, 'before_create', DDL('DROP TRIGGER users_trigger')) + + spow = DDL('ALTER TABLE %(table)s SET secretpowers TRUE') + event.listen(tbl, 'after_create', spow.execute_if(dialect='somedb')) + + drop_spow = DDL('ALTER TABLE users SET secretpowers FALSE') + connection.execute(drop_spow) + + When operating on Table events, the following ``statement`` + string substitions are available:: + + %(table)s - the Table name, with any required quoting applied + %(schema)s - the schema name, with any required quoting applied + %(fullname)s - the Table name including schema, quoted if needed + + The DDL's "context", if any, will be combined with the standard + substutions noted above. Keys present in the context will override + the standard substitutions. + + """ + + __visit_name__ = "ddl" + + def __init__(self, statement, on=None, context=None, bind=None): + """Create a DDL statement. + + :param statement: + A string or unicode string to be executed. Statements will be + processed with Python's string formatting operator. See the + ``context`` argument and the ``execute_at`` method. + + A literal '%' in a statement must be escaped as '%%'. + + SQL bind parameters are not available in DDL statements. + + :param on: + .. deprecated:: 0.7 + See :meth:`.DDLElement.execute_if`. + + Optional filtering criteria. May be a string, tuple or a callable + predicate. If a string, it will be compared to the name of the + executing database dialect:: + + DDL('something', on='postgresql') + + If a tuple, specifies multiple dialect names:: + + DDL('something', on=('postgresql', 'mysql')) + + If a callable, it will be invoked with four positional arguments + as well as optional keyword arguments: + + :ddl: + This DDL element. + + :event: + The name of the event that has triggered this DDL, such as + 'after-create' Will be None if the DDL is executed explicitly. + + :target: + The ``Table`` or ``MetaData`` object which is the target of + this event. May be None if the DDL is executed explicitly. + + :connection: + The ``Connection`` being used for DDL execution + + :tables: + Optional keyword argument - a list of Table objects which are to + be created/ dropped within a MetaData.create_all() or drop_all() + method call. + + + If the callable returns a true value, the DDL statement will be + executed. + + :param context: + Optional dictionary, defaults to None. These values will be + available for use in string substitutions on the DDL statement. + + :param bind: + Optional. A :class:`.Connectable`, used by + default when ``execute()`` is invoked without a bind argument. + + + .. seealso:: + + :class:`.DDLEvents` + + :mod:`sqlalchemy.event` + + """ + + if not isinstance(statement, util.string_types): + raise exc.ArgumentError( + "Expected a string or unicode SQL statement, got '%r'" % + statement) + + self.statement = statement + self.context = context or {} + + self._check_ddl_on(on) + self.on = on + self._bind = bind + + def __repr__(self): + return '<%s@%s; %s>' % ( + type(self).__name__, id(self), + ', '.join([repr(self.statement)] + + ['%s=%r' % (key, getattr(self, key)) + for key in ('on', 'context') + if getattr(self, key)])) + + + +class _CreateDropBase(DDLElement): + """Base class for DDL constucts that represent CREATE and DROP or + equivalents. + + The common theme of _CreateDropBase is a single + ``element`` attribute which refers to the element + to be created or dropped. + + """ + + def __init__(self, element, on=None, bind=None): + self.element = element + self._check_ddl_on(on) + self.on = on + self.bind = bind + + def _create_rule_disable(self, compiler): + """Allow disable of _create_rule using a callable. + + Pass to _create_rule using + util.portable_instancemethod(self._create_rule_disable) + to retain serializability. + + """ + return False + + +class CreateSchema(_CreateDropBase): + """Represent a CREATE SCHEMA statement. + + .. versionadded:: 0.7.4 + + The argument here is the string name of the schema. + + """ + + __visit_name__ = "create_schema" + + def __init__(self, name, quote=None, **kw): + """Create a new :class:`.CreateSchema` construct.""" + + self.quote = quote + super(CreateSchema, self).__init__(name, **kw) + + +class DropSchema(_CreateDropBase): + """Represent a DROP SCHEMA statement. + + The argument here is the string name of the schema. + + .. versionadded:: 0.7.4 + + """ + + __visit_name__ = "drop_schema" + + def __init__(self, name, quote=None, cascade=False, **kw): + """Create a new :class:`.DropSchema` construct.""" + + self.quote = quote + self.cascade = cascade + super(DropSchema, self).__init__(name, **kw) + + +class CreateTable(_CreateDropBase): + """Represent a CREATE TABLE statement.""" + + __visit_name__ = "create_table" + + def __init__(self, element, on=None, bind=None): + """Create a :class:`.CreateTable` construct. + + :param element: a :class:`.Table` that's the subject + of the CREATE + :param on: See the description for 'on' in :class:`.DDL`. + :param bind: See the description for 'bind' in :class:`.DDL`. + + """ + super(CreateTable, self).__init__(element, on=on, bind=bind) + self.columns = [CreateColumn(column) + for column in element.columns + ] + + +class _DropView(_CreateDropBase): + """Semi-public 'DROP VIEW' construct. + + Used by the test suite for dialect-agnostic drops of views. + This object will eventually be part of a public "view" API. + + """ + __visit_name__ = "drop_view" + + +class CreateColumn(_DDLCompiles): + """Represent a :class:`.Column` as rendered in a CREATE TABLE statement, + via the :class:`.CreateTable` construct. + + This is provided to support custom column DDL within the generation + of CREATE TABLE statements, by using the + compiler extension documented in :ref:`sqlalchemy.ext.compiler_toplevel` + to extend :class:`.CreateColumn`. + + Typical integration is to examine the incoming :class:`.Column` + object, and to redirect compilation if a particular flag or condition + is found:: + + from sqlalchemy import schema + from sqlalchemy.ext.compiler import compiles + + @compiles(schema.CreateColumn) + def compile(element, compiler, **kw): + column = element.element + + if "special" not in column.info: + return compiler.visit_create_column(element, **kw) + + text = "%s SPECIAL DIRECTIVE %s" % ( + column.name, + compiler.type_compiler.process(column.type) + ) + default = compiler.get_column_default_string(column) + if default is not None: + text += " DEFAULT " + default + + if not column.nullable: + text += " NOT NULL" + + if column.constraints: + text += " ".join( + compiler.process(const) + for const in column.constraints) + return text + + The above construct can be applied to a :class:`.Table` as follows:: + + from sqlalchemy import Table, Metadata, Column, Integer, String + from sqlalchemy import schema + + metadata = MetaData() + + table = Table('mytable', MetaData(), + Column('x', Integer, info={"special":True}, primary_key=True), + Column('y', String(50)), + Column('z', String(20), info={"special":True}) + ) + + metadata.create_all(conn) + + Above, the directives we've added to the :attr:`.Column.info` collection + will be detected by our custom compilation scheme:: + + CREATE TABLE mytable ( + x SPECIAL DIRECTIVE INTEGER NOT NULL, + y VARCHAR(50), + z SPECIAL DIRECTIVE VARCHAR(20), + PRIMARY KEY (x) + ) + + The :class:`.CreateColumn` construct can also be used to skip certain + columns when producing a ``CREATE TABLE``. This is accomplished by + creating a compilation rule that conditionally returns ``None``. + This is essentially how to produce the same effect as using the + ``system=True`` argument on :class:`.Column`, which marks a column + as an implicitly-present "system" column. + + For example, suppose we wish to produce a :class:`.Table` which skips + rendering of the Postgresql ``xmin`` column against the Postgresql backend, + but on other backends does render it, in anticipation of a triggered rule. + A conditional compilation rule could skip this name only on Postgresql:: + + from sqlalchemy.schema import CreateColumn + + @compiles(CreateColumn, "postgresql") + def skip_xmin(element, compiler, **kw): + if element.element.name == 'xmin': + return None + else: + return compiler.visit_create_column(element, **kw) + + + my_table = Table('mytable', metadata, + Column('id', Integer, primary_key=True), + Column('xmin', Integer) + ) + + Above, a :class:`.CreateTable` construct will generate a ``CREATE TABLE`` + which only includes the ``id`` column in the string; the ``xmin`` column + will be omitted, but only against the Postgresql backend. + + .. versionadded:: 0.8.3 The :class:`.CreateColumn` construct supports + skipping of columns by returning ``None`` from a custom compilation rule. + + .. versionadded:: 0.8 The :class:`.CreateColumn` construct was added + to support custom column creation styles. + + """ + __visit_name__ = 'create_column' + + def __init__(self, element): + self.element = element + + +class DropTable(_CreateDropBase): + """Represent a DROP TABLE statement.""" + + __visit_name__ = "drop_table" + + +class CreateSequence(_CreateDropBase): + """Represent a CREATE SEQUENCE statement.""" + + __visit_name__ = "create_sequence" + + +class DropSequence(_CreateDropBase): + """Represent a DROP SEQUENCE statement.""" + + __visit_name__ = "drop_sequence" + + +class CreateIndex(_CreateDropBase): + """Represent a CREATE INDEX statement.""" + + __visit_name__ = "create_index" + + +class DropIndex(_CreateDropBase): + """Represent a DROP INDEX statement.""" + + __visit_name__ = "drop_index" + + +class AddConstraint(_CreateDropBase): + """Represent an ALTER TABLE ADD CONSTRAINT statement.""" + + __visit_name__ = "add_constraint" + + def __init__(self, element, *args, **kw): + super(AddConstraint, self).__init__(element, *args, **kw) + element._create_rule = util.portable_instancemethod( + self._create_rule_disable) + + +class DropConstraint(_CreateDropBase): + """Represent an ALTER TABLE DROP CONSTRAINT statement.""" + + __visit_name__ = "drop_constraint" + + def __init__(self, element, cascade=False, **kw): + self.cascade = cascade + super(DropConstraint, self).__init__(element, **kw) + element._create_rule = util.portable_instancemethod( + self._create_rule_disable) + + +class DDLBase(SchemaVisitor): + def __init__(self, connection): + self.connection = connection + + +class SchemaGenerator(DDLBase): + + def __init__(self, dialect, connection, checkfirst=False, + tables=None, **kwargs): + super(SchemaGenerator, self).__init__(connection, **kwargs) + self.checkfirst = checkfirst + self.tables = tables + self.preparer = dialect.identifier_preparer + self.dialect = dialect + self.memo = {} + + def _can_create_table(self, table): + self.dialect.validate_identifier(table.name) + if table.schema: + self.dialect.validate_identifier(table.schema) + return not self.checkfirst or \ + not self.dialect.has_table(self.connection, + table.name, schema=table.schema) + + def _can_create_sequence(self, sequence): + return self.dialect.supports_sequences and \ + ( + (not self.dialect.sequences_optional or + not sequence.optional) and + ( + not self.checkfirst or + not self.dialect.has_sequence( + self.connection, + sequence.name, + schema=sequence.schema) + ) + ) + + def visit_metadata(self, metadata): + if self.tables is not None: + tables = self.tables + else: + tables = list(metadata.tables.values()) + collection = [t for t in sort_tables(tables) + if self._can_create_table(t)] + seq_coll = [s for s in metadata._sequences.values() + if s.column is None and self._can_create_sequence(s)] + + metadata.dispatch.before_create(metadata, self.connection, + tables=collection, + checkfirst=self.checkfirst, + _ddl_runner=self) + + for seq in seq_coll: + self.traverse_single(seq, create_ok=True) + + for table in collection: + self.traverse_single(table, create_ok=True) + + metadata.dispatch.after_create(metadata, self.connection, + tables=collection, + checkfirst=self.checkfirst, + _ddl_runner=self) + + def visit_table(self, table, create_ok=False): + if not create_ok and not self._can_create_table(table): + return + + table.dispatch.before_create(table, self.connection, + checkfirst=self.checkfirst, + _ddl_runner=self) + + for column in table.columns: + if column.default is not None: + self.traverse_single(column.default) + + self.connection.execute(CreateTable(table)) + + if hasattr(table, 'indexes'): + for index in table.indexes: + self.traverse_single(index) + + table.dispatch.after_create(table, self.connection, + checkfirst=self.checkfirst, + _ddl_runner=self) + + def visit_sequence(self, sequence, create_ok=False): + if not create_ok and not self._can_create_sequence(sequence): + return + self.connection.execute(CreateSequence(sequence)) + + def visit_index(self, index): + self.connection.execute(CreateIndex(index)) + + +class SchemaDropper(DDLBase): + + def __init__(self, dialect, connection, checkfirst=False, + tables=None, **kwargs): + super(SchemaDropper, self).__init__(connection, **kwargs) + self.checkfirst = checkfirst + self.tables = tables + self.preparer = dialect.identifier_preparer + self.dialect = dialect + self.memo = {} + + def visit_metadata(self, metadata): + if self.tables is not None: + tables = self.tables + else: + tables = list(metadata.tables.values()) + + collection = [ + t + for t in reversed(sort_tables(tables)) + if self._can_drop_table(t) + ] + + seq_coll = [ + s + for s in metadata._sequences.values() + if s.column is None and self._can_drop_sequence(s) + ] + + metadata.dispatch.before_drop( + metadata, self.connection, tables=collection, + checkfirst=self.checkfirst, _ddl_runner=self) + + for table in collection: + self.traverse_single(table, drop_ok=True) + + for seq in seq_coll: + self.traverse_single(seq, drop_ok=True) + + metadata.dispatch.after_drop( + metadata, self.connection, tables=collection, + checkfirst=self.checkfirst, _ddl_runner=self) + + def _can_drop_table(self, table): + self.dialect.validate_identifier(table.name) + if table.schema: + self.dialect.validate_identifier(table.schema) + return not self.checkfirst or self.dialect.has_table(self.connection, + table.name, schema=table.schema) + + def _can_drop_sequence(self, sequence): + return self.dialect.supports_sequences and \ + ((not self.dialect.sequences_optional or + not sequence.optional) and + (not self.checkfirst or + self.dialect.has_sequence( + self.connection, + sequence.name, + schema=sequence.schema)) + ) + + def visit_index(self, index): + self.connection.execute(DropIndex(index)) + + def visit_table(self, table, drop_ok=False): + if not drop_ok and not self._can_drop_table(table): + return + + table.dispatch.before_drop(table, self.connection, + checkfirst=self.checkfirst, + _ddl_runner=self) + + for column in table.columns: + if column.default is not None: + self.traverse_single(column.default) + + self.connection.execute(DropTable(table)) + + table.dispatch.after_drop(table, self.connection, + checkfirst=self.checkfirst, + _ddl_runner=self) + + def visit_sequence(self, sequence, drop_ok=False): + if not drop_ok and not self._can_drop_sequence(sequence): + return + self.connection.execute(DropSequence(sequence)) + +def sort_tables(tables, skip_fn=None, extra_dependencies=None): + """sort a collection of Table objects in order of + their foreign-key dependency.""" + + tables = list(tables) + tuples = [] + if extra_dependencies is not None: + tuples.extend(extra_dependencies) + + def visit_foreign_key(fkey): + if fkey.use_alter: + return + elif skip_fn and skip_fn(fkey): + return + parent_table = fkey.column.table + if parent_table in tables: + child_table = fkey.parent.table + if parent_table is not child_table: + tuples.append((parent_table, child_table)) + + for table in tables: + traverse(table, + {'schema_visitor': True}, + {'foreign_key': visit_foreign_key}) + + tuples.extend( + [parent, table] for parent in table._extra_dependencies + ) + + return list(topological.sort(tuples, tables)) + diff --git a/libs/sqlalchemy/sql/default_comparator.py b/libs/sqlalchemy/sql/default_comparator.py new file mode 100644 index 00000000..c39dce9c --- /dev/null +++ b/libs/sqlalchemy/sql/default_comparator.py @@ -0,0 +1,278 @@ +# sql/default_comparator.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Default implementation of SQL comparison operations. +""" + +from .. import exc, util +from . import operators +from . import type_api +from .elements import BindParameter, True_, False_, BinaryExpression, \ + Null, _const_expr, _clause_element_as_expr, \ + ClauseList, ColumnElement, TextClause, UnaryExpression, \ + collate, _is_literal, _literal_as_text +from .selectable import SelectBase, Alias, Selectable, ScalarSelect + +class _DefaultColumnComparator(operators.ColumnOperators): + """Defines comparison and math operations. + + See :class:`.ColumnOperators` and :class:`.Operators` for descriptions + of all operations. + + """ + + @util.memoized_property + def type(self): + return self.expr.type + + def operate(self, op, *other, **kwargs): + o = self.operators[op.__name__] + return o[0](self, self.expr, op, *(other + o[1:]), **kwargs) + + def reverse_operate(self, op, other, **kwargs): + o = self.operators[op.__name__] + return o[0](self, self.expr, op, other, reverse=True, *o[1:], **kwargs) + + def _adapt_expression(self, op, other_comparator): + """evaluate the return type of , + and apply any adaptations to the given operator. + + This method determines the type of a resulting binary expression + given two source types and an operator. For example, two + :class:`.Column` objects, both of the type :class:`.Integer`, will + produce a :class:`.BinaryExpression` that also has the type + :class:`.Integer` when compared via the addition (``+``) operator. + However, using the addition operator with an :class:`.Integer` + and a :class:`.Date` object will produce a :class:`.Date`, assuming + "days delta" behavior by the database (in reality, most databases + other than Postgresql don't accept this particular operation). + + The method returns a tuple of the form , . + The resulting operator and type will be those applied to the + resulting :class:`.BinaryExpression` as the final operator and the + right-hand side of the expression. + + Note that only a subset of operators make usage of + :meth:`._adapt_expression`, + including math operators and user-defined operators, but not + boolean comparison or special SQL keywords like MATCH or BETWEEN. + + """ + return op, other_comparator.type + + def _boolean_compare(self, expr, op, obj, negate=None, reverse=False, + _python_is_types=(util.NoneType, bool), + **kwargs): + + if isinstance(obj, _python_is_types + (Null, True_, False_)): + + # allow x ==/!= True/False to be treated as a literal. + # this comes out to "== / != true/false" or "1/0" if those + # constants aren't supported and works on all platforms + if op in (operators.eq, operators.ne) and \ + isinstance(obj, (bool, True_, False_)): + return BinaryExpression(expr, + _literal_as_text(obj), + op, + type_=type_api.BOOLEANTYPE, + negate=negate, modifiers=kwargs) + else: + # all other None/True/False uses IS, IS NOT + if op in (operators.eq, operators.is_): + return BinaryExpression(expr, _const_expr(obj), + operators.is_, + negate=operators.isnot) + elif op in (operators.ne, operators.isnot): + return BinaryExpression(expr, _const_expr(obj), + operators.isnot, + negate=operators.is_) + else: + raise exc.ArgumentError( + "Only '=', '!=', 'is_()', 'isnot()' operators can " + "be used with None/True/False") + else: + obj = self._check_literal(expr, op, obj) + + if reverse: + return BinaryExpression(obj, + expr, + op, + type_=type_api.BOOLEANTYPE, + negate=negate, modifiers=kwargs) + else: + return BinaryExpression(expr, + obj, + op, + type_=type_api.BOOLEANTYPE, + negate=negate, modifiers=kwargs) + + def _binary_operate(self, expr, op, obj, reverse=False, result_type=None, + **kw): + obj = self._check_literal(expr, op, obj) + + if reverse: + left, right = obj, expr + else: + left, right = expr, obj + + if result_type is None: + op, result_type = left.comparator._adapt_expression( + op, right.comparator) + + return BinaryExpression(left, right, op, type_=result_type) + + def _scalar(self, expr, op, fn, **kw): + return fn(expr) + + def _in_impl(self, expr, op, seq_or_selectable, negate_op, **kw): + seq_or_selectable = _clause_element_as_expr(seq_or_selectable) + + if isinstance(seq_or_selectable, ScalarSelect): + return self._boolean_compare(expr, op, seq_or_selectable, + negate=negate_op) + elif isinstance(seq_or_selectable, SelectBase): + + # TODO: if we ever want to support (x, y, z) IN (select x, + # y, z from table), we would need a multi-column version of + # as_scalar() to produce a multi- column selectable that + # does not export itself as a FROM clause + + return self._boolean_compare( + expr, op, seq_or_selectable.as_scalar(), + negate=negate_op, **kw) + elif isinstance(seq_or_selectable, (Selectable, TextClause)): + return self._boolean_compare(expr, op, seq_or_selectable, + negate=negate_op, **kw) + + # Handle non selectable arguments as sequences + args = [] + for o in seq_or_selectable: + if not _is_literal(o): + if not isinstance(o, operators.ColumnOperators): + raise exc.InvalidRequestError('in() function accept' + 's either a list of non-selectable values, ' + 'or a selectable: %r' % o) + elif o is None: + o = Null() + else: + o = expr._bind_param(op, o) + args.append(o) + if len(args) == 0: + + # Special case handling for empty IN's, behave like + # comparison against zero row selectable. We use != to + # build the contradiction as it handles NULL values + # appropriately, i.e. "not (x IN ())" should not return NULL + # values for x. + + util.warn('The IN-predicate on "%s" was invoked with an ' + 'empty sequence. This results in a ' + 'contradiction, which nonetheless can be ' + 'expensive to evaluate. Consider alternative ' + 'strategies for improved performance.' % expr) + if op is operators.in_op: + return expr != expr + else: + return expr == expr + + return self._boolean_compare(expr, op, + ClauseList(*args).self_group(against=op), + negate=negate_op) + + def _unsupported_impl(self, expr, op, *arg, **kw): + raise NotImplementedError("Operator '%s' is not supported on " + "this expression" % op.__name__) + + def _neg_impl(self, expr, op, **kw): + """See :meth:`.ColumnOperators.__neg__`.""" + return UnaryExpression(expr, operator=operators.neg) + + def _match_impl(self, expr, op, other, **kw): + """See :meth:`.ColumnOperators.match`.""" + return self._boolean_compare(expr, operators.match_op, + self._check_literal(expr, operators.match_op, + other)) + + def _distinct_impl(self, expr, op, **kw): + """See :meth:`.ColumnOperators.distinct`.""" + return UnaryExpression(expr, operator=operators.distinct_op, + type_=expr.type) + + def _between_impl(self, expr, op, cleft, cright, **kw): + """See :meth:`.ColumnOperators.between`.""" + return BinaryExpression( + expr, + ClauseList( + self._check_literal(expr, operators.and_, cleft), + self._check_literal(expr, operators.and_, cright), + operator=operators.and_, + group=False, group_contents=False), + operators.between_op) + + def _collate_impl(self, expr, op, other, **kw): + return collate(expr, other) + + # a mapping of operators with the method they use, along with + # their negated operator for comparison operators + operators = { + "add": (_binary_operate,), + "mul": (_binary_operate,), + "sub": (_binary_operate,), + "div": (_binary_operate,), + "mod": (_binary_operate,), + "truediv": (_binary_operate,), + "custom_op": (_binary_operate,), + "concat_op": (_binary_operate,), + "lt": (_boolean_compare, operators.ge), + "le": (_boolean_compare, operators.gt), + "ne": (_boolean_compare, operators.eq), + "gt": (_boolean_compare, operators.le), + "ge": (_boolean_compare, operators.lt), + "eq": (_boolean_compare, operators.ne), + "like_op": (_boolean_compare, operators.notlike_op), + "ilike_op": (_boolean_compare, operators.notilike_op), + "notlike_op": (_boolean_compare, operators.like_op), + "notilike_op": (_boolean_compare, operators.ilike_op), + "contains_op": (_boolean_compare, operators.notcontains_op), + "startswith_op": (_boolean_compare, operators.notstartswith_op), + "endswith_op": (_boolean_compare, operators.notendswith_op), + "desc_op": (_scalar, UnaryExpression._create_desc), + "asc_op": (_scalar, UnaryExpression._create_asc), + "nullsfirst_op": (_scalar, UnaryExpression._create_nullsfirst), + "nullslast_op": (_scalar, UnaryExpression._create_nullslast), + "in_op": (_in_impl, operators.notin_op), + "notin_op": (_in_impl, operators.in_op), + "is_": (_boolean_compare, operators.is_), + "isnot": (_boolean_compare, operators.isnot), + "collate": (_collate_impl,), + "match_op": (_match_impl,), + "distinct_op": (_distinct_impl,), + "between_op": (_between_impl, ), + "neg": (_neg_impl,), + "getitem": (_unsupported_impl,), + "lshift": (_unsupported_impl,), + "rshift": (_unsupported_impl,), + } + + def _check_literal(self, expr, operator, other): + if isinstance(other, (ColumnElement, TextClause)): + if isinstance(other, BindParameter) and \ + other.type._isnull: + other = other._clone() + other.type = expr.type + return other + elif hasattr(other, '__clause_element__'): + other = other.__clause_element__() + elif isinstance(other, type_api.TypeEngine.Comparator): + other = other.expr + + if isinstance(other, (SelectBase, Alias)): + return other.as_scalar() + elif not isinstance(other, (ColumnElement, TextClause)): + return expr._bind_param(operator, other) + else: + return other + diff --git a/libs/sqlalchemy/sql/dml.py b/libs/sqlalchemy/sql/dml.py new file mode 100644 index 00000000..22694348 --- /dev/null +++ b/libs/sqlalchemy/sql/dml.py @@ -0,0 +1,770 @@ +# sql/dml.py +# Copyright (C) 2009-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +""" +Provide :class:`.Insert`, :class:`.Update` and :class:`.Delete`. + +""" + +from .base import Executable, _generative, _from_objects +from .elements import ClauseElement, _literal_as_text, Null, and_, _clone +from .selectable import _interpret_as_from, _interpret_as_select, HasPrefixes +from .. import util +from .. import exc + +class UpdateBase(HasPrefixes, Executable, ClauseElement): + """Form the base for ``INSERT``, ``UPDATE``, and ``DELETE`` statements. + + """ + + __visit_name__ = 'update_base' + + _execution_options = \ + Executable._execution_options.union({'autocommit': True}) + kwargs = util.immutabledict() + _hints = util.immutabledict() + _prefixes = () + + def _process_colparams(self, parameters): + def process_single(p): + if isinstance(p, (list, tuple)): + return dict( + (c.key, pval) + for c, pval in zip(self.table.c, p) + ) + else: + return p + + if isinstance(parameters, (list, tuple)) and \ + isinstance(parameters[0], (list, tuple, dict)): + + if not self._supports_multi_parameters: + raise exc.InvalidRequestError( + "This construct does not support " + "multiple parameter sets.") + + return [process_single(p) for p in parameters], True + else: + return process_single(parameters), False + + def params(self, *arg, **kw): + """Set the parameters for the statement. + + This method raises ``NotImplementedError`` on the base class, + and is overridden by :class:`.ValuesBase` to provide the + SET/VALUES clause of UPDATE and INSERT. + + """ + raise NotImplementedError( + "params() is not supported for INSERT/UPDATE/DELETE statements." + " To set the values for an INSERT or UPDATE statement, use" + " stmt.values(**parameters).") + + def bind(self): + """Return a 'bind' linked to this :class:`.UpdateBase` + or a :class:`.Table` associated with it. + + """ + return self._bind or self.table.bind + + def _set_bind(self, bind): + self._bind = bind + bind = property(bind, _set_bind) + + @_generative + def returning(self, *cols): + """Add a :term:`RETURNING` or equivalent clause to this statement. + + e.g.:: + + stmt = table.update().\\ + where(table.c.data == 'value').\\ + values(status='X').\\ + returning(table.c.server_flag, table.c.updated_timestamp) + + for server_flag, updated_timestamp in connection.execute(stmt): + print(server_flag, updated_timestamp) + + The given collection of column expressions should be derived from + the table that is + the target of the INSERT, UPDATE, or DELETE. While :class:`.Column` + objects are typical, the elements can also be expressions:: + + stmt = table.insert().returning( + (table.c.first_name + " " + table.c.last_name).label('fullname') + ) + + Upon compilation, a RETURNING clause, or database equivalent, + will be rendered within the statement. For INSERT and UPDATE, + the values are the newly inserted/updated values. For DELETE, + the values are those of the rows which were deleted. + + Upon execution, the values of the columns to be returned + are made available via the result set and can be iterated + using :meth:`.ResultProxy.fetchone` and similar. For DBAPIs which do not + natively support returning values (i.e. cx_oracle), + SQLAlchemy will approximate this behavior at the result level + so that a reasonable amount of behavioral neutrality is + provided. + + Note that not all databases/DBAPIs + support RETURNING. For those backends with no support, + an exception is raised upon compilation and/or execution. + For those who do support it, the functionality across backends + varies greatly, including restrictions on executemany() + and other statements which return multiple rows. Please + read the documentation notes for the database in use in + order to determine the availability of RETURNING. + + .. seealso:: + + :meth:`.ValuesBase.return_defaults` - an alternative method tailored + towards efficient fetching of server-side defaults and triggers + for single-row INSERTs or UPDATEs. + + + """ + self._returning = cols + + + @_generative + def with_hint(self, text, selectable=None, dialect_name="*"): + """Add a table hint for a single table to this + INSERT/UPDATE/DELETE statement. + + .. note:: + + :meth:`.UpdateBase.with_hint` currently applies only to + Microsoft SQL Server. For MySQL INSERT/UPDATE/DELETE hints, use + :meth:`.UpdateBase.prefix_with`. + + The text of the hint is rendered in the appropriate + location for the database backend in use, relative + to the :class:`.Table` that is the subject of this + statement, or optionally to that of the given + :class:`.Table` passed as the ``selectable`` argument. + + The ``dialect_name`` option will limit the rendering of a particular + hint to a particular backend. Such as, to add a hint + that only takes effect for SQL Server:: + + mytable.insert().with_hint("WITH (PAGLOCK)", dialect_name="mssql") + + .. versionadded:: 0.7.6 + + :param text: Text of the hint. + :param selectable: optional :class:`.Table` that specifies + an element of the FROM clause within an UPDATE or DELETE + to be the subject of the hint - applies only to certain backends. + :param dialect_name: defaults to ``*``, if specified as the name + of a particular dialect, will apply these hints only when + that dialect is in use. + """ + if selectable is None: + selectable = self.table + + self._hints = self._hints.union( + {(selectable, dialect_name): text}) + + +class ValuesBase(UpdateBase): + """Supplies support for :meth:`.ValuesBase.values` to + INSERT and UPDATE constructs.""" + + __visit_name__ = 'values_base' + + _supports_multi_parameters = False + _has_multi_parameters = False + select = None + + def __init__(self, table, values, prefixes): + self.table = _interpret_as_from(table) + self.parameters, self._has_multi_parameters = \ + self._process_colparams(values) + if prefixes: + self._setup_prefixes(prefixes) + + @_generative + def values(self, *args, **kwargs): + """specify a fixed VALUES clause for an INSERT statement, or the SET + clause for an UPDATE. + + Note that the :class:`.Insert` and :class:`.Update` constructs support + per-execution time formatting of the VALUES and/or SET clauses, + based on the arguments passed to :meth:`.Connection.execute`. However, + the :meth:`.ValuesBase.values` method can be used to "fix" a particular + set of parameters into the statement. + + Multiple calls to :meth:`.ValuesBase.values` will produce a new + construct, each one with the parameter list modified to include + the new parameters sent. In the typical case of a single + dictionary of parameters, the newly passed keys will replace + the same keys in the previous construct. In the case of a list-based + "multiple values" construct, each new list of values is extended + onto the existing list of values. + + :param \**kwargs: key value pairs representing the string key + of a :class:`.Column` mapped to the value to be rendered into the + VALUES or SET clause:: + + users.insert().values(name="some name") + + users.update().where(users.c.id==5).values(name="some name") + + :param \*args: Alternatively, a dictionary, tuple or list + of dictionaries or tuples can be passed as a single positional + argument in order to form the VALUES or + SET clause of the statement. The single dictionary form + works the same as the kwargs form:: + + users.insert().values({"name": "some name"}) + + If a tuple is passed, the tuple should contain the same number + of columns as the target :class:`.Table`:: + + users.insert().values((5, "some name")) + + The :class:`.Insert` construct also supports multiply-rendered VALUES + construct, for those backends which support this SQL syntax + (SQLite, Postgresql, MySQL). This mode is indicated by passing a list + of one or more dictionaries/tuples:: + + users.insert().values([ + {"name": "some name"}, + {"name": "some other name"}, + {"name": "yet another name"}, + ]) + + In the case of an :class:`.Update` + construct, only the single dictionary/tuple form is accepted, + else an exception is raised. It is also an exception case to + attempt to mix the single-/multiple- value styles together, + either through multiple :meth:`.ValuesBase.values` calls + or by sending a list + kwargs at the same time. + + .. note:: + + Passing a multiple values list is *not* the same + as passing a multiple values list to the :meth:`.Connection.execute` + method. Passing a list of parameter sets to :meth:`.ValuesBase.values` + produces a construct of this form:: + + INSERT INTO table (col1, col2, col3) VALUES + (col1_0, col2_0, col3_0), + (col1_1, col2_1, col3_1), + ... + + whereas a multiple list passed to :meth:`.Connection.execute` + has the effect of using the DBAPI + `executemany() `_ + method, which provides a high-performance system of invoking + a single-row INSERT statement many times against a series + of parameter sets. The "executemany" style is supported by + all database backends, as it does not depend on a special SQL + syntax. + + .. versionadded:: 0.8 + Support for multiple-VALUES INSERT statements. + + + .. seealso:: + + :ref:`inserts_and_updates` - SQL Expression + Language Tutorial + + :func:`~.expression.insert` - produce an ``INSERT`` statement + + :func:`~.expression.update` - produce an ``UPDATE`` statement + + """ + if self.select is not None: + raise exc.InvalidRequestError( + "This construct already inserts from a SELECT") + if self._has_multi_parameters and kwargs: + raise exc.InvalidRequestError( + "This construct already has multiple parameter sets.") + + if args: + if len(args) > 1: + raise exc.ArgumentError( + "Only a single dictionary/tuple or list of " + "dictionaries/tuples is accepted positionally.") + v = args[0] + else: + v = {} + + if self.parameters is None: + self.parameters, self._has_multi_parameters = \ + self._process_colparams(v) + else: + if self._has_multi_parameters: + self.parameters = list(self.parameters) + p, self._has_multi_parameters = self._process_colparams(v) + if not self._has_multi_parameters: + raise exc.ArgumentError( + "Can't mix single-values and multiple values " + "formats in one statement") + + self.parameters.extend(p) + else: + self.parameters = self.parameters.copy() + p, self._has_multi_parameters = self._process_colparams(v) + if self._has_multi_parameters: + raise exc.ArgumentError( + "Can't mix single-values and multiple values " + "formats in one statement") + self.parameters.update(p) + + if kwargs: + if self._has_multi_parameters: + raise exc.ArgumentError( + "Can't pass kwargs and multiple parameter sets " + "simultaenously") + else: + self.parameters.update(kwargs) + + @_generative + def return_defaults(self, *cols): + """Make use of a :term:`RETURNING` clause for the purpose + of fetching server-side expressions and defaults. + + E.g.:: + + stmt = table.insert().values(data='newdata').return_defaults() + + result = connection.execute(stmt) + + server_created_at = result.returned_defaults['created_at'] + + When used against a backend that supports RETURNING, all column + values generated by SQL expression or server-side-default will be added + to any existing RETURNING clause, provided that + :meth:`.UpdateBase.returning` is not used simultaneously. The column values + will then be available on the result using the + :attr:`.ResultProxy.returned_defaults` accessor as a + dictionary, referring to values keyed to the :class:`.Column` object + as well as its ``.key``. + + This method differs from :meth:`.UpdateBase.returning` in these ways: + + 1. :meth:`.ValuesBase.return_defaults` is only intended for use with + an INSERT or an UPDATE statement that matches exactly one row. + While the RETURNING construct in the general sense supports multiple + rows for a multi-row UPDATE or DELETE statement, or for special + cases of INSERT that return multiple rows (e.g. INSERT from SELECT, + multi-valued VALUES clause), :meth:`.ValuesBase.return_defaults` + is intended only + for an "ORM-style" single-row INSERT/UPDATE statement. The row + returned by the statement is also consumed implcitly when + :meth:`.ValuesBase.return_defaults` is used. By contrast, + :meth:`.UpdateBase.returning` leaves the RETURNING result-set intact + with a collection of any number of rows. + + 2. It is compatible with the existing logic to fetch auto-generated + primary key values, also known as "implicit returning". Backends that + support RETURNING will automatically make use of RETURNING in order + to fetch the value of newly generated primary keys; while the + :meth:`.UpdateBase.returning` method circumvents this behavior, + :meth:`.ValuesBase.return_defaults` leaves it intact. + + 3. It can be called against any backend. Backends that don't support + RETURNING will skip the usage of the feature, rather than raising + an exception. The return value of :attr:`.ResultProxy.returned_defaults` + will be ``None`` + + :meth:`.ValuesBase.return_defaults` is used by the ORM to provide + an efficient implementation for the ``eager_defaults`` feature of + :func:`.mapper`. + + :param cols: optional list of column key names or :class:`.Column` + objects. If omitted, all column expressions evaulated on the server + are added to the returning list. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :meth:`.UpdateBase.returning` + + :attr:`.ResultProxy.returned_defaults` + + """ + self._return_defaults = cols or True + + +class Insert(ValuesBase): + """Represent an INSERT construct. + + The :class:`.Insert` object is created using the + :func:`~.expression.insert()` function. + + .. seealso:: + + :ref:`coretutorial_insert_expressions` + + """ + __visit_name__ = 'insert' + + _supports_multi_parameters = True + + def __init__(self, + table, + values=None, + inline=False, + bind=None, + prefixes=None, + returning=None, + return_defaults=False, + **kwargs): + """Construct an :class:`.Insert` object. + + Similar functionality is available via the + :meth:`~.TableClause.insert` method on + :class:`~.schema.Table`. + + :param table: :class:`.TableClause` which is the subject of the insert. + + :param values: collection of values to be inserted; see + :meth:`.Insert.values` for a description of allowed formats here. + Can be omitted entirely; a :class:`.Insert` construct will also + dynamically render the VALUES clause at execution time based on + the parameters passed to :meth:`.Connection.execute`. + + :param inline: if True, SQL defaults will be compiled 'inline' into the + statement and not pre-executed. + + If both `values` and compile-time bind parameters are present, the + compile-time bind parameters override the information specified + within `values` on a per-key basis. + + The keys within `values` can be either :class:`~sqlalchemy.schema.Column` + objects or their string identifiers. Each key may reference one of: + + * a literal data value (i.e. string, number, etc.); + * a Column object; + * a SELECT statement. + + If a ``SELECT`` statement is specified which references this + ``INSERT`` statement's table, the statement will be correlated + against the ``INSERT`` statement. + + .. seealso:: + + :ref:`coretutorial_insert_expressions` - SQL Expression Tutorial + + :ref:`inserts_and_updates` - SQL Expression Tutorial + + """ + ValuesBase.__init__(self, table, values, prefixes) + self._bind = bind + self.select = self.select_names = None + self.inline = inline + self._returning = returning + self.kwargs = kwargs + self._return_defaults = return_defaults + + def get_children(self, **kwargs): + if self.select is not None: + return self.select, + else: + return () + + @_generative + def from_select(self, names, select): + """Return a new :class:`.Insert` construct which represents + an ``INSERT...FROM SELECT`` statement. + + e.g.:: + + sel = select([table1.c.a, table1.c.b]).where(table1.c.c > 5) + ins = table2.insert().from_select(['a', 'b'], sel) + + :param names: a sequence of string column names or :class:`.Column` + objects representing the target columns. + :param select: a :func:`.select` construct, :class:`.FromClause` + or other construct which resolves into a :class:`.FromClause`, + such as an ORM :class:`.Query` object, etc. The order of + columns returned from this FROM clause should correspond to the + order of columns sent as the ``names`` parameter; while this + is not checked before passing along to the database, the database + would normally raise an exception if these column lists don't + correspond. + + .. note:: + + Depending on backend, it may be necessary for the :class:`.Insert` + statement to be constructed using the ``inline=True`` flag; this + flag will prevent the implicit usage of ``RETURNING`` when the + ``INSERT`` statement is rendered, which isn't supported on a backend + such as Oracle in conjunction with an ``INSERT..SELECT`` combination:: + + sel = select([table1.c.a, table1.c.b]).where(table1.c.c > 5) + ins = table2.insert(inline=True).from_select(['a', 'b'], sel) + + .. note:: + + A SELECT..INSERT construct in SQL has no VALUES clause. Therefore + :class:`.Column` objects which utilize Python-side defaults + (e.g. as described at :ref:`metadata_defaults_toplevel`) + will **not** take effect when using :meth:`.Insert.from_select`. + + .. versionadded:: 0.8.3 + + """ + if self.parameters: + raise exc.InvalidRequestError( + "This construct already inserts value expressions") + + self.parameters, self._has_multi_parameters = \ + self._process_colparams(dict((n, Null()) for n in names)) + + self.select_names = names + self.select = _interpret_as_select(select) + + def _copy_internals(self, clone=_clone, **kw): + # TODO: coverage + self.parameters = self.parameters.copy() + if self.select is not None: + self.select = _clone(self.select) + + +class Update(ValuesBase): + """Represent an Update construct. + + The :class:`.Update` object is created using the :func:`update()` function. + + """ + __visit_name__ = 'update' + + def __init__(self, + table, + whereclause=None, + values=None, + inline=False, + bind=None, + prefixes=None, + returning=None, + return_defaults=False, + **kwargs): + """Construct an :class:`.Update` object. + + E.g.:: + + from sqlalchemy import update + + stmt = update(users).where(users.c.id==5).\\ + values(name='user #5') + + Similar functionality is available via the + :meth:`~.TableClause.update` method on + :class:`.Table`:: + + stmt = users.update().\\ + where(users.c.id==5).\\ + values(name='user #5') + + :param table: A :class:`.Table` object representing the database + table to be updated. + + :param whereclause: Optional SQL expression describing the ``WHERE`` + condition of the ``UPDATE`` statement. Modern applications + may prefer to use the generative :meth:`~Update.where()` + method to specify the ``WHERE`` clause. + + The WHERE clause can refer to multiple tables. + For databases which support this, an ``UPDATE FROM`` clause will + be generated, or on MySQL, a multi-table update. The statement + will fail on databases that don't have support for multi-table + update statements. A SQL-standard method of referring to + additional tables in the WHERE clause is to use a correlated + subquery:: + + users.update().values(name='ed').where( + users.c.name==select([addresses.c.email_address]).\\ + where(addresses.c.user_id==users.c.id).\\ + as_scalar() + ) + + .. versionchanged:: 0.7.4 + The WHERE clause can refer to multiple tables. + + :param values: + Optional dictionary which specifies the ``SET`` conditions of the + ``UPDATE``. If left as ``None``, the ``SET`` + conditions are determined from those parameters passed to the + statement during the execution and/or compilation of the + statement. When compiled standalone without any parameters, + the ``SET`` clause generates for all columns. + + Modern applications may prefer to use the generative + :meth:`.Update.values` method to set the values of the + UPDATE statement. + + :param inline: + if True, SQL defaults present on :class:`.Column` objects via + the ``default`` keyword will be compiled 'inline' into the statement + and not pre-executed. This means that their values will not + be available in the dictionary returned from + :meth:`.ResultProxy.last_updated_params`. + + If both ``values`` and compile-time bind parameters are present, the + compile-time bind parameters override the information specified + within ``values`` on a per-key basis. + + The keys within ``values`` can be either :class:`.Column` + objects or their string identifiers (specifically the "key" of the + :class:`.Column`, normally but not necessarily equivalent to + its "name"). Normally, the + :class:`.Column` objects used here are expected to be + part of the target :class:`.Table` that is the table + to be updated. However when using MySQL, a multiple-table + UPDATE statement can refer to columns from any of + the tables referred to in the WHERE clause. + + The values referred to in ``values`` are typically: + + * a literal data value (i.e. string, number, etc.) + * a SQL expression, such as a related :class:`.Column`, + a scalar-returning :func:`.select` construct, + etc. + + When combining :func:`.select` constructs within the values + clause of an :func:`.update` construct, + the subquery represented by the :func:`.select` should be + *correlated* to the parent table, that is, providing criterion + which links the table inside the subquery to the outer table + being updated:: + + users.update().values( + name=select([addresses.c.email_address]).\\ + where(addresses.c.user_id==users.c.id).\\ + as_scalar() + ) + + .. seealso:: + + :ref:`inserts_and_updates` - SQL Expression + Language Tutorial + + + """ + ValuesBase.__init__(self, table, values, prefixes) + self._bind = bind + self._returning = returning + if whereclause is not None: + self._whereclause = _literal_as_text(whereclause) + else: + self._whereclause = None + self.inline = inline + self.kwargs = kwargs + self._return_defaults = return_defaults + + + def get_children(self, **kwargs): + if self._whereclause is not None: + return self._whereclause, + else: + return () + + def _copy_internals(self, clone=_clone, **kw): + # TODO: coverage + self._whereclause = clone(self._whereclause, **kw) + self.parameters = self.parameters.copy() + + @_generative + def where(self, whereclause): + """return a new update() construct with the given expression added to + its WHERE clause, joined to the existing clause via AND, if any. + + """ + if self._whereclause is not None: + self._whereclause = and_(self._whereclause, + _literal_as_text(whereclause)) + else: + self._whereclause = _literal_as_text(whereclause) + + @property + def _extra_froms(self): + # TODO: this could be made memoized + # if the memoization is reset on each generative call. + froms = [] + seen = set([self.table]) + + if self._whereclause is not None: + for item in _from_objects(self._whereclause): + if not seen.intersection(item._cloned_set): + froms.append(item) + seen.update(item._cloned_set) + + return froms + + +class Delete(UpdateBase): + """Represent a DELETE construct. + + The :class:`.Delete` object is created using the :func:`delete()` function. + + """ + + __visit_name__ = 'delete' + + def __init__(self, + table, + whereclause=None, + bind=None, + returning=None, + prefixes=None, + **kwargs): + """Construct :class:`.Delete` object. + + Similar functionality is available via the + :meth:`~.TableClause.delete` method on + :class:`~.schema.Table`. + + :param table: The table to be updated. + + :param whereclause: A :class:`.ClauseElement` describing the ``WHERE`` + condition of the ``UPDATE`` statement. Note that the + :meth:`~Delete.where()` generative method may be used instead. + + .. seealso:: + + :ref:`deletes` - SQL Expression Tutorial + + """ + self._bind = bind + self.table = _interpret_as_from(table) + self._returning = returning + + if prefixes: + self._setup_prefixes(prefixes) + + if whereclause is not None: + self._whereclause = _literal_as_text(whereclause) + else: + self._whereclause = None + + self.kwargs = kwargs + + def get_children(self, **kwargs): + if self._whereclause is not None: + return self._whereclause, + else: + return () + + @_generative + def where(self, whereclause): + """Add the given WHERE clause to a newly returned delete construct.""" + + if self._whereclause is not None: + self._whereclause = and_(self._whereclause, + _literal_as_text(whereclause)) + else: + self._whereclause = _literal_as_text(whereclause) + + def _copy_internals(self, clone=_clone, **kw): + # TODO: coverage + self._whereclause = clone(self._whereclause, **kw) + diff --git a/libs/sqlalchemy/sql/elements.py b/libs/sqlalchemy/sql/elements.py new file mode 100644 index 00000000..0e888fcf --- /dev/null +++ b/libs/sqlalchemy/sql/elements.py @@ -0,0 +1,2880 @@ +# sql/elements.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Core SQL expression elements, including :class:`.ClauseElement`, +:class:`.ColumnElement`, and derived classes. + +""" + +from __future__ import unicode_literals + +from .. import util, exc, inspection +from . import type_api +from . import operators +from .visitors import Visitable, cloned_traverse, traverse +from .annotation import Annotated +import itertools +from .base import Executable, PARSE_AUTOCOMMIT, Immutable, NO_ARG +from .base import _generative, Generative + +import re +import operator + +def _clone(element, **kw): + return element._clone() + +def collate(expression, collation): + """Return the clause ``expression COLLATE collation``. + + e.g.:: + + collate(mycolumn, 'utf8_bin') + + produces:: + + mycolumn COLLATE utf8_bin + + """ + + expr = _literal_as_binds(expression) + return BinaryExpression( + expr, + _literal_as_text(collation), + operators.collate, type_=expr.type) + +def between(ctest, cleft, cright): + """Return a ``BETWEEN`` predicate clause. + + Equivalent of SQL ``clausetest BETWEEN clauseleft AND clauseright``. + + The :func:`between()` method on all + :class:`.ColumnElement` subclasses provides + similar functionality. + + """ + ctest = _literal_as_binds(ctest) + return ctest.between(cleft, cright) + +def literal(value, type_=None): + """Return a literal clause, bound to a bind parameter. + + Literal clauses are created automatically when non- :class:`.ClauseElement` + objects (such as strings, ints, dates, etc.) are used in a comparison + operation with a :class:`.ColumnElement` + subclass, such as a :class:`~sqlalchemy.schema.Column` object. + Use this function to force the + generation of a literal clause, which will be created as a + :class:`BindParameter` with a bound value. + + :param value: the value to be bound. Can be any Python object supported by + the underlying DB-API, or is translatable via the given type argument. + + :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` which + will provide bind-parameter translation for this literal. + + """ + return BindParameter(None, value, type_=type_, unique=True) + + + +def type_coerce(expression, type_): + """Coerce the given expression into the given type, + on the Python side only. + + :func:`.type_coerce` is roughly similar to :func:`.cast`, except no + "CAST" expression is rendered - the given type is only applied towards + expression typing and against received result values. + + e.g.:: + + from sqlalchemy.types import TypeDecorator + import uuid + + class AsGuid(TypeDecorator): + impl = String + + def process_bind_param(self, value, dialect): + if value is not None: + return str(value) + else: + return None + + def process_result_value(self, value, dialect): + if value is not None: + return uuid.UUID(value) + else: + return None + + conn.execute( + select([type_coerce(mytable.c.ident, AsGuid)]).\\ + where( + type_coerce(mytable.c.ident, AsGuid) == + uuid.uuid3(uuid.NAMESPACE_URL, 'bar') + ) + ) + + :param expression: Column-oriented expression. + :param type_: A :class:`.TypeEngine` class or instance indicating + the type to which the CAST should apply. + + .. seealso:: + + :func:`.cast` + + """ + type_ = type_api.to_instance(type_) + + if hasattr(expression, '__clause_element__'): + return type_coerce(expression.__clause_element__(), type_) + elif isinstance(expression, BindParameter): + bp = expression._clone() + bp.type = type_ + return bp + elif not isinstance(expression, Visitable): + if expression is None: + return Null() + else: + return literal(expression, type_=type_) + else: + return Label(None, expression, type_=type_) + + + + + +def outparam(key, type_=None): + """Create an 'OUT' parameter for usage in functions (stored procedures), + for databases which support them. + + The ``outparam`` can be used like a regular function parameter. + The "output" value will be available from the + :class:`~sqlalchemy.engine.ResultProxy` object via its ``out_parameters`` + attribute, which returns a dictionary containing the values. + + """ + return BindParameter( + key, None, type_=type_, unique=False, isoutparam=True) + + + + +def not_(clause): + """Return a negation of the given clause, i.e. ``NOT(clause)``. + + The ``~`` operator is also overloaded on all + :class:`.ColumnElement` subclasses to produce the + same result. + + """ + return operators.inv(_literal_as_binds(clause)) + + + +@inspection._self_inspects +class ClauseElement(Visitable): + """Base class for elements of a programmatically constructed SQL + expression. + + """ + __visit_name__ = 'clause' + + _annotations = {} + supports_execution = False + _from_objects = [] + bind = None + _is_clone_of = None + is_selectable = False + is_clause_element = True + + _order_by_label_element = None + + def _clone(self): + """Create a shallow copy of this ClauseElement. + + This method may be used by a generative API. Its also used as + part of the "deep" copy afforded by a traversal that combines + the _copy_internals() method. + + """ + c = self.__class__.__new__(self.__class__) + c.__dict__ = self.__dict__.copy() + ClauseElement._cloned_set._reset(c) + ColumnElement.comparator._reset(c) + + # this is a marker that helps to "equate" clauses to each other + # when a Select returns its list of FROM clauses. the cloning + # process leaves around a lot of remnants of the previous clause + # typically in the form of column expressions still attached to the + # old table. + c._is_clone_of = self + + return c + + @property + def _constructor(self): + """return the 'constructor' for this ClauseElement. + + This is for the purposes for creating a new object of + this type. Usually, its just the element's __class__. + However, the "Annotated" version of the object overrides + to return the class of its proxied element. + + """ + return self.__class__ + + @util.memoized_property + def _cloned_set(self): + """Return the set consisting all cloned ancestors of this + ClauseElement. + + Includes this ClauseElement. This accessor tends to be used for + FromClause objects to identify 'equivalent' FROM clauses, regardless + of transformative operations. + + """ + s = util.column_set() + f = self + while f is not None: + s.add(f) + f = f._is_clone_of + return s + + def __getstate__(self): + d = self.__dict__.copy() + d.pop('_is_clone_of', None) + return d + + def _annotate(self, values): + """return a copy of this ClauseElement with annotations + updated by the given dictionary. + + """ + return Annotated(self, values) + + def _with_annotations(self, values): + """return a copy of this ClauseElement with annotations + replaced by the given dictionary. + + """ + return Annotated(self, values) + + def _deannotate(self, values=None, clone=False): + """return a copy of this :class:`.ClauseElement` with annotations + removed. + + :param values: optional tuple of individual values + to remove. + + """ + if clone: + # clone is used when we are also copying + # the expression for a deep deannotation + return self._clone() + else: + # if no clone, since we have no annotations we return + # self + return self + + def _execute_on_connection(self, connection, multiparams, params): + return connection._execute_clauseelement(self, multiparams, params) + + def unique_params(self, *optionaldict, **kwargs): + """Return a copy with :func:`bindparam()` elements replaced. + + Same functionality as ``params()``, except adds `unique=True` + to affected bind parameters so that multiple statements can be + used. + + """ + return self._params(True, optionaldict, kwargs) + + def params(self, *optionaldict, **kwargs): + """Return a copy with :func:`bindparam()` elements replaced. + + Returns a copy of this ClauseElement with :func:`bindparam()` + elements replaced with values taken from the given dictionary:: + + >>> clause = column('x') + bindparam('foo') + >>> print clause.compile().params + {'foo':None} + >>> print clause.params({'foo':7}).compile().params + {'foo':7} + + """ + return self._params(False, optionaldict, kwargs) + + def _params(self, unique, optionaldict, kwargs): + if len(optionaldict) == 1: + kwargs.update(optionaldict[0]) + elif len(optionaldict) > 1: + raise exc.ArgumentError( + "params() takes zero or one positional dictionary argument") + + def visit_bindparam(bind): + if bind.key in kwargs: + bind.value = kwargs[bind.key] + bind.required = False + if unique: + bind._convert_to_unique() + return cloned_traverse(self, {}, {'bindparam': visit_bindparam}) + + def compare(self, other, **kw): + """Compare this ClauseElement to the given ClauseElement. + + Subclasses should override the default behavior, which is a + straight identity comparison. + + \**kw are arguments consumed by subclass compare() methods and + may be used to modify the criteria for comparison. + (see :class:`.ColumnElement`) + + """ + return self is other + + def _copy_internals(self, clone=_clone, **kw): + """Reassign internal elements to be clones of themselves. + + Called during a copy-and-traverse operation on newly + shallow-copied elements to create a deep copy. + + The given clone function should be used, which may be applying + additional transformations to the element (i.e. replacement + traversal, cloned traversal, annotations). + + """ + pass + + def get_children(self, **kwargs): + """Return immediate child elements of this :class:`.ClauseElement`. + + This is used for visit traversal. + + \**kwargs may contain flags that change the collection that is + returned, for example to return a subset of items in order to + cut down on larger traversals, or to return child items from a + different context (such as schema-level collections instead of + clause-level). + + """ + return [] + + def self_group(self, against=None): + """Apply a 'grouping' to this :class:`.ClauseElement`. + + This method is overridden by subclasses to return a + "grouping" construct, i.e. parenthesis. In particular + it's used by "binary" expressions to provide a grouping + around themselves when placed into a larger expression, + as well as by :func:`.select` constructs when placed into + the FROM clause of another :func:`.select`. (Note that + subqueries should be normally created using the + :meth:`.Select.alias` method, as many platforms require + nested SELECT statements to be named). + + As expressions are composed together, the application of + :meth:`self_group` is automatic - end-user code should never + need to use this method directly. Note that SQLAlchemy's + clause constructs take operator precedence into account - + so parenthesis might not be needed, for example, in + an expression like ``x OR (y AND z)`` - AND takes precedence + over OR. + + The base :meth:`self_group` method of :class:`.ClauseElement` + just returns self. + """ + return self + + @util.dependencies("sqlalchemy.engine.default") + def compile(self, default, bind=None, dialect=None, **kw): + """Compile this SQL expression. + + The return value is a :class:`~.Compiled` object. + Calling ``str()`` or ``unicode()`` on the returned value will yield a + string representation of the result. The + :class:`~.Compiled` object also can return a + dictionary of bind parameter names and values + using the ``params`` accessor. + + :param bind: An ``Engine`` or ``Connection`` from which a + ``Compiled`` will be acquired. This argument takes precedence over + this :class:`.ClauseElement`'s bound engine, if any. + + :param column_keys: Used for INSERT and UPDATE statements, a list of + column names which should be present in the VALUES clause of the + compiled statement. If ``None``, all columns from the target table + object are rendered. + + :param dialect: A ``Dialect`` instance from which a ``Compiled`` + will be acquired. This argument takes precedence over the `bind` + argument as well as this :class:`.ClauseElement`'s bound engine, if + any. + + :param inline: Used for INSERT statements, for a dialect which does + not support inline retrieval of newly generated primary key + columns, will force the expression used to create the new primary + key value to be rendered inline within the INSERT statement's + VALUES clause. This typically refers to Sequence execution but may + also refer to any server-side default generation function + associated with a primary key `Column`. + + """ + + if not dialect: + if bind: + dialect = bind.dialect + elif self.bind: + dialect = self.bind.dialect + bind = self.bind + else: + dialect = default.DefaultDialect() + return self._compiler(dialect, bind=bind, **kw) + + def _compiler(self, dialect, **kw): + """Return a compiler appropriate for this ClauseElement, given a + Dialect.""" + + return dialect.statement_compiler(dialect, self, **kw) + + def __str__(self): + if util.py3k: + return str(self.compile()) + else: + return unicode(self.compile()).encode('ascii', 'backslashreplace') + + def __and__(self, other): + return and_(self, other) + + def __or__(self, other): + return or_(self, other) + + def __invert__(self): + if hasattr(self, 'negation_clause'): + return self.negation_clause + else: + return self._negate() + + def __bool__(self): + raise TypeError("Boolean value of this clause is not defined") + + __nonzero__ = __bool__ + + def _negate(self): + return UnaryExpression( + self.self_group(against=operators.inv), + operator=operators.inv, + negate=None) + + def __repr__(self): + friendly = getattr(self, 'description', None) + if friendly is None: + return object.__repr__(self) + else: + return '<%s.%s at 0x%x; %s>' % ( + self.__module__, self.__class__.__name__, id(self), friendly) + + + +class ColumnElement(ClauseElement, operators.ColumnOperators): + """Represent a column-oriented SQL expression suitable for usage in the + "columns" clause, WHERE clause etc. of a statement. + + While the most familiar kind of :class:`.ColumnElement` is the + :class:`.Column` object, :class:`.ColumnElement` serves as the basis + for any unit that may be present in a SQL expression, including + the expressions themselves, SQL functions, bound parameters, + literal expressions, keywords such as ``NULL``, etc. + :class:`.ColumnElement` is the ultimate base class for all such elements. + + A :class:`.ColumnElement` provides the ability to generate new + :class:`.ColumnElement` + objects using Python expressions. This means that Python operators + such as ``==``, ``!=`` and ``<`` are overloaded to mimic SQL operations, + and allow the instantiation of further :class:`.ColumnElement` instances + which are composed from other, more fundamental :class:`.ColumnElement` + objects. For example, two :class:`.ColumnClause` objects can be added + together with the addition operator ``+`` to produce + a :class:`.BinaryExpression`. + Both :class:`.ColumnClause` and :class:`.BinaryExpression` are subclasses + of :class:`.ColumnElement`:: + + >>> from sqlalchemy.sql import column + >>> column('a') + column('b') + + >>> print column('a') + column('b') + a + b + + :class:`.ColumnElement` supports the ability to be a *proxy* element, + which indicates that the :class:`.ColumnElement` may be associated with + a :class:`.Selectable` which was derived from another :class:`.Selectable`. + An example of a "derived" :class:`.Selectable` is an :class:`.Alias` of a + :class:`~sqlalchemy.schema.Table`. For the ambitious, an in-depth + discussion of this concept can be found at + `Expression Transformations `_. + + """ + + __visit_name__ = 'column' + primary_key = False + foreign_keys = [] + _label = None + _key_label = None + _alt_names = () + + def self_group(self, against=None): + if against in (operators.and_, operators.or_, operators._asbool) and \ + self.type._type_affinity is type_api.BOOLEANTYPE._type_affinity: + return AsBoolean(self, operators.istrue, operators.isfalse) + else: + return self + + def _negate(self): + if self.type._type_affinity is type_api.BOOLEANTYPE._type_affinity: + return AsBoolean(self, operators.isfalse, operators.istrue) + else: + return super(ColumnElement, self)._negate() + + @util.memoized_property + def type(self): + return type_api.NULLTYPE + + @util.memoized_property + def comparator(self): + return self.type.comparator_factory(self) + + def __getattr__(self, key): + try: + return getattr(self.comparator, key) + except AttributeError: + raise AttributeError( + 'Neither %r object nor %r object has an attribute %r' % ( + type(self).__name__, + type(self.comparator).__name__, + key) + ) + + def operate(self, op, *other, **kwargs): + return op(self.comparator, *other, **kwargs) + + def reverse_operate(self, op, other, **kwargs): + return op(other, self.comparator, **kwargs) + + def _bind_param(self, operator, obj): + return BindParameter(None, obj, + _compared_to_operator=operator, + _compared_to_type=self.type, unique=True) + + @property + def expression(self): + """Return a column expression. + + Part of the inspection interface; returns self. + + """ + return self + + @property + def _select_iterable(self): + return (self, ) + + @util.memoized_property + def base_columns(self): + return util.column_set(c for c in self.proxy_set + if not hasattr(c, '_proxies')) + + @util.memoized_property + def proxy_set(self): + s = util.column_set([self]) + if hasattr(self, '_proxies'): + for c in self._proxies: + s.update(c.proxy_set) + return s + + def shares_lineage(self, othercolumn): + """Return True if the given :class:`.ColumnElement` + has a common ancestor to this :class:`.ColumnElement`.""" + + return bool(self.proxy_set.intersection(othercolumn.proxy_set)) + + def _compare_name_for_result(self, other): + """Return True if the given column element compares to this one + when targeting within a result row.""" + + return hasattr(other, 'name') and hasattr(self, 'name') and \ + other.name == self.name + + def _make_proxy(self, selectable, name=None, name_is_truncatable=False, **kw): + """Create a new :class:`.ColumnElement` representing this + :class:`.ColumnElement` as it appears in the select list of a + descending selectable. + + """ + if name is None: + name = self.anon_label + try: + key = str(self) + except exc.UnsupportedCompilationError: + key = self.anon_label + else: + key = name + co = ColumnClause( + _as_truncated(name) if name_is_truncatable else name, + type_=getattr(self, 'type', None), + _selectable=selectable + ) + co._proxies = [self] + if selectable._is_clone_of is not None: + co._is_clone_of = \ + selectable._is_clone_of.columns.get(key) + selectable._columns[key] = co + return co + + def compare(self, other, use_proxies=False, equivalents=None, **kw): + """Compare this ColumnElement to another. + + Special arguments understood: + + :param use_proxies: when True, consider two columns that + share a common base column as equivalent (i.e. shares_lineage()) + + :param equivalents: a dictionary of columns as keys mapped to sets + of columns. If the given "other" column is present in this + dictionary, if any of the columns in the corresponding set() pass the + comparison test, the result is True. This is used to expand the + comparison to other columns that may be known to be equivalent to + this one via foreign key or other criterion. + + """ + to_compare = (other, ) + if equivalents and other in equivalents: + to_compare = equivalents[other].union(to_compare) + + for oth in to_compare: + if use_proxies and self.shares_lineage(oth): + return True + elif hash(oth) == hash(self): + return True + else: + return False + + def label(self, name): + """Produce a column label, i.e. `` AS ``. + + This is a shortcut to the :func:`~.expression.label` function. + + if 'name' is None, an anonymous label name will be generated. + + """ + return Label(name, self, self.type) + + @util.memoized_property + def anon_label(self): + """provides a constant 'anonymous label' for this ColumnElement. + + This is a label() expression which will be named at compile time. + The same label() is returned each time anon_label is called so + that expressions can reference anon_label multiple times, producing + the same label name at compile time. + + the compiler uses this function automatically at compile time + for expressions that are known to be 'unnamed' like binary + expressions and function calls. + + """ + return _anonymous_label('%%(%d %s)s' % (id(self), getattr(self, + 'name', 'anon'))) + + + +class BindParameter(ColumnElement): + """Represent a bound parameter value. + + """ + + __visit_name__ = 'bindparam' + + _is_crud = False + + def __init__(self, key, value=NO_ARG, type_=None, + unique=False, required=NO_ARG, + quote=None, callable_=None, + isoutparam=False, + _compared_to_operator=None, + _compared_to_type=None): + """Construct a new :class:`.BindParameter`. + + :param key: + the key for this bind param. Will be used in the generated + SQL statement for dialects that use named parameters. This + value may be modified when part of a compilation operation, + if other :class:`BindParameter` objects exist with the same + key, or if its length is too long and truncation is + required. + + :param value: + Initial value for this bind param. This value may be + overridden by the dictionary of parameters sent to statement + compilation/execution. + + Defaults to ``None``, however if neither ``value`` nor + ``callable`` are passed explicitly, the ``required`` flag will be + set to ``True`` which has the effect of requiring a value be present + when the statement is actually executed. + + .. versionchanged:: 0.8 The ``required`` flag is set to ``True`` + automatically if ``value`` or ``callable`` is not passed. + + :param callable\_: + A callable function that takes the place of "value". The function + will be called at statement execution time to determine the + ultimate value. Used for scenarios where the actual bind + value cannot be determined at the point at which the clause + construct is created, but embedded bind values are still desirable. + + :param type\_: + A ``TypeEngine`` object that will be used to pre-process the + value corresponding to this :class:`BindParameter` at + execution time. + + :param unique: + if True, the key name of this BindParamClause will be + modified if another :class:`BindParameter` of the same name + already has been located within the containing + :class:`.ClauseElement`. + + :param required: + If ``True``, a value is required at execution time. If not passed, + is set to ``True`` or ``False`` based on whether or not + one of ``value`` or ``callable`` were passed.. + + .. versionchanged:: 0.8 If the ``required`` flag is not specified, + it will be set automatically to ``True`` or ``False`` depending + on whether or not the ``value`` or ``callable`` parameters + were specified. + + :param quote: + True if this parameter name requires quoting and is not + currently known as a SQLAlchemy reserved word; this currently + only applies to the Oracle backend. + + :param isoutparam: + if True, the parameter should be treated like a stored procedure + "OUT" parameter. + + .. seealso:: + + :func:`.outparam` + + + + """ + if isinstance(key, ColumnClause): + type_ = key.type + key = key.name + if required is NO_ARG: + required = (value is NO_ARG and callable_ is None) + if value is NO_ARG: + value = None + + if quote is not None: + key = quoted_name(key, quote) + + if unique: + self.key = _anonymous_label('%%(%d %s)s' % (id(self), key + or 'param')) + else: + self.key = key or _anonymous_label('%%(%d param)s' + % id(self)) + + # identifying key that won't change across + # clones, used to identify the bind's logical + # identity + self._identifying_key = self.key + + # key that was passed in the first place, used to + # generate new keys + self._orig_key = key or 'param' + + self.unique = unique + self.value = value + self.callable = callable_ + self.isoutparam = isoutparam + self.required = required + if type_ is None: + if _compared_to_type is not None: + self.type = \ + _compared_to_type.coerce_compared_value( + _compared_to_operator, value) + else: + self.type = type_api._type_map.get(type(value), + type_api.NULLTYPE) + elif isinstance(type_, type): + self.type = type_() + else: + self.type = type_ + + def _with_value(self, value): + """Return a copy of this :class:`.BindParameter` with the given value set.""" + cloned = self._clone() + cloned.value = value + cloned.callable = None + cloned.required = False + if cloned.type is type_api.NULLTYPE: + cloned.type = type_api._type_map.get(type(value), + type_api.NULLTYPE) + return cloned + + @property + def effective_value(self): + """Return the value of this bound parameter, + taking into account if the ``callable`` parameter + was set. + + The ``callable`` value will be evaluated + and returned if present, else ``value``. + + """ + if self.callable: + return self.callable() + else: + return self.value + + def _clone(self): + c = ClauseElement._clone(self) + if self.unique: + c.key = _anonymous_label('%%(%d %s)s' % (id(c), c._orig_key + or 'param')) + return c + + def _convert_to_unique(self): + if not self.unique: + self.unique = True + self.key = _anonymous_label('%%(%d %s)s' % (id(self), + self._orig_key or 'param')) + + def compare(self, other, **kw): + """Compare this :class:`BindParameter` to the given + clause.""" + + return isinstance(other, BindParameter) \ + and self.type._compare_type_affinity(other.type) \ + and self.value == other.value + + def __getstate__(self): + """execute a deferred value for serialization purposes.""" + + d = self.__dict__.copy() + v = self.value + if self.callable: + v = self.callable() + d['callable'] = None + d['value'] = v + return d + + def __repr__(self): + return 'BindParameter(%r, %r, type_=%r)' % (self.key, + self.value, self.type) + + +class TypeClause(ClauseElement): + """Handle a type keyword in a SQL statement. + + Used by the ``Case`` statement. + + """ + + __visit_name__ = 'typeclause' + + def __init__(self, type): + self.type = type + + +class TextClause(Executable, ClauseElement): + """Represent a literal SQL text fragment. + + Public constructor is the :func:`text()` function. + + """ + + __visit_name__ = 'textclause' + + _bind_params_regex = re.compile(r'(?`` + to specify bind parameters; they will be compiled to their + engine-specific format. + + :param autocommit: + Deprecated. Use .execution_options(autocommit=) + to set the autocommit option. + + :param bind: + an optional connection or engine to be used for this text query. + + :param bindparams: + Deprecated. A list of :func:`.bindparam` instances used to + provide information about parameters embedded in the statement. + This argument now invokes the :meth:`.TextClause.bindparams` + method on the construct before returning it. E.g.:: + + stmt = text("SELECT * FROM table WHERE id=:id", + bindparams=[bindparam('id', value=5, type_=Integer)]) + + Is equivalent to:: + + stmt = text("SELECT * FROM table WHERE id=:id").\\ + bindparams(bindparam('id', value=5, type_=Integer)) + + .. deprecated:: 0.9.0 the :meth:`.TextClause.bindparams` method + supersedes the ``bindparams`` argument to :func:`.text`. + + :param typemap: + Deprecated. A dictionary mapping the names of columns + represented in the columns clause of a ``SELECT`` statement + to type objects, + which will be used to perform post-processing on columns within + the result set. This parameter now invokes the :meth:`.TextClause.columns` + method, which returns a :class:`.TextAsFrom` construct that gains + a ``.c`` collection and can be embedded in other expressions. E.g.:: + + stmt = text("SELECT * FROM table", + typemap={'id': Integer, 'name': String}, + ) + + Is equivalent to:: + + stmt = text("SELECT * FROM table").columns(id=Integer, name=String) + + Or alternatively:: + + from sqlalchemy.sql import column + stmt = text("SELECT * FROM table").columns( + column('id', Integer), + column('name', String) + ) + + .. deprecated:: 0.9.0 the :meth:`.TextClause.columns` method + supersedes the ``typemap`` argument to :func:`.text`. + + """ + stmt = TextClause(text, bind=bind) + if bindparams: + stmt = stmt.bindparams(*bindparams) + if typemap: + stmt = stmt.columns(**typemap) + if autocommit is not None: + util.warn_deprecated('autocommit on text() is deprecated. ' + 'Use .execution_options(autocommit=True)') + stmt = stmt.execution_options(autocommit=autocommit) + + return stmt + + @_generative + def bindparams(self, *binds, **names_to_values): + """Establish the values and/or types of bound parameters within + this :class:`.TextClause` construct. + + Given a text construct such as:: + + from sqlalchemy import text + stmt = text("SELECT id, name FROM user WHERE name=:name " + "AND timestamp=:timestamp") + + the :meth:`.TextClause.bindparams` method can be used to establish + the initial value of ``:name`` and ``:timestamp``, + using simple keyword arguments:: + + stmt = stmt.bindparams(name='jack', + timestamp=datetime.datetime(2012, 10, 8, 15, 12, 5)) + + Where above, new :class:`.BindParameter` objects + will be generated with the names ``name`` and ``timestamp``, and + values of ``jack`` and ``datetime.datetime(2012, 10, 8, 15, 12, 5)``, + respectively. The types will be + inferred from the values given, in this case :class:`.String` and + :class:`.DateTime`. + + When specific typing behavior is needed, the positional ``*binds`` + argument can be used in which to specify :func:`.bindparam` constructs + directly. These constructs must include at least the ``key`` argument, + then an optional value and type:: + + from sqlalchemy import bindparam + stmt = stmt.bindparams( + bindparam('name', value='jack', type_=String), + bindparam('timestamp', type_=DateTime) + ) + + Above, we specified the type of :class:`.DateTime` for the ``timestamp`` + bind, and the type of :class:`.String` for the ``name`` bind. In + the case of ``name`` we also set the default value of ``"jack"``. + + Additional bound parameters can be supplied at statement execution + time, e.g.:: + + result = connection.execute(stmt, + timestamp=datetime.datetime(2012, 10, 8, 15, 12, 5)) + + The :meth:`.TextClause.bindparams` method can be called repeatedly, where + it will re-use existing :class:`.BindParameter` objects to add new information. + For example, we can call :meth:`.TextClause.bindparams` first with + typing information, and a second time with value information, and it + will be combined:: + + stmt = text("SELECT id, name FROM user WHERE name=:name " + "AND timestamp=:timestamp") + stmt = stmt.bindparams( + bindparam('name', type_=String), + bindparam('timestamp', type_=DateTime) + ) + stmt = stmt.bindparams( + name='jack', + timestamp=datetime.datetime(2012, 10, 8, 15, 12, 5) + ) + + + .. versionadded:: 0.9.0 The :meth:`.TextClause.bindparams` method supersedes + the argument ``bindparams`` passed to :func:`~.expression.text`. + + + """ + self._bindparams = new_params = self._bindparams.copy() + + for bind in binds: + try: + existing = new_params[bind.key] + except KeyError: + raise exc.ArgumentError( + "This text() construct doesn't define a " + "bound parameter named %r" % bind.key) + else: + new_params[existing.key] = bind + + for key, value in names_to_values.items(): + try: + existing = new_params[key] + except KeyError: + raise exc.ArgumentError( + "This text() construct doesn't define a " + "bound parameter named %r" % key) + else: + new_params[key] = existing._with_value(value) + + + + @util.dependencies('sqlalchemy.sql.selectable') + def columns(self, selectable, *cols, **types): + """Turn this :class:`.TextClause` object into a :class:`.TextAsFrom` + object that can be embedded into another statement. + + This function essentially bridges the gap between an entirely + textual SELECT statement and the SQL expression language concept + of a "selectable":: + + from sqlalchemy.sql import column, text + + stmt = text("SELECT id, name FROM some_table") + stmt = stmt.columns(column('id'), column('name')).alias('st') + + stmt = select([mytable]).\\ + select_from( + mytable.join(stmt, mytable.c.name == stmt.c.name) + ).where(stmt.c.id > 5) + + Above, we used untyped :func:`.column` elements. These can also have + types specified, which will impact how the column behaves in expressions + as well as determining result set behavior:: + + stmt = text("SELECT id, name, timestamp FROM some_table") + stmt = stmt.columns( + column('id', Integer), + column('name', Unicode), + column('timestamp', DateTime) + ) + + for id, name, timestamp in connection.execute(stmt): + print(id, name, timestamp) + + Keyword arguments allow just the names and types of columns to be specified, + where the :func:`.column` elements will be generated automatically:: + + stmt = text("SELECT id, name, timestamp FROM some_table") + stmt = stmt.columns( + id=Integer, + name=Unicode, + timestamp=DateTime + ) + + for id, name, timestamp in connection.execute(stmt): + print(id, name, timestamp) + + The :meth:`.TextClause.columns` method provides a direct + route to calling :meth:`.FromClause.alias` as well as :meth:`.SelectBase.cte` + against a textual SELECT statement:: + + stmt = stmt.columns(id=Integer, name=String).cte('st') + + stmt = select([sometable]).where(sometable.c.id == stmt.c.id) + + .. versionadded:: 0.9.0 :func:`.text` can now be converted into a fully + featured "selectable" construct using the :meth:`.TextClause.columns` + method. This method supersedes the ``typemap`` argument to + :func:`.text`. + + """ + + col_by_name = dict( + (col.key, col) for col in cols + ) + for key, type_ in types.items(): + col_by_name[key] = ColumnClause(key, type_) + + return selectable.TextAsFrom(self, list(col_by_name.values())) + + @property + def type(self): + return type_api.NULLTYPE + + @property + def comparator(self): + return self.type.comparator_factory(self) + + def self_group(self, against=None): + if against is operators.in_op: + return Grouping(self) + else: + return self + + def _copy_internals(self, clone=_clone, **kw): + self._bindparams = dict((b.key, clone(b, **kw)) + for b in self._bindparams.values()) + + def get_children(self, **kwargs): + return list(self._bindparams.values()) + + +class Null(ColumnElement): + """Represent the NULL keyword in a SQL statement. + + :class:`.Null` is accessed as a constant via the + :func:`.null` function. + + """ + + __visit_name__ = 'null' + + @util.memoized_property + def type(self): + return type_api.NULLTYPE + + @classmethod + def _singleton(cls): + """Return a constant :class:`.Null` construct.""" + + return NULL + + def compare(self, other): + return isinstance(other, Null) + + +class False_(ColumnElement): + """Represent the ``false`` keyword, or equivalent, in a SQL statement. + + :class:`.False_` is accessed as a constant via the + :func:`.false` function. + + """ + + __visit_name__ = 'false' + + @util.memoized_property + def type(self): + return type_api.BOOLEANTYPE + + def _negate(self): + return TRUE + + @classmethod + def _singleton(cls): + """Return a constant :class:`.False_` construct. + + E.g.:: + + >>> from sqlalchemy import false + >>> print select([t.c.x]).where(false()) + SELECT x FROM t WHERE false + + A backend which does not support true/false constants will render as + an expression against 1 or 0:: + + >>> print select([t.c.x]).where(false()) + SELECT x FROM t WHERE 0 = 1 + + The :func:`.true` and :func:`.false` constants also feature + "short circuit" operation within an :func:`.and_` or :func:`.or_` + conjunction:: + + >>> print select([t.c.x]).where(or_(t.c.x > 5, true())) + SELECT x FROM t WHERE true + + >>> print select([t.c.x]).where(and_(t.c.x > 5, false())) + SELECT x FROM t WHERE false + + .. versionchanged:: 0.9 :func:`.true` and :func:`.false` feature + better integrated behavior within conjunctions and on dialects + that don't support true/false constants. + + .. seealso:: + + :func:`.true` + + """ + + return FALSE + + def compare(self, other): + return isinstance(other, False_) + +class True_(ColumnElement): + """Represent the ``true`` keyword, or equivalent, in a SQL statement. + + :class:`.True_` is accessed as a constant via the + :func:`.true` function. + + """ + + __visit_name__ = 'true' + + @util.memoized_property + def type(self): + return type_api.BOOLEANTYPE + + def _negate(self): + return FALSE + + @classmethod + def _ifnone(cls, other): + if other is None: + return cls._singleton() + else: + return other + + @classmethod + def _singleton(cls): + """Return a constant :class:`.True_` construct. + + E.g.:: + + >>> from sqlalchemy import true + >>> print select([t.c.x]).where(true()) + SELECT x FROM t WHERE true + + A backend which does not support true/false constants will render as + an expression against 1 or 0:: + + >>> print select([t.c.x]).where(true()) + SELECT x FROM t WHERE 1 = 1 + + The :func:`.true` and :func:`.false` constants also feature + "short circuit" operation within an :func:`.and_` or :func:`.or_` + conjunction:: + + >>> print select([t.c.x]).where(or_(t.c.x > 5, true())) + SELECT x FROM t WHERE true + + >>> print select([t.c.x]).where(and_(t.c.x > 5, false())) + SELECT x FROM t WHERE false + + .. versionchanged:: 0.9 :func:`.true` and :func:`.false` feature + better integrated behavior within conjunctions and on dialects + that don't support true/false constants. + + .. seealso:: + + :func:`.false` + + """ + + return TRUE + + def compare(self, other): + return isinstance(other, True_) + +NULL = Null() +FALSE = False_() +TRUE = True_() + +class ClauseList(ClauseElement): + """Describe a list of clauses, separated by an operator. + + By default, is comma-separated, such as a column listing. + + """ + __visit_name__ = 'clauselist' + + def __init__(self, *clauses, **kwargs): + self.operator = kwargs.pop('operator', operators.comma_op) + self.group = kwargs.pop('group', True) + self.group_contents = kwargs.pop('group_contents', True) + if self.group_contents: + self.clauses = [ + _literal_as_text(clause).self_group(against=self.operator) + for clause in clauses] + else: + self.clauses = [ + _literal_as_text(clause) + for clause in clauses] + + def __iter__(self): + return iter(self.clauses) + + def __len__(self): + return len(self.clauses) + + @property + def _select_iterable(self): + return iter(self) + + def append(self, clause): + if self.group_contents: + self.clauses.append(_literal_as_text(clause).\ + self_group(against=self.operator)) + else: + self.clauses.append(_literal_as_text(clause)) + + def _copy_internals(self, clone=_clone, **kw): + self.clauses = [clone(clause, **kw) for clause in self.clauses] + + def get_children(self, **kwargs): + return self.clauses + + @property + def _from_objects(self): + return list(itertools.chain(*[c._from_objects for c in self.clauses])) + + def self_group(self, against=None): + if self.group and operators.is_precedent(self.operator, against): + return Grouping(self) + else: + return self + + def compare(self, other, **kw): + """Compare this :class:`.ClauseList` to the given :class:`.ClauseList`, + including a comparison of all the clause items. + + """ + if not isinstance(other, ClauseList) and len(self.clauses) == 1: + return self.clauses[0].compare(other, **kw) + elif isinstance(other, ClauseList) and \ + len(self.clauses) == len(other.clauses): + for i in range(0, len(self.clauses)): + if not self.clauses[i].compare(other.clauses[i], **kw): + return False + else: + return self.operator == other.operator + else: + return False + + + +class BooleanClauseList(ClauseList, ColumnElement): + __visit_name__ = 'clauselist' + + def __init__(self, *arg, **kw): + raise NotImplementedError( + "BooleanClauseList has a private constructor") + + @classmethod + def _construct(cls, operator, continue_on, skip_on, *clauses, **kw): + convert_clauses = [] + + clauses = util.coerce_generator_arg(clauses) + for clause in clauses: + clause = _literal_as_text(clause) + + if isinstance(clause, continue_on): + continue + elif isinstance(clause, skip_on): + return clause.self_group(against=operators._asbool) + + convert_clauses.append(clause) + + if len(convert_clauses) == 1: + return convert_clauses[0].self_group(against=operators._asbool) + elif not convert_clauses and clauses: + return clauses[0].self_group(against=operators._asbool) + + convert_clauses = [c.self_group(against=operator) + for c in convert_clauses] + + self = cls.__new__(cls) + self.clauses = convert_clauses + self.group = True + self.operator = operator + self.group_contents = True + self.type = type_api.BOOLEANTYPE + return self + + @classmethod + def and_(cls, *clauses): + """Join a list of clauses together using the ``AND`` operator. + + The ``&`` operator is also overloaded on all :class:`.ColumnElement` + subclasses to produce the + same result. + + """ + return cls._construct(operators.and_, True_, False_, *clauses) + + @classmethod + def or_(cls, *clauses): + """Join a list of clauses together using the ``OR`` operator. + + The ``|`` operator is also overloaded on all + :class:`.ColumnElement` subclasses to produce the + same result. + + """ + return cls._construct(operators.or_, False_, True_, *clauses) + + @property + def _select_iterable(self): + return (self, ) + + def self_group(self, against=None): + if not self.clauses: + return self + else: + return super(BooleanClauseList, self).self_group(against=against) + + def _negate(self): + return ClauseList._negate(self) + + +and_ = BooleanClauseList.and_ +or_ = BooleanClauseList.or_ + +class Tuple(ClauseList, ColumnElement): + """Represent a SQL tuple.""" + + def __init__(self, *clauses, **kw): + """Return a :class:`.Tuple`. + + Main usage is to produce a composite IN construct:: + + from sqlalchemy import tuple_ + + tuple_(table.c.col1, table.c.col2).in_( + [(1, 2), (5, 12), (10, 19)] + ) + + .. warning:: + + The composite IN construct is not supported by all backends, + and is currently known to work on Postgresql and MySQL, + but not SQLite. Unsupported backends will raise + a subclass of :class:`~sqlalchemy.exc.DBAPIError` when such + an expression is invoked. + + """ + + clauses = [_literal_as_binds(c) for c in clauses] + self.type = kw.pop('type_', None) + if self.type is None: + self.type = _type_from_args(clauses) + super(Tuple, self).__init__(*clauses, **kw) + + @property + def _select_iterable(self): + return (self, ) + + def _bind_param(self, operator, obj): + return Tuple(*[ + BindParameter(None, o, _compared_to_operator=operator, + _compared_to_type=self.type, unique=True) + for o in obj + ]).self_group() + + +class Case(ColumnElement): + """Represent a SQL ``CASE`` construct. + + + """ + __visit_name__ = 'case' + + def __init__(self, whens, value=None, else_=None): + """Produce a :class:`.Case` object. + + :param whens: A sequence of pairs, or alternatively a dict, + to be translated into "WHEN / THEN" clauses. + + :param value: Optional for simple case statements, produces + a column expression as in "CASE WHEN ..." + + :param else\_: Optional as well, for case defaults produces + the "ELSE" portion of the "CASE" statement. + + The expressions used for THEN and ELSE, + when specified as strings, will be interpreted + as bound values. To specify textual SQL expressions + for these, use the :func:`literal_column` + construct. + + The expressions used for the WHEN criterion + may only be literal strings when "value" is + present, i.e. CASE table.somecol WHEN "x" THEN "y". + Otherwise, literal strings are not accepted + in this position, and either the text() + or literal() constructs must be used to + interpret raw string values. + + Usage examples:: + + case([(orderline.c.qty > 100, item.c.specialprice), + (orderline.c.qty > 10, item.c.bulkprice) + ], else_=item.c.regularprice) + + case(value=emp.c.type, whens={ + 'engineer': emp.c.salary * 1.1, + 'manager': emp.c.salary * 3, + }) + + Using :func:`.literal_column()`, to allow for databases that + do not support bind parameters in the ``then`` clause. The type + can be specified which determines the type of the :func:`case()` construct + overall:: + + case([(orderline.c.qty > 100, + literal_column("'greaterthan100'", String)), + (orderline.c.qty > 10, literal_column("'greaterthan10'", + String)) + ], else_=literal_column("'lethan10'", String)) + + """ + + try: + whens = util.dictlike_iteritems(whens) + except TypeError: + pass + + if value is not None: + whenlist = [ + (_literal_as_binds(c).self_group(), + _literal_as_binds(r)) for (c, r) in whens + ] + else: + whenlist = [ + (_no_literals(c).self_group(), + _literal_as_binds(r)) for (c, r) in whens + ] + + if whenlist: + type_ = list(whenlist[-1])[-1].type + else: + type_ = None + + if value is None: + self.value = None + else: + self.value = _literal_as_binds(value) + + self.type = type_ + self.whens = whenlist + if else_ is not None: + self.else_ = _literal_as_binds(else_) + else: + self.else_ = None + + def _copy_internals(self, clone=_clone, **kw): + if self.value is not None: + self.value = clone(self.value, **kw) + self.whens = [(clone(x, **kw), clone(y, **kw)) + for x, y in self.whens] + if self.else_ is not None: + self.else_ = clone(self.else_, **kw) + + def get_children(self, **kwargs): + if self.value is not None: + yield self.value + for x, y in self.whens: + yield x + yield y + if self.else_ is not None: + yield self.else_ + + @property + def _from_objects(self): + return list(itertools.chain(*[x._from_objects for x in + self.get_children()])) + + +def literal_column(text, type_=None): + """Return a textual column expression, as would be in the columns + clause of a ``SELECT`` statement. + + The object returned supports further expressions in the same way as any + other column object, including comparison, math and string operations. + The type\_ parameter is important to determine proper expression behavior + (such as, '+' means string concatenation or numerical addition based on + the type). + + :param text: the text of the expression; can be any SQL expression. + Quoting rules will not be applied. To specify a column-name expression + which should be subject to quoting rules, use the :func:`column` + function. + + :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` + object which will + provide result-set translation and additional expression semantics for + this column. If left as None the type will be NullType. + + """ + return ColumnClause(text, type_=type_, is_literal=True) + + + +class Cast(ColumnElement): + """Represent the SQL ``CAST`` construct.""" + + __visit_name__ = 'cast' + + def __init__(self, expression, type_): + """Return a :class:`.Cast` object. + + Equivalent of SQL ``CAST(clause AS totype)``. + + E.g.:: + + cast(table.c.unit_price * table.c.qty, Numeric(10,4)) + + or:: + + cast(table.c.timestamp, DATE) + + :param expression: Column-oriented expression. + :param type_: A :class:`.TypeEngine` class or instance indicating + the type to which the CAST should apply. + + .. seealso:: + + :func:`.type_coerce` - Python-side type coercion without emitting + CAST. + + """ + self.type = type_api.to_instance(type_) + self.clause = _literal_as_binds(expression, type_=self.type) + self.typeclause = TypeClause(self.type) + + def _copy_internals(self, clone=_clone, **kw): + self.clause = clone(self.clause, **kw) + self.typeclause = clone(self.typeclause, **kw) + + def get_children(self, **kwargs): + return self.clause, self.typeclause + + @property + def _from_objects(self): + return self.clause._from_objects + + +class Extract(ColumnElement): + """Represent a SQL EXTRACT clause, ``extract(field FROM expr)``.""" + + __visit_name__ = 'extract' + + def __init__(self, field, expr, **kwargs): + """Return a :class:`.Extract` construct. + + This is typically available as :func:`.extract` + as well as ``func.extract`` from the + :data:`.func` namespace. + + """ + self.type = type_api.INTEGERTYPE + self.field = field + self.expr = _literal_as_binds(expr, None) + + def _copy_internals(self, clone=_clone, **kw): + self.expr = clone(self.expr, **kw) + + def get_children(self, **kwargs): + return self.expr, + + @property + def _from_objects(self): + return self.expr._from_objects + + +class UnaryExpression(ColumnElement): + """Define a 'unary' expression. + + A unary expression has a single column expression + and an operator. The operator can be placed on the left + (where it is called the 'operator') or right (where it is called the + 'modifier') of the column expression. + + """ + __visit_name__ = 'unary' + + def __init__(self, element, operator=None, modifier=None, + type_=None, negate=None): + self.operator = operator + self.modifier = modifier + self.element = element.self_group(against=self.operator or self.modifier) + self.type = type_api.to_instance(type_) + self.negate = negate + + @classmethod + def _create_nullsfirst(cls, column): + """Return a NULLS FIRST ``ORDER BY`` clause element. + + e.g.:: + + someselect.order_by(desc(table1.mycol).nullsfirst()) + + produces:: + + ORDER BY mycol DESC NULLS FIRST + + """ + return UnaryExpression( + _literal_as_text(column), modifier=operators.nullsfirst_op) + + + @classmethod + def _create_nullslast(cls, column): + """Return a NULLS LAST ``ORDER BY`` clause element. + + e.g.:: + + someselect.order_by(desc(table1.mycol).nullslast()) + + produces:: + + ORDER BY mycol DESC NULLS LAST + + """ + return UnaryExpression( + _literal_as_text(column), modifier=operators.nullslast_op) + + + @classmethod + def _create_desc(cls, column): + """Return a descending ``ORDER BY`` clause element. + + e.g.:: + + someselect.order_by(desc(table1.mycol)) + + produces:: + + ORDER BY mycol DESC + + """ + return UnaryExpression( + _literal_as_text(column), modifier=operators.desc_op) + + @classmethod + def _create_asc(cls, column): + """Return an ascending ``ORDER BY`` clause element. + + e.g.:: + + someselect.order_by(asc(table1.mycol)) + + produces:: + + ORDER BY mycol ASC + + """ + return UnaryExpression( + _literal_as_text(column), modifier=operators.asc_op) + + @classmethod + def _create_distinct(cls, expr): + """Return a ``DISTINCT`` clause. + + e.g.:: + + distinct(a) + + renders:: + + DISTINCT a + + """ + expr = _literal_as_binds(expr) + return UnaryExpression(expr, + operator=operators.distinct_op, type_=expr.type) + + @util.memoized_property + def _order_by_label_element(self): + if self.modifier in (operators.desc_op, operators.asc_op): + return self.element._order_by_label_element + else: + return None + + @property + def _from_objects(self): + return self.element._from_objects + + def _copy_internals(self, clone=_clone, **kw): + self.element = clone(self.element, **kw) + + def get_children(self, **kwargs): + return self.element, + + def compare(self, other, **kw): + """Compare this :class:`UnaryExpression` against the given + :class:`.ClauseElement`.""" + + return ( + isinstance(other, UnaryExpression) and + self.operator == other.operator and + self.modifier == other.modifier and + self.element.compare(other.element, **kw) + ) + + def _negate(self): + if self.negate is not None: + return UnaryExpression( + self.element, + operator=self.negate, + negate=self.operator, + modifier=self.modifier, + type_=self.type) + else: + return ClauseElement._negate(self) + + def self_group(self, against=None): + if self.operator and operators.is_precedent(self.operator, against): + return Grouping(self) + else: + return self + + +class AsBoolean(UnaryExpression): + + def __init__(self, element, operator, negate): + self.element = element + self.type = type_api.BOOLEANTYPE + self.operator = operator + self.negate = negate + self.modifier = None + + def self_group(self, against=None): + return self + + def _negate(self): + return self.element._negate() + + +class BinaryExpression(ColumnElement): + """Represent an expression that is ``LEFT RIGHT``. + + A :class:`.BinaryExpression` is generated automatically + whenever two column expressions are used in a Python binary expresion:: + + >>> from sqlalchemy.sql import column + >>> column('a') + column('b') + + >>> print column('a') + column('b') + a + b + + """ + + __visit_name__ = 'binary' + + def __init__(self, left, right, operator, type_=None, + negate=None, modifiers=None): + # allow compatibility with libraries that + # refer to BinaryExpression directly and pass strings + if isinstance(operator, util.string_types): + operator = operators.custom_op(operator) + self._orig = (left, right) + self.left = left.self_group(against=operator) + self.right = right.self_group(against=operator) + self.operator = operator + self.type = type_api.to_instance(type_) + self.negate = negate + + if modifiers is None: + self.modifiers = {} + else: + self.modifiers = modifiers + + def __bool__(self): + if self.operator in (operator.eq, operator.ne): + return self.operator(hash(self._orig[0]), hash(self._orig[1])) + else: + raise TypeError("Boolean value of this clause is not defined") + + __nonzero__ = __bool__ + + @property + def is_comparison(self): + return operators.is_comparison(self.operator) + + @property + def _from_objects(self): + return self.left._from_objects + self.right._from_objects + + def _copy_internals(self, clone=_clone, **kw): + self.left = clone(self.left, **kw) + self.right = clone(self.right, **kw) + + def get_children(self, **kwargs): + return self.left, self.right + + def compare(self, other, **kw): + """Compare this :class:`BinaryExpression` against the + given :class:`BinaryExpression`.""" + + return ( + isinstance(other, BinaryExpression) and + self.operator == other.operator and + ( + self.left.compare(other.left, **kw) and + self.right.compare(other.right, **kw) or + ( + operators.is_commutative(self.operator) and + self.left.compare(other.right, **kw) and + self.right.compare(other.left, **kw) + ) + ) + ) + + def self_group(self, against=None): + if operators.is_precedent(self.operator, against): + return Grouping(self) + else: + return self + + def _negate(self): + if self.negate is not None: + return BinaryExpression( + self.left, + self.right, + self.negate, + negate=self.operator, + type_=type_api.BOOLEANTYPE, + modifiers=self.modifiers) + else: + return super(BinaryExpression, self)._negate() + + + + +class Grouping(ColumnElement): + """Represent a grouping within a column expression""" + + __visit_name__ = 'grouping' + + def __init__(self, element): + self.element = element + self.type = getattr(element, 'type', type_api.NULLTYPE) + + def self_group(self, against=None): + return self + + @property + def _label(self): + return getattr(self.element, '_label', None) or self.anon_label + + def _copy_internals(self, clone=_clone, **kw): + self.element = clone(self.element, **kw) + + def get_children(self, **kwargs): + return self.element, + + @property + def _from_objects(self): + return self.element._from_objects + + def __getattr__(self, attr): + return getattr(self.element, attr) + + def __getstate__(self): + return {'element': self.element, 'type': self.type} + + def __setstate__(self, state): + self.element = state['element'] + self.type = state['type'] + + def compare(self, other, **kw): + return isinstance(other, Grouping) and \ + self.element.compare(other.element) + + +class Over(ColumnElement): + """Represent an OVER clause. + + This is a special operator against a so-called + "window" function, as well as any aggregate function, + which produces results relative to the result set + itself. It's supported only by certain database + backends. + + """ + __visit_name__ = 'over' + + order_by = None + partition_by = None + + def __init__(self, func, partition_by=None, order_by=None): + """Produce an :class:`.Over` object against a function. + + Used against aggregate or so-called "window" functions, + for database backends that support window functions. + + E.g.:: + + from sqlalchemy import over + over(func.row_number(), order_by='x') + + Would produce "ROW_NUMBER() OVER(ORDER BY x)". + + :param func: a :class:`.FunctionElement` construct, typically + generated by :data:`~.expression.func`. + :param partition_by: a column element or string, or a list + of such, that will be used as the PARTITION BY clause + of the OVER construct. + :param order_by: a column element or string, or a list + of such, that will be used as the ORDER BY clause + of the OVER construct. + + This function is also available from the :data:`~.expression.func` + construct itself via the :meth:`.FunctionElement.over` method. + + .. versionadded:: 0.7 + + """ + self.func = func + if order_by is not None: + self.order_by = ClauseList(*util.to_list(order_by)) + if partition_by is not None: + self.partition_by = ClauseList(*util.to_list(partition_by)) + + @util.memoized_property + def type(self): + return self.func.type + + def get_children(self, **kwargs): + return [c for c in + (self.func, self.partition_by, self.order_by) + if c is not None] + + def _copy_internals(self, clone=_clone, **kw): + self.func = clone(self.func, **kw) + if self.partition_by is not None: + self.partition_by = clone(self.partition_by, **kw) + if self.order_by is not None: + self.order_by = clone(self.order_by, **kw) + + @property + def _from_objects(self): + return list(itertools.chain( + *[c._from_objects for c in + (self.func, self.partition_by, self.order_by) + if c is not None] + )) + + +class Label(ColumnElement): + """Represents a column label (AS). + + Represent a label, as typically applied to any column-level + element using the ``AS`` sql keyword. + + """ + + __visit_name__ = 'label' + + def __init__(self, name, element, type_=None): + """Return a :class:`Label` object for the + given :class:`.ColumnElement`. + + A label changes the name of an element in the columns clause of a + ``SELECT`` statement, typically via the ``AS`` SQL keyword. + + This functionality is more conveniently available via the + :meth:`.ColumnElement.label` method on :class:`.ColumnElement`. + + :param name: label name + + :param obj: a :class:`.ColumnElement`. + + """ + while isinstance(element, Label): + element = element.element + if name: + self.name = name + else: + self.name = _anonymous_label('%%(%d %s)s' % (id(self), + getattr(element, 'name', 'anon'))) + self.key = self._label = self._key_label = self.name + self._element = element + self._type = type_ + self._proxies = [element] + + def __reduce__(self): + return self.__class__, (self.name, self._element, self._type) + + @util.memoized_property + def _order_by_label_element(self): + return self + + @util.memoized_property + def type(self): + return type_api.to_instance( + self._type or getattr(self._element, 'type', None) + ) + + @util.memoized_property + def element(self): + return self._element.self_group(against=operators.as_) + + def self_group(self, against=None): + sub_element = self._element.self_group(against=against) + if sub_element is not self._element: + return Label(self.name, + sub_element, + type_=self._type) + else: + return self + + @property + def primary_key(self): + return self.element.primary_key + + @property + def foreign_keys(self): + return self.element.foreign_keys + + def get_children(self, **kwargs): + return self.element, + + def _copy_internals(self, clone=_clone, **kw): + self.element = clone(self.element, **kw) + + @property + def _from_objects(self): + return self.element._from_objects + + def _make_proxy(self, selectable, name=None, **kw): + e = self.element._make_proxy(selectable, + name=name if name else self.name) + e._proxies.append(self) + if self._type is not None: + e.type = self._type + return e + + +class ColumnClause(Immutable, ColumnElement): + """Represents a generic column expression from any textual string. + + This includes columns associated with tables, aliases and select + statements, but also any arbitrary text. May or may not be bound + to an underlying :class:`.Selectable`. + + :class:`.ColumnClause` is constructed by itself typically via + the :func:`~.expression.column` function. It may be placed directly + into constructs such as :func:`.select` constructs:: + + from sqlalchemy.sql import column, select + + c1, c2 = column("c1"), column("c2") + s = select([c1, c2]).where(c1==5) + + There is also a variant on :func:`~.expression.column` known + as :func:`~.expression.literal_column` - the difference is that + in the latter case, the string value is assumed to be an exact + expression, rather than a column name, so that no quoting rules + or similar are applied:: + + from sqlalchemy.sql import literal_column, select + + s = select([literal_column("5 + 7")]) + + :class:`.ColumnClause` can also be used in a table-like + fashion by combining the :func:`~.expression.column` function + with the :func:`~.expression.table` function, to produce + a "lightweight" form of table metadata:: + + from sqlalchemy.sql import table, column + + user = table("user", + column("id"), + column("name"), + column("description"), + ) + + The above construct can be created in an ad-hoc fashion and is + not associated with any :class:`.schema.MetaData`, unlike it's + more full fledged :class:`.schema.Table` counterpart. + + """ + __visit_name__ = 'column' + + onupdate = default = server_default = server_onupdate = None + + _memoized_property = util.group_expirable_memoized_property() + + def __init__(self, text, type_=None, is_literal=False, _selectable=None): + """Construct a :class:`.ColumnClause` object. + + :param text: the text of the element. + + :param type: :class:`.types.TypeEngine` object which can associate + this :class:`.ColumnClause` with a type. + + :param is_literal: if True, the :class:`.ColumnClause` is assumed to + be an exact expression that will be delivered to the output with no + quoting rules applied regardless of case sensitive settings. the + :func:`literal_column()` function is usually used to create such a + :class:`.ColumnClause`. + + :param text: the name of the column. Quoting rules will be applied + to the clause like any other column name. For textual column constructs + that are not to be quoted, use the :func:`literal_column` function. + + :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` object + which will provide result-set translation for this column. + + + """ + + self.key = self.name = text + self.table = _selectable + self.type = type_api.to_instance(type_) + self.is_literal = is_literal + + def _compare_name_for_result(self, other): + if self.is_literal or \ + self.table is None or \ + not hasattr(other, 'proxy_set') or ( + isinstance(other, ColumnClause) and other.is_literal + ): + return super(ColumnClause, self).\ + _compare_name_for_result(other) + else: + return other.proxy_set.intersection(self.proxy_set) + + def _get_table(self): + return self.__dict__['table'] + + def _set_table(self, table): + self._memoized_property.expire_instance(self) + self.__dict__['table'] = table + table = property(_get_table, _set_table) + + @_memoized_property + def _from_objects(self): + t = self.table + if t is not None: + return [t] + else: + return [] + + @util.memoized_property + def description(self): + if util.py3k: + return self.name + else: + return self.name.encode('ascii', 'backslashreplace') + + @_memoized_property + def _key_label(self): + if self.key != self.name: + return self._gen_label(self.key) + else: + return self._label + + @_memoized_property + def _label(self): + return self._gen_label(self.name) + + def _gen_label(self, name): + t = self.table + + if self.is_literal: + return None + + elif t is not None and t.named_with_column: + if getattr(t, 'schema', None): + label = t.schema.replace('.', '_') + "_" + \ + t.name + "_" + name + else: + label = t.name + "_" + name + + # propagate name quoting rules for labels. + if getattr(name, "quote", None) is not None: + if isinstance(label, quoted_name): + label.quote = name.quote + else: + label = quoted_name(label, name.quote) + elif getattr(t.name, "quote", None) is not None: + # can't get this situation to occur, so let's + # assert false on it for now + assert not isinstance(label, quoted_name) + label = quoted_name(label, t.name.quote) + + # ensure the label name doesn't conflict with that + # of an existing column + if label in t.c: + _label = label + counter = 1 + while _label in t.c: + _label = label + "_" + str(counter) + counter += 1 + label = _label + + return _as_truncated(label) + + else: + return name + + def _bind_param(self, operator, obj): + return BindParameter(self.name, obj, + _compared_to_operator=operator, + _compared_to_type=self.type, + unique=True) + + def _make_proxy(self, selectable, name=None, attach=True, + name_is_truncatable=False, **kw): + # propagate the "is_literal" flag only if we are keeping our name, + # otherwise its considered to be a label + is_literal = self.is_literal and (name is None or name == self.name) + c = self._constructor( + _as_truncated(name or self.name) if \ + name_is_truncatable else \ + (name or self.name), + type_=self.type, + _selectable=selectable, + is_literal=is_literal + ) + if name is None: + c.key = self.key + c._proxies = [self] + if selectable._is_clone_of is not None: + c._is_clone_of = \ + selectable._is_clone_of.columns.get(c.key) + + if attach: + selectable._columns[c.key] = c + return c + + +class _IdentifiedClause(Executable, ClauseElement): + + __visit_name__ = 'identified' + _execution_options = \ + Executable._execution_options.union({'autocommit': False}) + + def __init__(self, ident): + self.ident = ident + + +class SavepointClause(_IdentifiedClause): + __visit_name__ = 'savepoint' + + +class RollbackToSavepointClause(_IdentifiedClause): + __visit_name__ = 'rollback_to_savepoint' + + +class ReleaseSavepointClause(_IdentifiedClause): + __visit_name__ = 'release_savepoint' + + +class quoted_name(util.text_type): + """Represent a SQL identifier combined with quoting preferences. + + :class:`.quoted_name` is a Python unicode/str subclass which + represents a particular identifier name along with a + ``quote`` flag. This ``quote`` flag, when set to + ``True`` or ``False``, overrides automatic quoting behavior + for this identifier in order to either unconditionally quote + or to not quote the name. If left at its default of ``None``, + quoting behavior is applied to the identifier on a per-backend basis + based on an examination of the token itself. + + A :class:`.quoted_name` object with ``quote=True`` is also + prevented from being modified in the case of a so-called + "name normalize" option. Certain database backends, such as + Oracle, Firebird, and DB2 "normalize" case-insensitive names + as uppercase. The SQLAlchemy dialects for these backends + convert from SQLAlchemy's lower-case-means-insensitive convention + to the upper-case-means-insensitive conventions of those backends. + The ``quote=True`` flag here will prevent this conversion from occurring + to support an identifier that's quoted as all lower case against + such a backend. + + The :class:`.quoted_name` object is normally created automatically + when specifying the name for key schema constructs such as :class:`.Table`, + :class:`.Column`, and others. The class can also be passed explicitly + as the name to any function that receives a name which can be quoted. + Such as to use the :meth:`.Engine.has_table` method with an unconditionally + quoted name:: + + from sqlaclchemy import create_engine + from sqlalchemy.sql.elements import quoted_name + + engine = create_engine("oracle+cx_oracle://some_dsn") + engine.has_table(quoted_name("some_table", True)) + + The above logic will run the "has table" logic against the Oracle backend, + passing the name exactly as ``"some_table"`` without converting to + upper case. + + .. versionadded:: 0.9.0 + + """ + + def __new__(cls, value, quote): + if value is None: + return None + # experimental - don't bother with quoted_name + # if quote flag is None. doesn't seem to make any dent + # in performance however + # elif not sprcls and quote is None: + # return value + elif isinstance(value, cls) and ( + quote is None or value.quote == quote + ): + return value + self = super(quoted_name, cls).__new__(cls, value) + self.quote = quote + return self + + def __reduce__(self): + return quoted_name, (util.text_type(self), self.quote) + + @util.memoized_instancemethod + def lower(self): + if self.quote: + return self + else: + return util.text_type(self).lower() + + @util.memoized_instancemethod + def upper(self): + if self.quote: + return self + else: + return util.text_type(self).upper() + + def __repr__(self): + backslashed = self.encode('ascii', 'backslashreplace') + if not util.py2k: + backslashed = backslashed.decode('ascii') + return "'%s'" % backslashed + +class _truncated_label(quoted_name): + """A unicode subclass used to identify symbolic " + "names that may require truncation.""" + + def __new__(cls, value, quote=None): + quote = getattr(value, "quote", quote) + #return super(_truncated_label, cls).__new__(cls, value, quote, True) + return super(_truncated_label, cls).__new__(cls, value, quote) + + def __reduce__(self): + return self.__class__, (util.text_type(self), self.quote) + + def apply_map(self, map_): + return self + +# for backwards compatibility in case +# someone is re-implementing the +# _truncated_identifier() sequence in a custom +# compiler +_generated_label = _truncated_label + + +class _anonymous_label(_truncated_label): + """A unicode subclass used to identify anonymously + generated names.""" + + def __add__(self, other): + return _anonymous_label( + quoted_name( + util.text_type.__add__(self, util.text_type(other)), + self.quote) + ) + + def __radd__(self, other): + return _anonymous_label( + quoted_name( + util.text_type.__add__(util.text_type(other), self), + self.quote) + ) + + def apply_map(self, map_): + if self.quote is not None: + # preserve quoting only if necessary + return quoted_name(self % map_, self.quote) + else: + # else skip the constructor call + return self % map_ + + +def _as_truncated(value): + """coerce the given value to :class:`._truncated_label`. + + Existing :class:`._truncated_label` and + :class:`._anonymous_label` objects are passed + unchanged. + """ + + if isinstance(value, _truncated_label): + return value + else: + return _truncated_label(value) + + +def _string_or_unprintable(element): + if isinstance(element, util.string_types): + return element + else: + try: + return str(element) + except: + return "unprintable element %r" % element + + +def _expand_cloned(elements): + """expand the given set of ClauseElements to be the set of all 'cloned' + predecessors. + + """ + return itertools.chain(*[x._cloned_set for x in elements]) + + +def _select_iterables(elements): + """expand tables into individual columns in the + given list of column expressions. + + """ + return itertools.chain(*[c._select_iterable for c in elements]) + + +def _cloned_intersection(a, b): + """return the intersection of sets a and b, counting + any overlap between 'cloned' predecessors. + + The returned set is in terms of the entities present within 'a'. + + """ + all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) + return set(elem for elem in a + if all_overlap.intersection(elem._cloned_set)) + +def _cloned_difference(a, b): + all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) + return set(elem for elem in a + if not all_overlap.intersection(elem._cloned_set)) + + +def _labeled(element): + if not hasattr(element, 'name'): + return element.label(None) + else: + return element + + +def _is_column(col): + """True if ``col`` is an instance of :class:`.ColumnElement`.""" + + return isinstance(col, ColumnElement) + + +def _find_columns(clause): + """locate Column objects within the given expression.""" + + cols = util.column_set() + traverse(clause, {}, {'column': cols.add}) + return cols + + +# there is some inconsistency here between the usage of +# inspect() vs. checking for Visitable and __clause_element__. +# Ideally all functions here would derive from inspect(), +# however the inspect() versions add significant callcount +# overhead for critical functions like _interpret_as_column_or_from(). +# Generally, the column-based functions are more performance critical +# and are fine just checking for __clause_element__(). it's only +# _interpret_as_from() where we'd like to be able to receive ORM entities +# that have no defined namespace, hence inspect() is needed there. + + +def _column_as_key(element): + if isinstance(element, util.string_types): + return element + if hasattr(element, '__clause_element__'): + element = element.__clause_element__() + try: + return element.key + except AttributeError: + return None + + +def _clause_element_as_expr(element): + if hasattr(element, '__clause_element__'): + return element.__clause_element__() + else: + return element + + +def _literal_as_text(element): + if isinstance(element, Visitable): + return element + elif hasattr(element, '__clause_element__'): + return element.__clause_element__() + elif isinstance(element, util.string_types): + return TextClause(util.text_type(element)) + elif isinstance(element, (util.NoneType, bool)): + return _const_expr(element) + else: + raise exc.ArgumentError( + "SQL expression object or string expected." + ) + + +def _no_literals(element): + if hasattr(element, '__clause_element__'): + return element.__clause_element__() + elif not isinstance(element, Visitable): + raise exc.ArgumentError("Ambiguous literal: %r. Use the 'text()' " + "function to indicate a SQL expression " + "literal, or 'literal()' to indicate a " + "bound value." % element) + else: + return element + + +def _is_literal(element): + return not isinstance(element, Visitable) and \ + not hasattr(element, '__clause_element__') + + +def _only_column_elements_or_none(element, name): + if element is None: + return None + else: + return _only_column_elements(element, name) + + +def _only_column_elements(element, name): + if hasattr(element, '__clause_element__'): + element = element.__clause_element__() + if not isinstance(element, ColumnElement): + raise exc.ArgumentError( + "Column-based expression object expected for argument " + "'%s'; got: '%s', type %s" % (name, element, type(element))) + return element + +def _literal_as_binds(element, name=None, type_=None): + if hasattr(element, '__clause_element__'): + return element.__clause_element__() + elif not isinstance(element, Visitable): + if element is None: + return Null() + else: + return BindParameter(name, element, type_=type_, unique=True) + else: + return element + + +def _interpret_as_column_or_from(element): + if isinstance(element, Visitable): + return element + elif hasattr(element, '__clause_element__'): + return element.__clause_element__() + + insp = inspection.inspect(element, raiseerr=False) + if insp is None: + if isinstance(element, (util.NoneType, bool)): + return _const_expr(element) + elif hasattr(insp, "selectable"): + return insp.selectable + + return ColumnClause(str(element), is_literal=True) + + +def _const_expr(element): + if isinstance(element, (Null, False_, True_)): + return element + elif element is None: + return Null() + elif element is False: + return False_() + elif element is True: + return True_() + else: + raise exc.ArgumentError( + "Expected None, False, or True" + ) + + +def _type_from_args(args): + for a in args: + if not a.type._isnull: + return a.type + else: + return type_api.NULLTYPE + + +def _corresponding_column_or_error(fromclause, column, + require_embedded=False): + c = fromclause.corresponding_column(column, + require_embedded=require_embedded) + if c is None: + raise exc.InvalidRequestError( + "Given column '%s', attached to table '%s', " + "failed to locate a corresponding column from table '%s'" + % + (column, + getattr(column, 'table', None), + fromclause.description) + ) + return c + + +class AnnotatedColumnElement(Annotated): + def __init__(self, element, values): + Annotated.__init__(self, element, values) + ColumnElement.comparator._reset(self) + for attr in ('name', 'key', 'table'): + if self.__dict__.get(attr, False) is None: + self.__dict__.pop(attr) + + def _with_annotations(self, values): + clone = super(AnnotatedColumnElement, self)._with_annotations(values) + ColumnElement.comparator._reset(clone) + return clone + + @util.memoized_property + def name(self): + """pull 'name' from parent, if not present""" + return self._Annotated__element.name + + @util.memoized_property + def table(self): + """pull 'table' from parent, if not present""" + return self._Annotated__element.table + + @util.memoized_property + def key(self): + """pull 'key' from parent, if not present""" + return self._Annotated__element.key + + @util.memoized_property + def info(self): + return self._Annotated__element.info + diff --git a/libs/sqlalchemy/sql/expression.py b/libs/sqlalchemy/sql/expression.py index c90a3dcb..c99665b4 100644 --- a/libs/sqlalchemy/sql/expression.py +++ b/libs/sqlalchemy/sql/expression.py @@ -1,45 +1,18 @@ # sql/expression.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""Defines the base components of SQL expression trees. +"""Defines the public namespace for SQL expression constructs. -All components are derived from a common base class -:class:`.ClauseElement`. Common behaviors are organized -based on class hierarchies, in some cases via mixins. - -All object construction from this package occurs via functions which -in some cases will construct composite :class:`.ClauseElement` structures -together, and in other cases simply return a single :class:`.ClauseElement` -constructed directly. The function interface affords a more "DSL-ish" -feel to constructing SQL expressions and also allows future class -reorganizations. - -Even though classes are not constructed directly from the outside, -most classes which have additional public methods are considered to be -public (i.e. have no leading underscore). Other classes which are -"semi-public" are marked with a single leading underscore; these -classes usually have few or no public methods and are less guaranteed -to stay the same in future releases. +Prior to version 0.9, this module contained all of "elements", "dml", +"default_comparator" and "selectable". The module was broken up +and most "factory" functions were moved to be grouped with their associated +class. """ -import itertools, re -from operator import attrgetter - -from sqlalchemy import util, exc -from sqlalchemy.sql import operators -from sqlalchemy.sql.operators import Operators, ColumnOperators -from sqlalchemy.sql.visitors import Visitable, cloned_traverse -import operator - -functions = util.importlater("sqlalchemy.sql", "functions") -sqlutil = util.importlater("sqlalchemy.sql", "util") -sqltypes = util.importlater("sqlalchemy", "types") -default = util.importlater("sqlalchemy.engine", "default") - __all__ = [ 'Alias', 'ClauseElement', 'ColumnCollection', 'ColumnElement', 'CompoundSelect', 'Delete', 'FromClause', 'Insert', 'Join', 'Select', @@ -48,5683 +21,107 @@ __all__ = [ 'except_', 'except_all', 'exists', 'extract', 'func', 'modifier', 'collate', 'insert', 'intersect', 'intersect_all', 'join', 'label', 'literal', 'literal_column', 'not_', 'null', 'nullsfirst', 'nullslast', - 'or_', 'outparam', 'outerjoin', 'over', 'select', 'subquery', 'table', 'text', - 'tuple_', 'type_coerce', 'union', 'union_all', 'update', ] - -PARSE_AUTOCOMMIT = util.symbol('PARSE_AUTOCOMMIT') - -def nullsfirst(column): - """Return a NULLS FIRST ``ORDER BY`` clause element. - - e.g.:: - - someselect.order_by(desc(table1.mycol).nullsfirst()) - - produces:: - - ORDER BY mycol DESC NULLS FIRST - - """ - return _UnaryExpression(column, modifier=operators.nullsfirst_op) - -def nullslast(column): - """Return a NULLS LAST ``ORDER BY`` clause element. - - e.g.:: - - someselect.order_by(desc(table1.mycol).nullslast()) - - produces:: - - ORDER BY mycol DESC NULLS LAST - - """ - return _UnaryExpression(column, modifier=operators.nullslast_op) - -def desc(column): - """Return a descending ``ORDER BY`` clause element. - - e.g.:: - - someselect.order_by(desc(table1.mycol)) - - produces:: - - ORDER BY mycol DESC - - """ - return _UnaryExpression(column, modifier=operators.desc_op) - -def asc(column): - """Return an ascending ``ORDER BY`` clause element. - - e.g.:: - - someselect.order_by(asc(table1.mycol)) - - produces:: - - ORDER BY mycol ASC - - """ - return _UnaryExpression(column, modifier=operators.asc_op) - -def outerjoin(left, right, onclause=None): - """Return an ``OUTER JOIN`` clause element. - - The returned object is an instance of :class:`.Join`. - - Similar functionality is also available via the - :meth:`~.FromClause.outerjoin()` method on any - :class:`.FromClause`. - - :param left: The left side of the join. - - :param right: The right side of the join. - - :param onclause: Optional criterion for the ``ON`` clause, is - derived from foreign key relationships established between - left and right otherwise. - - To chain joins together, use the :meth:`.FromClause.join` or - :meth:`.FromClause.outerjoin` methods on the resulting - :class:`.Join` object. - - """ - return Join(left, right, onclause, isouter=True) - -def join(left, right, onclause=None, isouter=False): - """Return a ``JOIN`` clause element (regular inner join). - - The returned object is an instance of :class:`.Join`. - - Similar functionality is also available via the - :meth:`~.FromClause.join()` method on any - :class:`.FromClause`. - - :param left: The left side of the join. - - :param right: The right side of the join. - - :param onclause: Optional criterion for the ``ON`` clause, is - derived from foreign key relationships established between - left and right otherwise. - - To chain joins together, use the :meth:`.FromClause.join` or - :meth:`.FromClause.outerjoin` methods on the resulting - :class:`.Join` object. - - - """ - return Join(left, right, onclause, isouter) - -def select(columns=None, whereclause=None, from_obj=[], **kwargs): - """Returns a ``SELECT`` clause element. - - Similar functionality is also available via the :func:`select()` - method on any :class:`.FromClause`. - - The returned object is an instance of :class:`.Select`. - - All arguments which accept :class:`.ClauseElement` arguments also accept - string arguments, which will be converted as appropriate into - either :func:`text()` or :func:`literal_column()` constructs. - - See also: - - :ref:`coretutorial_selecting` - Core Tutorial description of :func:`.select`. - - :param columns: - A list of :class:`.ClauseElement` objects, typically - :class:`.ColumnElement` objects or subclasses, which will form the - columns clause of the resulting statement. For all members which are - instances of :class:`.Selectable`, the individual :class:`.ColumnElement` - members of the :class:`.Selectable` will be added individually to the - columns clause. For example, specifying a - :class:`~sqlalchemy.schema.Table` instance will result in all the - contained :class:`~sqlalchemy.schema.Column` objects within to be added - to the columns clause. - - This argument is not present on the form of :func:`select()` - available on :class:`~sqlalchemy.schema.Table`. - - :param whereclause: - A :class:`.ClauseElement` expression which will be used to form the - ``WHERE`` clause. - - :param from_obj: - A list of :class:`.ClauseElement` objects which will be added to the - ``FROM`` clause of the resulting statement. Note that "from" objects are - automatically located within the columns and whereclause ClauseElements. - Use this parameter to explicitly specify "from" objects which are not - automatically locatable. This could include - :class:`~sqlalchemy.schema.Table` objects that aren't otherwise present, - or :class:`.Join` objects whose presence will supercede that of the - :class:`~sqlalchemy.schema.Table` objects already located in the other - clauses. - - :param autocommit: - Deprecated. Use .execution_options(autocommit=) - to set the autocommit option. - - :param bind=None: - an :class:`~.base.Engine` or :class:`~.base.Connection` instance - to which the - resulting :class:`.Select` object will be bound. The :class:`.Select` - object will otherwise automatically bind to whatever - :class:`~.base.Connectable` instances can be located within its contained - :class:`.ClauseElement` members. - - :param correlate=True: - indicates that this :class:`.Select` object should have its - contained :class:`.FromClause` elements "correlated" to an enclosing - :class:`.Select` object. This means that any :class:`.ClauseElement` - instance within the "froms" collection of this :class:`.Select` - which is also present in the "froms" collection of an - enclosing select will not be rendered in the ``FROM`` clause - of this select statement. - - :param distinct=False: - when ``True``, applies a ``DISTINCT`` qualifier to the columns - clause of the resulting statement. - - The boolean argument may also be a column expression or list - of column expressions - this is a special calling form which - is understood by the Postgresql dialect to render the - ``DISTINCT ON ()`` syntax. - - ``distinct`` is also available via the :meth:`~.Select.distinct` - generative method. - - .. note:: - - The ``distinct`` keyword's acceptance of a string - argument for usage with MySQL is deprecated. Use - the ``prefixes`` argument or :meth:`~.Select.prefix_with`. - - :param for_update=False: - when ``True``, applies ``FOR UPDATE`` to the end of the - resulting statement. - - Certain database dialects also support - alternate values for this parameter: - - * With the MySQL dialect, the value ``"read"`` translates to - ``LOCK IN SHARE MODE``. - * With the Oracle and Postgresql dialects, the value ``"nowait"`` - translates to ``FOR UPDATE NOWAIT``. - * With the Postgresql dialect, the values "read" and ``"read_nowait"`` - translate to ``FOR SHARE`` and ``FOR SHARE NOWAIT``, respectively. - - .. versionadded:: 0.7.7 - - :param group_by: - a list of :class:`.ClauseElement` objects which will comprise the - ``GROUP BY`` clause of the resulting select. - - :param having: - a :class:`.ClauseElement` that will comprise the ``HAVING`` clause - of the resulting select when ``GROUP BY`` is used. - - :param limit=None: - a numerical value which usually compiles to a ``LIMIT`` - expression in the resulting select. Databases that don't - support ``LIMIT`` will attempt to provide similar - functionality. - - :param offset=None: - a numeric value which usually compiles to an ``OFFSET`` - expression in the resulting select. Databases that don't - support ``OFFSET`` will attempt to provide similar - functionality. - - :param order_by: - a scalar or list of :class:`.ClauseElement` objects which will - comprise the ``ORDER BY`` clause of the resulting select. - - :param prefixes: - a list of strings or :class:`.ClauseElement` objects to include - directly after the SELECT keyword in the generated statement, - for dialect-specific query features. ``prefixes`` is - also available via the :meth:`~.Select.prefix_with` - generative method. - - :param use_labels=False: - when ``True``, the statement will be generated using labels - for each column in the columns clause, which qualify each - column with its parent table's (or aliases) name so that name - conflicts between columns in different tables don't occur. - The format of the label is _. The "c" - collection of the resulting :class:`.Select` object will use these - names as well for targeting column members. - - use_labels is also available via the :meth:`~._SelectBase.apply_labels` - generative method. - - """ - return Select(columns, whereclause=whereclause, from_obj=from_obj, - **kwargs) - -def subquery(alias, *args, **kwargs): - """Return an :class:`.Alias` object derived - from a :class:`.Select`. - - name - alias name - - \*args, \**kwargs - - all other arguments are delivered to the - :func:`select` function. - - """ - return Select(*args, **kwargs).alias(alias) - -def insert(table, values=None, inline=False, **kwargs): - """Represent an ``INSERT`` statement via the :class:`.Insert` SQL - construct. - - Similar functionality is available via the :meth:`~.TableClause.insert` method on - :class:`~.schema.Table`. - - - :param table: The table to be inserted into. - - :param values: A dictionary which specifies the column specifications of - the ``INSERT``, and is optional. If left as None, the column - specifications are determined from the bind parameters used during the - compile phase of the ``INSERT`` statement. If the bind parameters also - are None during the compile phase, then the column specifications will be - generated from the full list of table columns. Note that the - :meth:`~Insert.values()` generative method may also be used for this. - - :param prefixes: A list of modifier keywords to be inserted between INSERT - and INTO. Alternatively, the :meth:`~Insert.prefix_with` generative - method may be used. - - :param inline: if True, SQL defaults will be compiled 'inline' into the - statement and not pre-executed. - - If both `values` and compile-time bind parameters are present, the - compile-time bind parameters override the information specified - within `values` on a per-key basis. - - The keys within `values` can be either :class:`~sqlalchemy.schema.Column` - objects or their string identifiers. Each key may reference one of: - - * a literal data value (i.e. string, number, etc.); - * a Column object; - * a SELECT statement. - - If a ``SELECT`` statement is specified which references this - ``INSERT`` statement's table, the statement will be correlated - against the ``INSERT`` statement. - - See also: - - :ref:`coretutorial_insert_expressions` - SQL Expression Tutorial - - :ref:`inserts_and_updates` - SQL Expression Tutorial - - """ - return Insert(table, values, inline=inline, **kwargs) - -def update(table, whereclause=None, values=None, inline=False, **kwargs): - """Represent an ``UPDATE`` statement via the :class:`.Update` SQL - construct. - - E.g.:: - - from sqlalchemy import update - - stmt = update(users).where(users.c.id==5).\\ - values(name='user #5') - - Similar functionality is available via the :meth:`~.TableClause.update` method on - :class:`.Table`:: - - - stmt = users.update().\\ - where(users.c.id==5).\\ - values(name='user #5') - - :param table: A :class:`.Table` object representing the database - table to be updated. - - :param whereclause: Optional SQL expression describing the ``WHERE`` - condition of the ``UPDATE`` statement. Modern applications - may prefer to use the generative :meth:`~Update.where()` - method to specify the ``WHERE`` clause. - - The WHERE clause can refer to multiple tables. - For databases which support this, an ``UPDATE FROM`` clause will - be generated, or on MySQL, a multi-table update. The statement - will fail on databases that don't have support for multi-table - update statements. A SQL-standard method of referring to - additional tables in the WHERE clause is to use a correlated - subquery:: - - users.update().values(name='ed').where( - users.c.name==select([addresses.c.email_address]).\\ - where(addresses.c.user_id==users.c.id).\\ - as_scalar() - ) - - .. versionchanged:: 0.7.4 - The WHERE clause can refer to multiple tables. - - :param values: - Optional dictionary which specifies the ``SET`` conditions of the - ``UPDATE``. If left as ``None``, the ``SET`` - conditions are determined from those parameters passed to the - statement during the execution and/or compilation of the - statement. When compiled standalone without any parameters, - the ``SET`` clause generates for all columns. - - Modern applications may prefer to use the generative - :meth:`.Update.values` method to set the values of the - UPDATE statement. - - :param inline: - if True, SQL defaults present on :class:`.Column` objects via - the ``default`` keyword will be compiled 'inline' into the statement - and not pre-executed. This means that their values will not - be available in the dictionary returned from - :meth:`.ResultProxy.last_updated_params`. - - If both ``values`` and compile-time bind parameters are present, the - compile-time bind parameters override the information specified - within ``values`` on a per-key basis. - - The keys within ``values`` can be either :class:`.Column` - objects or their string identifiers (specifically the "key" of the - :class:`.Column`, normally but not necessarily equivalent to - its "name"). Normally, the - :class:`.Column` objects used here are expected to be - part of the target :class:`.Table` that is the table - to be updated. However when using MySQL, a multiple-table - UPDATE statement can refer to columns from any of - the tables referred to in the WHERE clause. - - The values referred to in ``values`` are typically: - - * a literal data value (i.e. string, number, etc.) - * a SQL expression, such as a related :class:`.Column`, - a scalar-returning :func:`.select` construct, - etc. - - When combining :func:`.select` constructs within the values - clause of an :func:`.update` construct, - the subquery represented by the :func:`.select` should be - *correlated* to the parent table, that is, providing criterion - which links the table inside the subquery to the outer table - being updated:: - - users.update().values( - name=select([addresses.c.email_address]).\\ - where(addresses.c.user_id==users.c.id).\\ - as_scalar() - ) - - See also: - - :ref:`inserts_and_updates` - SQL Expression - Language Tutorial - - - """ - return Update( - table, - whereclause=whereclause, - values=values, - inline=inline, - **kwargs) - -def delete(table, whereclause = None, **kwargs): - """Represent a ``DELETE`` statement via the :class:`.Delete` SQL - construct. - - Similar functionality is available via the :meth:`~.TableClause.delete` method on - :class:`~.schema.Table`. - - :param table: The table to be updated. - - :param whereclause: A :class:`.ClauseElement` describing the ``WHERE`` - condition of the ``UPDATE`` statement. Note that the - :meth:`~Delete.where()` generative method may be used instead. - - See also: - - :ref:`deletes` - SQL Expression Tutorial - - """ - return Delete(table, whereclause, **kwargs) - -def and_(*clauses): - """Join a list of clauses together using the ``AND`` operator. - - The ``&`` operator is also overloaded on all - :class:`_CompareMixin` subclasses to produce the - same result. - - """ - if len(clauses) == 1: - return clauses[0] - return BooleanClauseList(operator=operators.and_, *clauses) - -def or_(*clauses): - """Join a list of clauses together using the ``OR`` operator. - - The ``|`` operator is also overloaded on all - :class:`_CompareMixin` subclasses to produce the - same result. - - """ - if len(clauses) == 1: - return clauses[0] - return BooleanClauseList(operator=operators.or_, *clauses) - -def not_(clause): - """Return a negation of the given clause, i.e. ``NOT(clause)``. - - The ``~`` operator is also overloaded on all - :class:`_CompareMixin` subclasses to produce the - same result. - - """ - return operators.inv(_literal_as_binds(clause)) - -def distinct(expr): - """Return a ``DISTINCT`` clause. - - e.g.:: - - distinct(a) - - renders:: - - DISTINCT a - - """ - expr = _literal_as_binds(expr) - return _UnaryExpression(expr, operator=operators.distinct_op, type_=expr.type) - -def between(ctest, cleft, cright): - """Return a ``BETWEEN`` predicate clause. - - Equivalent of SQL ``clausetest BETWEEN clauseleft AND clauseright``. - - The :func:`between()` method on all - :class:`_CompareMixin` subclasses provides - similar functionality. - - """ - ctest = _literal_as_binds(ctest) - return ctest.between(cleft, cright) - - -def case(whens, value=None, else_=None): - """Produce a ``CASE`` statement. - - whens - A sequence of pairs, or alternatively a dict, - to be translated into "WHEN / THEN" clauses. - - value - Optional for simple case statements, produces - a column expression as in "CASE WHEN ..." - - else\_ - Optional as well, for case defaults produces - the "ELSE" portion of the "CASE" statement. - - The expressions used for THEN and ELSE, - when specified as strings, will be interpreted - as bound values. To specify textual SQL expressions - for these, use the :func:`literal_column` - construct. - - The expressions used for the WHEN criterion - may only be literal strings when "value" is - present, i.e. CASE table.somecol WHEN "x" THEN "y". - Otherwise, literal strings are not accepted - in this position, and either the text() - or literal() constructs must be used to - interpret raw string values. - - Usage examples:: - - case([(orderline.c.qty > 100, item.c.specialprice), - (orderline.c.qty > 10, item.c.bulkprice) - ], else_=item.c.regularprice) - case(value=emp.c.type, whens={ - 'engineer': emp.c.salary * 1.1, - 'manager': emp.c.salary * 3, - }) - - Using :func:`literal_column()`, to allow for databases that - do not support bind parameters in the ``then`` clause. The type - can be specified which determines the type of the :func:`case()` construct - overall:: - - case([(orderline.c.qty > 100, - literal_column("'greaterthan100'", String)), - (orderline.c.qty > 10, literal_column("'greaterthan10'", - String)) - ], else_=literal_column("'lethan10'", String)) - - """ - - return _Case(whens, value=value, else_=else_) - -def cast(clause, totype, **kwargs): - """Return a ``CAST`` function. - - Equivalent of SQL ``CAST(clause AS totype)``. - - Use with a :class:`~sqlalchemy.types.TypeEngine` subclass, i.e:: - - cast(table.c.unit_price * table.c.qty, Numeric(10,4)) - - or:: - - cast(table.c.timestamp, DATE) - - """ - return _Cast(clause, totype, **kwargs) - -def extract(field, expr): - """Return the clause ``extract(field FROM expr)``.""" - - return _Extract(field, expr) - -def collate(expression, collation): - """Return the clause ``expression COLLATE collation``. - - e.g.:: - - collate(mycolumn, 'utf8_bin') - - produces:: - - mycolumn COLLATE utf8_bin - - """ - - expr = _literal_as_binds(expression) - return _BinaryExpression( - expr, - _literal_as_text(collation), - operators.collate, type_=expr.type) - -def exists(*args, **kwargs): - """Return an ``EXISTS`` clause as applied to a :class:`.Select` object. - - Calling styles are of the following forms:: - - # use on an existing select() - s = select([table.c.col1]).where(table.c.col2==5) - s = exists(s) - - # construct a select() at once - exists(['*'], **select_arguments).where(criterion) - - # columns argument is optional, generates "EXISTS (SELECT *)" - # by default. - exists().where(table.c.col2==5) - - """ - return _Exists(*args, **kwargs) - -def union(*selects, **kwargs): - """Return a ``UNION`` of multiple selectables. - - The returned object is an instance of - :class:`.CompoundSelect`. - - A similar :func:`union()` method is available on all - :class:`.FromClause` subclasses. - - \*selects - a list of :class:`.Select` instances. - - \**kwargs - available keyword arguments are the same as those of - :func:`select`. - - """ - return CompoundSelect(CompoundSelect.UNION, *selects, **kwargs) - -def union_all(*selects, **kwargs): - """Return a ``UNION ALL`` of multiple selectables. - - The returned object is an instance of - :class:`.CompoundSelect`. - - A similar :func:`union_all()` method is available on all - :class:`.FromClause` subclasses. - - \*selects - a list of :class:`.Select` instances. - - \**kwargs - available keyword arguments are the same as those of - :func:`select`. - - """ - return CompoundSelect(CompoundSelect.UNION_ALL, *selects, **kwargs) - -def except_(*selects, **kwargs): - """Return an ``EXCEPT`` of multiple selectables. - - The returned object is an instance of - :class:`.CompoundSelect`. - - \*selects - a list of :class:`.Select` instances. - - \**kwargs - available keyword arguments are the same as those of - :func:`select`. - - """ - return CompoundSelect(CompoundSelect.EXCEPT, *selects, **kwargs) - -def except_all(*selects, **kwargs): - """Return an ``EXCEPT ALL`` of multiple selectables. - - The returned object is an instance of - :class:`.CompoundSelect`. - - \*selects - a list of :class:`.Select` instances. - - \**kwargs - available keyword arguments are the same as those of - :func:`select`. - - """ - return CompoundSelect(CompoundSelect.EXCEPT_ALL, *selects, **kwargs) - -def intersect(*selects, **kwargs): - """Return an ``INTERSECT`` of multiple selectables. - - The returned object is an instance of - :class:`.CompoundSelect`. - - \*selects - a list of :class:`.Select` instances. - - \**kwargs - available keyword arguments are the same as those of - :func:`select`. - - """ - return CompoundSelect(CompoundSelect.INTERSECT, *selects, **kwargs) - -def intersect_all(*selects, **kwargs): - """Return an ``INTERSECT ALL`` of multiple selectables. - - The returned object is an instance of - :class:`.CompoundSelect`. - - \*selects - a list of :class:`.Select` instances. - - \**kwargs - available keyword arguments are the same as those of - :func:`select`. - - """ - return CompoundSelect(CompoundSelect.INTERSECT_ALL, *selects, **kwargs) - -def alias(selectable, name=None): - """Return an :class:`.Alias` object. - - An :class:`.Alias` represents any :class:`.FromClause` - with an alternate name assigned within SQL, typically using the ``AS`` - clause when generated, e.g. ``SELECT * FROM table AS aliasname``. - - Similar functionality is available via the - :meth:`~.FromClause.alias` method - available on all :class:`.FromClause` subclasses. - - When an :class:`.Alias` is created from a :class:`.Table` object, - this has the effect of the table being rendered - as ``tablename AS aliasname`` in a SELECT statement. - - For :func:`.select` objects, the effect is that of creating a named - subquery, i.e. ``(select ...) AS aliasname``. - - The ``name`` parameter is optional, and provides the name - to use in the rendered SQL. If blank, an "anonymous" name - will be deterministically generated at compile time. - Deterministic means the name is guaranteed to be unique against - other constructs used in the same statement, and will also be the - same name for each successive compilation of the same statement - object. - - :param selectable: any :class:`.FromClause` subclass, - such as a table, select statement, etc. - - :param name: string name to be assigned as the alias. - If ``None``, a name will be deterministically generated - at compile time. - - """ - return Alias(selectable, name=name) - - -def literal(value, type_=None): - """Return a literal clause, bound to a bind parameter. - - Literal clauses are created automatically when non- :class:`.ClauseElement` - objects (such as strings, ints, dates, etc.) are used in a comparison - operation with a :class:`_CompareMixin` - subclass, such as a :class:`~sqlalchemy.schema.Column` object. Use this function to force the - generation of a literal clause, which will be created as a - :class:`_BindParamClause` with a bound value. - - :param value: the value to be bound. Can be any Python object supported by - the underlying DB-API, or is translatable via the given type argument. - - :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` which - will provide bind-parameter translation for this literal. - - """ - return _BindParamClause(None, value, type_=type_, unique=True) - -def tuple_(*expr): - """Return a SQL tuple. - - Main usage is to produce a composite IN construct:: - - tuple_(table.c.col1, table.c.col2).in_( - [(1, 2), (5, 12), (10, 19)] - ) - - .. warning:: - - The composite IN construct is not supported by all backends, - and is currently known to work on Postgresql and MySQL, - but not SQLite. Unsupported backends will raise - a subclass of :class:`~sqlalchemy.exc.DBAPIError` when such - an expression is invoked. - - """ - return _Tuple(*expr) - -def type_coerce(expr, type_): - """Coerce the given expression into the given type, on the Python side only. - - :func:`.type_coerce` is roughly similar to :func:`.cast`, except no - "CAST" expression is rendered - the given type is only applied towards - expression typing and against received result values. - - e.g.:: - - from sqlalchemy.types import TypeDecorator - import uuid - - class AsGuid(TypeDecorator): - impl = String - - def process_bind_param(self, value, dialect): - if value is not None: - return str(value) - else: - return None - - def process_result_value(self, value, dialect): - if value is not None: - return uuid.UUID(value) - else: - return None - - conn.execute( - select([type_coerce(mytable.c.ident, AsGuid)]).\\ - where( - type_coerce(mytable.c.ident, AsGuid) == - uuid.uuid3(uuid.NAMESPACE_URL, 'bar') - ) - ) - - """ - if hasattr(expr, '__clause_expr__'): - return type_coerce(expr.__clause_expr__()) - - elif not isinstance(expr, Visitable): - if expr is None: - return null() - else: - return literal(expr, type_=type_) - else: - return _Label(None, expr, type_=type_) - - -def label(name, obj): - """Return a :class:`_Label` object for the - given :class:`.ColumnElement`. - - A label changes the name of an element in the columns clause of a - ``SELECT`` statement, typically via the ``AS`` SQL keyword. - - This functionality is more conveniently available via the - :func:`label()` method on :class:`.ColumnElement`. - - name - label name - - obj - a :class:`.ColumnElement`. - - """ - return _Label(name, obj) - -def column(text, type_=None): - """Return a textual column clause, as would be in the columns clause of a - ``SELECT`` statement. - - The object returned is an instance of :class:`.ColumnClause`, which - represents the "syntactical" portion of the schema-level - :class:`~sqlalchemy.schema.Column` object. It is often used directly - within :func:`~.expression.select` constructs or with lightweight :func:`~.expression.table` - constructs. - - Note that the :func:`~.expression.column` function is not part of - the ``sqlalchemy`` namespace. It must be imported from the ``sql`` package:: - - from sqlalchemy.sql import table, column - - :param text: the name of the column. Quoting rules will be applied - to the clause like any other column name. For textual column constructs - that are not to be quoted, use the :func:`literal_column` function. - - :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` object - which will provide result-set translation for this column. - - See :class:`.ColumnClause` for further examples. - - """ - return ColumnClause(text, type_=type_) - -def literal_column(text, type_=None): - """Return a textual column expression, as would be in the columns - clause of a ``SELECT`` statement. - - The object returned supports further expressions in the same way as any - other column object, including comparison, math and string operations. - The type\_ parameter is important to determine proper expression behavior - (such as, '+' means string concatenation or numerical addition based on - the type). - - :param text: the text of the expression; can be any SQL expression. - Quoting rules will not be applied. To specify a column-name expression - which should be subject to quoting rules, use the :func:`column` - function. - - :param type\_: an optional :class:`~sqlalchemy.types.TypeEngine` object which will - provide result-set translation and additional expression semantics for - this column. If left as None the type will be NullType. - - """ - return ColumnClause(text, type_=type_, is_literal=True) - -def table(name, *columns): - """Represent a textual table clause. - - The object returned is an instance of :class:`.TableClause`, which represents the - "syntactical" portion of the schema-level :class:`~.schema.Table` object. - It may be used to construct lightweight table constructs. - - Note that the :func:`~.expression.table` function is not part of - the ``sqlalchemy`` namespace. It must be imported from the ``sql`` package:: - - from sqlalchemy.sql import table, column - - :param name: Name of the table. - - :param columns: A collection of :func:`~.expression.column` constructs. - - See :class:`.TableClause` for further examples. - - """ - return TableClause(name, *columns) - -def bindparam(key, value=None, type_=None, unique=False, required=False, callable_=None): - """Create a bind parameter clause with the given key. - - :param key: - the key for this bind param. Will be used in the generated - SQL statement for dialects that use named parameters. This - value may be modified when part of a compilation operation, - if other :class:`_BindParamClause` objects exist with the same - key, or if its length is too long and truncation is - required. - - :param value: - Initial value for this bind param. This value may be - overridden by the dictionary of parameters sent to statement - compilation/execution. - - :param callable\_: - A callable function that takes the place of "value". The function - will be called at statement execution time to determine the - ultimate value. Used for scenarios where the actual bind - value cannot be determined at the point at which the clause - construct is created, but embedded bind values are still desirable. - - :param type\_: - A ``TypeEngine`` object that will be used to pre-process the - value corresponding to this :class:`_BindParamClause` at - execution time. - - :param unique: - if True, the key name of this BindParamClause will be - modified if another :class:`_BindParamClause` of the same name - already has been located within the containing - :class:`.ClauseElement`. - - :param required: - a value is required at execution time. - - """ - if isinstance(key, ColumnClause): - return _BindParamClause(key.name, value, type_=key.type, - callable_=callable_, - unique=unique, required=required) - else: - return _BindParamClause(key, value, type_=type_, - callable_=callable_, - unique=unique, required=required) - -def outparam(key, type_=None): - """Create an 'OUT' parameter for usage in functions (stored procedures), - for databases which support them. - - The ``outparam`` can be used like a regular function parameter. - The "output" value will be available from the - :class:`~sqlalchemy.engine.ResultProxy` object via its ``out_parameters`` - attribute, which returns a dictionary containing the values. - - """ - return _BindParamClause( - key, None, type_=type_, unique=False, isoutparam=True) - -def text(text, bind=None, *args, **kwargs): - """Create a SQL construct that is represented by a literal string. - - E.g.:: - - t = text("SELECT * FROM users") - result = connection.execute(t) - - The advantages :func:`text` provides over a plain string are - backend-neutral support for bind parameters, per-statement - execution options, as well as - bind parameter and result-column typing behavior, allowing - SQLAlchemy type constructs to play a role when executing - a statement that is specified literally. - - Bind parameters are specified by name, using the format ``:name``. - E.g.:: - - t = text("SELECT * FROM users WHERE id=:user_id") - result = connection.execute(t, user_id=12) - - To invoke SQLAlchemy typing logic for bind parameters, the - ``bindparams`` list allows specification of :func:`bindparam` - constructs which specify the type for a given name:: - - t = text("SELECT id FROM users WHERE updated_at>:updated", - bindparams=[bindparam('updated', DateTime())] - ) - - Typing during result row processing is also an important concern. - Result column types - are specified using the ``typemap`` dictionary, where the keys - match the names of columns. These names are taken from what - the DBAPI returns as ``cursor.description``:: - - t = text("SELECT id, name FROM users", - typemap={ - 'id':Integer, - 'name':Unicode - } - ) - - The :func:`text` construct is used internally for most cases when - a literal string is specified for part of a larger query, such as - within :func:`select()`, :func:`update()`, - :func:`insert()` or :func:`delete()`. In those cases, the same - bind parameter syntax is applied:: - - s = select([users.c.id, users.c.name]).where("id=:user_id") - result = connection.execute(s, user_id=12) - - Using :func:`text` explicitly usually implies the construction - of a full, standalone statement. As such, SQLAlchemy refers - to it as an :class:`.Executable` object, and it supports - the :meth:`Executable.execution_options` method. For example, - a :func:`text` construct that should be subject to "autocommit" - can be set explicitly so using the ``autocommit`` option:: - - t = text("EXEC my_procedural_thing()").\\ - execution_options(autocommit=True) - - Note that SQLAlchemy's usual "autocommit" behavior applies to - :func:`text` constructs - that is, statements which begin - with a phrase such as ``INSERT``, ``UPDATE``, ``DELETE``, - or a variety of other phrases specific to certain backends, will - be eligible for autocommit if no transaction is in progress. - - :param text: - the text of the SQL statement to be created. use ``:`` - to specify bind parameters; they will be compiled to their - engine-specific format. - - :param autocommit: - Deprecated. Use .execution_options(autocommit=) - to set the autocommit option. - - :param bind: - an optional connection or engine to be used for this text query. - - :param bindparams: - a list of :func:`bindparam()` instances which can be used to define - the types and/or initial values for the bind parameters within - the textual statement; the keynames of the bindparams must match - those within the text of the statement. The types will be used - for pre-processing on bind values. - - :param typemap: - a dictionary mapping the names of columns represented in the - columns clause of a ``SELECT`` statement to type objects, - which will be used to perform post-processing on columns within - the result set. This argument applies to any expression - that returns result sets. - - """ - return _TextClause(text, bind=bind, *args, **kwargs) - -def over(func, partition_by=None, order_by=None): - """Produce an OVER clause against a function. - - Used against aggregate or so-called "window" functions, - for database backends that support window functions. - - E.g.:: - - from sqlalchemy import over - over(func.row_number(), order_by='x') - - Would produce "ROW_NUMBER() OVER(ORDER BY x)". - - :param func: a :class:`.FunctionElement` construct, typically - generated by :attr:`~.expression.func`. - :param partition_by: a column element or string, or a list - of such, that will be used as the PARTITION BY clause - of the OVER construct. - :param order_by: a column element or string, or a list - of such, that will be used as the ORDER BY clause - of the OVER construct. - - This function is also available from the :attr:`~.expression.func` - construct itself via the :meth:`.FunctionElement.over` method. - - .. versionadded:: 0.7 - - """ - return _Over(func, partition_by=partition_by, order_by=order_by) - -def null(): - """Return a :class:`_Null` object, which compiles to ``NULL``. - - """ - return _Null() - -def true(): - """Return a :class:`_True` object, which compiles to ``true``, or the - boolean equivalent for the target dialect. - - """ - return _True() - -def false(): - """Return a :class:`_False` object, which compiles to ``false``, or the - boolean equivalent for the target dialect. - - """ - return _False() - -class _FunctionGenerator(object): - """Generate :class:`.Function` objects based on getattr calls.""" - - def __init__(self, **opts): - self.__names = [] - self.opts = opts - - def __getattr__(self, name): - # passthru __ attributes; fixes pydoc - if name.startswith('__'): - try: - return self.__dict__[name] - except KeyError: - raise AttributeError(name) - - elif name.endswith('_'): - name = name[0:-1] - f = _FunctionGenerator(**self.opts) - f.__names = list(self.__names) + [name] - return f - - def __call__(self, *c, **kwargs): - o = self.opts.copy() - o.update(kwargs) - if len(self.__names) == 1: - func = getattr(functions, self.__names[-1].lower(), None) - if func is not None and \ - isinstance(func, type) and \ - issubclass(func, Function): - return func(*c, **o) - - return Function(self.__names[-1], - packagenames=self.__names[0:-1], *c, **o) - -# "func" global - i.e. func.count() -func = _FunctionGenerator() -"""Generate SQL function expressions. - - ``func`` is a special object instance which generates SQL functions based on name-based attributes, e.g.:: - - >>> print func.count(1) - count(:param_1) - - The element is a column-oriented SQL element like any other, and is - used in that way:: - - >>> print select([func.count(table.c.id)]) - SELECT count(sometable.id) FROM sometable - - Any name can be given to ``func``. If the function name is unknown to - SQLAlchemy, it will be rendered exactly as is. For common SQL functions - which SQLAlchemy is aware of, the name may be interpreted as a *generic - function* which will be compiled appropriately to the target database:: - - >>> print func.current_timestamp() - CURRENT_TIMESTAMP - - To call functions which are present in dot-separated packages, specify them in the same manner:: - - >>> print func.stats.yield_curve(5, 10) - stats.yield_curve(:yield_curve_1, :yield_curve_2) - - SQLAlchemy can be made aware of the return type of functions to enable - type-specific lexical and result-based behavior. For example, to ensure - that a string-based function returns a Unicode value and is similarly - treated as a string in expressions, specify - :class:`~sqlalchemy.types.Unicode` as the type: - - >>> print func.my_string(u'hi', type_=Unicode) + ' ' + \ - ... func.my_string(u'there', type_=Unicode) - my_string(:my_string_1) || :my_string_2 || my_string(:my_string_3) - - The object returned by a ``func`` call is an instance of :class:`.Function`. - This object meets the "column" interface, including comparison and labeling - functions. The object can also be passed the :meth:`~.Connectable.execute` - method of a :class:`.Connection` or :class:`.Engine`, where it will be - wrapped inside of a SELECT statement first:: - - print connection.execute(func.current_timestamp()).scalar() - - A function can also be "bound" to a :class:`.Engine` or :class:`.Connection` - using the ``bind`` keyword argument, providing an execute() as well - as a scalar() method:: - - myfunc = func.current_timestamp(bind=some_engine) - print myfunc.scalar() - - Functions which are interpreted as "generic" functions know how to - calculate their return type automatically. For a listing of known generic - functions, see :ref:`generic_functions`. - -""" - -# "modifier" global - i.e. modifier.distinct -# TODO: use UnaryExpression for this instead ? -modifier = _FunctionGenerator(group=False) - -class _truncated_label(unicode): - """A unicode subclass used to identify symbolic " - "names that may require truncation.""" - - def apply_map(self, map_): - return self - -# for backwards compatibility in case -# someone is re-implementing the -# _truncated_identifier() sequence in a custom -# compiler -_generated_label = _truncated_label - -class _anonymous_label(_truncated_label): - """A unicode subclass used to identify anonymously - generated names.""" - - def __add__(self, other): - return _anonymous_label( - unicode(self) + - unicode(other)) - - def __radd__(self, other): - return _anonymous_label( - unicode(other) + - unicode(self)) - - def apply_map(self, map_): - return self % map_ - -def _as_truncated(value): - """coerce the given value to :class:`._truncated_label`. - - Existing :class:`._truncated_label` and - :class:`._anonymous_label` objects are passed - unchanged. - """ - - if isinstance(value, _truncated_label): - return value - else: - return _truncated_label(value) - -def _string_or_unprintable(element): - if isinstance(element, basestring): - return element - else: - try: - return str(element) - except: - return "unprintable element %r" % element - -def _clone(element, **kw): - return element._clone() - -def _expand_cloned(elements): - """expand the given set of ClauseElements to be the set of all 'cloned' - predecessors. - - """ - return itertools.chain(*[x._cloned_set for x in elements]) - -def _select_iterables(elements): - """expand tables into individual columns in the - given list of column expressions. - - """ - return itertools.chain(*[c._select_iterable for c in elements]) - -def _cloned_intersection(a, b): - """return the intersection of sets a and b, counting - any overlap between 'cloned' predecessors. - - The returned set is in terms of the entities present within 'a'. - - """ - all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) - return set(elem for elem in a - if all_overlap.intersection(elem._cloned_set)) - - -def _is_literal(element): - return not isinstance(element, Visitable) and \ - not hasattr(element, '__clause_element__') - -def _from_objects(*elements): - return itertools.chain(*[element._from_objects for element in elements]) - -def _labeled(element): - if not hasattr(element, 'name'): - return element.label(None) - else: - return element - -def _column_as_key(element): - if isinstance(element, basestring): - return element - if hasattr(element, '__clause_element__'): - element = element.__clause_element__() - return element.key - -def _literal_as_text(element): - if isinstance(element, Visitable): - return element - elif hasattr(element, '__clause_element__'): - return element.__clause_element__() - elif isinstance(element, basestring): - return _TextClause(unicode(element)) - elif isinstance(element, (util.NoneType, bool)): - return _const_expr(element) - else: - raise exc.ArgumentError( - "SQL expression object or string expected." - ) - -def _const_expr(element): - if element is None: - return null() - elif element is False: - return false() - elif element is True: - return true() - else: - raise exc.ArgumentError( - "Expected None, False, or True" - ) - -def _clause_element_as_expr(element): - if hasattr(element, '__clause_element__'): - return element.__clause_element__() - else: - return element - -def _literal_as_column(element): - if isinstance(element, Visitable): - return element - elif hasattr(element, '__clause_element__'): - return element.__clause_element__() - else: - return literal_column(str(element)) - -def _literal_as_binds(element, name=None, type_=None): - if hasattr(element, '__clause_element__'): - return element.__clause_element__() - elif not isinstance(element, Visitable): - if element is None: - return null() - else: - return _BindParamClause(name, element, type_=type_, unique=True) - else: - return element - -def _type_from_args(args): - for a in args: - if not isinstance(a.type, sqltypes.NullType): - return a.type - else: - return sqltypes.NullType - -def _no_literals(element): - if hasattr(element, '__clause_element__'): - return element.__clause_element__() - elif not isinstance(element, Visitable): - raise exc.ArgumentError("Ambiguous literal: %r. Use the 'text()' " - "function to indicate a SQL expression " - "literal, or 'literal()' to indicate a " - "bound value." % element) - else: - return element - -def _only_column_elements_or_none(element, name): - if element is None: - return None - else: - return _only_column_elements(element, name) - -def _only_column_elements(element, name): - if hasattr(element, '__clause_element__'): - element = element.__clause_element__() - if not isinstance(element, ColumnElement): - raise exc.ArgumentError( - "Column-based expression object expected for argument " - "'%s'; got: '%s', type %s" % (name, element, type(element))) - return element - -def _corresponding_column_or_error(fromclause, column, - require_embedded=False): - c = fromclause.corresponding_column(column, - require_embedded=require_embedded) - if c is None: - raise exc.InvalidRequestError( - "Given column '%s', attached to table '%s', " - "failed to locate a corresponding column from table '%s'" - % - (column, - getattr(column, 'table', None),fromclause.description) - ) - return c - -@util.decorator -def _generative(fn, *args, **kw): - """Mark a method as generative.""" - - self = args[0]._generate() - fn(self, *args[1:], **kw) - return self - - -def is_column(col): - """True if ``col`` is an instance of :class:`.ColumnElement`.""" - - return isinstance(col, ColumnElement) - - -class ClauseElement(Visitable): - """Base class for elements of a programmatically constructed SQL - expression. - - """ - __visit_name__ = 'clause' - - _annotations = {} - supports_execution = False - _from_objects = [] - bind = None - _is_clone_of = None - - def _clone(self): - """Create a shallow copy of this ClauseElement. - - This method may be used by a generative API. Its also used as - part of the "deep" copy afforded by a traversal that combines - the _copy_internals() method. - - """ - c = self.__class__.__new__(self.__class__) - c.__dict__ = self.__dict__.copy() - c.__dict__.pop('_cloned_set', None) - - # this is a marker that helps to "equate" clauses to each other - # when a Select returns its list of FROM clauses. the cloning - # process leaves around a lot of remnants of the previous clause - # typically in the form of column expressions still attached to the - # old table. - c._is_clone_of = self - - return c - - @property - def _constructor(self): - """return the 'constructor' for this ClauseElement. - - This is for the purposes for creating a new object of - this type. Usually, its just the element's __class__. - However, the "Annotated" version of the object overrides - to return the class of its proxied element. - - """ - return self.__class__ - - @util.memoized_property - def _cloned_set(self): - """Return the set consisting all cloned ancestors of this - ClauseElement. - - Includes this ClauseElement. This accessor tends to be used for - FromClause objects to identify 'equivalent' FROM clauses, regardless - of transformative operations. - - """ - s = util.column_set() - f = self - while f is not None: - s.add(f) - f = f._is_clone_of - return s - - def __getstate__(self): - d = self.__dict__.copy() - d.pop('_is_clone_of', None) - return d - - if util.jython: - def __hash__(self): - """Return a distinct hash code. - - ClauseElements may have special equality comparisons which - makes us rely on them having unique hash codes for use in - hash-based collections. Stock __hash__ doesn't guarantee - unique values on platforms with moving GCs. - """ - return id(self) - - def _annotate(self, values): - """return a copy of this ClauseElement with the given annotations - dictionary. - - """ - return sqlutil.Annotated(self, values) - - def _deannotate(self): - """return a copy of this ClauseElement with an empty annotations - dictionary. - - """ - return self._clone() - - def unique_params(self, *optionaldict, **kwargs): - """Return a copy with :func:`bindparam()` elements replaced. - - Same functionality as ``params()``, except adds `unique=True` - to affected bind parameters so that multiple statements can be - used. - - """ - return self._params(True, optionaldict, kwargs) - - def params(self, *optionaldict, **kwargs): - """Return a copy with :func:`bindparam()` elements replaced. - - Returns a copy of this ClauseElement with :func:`bindparam()` - elements replaced with values taken from the given dictionary:: - - >>> clause = column('x') + bindparam('foo') - >>> print clause.compile().params - {'foo':None} - >>> print clause.params({'foo':7}).compile().params - {'foo':7} - - """ - return self._params(False, optionaldict, kwargs) - - def _params(self, unique, optionaldict, kwargs): - if len(optionaldict) == 1: - kwargs.update(optionaldict[0]) - elif len(optionaldict) > 1: - raise exc.ArgumentError( - "params() takes zero or one positional dictionary argument") - - def visit_bindparam(bind): - if bind.key in kwargs: - bind.value = kwargs[bind.key] - if unique: - bind._convert_to_unique() - return cloned_traverse(self, {}, {'bindparam':visit_bindparam}) - - def compare(self, other, **kw): - """Compare this ClauseElement to the given ClauseElement. - - Subclasses should override the default behavior, which is a - straight identity comparison. - - \**kw are arguments consumed by subclass compare() methods and - may be used to modify the criteria for comparison. - (see :class:`.ColumnElement`) - - """ - return self is other - - def _copy_internals(self, clone=_clone, **kw): - """Reassign internal elements to be clones of themselves. - - Called during a copy-and-traverse operation on newly - shallow-copied elements to create a deep copy. - - The given clone function should be used, which may be applying - additional transformations to the element (i.e. replacement - traversal, cloned traversal, annotations). - - """ - pass - - def get_children(self, **kwargs): - """Return immediate child elements of this :class:`.ClauseElement`. - - This is used for visit traversal. - - \**kwargs may contain flags that change the collection that is - returned, for example to return a subset of items in order to - cut down on larger traversals, or to return child items from a - different context (such as schema-level collections instead of - clause-level). - - """ - return [] - - def self_group(self, against=None): - """Apply a 'grouping' to this :class:`.ClauseElement`. - - This method is overridden by subclasses to return a - "grouping" construct, i.e. parenthesis. In particular - it's used by "binary" expressions to provide a grouping - around themselves when placed into a larger expression, - as well as by :func:`.select` constructs when placed into - the FROM clause of another :func:`.select`. (Note that - subqueries should be normally created using the - :func:`.Select.alias` method, as many platforms require - nested SELECT statements to be named). - - As expressions are composed together, the application of - :meth:`self_group` is automatic - end-user code should never - need to use this method directly. Note that SQLAlchemy's - clause constructs take operator precedence into account - - so parenthesis might not be needed, for example, in - an expression like ``x OR (y AND z)`` - AND takes precedence - over OR. - - The base :meth:`self_group` method of :class:`.ClauseElement` - just returns self. - """ - return self - - - @util.deprecated('0.7', - 'Only SQL expressions which subclass ' - ':class:`.Executable` may provide the ' - ':func:`.execute` method.') - def execute(self, *multiparams, **params): - """Compile and execute this :class:`.ClauseElement`. - - """ - e = self.bind - if e is None: - label = getattr(self, 'description', self.__class__.__name__) - msg = ('This %s does not support direct execution.' % label) - raise exc.UnboundExecutionError(msg) - return e._execute_clauseelement(self, multiparams, params) - - @util.deprecated('0.7', - 'Only SQL expressions which subclass ' - ':class:`.Executable` may provide the ' - ':func:`.scalar` method.') - def scalar(self, *multiparams, **params): - """Compile and execute this :class:`.ClauseElement`, returning - the result's scalar representation. - - """ - return self.execute(*multiparams, **params).scalar() - - def compile(self, bind=None, dialect=None, **kw): - """Compile this SQL expression. - - The return value is a :class:`~sqlalchemy.engine.Compiled` object. - Calling ``str()`` or ``unicode()`` on the returned value will yield a - string representation of the result. The - :class:`~sqlalchemy.engine.Compiled` object also can return a - dictionary of bind parameter names and values - using the ``params`` accessor. - - :param bind: An ``Engine`` or ``Connection`` from which a - ``Compiled`` will be acquired. This argument takes precedence over - this :class:`.ClauseElement`'s bound engine, if any. - - :param column_keys: Used for INSERT and UPDATE statements, a list of - column names which should be present in the VALUES clause of the - compiled statement. If ``None``, all columns from the target table - object are rendered. - - :param dialect: A ``Dialect`` instance from which a ``Compiled`` - will be acquired. This argument takes precedence over the `bind` - argument as well as this :class:`.ClauseElement`'s bound engine, if - any. - - :param inline: Used for INSERT statements, for a dialect which does - not support inline retrieval of newly generated primary key - columns, will force the expression used to create the new primary - key value to be rendered inline within the INSERT statement's - VALUES clause. This typically refers to Sequence execution but may - also refer to any server-side default generation function - associated with a primary key `Column`. - - """ - - if not dialect: - if bind: - dialect = bind.dialect - elif self.bind: - dialect = self.bind.dialect - bind = self.bind - else: - dialect = default.DefaultDialect() - return self._compiler(dialect, bind=bind, **kw) - - def _compiler(self, dialect, **kw): - """Return a compiler appropriate for this ClauseElement, given a - Dialect.""" - - return dialect.statement_compiler(dialect, self, **kw) - - def __str__(self): - # Py3K - #return unicode(self.compile()) - # Py2K - return unicode(self.compile()).encode('ascii', 'backslashreplace') - # end Py2K - - def __and__(self, other): - return and_(self, other) - - def __or__(self, other): - return or_(self, other) - - def __invert__(self): - return self._negate() - - def __nonzero__(self): - raise TypeError("Boolean value of this clause is not defined") - - def _negate(self): - if hasattr(self, 'negation_clause'): - return self.negation_clause - else: - return _UnaryExpression( - self.self_group(against=operators.inv), - operator=operators.inv, - negate=None) - - def __repr__(self): - friendly = getattr(self, 'description', None) - if friendly is None: - return object.__repr__(self) - else: - return '<%s.%s at 0x%x; %s>' % ( - self.__module__, self.__class__.__name__, id(self), friendly) - - -class _Immutable(object): - """mark a ClauseElement as 'immutable' when expressions are cloned.""" - - def unique_params(self, *optionaldict, **kwargs): - raise NotImplementedError("Immutable objects do not support copying") - - def params(self, *optionaldict, **kwargs): - raise NotImplementedError("Immutable objects do not support copying") - - def _clone(self): - return self - - -class _CompareMixin(ColumnOperators): - """Defines comparison and math operations for :class:`.ClauseElement` - instances. - - See :class:`.ColumnOperators` and :class:`.Operators` for descriptions - of all operations. - - """ - - def __compare(self, op, obj, negate=None, reverse=False, - **kwargs - ): - if obj is None or isinstance(obj, _Null): - if op in (operators.eq, operators.is_): - return _BinaryExpression(self, null(), operators.is_, - negate=operators.isnot) - elif op in (operators.ne, operators.isnot): - return _BinaryExpression(self, null(), operators.isnot, - negate=operators.is_) - else: - raise exc.ArgumentError("Only '='/'!=' operators can " - "be used with NULL") - else: - obj = self._check_literal(op, obj) - - if reverse: - return _BinaryExpression(obj, - self, - op, - type_=sqltypes.BOOLEANTYPE, - negate=negate, modifiers=kwargs) - else: - return _BinaryExpression(self, - obj, - op, - type_=sqltypes.BOOLEANTYPE, - negate=negate, modifiers=kwargs) - - def __operate(self, op, obj, reverse=False): - obj = self._check_literal(op, obj) - - if reverse: - left, right = obj, self - else: - left, right = self, obj - - if left.type is None: - op, result_type = sqltypes.NULLTYPE._adapt_expression(op, - right.type) - elif right.type is None: - op, result_type = left.type._adapt_expression(op, - sqltypes.NULLTYPE) - else: - op, result_type = left.type._adapt_expression(op, - right.type) - return _BinaryExpression(left, right, op, type_=result_type) - - - # a mapping of operators with the method they use, along with their negated - # operator for comparison operators - operators = { - operators.add : (__operate,), - operators.mul : (__operate,), - operators.sub : (__operate,), - # Py2K - operators.div : (__operate,), - # end Py2K - operators.mod : (__operate,), - operators.truediv : (__operate,), - operators.lt : (__compare, operators.ge), - operators.le : (__compare, operators.gt), - operators.ne : (__compare, operators.eq), - operators.gt : (__compare, operators.le), - operators.ge : (__compare, operators.lt), - operators.eq : (__compare, operators.ne), - operators.like_op : (__compare, operators.notlike_op), - operators.ilike_op : (__compare, operators.notilike_op), - operators.is_ : (__compare, operators.is_), - operators.isnot : (__compare, operators.isnot), - } - - def operate(self, op, *other, **kwargs): - o = _CompareMixin.operators[op] - return o[0](self, op, other[0], *o[1:], **kwargs) - - def reverse_operate(self, op, other, **kwargs): - o = _CompareMixin.operators[op] - return o[0](self, op, other, reverse=True, *o[1:], **kwargs) - - def in_(self, other): - """See :meth:`.ColumnOperators.in_`.""" - return self._in_impl(operators.in_op, operators.notin_op, other) - - def _in_impl(self, op, negate_op, seq_or_selectable): - seq_or_selectable = _clause_element_as_expr(seq_or_selectable) - - if isinstance(seq_or_selectable, _ScalarSelect): - return self.__compare(op, seq_or_selectable, - negate=negate_op) - elif isinstance(seq_or_selectable, _SelectBase): - - # TODO: if we ever want to support (x, y, z) IN (select x, - # y, z from table), we would need a multi-column version of - # as_scalar() to produce a multi- column selectable that - # does not export itself as a FROM clause - - return self.__compare(op, seq_or_selectable.as_scalar(), - negate=negate_op) - elif isinstance(seq_or_selectable, (Selectable, _TextClause)): - return self.__compare(op, seq_or_selectable, - negate=negate_op) - - - # Handle non selectable arguments as sequences - - args = [] - for o in seq_or_selectable: - if not _is_literal(o): - if not isinstance(o, _CompareMixin): - raise exc.InvalidRequestError('in() function accept' - 's either a list of non-selectable values, ' - 'or a selectable: %r' % o) - else: - o = self._bind_param(op, o) - args.append(o) - if len(args) == 0: - - # Special case handling for empty IN's, behave like - # comparison against zero row selectable. We use != to - # build the contradiction as it handles NULL values - # appropriately, i.e. "not (x IN ())" should not return NULL - # values for x. - - util.warn('The IN-predicate on "%s" was invoked with an ' - 'empty sequence. This results in a ' - 'contradiction, which nonetheless can be ' - 'expensive to evaluate. Consider alternative ' - 'strategies for improved performance.' % self) - return self != self - - return self.__compare(op, - ClauseList(*args).self_group(against=op), - negate=negate_op) - - def __neg__(self): - """See :meth:`.ColumnOperators.__neg__`.""" - return _UnaryExpression(self, operator=operators.neg) - - def startswith(self, other, escape=None): - """See :meth:`.ColumnOperators.startswith`.""" - # use __radd__ to force string concat behavior - return self.__compare( - operators.like_op, - literal_column("'%'", type_=sqltypes.String).__radd__( - self._check_literal(operators.like_op, other) - ), - escape=escape) - - def endswith(self, other, escape=None): - """See :meth:`.ColumnOperators.endswith`.""" - return self.__compare( - operators.like_op, - literal_column("'%'", type_=sqltypes.String) + - self._check_literal(operators.like_op, other), - escape=escape) - - def contains(self, other, escape=None): - """See :meth:`.ColumnOperators.contains`.""" - return self.__compare( - operators.like_op, - literal_column("'%'", type_=sqltypes.String) + - self._check_literal(operators.like_op, other) + - literal_column("'%'", type_=sqltypes.String), - escape=escape) - - def match(self, other): - """See :meth:`.ColumnOperators.match`.""" - return self.__compare(operators.match_op, - self._check_literal(operators.match_op, - other)) - - def label(self, name): - """Produce a column label, i.e. `` AS ``. - - This is a shortcut to the :func:`~.expression.label` function. - - if 'name' is None, an anonymous label name will be generated. - - """ - return _Label(name, self, self.type) - - def desc(self): - """See :meth:`.ColumnOperators.desc`.""" - return desc(self) - - def asc(self): - """See :meth:`.ColumnOperators.asc`.""" - return asc(self) - - def nullsfirst(self): - """See :meth:`.ColumnOperators.nullsfirst`.""" - return nullsfirst(self) - - def nullslast(self): - """See :meth:`.ColumnOperators.nullslast`.""" - return nullslast(self) - - def distinct(self): - """See :meth:`.ColumnOperators.distinct`.""" - return _UnaryExpression(self, operator=operators.distinct_op, - type_=self.type) - - def between(self, cleft, cright): - """See :meth:`.ColumnOperators.between`.""" - return _BinaryExpression( - self, - ClauseList( - self._check_literal(operators.and_, cleft), - self._check_literal(operators.and_, cright), - operator=operators.and_, - group=False), - operators.between_op) - - def collate(self, collation): - """See :meth:`.ColumnOperators.collate`.""" - - return collate(self, collation) - - def op(self, operator): - """See :meth:`.ColumnOperators.op`.""" - - return lambda other: self.__operate(operator, other) - - def _bind_param(self, operator, obj): - return _BindParamClause(None, obj, - _compared_to_operator=operator, - _compared_to_type=self.type, unique=True) - - def _check_literal(self, operator, other): - if isinstance(other, _BindParamClause) and \ - isinstance(other.type, sqltypes.NullType): - # TODO: perhaps we should not mutate the incoming bindparam() - # here and instead make a copy of it. this might - # be the only place that we're mutating an incoming construct. - other.type = self.type - return other - elif hasattr(other, '__clause_element__'): - other = other.__clause_element__() - if isinstance(other, (_SelectBase, Alias)): - other = other.as_scalar() - return other - elif not isinstance(other, ClauseElement): - return self._bind_param(operator, other) - elif isinstance(other, (_SelectBase, Alias)): - return other.as_scalar() - else: - return other - - -class ColumnElement(ClauseElement, _CompareMixin): - """Represent an element that is usable within the "column clause" portion - of a ``SELECT`` statement. - - This includes columns associated with tables, aliases, and - subqueries, expressions, function calls, SQL keywords such as - ``NULL``, literals, etc. :class:`.ColumnElement` is the ultimate base - class for all such elements. - - :class:`.ColumnElement` supports the ability to be a *proxy* element, - which indicates that the :class:`.ColumnElement` may be associated with - a :class:`.Selectable` which was derived from another :class:`.Selectable`. - An example of a "derived" :class:`.Selectable` is an :class:`.Alias` of a - :class:`~sqlalchemy.schema.Table`. - - A :class:`.ColumnElement`, by subclassing the :class:`_CompareMixin` mixin - class, provides the ability to generate new :class:`.ClauseElement` - objects using Python expressions. See the :class:`_CompareMixin` - docstring for more details. - - """ - - __visit_name__ = 'column' - primary_key = False - foreign_keys = [] - quote = None - _label = None - _key_label = None - _alt_names = () - - @property - def _select_iterable(self): - return (self, ) - - @util.memoized_property - def base_columns(self): - return util.column_set(c for c in self.proxy_set - if not hasattr(c, 'proxies')) - - @util.memoized_property - def proxy_set(self): - s = util.column_set([self]) - if hasattr(self, 'proxies'): - for c in self.proxies: - s.update(c.proxy_set) - return s - - def shares_lineage(self, othercolumn): - """Return True if the given :class:`.ColumnElement` - has a common ancestor to this :class:`.ColumnElement`.""" - - return bool(self.proxy_set.intersection(othercolumn.proxy_set)) - - def _compare_name_for_result(self, other): - """Return True if the given column element compares to this one - when targeting within a result row.""" - - return hasattr(other, 'name') and hasattr(self, 'name') and \ - other.name == self.name - - def _make_proxy(self, selectable, name=None): - """Create a new :class:`.ColumnElement` representing this - :class:`.ColumnElement` as it appears in the select list of a - descending selectable. - - """ - if name is None: - name = self.anon_label - # TODO: may want to change this to anon_label, - # or some value that is more useful than the - # compiled form of the expression - key = str(self) - else: - key = name - - co = ColumnClause(_as_truncated(name), - selectable, - type_=getattr(self, - 'type', None)) - co.proxies = [self] - if selectable._is_clone_of is not None: - co._is_clone_of = \ - selectable._is_clone_of.columns.get(key) - selectable._columns[key] = co - return co - - def compare(self, other, use_proxies=False, equivalents=None, **kw): - """Compare this ColumnElement to another. - - Special arguments understood: - - :param use_proxies: when True, consider two columns that - share a common base column as equivalent (i.e. shares_lineage()) - - :param equivalents: a dictionary of columns as keys mapped to sets - of columns. If the given "other" column is present in this - dictionary, if any of the columns in the corresponding set() pass the - comparison test, the result is True. This is used to expand the - comparison to other columns that may be known to be equivalent to - this one via foreign key or other criterion. - - """ - to_compare = (other, ) - if equivalents and other in equivalents: - to_compare = equivalents[other].union(to_compare) - - for oth in to_compare: - if use_proxies and self.shares_lineage(oth): - return True - elif oth is self: - return True - else: - return False - - @util.memoized_property - def anon_label(self): - """provides a constant 'anonymous label' for this ColumnElement. - - This is a label() expression which will be named at compile time. - The same label() is returned each time anon_label is called so - that expressions can reference anon_label multiple times, producing - the same label name at compile time. - - the compiler uses this function automatically at compile time - for expressions that are known to be 'unnamed' like binary - expressions and function calls. - - """ - return _anonymous_label('%%(%d %s)s' % (id(self), getattr(self, - 'name', 'anon'))) - -class ColumnCollection(util.OrderedProperties): - """An ordered dictionary that stores a list of ColumnElement - instances. - - Overrides the ``__eq__()`` method to produce SQL clauses between - sets of correlated columns. - - """ - - def __init__(self, *cols): - super(ColumnCollection, self).__init__() - self._data.update((c.key, c) for c in cols) - self.__dict__['_all_cols'] = util.column_set(self) - - def __str__(self): - return repr([str(c) for c in self]) - - def replace(self, column): - """add the given column to this collection, removing unaliased - versions of this column as well as existing columns with the - same key. - - e.g.:: - - t = Table('sometable', metadata, Column('col1', Integer)) - t.columns.replace(Column('col1', Integer, key='columnone')) - - will remove the original 'col1' from the collection, and add - the new column under the name 'columnname'. - - Used by schema.Column to override columns during table reflection. - - """ - if column.name in self and column.key != column.name: - other = self[column.name] - if other.name == other.key: - del self._data[other.name] - self._all_cols.remove(other) - if column.key in self._data: - self._all_cols.remove(self._data[column.key]) - self._all_cols.add(column) - self._data[column.key] = column - - def add(self, column): - """Add a column to this collection. - - The key attribute of the column will be used as the hash key - for this dictionary. - - """ - self[column.key] = column - - def __delitem__(self, key): - raise NotImplementedError() - - def __setattr__(self, key, object): - raise NotImplementedError() - - def __setitem__(self, key, value): - if key in self: - - # this warning is primarily to catch select() statements - # which have conflicting column names in their exported - # columns collection - - existing = self[key] - if not existing.shares_lineage(value): - util.warn('Column %r on table %r being replaced by ' - 'another column with the same key. Consider ' - 'use_labels for select() statements.' % (key, - getattr(existing, 'table', None))) - self._all_cols.remove(existing) - # pop out memoized proxy_set as this - # operation may very well be occurring - # in a _make_proxy operation - value.__dict__.pop('proxy_set', None) - self._all_cols.add(value) - self._data[key] = value - - def clear(self): - self._data.clear() - self._all_cols.clear() - - def remove(self, column): - del self._data[column.key] - self._all_cols.remove(column) - - def update(self, value): - self._data.update(value) - self._all_cols.clear() - self._all_cols.update(self._data.values()) - - def extend(self, iter): - self.update((c.key, c) for c in iter) - - __hash__ = None - - def __eq__(self, other): - l = [] - for c in other: - for local in self: - if c.shares_lineage(local): - l.append(c==local) - return and_(*l) - - def __contains__(self, other): - if not isinstance(other, basestring): - raise exc.ArgumentError("__contains__ requires a string argument") - return util.OrderedProperties.__contains__(self, other) - - def __setstate__(self, state): - self.__dict__['_data'] = state['_data'] - self.__dict__['_all_cols'] = util.column_set(self._data.values()) - - def contains_column(self, col): - # this has to be done via set() membership - return col in self._all_cols - - def as_immutable(self): - return ImmutableColumnCollection(self._data, self._all_cols) - -class ImmutableColumnCollection(util.ImmutableProperties, ColumnCollection): - def __init__(self, data, colset): - util.ImmutableProperties.__init__(self, data) - self.__dict__['_all_cols'] = colset - - extend = remove = util.ImmutableProperties._immutable - - -class ColumnSet(util.ordered_column_set): - def contains_column(self, col): - return col in self - - def extend(self, cols): - for col in cols: - self.add(col) - - def __add__(self, other): - return list(self) + list(other) - - def __eq__(self, other): - l = [] - for c in other: - for local in self: - if c.shares_lineage(local): - l.append(c==local) - return and_(*l) - - def __hash__(self): - return hash(tuple(x for x in self)) - -class Selectable(ClauseElement): - """mark a class as being selectable""" - __visit_name__ = 'selectable' - -class FromClause(Selectable): - """Represent an element that can be used within the ``FROM`` - clause of a ``SELECT`` statement. - - """ - __visit_name__ = 'fromclause' - named_with_column = False - _hide_froms = [] - quote = None - schema = None - _memoized_property = util.group_expirable_memoized_property(["_columns"]) - - def count(self, whereclause=None, **params): - """return a SELECT COUNT generated against this - :class:`.FromClause`.""" - - if self.primary_key: - col = list(self.primary_key)[0] - else: - col = list(self.columns)[0] - return select( - [func.count(col).label('tbl_row_count')], - whereclause, - from_obj=[self], - **params) - - def select(self, whereclause=None, **params): - """return a SELECT of this :class:`.FromClause`.""" - - return select([self], whereclause, **params) - - def join(self, right, onclause=None, isouter=False): - """return a join of this :class:`.FromClause` against another - :class:`.FromClause`.""" - - return Join(self, right, onclause, isouter) - - def outerjoin(self, right, onclause=None): - """return an outer join of this :class:`.FromClause` against another - :class:`.FromClause`.""" - - return Join(self, right, onclause, True) - - def alias(self, name=None): - """return an alias of this :class:`.FromClause`. - - This is shorthand for calling:: - - from sqlalchemy import alias - a = alias(self, name=name) - - See :func:`~.expression.alias` for details. - - """ - - return Alias(self, name) - - def is_derived_from(self, fromclause): - """Return True if this FromClause is 'derived' from the given - FromClause. - - An example would be an Alias of a Table is derived from that Table. - - """ - # this is essentially an "identity" check in the base class. - # Other constructs override this to traverse through - # contained elements. - return fromclause in self._cloned_set - - def _is_lexical_equivalent(self, other): - """Return True if this FromClause and the other represent - the same lexical identity. - - This tests if either one is a copy of the other, or - if they are the same via annotation identity. - - """ - return self._cloned_set.intersection(other._cloned_set) - - def replace_selectable(self, old, alias): - """replace all occurrences of FromClause 'old' with the given Alias - object, returning a copy of this :class:`.FromClause`. - - """ - - return sqlutil.ClauseAdapter(alias).traverse(self) - - def correspond_on_equivalents(self, column, equivalents): - """Return corresponding_column for the given column, or if None - search for a match in the given dictionary. - - """ - col = self.corresponding_column(column, require_embedded=True) - if col is None and col in equivalents: - for equiv in equivalents[col]: - nc = self.corresponding_column(equiv, require_embedded=True) - if nc: - return nc - return col - - def corresponding_column(self, column, require_embedded=False): - """Given a :class:`.ColumnElement`, return the exported - :class:`.ColumnElement` object from this :class:`.Selectable` - which corresponds to that original - :class:`~sqlalchemy.schema.Column` via a common ancestor - column. - - :param column: the target :class:`.ColumnElement` to be matched - - :param require_embedded: only return corresponding columns for - the given :class:`.ColumnElement`, if the given - :class:`.ColumnElement` is actually present within a sub-element - of this :class:`.FromClause`. Normally the column will match if - it merely shares a common ancestor with one of the exported - columns of this :class:`.FromClause`. - - """ - - def embedded(expanded_proxy_set, target_set): - for t in target_set.difference(expanded_proxy_set): - if not set(_expand_cloned([t]) - ).intersection(expanded_proxy_set): - return False - return True - - # don't dig around if the column is locally present - if self.c.contains_column(column): - return column - col, intersect = None, None - target_set = column.proxy_set - cols = self.c - for c in cols: - expanded_proxy_set = set(_expand_cloned(c.proxy_set)) - i = target_set.intersection(expanded_proxy_set) - if i and (not require_embedded - or embedded(expanded_proxy_set, target_set)): - if col is None: - - # no corresponding column yet, pick this one. - - col, intersect = c, i - elif len(i) > len(intersect): - - # 'c' has a larger field of correspondence than - # 'col'. i.e. selectable.c.a1_x->a1.c.x->table.c.x - # matches a1.c.x->table.c.x better than - # selectable.c.x->table.c.x does. - - col, intersect = c, i - elif i == intersect: - - # they have the same field of correspondence. see - # which proxy_set has fewer columns in it, which - # indicates a closer relationship with the root - # column. Also take into account the "weight" - # attribute which CompoundSelect() uses to give - # higher precedence to columns based on vertical - # position in the compound statement, and discard - # columns that have no reference to the target - # column (also occurs with CompoundSelect) - - col_distance = util.reduce(operator.add, - [sc._annotations.get('weight', 1) for sc in - col.proxy_set if sc.shares_lineage(column)]) - c_distance = util.reduce(operator.add, - [sc._annotations.get('weight', 1) for sc in - c.proxy_set if sc.shares_lineage(column)]) - if c_distance < col_distance: - col, intersect = c, i - return col - - @property - def description(self): - """a brief description of this FromClause. - - Used primarily for error message formatting. - - """ - return getattr(self, 'name', self.__class__.__name__ + " object") - - def _reset_exported(self): - """delete memoized collections when a FromClause is cloned.""" - - self._memoized_property.expire_instance(self) - - @_memoized_property - def columns(self): - """Return the collection of Column objects contained by this - FromClause.""" - - if '_columns' not in self.__dict__: - self._init_collections() - self._populate_column_collection() - return self._columns.as_immutable() - - @_memoized_property - def primary_key(self): - """Return the collection of Column objects which comprise the - primary key of this FromClause.""" - - self._init_collections() - self._populate_column_collection() - return self.primary_key - - @_memoized_property - def foreign_keys(self): - """Return the collection of ForeignKey objects which this - FromClause references.""" - - self._init_collections() - self._populate_column_collection() - return self.foreign_keys - - c = property(attrgetter('columns')) - _select_iterable = property(attrgetter('columns')) - - def _init_collections(self): - assert '_columns' not in self.__dict__ - assert 'primary_key' not in self.__dict__ - assert 'foreign_keys' not in self.__dict__ - - self._columns = ColumnCollection() - self.primary_key = ColumnSet() - self.foreign_keys = set() - - def _populate_column_collection(self): - pass - -class _BindParamClause(ColumnElement): - """Represent a bind parameter. - - Public constructor is the :func:`bindparam()` function. - - """ - - __visit_name__ = 'bindparam' - quote = None - - def __init__(self, key, value, type_=None, unique=False, - callable_=None, - isoutparam=False, required=False, - _compared_to_operator=None, - _compared_to_type=None): - """Construct a _BindParamClause. - - :param key: - the key for this bind param. Will be used in the generated - SQL statement for dialects that use named parameters. This - value may be modified when part of a compilation operation, - if other :class:`_BindParamClause` objects exist with the same - key, or if its length is too long and truncation is - required. - - :param value: - Initial value for this bind param. This value may be - overridden by the dictionary of parameters sent to statement - compilation/execution. - - :param callable\_: - A callable function that takes the place of "value". The function - will be called at statement execution time to determine the - ultimate value. Used for scenarios where the actual bind - value cannot be determined at the point at which the clause - construct is created, but embedded bind values are still desirable. - - :param type\_: - A ``TypeEngine`` object that will be used to pre-process the - value corresponding to this :class:`_BindParamClause` at - execution time. - - :param unique: - if True, the key name of this BindParamClause will be - modified if another :class:`_BindParamClause` of the same name - already has been located within the containing - :class:`.ClauseElement`. - - :param required: - a value is required at execution time. - - :param isoutparam: - if True, the parameter should be treated like a stored procedure - "OUT" parameter. - - """ - if unique: - self.key = _anonymous_label('%%(%d %s)s' % (id(self), key - or 'param')) - else: - self.key = key or _anonymous_label('%%(%d param)s' - % id(self)) - - # identifying key that won't change across - # clones, used to identify the bind's logical - # identity - self._identifying_key = self.key - - # key that was passed in the first place, used to - # generate new keys - self._orig_key = key or 'param' - - self.unique = unique - self.value = value - self.callable = callable_ - self.isoutparam = isoutparam - self.required = required - if type_ is None: - if _compared_to_type is not None: - self.type = \ - _compared_to_type._coerce_compared_value( - _compared_to_operator, value) - else: - self.type = sqltypes._type_map.get(type(value), - sqltypes.NULLTYPE) - elif isinstance(type_, type): - self.type = type_() - else: - self.type = type_ - - @property - def effective_value(self): - """Return the value of this bound parameter, - taking into account if the ``callable`` parameter - was set. - - The ``callable`` value will be evaluated - and returned if present, else ``value``. - - """ - if self.callable: - return self.callable() - else: - return self.value - - def _clone(self): - c = ClauseElement._clone(self) - if self.unique: - c.key = _anonymous_label('%%(%d %s)s' % (id(c), c._orig_key - or 'param')) - return c - - def _convert_to_unique(self): - if not self.unique: - self.unique = True - self.key = _anonymous_label('%%(%d %s)s' % (id(self), - self._orig_key or 'param')) - - def compare(self, other, **kw): - """Compare this :class:`_BindParamClause` to the given - clause.""" - - return isinstance(other, _BindParamClause) \ - and self.type._compare_type_affinity(other.type) \ - and self.value == other.value - - def __getstate__(self): - """execute a deferred value for serialization purposes.""" - - d = self.__dict__.copy() - v = self.value - if self.callable: - v = self.callable() - d['callable'] = None - d['value'] = v - return d - - def __repr__(self): - return '_BindParamClause(%r, %r, type_=%r)' % (self.key, - self.value, self.type) - -class _TypeClause(ClauseElement): - """Handle a type keyword in a SQL statement. - - Used by the ``Case`` statement. - - """ - - __visit_name__ = 'typeclause' - - def __init__(self, type): - self.type = type - - -class _Generative(object): - """Allow a ClauseElement to generate itself via the - @_generative decorator. - - """ - - def _generate(self): - s = self.__class__.__new__(self.__class__) - s.__dict__ = self.__dict__.copy() - return s - - -class Executable(_Generative): - """Mark a ClauseElement as supporting execution. - - :class:`.Executable` is a superclass for all "statement" types - of objects, including :func:`select`, :func:`delete`, :func:`update`, - :func:`insert`, :func:`text`. - - """ - - supports_execution = True - _execution_options = util.immutabledict() - _bind = None - - @_generative - def execution_options(self, **kw): - """ Set non-SQL options for the statement which take effect during - execution. - - Execution options can be set on a per-statement or - per :class:`.Connection` basis. Additionally, the - :class:`.Engine` and ORM :class:`~.orm.query.Query` objects provide access - to execution options which they in turn configure upon connections. - - The :meth:`execution_options` method is generative. A new - instance of this statement is returned that contains the options:: - - statement = select([table.c.x, table.c.y]) - statement = statement.execution_options(autocommit=True) - - Note that only a subset of possible execution options can be applied - to a statement - these include "autocommit" and "stream_results", - but not "isolation_level" or "compiled_cache". - See :meth:`.Connection.execution_options` for a full list of - possible options. - - See also: - - :meth:`.Connection.execution_options()` - - :meth:`.Query.execution_options()` - - """ - if 'isolation_level' in kw: - raise exc.ArgumentError( - "'isolation_level' execution option may only be specified " - "on Connection.execution_options(), or " - "per-engine using the isolation_level " - "argument to create_engine()." - ) - if 'compiled_cache' in kw: - raise exc.ArgumentError( - "'compiled_cache' execution option may only be specified " - "on Connection.execution_options(), not per statement." - ) - self._execution_options = self._execution_options.union(kw) - - def execute(self, *multiparams, **params): - """Compile and execute this :class:`.Executable`.""" - - e = self.bind - if e is None: - label = getattr(self, 'description', self.__class__.__name__) - msg = ('This %s is not directly bound to a Connection or Engine.' - 'Use the .execute() method of a Connection or Engine ' - 'to execute this construct.' % label) - raise exc.UnboundExecutionError(msg) - return e._execute_clauseelement(self, multiparams, params) - - def scalar(self, *multiparams, **params): - """Compile and execute this :class:`.Executable`, returning the - result's scalar representation. - - """ - return self.execute(*multiparams, **params).scalar() - - @property - def bind(self): - """Returns the :class:`.Engine` or :class:`.Connection` to - which this :class:`.Executable` is bound, or None if none found. - - This is a traversal which checks locally, then - checks among the "from" clauses of associated objects - until a bound engine or connection is found. - - """ - if self._bind is not None: - return self._bind - - for f in _from_objects(self): - if f is self: - continue - engine = f.bind - if engine is not None: - return engine - else: - return None - - -# legacy, some outside users may be calling this + 'or_', 'outparam', 'outerjoin', 'over', 'select', 'subquery', + 'table', 'text', + 'tuple_', 'type_coerce', 'union', 'union_all', 'update'] + + +from .visitors import Visitable +from .functions import func, modifier, FunctionElement +from ..util.langhelpers import public_factory +from .elements import ClauseElement, ColumnElement,\ + BindParameter, UnaryExpression, BooleanClauseList, \ + Label, Cast, Case, ColumnClause, TextClause, Over, Null, \ + True_, False_, BinaryExpression, Tuple, TypeClause, Extract, \ + Grouping, not_, \ + collate, literal_column, between,\ + literal, outparam, type_coerce, ClauseList + +from .elements import SavepointClause, RollbackToSavepointClause, \ + ReleaseSavepointClause + +from .base import ColumnCollection, Generative, Executable, \ + PARSE_AUTOCOMMIT + +from .selectable import Alias, Join, Select, Selectable, TableClause, \ + CompoundSelect, CTE, FromClause, FromGrouping, SelectBase, \ + alias, GenerativeSelect, \ + subquery, HasPrefixes, Exists, ScalarSelect, TextAsFrom + + +from .dml import Insert, Update, Delete, UpdateBase, ValuesBase + +# factory functions - these pull class-bound constructors and classmethods +# from SQL elements and selectables into public functions. This allows +# the functions to be available in the sqlalchemy.sql.* namespace and +# to be auto-cross-documenting from the function to the class itself. + +and_ = public_factory(BooleanClauseList.and_, ".expression.and_") +or_ = public_factory(BooleanClauseList.or_, ".expression.or_") +bindparam = public_factory(BindParameter, ".expression.bindparam") +select = public_factory(Select, ".expression.select") +text = public_factory(TextClause._create_text, ".expression.text") +table = public_factory(TableClause, ".expression.table") +column = public_factory(ColumnClause, ".expression.column") +over = public_factory(Over, ".expression.over") +label = public_factory(Label, ".expression.label") +case = public_factory(Case, ".expression.case") +cast = public_factory(Cast, ".expression.cast") +extract = public_factory(Extract, ".expression.extract") +tuple_ = public_factory(Tuple, ".expression.tuple_") +except_ = public_factory(CompoundSelect._create_except, ".expression.except_") +except_all = public_factory(CompoundSelect._create_except_all, ".expression.except_all") +intersect = public_factory(CompoundSelect._create_intersect, ".expression.intersect") +intersect_all = public_factory(CompoundSelect._create_intersect_all, ".expression.intersect_all") +union = public_factory(CompoundSelect._create_union, ".expression.union") +union_all = public_factory(CompoundSelect._create_union_all, ".expression.union_all") +exists = public_factory(Exists, ".expression.exists") +nullsfirst = public_factory(UnaryExpression._create_nullsfirst, ".expression.nullsfirst") +nullslast = public_factory(UnaryExpression._create_nullslast, ".expression.nullslast") +asc = public_factory(UnaryExpression._create_asc, ".expression.asc") +desc = public_factory(UnaryExpression._create_desc, ".expression.desc") +distinct = public_factory(UnaryExpression._create_distinct, ".expression.distinct") +true = public_factory(True_._singleton, ".expression.true") +false = public_factory(False_._singleton, ".expression.false") +null = public_factory(Null._singleton, ".expression.null") +join = public_factory(Join._create_join, ".expression.join") +outerjoin = public_factory(Join._create_outerjoin, ".expression.outerjoin") +insert = public_factory(Insert, ".expression.insert") +update = public_factory(Update, ".expression.update") +delete = public_factory(Delete, ".expression.delete") + + +# internal functions still being called from tests and the ORM, +# these might be better off in some other namespace +from .base import _from_objects +from .elements import _literal_as_text, _clause_element_as_expr,\ + _is_column, _labeled, _only_column_elements, _string_or_unprintable, \ + _truncated_label, _clone, _cloned_difference, _cloned_intersection,\ + _column_as_key, _literal_as_binds, _select_iterables, \ + _corresponding_column_or_error +from .selectable import _interpret_as_from + + + +# old names for compatibility _Executable = Executable - -class _TextClause(Executable, ClauseElement): - """Represent a literal SQL text fragment. - - Public constructor is the :func:`text()` function. - - """ - - __visit_name__ = 'textclause' - - _bind_params_regex = re.compile(r'(? RIGHT``.""" - - __visit_name__ = 'binary' - - def __init__(self, left, right, operator, type_=None, - negate=None, modifiers=None): - self.left = _literal_as_text(left).self_group(against=operator) - self.right = _literal_as_text(right).self_group(against=operator) - self.operator = operator - self.type = sqltypes.to_instance(type_) - self.negate = negate - if modifiers is None: - self.modifiers = {} - else: - self.modifiers = modifiers - - def __nonzero__(self): - try: - return self.operator(hash(self.left), hash(self.right)) - except: - raise TypeError("Boolean value of this clause is not defined") - - @property - def _from_objects(self): - return self.left._from_objects + self.right._from_objects - - def _copy_internals(self, clone=_clone, **kw): - self.left = clone(self.left, **kw) - self.right = clone(self.right, **kw) - - def get_children(self, **kwargs): - return self.left, self.right - - def compare(self, other, **kw): - """Compare this :class:`_BinaryExpression` against the - given :class:`_BinaryExpression`.""" - - return ( - isinstance(other, _BinaryExpression) and - self.operator == other.operator and - ( - self.left.compare(other.left, **kw) and - self.right.compare(other.right, **kw) or - ( - operators.is_commutative(self.operator) and - self.left.compare(other.right, **kw) and - self.right.compare(other.left, **kw) - ) - ) - ) - - def self_group(self, against=None): - if operators.is_precedent(self.operator, against): - return _Grouping(self) - else: - return self - - def _negate(self): - if self.negate is not None: - return _BinaryExpression( - self.left, - self.right, - self.negate, - negate=self.operator, - type_=sqltypes.BOOLEANTYPE, - modifiers=self.modifiers) - else: - return super(_BinaryExpression, self)._negate() - -class _Exists(_UnaryExpression): - __visit_name__ = _UnaryExpression.__visit_name__ - _from_objects = [] - - def __init__(self, *args, **kwargs): - if args and isinstance(args[0], (_SelectBase, _ScalarSelect)): - s = args[0] - else: - if not args: - args = ([literal_column('*')],) - s = select(*args, **kwargs).as_scalar().self_group() - - _UnaryExpression.__init__(self, s, operator=operators.exists, - type_=sqltypes.Boolean) - - def select(self, whereclause=None, **params): - return select([self], whereclause, **params) - - def correlate(self, fromclause): - e = self._clone() - e.element = self.element.correlate(fromclause).self_group() - return e - - def select_from(self, clause): - """return a new :class:`._Exists` construct, applying the given expression - to the :meth:`.Select.select_from` method of the select statement - contained. - - """ - e = self._clone() - e.element = self.element.select_from(clause).self_group() - return e - - def where(self, clause): - """return a new exists() construct with the given expression added to - its WHERE clause, joined to the existing clause via AND, if any. - - """ - e = self._clone() - e.element = self.element.where(clause).self_group() - return e - -class Join(FromClause): - """represent a ``JOIN`` construct between two :class:`.FromClause` - elements. - - The public constructor function for :class:`.Join` is the module-level - :func:`join()` function, as well as the :func:`join()` method available - off all :class:`.FromClause` subclasses. - - """ - __visit_name__ = 'join' - - def __init__(self, left, right, onclause=None, isouter=False): - """Construct a new :class:`.Join`. - - The usual entrypoint here is the :func:`~.expression.join` - function or the :meth:`.FromClause.join` method of any - :class:`.FromClause` object. - - """ - self.left = _literal_as_text(left) - self.right = _literal_as_text(right).self_group() - - if onclause is None: - self.onclause = self._match_primaries(self.left, self.right) - else: - self.onclause = onclause - - self.isouter = isouter - self.__folded_equivalents = None - - @property - def description(self): - return "Join object on %s(%d) and %s(%d)" % ( - self.left.description, - id(self.left), - self.right.description, - id(self.right)) - - def is_derived_from(self, fromclause): - return fromclause is self or \ - self.left.is_derived_from(fromclause) or\ - self.right.is_derived_from(fromclause) - - def self_group(self, against=None): - return _FromGrouping(self) - - def _populate_column_collection(self): - columns = [c for c in self.left.columns] + \ - [c for c in self.right.columns] - - self.primary_key.extend(sqlutil.reduce_columns( - (c for c in columns if c.primary_key), self.onclause)) - self._columns.update((col._label, col) for col in columns) - self.foreign_keys.update(itertools.chain( - *[col.foreign_keys for col in columns])) - - def _copy_internals(self, clone=_clone, **kw): - self._reset_exported() - self.left = clone(self.left, **kw) - self.right = clone(self.right, **kw) - self.onclause = clone(self.onclause, **kw) - self.__folded_equivalents = None - - def get_children(self, **kwargs): - return self.left, self.right, self.onclause - - def _match_primaries(self, left, right): - if isinstance(left, Join): - left_right = left.right - else: - left_right = None - return sqlutil.join_condition(left, right, a_subset=left_right) - - def select(self, whereclause=None, fold_equivalents=False, **kwargs): - """Create a :class:`.Select` from this :class:`.Join`. - - The equivalent long-hand form, given a :class:`.Join` object - ``j``, is:: - - from sqlalchemy import select - j = select([j.left, j.right], **kw).\\ - where(whereclause).\\ - select_from(j) - - :param whereclause: the WHERE criterion that will be sent to - the :func:`select()` function - - :param fold_equivalents: based on the join criterion of this - :class:`.Join`, do not include - repeat column names in the column list of the resulting - select, for columns that are calculated to be "equivalent" - based on the join criterion of this :class:`.Join`. This will - recursively apply to any joins directly nested by this one - as well. - - :param \**kwargs: all other kwargs are sent to the - underlying :func:`select()` function. - - """ - if fold_equivalents: - collist = sqlutil.folded_equivalents(self) - else: - collist = [self.left, self.right] - - return select(collist, whereclause, from_obj=[self], **kwargs) - - @property - def bind(self): - return self.left.bind or self.right.bind - - def alias(self, name=None): - """return an alias of this :class:`.Join`. - - Used against a :class:`.Join` object, - :meth:`~.Join.alias` calls the :meth:`~.Join.select` - method first so that a subquery against a - :func:`.select` construct is generated. - the :func:`~expression.select` construct also has the - ``correlate`` flag set to ``False`` and will not - auto-correlate inside an enclosing :func:`~expression.select` - construct. - - The equivalent long-hand form, given a :class:`.Join` object - ``j``, is:: - - from sqlalchemy import select, alias - j = alias( - select([j.left, j.right]).\\ - select_from(j).\\ - with_labels(True).\\ - correlate(False), - name=name - ) - - See :func:`~.expression.alias` for further details on - aliases. - - """ - return self.select(use_labels=True, correlate=False).alias(name) - - @property - def _hide_froms(self): - return itertools.chain(*[_from_objects(x.left, x.right) - for x in self._cloned_set]) - - @property - def _from_objects(self): - return [self] + \ - self.onclause._from_objects + \ - self.left._from_objects + \ - self.right._from_objects - -class Alias(FromClause): - """Represents an table or selectable alias (AS). - - Represents an alias, as typically applied to any table or - sub-select within a SQL statement using the ``AS`` keyword (or - without the keyword on certain databases such as Oracle). - - This object is constructed from the :func:`~.expression.alias` module level - function as well as the :meth:`.FromClause.alias` method available on all - :class:`.FromClause` subclasses. - - """ - - __visit_name__ = 'alias' - named_with_column = True - - def __init__(self, selectable, name=None): - baseselectable = selectable - while isinstance(baseselectable, Alias): - baseselectable = baseselectable.element - self.original = baseselectable - self.supports_execution = baseselectable.supports_execution - if self.supports_execution: - self._execution_options = baseselectable._execution_options - self.element = selectable - if name is None: - if self.original.named_with_column: - name = getattr(self.original, 'name', None) - name = _anonymous_label('%%(%d %s)s' % (id(self), name - or 'anon')) - self.name = name - - @property - def description(self): - # Py3K - #return self.name - # Py2K - return self.name.encode('ascii', 'backslashreplace') - # end Py2K - - def as_scalar(self): - try: - return self.element.as_scalar() - except AttributeError: - raise AttributeError("Element %s does not support " - "'as_scalar()'" % self.element) - - def is_derived_from(self, fromclause): - if fromclause in self._cloned_set: - return True - return self.element.is_derived_from(fromclause) - - def _populate_column_collection(self): - for col in self.element.columns: - col._make_proxy(self) - - def _copy_internals(self, clone=_clone, **kw): - # don't apply anything to an aliased Table - # for now. May want to drive this from - # the given **kw. - if isinstance(self.element, TableClause): - return - self._reset_exported() - self.element = clone(self.element, **kw) - baseselectable = self.element - while isinstance(baseselectable, Alias): - baseselectable = baseselectable.element - self.original = baseselectable - - def get_children(self, column_collections=True, **kw): - if column_collections: - for c in self.c: - yield c - yield self.element - - @property - def _from_objects(self): - return [self] - - @property - def bind(self): - return self.element.bind - -class CTE(Alias): - """Represent a Common Table Expression. - - The :class:`.CTE` object is obtained using the - :meth:`._SelectBase.cte` method from any selectable. - See that method for complete examples. - - .. versionadded:: 0.7.6 - - """ - __visit_name__ = 'cte' - - def __init__(self, selectable, - name=None, - recursive=False, - cte_alias=False, - _restates=frozenset()): - self.recursive = recursive - self.cte_alias = cte_alias - self._restates = _restates - super(CTE, self).__init__(selectable, name=name) - - def alias(self, name=None): - return CTE( - self.original, - name=name, - recursive=self.recursive, - cte_alias = self.name - ) - - def union(self, other): - return CTE( - self.original.union(other), - name=self.name, - recursive=self.recursive, - _restates=self._restates.union([self]) - ) - - def union_all(self, other): - return CTE( - self.original.union_all(other), - name=self.name, - recursive=self.recursive, - _restates=self._restates.union([self]) - ) - - -class _Grouping(ColumnElement): - """Represent a grouping within a column expression""" - - __visit_name__ = 'grouping' - - def __init__(self, element): - self.element = element - self.type = getattr(element, 'type', None) - - @property - def _label(self): - return getattr(self.element, '_label', None) or self.anon_label - - def _copy_internals(self, clone=_clone, **kw): - self.element = clone(self.element, **kw) - - def get_children(self, **kwargs): - return self.element, - - @property - def _from_objects(self): - return self.element._from_objects - - def __getattr__(self, attr): - return getattr(self.element, attr) - - def __getstate__(self): - return {'element':self.element, 'type':self.type} - - def __setstate__(self, state): - self.element = state['element'] - self.type = state['type'] - -class _FromGrouping(FromClause): - """Represent a grouping of a FROM clause""" - __visit_name__ = 'grouping' - - def __init__(self, element): - self.element = element - - def _init_collections(self): - pass - - @property - def columns(self): - return self.element.columns - - @property - def primary_key(self): - return self.element.primary_key - - @property - def foreign_keys(self): - # this could be - # self.element.foreign_keys - # see SelectableTest.test_join_condition - return set() - - @property - def _hide_froms(self): - return self.element._hide_froms - - def get_children(self, **kwargs): - return self.element, - - def _copy_internals(self, clone=_clone, **kw): - self.element = clone(self.element, **kw) - - @property - def _from_objects(self): - return self.element._from_objects - - def __getattr__(self, attr): - return getattr(self.element, attr) - - def __getstate__(self): - return {'element':self.element} - - def __setstate__(self, state): - self.element = state['element'] - -class _Over(ColumnElement): - """Represent an OVER clause. - - This is a special operator against a so-called - "window" function, as well as any aggregate function, - which produces results relative to the result set - itself. It's supported only by certain database - backends. - - """ - __visit_name__ = 'over' - - order_by = None - partition_by = None - - def __init__(self, func, partition_by=None, order_by=None): - self.func = func - if order_by is not None: - self.order_by = ClauseList(*util.to_list(order_by)) - if partition_by is not None: - self.partition_by = ClauseList(*util.to_list(partition_by)) - - @util.memoized_property - def type(self): - return self.func.type - - def get_children(self, **kwargs): - return [c for c in - (self.func, self.partition_by, self.order_by) - if c is not None] - - def _copy_internals(self, clone=_clone, **kw): - self.func = clone(self.func, **kw) - if self.partition_by is not None: - self.partition_by = clone(self.partition_by, **kw) - if self.order_by is not None: - self.order_by = clone(self.order_by, **kw) - - @property - def _from_objects(self): - return list(itertools.chain( - *[c._from_objects for c in - (self.func, self.partition_by, self.order_by) - if c is not None] - )) - -class _Label(ColumnElement): - """Represents a column label (AS). - - Represent a label, as typically applied to any column-level - element using the ``AS`` sql keyword. - - This object is constructed from the :func:`label()` module level - function as well as the :func:`label()` method available on all - :class:`.ColumnElement` subclasses. - - """ - - __visit_name__ = 'label' - - def __init__(self, name, element, type_=None): - while isinstance(element, _Label): - element = element.element - if name: - self.name = name - else: - self.name = _anonymous_label('%%(%d %s)s' % (id(self), - getattr(element, 'name', 'anon'))) - self.key = self._label = self._key_label = self.name - self._element = element - self._type = type_ - self.quote = element.quote - self.proxies = [element] - - @util.memoized_property - def type(self): - return sqltypes.to_instance( - self._type or getattr(self._element, 'type', None) - ) - - @util.memoized_property - def element(self): - return self._element.self_group(against=operators.as_) - - def self_group(self, against=None): - sub_element = self._element.self_group(against=against) - if sub_element is not self._element: - return _Label(self.name, - sub_element, - type_=self._type) - else: - return self - - @property - def primary_key(self): - return self.element.primary_key - - @property - def foreign_keys(self): - return self.element.foreign_keys - - def get_children(self, **kwargs): - return self.element, - - def _copy_internals(self, clone=_clone, **kw): - self.element = clone(self.element, **kw) - - @property - def _from_objects(self): - return self.element._from_objects - - def _make_proxy(self, selectable, name = None): - e = self.element._make_proxy(selectable, name=name or self.name) - e.proxies.append(self) - return e - -class ColumnClause(_Immutable, ColumnElement): - """Represents a generic column expression from any textual string. - - This includes columns associated with tables, aliases and select - statements, but also any arbitrary text. May or may not be bound - to an underlying :class:`.Selectable`. - - :class:`.ColumnClause` is constructed by itself typically via - the :func:`~.expression.column` function. It may be placed directly - into constructs such as :func:`.select` constructs:: - - from sqlalchemy.sql import column, select - - c1, c2 = column("c1"), column("c2") - s = select([c1, c2]).where(c1==5) - - There is also a variant on :func:`~.expression.column` known - as :func:`~.expression.literal_column` - the difference is that - in the latter case, the string value is assumed to be an exact - expression, rather than a column name, so that no quoting rules - or similar are applied:: - - from sqlalchemy.sql import literal_column, select - - s = select([literal_column("5 + 7")]) - - :class:`.ColumnClause` can also be used in a table-like - fashion by combining the :func:`~.expression.column` function - with the :func:`~.expression.table` function, to produce - a "lightweight" form of table metadata:: - - from sqlalchemy.sql import table, column - - user = table("user", - column("id"), - column("name"), - column("description"), - ) - - The above construct can be created in an ad-hoc fashion and is - not associated with any :class:`.schema.MetaData`, unlike it's - more full fledged :class:`.schema.Table` counterpart. - - :param text: the text of the element. - - :param selectable: parent selectable. - - :param type: :class:`.types.TypeEngine` object which can associate - this :class:`.ColumnClause` with a type. - - :param is_literal: if True, the :class:`.ColumnClause` is assumed to - be an exact expression that will be delivered to the output with no - quoting rules applied regardless of case sensitive settings. the - :func:`literal_column()` function is usually used to create such a - :class:`.ColumnClause`. - - """ - __visit_name__ = 'column' - - onupdate = default = server_default = server_onupdate = None - - _memoized_property = util.group_expirable_memoized_property() - - def __init__(self, text, selectable=None, type_=None, is_literal=False): - self.key = self.name = text - self.table = selectable - self.type = sqltypes.to_instance(type_) - self.is_literal = is_literal - - def _compare_name_for_result(self, other): - if self.is_literal or \ - self.table is None or \ - not hasattr(other, 'proxy_set') or ( - isinstance(other, ColumnClause) and other.is_literal - ): - return super(ColumnClause, self).\ - _compare_name_for_result(other) - else: - return other.proxy_set.intersection(self.proxy_set) - - def _get_table(self): - return self.__dict__['table'] - def _set_table(self, table): - self._memoized_property.expire_instance(self) - self.__dict__['table'] = table - table = property(_get_table, _set_table) - - @_memoized_property - def _from_objects(self): - t = self.table - if t is not None: - return [t] - else: - return [] - - @util.memoized_property - def description(self): - # Py3K - #return self.name - # Py2K - return self.name.encode('ascii', 'backslashreplace') - # end Py2K - - @_memoized_property - def _key_label(self): - if self.key != self.name: - return self._gen_label(self.key) - else: - return self._label - - @_memoized_property - def _label(self): - return self._gen_label(self.name) - - def _gen_label(self, name): - t = self.table - if self.is_literal: - return None - - elif t is not None and t.named_with_column: - if getattr(t, 'schema', None): - label = t.schema.replace('.', '_') + "_" + \ - t.name + "_" + name - else: - label = t.name + "_" + name - - # ensure the label name doesn't conflict with that - # of an existing column - if label in t.c: - _label = label - counter = 1 - while _label in t.c: - _label = label + "_" + str(counter) - counter += 1 - label = _label - - return _as_truncated(label) - - else: - return name - - def label(self, name): - # currently, anonymous labels don't occur for - # ColumnClause. The use at the moment - # is that they do not generate nicely for - # is_literal clauses. We would like to change - # this so that label(None) acts as would be expected. - # See [ticket:2168]. - if name is None: - return self - else: - return super(ColumnClause, self).label(name) - - - def _bind_param(self, operator, obj): - return _BindParamClause(self.name, obj, - _compared_to_operator=operator, - _compared_to_type=self.type, - unique=True) - - def _make_proxy(self, selectable, name=None, attach=True): - # propagate the "is_literal" flag only if we are keeping our name, - # otherwise its considered to be a label - is_literal = self.is_literal and (name is None or name == self.name) - c = self._constructor( - _as_truncated(name or self.name), - selectable=selectable, - type_=self.type, - is_literal=is_literal - ) - c.proxies = [self] - if selectable._is_clone_of is not None: - c._is_clone_of = \ - selectable._is_clone_of.columns.get(c.name) - - if attach: - selectable._columns[c.name] = c - return c - -class TableClause(_Immutable, FromClause): - """Represents a minimal "table" construct. - - The constructor for :class:`.TableClause` is the - :func:`~.expression.table` function. This produces - a lightweight table object that has only a name and a - collection of columns, which are typically produced - by the :func:`~.expression.column` function:: - - from sqlalchemy.sql import table, column - - user = table("user", - column("id"), - column("name"), - column("description"), - ) - - The :class:`.TableClause` construct serves as the base for - the more commonly used :class:`~.schema.Table` object, providing - the usual set of :class:`~.expression.FromClause` services including - the ``.c.`` collection and statement generation methods. - - It does **not** provide all the additional schema-level services - of :class:`~.schema.Table`, including constraints, references to other - tables, or support for :class:`.MetaData`-level services. It's useful - on its own as an ad-hoc construct used to generate quick SQL - statements when a more fully fledged :class:`~.schema.Table` is not on hand. - - """ - - __visit_name__ = 'table' - - named_with_column = True - - def __init__(self, name, *columns): - super(TableClause, self).__init__() - self.name = self.fullname = name - self._columns = ColumnCollection() - self.primary_key = ColumnSet() - self.foreign_keys = set() - for c in columns: - self.append_column(c) - - def _init_collections(self): - pass - - @util.memoized_property - def description(self): - # Py3K - #return self.name - # Py2K - return self.name.encode('ascii', 'backslashreplace') - # end Py2K - - def append_column(self, c): - self._columns[c.name] = c - c.table = self - - def get_children(self, column_collections=True, **kwargs): - if column_collections: - return [c for c in self.c] - else: - return [] - - def count(self, whereclause=None, **params): - """return a SELECT COUNT generated against this - :class:`.TableClause`.""" - - if self.primary_key: - col = list(self.primary_key)[0] - else: - col = list(self.columns)[0] - return select( - [func.count(col).label('tbl_row_count')], - whereclause, - from_obj=[self], - **params) - - def insert(self, values=None, inline=False, **kwargs): - """Generate an :func:`.insert` construct against this - :class:`.TableClause`. - - E.g.:: - - table.insert().values(name='foo') - - See :func:`.insert` for argument and usage information. - - """ - - return insert(self, values=values, inline=inline, **kwargs) - - def update(self, whereclause=None, values=None, inline=False, **kwargs): - """Generate an :func:`.update` construct against this - :class:`.TableClause`. - - E.g.:: - - table.update().where(table.c.id==7).values(name='foo') - - See :func:`.update` for argument and usage information. - - """ - - return update(self, whereclause=whereclause, - values=values, inline=inline, **kwargs) - - def delete(self, whereclause=None, **kwargs): - """Generate a :func:`.delete` construct against this - :class:`.TableClause`. - - E.g.:: - - table.delete().where(table.c.id==7) - - See :func:`.delete` for argument and usage information. - - """ - - return delete(self, whereclause, **kwargs) - - @property - def _from_objects(self): - return [self] - -class _SelectBase(Executable, FromClause): - """Base class for :class:`.Select` and ``CompoundSelects``.""" - - _order_by_clause = ClauseList() - _group_by_clause = ClauseList() - _limit = None - _offset = None - - def __init__(self, - use_labels=False, - for_update=False, - limit=None, - offset=None, - order_by=None, - group_by=None, - bind=None, - autocommit=None): - self.use_labels = use_labels - self.for_update = for_update - if autocommit is not None: - util.warn_deprecated('autocommit on select() is ' - 'deprecated. Use .execution_options(a' - 'utocommit=True)') - self._execution_options = \ - self._execution_options.union({'autocommit' - : autocommit}) - if limit is not None: - self._limit = util.asint(limit) - if offset is not None: - self._offset = util.asint(offset) - self._bind = bind - - if order_by is not None: - self._order_by_clause = ClauseList(*util.to_list(order_by)) - if group_by is not None: - self._group_by_clause = ClauseList(*util.to_list(group_by)) - - def as_scalar(self): - """return a 'scalar' representation of this selectable, which can be - used as a column expression. - - Typically, a select statement which has only one column in its columns - clause is eligible to be used as a scalar expression. - - The returned object is an instance of - :class:`_ScalarSelect`. - - """ - return _ScalarSelect(self) - - @_generative - def apply_labels(self): - """return a new selectable with the 'use_labels' flag set to True. - - This will result in column expressions being generated using labels - against their table name, such as "SELECT somecolumn AS - tablename_somecolumn". This allows selectables which contain multiple - FROM clauses to produce a unique set of column names regardless of - name conflicts among the individual FROM clauses. - - """ - self.use_labels = True - - def label(self, name): - """return a 'scalar' representation of this selectable, embedded as a - subquery with a label. - - See also :meth:`~._SelectBase.as_scalar`. - - """ - return self.as_scalar().label(name) - - def cte(self, name=None, recursive=False): - """Return a new :class:`.CTE`, or Common Table Expression instance. - - Common table expressions are a SQL standard whereby SELECT - statements can draw upon secondary statements specified along - with the primary statement, using a clause called "WITH". - Special semantics regarding UNION can also be employed to - allow "recursive" queries, where a SELECT statement can draw - upon the set of rows that have previously been selected. - - SQLAlchemy detects :class:`.CTE` objects, which are treated - similarly to :class:`.Alias` objects, as special elements - to be delivered to the FROM clause of the statement as well - as to a WITH clause at the top of the statement. - - .. versionadded:: 0.7.6 - - :param name: name given to the common table expression. Like - :meth:`._FromClause.alias`, the name can be left as ``None`` - in which case an anonymous symbol will be used at query - compile time. - :param recursive: if ``True``, will render ``WITH RECURSIVE``. - A recursive common table expression is intended to be used in - conjunction with UNION ALL in order to derive rows - from those already selected. - - The following examples illustrate two examples from - Postgresql's documentation at - http://www.postgresql.org/docs/8.4/static/queries-with.html. - - Example 1, non recursive:: - - from sqlalchemy import Table, Column, String, Integer, MetaData, \\ - select, func - - metadata = MetaData() - - orders = Table('orders', metadata, - Column('region', String), - Column('amount', Integer), - Column('product', String), - Column('quantity', Integer) - ) - - regional_sales = select([ - orders.c.region, - func.sum(orders.c.amount).label('total_sales') - ]).group_by(orders.c.region).cte("regional_sales") - - - top_regions = select([regional_sales.c.region]).\\ - where( - regional_sales.c.total_sales > - select([ - func.sum(regional_sales.c.total_sales)/10 - ]) - ).cte("top_regions") - - statement = select([ - orders.c.region, - orders.c.product, - func.sum(orders.c.quantity).label("product_units"), - func.sum(orders.c.amount).label("product_sales") - ]).where(orders.c.region.in_( - select([top_regions.c.region]) - )).group_by(orders.c.region, orders.c.product) - - result = conn.execute(statement).fetchall() - - Example 2, WITH RECURSIVE:: - - from sqlalchemy import Table, Column, String, Integer, MetaData, \\ - select, func - - metadata = MetaData() - - parts = Table('parts', metadata, - Column('part', String), - Column('sub_part', String), - Column('quantity', Integer), - ) - - included_parts = select([ - parts.c.sub_part, - parts.c.part, - parts.c.quantity]).\\ - where(parts.c.part=='our part').\\ - cte(recursive=True) - - - incl_alias = included_parts.alias() - parts_alias = parts.alias() - included_parts = included_parts.union_all( - select([ - parts_alias.c.part, - parts_alias.c.sub_part, - parts_alias.c.quantity - ]). - where(parts_alias.c.part==incl_alias.c.sub_part) - ) - - statement = select([ - included_parts.c.sub_part, - func.sum(included_parts.c.quantity).label('total_quantity') - ]).\ - select_from(included_parts.join(parts, - included_parts.c.part==parts.c.part)).\\ - group_by(included_parts.c.sub_part) - - result = conn.execute(statement).fetchall() - - - See also: - - :meth:`.orm.query.Query.cte` - ORM version of :meth:`._SelectBase.cte`. - - """ - return CTE(self, name=name, recursive=recursive) - - @_generative - @util.deprecated('0.6', - message=":func:`.autocommit` is deprecated. Use " - ":func:`.Executable.execution_options` with the " - "'autocommit' flag.") - def autocommit(self): - """return a new selectable with the 'autocommit' flag set to - True.""" - - self._execution_options = \ - self._execution_options.union({'autocommit': True}) - - def _generate(self): - """Override the default _generate() method to also clear out - exported collections.""" - - s = self.__class__.__new__(self.__class__) - s.__dict__ = self.__dict__.copy() - s._reset_exported() - return s - - @_generative - def limit(self, limit): - """return a new selectable with the given LIMIT criterion - applied.""" - - self._limit = util.asint(limit) - - @_generative - def offset(self, offset): - """return a new selectable with the given OFFSET criterion - applied.""" - - self._offset = util.asint(offset) - - @_generative - def order_by(self, *clauses): - """return a new selectable with the given list of ORDER BY - criterion applied. - - The criterion will be appended to any pre-existing ORDER BY - criterion. - - """ - - self.append_order_by(*clauses) - - @_generative - def group_by(self, *clauses): - """return a new selectable with the given list of GROUP BY - criterion applied. - - The criterion will be appended to any pre-existing GROUP BY - criterion. - - """ - - self.append_group_by(*clauses) - - def append_order_by(self, *clauses): - """Append the given ORDER BY criterion applied to this selectable. - - The criterion will be appended to any pre-existing ORDER BY criterion. - - """ - if len(clauses) == 1 and clauses[0] is None: - self._order_by_clause = ClauseList() - else: - if getattr(self, '_order_by_clause', None) is not None: - clauses = list(self._order_by_clause) + list(clauses) - self._order_by_clause = ClauseList(*clauses) - - def append_group_by(self, *clauses): - """Append the given GROUP BY criterion applied to this selectable. - - The criterion will be appended to any pre-existing GROUP BY criterion. - - """ - if len(clauses) == 1 and clauses[0] is None: - self._group_by_clause = ClauseList() - else: - if getattr(self, '_group_by_clause', None) is not None: - clauses = list(self._group_by_clause) + list(clauses) - self._group_by_clause = ClauseList(*clauses) - - @property - def _from_objects(self): - return [self] - - -class _ScalarSelect(_Grouping): - _from_objects = [] - - def __init__(self, element): - self.element = element - self.type = element._scalar_type() - - @property - def columns(self): - raise exc.InvalidRequestError('Scalar Select expression has no ' - 'columns; use this object directly within a ' - 'column-level expression.') - c = columns - - def self_group(self, **kwargs): - return self - - def _make_proxy(self, selectable, name): - return list(self.inner_columns)[0]._make_proxy(selectable, name) - -class CompoundSelect(_SelectBase): - """Forms the basis of ``UNION``, ``UNION ALL``, and other - SELECT-based set operations.""" - - __visit_name__ = 'compound_select' - - UNION = util.symbol('UNION') - UNION_ALL = util.symbol('UNION ALL') - EXCEPT = util.symbol('EXCEPT') - EXCEPT_ALL = util.symbol('EXCEPT ALL') - INTERSECT = util.symbol('INTERSECT') - INTERSECT_ALL = util.symbol('INTERSECT ALL') - - def __init__(self, keyword, *selects, **kwargs): - self._should_correlate = kwargs.pop('correlate', False) - self.keyword = keyword - self.selects = [] - - numcols = None - - # some DBs do not like ORDER BY in the inner queries of a UNION, etc. - for n, s in enumerate(selects): - s = _clause_element_as_expr(s) - - if not numcols: - numcols = len(s.c) - elif len(s.c) != numcols: - raise exc.ArgumentError('All selectables passed to ' - 'CompoundSelect must have identical numbers of ' - 'columns; select #%d has %d columns, select ' - '#%d has %d' % (1, len(self.selects[0].c), n - + 1, len(s.c))) - - self.selects.append(s.self_group(self)) - - _SelectBase.__init__(self, **kwargs) - - def _scalar_type(self): - return self.selects[0]._scalar_type() - - def self_group(self, against=None): - return _FromGrouping(self) - - def is_derived_from(self, fromclause): - for s in self.selects: - if s.is_derived_from(fromclause): - return True - return False - - def _populate_column_collection(self): - for cols in zip(*[s.c for s in self.selects]): - - # this is a slightly hacky thing - the union exports a - # column that resembles just that of the *first* selectable. - # to get at a "composite" column, particularly foreign keys, - # you have to dig through the proxies collection which we - # generate below. We may want to improve upon this, such as - # perhaps _make_proxy can accept a list of other columns - # that are "shared" - schema.column can then copy all the - # ForeignKeys in. this would allow the union() to have all - # those fks too. - - proxy = cols[0]._make_proxy(self, name=self.use_labels - and cols[0]._label or None) - - # hand-construct the "proxies" collection to include all - # derived columns place a 'weight' annotation corresponding - # to how low in the list of select()s the column occurs, so - # that the corresponding_column() operation can resolve - # conflicts - - proxy.proxies = [c._annotate({'weight': i + 1}) for (i, - c) in enumerate(cols)] - - def _copy_internals(self, clone=_clone, **kw): - self._reset_exported() - self.selects = [clone(s, **kw) for s in self.selects] - if hasattr(self, '_col_map'): - del self._col_map - for attr in ('_order_by_clause', '_group_by_clause'): - if getattr(self, attr) is not None: - setattr(self, attr, clone(getattr(self, attr), **kw)) - - def get_children(self, column_collections=True, **kwargs): - return (column_collections and list(self.c) or []) \ - + [self._order_by_clause, self._group_by_clause] \ - + list(self.selects) - - def bind(self): - if self._bind: - return self._bind - for s in self.selects: - e = s.bind - if e: - return e - else: - return None - def _set_bind(self, bind): - self._bind = bind - bind = property(bind, _set_bind) - -class Select(_SelectBase): - """Represents a ``SELECT`` statement. - - See also: - - :func:`~.expression.select` - the function which creates a :class:`.Select` object. - - :ref:`coretutorial_selecting` - Core Tutorial description of :func:`.select`. - - """ - - __visit_name__ = 'select' - - _prefixes = () - _hints = util.immutabledict() - _distinct = False - _from_cloned = None - - _memoized_property = _SelectBase._memoized_property - - def __init__(self, - columns, - whereclause=None, - from_obj=None, - distinct=False, - having=None, - correlate=True, - prefixes=None, - **kwargs): - """Construct a Select object. - - The public constructor for Select is the - :func:`select` function; see that function for - argument descriptions. - - Additional generative and mutator methods are available on the - :class:`_SelectBase` superclass. - - """ - self._should_correlate = correlate - if distinct is not False: - if isinstance(distinct, basestring): - util.warn_deprecated( - "A string argument passed to the 'distinct' " - "keyword argument of 'select()' is deprecated " - "- please use 'prefixes' or 'prefix_with()' " - "to specify additional prefixes") - if prefixes: - prefixes = util.to_list(prefixes) + [distinct] - else: - prefixes = [distinct] - elif distinct is True: - self._distinct = True - else: - self._distinct = [ - _literal_as_text(e) - for e in util.to_list(distinct) - ] - - self._correlate = set() - if from_obj is not None: - self._from_obj = util.OrderedSet( - _literal_as_text(f) - for f in util.to_list(from_obj)) - else: - self._from_obj = util.OrderedSet() - - try: - cols_present = bool(columns) - except TypeError: - raise exc.ArgumentError("columns argument to select() must " - "be a Python list or other iterable") - - if cols_present: - self._raw_columns = [] - for c in columns: - c = _literal_as_column(c) - if isinstance(c, _ScalarSelect): - c = c.self_group(against=operators.comma_op) - self._raw_columns.append(c) - else: - self._raw_columns = [] - - if whereclause is not None: - self._whereclause = _literal_as_text(whereclause) - else: - self._whereclause = None - - if having is not None: - self._having = _literal_as_text(having) - else: - self._having = None - - if prefixes: - self._prefixes = tuple([_literal_as_text(p) for p in prefixes]) - - _SelectBase.__init__(self, **kwargs) - - @property - def _froms(self): - # would love to cache this, - # but there's just enough edge cases, particularly now that - # declarative encourages construction of SQL expressions - # without tables present, to just regen this each time. - froms = [] - seen = set() - translate = self._from_cloned - def add(items): - for item in items: - if translate and item in translate: - item = translate[item] - if not seen.intersection(item._cloned_set): - froms.append(item) - seen.update(item._cloned_set) - - add(_from_objects(*self._raw_columns)) - if self._whereclause is not None: - add(_from_objects(self._whereclause)) - add(self._from_obj) - - return froms - - def _get_display_froms(self, existing_froms=None): - """Return the full list of 'from' clauses to be displayed. - - Takes into account a set of existing froms which may be - rendered in the FROM clause of enclosing selects; this Select - may want to leave those absent if it is automatically - correlating. - - """ - froms = self._froms - - toremove = set(itertools.chain(*[f._hide_froms for f in froms])) - if toremove: - # if we're maintaining clones of froms, - # add the copies out to the toremove list. only include - # clones that are lexical equivalents. - if self._from_cloned: - toremove.update( - self._from_cloned[f] for f in - toremove.intersection(self._from_cloned) - if self._from_cloned[f]._is_lexical_equivalent(f) - ) - # filter out to FROM clauses not in the list, - # using a list to maintain ordering - froms = [f for f in froms if f not in toremove] - - if len(froms) > 1 or self._correlate: - if self._correlate: - froms = [f for f in froms if f not in _cloned_intersection(froms, - self._correlate)] - if self._should_correlate and existing_froms: - froms = [f for f in froms if f not in _cloned_intersection(froms, - existing_froms)] - - if not len(froms): - raise exc.InvalidRequestError("Select statement '%s" - "' returned no FROM clauses due to " - "auto-correlation; specify " - "correlate() to control " - "correlation manually." % self) - - return froms - - def _scalar_type(self): - elem = self._raw_columns[0] - cols = list(elem._select_iterable) - return cols[0].type - - @property - def froms(self): - """Return the displayed list of FromClause elements.""" - - return self._get_display_froms() - - @_generative - def with_hint(self, selectable, text, dialect_name='*'): - """Add an indexing hint for the given selectable to this - :class:`.Select`. - - The text of the hint is rendered in the appropriate - location for the database backend in use, relative - to the given :class:`.Table` or :class:`.Alias` passed as the - ``selectable`` argument. The dialect implementation - typically uses Python string substitution syntax - with the token ``%(name)s`` to render the name of - the table or alias. E.g. when using Oracle, the - following:: - - select([mytable]).\\ - with_hint(mytable, "+ index(%(name)s ix_mytable)") - - Would render SQL as:: - - select /*+ index(mytable ix_mytable) */ ... from mytable - - The ``dialect_name`` option will limit the rendering of a particular - hint to a particular backend. Such as, to add hints for both Oracle - and Sybase simultaneously:: - - select([mytable]).\\ - with_hint(mytable, "+ index(%(name)s ix_mytable)", 'oracle').\\ - with_hint(mytable, "WITH INDEX ix_mytable", 'sybase') - - """ - self._hints = self._hints.union({(selectable, dialect_name):text}) - - @property - def type(self): - raise exc.InvalidRequestError("Select objects don't have a type. " - "Call as_scalar() on this Select object " - "to return a 'scalar' version of this Select.") - - @_memoized_property.method - def locate_all_froms(self): - """return a Set of all FromClause elements referenced by this Select. - - This set is a superset of that returned by the ``froms`` property, - which is specifically for those FromClause elements that would - actually be rendered. - - """ - froms = self._froms - return froms + list(_from_objects(*froms)) - - @property - def inner_columns(self): - """an iterator of all ColumnElement expressions which would - be rendered into the columns clause of the resulting SELECT statement. - - """ - return _select_iterables(self._raw_columns) - - def is_derived_from(self, fromclause): - if self in fromclause._cloned_set: - return True - - for f in self.locate_all_froms(): - if f.is_derived_from(fromclause): - return True - return False - - def _copy_internals(self, clone=_clone, **kw): - - # Select() object has been cloned and probably adapted by the - # given clone function. Apply the cloning function to internal - # objects - - # 1. keep a dictionary of the froms we've cloned, and what - # they've become. This is consulted later when we derive - # additional froms from "whereclause" and the columns clause, - # which may still reference the uncloned parent table. - # as of 0.7.4 we also put the current version of _froms, which - # gets cleared on each generation. previously we were "baking" - # _froms into self._from_obj. - self._from_cloned = from_cloned = dict((f, clone(f, **kw)) - for f in self._from_obj.union(self._froms)) - - # 3. update persistent _from_obj with the cloned versions. - self._from_obj = util.OrderedSet(from_cloned[f] for f in - self._from_obj) - - # the _correlate collection is done separately, what can happen - # here is the same item is _correlate as in _from_obj but the - # _correlate version has an annotation on it - (specifically - # RelationshipProperty.Comparator._criterion_exists() does - # this). Also keep _correlate liberally open with it's previous - # contents, as this set is used for matching, not rendering. - self._correlate = set(clone(f) for f in - self._correlate).union(self._correlate) - - # 4. clone other things. The difficulty here is that Column - # objects are not actually cloned, and refer to their original - # .table, resulting in the wrong "from" parent after a clone - # operation. Hence _from_cloned and _from_obj supercede what is - # present here. - self._raw_columns = [clone(c, **kw) for c in self._raw_columns] - for attr in '_whereclause', '_having', '_order_by_clause', \ - '_group_by_clause': - if getattr(self, attr) is not None: - setattr(self, attr, clone(getattr(self, attr), **kw)) - - # erase exported column list, _froms collection, - # etc. - self._reset_exported() - - def get_children(self, column_collections=True, **kwargs): - """return child elements as per the ClauseElement specification.""" - - return (column_collections and list(self.columns) or []) + \ - self._raw_columns + list(self._froms) + \ - [x for x in - (self._whereclause, self._having, - self._order_by_clause, self._group_by_clause) - if x is not None] - - @_generative - def column(self, column): - """return a new select() construct with the given column expression - added to its columns clause. - - """ - self.append_column(column) - - @_generative - def with_only_columns(self, columns): - """Return a new :func:`.select` construct with its columns - clause replaced with the given columns. - - .. versionchanged:: 0.7.3 - Due to a bug fix, this method has a slight - behavioral change as of version 0.7.3. - Prior to version 0.7.3, the FROM clause of - a :func:`.select` was calculated upfront and as new columns - were added; in 0.7.3 and later it's calculated - at compile time, fixing an issue regarding late binding - of columns to parent tables. This changes the behavior of - :meth:`.Select.with_only_columns` in that FROM clauses no - longer represented in the new list are dropped, - but this behavior is more consistent in - that the FROM clauses are consistently derived from the - current columns clause. The original intent of this method - is to allow trimming of the existing columns list to be fewer - columns than originally present; the use case of replacing - the columns list with an entirely different one hadn't - been anticipated until 0.7.3 was released; the usage - guidelines below illustrate how this should be done. - - This method is exactly equivalent to as if the original - :func:`.select` had been called with the given columns - clause. I.e. a statement:: - - s = select([table1.c.a, table1.c.b]) - s = s.with_only_columns([table1.c.b]) - - should be exactly equivalent to:: - - s = select([table1.c.b]) - - This means that FROM clauses which are only derived - from the column list will be discarded if the new column - list no longer contains that FROM:: - - >>> table1 = table('t1', column('a'), column('b')) - >>> table2 = table('t2', column('a'), column('b')) - >>> s1 = select([table1.c.a, table2.c.b]) - >>> print s1 - SELECT t1.a, t2.b FROM t1, t2 - >>> s2 = s1.with_only_columns([table2.c.b]) - >>> print s2 - SELECT t2.b FROM t1 - - The preferred way to maintain a specific FROM clause - in the construct, assuming it won't be represented anywhere - else (i.e. not in the WHERE clause, etc.) is to set it using - :meth:`.Select.select_from`:: - - >>> s1 = select([table1.c.a, table2.c.b]).\\ - ... select_from(table1.join(table2, table1.c.a==table2.c.a)) - >>> s2 = s1.with_only_columns([table2.c.b]) - >>> print s2 - SELECT t2.b FROM t1 JOIN t2 ON t1.a=t2.a - - Care should also be taken to use the correct - set of column objects passed to :meth:`.Select.with_only_columns`. - Since the method is essentially equivalent to calling the - :func:`.select` construct in the first place with the given - columns, the columns passed to :meth:`.Select.with_only_columns` - should usually be a subset of those which were passed - to the :func:`.select` construct, not those which are available - from the ``.c`` collection of that :func:`.select`. That - is:: - - s = select([table1.c.a, table1.c.b]).select_from(table1) - s = s.with_only_columns([table1.c.b]) - - and **not**:: - - # usually incorrect - s = s.with_only_columns([s.c.b]) - - The latter would produce the SQL:: - - SELECT b - FROM (SELECT t1.a AS a, t1.b AS b - FROM t1), t1 - - Since the :func:`.select` construct is essentially being - asked to select both from ``table1`` as well as itself. - - """ - self._reset_exported() - rc = [] - for c in columns: - c = _literal_as_column(c) - if isinstance(c, _ScalarSelect): - c = c.self_group(against=operators.comma_op) - rc.append(c) - self._raw_columns = rc - - @_generative - def where(self, whereclause): - """return a new select() construct with the given expression added to - its WHERE clause, joined to the existing clause via AND, if any. - - """ - - self.append_whereclause(whereclause) - - @_generative - def having(self, having): - """return a new select() construct with the given expression added to - its HAVING clause, joined to the existing clause via AND, if any. - - """ - self.append_having(having) - - @_generative - def distinct(self, *expr): - """Return a new select() construct which will apply DISTINCT to its - columns clause. - - :param \*expr: optional column expressions. When present, - the Postgresql dialect will render a ``DISTINCT ON (>)`` - construct. - - """ - if expr: - expr = [_literal_as_text(e) for e in expr] - if isinstance(self._distinct, list): - self._distinct = self._distinct + expr - else: - self._distinct = expr - else: - self._distinct = True - - @_generative - def prefix_with(self, *expr): - """return a new select() construct which will apply the given - expressions, typically strings, to the start of its columns clause, - not using any commas. In particular is useful for MySQL - keywords. - - e.g.:: - - select(['a', 'b']).prefix_with('HIGH_PRIORITY', - 'SQL_SMALL_RESULT', - 'ALL') - - Would render:: - - SELECT HIGH_PRIORITY SQL_SMALL_RESULT ALL a, b - - """ - expr = tuple(_literal_as_text(e) for e in expr) - self._prefixes = self._prefixes + expr - - @_generative - def select_from(self, fromclause): - """return a new :func:`.select` construct with the given FROM expression - merged into its list of FROM objects. - - E.g.:: - - table1 = table('t1', column('a')) - table2 = table('t2', column('b')) - s = select([table1.c.a]).\\ - select_from( - table1.join(table2, table1.c.a==table2.c.b) - ) - - The "from" list is a unique set on the identity of each element, - so adding an already present :class:`.Table` or other selectable - will have no effect. Passing a :class:`.Join` that refers - to an already present :class:`.Table` or other selectable will have - the effect of concealing the presence of that selectable as - an individual element in the rendered FROM list, instead rendering it into a - JOIN clause. - - While the typical purpose of :meth:`.Select.select_from` is to replace - the default, derived FROM clause with a join, it can also be called with - individual table elements, multiple times if desired, in the case that the - FROM clause cannot be fully derived from the columns clause:: - - select([func.count('*')]).select_from(table1) - - """ - self.append_from(fromclause) - - @_generative - def correlate(self, *fromclauses): - """return a new select() construct which will correlate the given FROM - clauses to that of an enclosing select(), if a match is found. - - By "match", the given fromclause must be present in this select's - list of FROM objects and also present in an enclosing select's list of - FROM objects. - - Calling this method turns off the select's default behavior of - "auto-correlation". Normally, select() auto-correlates all of its FROM - clauses to those of an embedded select when compiled. - - If the fromclause is None, correlation is disabled for the returned - select(). - - """ - self._should_correlate = False - if fromclauses and fromclauses[0] is None: - self._correlate = set() - else: - self._correlate = self._correlate.union(fromclauses) - - def append_correlation(self, fromclause): - """append the given correlation expression to this select() - construct.""" - - self._should_correlate = False - self._correlate = self._correlate.union([fromclause]) - - def append_column(self, column): - """append the given column expression to the columns clause of this - select() construct. - - """ - self._reset_exported() - column = _literal_as_column(column) - - if isinstance(column, _ScalarSelect): - column = column.self_group(against=operators.comma_op) - - self._raw_columns = self._raw_columns + [column] - - def append_prefix(self, clause): - """append the given columns clause prefix expression to this select() - construct. - - """ - clause = _literal_as_text(clause) - self._prefixes = self._prefixes + (clause,) - - def append_whereclause(self, whereclause): - """append the given expression to this select() construct's WHERE - criterion. - - The expression will be joined to existing WHERE criterion via AND. - - """ - self._reset_exported() - whereclause = _literal_as_text(whereclause) - - if self._whereclause is not None: - self._whereclause = and_(self._whereclause, whereclause) - else: - self._whereclause = whereclause - - def append_having(self, having): - """append the given expression to this select() construct's HAVING - criterion. - - The expression will be joined to existing HAVING criterion via AND. - - """ - if self._having is not None: - self._having = and_(self._having, _literal_as_text(having)) - else: - self._having = _literal_as_text(having) - - def append_from(self, fromclause): - """append the given FromClause expression to this select() construct's - FROM clause. - - """ - self._reset_exported() - fromclause = _literal_as_text(fromclause) - self._from_obj = self._from_obj.union([fromclause]) - - def _populate_column_collection(self): - for c in self.inner_columns: - if hasattr(c, '_make_proxy'): - c._make_proxy(self, - name=self.use_labels - and c._label or None) - - def self_group(self, against=None): - """return a 'grouping' construct as per the ClauseElement - specification. - - This produces an element that can be embedded in an expression. Note - that this method is called automatically as needed when constructing - expressions and should not require explicit use. - - """ - if isinstance(against, CompoundSelect): - return self - return _FromGrouping(self) - - def union(self, other, **kwargs): - """return a SQL UNION of this select() construct against the given - selectable.""" - - return union(self, other, **kwargs) - - def union_all(self, other, **kwargs): - """return a SQL UNION ALL of this select() construct against the given - selectable. - - """ - return union_all(self, other, **kwargs) - - def except_(self, other, **kwargs): - """return a SQL EXCEPT of this select() construct against the given - selectable.""" - - return except_(self, other, **kwargs) - - def except_all(self, other, **kwargs): - """return a SQL EXCEPT ALL of this select() construct against the - given selectable. - - """ - return except_all(self, other, **kwargs) - - def intersect(self, other, **kwargs): - """return a SQL INTERSECT of this select() construct against the given - selectable. - - """ - return intersect(self, other, **kwargs) - - def intersect_all(self, other, **kwargs): - """return a SQL INTERSECT ALL of this select() construct against the - given selectable. - - """ - return intersect_all(self, other, **kwargs) - - def bind(self): - if self._bind: - return self._bind - froms = self._froms - if not froms: - for c in self._raw_columns: - e = c.bind - if e: - self._bind = e - return e - else: - e = list(froms)[0].bind - if e: - self._bind = e - return e - - return None - - def _set_bind(self, bind): - self._bind = bind - bind = property(bind, _set_bind) - -class UpdateBase(Executable, ClauseElement): - """Form the base for ``INSERT``, ``UPDATE``, and ``DELETE`` statements. - - """ - - __visit_name__ = 'update_base' - - _execution_options = \ - Executable._execution_options.union({'autocommit': True}) - kwargs = util.immutabledict() - _hints = util.immutabledict() - - def _process_colparams(self, parameters): - if isinstance(parameters, (list, tuple)): - pp = {} - for i, c in enumerate(self.table.c): - pp[c.key] = parameters[i] - return pp - else: - return parameters - - def params(self, *arg, **kw): - """Set the parameters for the statement. - - This method raises ``NotImplementedError`` on the base class, - and is overridden by :class:`.ValuesBase` to provide the - SET/VALUES clause of UPDATE and INSERT. - - """ - raise NotImplementedError( - "params() is not supported for INSERT/UPDATE/DELETE statements." - " To set the values for an INSERT or UPDATE statement, use" - " stmt.values(**parameters).") - - def bind(self): - """Return a 'bind' linked to this :class:`.UpdateBase` - or a :class:`.Table` associated with it. - - """ - return self._bind or self.table.bind - - def _set_bind(self, bind): - self._bind = bind - bind = property(bind, _set_bind) - - _returning_re = re.compile(r'(?:firebird|postgres(?:ql)?)_returning') - def _process_deprecated_kw(self, kwargs): - for k in list(kwargs): - m = self._returning_re.match(k) - if m: - self._returning = kwargs.pop(k) - util.warn_deprecated( - "The %r argument is deprecated. Please " - "use statement.returning(col1, col2, ...)" % k - ) - return kwargs - - @_generative - def returning(self, *cols): - """Add a RETURNING or equivalent clause to this statement. - - The given list of columns represent columns within the table that is - the target of the INSERT, UPDATE, or DELETE. Each element can be any - column expression. :class:`~sqlalchemy.schema.Table` objects will be - expanded into their individual columns. - - Upon compilation, a RETURNING clause, or database equivalent, - will be rendered within the statement. For INSERT and UPDATE, - the values are the newly inserted/updated values. For DELETE, - the values are those of the rows which were deleted. - - Upon execution, the values of the columns to be returned - are made available via the result set and can be iterated - using ``fetchone()`` and similar. For DBAPIs which do not - natively support returning values (i.e. cx_oracle), - SQLAlchemy will approximate this behavior at the result level - so that a reasonable amount of behavioral neutrality is - provided. - - Note that not all databases/DBAPIs - support RETURNING. For those backends with no support, - an exception is raised upon compilation and/or execution. - For those who do support it, the functionality across backends - varies greatly, including restrictions on executemany() - and other statements which return multiple rows. Please - read the documentation notes for the database in use in - order to determine the availability of RETURNING. - - """ - self._returning = cols - - @_generative - def with_hint(self, text, selectable=None, dialect_name="*"): - """Add a table hint for a single table to this - INSERT/UPDATE/DELETE statement. - - .. note:: - - :meth:`.UpdateBase.with_hint` currently applies only to - Microsoft SQL Server. For MySQL INSERT hints, use - :meth:`.Insert.prefix_with`. UPDATE/DELETE hints for - MySQL will be added in a future release. - - The text of the hint is rendered in the appropriate - location for the database backend in use, relative - to the :class:`.Table` that is the subject of this - statement, or optionally to that of the given - :class:`.Table` passed as the ``selectable`` argument. - - The ``dialect_name`` option will limit the rendering of a particular - hint to a particular backend. Such as, to add a hint - that only takes effect for SQL Server:: - - mytable.insert().with_hint("WITH (PAGLOCK)", dialect_name="mssql") - - .. versionadded:: 0.7.6 - - :param text: Text of the hint. - :param selectable: optional :class:`.Table` that specifies - an element of the FROM clause within an UPDATE or DELETE - to be the subject of the hint - applies only to certain backends. - :param dialect_name: defaults to ``*``, if specified as the name - of a particular dialect, will apply these hints only when - that dialect is in use. - """ - if selectable is None: - selectable = self.table - - self._hints = self._hints.union({(selectable, dialect_name):text}) - -class ValuesBase(UpdateBase): - """Supplies support for :meth:`.ValuesBase.values` to INSERT and UPDATE constructs.""" - - __visit_name__ = 'values_base' - - def __init__(self, table, values): - self.table = table - self.parameters = self._process_colparams(values) - - @_generative - def values(self, *args, **kwargs): - """specify the VALUES clause for an INSERT statement, or the SET - clause for an UPDATE. - - :param \**kwargs: key value pairs representing the string key - of a :class:`.Column` mapped to the value to be rendered into the - VALUES or SET clause:: - - users.insert().values(name="some name") - - users.update().where(users.c.id==5).values(name="some name") - - :param \*args: A single dictionary can be sent as the first positional - argument. This allows non-string based keys, such as Column - objects, to be used:: - - users.insert().values({users.c.name : "some name"}) - - users.update().where(users.c.id==5).values({users.c.name : "some name"}) - - See also: - - :ref:`inserts_and_updates` - SQL Expression - Language Tutorial - - :func:`~.expression.insert` - produce an ``INSERT`` statement - - :func:`~.expression.update` - produce an ``UPDATE`` statement - - """ - if args: - v = args[0] - else: - v = {} - - if self.parameters is None: - self.parameters = self._process_colparams(v) - self.parameters.update(kwargs) - else: - self.parameters = self.parameters.copy() - self.parameters.update(self._process_colparams(v)) - self.parameters.update(kwargs) - -class Insert(ValuesBase): - """Represent an INSERT construct. - - The :class:`.Insert` object is created using the :func:`~.expression.insert()` function. - - See also: - - :ref:`coretutorial_insert_expressions` - - """ - __visit_name__ = 'insert' - - _prefixes = () - - def __init__(self, - table, - values=None, - inline=False, - bind=None, - prefixes=None, - returning=None, - **kwargs): - ValuesBase.__init__(self, table, values) - self._bind = bind - self.select = None - self.inline = inline - self._returning = returning - if prefixes: - self._prefixes = tuple([_literal_as_text(p) for p in prefixes]) - - if kwargs: - self.kwargs = self._process_deprecated_kw(kwargs) - - def get_children(self, **kwargs): - if self.select is not None: - return self.select, - else: - return () - - def _copy_internals(self, clone=_clone, **kw): - # TODO: coverage - self.parameters = self.parameters.copy() - - @_generative - def prefix_with(self, clause): - """Add a word or expression between INSERT and INTO. Generative. - - If multiple prefixes are supplied, they will be separated with - spaces. - - """ - clause = _literal_as_text(clause) - self._prefixes = self._prefixes + (clause,) - -class Update(ValuesBase): - """Represent an Update construct. - - The :class:`.Update` object is created using the :func:`update()` function. - - """ - __visit_name__ = 'update' - - def __init__(self, - table, - whereclause, - values=None, - inline=False, - bind=None, - returning=None, - **kwargs): - ValuesBase.__init__(self, table, values) - self._bind = bind - self._returning = returning - if whereclause is not None: - self._whereclause = _literal_as_text(whereclause) - else: - self._whereclause = None - self.inline = inline - - if kwargs: - self.kwargs = self._process_deprecated_kw(kwargs) - - def get_children(self, **kwargs): - if self._whereclause is not None: - return self._whereclause, - else: - return () - - def _copy_internals(self, clone=_clone, **kw): - # TODO: coverage - self._whereclause = clone(self._whereclause, **kw) - self.parameters = self.parameters.copy() - - @_generative - def where(self, whereclause): - """return a new update() construct with the given expression added to - its WHERE clause, joined to the existing clause via AND, if any. - - """ - if self._whereclause is not None: - self._whereclause = and_(self._whereclause, - _literal_as_text(whereclause)) - else: - self._whereclause = _literal_as_text(whereclause) - - @property - def _extra_froms(self): - # TODO: this could be made memoized - # if the memoization is reset on each generative call. - froms = [] - seen = set([self.table]) - - if self._whereclause is not None: - for item in _from_objects(self._whereclause): - if not seen.intersection(item._cloned_set): - froms.append(item) - seen.update(item._cloned_set) - - return froms - -class Delete(UpdateBase): - """Represent a DELETE construct. - - The :class:`.Delete` object is created using the :func:`delete()` function. - - """ - - __visit_name__ = 'delete' - - def __init__(self, - table, - whereclause, - bind=None, - returning =None, - **kwargs): - self._bind = bind - self.table = table - self._returning = returning - - if whereclause is not None: - self._whereclause = _literal_as_text(whereclause) - else: - self._whereclause = None - - if kwargs: - self.kwargs = self._process_deprecated_kw(kwargs) - - def get_children(self, **kwargs): - if self._whereclause is not None: - return self._whereclause, - else: - return () - - @_generative - def where(self, whereclause): - """Add the given WHERE clause to a newly returned delete construct.""" - - if self._whereclause is not None: - self._whereclause = and_(self._whereclause, - _literal_as_text(whereclause)) - else: - self._whereclause = _literal_as_text(whereclause) - - def _copy_internals(self, clone=_clone, **kw): - # TODO: coverage - self._whereclause = clone(self._whereclause, **kw) - -class _IdentifiedClause(Executable, ClauseElement): - - __visit_name__ = 'identified' - _execution_options = \ - Executable._execution_options.union({'autocommit': False}) - quote = None - - def __init__(self, ident): - self.ident = ident - -class SavepointClause(_IdentifiedClause): - __visit_name__ = 'savepoint' - -class RollbackToSavepointClause(_IdentifiedClause): - __visit_name__ = 'rollback_to_savepoint' - -class ReleaseSavepointClause(_IdentifiedClause): - __visit_name__ = 'release_savepoint' - - +_BindParamClause = BindParameter +_Label = Label +_SelectBase = SelectBase +_BinaryExpression = BinaryExpression +_Cast = Cast +_Null = Null +_False = False_ +_True = True_ +_TextClause = TextClause +_UnaryExpression = UnaryExpression +_Case = Case +_Tuple = Tuple +_Over = Over +_Generative = Generative +_TypeClause = TypeClause +_Extract = Extract +_Exists = Exists +_Grouping = Grouping +_FromGrouping = FromGrouping +_ScalarSelect = ScalarSelect diff --git a/libs/sqlalchemy/sql/functions.py b/libs/sqlalchemy/sql/functions.py index 95781d70..f300e241 100644 --- a/libs/sqlalchemy/sql/functions.py +++ b/libs/sqlalchemy/sql/functions.py @@ -1,36 +1,418 @@ # sql/functions.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy import types as sqltypes, schema -from sqlalchemy.sql.expression import ( - ClauseList, Function, _literal_as_binds, text, _type_from_args - ) -from sqlalchemy.sql import operators -from sqlalchemy.sql.visitors import VisitableType +"""SQL function API, factories, and built-in functions. + +""" +from . import sqltypes, schema +from .base import Executable +from .elements import ClauseList, Cast, Extract, _literal_as_binds, \ + literal_column, _type_from_args, ColumnElement, _clone,\ + Over, BindParameter +from .selectable import FromClause, Select + +from . import operators +from .visitors import VisitableType +from .. import util + +_registry = util.defaultdict(dict) + + +def register_function(identifier, fn, package="_default"): + """Associate a callable with a particular func. name. + + This is normally called by _GenericMeta, but is also + available by itself so that a non-Function construct + can be associated with the :data:`.func` accessor (i.e. + CAST, EXTRACT). + + """ + reg = _registry[package] + reg[identifier] = fn + + +class FunctionElement(Executable, ColumnElement, FromClause): + """Base for SQL function-oriented constructs. + + .. seealso:: + + :class:`.Function` - named SQL function. + + :data:`.func` - namespace which produces registered or ad-hoc + :class:`.Function` instances. + + :class:`.GenericFunction` - allows creation of registered function + types. + + """ + + packagenames = () + + def __init__(self, *clauses, **kwargs): + """Construct a :class:`.FunctionElement`. + """ + args = [_literal_as_binds(c, self.name) for c in clauses] + self.clause_expr = ClauseList( + operator=operators.comma_op, + group_contents=True, *args).\ + self_group() + + def _execute_on_connection(self, connection, multiparams, params): + return connection._execute_function(self, multiparams, params) + + @property + def columns(self): + """Fulfill the 'columns' contract of :class:`.ColumnElement`. + + Returns a single-element list consisting of this object. + + """ + return [self] + + @util.memoized_property + def clauses(self): + """Return the underlying :class:`.ClauseList` which contains + the arguments for this :class:`.FunctionElement`. + + """ + return self.clause_expr.element + + def over(self, partition_by=None, order_by=None): + """Produce an OVER clause against this function. + + Used against aggregate or so-called "window" functions, + for database backends that support window functions. + + The expression:: + + func.row_number().over(order_by='x') + + is shorthand for:: + + from sqlalchemy import over + over(func.row_number(), order_by='x') + + See :func:`~.expression.over` for a full description. + + .. versionadded:: 0.7 + + """ + return Over(self, partition_by=partition_by, order_by=order_by) + + @property + def _from_objects(self): + return self.clauses._from_objects + + def get_children(self, **kwargs): + return self.clause_expr, + + def _copy_internals(self, clone=_clone, **kw): + self.clause_expr = clone(self.clause_expr, **kw) + self._reset_exported() + FunctionElement.clauses._reset(self) + + def select(self): + """Produce a :func:`~.expression.select` construct + against this :class:`.FunctionElement`. + + This is shorthand for:: + + s = select([function_element]) + + """ + s = Select([self]) + if self._execution_options: + s = s.execution_options(**self._execution_options) + return s + + def scalar(self): + """Execute this :class:`.FunctionElement` against an embedded + 'bind' and return a scalar value. + + This first calls :meth:`~.FunctionElement.select` to + produce a SELECT construct. + + Note that :class:`.FunctionElement` can be passed to + the :meth:`.Connectable.scalar` method of :class:`.Connection` + or :class:`.Engine`. + + """ + return self.select().execute().scalar() + + def execute(self): + """Execute this :class:`.FunctionElement` against an embedded + 'bind'. + + This first calls :meth:`~.FunctionElement.select` to + produce a SELECT construct. + + Note that :class:`.FunctionElement` can be passed to + the :meth:`.Connectable.execute` method of :class:`.Connection` + or :class:`.Engine`. + + """ + return self.select().execute() + + def _bind_param(self, operator, obj): + return BindParameter(None, obj, _compared_to_operator=operator, + _compared_to_type=self.type, unique=True) + + +class _FunctionGenerator(object): + """Generate :class:`.Function` objects based on getattr calls.""" + + def __init__(self, **opts): + self.__names = [] + self.opts = opts + + def __getattr__(self, name): + # passthru __ attributes; fixes pydoc + if name.startswith('__'): + try: + return self.__dict__[name] + except KeyError: + raise AttributeError(name) + + elif name.endswith('_'): + name = name[0:-1] + f = _FunctionGenerator(**self.opts) + f.__names = list(self.__names) + [name] + return f + + def __call__(self, *c, **kwargs): + o = self.opts.copy() + o.update(kwargs) + + tokens = len(self.__names) + + if tokens == 2: + package, fname = self.__names + elif tokens == 1: + package, fname = "_default", self.__names[0] + else: + package = None + + if package is not None: + func = _registry[package].get(fname) + if func is not None: + return func(*c, **o) + + return Function(self.__names[-1], + packagenames=self.__names[0:-1], *c, **o) + + +func = _FunctionGenerator() +"""Generate SQL function expressions. + + :data:`.func` is a special object instance which generates SQL + functions based on name-based attributes, e.g.:: + + >>> print func.count(1) + count(:param_1) + + The element is a column-oriented SQL element like any other, and is + used in that way:: + + >>> print select([func.count(table.c.id)]) + SELECT count(sometable.id) FROM sometable + + Any name can be given to :data:`.func`. If the function name is unknown to + SQLAlchemy, it will be rendered exactly as is. For common SQL functions + which SQLAlchemy is aware of, the name may be interpreted as a *generic + function* which will be compiled appropriately to the target database:: + + >>> print func.current_timestamp() + CURRENT_TIMESTAMP + + To call functions which are present in dot-separated packages, + specify them in the same manner:: + + >>> print func.stats.yield_curve(5, 10) + stats.yield_curve(:yield_curve_1, :yield_curve_2) + + SQLAlchemy can be made aware of the return type of functions to enable + type-specific lexical and result-based behavior. For example, to ensure + that a string-based function returns a Unicode value and is similarly + treated as a string in expressions, specify + :class:`~sqlalchemy.types.Unicode` as the type: + + >>> print func.my_string(u'hi', type_=Unicode) + ' ' + \ + ... func.my_string(u'there', type_=Unicode) + my_string(:my_string_1) || :my_string_2 || my_string(:my_string_3) + + The object returned by a :data:`.func` call is usually an instance of + :class:`.Function`. + This object meets the "column" interface, including comparison and labeling + functions. The object can also be passed the :meth:`~.Connectable.execute` + method of a :class:`.Connection` or :class:`.Engine`, where it will be + wrapped inside of a SELECT statement first:: + + print connection.execute(func.current_timestamp()).scalar() + + In a few exception cases, the :data:`.func` accessor + will redirect a name to a built-in expression such as :func:`.cast` + or :func:`.extract`, as these names have well-known meaning + but are not exactly the same as "functions" from a SQLAlchemy + perspective. + + .. versionadded:: 0.8 :data:`.func` can return non-function expression + constructs for common quasi-functional names like :func:`.cast` + and :func:`.extract`. + + Functions which are interpreted as "generic" functions know how to + calculate their return type automatically. For a listing of known generic + functions, see :ref:`generic_functions`. + +""" + +modifier = _FunctionGenerator(group=False) + +class Function(FunctionElement): + """Describe a named SQL function. + + See the superclass :class:`.FunctionElement` for a description + of public methods. + + .. seealso:: + + :data:`.func` - namespace which produces registered or ad-hoc + :class:`.Function` instances. + + :class:`.GenericFunction` - allows creation of registered function + types. + + """ + + __visit_name__ = 'function' + + def __init__(self, name, *clauses, **kw): + """Construct a :class:`.Function`. + + The :data:`.func` construct is normally used to construct + new :class:`.Function` instances. + + """ + self.packagenames = kw.pop('packagenames', None) or [] + self.name = name + self._bind = kw.get('bind', None) + self.type = sqltypes.to_instance(kw.get('type_', None)) + + FunctionElement.__init__(self, *clauses, **kw) + + def _bind_param(self, operator, obj): + return BindParameter(self.name, obj, + _compared_to_operator=operator, + _compared_to_type=self.type, + unique=True) + class _GenericMeta(VisitableType): - def __call__(self, *args, **kwargs): - args = [_literal_as_binds(c) for c in args] - return type.__call__(self, *args, **kwargs) + def __init__(cls, clsname, bases, clsdict): + cls.name = name = clsdict.get('name', clsname) + cls.identifier = identifier = clsdict.get('identifier', name) + package = clsdict.pop('package', '_default') + # legacy + if '__return_type__' in clsdict: + cls.type = clsdict['__return_type__'] + register_function(identifier, cls, package) + super(_GenericMeta, cls).__init__(clsname, bases, clsdict) -class GenericFunction(Function): - __metaclass__ = _GenericMeta - def __init__(self, type_=None, args=(), **kwargs): +class GenericFunction(util.with_metaclass(_GenericMeta, Function)): + """Define a 'generic' function. + + A generic function is a pre-established :class:`.Function` + class that is instantiated automatically when called + by name from the :data:`.func` attribute. Note that + calling any name from :data:`.func` has the effect that + a new :class:`.Function` instance is created automatically, + given that name. The primary use case for defining + a :class:`.GenericFunction` class is so that a function + of a particular name may be given a fixed return type. + It can also include custom argument parsing schemes as well + as additional methods. + + Subclasses of :class:`.GenericFunction` are automatically + registered under the name of the class. For + example, a user-defined function ``as_utc()`` would + be available immediately:: + + from sqlalchemy.sql.functions import GenericFunction + from sqlalchemy.types import DateTime + + class as_utc(GenericFunction): + type = DateTime + + print select([func.as_utc()]) + + User-defined generic functions can be organized into + packages by specifying the "package" attribute when defining + :class:`.GenericFunction`. Third party libraries + containing many functions may want to use this in order + to avoid name conflicts with other systems. For example, + if our ``as_utc()`` function were part of a package + "time":: + + class as_utc(GenericFunction): + type = DateTime + package = "time" + + The above function would be available from :data:`.func` + using the package name ``time``:: + + print select([func.time.as_utc()]) + + A final option is to allow the function to be accessed + from one name in :data:`.func` but to render as a different name. + The ``identifier`` attribute will override the name used to + access the function as loaded from :data:`.func`, but will retain + the usage of ``name`` as the rendered name:: + + class GeoBuffer(GenericFunction): + type = Geometry + package = "geo" + name = "ST_Buffer" + identifier = "buffer" + + The above function will render as follows:: + + >>> print func.geo.buffer() + ST_Buffer() + + .. versionadded:: 0.8 :class:`.GenericFunction` now supports + automatic registration of new functions as well as package + and custom naming support. + + .. versionchanged:: 0.8 The attribute name ``type`` is used + to specify the function's return type at the class level. + Previously, the name ``__return_type__`` was used. This + name is still recognized for backwards-compatibility. + + """ + + coerce_arguments = True + + def __init__(self, *args, **kwargs): + parsed_args = kwargs.pop('_parsed_args', None) + if parsed_args is None: + parsed_args = [_literal_as_binds(c) for c in args] self.packagenames = [] - self.name = self.__class__.__name__ self._bind = kwargs.get('bind', None) self.clause_expr = ClauseList( operator=operators.comma_op, - group_contents=True, *args).self_group() + group_contents=True, *parsed_args).self_group() self.type = sqltypes.to_instance( - type_ or getattr(self, '__return_type__', None)) + kwargs.pop("type_", None) or getattr(self, 'type', None)) -class next_value(Function): +register_function("cast", Cast) +register_function("extract", Extract) + + +class next_value(GenericFunction): """Represent the 'next value', given a :class:`.Sequence` as it's single argument. @@ -52,83 +434,101 @@ class next_value(Function): def _from_objects(self): return [] + class AnsiFunction(GenericFunction): def __init__(self, **kwargs): GenericFunction.__init__(self, **kwargs) + class ReturnTypeFromArgs(GenericFunction): """Define a function whose return type is the same as its arguments.""" def __init__(self, *args, **kwargs): + args = [_literal_as_binds(c) for c in args] kwargs.setdefault('type_', _type_from_args(args)) - GenericFunction.__init__(self, args=args, **kwargs) + kwargs['_parsed_args'] = args + GenericFunction.__init__(self, *args, **kwargs) + class coalesce(ReturnTypeFromArgs): pass + class max(ReturnTypeFromArgs): pass + class min(ReturnTypeFromArgs): pass + class sum(ReturnTypeFromArgs): pass class now(GenericFunction): - __return_type__ = sqltypes.DateTime + type = sqltypes.DateTime + class concat(GenericFunction): - __return_type__ = sqltypes.String - def __init__(self, *args, **kwargs): - GenericFunction.__init__(self, args=args, **kwargs) + type = sqltypes.String + class char_length(GenericFunction): - __return_type__ = sqltypes.Integer + type = sqltypes.Integer def __init__(self, arg, **kwargs): - GenericFunction.__init__(self, args=[arg], **kwargs) + GenericFunction.__init__(self, arg, **kwargs) + class random(GenericFunction): - def __init__(self, *args, **kwargs): - kwargs.setdefault('type_', None) - GenericFunction.__init__(self, args=args, **kwargs) + pass + class count(GenericFunction): - """The ANSI COUNT aggregate function. With no arguments, emits COUNT \*.""" + """The ANSI COUNT aggregate function. With no arguments, + emits COUNT \*. - __return_type__ = sqltypes.Integer + """ + type = sqltypes.Integer def __init__(self, expression=None, **kwargs): if expression is None: - expression = text('*') - GenericFunction.__init__(self, args=(expression,), **kwargs) + expression = literal_column('*') + GenericFunction.__init__(self, expression, **kwargs) + class current_date(AnsiFunction): - __return_type__ = sqltypes.Date + type = sqltypes.Date + class current_time(AnsiFunction): - __return_type__ = sqltypes.Time + type = sqltypes.Time + class current_timestamp(AnsiFunction): - __return_type__ = sqltypes.DateTime + type = sqltypes.DateTime + class current_user(AnsiFunction): - __return_type__ = sqltypes.String + type = sqltypes.String + class localtime(AnsiFunction): - __return_type__ = sqltypes.DateTime + type = sqltypes.DateTime + class localtimestamp(AnsiFunction): - __return_type__ = sqltypes.DateTime + type = sqltypes.DateTime + class session_user(AnsiFunction): - __return_type__ = sqltypes.String + type = sqltypes.String + class sysdate(AnsiFunction): - __return_type__ = sqltypes.DateTime + type = sqltypes.DateTime + class user(AnsiFunction): - __return_type__ = sqltypes.String - + type = sqltypes.String diff --git a/libs/sqlalchemy/sql/operators.py b/libs/sqlalchemy/sql/operators.py index 9e796506..d7ec977a 100644 --- a/libs/sqlalchemy/sql/operators.py +++ b/libs/sqlalchemy/sql/operators.py @@ -1,5 +1,5 @@ # sql/operators.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -9,21 +9,29 @@ """Defines operators used in SQL expressions.""" +from .. import util + + from operator import ( - and_, or_, inv, add, mul, sub, mod, truediv, lt, le, ne, gt, ge, eq, neg + and_, or_, inv, add, mul, sub, mod, truediv, lt, le, ne, gt, ge, eq, neg, + getitem, lshift, rshift ) -# Py2K -from operator import (div,) -# end Py2K +if util.py2k: + from operator import div +else: + div = truediv + -from sqlalchemy.util import symbol class Operators(object): """Base of comparison and logical operators. - Implements base methods :meth:`operate` and :meth:`reverse_operate`, - as well as :meth:`__and__`, :meth:`__or__`, :meth:`__invert__`. + Implements base methods :meth:`~sqlalchemy.sql.operators.Operators.operate` and + :meth:`~sqlalchemy.sql.operators.Operators.reverse_operate`, as well as + :meth:`~sqlalchemy.sql.operators.Operators.__and__`, + :meth:`~sqlalchemy.sql.operators.Operators.__or__`, + :meth:`~sqlalchemy.sql.operators.Operators.__invert__`. Usually is used via its most common subclass :class:`.ColumnOperators`. @@ -94,7 +102,7 @@ class Operators(object): """ return self.operate(inv) - def op(self, opstring): + def op(self, opstring, precedence=0): """produce a generic operator function. e.g.:: @@ -105,21 +113,37 @@ class Operators(object): somecolumn * 5 - :param operator: a string which will be output as the infix operator - between this :class:`.ClauseElement` and the expression passed to the - generated function. - This function can also be used to make bitwise operators explicit. For example:: somecolumn.op('&')(0xff) - is a bitwise AND of the value in somecolumn. + is a bitwise AND of the value in ``somecolumn``. + + :param operator: a string which will be output as the infix operator + between this element and the expression passed to the + generated function. + + :param precedence: precedence to apply to the operator, when + parenthesizing expressions. A lower number will cause the expression + to be parenthesized when applied against another operator with + higher precedence. The default value of ``0`` is lower than all + operators except for the comma (``,``) and ``AS`` operators. + A value of 100 will be higher or equal to all operators, and -100 + will be lower than or equal to all operators. + + .. versionadded:: 0.8 - added the 'precedence' argument. + + .. seealso:: + + :ref:`types_operators` """ - def _op(b): - return self.operate(op, opstring, b) - return _op + operator = custom_op(opstring, precedence) + + def against(other): + return operator(self, other) + return against def operate(self, op, *other, **kwargs): """Operate on an argument. @@ -155,11 +179,48 @@ class Operators(object): """ raise NotImplementedError(str(op)) -class ColumnOperators(Operators): - """Defines comparison and math operations. - By default all methods call down to - :meth:`Operators.operate` or :meth:`Operators.reverse_operate` +class custom_op(object): + """Represent a 'custom' operator. + + :class:`.custom_op` is normally instantitated when the + :meth:`.ColumnOperators.op` method is used to create a + custom operator callable. The class can also be used directly + when programmatically constructing expressions. E.g. + to represent the "factorial" operation:: + + from sqlalchemy.sql import UnaryExpression + from sqlalchemy.sql import operators + from sqlalchemy import Numeric + + unary = UnaryExpression(table.c.somecolumn, + modifier=operators.custom_op("!"), + type_=Numeric) + + """ + __name__ = 'custom_op' + + def __init__(self, opstring, precedence=0): + self.opstring = opstring + self.precedence = precedence + + def __eq__(self, other): + return isinstance(other, custom_op) and \ + other.opstring == self.opstring + + def __hash__(self): + return id(self) + + def __call__(self, left, right, **kw): + return left.operate(self, right, **kw) + + +class ColumnOperators(Operators): + """Defines boolean, comparison, and other operators for + :class:`.ColumnElement` expressions. + + By default, all methods call down to + :meth:`.operate` or :meth:`.reverse_operate`, passing in the appropriate operator function from the Python builtin ``operator`` module or a SQLAlchemy-specific operator function from @@ -174,15 +235,21 @@ class ColumnOperators(Operators): def eq(a, b): return a == b - A SQLAlchemy construct like :class:`.ColumnElement` ultimately + The core column expression unit :class:`.ColumnElement` overrides :meth:`.Operators.operate` and others - to return further :class:`.ClauseElement` constructs, + to return further :class:`.ColumnElement` constructs, so that the ``==`` operation above is replaced by a clause construct. - The docstrings here will describe column-oriented - behavior of each operator. For ORM-based operators - on related objects and collections, see :class:`.RelationshipProperty.Comparator`. + See also: + + :ref:`types_operators` + + :attr:`.TypeEngine.comparator_factory` + + :class:`.ColumnOperators` + + :class:`.PropComparator` """ @@ -249,6 +316,33 @@ class ColumnOperators(Operators): """ return self.operate(neg) + def __getitem__(self, index): + """Implement the [] operator. + + This can be used by some database-specific types + such as Postgresql ARRAY and HSTORE. + + """ + return self.operate(getitem, index) + + def __lshift__(self, other): + """implement the << operator. + + Not used by SQLAlchemy core, this is provided + for custom operator systems which want to use + << as an extension point. + """ + return self.operate(lshift, other) + + def __rshift__(self, other): + """implement the >> operator. + + Not used by SQLAlchemy core, this is provided + for custom operator systems which want to use + >> as an extension point. + """ + return self.operate(rshift, other) + def concat(self, other): """Implement the 'concat' operator. @@ -263,6 +357,20 @@ class ColumnOperators(Operators): In a column context, produces the clause ``a LIKE other``. + E.g.:: + + select([sometable]).where(sometable.c.column.like("%foobar%")) + + :param other: expression to be compared + :param escape: optional escape character, renders the ``ESCAPE`` + keyword, e.g.:: + + somecolumn.like("foo/%bar", escape="/") + + .. seealso:: + + :meth:`.ColumnOperators.ilike` + """ return self.operate(like_op, other, escape=escape) @@ -271,6 +379,20 @@ class ColumnOperators(Operators): In a column context, produces the clause ``a ILIKE other``. + E.g.:: + + select([sometable]).where(sometable.c.column.ilike("%foobar%")) + + :param other: expression to be compared + :param escape: optional escape character, renders the ``ESCAPE`` + keyword, e.g.:: + + somecolumn.ilike("foo/%bar", escape="/") + + .. seealso:: + + :meth:`.ColumnOperators.like` + """ return self.operate(ilike_op, other, escape=escape) @@ -284,6 +406,51 @@ class ColumnOperators(Operators): """ return self.operate(in_op, other) + def notin_(self, other): + """implement the ``NOT IN`` operator. + + This is equivalent to using negation with :meth:`.ColumnOperators.in_`, + i.e. ``~x.in_(y)``. + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`.ColumnOperators.in_` + + """ + return self.operate(notin_op, other) + + def notlike(self, other, escape=None): + """implement the ``NOT LIKE`` operator. + + This is equivalent to using negation with + :meth:`.ColumnOperators.like`, i.e. ``~x.like(y)``. + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`.ColumnOperators.like` + + """ + return self.operate(notlike_op, other, escape=escape) + + def notilike(self, other, escape=None): + """implement the ``NOT ILIKE`` operator. + + This is equivalent to using negation with + :meth:`.ColumnOperators.ilike`, i.e. ``~x.ilike(y)``. + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`.ColumnOperators.ilike` + + """ + return self.operate(notilike_op, other, escape=escape) + def is_(self, other): """Implement the ``IS`` operator. @@ -376,7 +543,7 @@ class ColumnOperators(Operators): def __radd__(self, other): """Implement the ``+`` operator in reverse. - See :meth:`__add__`. + See :meth:`.ColumnOperators.__add__`. """ return self.reverse_operate(add, other) @@ -384,7 +551,7 @@ class ColumnOperators(Operators): def __rsub__(self, other): """Implement the ``-`` operator in reverse. - See :meth:`__sub__`. + See :meth:`.ColumnOperators.__sub__`. """ return self.reverse_operate(sub, other) @@ -392,7 +559,7 @@ class ColumnOperators(Operators): def __rmul__(self, other): """Implement the ``*`` operator in reverse. - See :meth:`__mul__`. + See :meth:`.ColumnOperators.__mul__`. """ return self.reverse_operate(mul, other) @@ -400,7 +567,7 @@ class ColumnOperators(Operators): def __rdiv__(self, other): """Implement the ``/`` operator in reverse. - See :meth:`__div__`. + See :meth:`.ColumnOperators.__div__`. """ return self.reverse_operate(div, other) @@ -411,7 +578,10 @@ class ColumnOperators(Operators): return self.operate(between_op, cleft, cright) def distinct(self): - """Produce a :func:`~.expression.distinct` clause against the parent object.""" + """Produce a :func:`~.expression.distinct` clause against the + parent object. + + """ return self.operate(distinct_op) def __add__(self, other): @@ -421,7 +591,7 @@ class ColumnOperators(Operators): if the parent object has non-string affinity. If the parent object has a string affinity, produces the concatenation operator, ``a || b`` - - see :meth:`concat`. + see :meth:`.ColumnOperators.concat`. """ return self.operate(add, other) @@ -469,144 +639,216 @@ class ColumnOperators(Operators): def __rtruediv__(self, other): """Implement the ``//`` operator in reverse. - See :meth:`__truediv__`. + See :meth:`.ColumnOperators.__truediv__`. """ return self.reverse_operate(truediv, other) + def from_(): raise NotImplementedError() + def as_(): raise NotImplementedError() + def exists(): raise NotImplementedError() + +def istrue(a): + raise NotImplementedError() + +def isfalse(a): + raise NotImplementedError() + def is_(a, b): return a.is_(b) + def isnot(a, b): return a.isnot(b) + def collate(a, b): return a.collate(b) + def op(a, opstring, b): return a.op(opstring)(b) + def like_op(a, b, escape=None): return a.like(b, escape=escape) + def notlike_op(a, b, escape=None): - return ~a.like(b, escape=escape) + return a.notlike(b, escape=escape) + def ilike_op(a, b, escape=None): return a.ilike(b, escape=escape) + def notilike_op(a, b, escape=None): - return ~a.ilike(b, escape=escape) + return a.notilike(b, escape=escape) + def between_op(a, b, c): return a.between(b, c) + def in_op(a, b): return a.in_(b) + def notin_op(a, b): - return ~a.in_(b) + return a.notin_(b) + def distinct_op(a): return a.distinct() + def startswith_op(a, b, escape=None): return a.startswith(b, escape=escape) + +def notstartswith_op(a, b, escape=None): + return ~a.startswith(b, escape=escape) + + def endswith_op(a, b, escape=None): return a.endswith(b, escape=escape) + +def notendswith_op(a, b, escape=None): + return ~a.endswith(b, escape=escape) + + def contains_op(a, b, escape=None): return a.contains(b, escape=escape) + +def notcontains_op(a, b, escape=None): + return ~a.contains(b, escape=escape) + + def match_op(a, b): return a.match(b) + def comma_op(a, b): raise NotImplementedError() + def concat_op(a, b): return a.concat(b) + def desc_op(a): return a.desc() + def asc_op(a): return a.asc() + def nullsfirst_op(a): return a.nullsfirst() + def nullslast_op(a): return a.nullslast() + _commutative = set([eq, ne, add, mul]) +_comparison = set([eq, ne, lt, gt, ge, le, between_op]) + + +def is_comparison(op): + return op in _comparison + + def is_commutative(op): return op in _commutative + def is_ordering_modifier(op): return op in (asc_op, desc_op, nullsfirst_op, nullslast_op) _associative = _commutative.union([concat_op, and_, or_]) +_natural_self_precedent = _associative.union([getitem]) +"""Operators where if we have (a op b) op c, we don't want to +parenthesize (a op b). -_smallest = symbol('_smallest') -_largest = symbol('_largest') +""" + +_asbool = util.symbol('_asbool', canonical=-10) +_smallest = util.symbol('_smallest', canonical=-100) +_largest = util.symbol('_largest', canonical=100) _PRECEDENCE = { from_: 15, - mul: 7, - truediv: 7, - # Py2K - div: 7, - # end Py2K - mod: 7, - neg: 7, - add: 6, - sub: 6, + getitem: 15, + mul: 8, + truediv: 8, + div: 8, + mod: 8, + neg: 8, + add: 7, + sub: 7, + concat_op: 6, match_op: 6, - ilike_op: 5, - notilike_op: 5, - like_op: 5, - notlike_op: 5, - in_op: 5, - notin_op: 5, - is_: 5, - isnot: 5, + + ilike_op: 6, + notilike_op: 6, + like_op: 6, + notlike_op: 6, + in_op: 6, + notin_op: 6, + + is_: 6, + isnot: 6, + eq: 5, ne: 5, gt: 5, lt: 5, ge: 5, le: 5, + between_op: 5, distinct_op: 5, inv: 5, + istrue: 5, + isfalse: 5, and_: 3, or_: 2, comma_op: -1, - collate: 7, + + desc_op: 3, + asc_op: 3, + collate: 4, + as_: -1, exists: 0, - _smallest: -1000, - _largest: 1000 + _asbool: -10, + _smallest: _smallest, + _largest: _largest } + def is_precedent(operator, against): - if operator is against and operator in _associative: + if operator is against and operator in _natural_self_precedent: return False else: - return (_PRECEDENCE.get(operator, _PRECEDENCE[_smallest]) <= - _PRECEDENCE.get(against, _PRECEDENCE[_largest])) + return (_PRECEDENCE.get(operator, + getattr(operator, 'precedence', _smallest)) <= + _PRECEDENCE.get(against, + getattr(against, 'precedence', _largest))) diff --git a/libs/sqlalchemy/sql/schema.py b/libs/sqlalchemy/sql/schema.py new file mode 100644 index 00000000..6ee92871 --- /dev/null +++ b/libs/sqlalchemy/sql/schema.py @@ -0,0 +1,3150 @@ +# sql/schema.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""The schema module provides the building blocks for database metadata. + +Each element within this module describes a database entity which can be +created and dropped, or is otherwise part of such an entity. Examples include +tables, columns, sequences, and indexes. + +All entities are subclasses of :class:`~sqlalchemy.schema.SchemaItem`, and as +defined in this module they are intended to be agnostic of any vendor-specific +constructs. + +A collection of entities are grouped into a unit called +:class:`~sqlalchemy.schema.MetaData`. MetaData serves as a logical grouping of +schema elements, and can also be associated with an actual database connection +such that operations involving the contained elements can contact the database +as needed. + +Two of the elements here also build upon their "syntactic" counterparts, which +are defined in :class:`~sqlalchemy.sql.expression.`, specifically +:class:`~sqlalchemy.schema.Table` and :class:`~sqlalchemy.schema.Column`. +Since these objects are part of the SQL expression language, they are usable +as components in SQL expressions. + +""" + +import re +import inspect +from .. import exc, util, event, inspection +from .base import SchemaEventTarget +from . import visitors +from . import type_api +from .base import _bind_or_error, ColumnCollection +from .elements import ClauseElement, ColumnClause, _truncated_label, \ + _as_truncated, TextClause, _literal_as_text,\ + ColumnElement, _find_columns, quoted_name +from .selectable import TableClause +import collections +import sqlalchemy +from . import ddl + +RETAIN_SCHEMA = util.symbol('retain_schema') + + +def _get_table_key(name, schema): + if schema is None: + return name + else: + return schema + "." + name + + +def _validate_dialect_kwargs(kwargs, name): + # validate remaining kwargs that they all specify DB prefixes + + for k in kwargs: + m = re.match('^(.+?)_.*', k) + if m is None: + raise TypeError("Additional arguments should be " + "named _, got '%s'" % k) + +@inspection._self_inspects +class SchemaItem(SchemaEventTarget, visitors.Visitable): + """Base class for items that define a database schema.""" + + __visit_name__ = 'schema_item' + + def _execute_on_connection(self, connection, multiparams, params): + return connection._execute_default(self, multiparams, params) + + def _init_items(self, *args): + """Initialize the list of child items for this SchemaItem.""" + + for item in args: + if item is not None: + item._set_parent_with_dispatch(self) + + def get_children(self, **kwargs): + """used to allow SchemaVisitor access""" + return [] + + def __repr__(self): + return util.generic_repr(self) + + @property + @util.deprecated('0.9', 'Use ``.name.quote``') + def quote(self): + """Return the value of the ``quote`` flag passed + to this schema object, for those schema items which + have a ``name`` field. + + """ + + return self.name.quote + + @util.memoized_property + def info(self): + """Info dictionary associated with the object, allowing user-defined + data to be associated with this :class:`.SchemaItem`. + + The dictionary is automatically generated when first accessed. + It can also be specified in the constructor of some objects, + such as :class:`.Table` and :class:`.Column`. + + """ + return {} + + def _schema_item_copy(self, schema_item): + if 'info' in self.__dict__: + schema_item.info = self.info.copy() + schema_item.dispatch._update(self.dispatch) + return schema_item + + +class Table(SchemaItem, TableClause): + """Represent a table in a database. + + e.g.:: + + mytable = Table("mytable", metadata, + Column('mytable_id', Integer, primary_key=True), + Column('value', String(50)) + ) + + The :class:`.Table` object constructs a unique instance of itself based + on its name and optional schema name within the given + :class:`.MetaData` object. Calling the :class:`.Table` + constructor with the same name and same :class:`.MetaData` argument + a second time will return the *same* :class:`.Table` object - in this way + the :class:`.Table` constructor acts as a registry function. + + .. seealso:: + + :ref:`metadata_describing` - Introduction to database metadata + + Constructor arguments are as follows: + + :param name: The name of this table as represented in the database. + + The table name, along with the value of the ``schema`` parameter, + forms a key which uniquely identifies this :class:`.Table` within + the owning :class:`.MetaData` collection. + Additional calls to :class:`.Table` with the same name, metadata, + and schema name will return the same :class:`.Table` object. + + Names which contain no upper case characters + will be treated as case insensitive names, and will not be quoted + unless they are a reserved word or contain special characters. + A name with any number of upper case characters is considered + to be case sensitive, and will be sent as quoted. + + To enable unconditional quoting for the table name, specify the flag + ``quote=True`` to the constructor, or use the :class:`.quoted_name` + construct to specify the name. + + :param metadata: a :class:`.MetaData` object which will contain this + table. The metadata is used as a point of association of this table + with other tables which are referenced via foreign key. It also + may be used to associate this table with a particular + :class:`.Connectable`. + + :param \*args: Additional positional arguments are used primarily + to add the list of :class:`.Column` objects contained within this + table. Similar to the style of a CREATE TABLE statement, other + :class:`.SchemaItem` constructs may be added here, including + :class:`.PrimaryKeyConstraint`, and :class:`.ForeignKeyConstraint`. + + :param autoload: Defaults to False: the Columns for this table should + be reflected from the database. Usually there will be no Column + objects in the constructor if this property is set. + + :param autoload_replace: If ``True``, when using ``autoload=True`` + and ``extend_existing=True``, + replace ``Column`` objects already present in the ``Table`` that's + in the ``MetaData`` registry with + what's reflected. Otherwise, all existing columns will be + excluded from the reflection process. Note that this does + not impact ``Column`` objects specified in the same call to ``Table`` + which includes ``autoload``, those always take precedence. + Defaults to ``True``. + + .. versionadded:: 0.7.5 + + :param autoload_with: If autoload==True, this is an optional Engine + or Connection instance to be used for the table reflection. If + ``None``, the underlying MetaData's bound connectable will be used. + + :param extend_existing: When ``True``, indicates that if this + :class:`.Table` is already present in the given :class:`.MetaData`, + apply further arguments within the constructor to the existing + :class:`.Table`. + + If ``extend_existing`` or ``keep_existing`` are not set, an error is + raised if additional table modifiers are specified when + the given :class:`.Table` is already present in the :class:`.MetaData`. + + .. versionchanged:: 0.7.4 + ``extend_existing`` will work in conjunction + with ``autoload=True`` to run a new reflection operation against + the database; new :class:`.Column` objects will be produced + from database metadata to replace those existing with the same + name, and additional :class:`.Column` objects not present + in the :class:`.Table` will be added. + + As is always the case with ``autoload=True``, :class:`.Column` + objects can be specified in the same :class:`.Table` constructor, + which will take precedence. I.e.:: + + Table("mytable", metadata, + Column('y', Integer), + extend_existing=True, + autoload=True, + autoload_with=engine + ) + + The above will overwrite all columns within ``mytable`` which + are present in the database, except for ``y`` which will be used as is + from the above definition. If the ``autoload_replace`` flag + is set to False, no existing columns will be replaced. + + :param implicit_returning: True by default - indicates that + RETURNING can be used by default to fetch newly inserted primary key + values, for backends which support this. Note that + create_engine() also provides an implicit_returning flag. + + :param include_columns: A list of strings indicating a subset of + columns to be loaded via the ``autoload`` operation; table columns who + aren't present in this list will not be represented on the resulting + ``Table`` object. Defaults to ``None`` which indicates all columns + should be reflected. + + :param info: Optional data dictionary which will be populated into the + :attr:`.SchemaItem.info` attribute of this object. + + :param keep_existing: When ``True``, indicates that if this Table + is already present in the given :class:`.MetaData`, ignore + further arguments within the constructor to the existing + :class:`.Table`, and return the :class:`.Table` object as + originally created. This is to allow a function that wishes + to define a new :class:`.Table` on first call, but on + subsequent calls will return the same :class:`.Table`, + without any of the declarations (particularly constraints) + being applied a second time. Also see extend_existing. + + If extend_existing or keep_existing are not set, an error is + raised if additional table modifiers are specified when + the given :class:`.Table` is already present in the :class:`.MetaData`. + + :param listeners: A list of tuples of the form ``(, )`` + which will be passed to :func:`.event.listen` upon construction. + This alternate hook to :func:`.event.listen` allows the establishment + of a listener function specific to this :class:`.Table` before + the "autoload" process begins. Particularly useful for + the :meth:`.DDLEvents.column_reflect` event:: + + def listen_for_reflect(table, column_info): + "handle the column reflection event" + # ... + + t = Table( + 'sometable', + autoload=True, + listeners=[ + ('column_reflect', listen_for_reflect) + ]) + + :param mustexist: When ``True``, indicates that this Table must already + be present in the given :class:`.MetaData` collection, else + an exception is raised. + + :param prefixes: + A list of strings to insert after CREATE in the CREATE TABLE + statement. They will be separated by spaces. + + :param quote: Force quoting of this table's name on or off, corresponding + to ``True`` or ``False``. When left at its default of ``None``, + the column identifier will be quoted according to whether the name is + case sensitive (identifiers with at least one upper case character are + treated as case sensitive), or if it's a reserved word. This flag + is only needed to force quoting of a reserved word which is not known + by the SQLAlchemy dialect. + + :param quote_schema: same as 'quote' but applies to the schema identifier. + + :param schema: The schema name for this table, which is required if + the table resides in a schema other than the default selected schema + for the engine's database connection. Defaults to ``None``. + + The quoting rules for the schema name are the same as those for the + ``name`` parameter, in that quoting is applied for reserved words or + case-sensitive names; to enable unconditional quoting for the + schema name, specify the flag + ``quote_schema=True`` to the constructor, or use the :class:`.quoted_name` + construct to specify the name. + + + :param useexisting: Deprecated. Use extend_existing. + + """ + + __visit_name__ = 'table' + + def __new__(cls, *args, **kw): + if not args: + # python3k pickle seems to call this + return object.__new__(cls) + + try: + name, metadata, args = args[0], args[1], args[2:] + except IndexError: + raise TypeError("Table() takes at least two arguments") + + schema = kw.get('schema', None) + if schema is None: + schema = metadata.schema + keep_existing = kw.pop('keep_existing', False) + extend_existing = kw.pop('extend_existing', False) + if 'useexisting' in kw: + msg = "useexisting is deprecated. Use extend_existing." + util.warn_deprecated(msg) + if extend_existing: + msg = "useexisting is synonymous with extend_existing." + raise exc.ArgumentError(msg) + extend_existing = kw.pop('useexisting', False) + + if keep_existing and extend_existing: + msg = "keep_existing and extend_existing are mutually exclusive." + raise exc.ArgumentError(msg) + + mustexist = kw.pop('mustexist', False) + key = _get_table_key(name, schema) + if key in metadata.tables: + if not keep_existing and not extend_existing and bool(args): + raise exc.InvalidRequestError( + "Table '%s' is already defined for this MetaData " + "instance. Specify 'extend_existing=True' " + "to redefine " + "options and columns on an " + "existing Table object." % key) + table = metadata.tables[key] + if extend_existing: + table._init_existing(*args, **kw) + return table + else: + if mustexist: + raise exc.InvalidRequestError( + "Table '%s' not defined" % (key)) + table = object.__new__(cls) + table.dispatch.before_parent_attach(table, metadata) + metadata._add_table(name, schema, table) + try: + table._init(name, metadata, *args, **kw) + table.dispatch.after_parent_attach(table, metadata) + return table + except: + #metadata._remove_table(name, schema) + raise + + + @property + @util.deprecated('0.9', 'Use ``table.schema.quote``') + def quote_schema(self): + """Return the value of the ``quote_schema`` flag passed + to this :class:`.Table`. + """ + + return self.schema.quote + + def __init__(self, *args, **kw): + """Constructor for :class:`~.schema.Table`. + + This method is a no-op. See the top-level + documentation for :class:`~.schema.Table` + for constructor arguments. + + """ + # __init__ is overridden to prevent __new__ from + # calling the superclass constructor. + + def _init(self, name, metadata, *args, **kwargs): + super(Table, self).__init__(quoted_name(name, kwargs.pop('quote', None))) + self.metadata = metadata + + self.schema = kwargs.pop('schema', None) + if self.schema is None: + self.schema = metadata.schema + else: + quote_schema = kwargs.pop('quote_schema', None) + self.schema = quoted_name(self.schema, quote_schema) + + self.indexes = set() + self.constraints = set() + self._columns = ColumnCollection() + PrimaryKeyConstraint()._set_parent_with_dispatch(self) + self.foreign_keys = set() + self._extra_dependencies = set() + self.kwargs = {} + if self.schema is not None: + self.fullname = "%s.%s" % (self.schema, self.name) + else: + self.fullname = self.name + + autoload = kwargs.pop('autoload', False) + autoload_with = kwargs.pop('autoload_with', None) + # this argument is only used with _init_existing() + kwargs.pop('autoload_replace', True) + include_columns = kwargs.pop('include_columns', None) + + self.implicit_returning = kwargs.pop('implicit_returning', True) + + if 'info' in kwargs: + self.info = kwargs.pop('info') + if 'listeners' in kwargs: + listeners = kwargs.pop('listeners') + for evt, fn in listeners: + event.listen(self, evt, fn) + + self._prefixes = kwargs.pop('prefixes', []) + + self._extra_kwargs(**kwargs) + + # load column definitions from the database if 'autoload' is defined + # we do it after the table is in the singleton dictionary to support + # circular foreign keys + if autoload: + self._autoload(metadata, autoload_with, include_columns) + + # initialize all the column, etc. objects. done after reflection to + # allow user-overrides + self._init_items(*args) + + def _autoload(self, metadata, autoload_with, include_columns, + exclude_columns=()): + if self.primary_key.columns: + PrimaryKeyConstraint(*[ + c for c in self.primary_key.columns + if c.key in exclude_columns + ])._set_parent_with_dispatch(self) + + if autoload_with: + autoload_with.run_callable( + autoload_with.dialect.reflecttable, + self, include_columns, exclude_columns + ) + else: + bind = _bind_or_error(metadata, + msg="No engine is bound to this Table's MetaData. " + "Pass an engine to the Table via " + "autoload_with=, " + "or associate the MetaData with an engine via " + "metadata.bind=") + bind.run_callable( + bind.dialect.reflecttable, + self, include_columns, exclude_columns + ) + + @property + def _sorted_constraints(self): + """Return the set of constraints as a list, sorted by creation + order. + + """ + return sorted(self.constraints, key=lambda c: c._creation_order) + + def _init_existing(self, *args, **kwargs): + autoload = kwargs.pop('autoload', False) + autoload_with = kwargs.pop('autoload_with', None) + autoload_replace = kwargs.pop('autoload_replace', True) + schema = kwargs.pop('schema', None) + if schema and schema != self.schema: + raise exc.ArgumentError( + "Can't change schema of existing table from '%s' to '%s'", + (self.schema, schema)) + + include_columns = kwargs.pop('include_columns', None) + + if include_columns is not None: + for c in self.c: + if c.name not in include_columns: + self._columns.remove(c) + + for key in ('quote', 'quote_schema'): + if key in kwargs: + raise exc.ArgumentError( + "Can't redefine 'quote' or 'quote_schema' arguments") + + if 'info' in kwargs: + self.info = kwargs.pop('info') + + if autoload: + if not autoload_replace: + exclude_columns = [c.name for c in self.c] + else: + exclude_columns = () + self._autoload( + self.metadata, autoload_with, include_columns, exclude_columns) + + self._extra_kwargs(**kwargs) + self._init_items(*args) + + def _extra_kwargs(self, **kwargs): + # validate remaining kwargs that they all specify DB prefixes + _validate_dialect_kwargs(kwargs, "Table") + self.kwargs.update(kwargs) + + def _init_collections(self): + pass + + @util.memoized_property + def _autoincrement_column(self): + for col in self.primary_key: + if col.autoincrement and \ + col.type._type_affinity is not None and \ + issubclass(col.type._type_affinity, type_api.INTEGERTYPE._type_affinity) and \ + (not col.foreign_keys or col.autoincrement == 'ignore_fk') and \ + isinstance(col.default, (type(None), Sequence)) and \ + (col.server_default is None or col.server_default.reflected): + return col + + @property + def key(self): + """Return the 'key' for this :class:`.Table`. + + This value is used as the dictionary key within the + :attr:`.MetaData.tables` collection. It is typically the same + as that of :attr:`.Table.name` for a table with no :attr:`.Table.schema` + set; otherwise it is typically of the form ``schemaname.tablename``. + + """ + return _get_table_key(self.name, self.schema) + + def __repr__(self): + return "Table(%s)" % ', '.join( + [repr(self.name)] + [repr(self.metadata)] + + [repr(x) for x in self.columns] + + ["%s=%s" % (k, repr(getattr(self, k))) for k in ['schema']]) + + def __str__(self): + return _get_table_key(self.description, self.schema) + + @property + def bind(self): + """Return the connectable associated with this Table.""" + + return self.metadata and self.metadata.bind or None + + def add_is_dependent_on(self, table): + """Add a 'dependency' for this Table. + + This is another Table object which must be created + first before this one can, or dropped after this one. + + Usually, dependencies between tables are determined via + ForeignKey objects. However, for other situations that + create dependencies outside of foreign keys (rules, inheriting), + this method can manually establish such a link. + + """ + self._extra_dependencies.add(table) + + def append_column(self, column): + """Append a :class:`~.schema.Column` to this :class:`~.schema.Table`. + + The "key" of the newly added :class:`~.schema.Column`, i.e. the + value of its ``.key`` attribute, will then be available + in the ``.c`` collection of this :class:`~.schema.Table`, and the + column definition will be included in any CREATE TABLE, SELECT, + UPDATE, etc. statements generated from this :class:`~.schema.Table` + construct. + + Note that this does **not** change the definition of the table + as it exists within any underlying database, assuming that + table has already been created in the database. Relational + databases support the addition of columns to existing tables + using the SQL ALTER command, which would need to be + emitted for an already-existing table that doesn't contain + the newly added column. + + """ + + column._set_parent_with_dispatch(self) + + def append_constraint(self, constraint): + """Append a :class:`~.schema.Constraint` to this + :class:`~.schema.Table`. + + This has the effect of the constraint being included in any + future CREATE TABLE statement, assuming specific DDL creation + events have not been associated with the given + :class:`~.schema.Constraint` object. + + Note that this does **not** produce the constraint within the + relational database automatically, for a table that already exists + in the database. To add a constraint to an + existing relational database table, the SQL ALTER command must + be used. SQLAlchemy also provides the + :class:`.AddConstraint` construct which can produce this SQL when + invoked as an executable clause. + + """ + + constraint._set_parent_with_dispatch(self) + + def append_ddl_listener(self, event_name, listener): + """Append a DDL event listener to this ``Table``. + + .. deprecated:: 0.7 + See :class:`.DDLEvents`. + + """ + + def adapt_listener(target, connection, **kw): + listener(event_name, target, connection) + + event.listen(self, "" + event_name.replace('-', '_'), adapt_listener) + + def _set_parent(self, metadata): + metadata._add_table(self.name, self.schema, self) + self.metadata = metadata + + def get_children(self, column_collections=True, + schema_visitor=False, **kw): + if not schema_visitor: + return TableClause.get_children( + self, column_collections=column_collections, **kw) + else: + if column_collections: + return list(self.columns) + else: + return [] + + def exists(self, bind=None): + """Return True if this table exists.""" + + if bind is None: + bind = _bind_or_error(self) + + return bind.run_callable(bind.dialect.has_table, + self.name, schema=self.schema) + + def create(self, bind=None, checkfirst=False): + """Issue a ``CREATE`` statement for this + :class:`.Table`, using the given :class:`.Connectable` + for connectivity. + + .. seealso:: + + :meth:`.MetaData.create_all`. + + """ + + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaGenerator, + self, + checkfirst=checkfirst) + + def drop(self, bind=None, checkfirst=False): + """Issue a ``DROP`` statement for this + :class:`.Table`, using the given :class:`.Connectable` + for connectivity. + + .. seealso:: + + :meth:`.MetaData.drop_all`. + + """ + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaDropper, + self, + checkfirst=checkfirst) + + def tometadata(self, metadata, schema=RETAIN_SCHEMA): + """Return a copy of this :class:`.Table` associated with a different + :class:`.MetaData`. + + E.g.:: + + some_engine = create_engine("sqlite:///some.db") + + # create two metadata + meta1 = MetaData() + meta2 = MetaData() + + # load 'users' from the sqlite engine + users_table = Table('users', meta1, autoload=True, + autoload_with=some_engine) + + # create the same Table object for the plain metadata + users_table_2 = users_table.tometadata(meta2) + + :param metadata: Target :class:`.MetaData` object. + :param schema: Optional string name of a target schema, or + ``None`` for no schema. The :class:`.Table` object will be + given this schema name upon copy. Defaults to the special + symbol :attr:`.RETAIN_SCHEMA` which indicates no change should be + made to the schema name of the resulting :class:`.Table`. + + """ + + if schema is RETAIN_SCHEMA: + schema = self.schema + elif schema is None: + schema = metadata.schema + key = _get_table_key(self.name, schema) + if key in metadata.tables: + util.warn("Table '%s' already exists within the given " + "MetaData - not copying." % self.description) + return metadata.tables[key] + + args = [] + for c in self.columns: + args.append(c.copy(schema=schema)) + table = Table( + self.name, metadata, schema=schema, + *args, **self.kwargs + ) + for c in self.constraints: + table.append_constraint(c.copy(schema=schema, target_table=table)) + + for index in self.indexes: + # skip indexes that would be generated + # by the 'index' flag on Column + if len(index.columns) == 1 and \ + list(index.columns)[0].index: + continue + Index(index.name, + unique=index.unique, + *[table.c[col] for col in index.columns.keys()], + **index.kwargs) + return self._schema_item_copy(table) + + +class Column(SchemaItem, ColumnClause): + """Represents a column in a database table.""" + + __visit_name__ = 'column' + + def __init__(self, *args, **kwargs): + """ + Construct a new ``Column`` object. + + :param name: The name of this column as represented in the database. + This argument may be the first positional argument, or specified + via keyword. + + Names which contain no upper case characters + will be treated as case insensitive names, and will not be quoted + unless they are a reserved word. Names with any number of upper + case characters will be quoted and sent exactly. Note that this + behavior applies even for databases which standardize upper + case names as case insensitive such as Oracle. + + The name field may be omitted at construction time and applied + later, at any time before the Column is associated with a + :class:`.Table`. This is to support convenient + usage within the :mod:`~sqlalchemy.ext.declarative` extension. + + :param type\_: The column's type, indicated using an instance which + subclasses :class:`~sqlalchemy.types.TypeEngine`. If no arguments + are required for the type, the class of the type can be sent + as well, e.g.:: + + # use a type with arguments + Column('data', String(50)) + + # use no arguments + Column('level', Integer) + + The ``type`` argument may be the second positional argument + or specified by keyword. + + If the ``type`` is ``None`` or is omitted, it will first default to the special + type :class:`.NullType`. If and when this :class:`.Column` is + made to refer to another column using :class:`.ForeignKey` + and/or :class:`.ForeignKeyConstraint`, the type of the remote-referenced + column will be copied to this column as well, at the moment that + the foreign key is resolved against that remote :class:`.Column` + object. + + .. versionchanged:: 0.9.0 + Support for propagation of type to a :class:`.Column` from its + :class:`.ForeignKey` object has been improved and should be + more reliable and timely. + + :param \*args: Additional positional arguments include various + :class:`.SchemaItem` derived constructs which will be applied + as options to the column. These include instances of + :class:`.Constraint`, :class:`.ForeignKey`, :class:`.ColumnDefault`, + and :class:`.Sequence`. In some cases an equivalent keyword + argument is available such as ``server_default``, ``default`` + and ``unique``. + + :param autoincrement: This flag may be set to ``False`` to + indicate an integer primary key column that should not be + considered to be the "autoincrement" column, that is + the integer primary key column which generates values + implicitly upon INSERT and whose value is usually returned + via the DBAPI cursor.lastrowid attribute. It defaults + to ``True`` to satisfy the common use case of a table + with a single integer primary key column. If the table + has a composite primary key consisting of more than one + integer column, set this flag to True only on the + column that should be considered "autoincrement". + + The setting *only* has an effect for columns which are: + + * Integer derived (i.e. INT, SMALLINT, BIGINT). + + * Part of the primary key + + * Are not referenced by any foreign keys, unless + the value is specified as ``'ignore_fk'`` + + .. versionadded:: 0.7.4 + + * have no server side or client side defaults (with the exception + of Postgresql SERIAL). + + The setting has these two effects on columns that meet the + above criteria: + + * DDL issued for the column will include database-specific + keywords intended to signify this column as an + "autoincrement" column, such as AUTO INCREMENT on MySQL, + SERIAL on Postgresql, and IDENTITY on MS-SQL. It does + *not* issue AUTOINCREMENT for SQLite since this is a + special SQLite flag that is not required for autoincrementing + behavior. See the SQLite dialect documentation for + information on SQLite's AUTOINCREMENT. + + * The column will be considered to be available as + cursor.lastrowid or equivalent, for those dialects which + "post fetch" newly inserted identifiers after a row has + been inserted (SQLite, MySQL, MS-SQL). It does not have + any effect in this regard for databases that use sequences + to generate primary key identifiers (i.e. Firebird, Postgresql, + Oracle). + + .. versionchanged:: 0.7.4 + ``autoincrement`` accepts a special value ``'ignore_fk'`` + to indicate that autoincrementing status regardless of foreign + key references. This applies to certain composite foreign key + setups, such as the one demonstrated in the ORM documentation + at :ref:`post_update`. + + :param default: A scalar, Python callable, or + :class:`.ColumnElement` expression representing the + *default value* for this column, which will be invoked upon insert + if this column is otherwise not specified in the VALUES clause of + the insert. This is a shortcut to using :class:`.ColumnDefault` as + a positional argument; see that class for full detail on the + structure of the argument. + + Contrast this argument to ``server_default`` which creates a + default generator on the database side. + + :param doc: optional String that can be used by the ORM or similar + to document attributes. This attribute does not render SQL + comments (a future attribute 'comment' will achieve that). + + :param key: An optional string identifier which will identify this + ``Column`` object on the :class:`.Table`. When a key is provided, + this is the only identifier referencing the ``Column`` within the + application, including ORM attribute mapping; the ``name`` field + is used only when rendering SQL. + + :param index: When ``True``, indicates that the column is indexed. + This is a shortcut for using a :class:`.Index` construct on the + table. To specify indexes with explicit names or indexes that + contain multiple columns, use the :class:`.Index` construct + instead. + + :param info: Optional data dictionary which will be populated into the + :attr:`.SchemaItem.info` attribute of this object. + + :param nullable: If set to the default of ``True``, indicates the + column will be rendered as allowing NULL, else it's rendered as + NOT NULL. This parameter is only used when issuing CREATE TABLE + statements. + + :param onupdate: A scalar, Python callable, or + :class:`~sqlalchemy.sql.expression.ClauseElement` representing a + default value to be applied to the column within UPDATE + statements, which wil be invoked upon update if this column is not + present in the SET clause of the update. This is a shortcut to + using :class:`.ColumnDefault` as a positional argument with + ``for_update=True``. + + :param primary_key: If ``True``, marks this column as a primary key + column. Multiple columns can have this flag set to specify + composite primary keys. As an alternative, the primary key of a + :class:`.Table` can be specified via an explicit + :class:`.PrimaryKeyConstraint` object. + + :param server_default: A :class:`.FetchedValue` instance, str, Unicode + or :func:`~sqlalchemy.sql.expression.text` construct representing + the DDL DEFAULT value for the column. + + String types will be emitted as-is, surrounded by single quotes:: + + Column('x', Text, server_default="val") + + x TEXT DEFAULT 'val' + + A :func:`~sqlalchemy.sql.expression.text` expression will be + rendered as-is, without quotes:: + + Column('y', DateTime, server_default=text('NOW()')) + + y DATETIME DEFAULT NOW() + + Strings and text() will be converted into a :class:`.DefaultClause` + object upon initialization. + + Use :class:`.FetchedValue` to indicate that an already-existing + column will generate a default value on the database side which + will be available to SQLAlchemy for post-fetch after inserts. This + construct does not specify any DDL and the implementation is left + to the database, such as via a trigger. + + :param server_onupdate: A :class:`.FetchedValue` instance + representing a database-side default generation function. This + indicates to SQLAlchemy that a newly generated value will be + available after updates. This construct does not specify any DDL + and the implementation is left to the database, such as via a + trigger. + + :param quote: Force quoting of this column's name on or off, + corresponding to ``True`` or ``False``. When left at its default + of ``None``, the column identifier will be quoted according to + whether the name is case sensitive (identifiers with at least one + upper case character are treated as case sensitive), or if it's a + reserved word. This flag is only needed to force quoting of a + reserved word which is not known by the SQLAlchemy dialect. + + :param unique: When ``True``, indicates that this column contains a + unique constraint, or if ``index`` is ``True`` as well, indicates + that the :class:`.Index` should be created with the unique flag. + To specify multiple columns in the constraint/index or to specify + an explicit name, use the :class:`.UniqueConstraint` or + :class:`.Index` constructs explicitly. + + :param system: When ``True``, indicates this is a "system" column, + that is a column which is automatically made available by the + database, and should not be included in the columns list for a + ``CREATE TABLE`` statement. + + For more elaborate scenarios where columns should be conditionally + rendered differently on different backends, consider custom + compilation rules for :class:`.CreateColumn`. + + ..versionadded:: 0.8.3 Added the ``system=True`` parameter to + :class:`.Column`. + + """ + + name = kwargs.pop('name', None) + type_ = kwargs.pop('type_', None) + args = list(args) + if args: + if isinstance(args[0], util.string_types): + if name is not None: + raise exc.ArgumentError( + "May not pass name positionally and as a keyword.") + name = args.pop(0) + if args: + coltype = args[0] + + if hasattr(coltype, "_sqla_type"): + if type_ is not None: + raise exc.ArgumentError( + "May not pass type_ positionally and as a keyword.") + type_ = args.pop(0) + + if name is not None: + name = quoted_name(name, kwargs.pop('quote', None)) + elif "quote" in kwargs: + raise exc.ArgumentError("Explicit 'name' is required when " + "sending 'quote' argument") + + super(Column, self).__init__(name, type_) + self.key = kwargs.pop('key', name) + self.primary_key = kwargs.pop('primary_key', False) + self.nullable = kwargs.pop('nullable', not self.primary_key) + self.default = kwargs.pop('default', None) + self.server_default = kwargs.pop('server_default', None) + self.server_onupdate = kwargs.pop('server_onupdate', None) + + # these default to None because .index and .unique is *not* + # an informational flag about Column - there can still be an + # Index or UniqueConstraint referring to this Column. + self.index = kwargs.pop('index', None) + self.unique = kwargs.pop('unique', None) + + self.system = kwargs.pop('system', False) + self.doc = kwargs.pop('doc', None) + self.onupdate = kwargs.pop('onupdate', None) + self.autoincrement = kwargs.pop('autoincrement', True) + self.constraints = set() + self.foreign_keys = set() + + # check if this Column is proxying another column + if '_proxies' in kwargs: + self._proxies = kwargs.pop('_proxies') + # otherwise, add DDL-related events + elif isinstance(self.type, SchemaEventTarget): + self.type._set_parent_with_dispatch(self) + + if self.default is not None: + if isinstance(self.default, (ColumnDefault, Sequence)): + args.append(self.default) + else: + if getattr(self.type, '_warn_on_bytestring', False): + if isinstance(self.default, util.binary_type): + util.warn("Unicode column received non-unicode " + "default value.") + args.append(ColumnDefault(self.default)) + + if self.server_default is not None: + if isinstance(self.server_default, FetchedValue): + args.append(self.server_default._as_for_update(False)) + else: + args.append(DefaultClause(self.server_default)) + + if self.onupdate is not None: + if isinstance(self.onupdate, (ColumnDefault, Sequence)): + args.append(self.onupdate) + else: + args.append(ColumnDefault(self.onupdate, for_update=True)) + + if self.server_onupdate is not None: + if isinstance(self.server_onupdate, FetchedValue): + args.append(self.server_onupdate._as_for_update(True)) + else: + args.append(DefaultClause(self.server_onupdate, + for_update=True)) + self._init_items(*args) + + util.set_creation_order(self) + + if 'info' in kwargs: + self.info = kwargs.pop('info') + + if kwargs: + raise exc.ArgumentError( + "Unknown arguments passed to Column: " + repr(list(kwargs))) + +# @property +# def quote(self): +# return getattr(self.name, "quote", None) + + def __str__(self): + if self.name is None: + return "(no name)" + elif self.table is not None: + if self.table.named_with_column: + return (self.table.description + "." + self.description) + else: + return self.description + else: + return self.description + + def references(self, column): + """Return True if this Column references the given column via foreign + key.""" + + for fk in self.foreign_keys: + if fk.column.proxy_set.intersection(column.proxy_set): + return True + else: + return False + + def append_foreign_key(self, fk): + fk._set_parent_with_dispatch(self) + + def __repr__(self): + kwarg = [] + if self.key != self.name: + kwarg.append('key') + if self.primary_key: + kwarg.append('primary_key') + if not self.nullable: + kwarg.append('nullable') + if self.onupdate: + kwarg.append('onupdate') + if self.default: + kwarg.append('default') + if self.server_default: + kwarg.append('server_default') + return "Column(%s)" % ', '.join( + [repr(self.name)] + [repr(self.type)] + + [repr(x) for x in self.foreign_keys if x is not None] + + [repr(x) for x in self.constraints] + + [(self.table is not None and "table=<%s>" % + self.table.description or "table=None")] + + ["%s=%s" % (k, repr(getattr(self, k))) for k in kwarg]) + + def _set_parent(self, table): + if not self.name: + raise exc.ArgumentError( + "Column must be constructed with a non-blank name or " + "assign a non-blank .name before adding to a Table.") + if self.key is None: + self.key = self.name + + existing = getattr(self, 'table', None) + if existing is not None and existing is not table: + raise exc.ArgumentError( + "Column object already assigned to Table '%s'" % + existing.description) + + if self.key in table._columns: + col = table._columns.get(self.key) + if col is not self: + for fk in col.foreign_keys: + table.foreign_keys.remove(fk) + if fk.constraint in table.constraints: + # this might have been removed + # already, if it's a composite constraint + # and more than one col being replaced + table.constraints.remove(fk.constraint) + + table._columns.replace(self) + + if self.primary_key: + table.primary_key._replace(self) + Table._autoincrement_column._reset(table) + elif self.key in table.primary_key: + raise exc.ArgumentError( + "Trying to redefine primary-key column '%s' as a " + "non-primary-key column on table '%s'" % ( + self.key, table.fullname)) + self.table = table + + if self.index: + if isinstance(self.index, util.string_types): + raise exc.ArgumentError( + "The 'index' keyword argument on Column is boolean only. " + "To create indexes with a specific name, create an " + "explicit Index object external to the Table.") + Index(_truncated_label('ix_%s' % self._label), + self, unique=bool(self.unique)) + elif self.unique: + if isinstance(self.unique, util.string_types): + raise exc.ArgumentError( + "The 'unique' keyword argument on Column is boolean " + "only. To create unique constraints or indexes with a " + "specific name, append an explicit UniqueConstraint to " + "the Table's list of elements, or create an explicit " + "Index object external to the Table.") + table.append_constraint(UniqueConstraint(self.key)) + + fk_key = (table.key, self.key) + if fk_key in self.table.metadata._fk_memos: + for fk in self.table.metadata._fk_memos[fk_key]: + fk._set_remote_table(table) + + def _on_table_attach(self, fn): + if self.table is not None: + fn(self, self.table) + event.listen(self, 'after_parent_attach', fn) + + def copy(self, **kw): + """Create a copy of this ``Column``, unitialized. + + This is used in ``Table.tometadata``. + + """ + + # Constraint objects plus non-constraint-bound ForeignKey objects + args = \ + [c.copy(**kw) for c in self.constraints] + \ + [c.copy(**kw) for c in self.foreign_keys if not c.constraint] + + type_ = self.type + if isinstance(type_, SchemaEventTarget): + type_ = type_.copy(**kw) + + c = self._constructor( + name=self.name, + type_=type_, + key=self.key, + primary_key=self.primary_key, + nullable=self.nullable, + unique=self.unique, + system=self.system, + #quote=self.quote, + index=self.index, + autoincrement=self.autoincrement, + default=self.default, + server_default=self.server_default, + onupdate=self.onupdate, + server_onupdate=self.server_onupdate, + doc=self.doc, + *args + ) + return self._schema_item_copy(c) + + def _make_proxy(self, selectable, name=None, key=None, + name_is_truncatable=False, **kw): + """Create a *proxy* for this column. + + This is a copy of this ``Column`` referenced by a different parent + (such as an alias or select statement). The column should + be used only in select scenarios, as its full DDL/default + information is not transferred. + + """ + fk = [ForeignKey(f.column, _constraint=f.constraint) + for f in self.foreign_keys] + if name is None and self.name is None: + raise exc.InvalidRequestError("Cannot initialize a sub-selectable" + " with this Column object until it's 'name' has " + "been assigned.") + try: + c = self._constructor( + _as_truncated(name or self.name) if \ + name_is_truncatable else (name or self.name), + self.type, + key=key if key else name if name else self.key, + primary_key=self.primary_key, + nullable=self.nullable, + _proxies=[self], *fk) + except TypeError: + util.raise_from_cause( + TypeError( + "Could not create a copy of this %r object. " + "Ensure the class includes a _constructor() " + "attribute or method which accepts the " + "standard Column constructor arguments, or " + "references the Column class itself." % self.__class__) + ) + + c.table = selectable + selectable._columns.add(c) + if selectable._is_clone_of is not None: + c._is_clone_of = selectable._is_clone_of.columns[c.key] + if self.primary_key: + selectable.primary_key.add(c) + c.dispatch.after_parent_attach(c, selectable) + return c + + def get_children(self, schema_visitor=False, **kwargs): + if schema_visitor: + return [x for x in (self.default, self.onupdate) + if x is not None] + \ + list(self.foreign_keys) + list(self.constraints) + else: + return ColumnClause.get_children(self, **kwargs) + + +class ForeignKey(SchemaItem): + """Defines a dependency between two columns. + + ``ForeignKey`` is specified as an argument to a :class:`.Column` object, + e.g.:: + + t = Table("remote_table", metadata, + Column("remote_id", ForeignKey("main_table.id")) + ) + + Note that ``ForeignKey`` is only a marker object that defines + a dependency between two columns. The actual constraint + is in all cases represented by the :class:`.ForeignKeyConstraint` + object. This object will be generated automatically when + a ``ForeignKey`` is associated with a :class:`.Column` which + in turn is associated with a :class:`.Table`. Conversely, + when :class:`.ForeignKeyConstraint` is applied to a :class:`.Table`, + ``ForeignKey`` markers are automatically generated to be + present on each associated :class:`.Column`, which are also + associated with the constraint object. + + Note that you cannot define a "composite" foreign key constraint, + that is a constraint between a grouping of multiple parent/child + columns, using ``ForeignKey`` objects. To define this grouping, + the :class:`.ForeignKeyConstraint` object must be used, and applied + to the :class:`.Table`. The associated ``ForeignKey`` objects + are created automatically. + + The ``ForeignKey`` objects associated with an individual + :class:`.Column` object are available in the `foreign_keys` collection + of that column. + + Further examples of foreign key configuration are in + :ref:`metadata_foreignkeys`. + + """ + + __visit_name__ = 'foreign_key' + + def __init__(self, column, _constraint=None, use_alter=False, name=None, + onupdate=None, ondelete=None, deferrable=None, + initially=None, link_to_name=False, match=None): + """ + Construct a column-level FOREIGN KEY. + + The :class:`.ForeignKey` object when constructed generates a + :class:`.ForeignKeyConstraint` which is associated with the parent + :class:`.Table` object's collection of constraints. + + :param column: A single target column for the key relationship. A + :class:`.Column` object or a column name as a string: + ``tablename.columnkey`` or ``schema.tablename.columnkey``. + ``columnkey`` is the ``key`` which has been assigned to the column + (defaults to the column name itself), unless ``link_to_name`` is + ``True`` in which case the rendered name of the column is used. + + .. versionadded:: 0.7.4 + Note that if the schema name is not included, and the + underlying :class:`.MetaData` has a "schema", that value will + be used. + + :param name: Optional string. An in-database name for the key if + `constraint` is not provided. + + :param onupdate: Optional string. If set, emit ON UPDATE when + issuing DDL for this constraint. Typical values include CASCADE, + DELETE and RESTRICT. + + :param ondelete: Optional string. If set, emit ON DELETE when + issuing DDL for this constraint. Typical values include CASCADE, + DELETE and RESTRICT. + + :param deferrable: Optional bool. If set, emit DEFERRABLE or NOT + DEFERRABLE when issuing DDL for this constraint. + + :param initially: Optional string. If set, emit INITIALLY when + issuing DDL for this constraint. + + :param link_to_name: if True, the string name given in ``column`` is + the rendered name of the referenced column, not its locally + assigned ``key``. + + :param use_alter: passed to the underlying + :class:`.ForeignKeyConstraint` to indicate the constraint should be + generated/dropped externally from the CREATE TABLE/ DROP TABLE + statement. See that classes' constructor for details. + + :param match: Optional string. If set, emit MATCH when issuing + DDL for this constraint. Typical values include SIMPLE, PARTIAL + and FULL. + + """ + + self._colspec = column + if isinstance(self._colspec, util.string_types): + self._table_column = None + else: + if hasattr(self._colspec, '__clause_element__'): + self._table_column = self._colspec.__clause_element__() + else: + self._table_column = self._colspec + + if not isinstance(self._table_column, ColumnClause): + raise exc.ArgumentError( + "String, Column, or Column-bound argument " + "expected, got %r" % self._table_column) + elif not isinstance(self._table_column.table, (util.NoneType, TableClause)): + raise exc.ArgumentError( + "ForeignKey received Column not bound " + "to a Table, got: %r" % self._table_column.table + ) + + # the linked ForeignKeyConstraint. + # ForeignKey will create this when parent Column + # is attached to a Table, *or* ForeignKeyConstraint + # object passes itself in when creating ForeignKey + # markers. + self.constraint = _constraint + self.parent = None + self.use_alter = use_alter + self.name = name + self.onupdate = onupdate + self.ondelete = ondelete + self.deferrable = deferrable + self.initially = initially + self.link_to_name = link_to_name + self.match = match + + def __repr__(self): + return "ForeignKey(%r)" % self._get_colspec() + + def copy(self, schema=None): + """Produce a copy of this :class:`.ForeignKey` object. + + The new :class:`.ForeignKey` will not be bound + to any :class:`.Column`. + + This method is usually used by the internal + copy procedures of :class:`.Column`, :class:`.Table`, + and :class:`.MetaData`. + + :param schema: The returned :class:`.ForeignKey` will + reference the original table and column name, qualified + by the given string schema name. + + """ + + fk = ForeignKey( + self._get_colspec(schema=schema), + use_alter=self.use_alter, + name=self.name, + onupdate=self.onupdate, + ondelete=self.ondelete, + deferrable=self.deferrable, + initially=self.initially, + link_to_name=self.link_to_name, + match=self.match + ) + return self._schema_item_copy(fk) + + + def _get_colspec(self, schema=None): + """Return a string based 'column specification' for this + :class:`.ForeignKey`. + + This is usually the equivalent of the string-based "tablename.colname" + argument first passed to the object's constructor. + + """ + if schema: + _schema, tname, colname = self._column_tokens + return "%s.%s.%s" % (schema, tname, colname) + elif self._table_column is not None: + return "%s.%s" % ( + self._table_column.table.fullname, self._table_column.key) + else: + return self._colspec + + + def _table_key(self): + if self._table_column is not None: + if self._table_column.table is None: + return None + else: + return self._table_column.table.key + else: + schema, tname, colname = self._column_tokens + return _get_table_key(tname, schema) + + + + target_fullname = property(_get_colspec) + + def references(self, table): + """Return True if the given :class:`.Table` is referenced by this + :class:`.ForeignKey`.""" + + return table.corresponding_column(self.column) is not None + + def get_referent(self, table): + """Return the :class:`.Column` in the given :class:`.Table` + referenced by this :class:`.ForeignKey`. + + Returns None if this :class:`.ForeignKey` does not reference the given + :class:`.Table`. + + """ + + return table.corresponding_column(self.column) + + @util.memoized_property + def _column_tokens(self): + """parse a string-based _colspec into its component parts.""" + + m = self._colspec.split('.') + if m is None: + raise exc.ArgumentError( + "Invalid foreign key column specification: %s" % + self._colspec) + if (len(m) == 1): + tname = m.pop() + colname = None + else: + colname = m.pop() + tname = m.pop() + + # A FK between column 'bar' and table 'foo' can be + # specified as 'foo', 'foo.bar', 'dbo.foo.bar', + # 'otherdb.dbo.foo.bar'. Once we have the column name and + # the table name, treat everything else as the schema + # name. Some databases (e.g. Sybase) support + # inter-database foreign keys. See tickets#1341 and -- + # indirectly related -- Ticket #594. This assumes that '.' + # will never appear *within* any component of the FK. + + if (len(m) > 0): + schema = '.'.join(m) + else: + schema = None + return schema, tname, colname + + def _resolve_col_tokens(self): + if self.parent is None: + raise exc.InvalidRequestError( + "this ForeignKey object does not yet have a " + "parent Column associated with it.") + + elif self.parent.table is None: + raise exc.InvalidRequestError( + "this ForeignKey's parent column is not yet associated " + "with a Table.") + + parenttable = self.parent.table + + # assertion, can be commented out. + # basically Column._make_proxy() sends the actual + # target Column to the ForeignKey object, so the + # string resolution here is never called. + for c in self.parent.base_columns: + if isinstance(c, Column): + assert c.table is parenttable + break + else: + assert False + ###################### + + schema, tname, colname = self._column_tokens + + if schema is None and parenttable.metadata.schema is not None: + schema = parenttable.metadata.schema + + tablekey = _get_table_key(tname, schema) + return parenttable, tablekey, colname + + + def _link_to_col_by_colstring(self, parenttable, table, colname): + if not hasattr(self.constraint, '_referred_table'): + self.constraint._referred_table = table + else: + assert self.constraint._referred_table is table + + _column = None + if colname is None: + # colname is None in the case that ForeignKey argument + # was specified as table name only, in which case we + # match the column name to the same column on the + # parent. + key = self.parent + _column = table.c.get(self.parent.key, None) + elif self.link_to_name: + key = colname + for c in table.c: + if c.name == colname: + _column = c + else: + key = colname + _column = table.c.get(colname, None) + + if _column is None: + raise exc.NoReferencedColumnError( + "Could not initialize target column for ForeignKey '%s' on table '%s': " + "table '%s' has no column named '%s'" % ( + self._colspec, parenttable.name, table.name, key), + table.name, key) + + self._set_target_column(_column) + + def _set_target_column(self, column): + # propagate TypeEngine to parent if it didn't have one + if self.parent.type._isnull: + self.parent.type = column.type + + # super-edgy case, if other FKs point to our column, + # they'd get the type propagated out also. + if isinstance(self.parent.table, Table): + fk_key = (self.parent.table.key, self.parent.key) + if fk_key in self.parent.table.metadata._fk_memos: + for fk in self.parent.table.metadata._fk_memos[fk_key]: + if fk.parent.type._isnull: + fk.parent.type = column.type + + self.column = column + + @util.memoized_property + def column(self): + """Return the target :class:`.Column` referenced by this + :class:`.ForeignKey`. + + If no target column has been established, an exception + is raised. + + .. versionchanged:: 0.9.0 + Foreign key target column resolution now occurs as soon as both + the ForeignKey object and the remote Column to which it refers + are both associated with the same MetaData object. + + """ + + if isinstance(self._colspec, util.string_types): + + parenttable, tablekey, colname = self._resolve_col_tokens() + + if tablekey not in parenttable.metadata: + raise exc.NoReferencedTableError( + "Foreign key associated with column '%s' could not find " + "table '%s' with which to generate a " + "foreign key to target column '%s'" % + (self.parent, tablekey, colname), + tablekey) + elif parenttable.key not in parenttable.metadata: + raise exc.InvalidRequestError( + "Table %s is no longer associated with its " + "parent MetaData" % parenttable) + else: + raise exc.NoReferencedColumnError( + "Could not initialize target column for " + "ForeignKey '%s' on table '%s': " + "table '%s' has no column named '%s'" % ( + self._colspec, parenttable.name, tablekey, colname), + tablekey, colname) + elif hasattr(self._colspec, '__clause_element__'): + _column = self._colspec.__clause_element__() + return _column + else: + _column = self._colspec + return _column + + def _set_parent(self, column): + if self.parent is not None and self.parent is not column: + raise exc.InvalidRequestError( + "This ForeignKey already has a parent !") + self.parent = column + self.parent.foreign_keys.add(self) + self.parent._on_table_attach(self._set_table) + + def _set_remote_table(self, table): + parenttable, tablekey, colname = self._resolve_col_tokens() + self._link_to_col_by_colstring(parenttable, table, colname) + self.constraint._validate_dest_table(table) + + def _remove_from_metadata(self, metadata): + parenttable, table_key, colname = self._resolve_col_tokens() + fk_key = (table_key, colname) + + if self in metadata._fk_memos[fk_key]: + # TODO: no test coverage for self not in memos + metadata._fk_memos[fk_key].remove(self) + + def _set_table(self, column, table): + # standalone ForeignKey - create ForeignKeyConstraint + # on the hosting Table when attached to the Table. + if self.constraint is None and isinstance(table, Table): + self.constraint = ForeignKeyConstraint( + [], [], use_alter=self.use_alter, name=self.name, + onupdate=self.onupdate, ondelete=self.ondelete, + deferrable=self.deferrable, initially=self.initially, + match=self.match, + ) + self.constraint._elements[self.parent] = self + self.constraint._set_parent_with_dispatch(table) + table.foreign_keys.add(self) + + # set up remote ".column" attribute, or a note to pick it + # up when the other Table/Column shows up + if isinstance(self._colspec, util.string_types): + parenttable, table_key, colname = self._resolve_col_tokens() + fk_key = (table_key, colname) + if table_key in parenttable.metadata.tables: + table = parenttable.metadata.tables[table_key] + try: + self._link_to_col_by_colstring(parenttable, table, colname) + except exc.NoReferencedColumnError: + # this is OK, we'll try later + pass + parenttable.metadata._fk_memos[fk_key].append(self) + elif hasattr(self._colspec, '__clause_element__'): + _column = self._colspec.__clause_element__() + self._set_target_column(_column) + else: + _column = self._colspec + self._set_target_column(_column) + + + +class _NotAColumnExpr(object): + def _not_a_column_expr(self): + raise exc.InvalidRequestError( + "This %s cannot be used directly " + "as a column expression." % self.__class__.__name__) + + __clause_element__ = self_group = lambda self: self._not_a_column_expr() + _from_objects = property(lambda self: self._not_a_column_expr()) + + +class DefaultGenerator(_NotAColumnExpr, SchemaItem): + """Base class for column *default* values.""" + + __visit_name__ = 'default_generator' + + is_sequence = False + is_server_default = False + column = None + + def __init__(self, for_update=False): + self.for_update = for_update + + def _set_parent(self, column): + self.column = column + if self.for_update: + self.column.onupdate = self + else: + self.column.default = self + + def execute(self, bind=None, **kwargs): + if bind is None: + bind = _bind_or_error(self) + return bind._execute_default(self, **kwargs) + + @property + def bind(self): + """Return the connectable associated with this default.""" + if getattr(self, 'column', None) is not None: + return self.column.table.bind + else: + return None + + +class ColumnDefault(DefaultGenerator): + """A plain default value on a column. + + This could correspond to a constant, a callable function, + or a SQL clause. + + :class:`.ColumnDefault` is generated automatically + whenever the ``default``, ``onupdate`` arguments of + :class:`.Column` are used. A :class:`.ColumnDefault` + can be passed positionally as well. + + For example, the following:: + + Column('foo', Integer, default=50) + + Is equivalent to:: + + Column('foo', Integer, ColumnDefault(50)) + + + """ + + def __init__(self, arg, **kwargs): + """"Construct a new :class:`.ColumnDefault`. + + + :param arg: argument representing the default value. + May be one of the following: + + * a plain non-callable Python value, such as a + string, integer, boolean, or other simple type. + The default value will be used as is each time. + * a SQL expression, that is one which derives from + :class:`.ColumnElement`. The SQL expression will + be rendered into the INSERT or UPDATE statement, + or in the case of a primary key column when + RETURNING is not used may be + pre-executed before an INSERT within a SELECT. + * A Python callable. The function will be invoked for each + new row subject to an INSERT or UPDATE. + The callable must accept exactly + zero or one positional arguments. The one-argument form + will receive an instance of the :class:`.ExecutionContext`, + which provides contextual information as to the current + :class:`.Connection` in use as well as the current + statement and parameters. + + """ + super(ColumnDefault, self).__init__(**kwargs) + if isinstance(arg, FetchedValue): + raise exc.ArgumentError( + "ColumnDefault may not be a server-side default type.") + if util.callable(arg): + arg = self._maybe_wrap_callable(arg) + self.arg = arg + + @util.memoized_property + def is_callable(self): + return util.callable(self.arg) + + @util.memoized_property + def is_clause_element(self): + return isinstance(self.arg, ClauseElement) + + @util.memoized_property + def is_scalar(self): + return not self.is_callable and \ + not self.is_clause_element and \ + not self.is_sequence + + def _maybe_wrap_callable(self, fn): + """Wrap callables that don't accept a context. + + The alternative here is to require that + a simple callable passed to "default" would need + to be of the form "default=lambda ctx: datetime.now". + That is the more "correct" way to go, but the case + of using a zero-arg callable for "default" is so + much more prominent than the context-specific one + I'm having trouble justifying putting that inconvenience + on everyone. + + """ + if inspect.isfunction(fn) or inspect.ismethod(fn): + inspectable = fn + elif inspect.isclass(fn): + inspectable = fn.__init__ + elif hasattr(fn, '__call__'): + inspectable = fn.__call__ + else: + # probably not inspectable, try anyways. + inspectable = fn + try: + argspec = inspect.getargspec(inspectable) + except TypeError: + return lambda ctx: fn() + + defaulted = argspec[3] is not None and len(argspec[3]) or 0 + positionals = len(argspec[0]) - defaulted + + # Py3K compat - no unbound methods + if inspect.ismethod(inspectable) or inspect.isclass(fn): + positionals -= 1 + + if positionals == 0: + return lambda ctx: fn() + elif positionals == 1: + return fn + else: + raise exc.ArgumentError( + "ColumnDefault Python function takes zero or one " + "positional arguments") + + def _visit_name(self): + if self.for_update: + return "column_onupdate" + else: + return "column_default" + __visit_name__ = property(_visit_name) + + def __repr__(self): + return "ColumnDefault(%r)" % self.arg + + +class Sequence(DefaultGenerator): + """Represents a named database sequence. + + The :class:`.Sequence` object represents the name and configurational + parameters of a database sequence. It also represents + a construct that can be "executed" by a SQLAlchemy :class:`.Engine` + or :class:`.Connection`, rendering the appropriate "next value" function + for the target database and returning a result. + + The :class:`.Sequence` is typically associated with a primary key column:: + + some_table = Table('some_table', metadata, + Column('id', Integer, Sequence('some_table_seq'), primary_key=True) + ) + + When CREATE TABLE is emitted for the above :class:`.Table`, if the + target platform supports sequences, a CREATE SEQUENCE statement will + be emitted as well. For platforms that don't support sequences, + the :class:`.Sequence` construct is ignored. + + .. seealso:: + + :class:`.CreateSequence` + + :class:`.DropSequence` + + """ + + __visit_name__ = 'sequence' + + is_sequence = True + + def __init__(self, name, start=None, increment=None, schema=None, + optional=False, quote=None, metadata=None, + quote_schema=None, + for_update=False): + """Construct a :class:`.Sequence` object. + + :param name: The name of the sequence. + :param start: the starting index of the sequence. This value is + used when the CREATE SEQUENCE command is emitted to the database + as the value of the "START WITH" clause. If ``None``, the + clause is omitted, which on most platforms indicates a starting + value of 1. + :param increment: the increment value of the sequence. This + value is used when the CREATE SEQUENCE command is emitted to + the database as the value of the "INCREMENT BY" clause. If ``None``, + the clause is omitted, which on most platforms indicates an + increment of 1. + :param schema: Optional schema name for the sequence, if located + in a schema other than the default. + :param optional: boolean value, when ``True``, indicates that this + :class:`.Sequence` object only needs to be explicitly generated + on backends that don't provide another way to generate primary + key identifiers. Currently, it essentially means, "don't create + this sequence on the Postgresql backend, where the SERIAL keyword + creates a sequence for us automatically". + :param quote: boolean value, when ``True`` or ``False``, explicitly + forces quoting of the schema name on or off. When left at its + default of ``None``, normal quoting rules based on casing and reserved + words take place. + :param quote_schema: set the quoting preferences for the ``schema`` + name. + :param metadata: optional :class:`.MetaData` object which will be + associated with this :class:`.Sequence`. A :class:`.Sequence` + that is associated with a :class:`.MetaData` gains access to the + ``bind`` of that :class:`.MetaData`, meaning the + :meth:`.Sequence.create` and :meth:`.Sequence.drop` methods will + make usage of that engine automatically. + + .. versionchanged:: 0.7 + Additionally, the appropriate CREATE SEQUENCE/ + DROP SEQUENCE DDL commands will be emitted corresponding to this + :class:`.Sequence` when :meth:`.MetaData.create_all` and + :meth:`.MetaData.drop_all` are invoked. + + Note that when a :class:`.Sequence` is applied to a :class:`.Column`, + the :class:`.Sequence` is automatically associated with the + :class:`.MetaData` object of that column's parent :class:`.Table`, + when that association is made. The :class:`.Sequence` will then + be subject to automatic CREATE SEQUENCE/DROP SEQUENCE corresponding + to when the :class:`.Table` object itself is created or dropped, + rather than that of the :class:`.MetaData` object overall. + :param for_update: Indicates this :class:`.Sequence`, when associated + with a :class:`.Column`, should be invoked for UPDATE statements + on that column's table, rather than for INSERT statements, when + no value is otherwise present for that column in the statement. + + """ + super(Sequence, self).__init__(for_update=for_update) + self.name = quoted_name(name, quote) + self.start = start + self.increment = increment + self.optional = optional + if metadata is not None and schema is None and metadata.schema: + self.schema = schema = metadata.schema + else: + self.schema = quoted_name(schema, quote_schema) + self.metadata = metadata + self._key = _get_table_key(name, schema) + if metadata: + self._set_metadata(metadata) + + @util.memoized_property + def is_callable(self): + return False + + @util.memoized_property + def is_clause_element(self): + return False + + @util.dependencies("sqlalchemy.sql.functions.func") + def next_value(self, func): + """Return a :class:`.next_value` function element + which will render the appropriate increment function + for this :class:`.Sequence` within any SQL expression. + + """ + return func.next_value(self, bind=self.bind) + + def _set_parent(self, column): + super(Sequence, self)._set_parent(column) + column._on_table_attach(self._set_table) + + def _set_table(self, column, table): + self._set_metadata(table.metadata) + + def _set_metadata(self, metadata): + self.metadata = metadata + self.metadata._sequences[self._key] = self + + @property + def bind(self): + if self.metadata: + return self.metadata.bind + else: + return None + + def create(self, bind=None, checkfirst=True): + """Creates this sequence in the database.""" + + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaGenerator, + self, + checkfirst=checkfirst) + + def drop(self, bind=None, checkfirst=True): + """Drops this sequence from the database.""" + + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaDropper, + self, + checkfirst=checkfirst) + + def _not_a_column_expr(self): + raise exc.InvalidRequestError( + "This %s cannot be used directly " + "as a column expression. Use func.next_value(sequence) " + "to produce a 'next value' function that's usable " + "as a column element." + % self.__class__.__name__) + + +@inspection._self_inspects +class FetchedValue(_NotAColumnExpr, SchemaEventTarget): + """A marker for a transparent database-side default. + + Use :class:`.FetchedValue` when the database is configured + to provide some automatic default for a column. + + E.g.:: + + Column('foo', Integer, FetchedValue()) + + Would indicate that some trigger or default generator + will create a new value for the ``foo`` column during an + INSERT. + + .. seealso:: + + :ref:`triggered_columns` + + """ + is_server_default = True + reflected = False + has_argument = False + + def __init__(self, for_update=False): + self.for_update = for_update + + def _as_for_update(self, for_update): + if for_update == self.for_update: + return self + else: + return self._clone(for_update) + + def _clone(self, for_update): + n = self.__class__.__new__(self.__class__) + n.__dict__.update(self.__dict__) + n.__dict__.pop('column', None) + n.for_update = for_update + return n + + def _set_parent(self, column): + self.column = column + if self.for_update: + self.column.server_onupdate = self + else: + self.column.server_default = self + + def __repr__(self): + return util.generic_repr(self) + + +class DefaultClause(FetchedValue): + """A DDL-specified DEFAULT column value. + + :class:`.DefaultClause` is a :class:`.FetchedValue` + that also generates a "DEFAULT" clause when + "CREATE TABLE" is emitted. + + :class:`.DefaultClause` is generated automatically + whenever the ``server_default``, ``server_onupdate`` arguments of + :class:`.Column` are used. A :class:`.DefaultClause` + can be passed positionally as well. + + For example, the following:: + + Column('foo', Integer, server_default="50") + + Is equivalent to:: + + Column('foo', Integer, DefaultClause("50")) + + """ + + has_argument = True + + def __init__(self, arg, for_update=False, _reflected=False): + util.assert_arg_type(arg, (util.string_types[0], + ClauseElement, + TextClause), 'arg') + super(DefaultClause, self).__init__(for_update) + self.arg = arg + self.reflected = _reflected + + def __repr__(self): + return "DefaultClause(%r, for_update=%r)" % \ + (self.arg, self.for_update) + + +class PassiveDefault(DefaultClause): + """A DDL-specified DEFAULT column value. + + .. deprecated:: 0.6 + :class:`.PassiveDefault` is deprecated. + Use :class:`.DefaultClause`. + """ + @util.deprecated("0.6", + ":class:`.PassiveDefault` is deprecated. " + "Use :class:`.DefaultClause`.", + False) + def __init__(self, *arg, **kw): + DefaultClause.__init__(self, *arg, **kw) + + +class Constraint(SchemaItem): + """A table-level SQL constraint.""" + + __visit_name__ = 'constraint' + + def __init__(self, name=None, deferrable=None, initially=None, + _create_rule=None, + **kw): + """Create a SQL constraint. + + :param name: + Optional, the in-database name of this ``Constraint``. + + :param deferrable: + Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when + issuing DDL for this constraint. + + :param initially: + Optional string. If set, emit INITIALLY when issuing DDL + for this constraint. + + :param _create_rule: + a callable which is passed the DDLCompiler object during + compilation. Returns True or False to signal inline generation of + this Constraint. + + The AddConstraint and DropConstraint DDL constructs provide + DDLElement's more comprehensive "conditional DDL" approach that is + passed a database connection when DDL is being issued. _create_rule + is instead called during any CREATE TABLE compilation, where there + may not be any transaction/connection in progress. However, it + allows conditional compilation of the constraint even for backends + which do not support addition of constraints through ALTER TABLE, + which currently includes SQLite. + + _create_rule is used by some types to create constraints. + Currently, its call signature is subject to change at any time. + + :param \**kwargs: + Dialect-specific keyword parameters, see the documentation + for various dialects and constraints regarding options here. + + """ + + self.name = name + self.deferrable = deferrable + self.initially = initially + self._create_rule = _create_rule + util.set_creation_order(self) + _validate_dialect_kwargs(kw, self.__class__.__name__) + self.kwargs = kw + + @property + def table(self): + try: + if isinstance(self.parent, Table): + return self.parent + except AttributeError: + pass + raise exc.InvalidRequestError( + "This constraint is not bound to a table. Did you " + "mean to call table.append_constraint(constraint) ?") + + def _set_parent(self, parent): + self.parent = parent + parent.constraints.add(self) + + def copy(self, **kw): + raise NotImplementedError() + + +def _to_schema_column(element): + if hasattr(element, '__clause_element__'): + element = element.__clause_element__() + if not isinstance(element, Column): + raise exc.ArgumentError("schema.Column object expected") + return element + + +def _to_schema_column_or_string(element): + if hasattr(element, '__clause_element__'): + element = element.__clause_element__() + if not isinstance(element, util.string_types + (ColumnElement, )): + msg = "Element %r is not a string name or column element" + raise exc.ArgumentError(msg % element) + return element + + +class ColumnCollectionMixin(object): + def __init__(self, *columns): + self.columns = ColumnCollection() + self._pending_colargs = [_to_schema_column_or_string(c) + for c in columns] + if self._pending_colargs and \ + isinstance(self._pending_colargs[0], Column) and \ + isinstance(self._pending_colargs[0].table, Table): + self._set_parent_with_dispatch(self._pending_colargs[0].table) + + def _set_parent(self, table): + for col in self._pending_colargs: + if isinstance(col, util.string_types): + col = table.c[col] + self.columns.add(col) + + +class ColumnCollectionConstraint(ColumnCollectionMixin, Constraint): + """A constraint that proxies a ColumnCollection.""" + + def __init__(self, *columns, **kw): + """ + :param \*columns: + A sequence of column names or Column objects. + + :param name: + Optional, the in-database name of this constraint. + + :param deferrable: + Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when + issuing DDL for this constraint. + + :param initially: + Optional string. If set, emit INITIALLY when issuing DDL + for this constraint. + + """ + ColumnCollectionMixin.__init__(self, *columns) + Constraint.__init__(self, **kw) + + def _set_parent(self, table): + ColumnCollectionMixin._set_parent(self, table) + Constraint._set_parent(self, table) + + def __contains__(self, x): + return x in self.columns + + def copy(self, **kw): + c = self.__class__(name=self.name, deferrable=self.deferrable, + initially=self.initially, *self.columns.keys()) + return self._schema_item_copy(c) + + def contains_column(self, col): + return self.columns.contains_column(col) + + def __iter__(self): + # inlining of + # return iter(self.columns) + # ColumnCollection->OrderedProperties->OrderedDict + ordered_dict = self.columns._data + return (ordered_dict[key] for key in ordered_dict._list) + + def __len__(self): + return len(self.columns._data) + + +class CheckConstraint(Constraint): + """A table- or column-level CHECK constraint. + + Can be included in the definition of a Table or Column. + """ + + def __init__(self, sqltext, name=None, deferrable=None, + initially=None, table=None, _create_rule=None, + _autoattach=True): + """Construct a CHECK constraint. + + :param sqltext: + A string containing the constraint definition, which will be used + verbatim, or a SQL expression construct. If given as a string, + the object is converted to a :class:`.Text` object. If the textual + string includes a colon character, escape this using a backslash:: + + CheckConstraint(r"foo ~ E'a(?\:b|c)d") + + :param name: + Optional, the in-database name of the constraint. + + :param deferrable: + Optional bool. If set, emit DEFERRABLE or NOT DEFERRABLE when + issuing DDL for this constraint. + + :param initially: + Optional string. If set, emit INITIALLY when issuing DDL + for this constraint. + + """ + + super(CheckConstraint, self).\ + __init__(name, deferrable, initially, _create_rule) + self.sqltext = _literal_as_text(sqltext) + if table is not None: + self._set_parent_with_dispatch(table) + elif _autoattach: + cols = _find_columns(self.sqltext) + tables = set([c.table for c in cols + if isinstance(c.table, Table)]) + if len(tables) == 1: + self._set_parent_with_dispatch( + tables.pop()) + + def __visit_name__(self): + if isinstance(self.parent, Table): + return "check_constraint" + else: + return "column_check_constraint" + __visit_name__ = property(__visit_name__) + + def copy(self, target_table=None, **kw): + if target_table is not None: + def replace(col): + if self.table.c.contains_column(col): + return target_table.c[col.key] + else: + return None + sqltext = visitors.replacement_traverse(self.sqltext, {}, replace) + else: + sqltext = self.sqltext + c = CheckConstraint(sqltext, + name=self.name, + initially=self.initially, + deferrable=self.deferrable, + _create_rule=self._create_rule, + table=target_table, + _autoattach=False) + return self._schema_item_copy(c) + + +class ForeignKeyConstraint(Constraint): + """A table-level FOREIGN KEY constraint. + + Defines a single column or composite FOREIGN KEY ... REFERENCES + constraint. For a no-frills, single column foreign key, adding a + :class:`.ForeignKey` to the definition of a :class:`.Column` is a shorthand + equivalent for an unnamed, single column :class:`.ForeignKeyConstraint`. + + Examples of foreign key configuration are in :ref:`metadata_foreignkeys`. + + """ + __visit_name__ = 'foreign_key_constraint' + + def __init__(self, columns, refcolumns, name=None, onupdate=None, + ondelete=None, deferrable=None, initially=None, use_alter=False, + link_to_name=False, match=None, table=None): + """Construct a composite-capable FOREIGN KEY. + + :param columns: A sequence of local column names. The named columns + must be defined and present in the parent Table. The names should + match the ``key`` given to each column (defaults to the name) unless + ``link_to_name`` is True. + + :param refcolumns: A sequence of foreign column names or Column + objects. The columns must all be located within the same Table. + + :param name: Optional, the in-database name of the key. + + :param onupdate: Optional string. If set, emit ON UPDATE when + issuing DDL for this constraint. Typical values include CASCADE, + DELETE and RESTRICT. + + :param ondelete: Optional string. If set, emit ON DELETE when + issuing DDL for this constraint. Typical values include CASCADE, + DELETE and RESTRICT. + + :param deferrable: Optional bool. If set, emit DEFERRABLE or NOT + DEFERRABLE when issuing DDL for this constraint. + + :param initially: Optional string. If set, emit INITIALLY when + issuing DDL for this constraint. + + :param link_to_name: if True, the string name given in ``column`` is + the rendered name of the referenced column, not its locally assigned + ``key``. + + :param use_alter: If True, do not emit the DDL for this constraint as + part of the CREATE TABLE definition. Instead, generate it via an + ALTER TABLE statement issued after the full collection of tables + have been created, and drop it via an ALTER TABLE statement before + the full collection of tables are dropped. This is shorthand for the + usage of :class:`.AddConstraint` and :class:`.DropConstraint` applied + as "after-create" and "before-drop" events on the MetaData object. + This is normally used to generate/drop constraints on objects that + are mutually dependent on each other. + + :param match: Optional string. If set, emit MATCH when issuing + DDL for this constraint. Typical values include SIMPLE, PARTIAL + and FULL. + + """ + super(ForeignKeyConstraint, self).\ + __init__(name, deferrable, initially) + + self.onupdate = onupdate + self.ondelete = ondelete + self.link_to_name = link_to_name + if self.name is None and use_alter: + raise exc.ArgumentError("Alterable Constraint requires a name") + self.use_alter = use_alter + self.match = match + + self._elements = util.OrderedDict() + + # standalone ForeignKeyConstraint - create + # associated ForeignKey objects which will be applied to hosted + # Column objects (in col.foreign_keys), either now or when attached + # to the Table for string-specified names + for col, refcol in zip(columns, refcolumns): + self._elements[col] = ForeignKey( + refcol, + _constraint=self, + name=self.name, + onupdate=self.onupdate, + ondelete=self.ondelete, + use_alter=self.use_alter, + link_to_name=self.link_to_name, + match=self.match, + deferrable=self.deferrable, + initially=self.initially + ) + + if table is not None: + self._set_parent_with_dispatch(table) + elif columns and \ + isinstance(columns[0], Column) and \ + columns[0].table is not None: + self._set_parent_with_dispatch(columns[0].table) + + def _validate_dest_table(self, table): + table_keys = set([elem._table_key() for elem in self._elements.values()]) + if None not in table_keys and len(table_keys) > 1: + elem0, elem1 = sorted(table_keys)[0:2] + raise exc.ArgumentError( + 'ForeignKeyConstraint on %s(%s) refers to ' + 'multiple remote tables: %s and %s' % ( + table.fullname, + self._col_description, + elem0, + elem1 + )) + + @property + def _col_description(self): + return ", ".join(self._elements) + + @property + def columns(self): + return list(self._elements) + + @property + def elements(self): + return list(self._elements.values()) + + def _set_parent(self, table): + super(ForeignKeyConstraint, self)._set_parent(table) + + self._validate_dest_table(table) + + for col, fk in self._elements.items(): + # string-specified column names now get + # resolved to Column objects + if isinstance(col, util.string_types): + try: + col = table.c[col] + except KeyError: + raise exc.ArgumentError( + "Can't create ForeignKeyConstraint " + "on table '%s': no column " + "named '%s' is present." % (table.description, col)) + + if not hasattr(fk, 'parent') or \ + fk.parent is not col: + fk._set_parent_with_dispatch(col) + + if self.use_alter: + def supports_alter(ddl, event, schema_item, bind, **kw): + return table in set(kw['tables']) and \ + bind.dialect.supports_alter + + event.listen(table.metadata, "after_create", + ddl.AddConstraint(self, on=supports_alter)) + event.listen(table.metadata, "before_drop", + ddl.DropConstraint(self, on=supports_alter)) + + def copy(self, schema=None, **kw): + fkc = ForeignKeyConstraint( + [x.parent.key for x in self._elements.values()], + [x._get_colspec(schema=schema) for x in self._elements.values()], + name=self.name, + onupdate=self.onupdate, + ondelete=self.ondelete, + use_alter=self.use_alter, + deferrable=self.deferrable, + initially=self.initially, + link_to_name=self.link_to_name, + match=self.match + ) + for self_fk, other_fk in zip( + self._elements.values(), + fkc._elements.values()): + self_fk._schema_item_copy(other_fk) + return self._schema_item_copy(fkc) + + +class PrimaryKeyConstraint(ColumnCollectionConstraint): + """A table-level PRIMARY KEY constraint. + + Defines a single column or composite PRIMARY KEY constraint. For a + no-frills primary key, adding ``primary_key=True`` to one or more + ``Column`` definitions is a shorthand equivalent for an unnamed single- or + multiple-column PrimaryKeyConstraint. + """ + + __visit_name__ = 'primary_key_constraint' + + def _set_parent(self, table): + super(PrimaryKeyConstraint, self)._set_parent(table) + + if table.primary_key in table.constraints: + table.constraints.remove(table.primary_key) + table.primary_key = self + table.constraints.add(self) + + for c in self.columns: + c.primary_key = True + + def _replace(self, col): + self.columns.replace(col) + + +class UniqueConstraint(ColumnCollectionConstraint): + """A table-level UNIQUE constraint. + + Defines a single column or composite UNIQUE constraint. For a no-frills, + single column constraint, adding ``unique=True`` to the ``Column`` + definition is a shorthand equivalent for an unnamed, single column + UniqueConstraint. + """ + + __visit_name__ = 'unique_constraint' + + +class Index(ColumnCollectionMixin, SchemaItem): + """A table-level INDEX. + + Defines a composite (one or more column) INDEX. + + E.g.:: + + sometable = Table("sometable", metadata, + Column("name", String(50)), + Column("address", String(100)) + ) + + Index("some_index", sometable.c.name) + + For a no-frills, single column index, adding + :class:`.Column` also supports ``index=True``:: + + sometable = Table("sometable", metadata, + Column("name", String(50), index=True) + ) + + For a composite index, multiple columns can be specified:: + + Index("some_index", sometable.c.name, sometable.c.address) + + Functional indexes are supported as well, keeping in mind that at least + one :class:`.Column` must be present:: + + Index("some_index", func.lower(sometable.c.name)) + + .. versionadded:: 0.8 support for functional and expression-based indexes. + + .. seealso:: + + :ref:`schema_indexes` - General information on :class:`.Index`. + + :ref:`postgresql_indexes` - PostgreSQL-specific options available for the + :class:`.Index` construct. + + :ref:`mysql_indexes` - MySQL-specific options available for the + :class:`.Index` construct. + + :ref:`mssql_indexes` - MSSQL-specific options available for the + :class:`.Index` construct. + + """ + + __visit_name__ = 'index' + + def __init__(self, name, *expressions, **kw): + """Construct an index object. + + :param name: + The name of the index + + :param \*expressions: + Column expressions to include in the index. The expressions + are normally instances of :class:`.Column`, but may also + be arbitrary SQL expressions which ultmately refer to a + :class:`.Column`. + + :param unique: + Defaults to False: create a unique index. + + :param \**kw: + Other keyword arguments may be interpreted by specific dialects. + + """ + self.table = None + + columns = [] + for expr in expressions: + if not isinstance(expr, ClauseElement): + columns.append(expr) + else: + cols = [] + visitors.traverse(expr, {}, {'column': cols.append}) + if cols: + columns.append(cols[0]) + else: + columns.append(expr) + + self.expressions = expressions + self.name = quoted_name(name, kw.pop("quote", None)) + self.unique = kw.pop('unique', False) + self.kwargs = kw + + # will call _set_parent() if table-bound column + # objects are present + ColumnCollectionMixin.__init__(self, *columns) + + + + def _set_parent(self, table): + ColumnCollectionMixin._set_parent(self, table) + + if self.table is not None and table is not self.table: + raise exc.ArgumentError( + "Index '%s' is against table '%s', and " + "cannot be associated with table '%s'." % ( + self.name, + self.table.description, + table.description + ) + ) + self.table = table + for c in self.columns: + if c.table != self.table: + raise exc.ArgumentError( + "Column '%s' is not part of table '%s'." % + (c, self.table.description) + ) + table.indexes.add(self) + + self.expressions = [ + expr if isinstance(expr, ClauseElement) + else colexpr + for expr, colexpr in zip(self.expressions, self.columns) + ] + + @property + def bind(self): + """Return the connectable associated with this Index.""" + + return self.table.bind + + def create(self, bind=None): + """Issue a ``CREATE`` statement for this + :class:`.Index`, using the given :class:`.Connectable` + for connectivity. + + .. seealso:: + + :meth:`.MetaData.create_all`. + + """ + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaGenerator, self) + return self + + def drop(self, bind=None): + """Issue a ``DROP`` statement for this + :class:`.Index`, using the given :class:`.Connectable` + for connectivity. + + .. seealso:: + + :meth:`.MetaData.drop_all`. + + """ + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaDropper, self) + + def __repr__(self): + return 'Index(%s)' % ( + ", ".join( + [repr(self.name)] + + [repr(c) for c in self.columns] + + (self.unique and ["unique=True"] or []) + )) + + +class MetaData(SchemaItem): + """A collection of :class:`.Table` objects and their associated schema + constructs. + + Holds a collection of :class:`.Table` objects as well as + an optional binding to an :class:`.Engine` or + :class:`.Connection`. If bound, the :class:`.Table` objects + in the collection and their columns may participate in implicit SQL + execution. + + The :class:`.Table` objects themselves are stored in the + :attr:`.MetaData.tables` dictionary. + + :class:`.MetaData` is a thread-safe object for read operations. Construction + of new tables within a single :class:`.MetaData` object, either explicitly + or via reflection, may not be completely thread-safe. + + .. seealso:: + + :ref:`metadata_describing` - Introduction to database metadata + + """ + + __visit_name__ = 'metadata' + + def __init__(self, bind=None, reflect=False, schema=None, + quote_schema=None): + """Create a new MetaData object. + + :param bind: + An Engine or Connection to bind to. May also be a string or URL + instance, these are passed to create_engine() and this MetaData will + be bound to the resulting engine. + + :param reflect: + Optional, automatically load all tables from the bound database. + Defaults to False. ``bind`` is required when this option is set. + + .. deprecated:: 0.8 + Please use the :meth:`.MetaData.reflect` method. + + :param schema: + The default schema to use for the :class:`.Table`, + :class:`.Sequence`, and other objects associated with this + :class:`.MetaData`. Defaults to ``None``. + + :param quote_schema: + Sets the ``quote_schema`` flag for those :class:`.Table`, + :class:`.Sequence`, and other objects which make usage of the + local ``schema`` name. + + .. versionadded:: 0.7.4 + ``schema`` and ``quote_schema`` parameters. + + """ + self.tables = util.immutabledict() + self.schema = quoted_name(schema, quote_schema) + self._schemas = set() + self._sequences = {} + self._fk_memos = collections.defaultdict(list) + + self.bind = bind + if reflect: + util.warn("reflect=True is deprecate; please " + "use the reflect() method.") + if not bind: + raise exc.ArgumentError( + "A bind must be supplied in conjunction " + "with reflect=True") + self.reflect() + + tables = None + """A dictionary of :class:`.Table` objects keyed to their name or "table key". + + The exact key is that determined by the :attr:`.Table.key` attribute; + for a table with no :attr:`.Table.schema` attribute, this is the same + as :attr:`.Table.name`. For a table with a schema, it is typically of the + form ``schemaname.tablename``. + + .. seealso:: + + :attr:`.MetaData.sorted_tables` + + """ + + def __repr__(self): + return 'MetaData(bind=%r)' % self.bind + + def __contains__(self, table_or_key): + if not isinstance(table_or_key, util.string_types): + table_or_key = table_or_key.key + return table_or_key in self.tables + + def _add_table(self, name, schema, table): + key = _get_table_key(name, schema) + dict.__setitem__(self.tables, key, table) + if schema: + self._schemas.add(schema) + + + + def _remove_table(self, name, schema): + key = _get_table_key(name, schema) + removed = dict.pop(self.tables, key, None) + if removed is not None: + for fk in removed.foreign_keys: + fk._remove_from_metadata(self) + if self._schemas: + self._schemas = set([t.schema + for t in self.tables.values() + if t.schema is not None]) + + + def __getstate__(self): + return {'tables': self.tables, + 'schema': self.schema, + 'schemas': self._schemas, + 'sequences': self._sequences, + 'fk_memos': self._fk_memos} + + def __setstate__(self, state): + self.tables = state['tables'] + self.schema = state['schema'] + self._bind = None + self._sequences = state['sequences'] + self._schemas = state['schemas'] + self._fk_memos = state['fk_memos'] + + def is_bound(self): + """True if this MetaData is bound to an Engine or Connection.""" + + return self._bind is not None + + def bind(self): + """An :class:`.Engine` or :class:`.Connection` to which this + :class:`.MetaData` is bound. + + Typically, a :class:`.Engine` is assigned to this attribute + so that "implicit execution" may be used, or alternatively + as a means of providing engine binding information to an + ORM :class:`.Session` object:: + + engine = create_engine("someurl://") + metadata.bind = engine + + .. seealso:: + + :ref:`dbengine_implicit` - background on "bound metadata" + + """ + return self._bind + + @util.dependencies("sqlalchemy.engine.url") + def _bind_to(self, url, bind): + """Bind this MetaData to an Engine, Connection, string or URL.""" + + if isinstance(bind, util.string_types + (url.URL, )): + self._bind = sqlalchemy.create_engine(bind) + else: + self._bind = bind + bind = property(bind, _bind_to) + + def clear(self): + """Clear all Table objects from this MetaData.""" + + dict.clear(self.tables) + self._schemas.clear() + self._fk_memos.clear() + + def remove(self, table): + """Remove the given Table object from this MetaData.""" + + self._remove_table(table.name, table.schema) + + @property + def sorted_tables(self): + """Returns a list of :class:`.Table` objects sorted in order of + foreign key dependency. + + The sorting will place :class:`.Table` objects that have dependencies + first, before the dependencies themselves, representing the + order in which they can be created. To get the order in which + the tables would be dropped, use the ``reversed()`` Python built-in. + + .. seealso:: + + :attr:`.MetaData.tables` + + :meth:`.Inspector.get_table_names` + + """ + return ddl.sort_tables(self.tables.values()) + + def reflect(self, bind=None, schema=None, views=False, only=None, + extend_existing=False, + autoload_replace=True): + """Load all available table definitions from the database. + + Automatically creates ``Table`` entries in this ``MetaData`` for any + table available in the database but not yet present in the + ``MetaData``. May be called multiple times to pick up tables recently + added to the database, however no special action is taken if a table + in this ``MetaData`` no longer exists in the database. + + :param bind: + A :class:`.Connectable` used to access the database; if None, uses + the existing bind on this ``MetaData``, if any. + + :param schema: + Optional, query and reflect tables from an alterate schema. + If None, the schema associated with this :class:`.MetaData` + is used, if any. + + :param views: + If True, also reflect views. + + :param only: + Optional. Load only a sub-set of available named tables. May be + specified as a sequence of names or a callable. + + If a sequence of names is provided, only those tables will be + reflected. An error is raised if a table is requested but not + available. Named tables already present in this ``MetaData`` are + ignored. + + If a callable is provided, it will be used as a boolean predicate to + filter the list of potential table names. The callable is called + with a table name and this ``MetaData`` instance as positional + arguments and should return a true value for any table to reflect. + + :param extend_existing: Passed along to each :class:`.Table` as + :paramref:`.Table.extend_existing`. + + .. versionadded:: 0.9.1 + + :param autoload_replace: Passed along to each :class:`.Table` as + :paramref:`.Table.autoload_replace`. + + .. versionadded:: 0.9.1 + + + """ + if bind is None: + bind = _bind_or_error(self) + + with bind.connect() as conn: + + reflect_opts = { + 'autoload': True, + 'autoload_with': conn, + 'extend_existing': extend_existing, + 'autoload_replace': autoload_replace + } + + if schema is None: + schema = self.schema + + if schema is not None: + reflect_opts['schema'] = schema + + available = util.OrderedSet(bind.engine.table_names(schema, + connection=conn)) + if views: + available.update( + bind.dialect.get_view_names(conn, schema) + ) + + if schema is not None: + available_w_schema = util.OrderedSet(["%s.%s" % (schema, name) + for name in available]) + else: + available_w_schema = available + + current = set(self.tables) + + if only is None: + load = [name for name, schname in + zip(available, available_w_schema) + if extend_existing or schname not in current] + elif util.callable(only): + load = [name for name, schname in + zip(available, available_w_schema) + if (extend_existing or schname not in current) + and only(name, self)] + else: + missing = [name for name in only if name not in available] + if missing: + s = schema and (" schema '%s'" % schema) or '' + raise exc.InvalidRequestError( + 'Could not reflect: requested table(s) not available ' + 'in %s%s: (%s)' % + (bind.engine.url, s, ', '.join(missing))) + load = [name for name in only if extend_existing or + name not in current] + + for name in load: + Table(name, self, **reflect_opts) + + def append_ddl_listener(self, event_name, listener): + """Append a DDL event listener to this ``MetaData``. + + .. deprecated:: 0.7 + See :class:`.DDLEvents`. + + """ + def adapt_listener(target, connection, **kw): + tables = kw['tables'] + listener(event, target, connection, tables=tables) + + event.listen(self, "" + event_name.replace('-', '_'), adapt_listener) + + def create_all(self, bind=None, tables=None, checkfirst=True): + """Create all tables stored in this metadata. + + Conditional by default, will not attempt to recreate tables already + present in the target database. + + :param bind: + A :class:`.Connectable` used to access the + database; if None, uses the existing bind on this ``MetaData``, if + any. + + :param tables: + Optional list of ``Table`` objects, which is a subset of the total + tables in the ``MetaData`` (others are ignored). + + :param checkfirst: + Defaults to True, don't issue CREATEs for tables already present + in the target database. + + """ + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaGenerator, + self, + checkfirst=checkfirst, + tables=tables) + + def drop_all(self, bind=None, tables=None, checkfirst=True): + """Drop all tables stored in this metadata. + + Conditional by default, will not attempt to drop tables not present in + the target database. + + :param bind: + A :class:`.Connectable` used to access the + database; if None, uses the existing bind on this ``MetaData``, if + any. + + :param tables: + Optional list of ``Table`` objects, which is a subset of the + total tables in the ``MetaData`` (others are ignored). + + :param checkfirst: + Defaults to True, only issue DROPs for tables confirmed to be + present in the target database. + + """ + if bind is None: + bind = _bind_or_error(self) + bind._run_visitor(ddl.SchemaDropper, + self, + checkfirst=checkfirst, + tables=tables) + + +class ThreadLocalMetaData(MetaData): + """A MetaData variant that presents a different ``bind`` in every thread. + + Makes the ``bind`` property of the MetaData a thread-local value, allowing + this collection of tables to be bound to different ``Engine`` + implementations or connections in each thread. + + The ThreadLocalMetaData starts off bound to None in each thread. Binds + must be made explicitly by assigning to the ``bind`` property or using + ``connect()``. You can also re-bind dynamically multiple times per + thread, just like a regular ``MetaData``. + + """ + + __visit_name__ = 'metadata' + + def __init__(self): + """Construct a ThreadLocalMetaData.""" + + self.context = util.threading.local() + self.__engines = {} + super(ThreadLocalMetaData, self).__init__() + + def bind(self): + """The bound Engine or Connection for this thread. + + This property may be assigned an Engine or Connection, or assigned a + string or URL to automatically create a basic Engine for this bind + with ``create_engine()``.""" + + return getattr(self.context, '_engine', None) + + @util.dependencies("sqlalchemy.engine.url") + def _bind_to(self, url, bind): + """Bind to a Connectable in the caller's thread.""" + + if isinstance(bind, util.string_types + (url.URL, )): + try: + self.context._engine = self.__engines[bind] + except KeyError: + e = sqlalchemy.create_engine(bind) + self.__engines[bind] = e + self.context._engine = e + else: + # TODO: this is squirrely. we shouldnt have to hold onto engines + # in a case like this + if bind not in self.__engines: + self.__engines[bind] = bind + self.context._engine = bind + + bind = property(bind, _bind_to) + + def is_bound(self): + """True if there is a bind for this thread.""" + return (hasattr(self.context, '_engine') and + self.context._engine is not None) + + def dispose(self): + """Dispose all bound engines, in all thread contexts.""" + + for e in self.__engines.values(): + if hasattr(e, 'dispose'): + e.dispose() + + + diff --git a/libs/sqlalchemy/sql/selectable.py b/libs/sqlalchemy/sql/selectable.py new file mode 100644 index 00000000..951268b2 --- /dev/null +++ b/libs/sqlalchemy/sql/selectable.py @@ -0,0 +1,3001 @@ +# sql/selectable.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""The :class:`.FromClause` class of SQL expression elements, representing +SQL tables and derived rowsets. + +""" + +from .elements import ClauseElement, TextClause, ClauseList, \ + and_, Grouping, UnaryExpression, literal_column +from .elements import _clone, \ + _literal_as_text, _interpret_as_column_or_from, _expand_cloned,\ + _select_iterables, _anonymous_label, _clause_element_as_expr,\ + _cloned_intersection, _cloned_difference, True_, _only_column_elements +from .base import Immutable, Executable, _generative, \ + ColumnCollection, ColumnSet, _from_objects, Generative +from . import type_api +from .. import inspection +from .. import util +from .. import exc +from operator import attrgetter +from . import operators +import operator +from .annotation import Annotated +import itertools + +def _interpret_as_from(element): + insp = inspection.inspect(element, raiseerr=False) + if insp is None: + if isinstance(element, util.string_types): + return TextClause(util.text_type(element)) + elif hasattr(insp, "selectable"): + return insp.selectable + raise exc.ArgumentError("FROM expression expected") + +def _interpret_as_select(element): + element = _interpret_as_from(element) + if isinstance(element, Alias): + element = element.original + if not isinstance(element, Select): + element = element.select() + return element + +def subquery(alias, *args, **kwargs): + """Return an :class:`.Alias` object derived + from a :class:`.Select`. + + name + alias name + + \*args, \**kwargs + + all other arguments are delivered to the + :func:`select` function. + + """ + return Select(*args, **kwargs).alias(alias) + + + +def alias(selectable, name=None, flat=False): + """Return an :class:`.Alias` object. + + An :class:`.Alias` represents any :class:`.FromClause` + with an alternate name assigned within SQL, typically using the ``AS`` + clause when generated, e.g. ``SELECT * FROM table AS aliasname``. + + Similar functionality is available via the + :meth:`~.FromClause.alias` method + available on all :class:`.FromClause` subclasses. + + When an :class:`.Alias` is created from a :class:`.Table` object, + this has the effect of the table being rendered + as ``tablename AS aliasname`` in a SELECT statement. + + For :func:`.select` objects, the effect is that of creating a named + subquery, i.e. ``(select ...) AS aliasname``. + + The ``name`` parameter is optional, and provides the name + to use in the rendered SQL. If blank, an "anonymous" name + will be deterministically generated at compile time. + Deterministic means the name is guaranteed to be unique against + other constructs used in the same statement, and will also be the + same name for each successive compilation of the same statement + object. + + :param selectable: any :class:`.FromClause` subclass, + such as a table, select statement, etc. + + :param name: string name to be assigned as the alias. + If ``None``, a name will be deterministically generated + at compile time. + + :param flat: Will be passed through to if the given selectable + is an instance of :class:`.Join` - see :meth:`.Join.alias` + for details. + + .. versionadded:: 0.9.0 + + """ + return selectable.alias(name=name, flat=flat) + + +class Selectable(ClauseElement): + """mark a class as being selectable""" + __visit_name__ = 'selectable' + + is_selectable = True + + @property + def selectable(self): + return self + + +class FromClause(Selectable): + """Represent an element that can be used within the ``FROM`` + clause of a ``SELECT`` statement. + + The most common forms of :class:`.FromClause` are the + :class:`.Table` and the :func:`.select` constructs. Key + features common to all :class:`.FromClause` objects include: + + * a :attr:`.c` collection, which provides per-name access to a collection + of :class:`.ColumnElement` objects. + * a :attr:`.primary_key` attribute, which is a collection of all those + :class:`.ColumnElement` objects that indicate the ``primary_key`` flag. + * Methods to generate various derivations of a "from" clause, including + :meth:`.FromClause.alias`, :meth:`.FromClause.join`, + :meth:`.FromClause.select`. + + + """ + __visit_name__ = 'fromclause' + named_with_column = False + _hide_froms = [] + + schema = None + """Define the 'schema' attribute for this :class:`.FromClause`. + + This is typically ``None`` for most objects except that of :class:`.Table`, + where it is taken as the value of the :paramref:`.Table.schema` argument. + + """ + + _memoized_property = util.group_expirable_memoized_property(["_columns"]) + + @util.dependencies("sqlalchemy.sql.functions") + def count(self, functions, whereclause=None, **params): + """return a SELECT COUNT generated against this + :class:`.FromClause`.""" + + if self.primary_key: + col = list(self.primary_key)[0] + else: + col = list(self.columns)[0] + return Select( + [functions.func.count(col).label('tbl_row_count')], + whereclause, + from_obj=[self], + **params) + + def select(self, whereclause=None, **params): + """return a SELECT of this :class:`.FromClause`. + + .. seealso:: + + :func:`~.sql.expression.select` - general purpose + method which allows for arbitrary column lists. + + """ + + return Select([self], whereclause, **params) + + def join(self, right, onclause=None, isouter=False): + """return a join of this :class:`.FromClause` against another + :class:`.FromClause`.""" + + return Join(self, right, onclause, isouter) + + def outerjoin(self, right, onclause=None): + """return an outer join of this :class:`.FromClause` against another + :class:`.FromClause`.""" + + return Join(self, right, onclause, True) + + def alias(self, name=None, flat=False): + """return an alias of this :class:`.FromClause`. + + This is shorthand for calling:: + + from sqlalchemy import alias + a = alias(self, name=name) + + See :func:`~.expression.alias` for details. + + """ + + return Alias(self, name) + + def is_derived_from(self, fromclause): + """Return True if this FromClause is 'derived' from the given + FromClause. + + An example would be an Alias of a Table is derived from that Table. + + """ + # this is essentially an "identity" check in the base class. + # Other constructs override this to traverse through + # contained elements. + return fromclause in self._cloned_set + + def _is_lexical_equivalent(self, other): + """Return True if this FromClause and the other represent + the same lexical identity. + + This tests if either one is a copy of the other, or + if they are the same via annotation identity. + + """ + return self._cloned_set.intersection(other._cloned_set) + + @util.dependencies("sqlalchemy.sql.util") + def replace_selectable(self, sqlutil, old, alias): + """replace all occurrences of FromClause 'old' with the given Alias + object, returning a copy of this :class:`.FromClause`. + + """ + + return sqlutil.ClauseAdapter(alias).traverse(self) + + def correspond_on_equivalents(self, column, equivalents): + """Return corresponding_column for the given column, or if None + search for a match in the given dictionary. + + """ + col = self.corresponding_column(column, require_embedded=True) + if col is None and col in equivalents: + for equiv in equivalents[col]: + nc = self.corresponding_column(equiv, require_embedded=True) + if nc: + return nc + return col + + def corresponding_column(self, column, require_embedded=False): + """Given a :class:`.ColumnElement`, return the exported + :class:`.ColumnElement` object from this :class:`.Selectable` + which corresponds to that original + :class:`~sqlalchemy.schema.Column` via a common ancestor + column. + + :param column: the target :class:`.ColumnElement` to be matched + + :param require_embedded: only return corresponding columns for + the given :class:`.ColumnElement`, if the given :class:`.ColumnElement` + is actually present within a sub-element + of this :class:`.FromClause`. Normally the column will match if + it merely shares a common ancestor with one of the exported + columns of this :class:`.FromClause`. + + """ + + def embedded(expanded_proxy_set, target_set): + for t in target_set.difference(expanded_proxy_set): + if not set(_expand_cloned([t]) + ).intersection(expanded_proxy_set): + return False + return True + + # don't dig around if the column is locally present + if self.c.contains_column(column): + return column + col, intersect = None, None + target_set = column.proxy_set + cols = self.c + for c in cols: + expanded_proxy_set = set(_expand_cloned(c.proxy_set)) + i = target_set.intersection(expanded_proxy_set) + if i and (not require_embedded + or embedded(expanded_proxy_set, target_set)): + if col is None: + + # no corresponding column yet, pick this one. + + col, intersect = c, i + elif len(i) > len(intersect): + + # 'c' has a larger field of correspondence than + # 'col'. i.e. selectable.c.a1_x->a1.c.x->table.c.x + # matches a1.c.x->table.c.x better than + # selectable.c.x->table.c.x does. + + col, intersect = c, i + elif i == intersect: + + # they have the same field of correspondence. see + # which proxy_set has fewer columns in it, which + # indicates a closer relationship with the root + # column. Also take into account the "weight" + # attribute which CompoundSelect() uses to give + # higher precedence to columns based on vertical + # position in the compound statement, and discard + # columns that have no reference to the target + # column (also occurs with CompoundSelect) + + col_distance = util.reduce(operator.add, + [sc._annotations.get('weight', 1) for sc in + col.proxy_set if sc.shares_lineage(column)]) + c_distance = util.reduce(operator.add, + [sc._annotations.get('weight', 1) for sc in + c.proxy_set if sc.shares_lineage(column)]) + if c_distance < col_distance: + col, intersect = c, i + return col + + @property + def description(self): + """a brief description of this FromClause. + + Used primarily for error message formatting. + + """ + return getattr(self, 'name', self.__class__.__name__ + " object") + + def _reset_exported(self): + """delete memoized collections when a FromClause is cloned.""" + + self._memoized_property.expire_instance(self) + + @_memoized_property + def columns(self): + """A named-based collection of :class:`.ColumnElement` objects + maintained by this :class:`.FromClause`. + + The :attr:`.columns`, or :attr:`.c` collection, is the gateway + to the construction of SQL expressions using table-bound or + other selectable-bound columns:: + + select([mytable]).where(mytable.c.somecolumn == 5) + + """ + + if '_columns' not in self.__dict__: + self._init_collections() + self._populate_column_collection() + return self._columns.as_immutable() + + @_memoized_property + def primary_key(self): + """Return the collection of Column objects which comprise the + primary key of this FromClause.""" + + self._init_collections() + self._populate_column_collection() + return self.primary_key + + @_memoized_property + def foreign_keys(self): + """Return the collection of ForeignKey objects which this + FromClause references.""" + + self._init_collections() + self._populate_column_collection() + return self.foreign_keys + + c = property(attrgetter('columns'), + doc="An alias for the :attr:`.columns` attribute.") + _select_iterable = property(attrgetter('columns')) + + def _init_collections(self): + assert '_columns' not in self.__dict__ + assert 'primary_key' not in self.__dict__ + assert 'foreign_keys' not in self.__dict__ + + self._columns = ColumnCollection() + self.primary_key = ColumnSet() + self.foreign_keys = set() + + @property + def _cols_populated(self): + return '_columns' in self.__dict__ + + def _populate_column_collection(self): + """Called on subclasses to establish the .c collection. + + Each implementation has a different way of establishing + this collection. + + """ + + def _refresh_for_new_column(self, column): + """Given a column added to the .c collection of an underlying + selectable, produce the local version of that column, assuming this + selectable ultimately should proxy this column. + + this is used to "ping" a derived selectable to add a new column + to its .c. collection when a Column has been added to one of the + Table objects it ultimtely derives from. + + If the given selectable hasn't populated it's .c. collection yet, + it should at least pass on the message to the contained selectables, + but it will return None. + + This method is currently used by Declarative to allow Table + columns to be added to a partially constructed inheritance + mapping that may have already produced joins. The method + isn't public right now, as the full span of implications + and/or caveats aren't yet clear. + + It's also possible that this functionality could be invoked by + default via an event, which would require that + selectables maintain a weak referencing collection of all + derivations. + + """ + if not self._cols_populated: + return None + elif column.key in self.columns and self.columns[column.key] is column: + return column + else: + return None + + +class Join(FromClause): + """represent a ``JOIN`` construct between two :class:`.FromClause` + elements. + + The public constructor function for :class:`.Join` is the module-level + :func:`join()` function, as well as the :func:`join()` method available + off all :class:`.FromClause` subclasses. + + """ + __visit_name__ = 'join' + + def __init__(self, left, right, onclause=None, isouter=False): + """Construct a new :class:`.Join`. + + The usual entrypoint here is the :func:`~.expression.join` + function or the :meth:`.FromClause.join` method of any + :class:`.FromClause` object. + + """ + self.left = _interpret_as_from(left) + self.right = _interpret_as_from(right).self_group() + + if onclause is None: + self.onclause = self._match_primaries(self.left, self.right) + else: + self.onclause = onclause + + self.isouter = isouter + + @classmethod + def _create_outerjoin(cls, left, right, onclause=None): + """Return an ``OUTER JOIN`` clause element. + + The returned object is an instance of :class:`.Join`. + + Similar functionality is also available via the + :meth:`~.FromClause.outerjoin()` method on any + :class:`.FromClause`. + + :param left: The left side of the join. + + :param right: The right side of the join. + + :param onclause: Optional criterion for the ``ON`` clause, is + derived from foreign key relationships established between + left and right otherwise. + + To chain joins together, use the :meth:`.FromClause.join` or + :meth:`.FromClause.outerjoin` methods on the resulting + :class:`.Join` object. + + """ + return cls(left, right, onclause, isouter=True) + + + @classmethod + def _create_join(cls, left, right, onclause=None, isouter=False): + """Return a ``JOIN`` clause element (regular inner join). + + The returned object is an instance of :class:`.Join`. + + Similar functionality is also available via the + :meth:`~.FromClause.join()` method on any + :class:`.FromClause`. + + :param left: The left side of the join. + + :param right: The right side of the join. + + :param onclause: Optional criterion for the ``ON`` clause, is + derived from foreign key relationships established between + left and right otherwise. + + :param isouter: if True, produce an outer join; synonymous + with :func:`.outerjoin`. + + To chain joins together, use the :meth:`.FromClause.join` or + :meth:`.FromClause.outerjoin` methods on the resulting + :class:`.Join` object. + + + """ + return cls(left, right, onclause, isouter) + + + @property + def description(self): + return "Join object on %s(%d) and %s(%d)" % ( + self.left.description, + id(self.left), + self.right.description, + id(self.right)) + + def is_derived_from(self, fromclause): + return fromclause is self or \ + self.left.is_derived_from(fromclause) or \ + self.right.is_derived_from(fromclause) + + def self_group(self, against=None): + return FromGrouping(self) + + @util.dependencies("sqlalchemy.sql.util") + def _populate_column_collection(self, sqlutil): + columns = [c for c in self.left.columns] + \ + [c for c in self.right.columns] + + self.primary_key.extend(sqlutil.reduce_columns( + (c for c in columns if c.primary_key), self.onclause)) + self._columns.update((col._label, col) for col in columns) + self.foreign_keys.update(itertools.chain( + *[col.foreign_keys for col in columns])) + + def _refresh_for_new_column(self, column): + col = self.left._refresh_for_new_column(column) + if col is None: + col = self.right._refresh_for_new_column(column) + if col is not None: + if self._cols_populated: + self._columns[col._label] = col + self.foreign_keys.add(col) + if col.primary_key: + self.primary_key.add(col) + return col + return None + + def _copy_internals(self, clone=_clone, **kw): + self._reset_exported() + self.left = clone(self.left, **kw) + self.right = clone(self.right, **kw) + self.onclause = clone(self.onclause, **kw) + + def get_children(self, **kwargs): + return self.left, self.right, self.onclause + + def _match_primaries(self, left, right): + if isinstance(left, Join): + left_right = left.right + else: + left_right = None + return self._join_condition(left, right, a_subset=left_right) + + @classmethod + def _join_condition(cls, a, b, ignore_nonexistent_tables=False, + a_subset=None, + consider_as_foreign_keys=None): + """create a join condition between two tables or selectables. + + e.g.:: + + join_condition(tablea, tableb) + + would produce an expression along the lines of:: + + tablea.c.id==tableb.c.tablea_id + + The join is determined based on the foreign key relationships + between the two selectables. If there are multiple ways + to join, or no way to join, an error is raised. + + :param ignore_nonexistent_tables: Deprecated - this + flag is no longer used. Only resolution errors regarding + the two given tables are propagated. + + :param a_subset: An optional expression that is a sub-component + of ``a``. An attempt will be made to join to just this sub-component + first before looking at the full ``a`` construct, and if found + will be successful even if there are other ways to join to ``a``. + This allows the "right side" of a join to be passed thereby + providing a "natural join". + + """ + crit = [] + constraints = set() + + for left in (a_subset, a): + if left is None: + continue + for fk in sorted( + b.foreign_keys, + key=lambda fk: fk.parent._creation_order): + if consider_as_foreign_keys is not None and \ + fk.parent not in consider_as_foreign_keys: + continue + try: + col = fk.get_referent(left) + except exc.NoReferenceError as nrte: + if nrte.table_name == left.name: + raise + else: + continue + + if col is not None: + crit.append(col == fk.parent) + constraints.add(fk.constraint) + if left is not b: + for fk in sorted( + left.foreign_keys, + key=lambda fk: fk.parent._creation_order): + if consider_as_foreign_keys is not None and \ + fk.parent not in consider_as_foreign_keys: + continue + try: + col = fk.get_referent(b) + except exc.NoReferenceError as nrte: + if nrte.table_name == b.name: + raise + else: + # this is totally covered. can't get + # coverage to mark it. + continue + + if col is not None: + crit.append(col == fk.parent) + constraints.add(fk.constraint) + if crit: + break + + if len(crit) == 0: + if isinstance(b, FromGrouping): + hint = " Perhaps you meant to convert the right side to a "\ + "subquery using alias()?" + else: + hint = "" + raise exc.NoForeignKeysError( + "Can't find any foreign key relationships " + "between '%s' and '%s'.%s" % (a.description, b.description, hint)) + elif len(constraints) > 1: + raise exc.AmbiguousForeignKeysError( + "Can't determine join between '%s' and '%s'; " + "tables have more than one foreign key " + "constraint relationship between them. " + "Please specify the 'onclause' of this " + "join explicitly." % (a.description, b.description)) + elif len(crit) == 1: + return (crit[0]) + else: + return and_(*crit) + + + def select(self, whereclause=None, **kwargs): + """Create a :class:`.Select` from this :class:`.Join`. + + The equivalent long-hand form, given a :class:`.Join` object + ``j``, is:: + + from sqlalchemy import select + j = select([j.left, j.right], **kw).\\ + where(whereclause).\\ + select_from(j) + + :param whereclause: the WHERE criterion that will be sent to + the :func:`select()` function + + :param \**kwargs: all other kwargs are sent to the + underlying :func:`select()` function. + + """ + collist = [self.left, self.right] + + return Select(collist, whereclause, from_obj=[self], **kwargs) + + @property + def bind(self): + return self.left.bind or self.right.bind + + @util.dependencies("sqlalchemy.sql.util") + def alias(self, sqlutil, name=None, flat=False): + """return an alias of this :class:`.Join`. + + The default behavior here is to first produce a SELECT + construct from this :class:`.Join`, then to produce a + :class:`.Alias` from that. So given a join of the form:: + + j = table_a.join(table_b, table_a.c.id == table_b.c.a_id) + + The JOIN by itself would look like:: + + table_a JOIN table_b ON table_a.id = table_b.a_id + + Whereas the alias of the above, ``j.alias()``, would in a + SELECT context look like:: + + (SELECT table_a.id AS table_a_id, table_b.id AS table_b_id, + table_b.a_id AS table_b_a_id + FROM table_a + JOIN table_b ON table_a.id = table_b.a_id) AS anon_1 + + The equivalent long-hand form, given a :class:`.Join` object + ``j``, is:: + + from sqlalchemy import select, alias + j = alias( + select([j.left, j.right]).\\ + select_from(j).\\ + with_labels(True).\\ + correlate(False), + name=name + ) + + The selectable produced by :meth:`.Join.alias` features the same + columns as that of the two individual selectables presented under + a single name - the individual columns are "auto-labeled", meaning + the ``.c.`` collection of the resulting :class:`.Alias` represents + the names of the individual columns using a ``_`` + scheme:: + + j.c.table_a_id + j.c.table_b_a_id + + :meth:`.Join.alias` also features an alternate + option for aliasing joins which produces no enclosing SELECT and + does not normally apply labels to the column names. The + ``flat=True`` option will call :meth:`.FromClause.alias` + against the left and right sides individually. + Using this option, no new ``SELECT`` is produced; + we instead, from a construct as below:: + + j = table_a.join(table_b, table_a.c.id == table_b.c.a_id) + j = j.alias(flat=True) + + we get a result like this:: + + table_a AS table_a_1 JOIN table_b AS table_b_1 ON + table_a_1.id = table_b_1.a_id + + The ``flat=True`` argument is also propagated to the contained + selectables, so that a composite join such as:: + + j = table_a.join( + table_b.join(table_c, + table_b.c.id == table_c.c.b_id), + table_b.c.a_id == table_a.c.id + ).alias(flat=True) + + Will produce an expression like:: + + table_a AS table_a_1 JOIN ( + table_b AS table_b_1 JOIN table_c AS table_c_1 + ON table_b_1.id = table_c_1.b_id + ) ON table_a_1.id = table_b_1.a_id + + The standalone :func:`~.expression.alias` function as well as the + base :meth:`.FromClause.alias` method also support the ``flat=True`` + argument as a no-op, so that the argument can be passed to the + ``alias()`` method of any selectable. + + .. versionadded:: 0.9.0 Added the ``flat=True`` option to create + "aliases" of joins without enclosing inside of a SELECT + subquery. + + :param name: name given to the alias. + + :param flat: if True, produce an alias of the left and right + sides of this :class:`.Join` and return the join of those + two selectables. This produces join expression that does not + include an enclosing SELECT. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :func:`~.expression.alias` + + """ + if flat: + assert name is None, "Can't send name argument with flat" + left_a, right_a = self.left.alias(flat=True), \ + self.right.alias(flat=True) + adapter = sqlutil.ClauseAdapter(left_a).\ + chain(sqlutil.ClauseAdapter(right_a)) + + return left_a.join(right_a, + adapter.traverse(self.onclause), isouter=self.isouter) + else: + return self.select(use_labels=True, correlate=False).alias(name) + + @property + def _hide_froms(self): + return itertools.chain(*[_from_objects(x.left, x.right) + for x in self._cloned_set]) + + @property + def _from_objects(self): + return [self] + \ + self.onclause._from_objects + \ + self.left._from_objects + \ + self.right._from_objects + + +class Alias(FromClause): + """Represents an table or selectable alias (AS). + + Represents an alias, as typically applied to any table or + sub-select within a SQL statement using the ``AS`` keyword (or + without the keyword on certain databases such as Oracle). + + This object is constructed from the :func:`~.expression.alias` module level + function as well as the :meth:`.FromClause.alias` method available on all + :class:`.FromClause` subclasses. + + """ + + __visit_name__ = 'alias' + named_with_column = True + + def __init__(self, selectable, name=None): + baseselectable = selectable + while isinstance(baseselectable, Alias): + baseselectable = baseselectable.element + self.original = baseselectable + self.supports_execution = baseselectable.supports_execution + if self.supports_execution: + self._execution_options = baseselectable._execution_options + self.element = selectable + if name is None: + if self.original.named_with_column: + name = getattr(self.original, 'name', None) + name = _anonymous_label('%%(%d %s)s' % (id(self), name + or 'anon')) + self.name = name + + @property + def description(self): + if util.py3k: + return self.name + else: + return self.name.encode('ascii', 'backslashreplace') + + def as_scalar(self): + try: + return self.element.as_scalar() + except AttributeError: + raise AttributeError("Element %s does not support " + "'as_scalar()'" % self.element) + + def is_derived_from(self, fromclause): + if fromclause in self._cloned_set: + return True + return self.element.is_derived_from(fromclause) + + def _populate_column_collection(self): + for col in self.element.columns: + col._make_proxy(self) + + def _refresh_for_new_column(self, column): + col = self.element._refresh_for_new_column(column) + if col is not None: + if not self._cols_populated: + return None + else: + return col._make_proxy(self) + else: + return None + + def _copy_internals(self, clone=_clone, **kw): + # don't apply anything to an aliased Table + # for now. May want to drive this from + # the given **kw. + if isinstance(self.element, TableClause): + return + self._reset_exported() + self.element = clone(self.element, **kw) + baseselectable = self.element + while isinstance(baseselectable, Alias): + baseselectable = baseselectable.element + self.original = baseselectable + + def get_children(self, column_collections=True, **kw): + if column_collections: + for c in self.c: + yield c + yield self.element + + @property + def _from_objects(self): + return [self] + + @property + def bind(self): + return self.element.bind + + +class CTE(Alias): + """Represent a Common Table Expression. + + The :class:`.CTE` object is obtained using the + :meth:`.SelectBase.cte` method from any selectable. + See that method for complete examples. + + .. versionadded:: 0.7.6 + + """ + __visit_name__ = 'cte' + + def __init__(self, selectable, + name=None, + recursive=False, + _cte_alias=None, + _restates=frozenset()): + self.recursive = recursive + self._cte_alias = _cte_alias + self._restates = _restates + super(CTE, self).__init__(selectable, name=name) + + def alias(self, name=None, flat=False): + return CTE( + self.original, + name=name, + recursive=self.recursive, + _cte_alias=self, + ) + + def union(self, other): + return CTE( + self.original.union(other), + name=self.name, + recursive=self.recursive, + _restates=self._restates.union([self]) + ) + + def union_all(self, other): + return CTE( + self.original.union_all(other), + name=self.name, + recursive=self.recursive, + _restates=self._restates.union([self]) + ) + + + + +class FromGrouping(FromClause): + """Represent a grouping of a FROM clause""" + __visit_name__ = 'grouping' + + def __init__(self, element): + self.element = element + + def _init_collections(self): + pass + + @property + def columns(self): + return self.element.columns + + @property + def primary_key(self): + return self.element.primary_key + + @property + def foreign_keys(self): + return self.element.foreign_keys + + def is_derived_from(self, element): + return self.element.is_derived_from(element) + + def alias(self, **kw): + return FromGrouping(self.element.alias(**kw)) + + @property + def _hide_froms(self): + return self.element._hide_froms + + def get_children(self, **kwargs): + return self.element, + + def _copy_internals(self, clone=_clone, **kw): + self.element = clone(self.element, **kw) + + @property + def _from_objects(self): + return self.element._from_objects + + def __getattr__(self, attr): + return getattr(self.element, attr) + + def __getstate__(self): + return {'element': self.element} + + def __setstate__(self, state): + self.element = state['element'] + +class TableClause(Immutable, FromClause): + """Represents a minimal "table" construct. + + This is a lightweight table object that has only a name and a + collection of columns, which are typically produced + by the :func:`.expression.column` function:: + + from sqlalchemy.sql import table, column + + user = table("user", + column("id"), + column("name"), + column("description"), + ) + + The :class:`.TableClause` construct serves as the base for + the more commonly used :class:`~.schema.Table` object, providing + the usual set of :class:`~.expression.FromClause` services including + the ``.c.`` collection and statement generation methods. + + It does **not** provide all the additional schema-level services + of :class:`~.schema.Table`, including constraints, references to other + tables, or support for :class:`.MetaData`-level services. It's useful + on its own as an ad-hoc construct used to generate quick SQL + statements when a more fully fledged :class:`~.schema.Table` + is not on hand. + + """ + + __visit_name__ = 'table' + + named_with_column = True + + implicit_returning = False + """:class:`.TableClause` doesn't support having a primary key or column + -level defaults, so implicit returning doesn't apply.""" + + _autoincrement_column = None + """No PK or default support so no autoincrement column.""" + + def __init__(self, name, *columns): + """Produce a new :class:`.TableClause`. + + The object returned is an instance of :class:`.TableClause`, which + represents the "syntactical" portion of the schema-level + :class:`~.schema.Table` object. + It may be used to construct lightweight table constructs. + + Note that the :func:`.expression.table` function is not part of + the ``sqlalchemy`` namespace. It must be imported from the + ``sql`` package:: + + from sqlalchemy.sql import table, column + + :param name: Name of the table. + + :param columns: A collection of :func:`.expression.column` constructs. + + """ + + super(TableClause, self).__init__() + self.name = self.fullname = name + self._columns = ColumnCollection() + self.primary_key = ColumnSet() + self.foreign_keys = set() + for c in columns: + self.append_column(c) + + def _init_collections(self): + pass + + @util.memoized_property + def description(self): + if util.py3k: + return self.name + else: + return self.name.encode('ascii', 'backslashreplace') + + def append_column(self, c): + self._columns[c.key] = c + c.table = self + + def get_children(self, column_collections=True, **kwargs): + if column_collections: + return [c for c in self.c] + else: + return [] + + @util.dependencies("sqlalchemy.sql.functions") + def count(self, functions, whereclause=None, **params): + """return a SELECT COUNT generated against this + :class:`.TableClause`.""" + + if self.primary_key: + col = list(self.primary_key)[0] + else: + col = list(self.columns)[0] + return Select( + [functions.func.count(col).label('tbl_row_count')], + whereclause, + from_obj=[self], + **params) + + @util.dependencies("sqlalchemy.sql.dml") + def insert(self, dml, values=None, inline=False, **kwargs): + """Generate an :func:`.insert` construct against this + :class:`.TableClause`. + + E.g.:: + + table.insert().values(name='foo') + + See :func:`.insert` for argument and usage information. + + """ + + return dml.Insert(self, values=values, inline=inline, **kwargs) + + @util.dependencies("sqlalchemy.sql.dml") + def update(self, dml, whereclause=None, values=None, inline=False, **kwargs): + """Generate an :func:`.update` construct against this + :class:`.TableClause`. + + E.g.:: + + table.update().where(table.c.id==7).values(name='foo') + + See :func:`.update` for argument and usage information. + + """ + + return dml.Update(self, whereclause=whereclause, + values=values, inline=inline, **kwargs) + + @util.dependencies("sqlalchemy.sql.dml") + def delete(self, dml, whereclause=None, **kwargs): + """Generate a :func:`.delete` construct against this + :class:`.TableClause`. + + E.g.:: + + table.delete().where(table.c.id==7) + + See :func:`.delete` for argument and usage information. + + """ + + return dml.Delete(self, whereclause, **kwargs) + + @property + def _from_objects(self): + return [self] + + +class ForUpdateArg(ClauseElement): + + @classmethod + def parse_legacy_select(self, arg): + """Parse the for_update arugment of :func:`.select`. + + :param mode: Defines the lockmode to use. + + ``None`` - translates to no lockmode + + ``'update'`` - translates to ``FOR UPDATE`` + (standard SQL, supported by most dialects) + + ``'nowait'`` - translates to ``FOR UPDATE NOWAIT`` + (supported by Oracle, PostgreSQL 8.1 upwards) + + ``'read'`` - translates to ``LOCK IN SHARE MODE`` (for MySQL), + and ``FOR SHARE`` (for PostgreSQL) + + ``'read_nowait'`` - translates to ``FOR SHARE NOWAIT`` + (supported by PostgreSQL). ``FOR SHARE`` and + ``FOR SHARE NOWAIT`` (PostgreSQL). + + """ + if arg in (None, False): + return None + + nowait = read = False + if arg == 'nowait': + nowait = True + elif arg == 'read': + read = True + elif arg == 'read_nowait': + read = nowait = True + elif arg is not True: + raise exc.ArgumentError("Unknown for_update argument: %r" % arg) + + return ForUpdateArg(read=read, nowait=nowait) + + @property + def legacy_for_update_value(self): + if self.read and not self.nowait: + return "read" + elif self.read and self.nowait: + return "read_nowait" + elif self.nowait: + return "nowait" + else: + return True + + def _copy_internals(self, clone=_clone, **kw): + if self.of is not None: + self.of = [clone(col, **kw) for col in self.of] + + def __init__(self, nowait=False, read=False, of=None): + """Represents arguments specified to :meth:`.Select.for_update`. + + .. versionadded:: 0.9.0 + """ + + self.nowait = nowait + self.read = read + if of is not None: + self.of = [_interpret_as_column_or_from(elem) + for elem in util.to_list(of)] + else: + self.of = None + + +class SelectBase(Executable, FromClause): + """Base class for SELECT statements. + + + This includes :class:`.Select`, :class:`.CompoundSelect` and + :class:`.TextAsFrom`. + + + """ + + def as_scalar(self): + """return a 'scalar' representation of this selectable, which can be + used as a column expression. + + Typically, a select statement which has only one column in its columns + clause is eligible to be used as a scalar expression. + + The returned object is an instance of + :class:`ScalarSelect`. + + """ + return ScalarSelect(self) + + + def label(self, name): + """return a 'scalar' representation of this selectable, embedded as a + subquery with a label. + + .. seealso:: + + :meth:`~.SelectBase.as_scalar`. + + """ + return self.as_scalar().label(name) + + def cte(self, name=None, recursive=False): + """Return a new :class:`.CTE`, or Common Table Expression instance. + + Common table expressions are a SQL standard whereby SELECT + statements can draw upon secondary statements specified along + with the primary statement, using a clause called "WITH". + Special semantics regarding UNION can also be employed to + allow "recursive" queries, where a SELECT statement can draw + upon the set of rows that have previously been selected. + + SQLAlchemy detects :class:`.CTE` objects, which are treated + similarly to :class:`.Alias` objects, as special elements + to be delivered to the FROM clause of the statement as well + as to a WITH clause at the top of the statement. + + .. versionadded:: 0.7.6 + + :param name: name given to the common table expression. Like + :meth:`._FromClause.alias`, the name can be left as ``None`` + in which case an anonymous symbol will be used at query + compile time. + :param recursive: if ``True``, will render ``WITH RECURSIVE``. + A recursive common table expression is intended to be used in + conjunction with UNION ALL in order to derive rows + from those already selected. + + The following examples illustrate two examples from + Postgresql's documentation at + http://www.postgresql.org/docs/8.4/static/queries-with.html. + + Example 1, non recursive:: + + from sqlalchemy import Table, Column, String, Integer, MetaData, \\ + select, func + + metadata = MetaData() + + orders = Table('orders', metadata, + Column('region', String), + Column('amount', Integer), + Column('product', String), + Column('quantity', Integer) + ) + + regional_sales = select([ + orders.c.region, + func.sum(orders.c.amount).label('total_sales') + ]).group_by(orders.c.region).cte("regional_sales") + + + top_regions = select([regional_sales.c.region]).\\ + where( + regional_sales.c.total_sales > + select([ + func.sum(regional_sales.c.total_sales)/10 + ]) + ).cte("top_regions") + + statement = select([ + orders.c.region, + orders.c.product, + func.sum(orders.c.quantity).label("product_units"), + func.sum(orders.c.amount).label("product_sales") + ]).where(orders.c.region.in_( + select([top_regions.c.region]) + )).group_by(orders.c.region, orders.c.product) + + result = conn.execute(statement).fetchall() + + Example 2, WITH RECURSIVE:: + + from sqlalchemy import Table, Column, String, Integer, MetaData, \\ + select, func + + metadata = MetaData() + + parts = Table('parts', metadata, + Column('part', String), + Column('sub_part', String), + Column('quantity', Integer), + ) + + included_parts = select([ + parts.c.sub_part, + parts.c.part, + parts.c.quantity]).\\ + where(parts.c.part=='our part').\\ + cte(recursive=True) + + + incl_alias = included_parts.alias() + parts_alias = parts.alias() + included_parts = included_parts.union_all( + select([ + parts_alias.c.part, + parts_alias.c.sub_part, + parts_alias.c.quantity + ]). + where(parts_alias.c.part==incl_alias.c.sub_part) + ) + + statement = select([ + included_parts.c.sub_part, + func.sum(included_parts.c.quantity). + label('total_quantity') + ]).\ + select_from(included_parts.join(parts, + included_parts.c.part==parts.c.part)).\\ + group_by(included_parts.c.sub_part) + + result = conn.execute(statement).fetchall() + + + .. seealso:: + + :meth:`.orm.query.Query.cte` - ORM version of :meth:`.SelectBase.cte`. + + """ + return CTE(self, name=name, recursive=recursive) + + @_generative + @util.deprecated('0.6', + message="``autocommit()`` is deprecated. Use " + ":meth:`.Executable.execution_options` with the " + "'autocommit' flag.") + def autocommit(self): + """return a new selectable with the 'autocommit' flag set to + True. + """ + + self._execution_options = \ + self._execution_options.union({'autocommit': True}) + + def _generate(self): + """Override the default _generate() method to also clear out + exported collections.""" + + s = self.__class__.__new__(self.__class__) + s.__dict__ = self.__dict__.copy() + s._reset_exported() + return s + + @property + def _from_objects(self): + return [self] + +class GenerativeSelect(SelectBase): + """Base class for SELECT statements where additional elements can be + added. + + This serves as the base for :class:`.Select` and :class:`.CompoundSelect` + where elements such as ORDER BY, GROUP BY can be added and column rendering + can be controlled. Compare to :class:`.TextAsFrom`, which, while it + subclasses :class:`.SelectBase` and is also a SELECT construct, represents + a fixed textual string which cannot be altered at this level, only + wrapped as a subquery. + + .. versionadded:: 0.9.0 :class:`.GenerativeSelect` was added to + provide functionality specific to :class:`.Select` and :class:`.CompoundSelect` + while allowing :class:`.SelectBase` to be used for other SELECT-like + objects, e.g. :class:`.TextAsFrom`. + + """ + _order_by_clause = ClauseList() + _group_by_clause = ClauseList() + _limit = None + _offset = None + _for_update_arg = None + + def __init__(self, + use_labels=False, + for_update=False, + limit=None, + offset=None, + order_by=None, + group_by=None, + bind=None, + autocommit=None): + self.use_labels = use_labels + + if for_update is not False: + self._for_update_arg = ForUpdateArg.parse_legacy_select(for_update) + + if autocommit is not None: + util.warn_deprecated('autocommit on select() is ' + 'deprecated. Use .execution_options(a' + 'utocommit=True)') + self._execution_options = \ + self._execution_options.union( + {'autocommit': autocommit}) + if limit is not None: + self._limit = util.asint(limit) + if offset is not None: + self._offset = util.asint(offset) + self._bind = bind + + if order_by is not None: + self._order_by_clause = ClauseList(*util.to_list(order_by)) + if group_by is not None: + self._group_by_clause = ClauseList(*util.to_list(group_by)) + + @property + def for_update(self): + """Provide legacy dialect support for the ``for_update`` attribute. + """ + if self._for_update_arg is not None: + return self._for_update_arg.legacy_for_update_value + else: + return None + + @for_update.setter + def for_update(self, value): + self._for_update_arg = ForUpdateArg.parse_legacy_select(value) + + @_generative + def with_for_update(self, nowait=False, read=False, of=None): + """Specify a ``FOR UPDATE`` clause for this :class:`.GenerativeSelect`. + + E.g.:: + + stmt = select([table]).with_for_update(nowait=True) + + On a database like Postgresql or Oracle, the above would render a + statement like:: + + SELECT table.a, table.b FROM table FOR UPDATE NOWAIT + + on other backends, the ``nowait`` option is ignored and instead + would produce:: + + SELECT table.a, table.b FROM table FOR UPDATE + + When called with no arguments, the statement will render with + the suffix ``FOR UPDATE``. Additional arguments can then be + provided which allow for common database-specific + variants. + + :param nowait: boolean; will render ``FOR UPDATE NOWAIT`` on Oracle and + Postgresql dialects. + + :param read: boolean; will render ``LOCK IN SHARE MODE`` on MySQL, + ``FOR SHARE`` on Postgresql. On Postgresql, when combined with + ``nowait``, will render ``FOR SHARE NOWAIT``. + + :param of: SQL expression or list of SQL expression elements + (typically :class:`.Column` objects or a compatible expression) which + will render into a ``FOR UPDATE OF`` clause; supported by PostgreSQL + and Oracle. May render as a table or as a column depending on + backend. + + .. versionadded:: 0.9.0 + + """ + self._for_update_arg = ForUpdateArg(nowait=nowait, read=read, of=of) + + @_generative + def apply_labels(self): + """return a new selectable with the 'use_labels' flag set to True. + + This will result in column expressions being generated using labels + against their table name, such as "SELECT somecolumn AS + tablename_somecolumn". This allows selectables which contain multiple + FROM clauses to produce a unique set of column names regardless of + name conflicts among the individual FROM clauses. + + """ + self.use_labels = True + + @_generative + def limit(self, limit): + """return a new selectable with the given LIMIT criterion + applied.""" + + self._limit = util.asint(limit) + + @_generative + def offset(self, offset): + """return a new selectable with the given OFFSET criterion + applied.""" + + self._offset = util.asint(offset) + + @_generative + def order_by(self, *clauses): + """return a new selectable with the given list of ORDER BY + criterion applied. + + The criterion will be appended to any pre-existing ORDER BY + criterion. + + """ + + self.append_order_by(*clauses) + + @_generative + def group_by(self, *clauses): + """return a new selectable with the given list of GROUP BY + criterion applied. + + The criterion will be appended to any pre-existing GROUP BY + criterion. + + """ + + self.append_group_by(*clauses) + + def append_order_by(self, *clauses): + """Append the given ORDER BY criterion applied to this selectable. + + The criterion will be appended to any pre-existing ORDER BY criterion. + + This is an **in-place** mutation method; the + :meth:`~.GenerativeSelect.order_by` method is preferred, as it provides standard + :term:`method chaining`. + + """ + if len(clauses) == 1 and clauses[0] is None: + self._order_by_clause = ClauseList() + else: + if getattr(self, '_order_by_clause', None) is not None: + clauses = list(self._order_by_clause) + list(clauses) + self._order_by_clause = ClauseList(*clauses) + + def append_group_by(self, *clauses): + """Append the given GROUP BY criterion applied to this selectable. + + The criterion will be appended to any pre-existing GROUP BY criterion. + + This is an **in-place** mutation method; the + :meth:`~.GenerativeSelect.group_by` method is preferred, as it provides standard + :term:`method chaining`. + + """ + if len(clauses) == 1 and clauses[0] is None: + self._group_by_clause = ClauseList() + else: + if getattr(self, '_group_by_clause', None) is not None: + clauses = list(self._group_by_clause) + list(clauses) + self._group_by_clause = ClauseList(*clauses) + + +class CompoundSelect(GenerativeSelect): + """Forms the basis of ``UNION``, ``UNION ALL``, and other + SELECT-based set operations. + + + .. seealso:: + + :func:`.union` + + :func:`.union_all` + + :func:`.intersect` + + :func:`.intersect_all` + + :func:`.except` + + :func:`.except_all` + + """ + + __visit_name__ = 'compound_select' + + UNION = util.symbol('UNION') + UNION_ALL = util.symbol('UNION ALL') + EXCEPT = util.symbol('EXCEPT') + EXCEPT_ALL = util.symbol('EXCEPT ALL') + INTERSECT = util.symbol('INTERSECT') + INTERSECT_ALL = util.symbol('INTERSECT ALL') + + def __init__(self, keyword, *selects, **kwargs): + self._auto_correlate = kwargs.pop('correlate', False) + self.keyword = keyword + self.selects = [] + + numcols = None + + # some DBs do not like ORDER BY in the inner queries of a UNION, etc. + for n, s in enumerate(selects): + s = _clause_element_as_expr(s) + + if not numcols: + numcols = len(s.c) + elif len(s.c) != numcols: + raise exc.ArgumentError('All selectables passed to ' + 'CompoundSelect must have identical numbers of ' + 'columns; select #%d has %d columns, select ' + '#%d has %d' % (1, len(self.selects[0].c), n + + 1, len(s.c))) + + self.selects.append(s.self_group(self)) + + GenerativeSelect.__init__(self, **kwargs) + + @classmethod + def _create_union(cls, *selects, **kwargs): + """Return a ``UNION`` of multiple selectables. + + The returned object is an instance of + :class:`.CompoundSelect`. + + A similar :func:`union()` method is available on all + :class:`.FromClause` subclasses. + + \*selects + a list of :class:`.Select` instances. + + \**kwargs + available keyword arguments are the same as those of + :func:`select`. + + """ + return CompoundSelect(CompoundSelect.UNION, *selects, **kwargs) + + @classmethod + def _create_union_all(cls, *selects, **kwargs): + """Return a ``UNION ALL`` of multiple selectables. + + The returned object is an instance of + :class:`.CompoundSelect`. + + A similar :func:`union_all()` method is available on all + :class:`.FromClause` subclasses. + + \*selects + a list of :class:`.Select` instances. + + \**kwargs + available keyword arguments are the same as those of + :func:`select`. + + """ + return CompoundSelect(CompoundSelect.UNION_ALL, *selects, **kwargs) + + + @classmethod + def _create_except(cls, *selects, **kwargs): + """Return an ``EXCEPT`` of multiple selectables. + + The returned object is an instance of + :class:`.CompoundSelect`. + + \*selects + a list of :class:`.Select` instances. + + \**kwargs + available keyword arguments are the same as those of + :func:`select`. + + """ + return CompoundSelect(CompoundSelect.EXCEPT, *selects, **kwargs) + + + @classmethod + def _create_except_all(cls, *selects, **kwargs): + """Return an ``EXCEPT ALL`` of multiple selectables. + + The returned object is an instance of + :class:`.CompoundSelect`. + + \*selects + a list of :class:`.Select` instances. + + \**kwargs + available keyword arguments are the same as those of + :func:`select`. + + """ + return CompoundSelect(CompoundSelect.EXCEPT_ALL, *selects, **kwargs) + + + @classmethod + def _create_intersect(cls, *selects, **kwargs): + """Return an ``INTERSECT`` of multiple selectables. + + The returned object is an instance of + :class:`.CompoundSelect`. + + \*selects + a list of :class:`.Select` instances. + + \**kwargs + available keyword arguments are the same as those of + :func:`select`. + + """ + return CompoundSelect(CompoundSelect.INTERSECT, *selects, **kwargs) + + + @classmethod + def _create_intersect_all(cls, *selects, **kwargs): + """Return an ``INTERSECT ALL`` of multiple selectables. + + The returned object is an instance of + :class:`.CompoundSelect`. + + \*selects + a list of :class:`.Select` instances. + + \**kwargs + available keyword arguments are the same as those of + :func:`select`. + + """ + return CompoundSelect(CompoundSelect.INTERSECT_ALL, *selects, **kwargs) + + + def _scalar_type(self): + return self.selects[0]._scalar_type() + + def self_group(self, against=None): + return FromGrouping(self) + + def is_derived_from(self, fromclause): + for s in self.selects: + if s.is_derived_from(fromclause): + return True + return False + + def _populate_column_collection(self): + for cols in zip(*[s.c for s in self.selects]): + + # this is a slightly hacky thing - the union exports a + # column that resembles just that of the *first* selectable. + # to get at a "composite" column, particularly foreign keys, + # you have to dig through the proxies collection which we + # generate below. We may want to improve upon this, such as + # perhaps _make_proxy can accept a list of other columns + # that are "shared" - schema.column can then copy all the + # ForeignKeys in. this would allow the union() to have all + # those fks too. + + proxy = cols[0]._make_proxy(self, + name=cols[0]._label if self.use_labels else None, + key=cols[0]._key_label if self.use_labels else None) + + # hand-construct the "_proxies" collection to include all + # derived columns place a 'weight' annotation corresponding + # to how low in the list of select()s the column occurs, so + # that the corresponding_column() operation can resolve + # conflicts + + proxy._proxies = [c._annotate({'weight': i + 1}) for (i, + c) in enumerate(cols)] + + def _refresh_for_new_column(self, column): + for s in self.selects: + s._refresh_for_new_column(column) + + if not self._cols_populated: + return None + + raise NotImplementedError("CompoundSelect constructs don't support " + "addition of columns to underlying selectables") + + def _copy_internals(self, clone=_clone, **kw): + self._reset_exported() + self.selects = [clone(s, **kw) for s in self.selects] + if hasattr(self, '_col_map'): + del self._col_map + for attr in ('_order_by_clause', '_group_by_clause', '_for_update_arg'): + if getattr(self, attr) is not None: + setattr(self, attr, clone(getattr(self, attr), **kw)) + + def get_children(self, column_collections=True, **kwargs): + return (column_collections and list(self.c) or []) \ + + [self._order_by_clause, self._group_by_clause] \ + + list(self.selects) + + def bind(self): + if self._bind: + return self._bind + for s in self.selects: + e = s.bind + if e: + return e + else: + return None + + def _set_bind(self, bind): + self._bind = bind + bind = property(bind, _set_bind) + + +class HasPrefixes(object): + _prefixes = () + + @_generative + def prefix_with(self, *expr, **kw): + """Add one or more expressions following the statement keyword, i.e. + SELECT, INSERT, UPDATE, or DELETE. Generative. + + This is used to support backend-specific prefix keywords such as those + provided by MySQL. + + E.g.:: + + stmt = table.insert().prefix_with("LOW_PRIORITY", dialect="mysql") + + Multiple prefixes can be specified by multiple calls + to :meth:`.prefix_with`. + + :param \*expr: textual or :class:`.ClauseElement` construct which + will be rendered following the INSERT, UPDATE, or DELETE + keyword. + :param \**kw: A single keyword 'dialect' is accepted. This is an + optional string dialect name which will + limit rendering of this prefix to only that dialect. + + """ + dialect = kw.pop('dialect', None) + if kw: + raise exc.ArgumentError("Unsupported argument(s): %s" % + ",".join(kw)) + self._setup_prefixes(expr, dialect) + + def _setup_prefixes(self, prefixes, dialect=None): + self._prefixes = self._prefixes + tuple( + [(_literal_as_text(p), dialect) for p in prefixes]) + + + +class Select(HasPrefixes, GenerativeSelect): + """Represents a ``SELECT`` statement. + + """ + + __visit_name__ = 'select' + + _prefixes = () + _hints = util.immutabledict() + _distinct = False + _from_cloned = None + _correlate = () + _correlate_except = None + _memoized_property = SelectBase._memoized_property + + def __init__(self, + columns=None, + whereclause=None, + from_obj=None, + distinct=False, + having=None, + correlate=True, + prefixes=None, + **kwargs): + """Construct a new :class:`.Select`. + + Similar functionality is also available via the :meth:`.FromClause.select` + method on any :class:`.FromClause`. + + All arguments which accept :class:`.ClauseElement` arguments also accept + string arguments, which will be converted as appropriate into + either :func:`text()` or :func:`literal_column()` constructs. + + .. seealso:: + + :ref:`coretutorial_selecting` - Core Tutorial description of + :func:`.select`. + + :param columns: + A list of :class:`.ClauseElement` objects, typically + :class:`.ColumnElement` objects or subclasses, which will form the + columns clause of the resulting statement. For all members which are + instances of :class:`.Selectable`, the individual :class:`.ColumnElement` + members of the :class:`.Selectable` will be added individually to the + columns clause. For example, specifying a + :class:`~sqlalchemy.schema.Table` instance will result in all the + contained :class:`~sqlalchemy.schema.Column` objects within to be added + to the columns clause. + + This argument is not present on the form of :func:`select()` + available on :class:`~sqlalchemy.schema.Table`. + + :param whereclause: + A :class:`.ClauseElement` expression which will be used to form the + ``WHERE`` clause. + + :param from_obj: + A list of :class:`.ClauseElement` objects which will be added to the + ``FROM`` clause of the resulting statement. Note that "from" objects are + automatically located within the columns and whereclause ClauseElements. + Use this parameter to explicitly specify "from" objects which are not + automatically locatable. This could include + :class:`~sqlalchemy.schema.Table` objects that aren't otherwise present, + or :class:`.Join` objects whose presence will supercede that of the + :class:`~sqlalchemy.schema.Table` objects already located in the other + clauses. + + :param autocommit: + Deprecated. Use .execution_options(autocommit=) + to set the autocommit option. + + :param bind=None: + an :class:`~.Engine` or :class:`~.Connection` instance + to which the + resulting :class:`.Select` object will be bound. The :class:`.Select` + object will otherwise automatically bind to whatever + :class:`~.base.Connectable` instances can be located within its contained + :class:`.ClauseElement` members. + + :param correlate=True: + indicates that this :class:`.Select` object should have its + contained :class:`.FromClause` elements "correlated" to an enclosing + :class:`.Select` object. This means that any :class:`.ClauseElement` + instance within the "froms" collection of this :class:`.Select` + which is also present in the "froms" collection of an + enclosing select will not be rendered in the ``FROM`` clause + of this select statement. + + :param distinct=False: + when ``True``, applies a ``DISTINCT`` qualifier to the columns + clause of the resulting statement. + + The boolean argument may also be a column expression or list + of column expressions - this is a special calling form which + is understood by the Postgresql dialect to render the + ``DISTINCT ON ()`` syntax. + + ``distinct`` is also available via the :meth:`~.Select.distinct` + generative method. + + :param for_update=False: + when ``True``, applies ``FOR UPDATE`` to the end of the + resulting statement. + + .. deprecated:: 0.9.0 - use :meth:`.GenerativeSelect.with_for_update` + to specify the structure of the ``FOR UPDATE`` clause. + + ``for_update`` accepts various string values interpreted by + specific backends, including: + + * ``"read"`` - on MySQL, translates to ``LOCK IN SHARE MODE``; + on Postgresql, translates to ``FOR SHARE``. + * ``"nowait"`` - on Postgresql and Oracle, translates to + ``FOR UPDATE NOWAIT``. + * ``"read_nowait"`` - on Postgresql, translates to + ``FOR SHARE NOWAIT``. + + .. seealso:: + + :meth:`.GenerativeSelect.with_for_update` - improved API for + specifying the ``FOR UPDATE`` clause. + + :param group_by: + a list of :class:`.ClauseElement` objects which will comprise the + ``GROUP BY`` clause of the resulting select. + + :param having: + a :class:`.ClauseElement` that will comprise the ``HAVING`` clause + of the resulting select when ``GROUP BY`` is used. + + :param limit=None: + a numerical value which usually compiles to a ``LIMIT`` + expression in the resulting select. Databases that don't + support ``LIMIT`` will attempt to provide similar + functionality. + + :param offset=None: + a numeric value which usually compiles to an ``OFFSET`` + expression in the resulting select. Databases that don't + support ``OFFSET`` will attempt to provide similar + functionality. + + :param order_by: + a scalar or list of :class:`.ClauseElement` objects which will + comprise the ``ORDER BY`` clause of the resulting select. + + :param use_labels=False: + when ``True``, the statement will be generated using labels + for each column in the columns clause, which qualify each + column with its parent table's (or aliases) name so that name + conflicts between columns in different tables don't occur. + The format of the label is _. The "c" + collection of the resulting :class:`.Select` object will use these + names as well for targeting column members. + + use_labels is also available via the :meth:`~.GenerativeSelect.apply_labels` + generative method. + + """ + self._auto_correlate = correlate + if distinct is not False: + if distinct is True: + self._distinct = True + else: + self._distinct = [ + _literal_as_text(e) + for e in util.to_list(distinct) + ] + + if from_obj is not None: + self._from_obj = util.OrderedSet( + _interpret_as_from(f) + for f in util.to_list(from_obj)) + else: + self._from_obj = util.OrderedSet() + + try: + cols_present = bool(columns) + except TypeError: + raise exc.ArgumentError("columns argument to select() must " + "be a Python list or other iterable") + + if cols_present: + self._raw_columns = [] + for c in columns: + c = _interpret_as_column_or_from(c) + if isinstance(c, ScalarSelect): + c = c.self_group(against=operators.comma_op) + self._raw_columns.append(c) + else: + self._raw_columns = [] + + if whereclause is not None: + self._whereclause = _literal_as_text(whereclause) + else: + self._whereclause = None + + if having is not None: + self._having = _literal_as_text(having) + else: + self._having = None + + if prefixes: + self._setup_prefixes(prefixes) + + GenerativeSelect.__init__(self, **kwargs) + + @property + def _froms(self): + # would love to cache this, + # but there's just enough edge cases, particularly now that + # declarative encourages construction of SQL expressions + # without tables present, to just regen this each time. + froms = [] + seen = set() + translate = self._from_cloned + + def add(items): + for item in items: + if item is self: + raise exc.InvalidRequestError( + "select() construct refers to itself as a FROM") + if translate and item in translate: + item = translate[item] + if not seen.intersection(item._cloned_set): + froms.append(item) + seen.update(item._cloned_set) + + add(_from_objects(*self._raw_columns)) + if self._whereclause is not None: + add(_from_objects(self._whereclause)) + add(self._from_obj) + + return froms + + def _get_display_froms(self, explicit_correlate_froms=None, + implicit_correlate_froms=None): + """Return the full list of 'from' clauses to be displayed. + + Takes into account a set of existing froms which may be + rendered in the FROM clause of enclosing selects; this Select + may want to leave those absent if it is automatically + correlating. + + """ + froms = self._froms + + toremove = set(itertools.chain(*[ + _expand_cloned(f._hide_froms) + for f in froms])) + if toremove: + # if we're maintaining clones of froms, + # add the copies out to the toremove list. only include + # clones that are lexical equivalents. + if self._from_cloned: + toremove.update( + self._from_cloned[f] for f in + toremove.intersection(self._from_cloned) + if self._from_cloned[f]._is_lexical_equivalent(f) + ) + # filter out to FROM clauses not in the list, + # using a list to maintain ordering + froms = [f for f in froms if f not in toremove] + + if self._correlate: + to_correlate = self._correlate + if to_correlate: + froms = [ + f for f in froms if f not in + _cloned_intersection( + _cloned_intersection(froms, explicit_correlate_froms or ()), + to_correlate + ) + ] + + if self._correlate_except is not None: + + froms = [ + f for f in froms if f not in + _cloned_difference( + _cloned_intersection(froms, explicit_correlate_froms or ()), + self._correlate_except + ) + ] + + if self._auto_correlate and \ + implicit_correlate_froms and \ + len(froms) > 1: + + froms = [ + f for f in froms if f not in + _cloned_intersection(froms, implicit_correlate_froms) + ] + + if not len(froms): + raise exc.InvalidRequestError("Select statement '%s" + "' returned no FROM clauses due to " + "auto-correlation; specify " + "correlate() to control " + "correlation manually." % self) + + return froms + + def _scalar_type(self): + elem = self._raw_columns[0] + cols = list(elem._select_iterable) + return cols[0].type + + @property + def froms(self): + """Return the displayed list of FromClause elements.""" + + return self._get_display_froms() + + @_generative + def with_hint(self, selectable, text, dialect_name='*'): + """Add an indexing hint for the given selectable to this + :class:`.Select`. + + The text of the hint is rendered in the appropriate + location for the database backend in use, relative + to the given :class:`.Table` or :class:`.Alias` passed as the + ``selectable`` argument. The dialect implementation + typically uses Python string substitution syntax + with the token ``%(name)s`` to render the name of + the table or alias. E.g. when using Oracle, the + following:: + + select([mytable]).\\ + with_hint(mytable, "+ index(%(name)s ix_mytable)") + + Would render SQL as:: + + select /*+ index(mytable ix_mytable) */ ... from mytable + + The ``dialect_name`` option will limit the rendering of a particular + hint to a particular backend. Such as, to add hints for both Oracle + and Sybase simultaneously:: + + select([mytable]).\\ + with_hint(mytable, "+ index(%(name)s ix_mytable)", 'oracle').\\ + with_hint(mytable, "WITH INDEX ix_mytable", 'sybase') + + """ + self._hints = self._hints.union( + {(selectable, dialect_name): text}) + + @property + def type(self): + raise exc.InvalidRequestError("Select objects don't have a type. " + "Call as_scalar() on this Select object " + "to return a 'scalar' version of this Select.") + + @_memoized_property.method + def locate_all_froms(self): + """return a Set of all FromClause elements referenced by this Select. + + This set is a superset of that returned by the ``froms`` property, + which is specifically for those FromClause elements that would + actually be rendered. + + """ + froms = self._froms + return froms + list(_from_objects(*froms)) + + @property + def inner_columns(self): + """an iterator of all ColumnElement expressions which would + be rendered into the columns clause of the resulting SELECT statement. + + """ + return _select_iterables(self._raw_columns) + + def is_derived_from(self, fromclause): + if self in fromclause._cloned_set: + return True + + for f in self.locate_all_froms(): + if f.is_derived_from(fromclause): + return True + return False + + def _copy_internals(self, clone=_clone, **kw): + + # Select() object has been cloned and probably adapted by the + # given clone function. Apply the cloning function to internal + # objects + + # 1. keep a dictionary of the froms we've cloned, and what + # they've become. This is consulted later when we derive + # additional froms from "whereclause" and the columns clause, + # which may still reference the uncloned parent table. + # as of 0.7.4 we also put the current version of _froms, which + # gets cleared on each generation. previously we were "baking" + # _froms into self._from_obj. + self._from_cloned = from_cloned = dict((f, clone(f, **kw)) + for f in self._from_obj.union(self._froms)) + + # 3. update persistent _from_obj with the cloned versions. + self._from_obj = util.OrderedSet(from_cloned[f] for f in + self._from_obj) + + # the _correlate collection is done separately, what can happen + # here is the same item is _correlate as in _from_obj but the + # _correlate version has an annotation on it - (specifically + # RelationshipProperty.Comparator._criterion_exists() does + # this). Also keep _correlate liberally open with it's previous + # contents, as this set is used for matching, not rendering. + self._correlate = set(clone(f) for f in + self._correlate).union(self._correlate) + + # 4. clone other things. The difficulty here is that Column + # objects are not actually cloned, and refer to their original + # .table, resulting in the wrong "from" parent after a clone + # operation. Hence _from_cloned and _from_obj supercede what is + # present here. + self._raw_columns = [clone(c, **kw) for c in self._raw_columns] + for attr in '_whereclause', '_having', '_order_by_clause', \ + '_group_by_clause', '_for_update_arg': + if getattr(self, attr) is not None: + setattr(self, attr, clone(getattr(self, attr), **kw)) + + # erase exported column list, _froms collection, + # etc. + self._reset_exported() + + def get_children(self, column_collections=True, **kwargs): + """return child elements as per the ClauseElement specification.""" + + return (column_collections and list(self.columns) or []) + \ + self._raw_columns + list(self._froms) + \ + [x for x in + (self._whereclause, self._having, + self._order_by_clause, self._group_by_clause) + if x is not None] + + @_generative + def column(self, column): + """return a new select() construct with the given column expression + added to its columns clause. + + """ + self.append_column(column) + + @util.dependencies("sqlalchemy.sql.util") + def reduce_columns(self, sqlutil, only_synonyms=True): + """Return a new :func`.select` construct with redundantly + named, equivalently-valued columns removed from the columns clause. + + "Redundant" here means two columns where one refers to the + other either based on foreign key, or via a simple equality + comparison in the WHERE clause of the statement. The primary purpose + of this method is to automatically construct a select statement + with all uniquely-named columns, without the need to use + table-qualified labels as :meth:`.apply_labels` does. + + When columns are omitted based on foreign key, the referred-to + column is the one that's kept. When columns are omitted based on + WHERE eqivalence, the first column in the columns clause is the + one that's kept. + + :param only_synonyms: when True, limit the removal of columns + to those which have the same name as the equivalent. Otherwise, + all columns that are equivalent to another are removed. + + .. versionadded:: 0.8 + + """ + return self.with_only_columns( + sqlutil.reduce_columns( + self.inner_columns, + only_synonyms=only_synonyms, + *(self._whereclause, ) + tuple(self._from_obj) + ) + ) + + @_generative + def with_only_columns(self, columns): + """Return a new :func:`.select` construct with its columns + clause replaced with the given columns. + + .. versionchanged:: 0.7.3 + Due to a bug fix, this method has a slight + behavioral change as of version 0.7.3. + Prior to version 0.7.3, the FROM clause of + a :func:`.select` was calculated upfront and as new columns + were added; in 0.7.3 and later it's calculated + at compile time, fixing an issue regarding late binding + of columns to parent tables. This changes the behavior of + :meth:`.Select.with_only_columns` in that FROM clauses no + longer represented in the new list are dropped, + but this behavior is more consistent in + that the FROM clauses are consistently derived from the + current columns clause. The original intent of this method + is to allow trimming of the existing columns list to be fewer + columns than originally present; the use case of replacing + the columns list with an entirely different one hadn't + been anticipated until 0.7.3 was released; the usage + guidelines below illustrate how this should be done. + + This method is exactly equivalent to as if the original + :func:`.select` had been called with the given columns + clause. I.e. a statement:: + + s = select([table1.c.a, table1.c.b]) + s = s.with_only_columns([table1.c.b]) + + should be exactly equivalent to:: + + s = select([table1.c.b]) + + This means that FROM clauses which are only derived + from the column list will be discarded if the new column + list no longer contains that FROM:: + + >>> table1 = table('t1', column('a'), column('b')) + >>> table2 = table('t2', column('a'), column('b')) + >>> s1 = select([table1.c.a, table2.c.b]) + >>> print s1 + SELECT t1.a, t2.b FROM t1, t2 + >>> s2 = s1.with_only_columns([table2.c.b]) + >>> print s2 + SELECT t2.b FROM t1 + + The preferred way to maintain a specific FROM clause + in the construct, assuming it won't be represented anywhere + else (i.e. not in the WHERE clause, etc.) is to set it using + :meth:`.Select.select_from`:: + + >>> s1 = select([table1.c.a, table2.c.b]).\\ + ... select_from(table1.join(table2, + ... table1.c.a==table2.c.a)) + >>> s2 = s1.with_only_columns([table2.c.b]) + >>> print s2 + SELECT t2.b FROM t1 JOIN t2 ON t1.a=t2.a + + Care should also be taken to use the correct + set of column objects passed to :meth:`.Select.with_only_columns`. + Since the method is essentially equivalent to calling the + :func:`.select` construct in the first place with the given + columns, the columns passed to :meth:`.Select.with_only_columns` + should usually be a subset of those which were passed + to the :func:`.select` construct, not those which are available + from the ``.c`` collection of that :func:`.select`. That + is:: + + s = select([table1.c.a, table1.c.b]).select_from(table1) + s = s.with_only_columns([table1.c.b]) + + and **not**:: + + # usually incorrect + s = s.with_only_columns([s.c.b]) + + The latter would produce the SQL:: + + SELECT b + FROM (SELECT t1.a AS a, t1.b AS b + FROM t1), t1 + + Since the :func:`.select` construct is essentially being + asked to select both from ``table1`` as well as itself. + + """ + self._reset_exported() + rc = [] + for c in columns: + c = _interpret_as_column_or_from(c) + if isinstance(c, ScalarSelect): + c = c.self_group(against=operators.comma_op) + rc.append(c) + self._raw_columns = rc + + @_generative + def where(self, whereclause): + """return a new select() construct with the given expression added to + its WHERE clause, joined to the existing clause via AND, if any. + + """ + + self.append_whereclause(whereclause) + + @_generative + def having(self, having): + """return a new select() construct with the given expression added to + its HAVING clause, joined to the existing clause via AND, if any. + + """ + self.append_having(having) + + @_generative + def distinct(self, *expr): + """Return a new select() construct which will apply DISTINCT to its + columns clause. + + :param \*expr: optional column expressions. When present, + the Postgresql dialect will render a ``DISTINCT ON (>)`` + construct. + + """ + if expr: + expr = [_literal_as_text(e) for e in expr] + if isinstance(self._distinct, list): + self._distinct = self._distinct + expr + else: + self._distinct = expr + else: + self._distinct = True + + @_generative + def select_from(self, fromclause): + """return a new :func:`.select` construct with the + given FROM expression + merged into its list of FROM objects. + + E.g.:: + + table1 = table('t1', column('a')) + table2 = table('t2', column('b')) + s = select([table1.c.a]).\\ + select_from( + table1.join(table2, table1.c.a==table2.c.b) + ) + + The "from" list is a unique set on the identity of each element, + so adding an already present :class:`.Table` or other selectable + will have no effect. Passing a :class:`.Join` that refers + to an already present :class:`.Table` or other selectable will have + the effect of concealing the presence of that selectable as + an individual element in the rendered FROM list, instead + rendering it into a JOIN clause. + + While the typical purpose of :meth:`.Select.select_from` is to + replace the default, derived FROM clause with a join, it can + also be called with individual table elements, multiple times + if desired, in the case that the FROM clause cannot be fully + derived from the columns clause:: + + select([func.count('*')]).select_from(table1) + + """ + self.append_from(fromclause) + + @_generative + def correlate(self, *fromclauses): + """return a new :class:`.Select` which will correlate the given FROM + clauses to that of an enclosing :class:`.Select`. + + Calling this method turns off the :class:`.Select` object's + default behavior of "auto-correlation". Normally, FROM elements + which appear in a :class:`.Select` that encloses this one via + its :term:`WHERE clause`, ORDER BY, HAVING or + :term:`columns clause` will be omitted from this :class:`.Select` + object's :term:`FROM clause`. + Setting an explicit correlation collection using the + :meth:`.Select.correlate` method provides a fixed list of FROM objects + that can potentially take place in this process. + + When :meth:`.Select.correlate` is used to apply specific FROM clauses + for correlation, the FROM elements become candidates for + correlation regardless of how deeply nested this :class:`.Select` + object is, relative to an enclosing :class:`.Select` which refers to + the same FROM object. This is in contrast to the behavior of + "auto-correlation" which only correlates to an immediate enclosing + :class:`.Select`. Multi-level correlation ensures that the link + between enclosed and enclosing :class:`.Select` is always via + at least one WHERE/ORDER BY/HAVING/columns clause in order for + correlation to take place. + + If ``None`` is passed, the :class:`.Select` object will correlate + none of its FROM entries, and all will render unconditionally + in the local FROM clause. + + :param \*fromclauses: a list of one or more :class:`.FromClause` + constructs, or other compatible constructs (i.e. ORM-mapped + classes) to become part of the correlate collection. + + .. versionchanged:: 0.8.0 ORM-mapped classes are accepted by + :meth:`.Select.correlate`. + + .. versionchanged:: 0.8.0 The :meth:`.Select.correlate` method no + longer unconditionally removes entries from the FROM clause; instead, + the candidate FROM entries must also be matched by a FROM entry + located in an enclosing :class:`.Select`, which ultimately encloses + this one as present in the WHERE clause, ORDER BY clause, HAVING + clause, or columns clause of an enclosing :meth:`.Select`. + + .. versionchanged:: 0.8.2 explicit correlation takes place + via any level of nesting of :class:`.Select` objects; in previous + 0.8 versions, correlation would only occur relative to the immediate + enclosing :class:`.Select` construct. + + .. seealso:: + + :meth:`.Select.correlate_except` + + :ref:`correlated_subqueries` + + """ + self._auto_correlate = False + if fromclauses and fromclauses[0] is None: + self._correlate = () + else: + self._correlate = set(self._correlate).union( + _interpret_as_from(f) for f in fromclauses) + + @_generative + def correlate_except(self, *fromclauses): + """return a new :class:`.Select` which will omit the given FROM + clauses from the auto-correlation process. + + Calling :meth:`.Select.correlate_except` turns off the + :class:`.Select` object's default behavior of + "auto-correlation" for the given FROM elements. An element + specified here will unconditionally appear in the FROM list, while + all other FROM elements remain subject to normal auto-correlation + behaviors. + + .. versionchanged:: 0.8.2 The :meth:`.Select.correlate_except` + method was improved to fully prevent FROM clauses specified here + from being omitted from the immediate FROM clause of this + :class:`.Select`. + + If ``None`` is passed, the :class:`.Select` object will correlate + all of its FROM entries. + + .. versionchanged:: 0.8.2 calling ``correlate_except(None)`` will + correctly auto-correlate all FROM clauses. + + :param \*fromclauses: a list of one or more :class:`.FromClause` + constructs, or other compatible constructs (i.e. ORM-mapped + classes) to become part of the correlate-exception collection. + + .. seealso:: + + :meth:`.Select.correlate` + + :ref:`correlated_subqueries` + + """ + + self._auto_correlate = False + if fromclauses and fromclauses[0] is None: + self._correlate_except = () + else: + self._correlate_except = set(self._correlate_except or ()).union( + _interpret_as_from(f) for f in fromclauses) + + def append_correlation(self, fromclause): + """append the given correlation expression to this select() + construct. + + This is an **in-place** mutation method; the + :meth:`~.Select.correlate` method is preferred, as it provides standard + :term:`method chaining`. + + """ + + self._auto_correlate = False + self._correlate = set(self._correlate).union( + _interpret_as_from(f) for f in fromclause) + + def append_column(self, column): + """append the given column expression to the columns clause of this + select() construct. + + This is an **in-place** mutation method; the + :meth:`~.Select.column` method is preferred, as it provides standard + :term:`method chaining`. + + """ + self._reset_exported() + column = _interpret_as_column_or_from(column) + + if isinstance(column, ScalarSelect): + column = column.self_group(against=operators.comma_op) + + self._raw_columns = self._raw_columns + [column] + + def append_prefix(self, clause): + """append the given columns clause prefix expression to this select() + construct. + + This is an **in-place** mutation method; the + :meth:`~.Select.prefix_with` method is preferred, as it provides standard + :term:`method chaining`. + + """ + clause = _literal_as_text(clause) + self._prefixes = self._prefixes + (clause,) + + def append_whereclause(self, whereclause): + """append the given expression to this select() construct's WHERE + criterion. + + The expression will be joined to existing WHERE criterion via AND. + + This is an **in-place** mutation method; the + :meth:`~.Select.where` method is preferred, as it provides standard + :term:`method chaining`. + + """ + + self._reset_exported() + self._whereclause = and_(True_._ifnone(self._whereclause), whereclause) + + def append_having(self, having): + """append the given expression to this select() construct's HAVING + criterion. + + The expression will be joined to existing HAVING criterion via AND. + + This is an **in-place** mutation method; the + :meth:`~.Select.having` method is preferred, as it provides standard + :term:`method chaining`. + + """ + self._reset_exported() + self._having = and_(True_._ifnone(self._having), having) + + def append_from(self, fromclause): + """append the given FromClause expression to this select() construct's + FROM clause. + + This is an **in-place** mutation method; the + :meth:`~.Select.select_from` method is preferred, as it provides standard + :term:`method chaining`. + + """ + self._reset_exported() + fromclause = _interpret_as_from(fromclause) + self._from_obj = self._from_obj.union([fromclause]) + + + @_memoized_property + def _columns_plus_names(self): + if self.use_labels: + names = set() + def name_for_col(c): + if c._label is None: + return (None, c) + name = c._label + if name in names: + name = c.anon_label + else: + names.add(name) + return name, c + + return [ + name_for_col(c) + for c in util.unique_list(_select_iterables(self._raw_columns)) + ] + else: + return [ + (None, c) + for c in util.unique_list(_select_iterables(self._raw_columns)) + ] + + def _populate_column_collection(self): + for name, c in self._columns_plus_names: + if not hasattr(c, '_make_proxy'): + continue + if name is None: + key = None + elif self.use_labels: + key = c._key_label + if key is not None and key in self.c: + key = c.anon_label + else: + key = None + + c._make_proxy(self, key=key, + name=name, + name_is_truncatable=True) + + def _refresh_for_new_column(self, column): + for fromclause in self._froms: + col = fromclause._refresh_for_new_column(column) + if col is not None: + if col in self.inner_columns and self._cols_populated: + our_label = col._key_label if self.use_labels else col.key + if our_label not in self.c: + return col._make_proxy(self, + name=col._label if self.use_labels else None, + key=col._key_label if self.use_labels else None, + name_is_truncatable=True) + return None + return None + + def self_group(self, against=None): + """return a 'grouping' construct as per the ClauseElement + specification. + + This produces an element that can be embedded in an expression. Note + that this method is called automatically as needed when constructing + expressions and should not require explicit use. + + """ + if isinstance(against, CompoundSelect): + return self + return FromGrouping(self) + + def union(self, other, **kwargs): + """return a SQL UNION of this select() construct against the given + selectable.""" + + return CompoundSelect._create_union(self, other, **kwargs) + + def union_all(self, other, **kwargs): + """return a SQL UNION ALL of this select() construct against the given + selectable. + + """ + return CompoundSelect._create_union_all(self, other, **kwargs) + + def except_(self, other, **kwargs): + """return a SQL EXCEPT of this select() construct against the given + selectable.""" + + return CompoundSelect._create_except(self, other, **kwargs) + + def except_all(self, other, **kwargs): + """return a SQL EXCEPT ALL of this select() construct against the + given selectable. + + """ + return CompoundSelect._create_except_all(self, other, **kwargs) + + def intersect(self, other, **kwargs): + """return a SQL INTERSECT of this select() construct against the given + selectable. + + """ + return CompoundSelect._create_intersect(self, other, **kwargs) + + def intersect_all(self, other, **kwargs): + """return a SQL INTERSECT ALL of this select() construct against the + given selectable. + + """ + return CompoundSelect._create_intersect_all(self, other, **kwargs) + + def bind(self): + if self._bind: + return self._bind + froms = self._froms + if not froms: + for c in self._raw_columns: + e = c.bind + if e: + self._bind = e + return e + else: + e = list(froms)[0].bind + if e: + self._bind = e + return e + + return None + + def _set_bind(self, bind): + self._bind = bind + bind = property(bind, _set_bind) + + +class ScalarSelect(Generative, Grouping): + _from_objects = [] + + def __init__(self, element): + self.element = element + self.type = element._scalar_type() + + @property + def columns(self): + raise exc.InvalidRequestError('Scalar Select expression has no ' + 'columns; use this object directly within a ' + 'column-level expression.') + c = columns + + @_generative + def where(self, crit): + """Apply a WHERE clause to the SELECT statement referred to + by this :class:`.ScalarSelect`. + + """ + self.element = self.element.where(crit) + + def self_group(self, **kwargs): + return self + + +class Exists(UnaryExpression): + """Represent an ``EXISTS`` clause. + + """ + __visit_name__ = UnaryExpression.__visit_name__ + _from_objects = [] + + + def __init__(self, *args, **kwargs): + """Construct a new :class:`.Exists` against an existing + :class:`.Select` object. + + Calling styles are of the following forms:: + + # use on an existing select() + s = select([table.c.col1]).where(table.c.col2==5) + s = exists(s) + + # construct a select() at once + exists(['*'], **select_arguments).where(criterion) + + # columns argument is optional, generates "EXISTS (SELECT *)" + # by default. + exists().where(table.c.col2==5) + + """ + if args and isinstance(args[0], (SelectBase, ScalarSelect)): + s = args[0] + else: + if not args: + args = ([literal_column('*')],) + s = Select(*args, **kwargs).as_scalar().self_group() + + UnaryExpression.__init__(self, s, operator=operators.exists, + type_=type_api.BOOLEANTYPE) + + def select(self, whereclause=None, **params): + return Select([self], whereclause, **params) + + def correlate(self, *fromclause): + e = self._clone() + e.element = self.element.correlate(*fromclause).self_group() + return e + + def correlate_except(self, *fromclause): + e = self._clone() + e.element = self.element.correlate_except(*fromclause).self_group() + return e + + def select_from(self, clause): + """return a new :class:`.Exists` construct, applying the given + expression to the :meth:`.Select.select_from` method of the select + statement contained. + + """ + e = self._clone() + e.element = self.element.select_from(clause).self_group() + return e + + def where(self, clause): + """return a new exists() construct with the given expression added to + its WHERE clause, joined to the existing clause via AND, if any. + + """ + e = self._clone() + e.element = self.element.where(clause).self_group() + return e + + +class TextAsFrom(SelectBase): + """Wrap a :class:`.TextClause` construct within a :class:`.SelectBase` + interface. + + This allows the :class:`.TextClause` object to gain a ``.c`` collection and + other FROM-like capabilities such as :meth:`.FromClause.alias`, + :meth:`.SelectBase.cte`, etc. + + The :class:`.TextAsFrom` construct is produced via the + :meth:`.TextClause.columns` method - see that method for details. + + .. versionadded:: 0.9.0 + + .. seealso:: + + :func:`.text` + + :meth:`.TextClause.columns` + + """ + __visit_name__ = "text_as_from" + + def __init__(self, text, columns): + self.element = text + self.column_args = columns + + @property + def _bind(self): + return self.element._bind + + @_generative + def bindparams(self, *binds, **bind_as_values): + self.element = self.element.bindparams(*binds, **bind_as_values) + + def _populate_column_collection(self): + for c in self.column_args: + c._make_proxy(self) + + def _copy_internals(self, clone=_clone, **kw): + self._reset_exported() + self.element = clone(self.element, **kw) + + def _scalar_type(self): + return self.column_args[0].type + +class AnnotatedFromClause(Annotated): + def __init__(self, element, values): + # force FromClause to generate their internal + # collections into __dict__ + element.c + Annotated.__init__(self, element, values) + + diff --git a/libs/sqlalchemy/sql/sqltypes.py b/libs/sqlalchemy/sql/sqltypes.py new file mode 100644 index 00000000..702e7736 --- /dev/null +++ b/libs/sqlalchemy/sql/sqltypes.py @@ -0,0 +1,1628 @@ +# sql/sqltypes.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""SQL specific types. + +""" + +import datetime as dt +import codecs + +from .type_api import TypeEngine, TypeDecorator, to_instance +from .elements import quoted_name, type_coerce +from .default_comparator import _DefaultColumnComparator +from .. import exc, util, processors +from .base import _bind_or_error, SchemaEventTarget +from . import operators +from .. import event +from ..util import pickle +import decimal + +if util.jython: + import array + +class _DateAffinity(object): + """Mixin date/time specific expression adaptations. + + Rules are implemented within Date,Time,Interval,DateTime, Numeric, + Integer. Based on http://www.postgresql.org/docs/current/static + /functions-datetime.html. + + """ + + @property + def _expression_adaptations(self): + raise NotImplementedError() + + class Comparator(TypeEngine.Comparator): + _blank_dict = util.immutabledict() + + def _adapt_expression(self, op, other_comparator): + othertype = other_comparator.type._type_affinity + return op, \ + to_instance(self.type._expression_adaptations.get(op, self._blank_dict).\ + get(othertype, NULLTYPE)) + comparator_factory = Comparator + +class Concatenable(object): + """A mixin that marks a type as supporting 'concatenation', + typically strings.""" + + class Comparator(TypeEngine.Comparator): + def _adapt_expression(self, op, other_comparator): + if op is operators.add and isinstance(other_comparator, + (Concatenable.Comparator, NullType.Comparator)): + return operators.concat_op, self.expr.type + else: + return op, self.expr.type + + comparator_factory = Comparator + + +class String(Concatenable, TypeEngine): + """The base for all string and character types. + + In SQL, corresponds to VARCHAR. Can also take Python unicode objects + and encode to the database's encoding in bind params (and the reverse for + result sets.) + + The `length` field is usually required when the `String` type is + used within a CREATE TABLE statement, as VARCHAR requires a length + on most databases. + + """ + + __visit_name__ = 'string' + + def __init__(self, length=None, collation=None, + convert_unicode=False, + unicode_error=None, + _warn_on_bytestring=False + ): + """ + Create a string-holding type. + + :param length: optional, a length for the column for use in + DDL and CAST expressions. May be safely omitted if no ``CREATE + TABLE`` will be issued. Certain databases may require a + ``length`` for use in DDL, and will raise an exception when + the ``CREATE TABLE`` DDL is issued if a ``VARCHAR`` + with no length is included. Whether the value is + interpreted as bytes or characters is database specific. + + :param collation: Optional, a column-level collation for + use in DDL and CAST expressions. Renders using the + COLLATE keyword supported by SQLite, MySQL, and Postgresql. + E.g.:: + + >>> from sqlalchemy import cast, select, String + >>> print select([cast('some string', String(collation='utf8'))]) + SELECT CAST(:param_1 AS VARCHAR COLLATE utf8) AS anon_1 + + .. versionadded:: 0.8 Added support for COLLATE to all + string types. + + :param convert_unicode: When set to ``True``, the + :class:`.String` type will assume that + input is to be passed as Python ``unicode`` objects, + and results returned as Python ``unicode`` objects. + If the DBAPI in use does not support Python unicode + (which is fewer and fewer these days), SQLAlchemy + will encode/decode the value, using the + value of the ``encoding`` parameter passed to + :func:`.create_engine` as the encoding. + + When using a DBAPI that natively supports Python + unicode objects, this flag generally does not + need to be set. For columns that are explicitly + intended to store non-ASCII data, the :class:`.Unicode` + or :class:`.UnicodeText` + types should be used regardless, which feature + the same behavior of ``convert_unicode`` but + also indicate an underlying column type that + directly supports unicode, such as ``NVARCHAR``. + + For the extremely rare case that Python ``unicode`` + is to be encoded/decoded by SQLAlchemy on a backend + that does natively support Python ``unicode``, + the value ``force`` can be passed here which will + cause SQLAlchemy's encode/decode services to be + used unconditionally. + + :param unicode_error: Optional, a method to use to handle Unicode + conversion errors. Behaves like the ``errors`` keyword argument to + the standard library's ``string.decode()`` functions. This flag + requires that ``convert_unicode`` is set to ``force`` - otherwise, + SQLAlchemy is not guaranteed to handle the task of unicode + conversion. Note that this flag adds significant performance + overhead to row-fetching operations for backends that already + return unicode objects natively (which most DBAPIs do). This + flag should only be used as a last resort for reading + strings from a column with varied or corrupted encodings. + + """ + if unicode_error is not None and convert_unicode != 'force': + raise exc.ArgumentError("convert_unicode must be 'force' " + "when unicode_error is set.") + + self.length = length + self.collation = collation + self.convert_unicode = convert_unicode + self.unicode_error = unicode_error + self._warn_on_bytestring = _warn_on_bytestring + + def literal_processor(self, dialect): + def process(value): + value = value.replace("'", "''") + return "'%s'" % value + return process + + def bind_processor(self, dialect): + if self.convert_unicode or dialect.convert_unicode: + if dialect.supports_unicode_binds and \ + self.convert_unicode != 'force': + if self._warn_on_bytestring: + def process(value): + if isinstance(value, util.binary_type): + util.warn("Unicode type received non-unicode bind " + "param value.") + return value + return process + else: + return None + else: + encoder = codecs.getencoder(dialect.encoding) + warn_on_bytestring = self._warn_on_bytestring + + def process(value): + if isinstance(value, util.text_type): + return encoder(value, self.unicode_error)[0] + elif warn_on_bytestring and value is not None: + util.warn("Unicode type received non-unicode bind " + "param value") + return value + return process + else: + return None + + def result_processor(self, dialect, coltype): + wants_unicode = self.convert_unicode or dialect.convert_unicode + needs_convert = wants_unicode and \ + (dialect.returns_unicode_strings is not True or + self.convert_unicode in ('force', 'force_nocheck')) + needs_isinstance = ( + needs_convert and + dialect.returns_unicode_strings and + self.convert_unicode != 'force_nocheck' + ) + + if needs_convert: + to_unicode = processors.to_unicode_processor_factory( + dialect.encoding, self.unicode_error) + + if needs_isinstance: + # we wouldn't be here unless convert_unicode='force' + # was specified, or the driver has erratic unicode-returning + # habits. since we will be getting back unicode + # in most cases, we check for it (decode will fail). + def process(value): + if isinstance(value, util.text_type): + return value + else: + return to_unicode(value) + return process + else: + # here, we assume that the object is not unicode, + # avoiding expensive isinstance() check. + return to_unicode + else: + return None + + @property + def python_type(self): + if self.convert_unicode: + return util.text_type + else: + return str + + def get_dbapi_type(self, dbapi): + return dbapi.STRING + + +class Text(String): + """A variably sized string type. + + In SQL, usually corresponds to CLOB or TEXT. Can also take Python + unicode objects and encode to the database's encoding in bind + params (and the reverse for result sets.) In general, TEXT objects + do not have a length; while some databases will accept a length + argument here, it will be rejected by others. + + """ + __visit_name__ = 'text' + + +class Unicode(String): + """A variable length Unicode string type. + + The :class:`.Unicode` type is a :class:`.String` subclass + that assumes input and output as Python ``unicode`` data, + and in that regard is equivalent to the usage of the + ``convert_unicode`` flag with the :class:`.String` type. + However, unlike plain :class:`.String`, it also implies an + underlying column type that is explicitly supporting of non-ASCII + data, such as ``NVARCHAR`` on Oracle and SQL Server. + This can impact the output of ``CREATE TABLE`` statements + and ``CAST`` functions at the dialect level, and can + also affect the handling of bound parameters in some + specific DBAPI scenarios. + + The encoding used by the :class:`.Unicode` type is usually + determined by the DBAPI itself; most modern DBAPIs + feature support for Python ``unicode`` objects as bound + values and result set values, and the encoding should + be configured as detailed in the notes for the target + DBAPI in the :ref:`dialect_toplevel` section. + + For those DBAPIs which do not support, or are not configured + to accommodate Python ``unicode`` objects + directly, SQLAlchemy does the encoding and decoding + outside of the DBAPI. The encoding in this scenario + is determined by the ``encoding`` flag passed to + :func:`.create_engine`. + + When using the :class:`.Unicode` type, it is only appropriate + to pass Python ``unicode`` objects, and not plain ``str``. + If a plain ``str`` is passed under Python 2, a warning + is emitted. If you notice your application emitting these warnings but + you're not sure of the source of them, the Python + ``warnings`` filter, documented at + http://docs.python.org/library/warnings.html, + can be used to turn these warnings into exceptions + which will illustrate a stack trace:: + + import warnings + warnings.simplefilter('error') + + For an application that wishes to pass plain bytestrings + and Python ``unicode`` objects to the ``Unicode`` type + equally, the bytestrings must first be decoded into + unicode. The recipe at :ref:`coerce_to_unicode` illustrates + how this is done. + + See also: + + :class:`.UnicodeText` - unlengthed textual counterpart + to :class:`.Unicode`. + + """ + + __visit_name__ = 'unicode' + + def __init__(self, length=None, **kwargs): + """ + Create a :class:`.Unicode` object. + + Parameters are the same as that of :class:`.String`, + with the exception that ``convert_unicode`` + defaults to ``True``. + + """ + kwargs.setdefault('convert_unicode', True) + kwargs.setdefault('_warn_on_bytestring', True) + super(Unicode, self).__init__(length=length, **kwargs) + + +class UnicodeText(Text): + """An unbounded-length Unicode string type. + + See :class:`.Unicode` for details on the unicode + behavior of this object. + + Like :class:`.Unicode`, usage the :class:`.UnicodeText` type implies a + unicode-capable type being used on the backend, such as + ``NCLOB``, ``NTEXT``. + + """ + + __visit_name__ = 'unicode_text' + + def __init__(self, length=None, **kwargs): + """ + Create a Unicode-converting Text type. + + Parameters are the same as that of :class:`.Text`, + with the exception that ``convert_unicode`` + defaults to ``True``. + + """ + kwargs.setdefault('convert_unicode', True) + kwargs.setdefault('_warn_on_bytestring', True) + super(UnicodeText, self).__init__(length=length, **kwargs) + + +class Integer(_DateAffinity, TypeEngine): + """A type for ``int`` integers.""" + + __visit_name__ = 'integer' + + def get_dbapi_type(self, dbapi): + return dbapi.NUMBER + + @property + def python_type(self): + return int + + def literal_processor(self, dialect): + def process(value): + return str(value) + return process + + @util.memoized_property + def _expression_adaptations(self): + # TODO: need a dictionary object that will + # handle operators generically here, this is incomplete + return { + operators.add: { + Date: Date, + Integer: self.__class__, + Numeric: Numeric, + }, + operators.mul: { + Interval: Interval, + Integer: self.__class__, + Numeric: Numeric, + }, + operators.div: { + Integer: self.__class__, + Numeric: Numeric, + }, + operators.truediv: { + Integer: self.__class__, + Numeric: Numeric, + }, + operators.sub: { + Integer: self.__class__, + Numeric: Numeric, + }, + } + + + +class SmallInteger(Integer): + """A type for smaller ``int`` integers. + + Typically generates a ``SMALLINT`` in DDL, and otherwise acts like + a normal :class:`.Integer` on the Python side. + + """ + + __visit_name__ = 'small_integer' + + +class BigInteger(Integer): + """A type for bigger ``int`` integers. + + Typically generates a ``BIGINT`` in DDL, and otherwise acts like + a normal :class:`.Integer` on the Python side. + + """ + + __visit_name__ = 'big_integer' + + + +class Numeric(_DateAffinity, TypeEngine): + """A type for fixed precision numbers. + + Typically generates DECIMAL or NUMERIC. Returns + ``decimal.Decimal`` objects by default, applying + conversion as needed. + + .. note:: + + The `cdecimal `_ library + is a high performing alternative to Python's built-in + ``decimal.Decimal`` type, which performs very poorly in high volume + situations. SQLAlchemy 0.7 is tested against ``cdecimal`` and supports + it fully. The type is not necessarily supported by DBAPI + implementations however, most of which contain an import for plain + ``decimal`` in their source code, even though some such as psycopg2 + provide hooks for alternate adapters. SQLAlchemy imports ``decimal`` + globally as well. The most straightforward and + foolproof way to use "cdecimal" given current DBAPI and Python support + is to patch it directly into sys.modules before anything else is + imported:: + + import sys + import cdecimal + sys.modules["decimal"] = cdecimal + + While the global patch is a little ugly, it's particularly + important to use just one decimal library at a time since + Python Decimal and cdecimal Decimal objects + are not currently compatible *with each other*:: + + >>> import cdecimal + >>> import decimal + >>> decimal.Decimal("10") == cdecimal.Decimal("10") + False + + SQLAlchemy will provide more natural support of + cdecimal if and when it becomes a standard part of Python + installations and is supported by all DBAPIs. + + """ + + __visit_name__ = 'numeric' + + _default_decimal_return_scale = 10 + + def __init__(self, precision=None, scale=None, + decimal_return_scale=None, asdecimal=True): + """ + Construct a Numeric. + + :param precision: the numeric precision for use in DDL ``CREATE + TABLE``. + + :param scale: the numeric scale for use in DDL ``CREATE TABLE``. + + :param asdecimal: default True. Return whether or not + values should be sent as Python Decimal objects, or + as floats. Different DBAPIs send one or the other based on + datatypes - the Numeric type will ensure that return values + are one or the other across DBAPIs consistently. + + :param decimal_return_scale: Default scale to use when converting + from floats to Python decimals. Floating point values will typically + be much longer due to decimal inaccuracy, and most floating point + database types don't have a notion of "scale", so by default the + float type looks for the first ten decimal places when converting. + Specfiying this value will override that length. Types which + do include an explicit ".scale" value, such as the base :class:`.Numeric` + as well as the MySQL float types, will use the value of ".scale" + as the default for decimal_return_scale, if not otherwise specified. + + .. versionadded:: 0.9.0 + + When using the ``Numeric`` type, care should be taken to ensure + that the asdecimal setting is apppropriate for the DBAPI in use - + when Numeric applies a conversion from Decimal->float or float-> + Decimal, this conversion incurs an additional performance overhead + for all result columns received. + + DBAPIs that return Decimal natively (e.g. psycopg2) will have + better accuracy and higher performance with a setting of ``True``, + as the native translation to Decimal reduces the amount of floating- + point issues at play, and the Numeric type itself doesn't need + to apply any further conversions. However, another DBAPI which + returns floats natively *will* incur an additional conversion + overhead, and is still subject to floating point data loss - in + which case ``asdecimal=False`` will at least remove the extra + conversion overhead. + + """ + self.precision = precision + self.scale = scale + self.decimal_return_scale = decimal_return_scale + self.asdecimal = asdecimal + + @property + def _effective_decimal_return_scale(self): + if self.decimal_return_scale is not None: + return self.decimal_return_scale + elif getattr(self, "scale", None) is not None: + return self.scale + else: + return self._default_decimal_return_scale + + def get_dbapi_type(self, dbapi): + return dbapi.NUMBER + + def literal_processor(self, dialect): + def process(value): + return str(value) + return process + + @property + def python_type(self): + if self.asdecimal: + return decimal.Decimal + else: + return float + + def bind_processor(self, dialect): + if dialect.supports_native_decimal: + return None + else: + return processors.to_float + + def result_processor(self, dialect, coltype): + if self.asdecimal: + if dialect.supports_native_decimal: + # we're a "numeric", DBAPI will give us Decimal directly + return None + else: + util.warn('Dialect %s+%s does *not* support Decimal ' + 'objects natively, and SQLAlchemy must ' + 'convert from floating point - rounding ' + 'errors and other issues may occur. Please ' + 'consider storing Decimal numbers as strings ' + 'or integers on this platform for lossless ' + 'storage.' % (dialect.name, dialect.driver)) + + # we're a "numeric", DBAPI returns floats, convert. + return processors.to_decimal_processor_factory( + decimal.Decimal, + self.scale if self.scale is not None + else self._default_decimal_return_scale) + else: + if dialect.supports_native_decimal: + return processors.to_float + else: + return None + + @util.memoized_property + def _expression_adaptations(self): + return { + operators.mul: { + Interval: Interval, + Numeric: self.__class__, + Integer: self.__class__, + }, + operators.div: { + Numeric: self.__class__, + Integer: self.__class__, + }, + operators.truediv: { + Numeric: self.__class__, + Integer: self.__class__, + }, + operators.add: { + Numeric: self.__class__, + Integer: self.__class__, + }, + operators.sub: { + Numeric: self.__class__, + Integer: self.__class__, + } + } + + +class Float(Numeric): + """A type for ``float`` numbers. + + Returns Python ``float`` objects by default, applying + conversion as needed. + + """ + + __visit_name__ = 'float' + + scale = None + + def __init__(self, precision=None, asdecimal=False, + decimal_return_scale=None, **kwargs): + """ + Construct a Float. + + :param precision: the numeric precision for use in DDL ``CREATE + TABLE``. + + :param asdecimal: the same flag as that of :class:`.Numeric`, but + defaults to ``False``. Note that setting this flag to ``True`` + results in floating point conversion. + + :param decimal_return_scale: Default scale to use when converting + from floats to Python decimals. Floating point values will typically + be much longer due to decimal inaccuracy, and most floating point + database types don't have a notion of "scale", so by default the + float type looks for the first ten decimal places when converting. + Specfiying this value will override that length. Note that the + MySQL float types, which do include "scale", will use "scale" + as the default for decimal_return_scale, if not otherwise specified. + + .. versionadded:: 0.9.0 + + :param \**kwargs: deprecated. Additional arguments here are ignored + by the default :class:`.Float` type. For database specific + floats that support additional arguments, see that dialect's + documentation for details, such as + :class:`sqlalchemy.dialects.mysql.FLOAT`. + + """ + self.precision = precision + self.asdecimal = asdecimal + self.decimal_return_scale = decimal_return_scale + if kwargs: + util.warn_deprecated("Additional keyword arguments " + "passed to Float ignored.") + + def result_processor(self, dialect, coltype): + if self.asdecimal: + return processors.to_decimal_processor_factory( + decimal.Decimal, + self._effective_decimal_return_scale) + else: + return None + + @util.memoized_property + def _expression_adaptations(self): + return { + operators.mul: { + Interval: Interval, + Numeric: self.__class__, + }, + operators.div: { + Numeric: self.__class__, + }, + operators.truediv: { + Numeric: self.__class__, + }, + operators.add: { + Numeric: self.__class__, + }, + operators.sub: { + Numeric: self.__class__, + } + } + + +class DateTime(_DateAffinity, TypeEngine): + """A type for ``datetime.datetime()`` objects. + + Date and time types return objects from the Python ``datetime`` + module. Most DBAPIs have built in support for the datetime + module, with the noted exception of SQLite. In the case of + SQLite, date and time types are stored as strings which are then + converted back to datetime objects when rows are returned. + + """ + + __visit_name__ = 'datetime' + + def __init__(self, timezone=False): + """Construct a new :class:`.DateTime`. + + :param timezone: boolean. If True, and supported by the + backend, will produce 'TIMESTAMP WITH TIMEZONE'. For backends + that don't support timezone aware timestamps, has no + effect. + + """ + self.timezone = timezone + + def get_dbapi_type(self, dbapi): + return dbapi.DATETIME + + @property + def python_type(self): + return dt.datetime + + @util.memoized_property + def _expression_adaptations(self): + return { + operators.add: { + Interval: self.__class__, + }, + operators.sub: { + Interval: self.__class__, + DateTime: Interval, + }, + } + + +class Date(_DateAffinity, TypeEngine): + """A type for ``datetime.date()`` objects.""" + + __visit_name__ = 'date' + + def get_dbapi_type(self, dbapi): + return dbapi.DATETIME + + @property + def python_type(self): + return dt.date + + @util.memoized_property + def _expression_adaptations(self): + return { + operators.add: { + Integer: self.__class__, + Interval: DateTime, + Time: DateTime, + }, + operators.sub: { + # date - integer = date + Integer: self.__class__, + + # date - date = integer. + Date: Integer, + + Interval: DateTime, + + # date - datetime = interval, + # this one is not in the PG docs + # but works + DateTime: Interval, + }, + } + + +class Time(_DateAffinity, TypeEngine): + """A type for ``datetime.time()`` objects.""" + + __visit_name__ = 'time' + + def __init__(self, timezone=False): + self.timezone = timezone + + def get_dbapi_type(self, dbapi): + return dbapi.DATETIME + + @property + def python_type(self): + return dt.time + + @util.memoized_property + def _expression_adaptations(self): + return { + operators.add: { + Date: DateTime, + Interval: self.__class__ + }, + operators.sub: { + Time: Interval, + Interval: self.__class__, + }, + } + + +class _Binary(TypeEngine): + """Define base behavior for binary types.""" + + def __init__(self, length=None): + self.length = length + + def literal_processor(self, dialect): + def process(value): + value = value.decode(self.dialect.encoding).replace("'", "''") + return "'%s'" % value + return process + + @property + def python_type(self): + return util.binary_type + + # Python 3 - sqlite3 doesn't need the `Binary` conversion + # here, though pg8000 does to indicate "bytea" + def bind_processor(self, dialect): + DBAPIBinary = dialect.dbapi.Binary + + def process(value): + if value is not None: + return DBAPIBinary(value) + else: + return None + return process + + # Python 3 has native bytes() type + # both sqlite3 and pg8000 seem to return it, + # psycopg2 as of 2.5 returns 'memoryview' + if util.py2k: + def result_processor(self, dialect, coltype): + if util.jython: + def process(value): + if value is not None: + if isinstance(value, array.array): + return value.tostring() + return str(value) + else: + return None + else: + process = processors.to_str + return process + else: + def result_processor(self, dialect, coltype): + def process(value): + if value is not None: + value = bytes(value) + return value + return process + + def coerce_compared_value(self, op, value): + """See :meth:`.TypeEngine.coerce_compared_value` for a description.""" + + if isinstance(value, util.string_types): + return self + else: + return super(_Binary, self).coerce_compared_value(op, value) + + def get_dbapi_type(self, dbapi): + return dbapi.BINARY + + +class LargeBinary(_Binary): + """A type for large binary byte data. + + The Binary type generates BLOB or BYTEA when tables are created, + and also converts incoming values using the ``Binary`` callable + provided by each DB-API. + + """ + + __visit_name__ = 'large_binary' + + def __init__(self, length=None): + """ + Construct a LargeBinary type. + + :param length: optional, a length for the column for use in + DDL statements, for those BLOB types that accept a length + (i.e. MySQL). It does *not* produce a small BINARY/VARBINARY + type - use the BINARY/VARBINARY types specifically for those. + May be safely omitted if no ``CREATE + TABLE`` will be issued. Certain databases may require a + *length* for use in DDL, and will raise an exception when + the ``CREATE TABLE`` DDL is issued. + + """ + _Binary.__init__(self, length=length) + + +class Binary(LargeBinary): + """Deprecated. Renamed to LargeBinary.""" + + def __init__(self, *arg, **kw): + util.warn_deprecated('The Binary type has been renamed to ' + 'LargeBinary.') + LargeBinary.__init__(self, *arg, **kw) + + + +class SchemaType(SchemaEventTarget): + """Mark a type as possibly requiring schema-level DDL for usage. + + Supports types that must be explicitly created/dropped (i.e. PG ENUM type) + as well as types that are complimented by table or schema level + constraints, triggers, and other rules. + + :class:`.SchemaType` classes can also be targets for the + :meth:`.DDLEvents.before_parent_attach` and + :meth:`.DDLEvents.after_parent_attach` events, where the events fire off + surrounding the association of the type object with a parent + :class:`.Column`. + + .. seealso:: + + :class:`.Enum` + + :class:`.Boolean` + + + """ + + def __init__(self, name=None, schema=None, metadata=None, + inherit_schema=False, quote=None): + if name is not None: + self.name = quoted_name(name, quote) + else: + self.name = None + self.schema = schema + self.metadata = metadata + self.inherit_schema = inherit_schema + if self.metadata: + event.listen( + self.metadata, + "before_create", + util.portable_instancemethod(self._on_metadata_create) + ) + event.listen( + self.metadata, + "after_drop", + util.portable_instancemethod(self._on_metadata_drop) + ) + + def _set_parent(self, column): + column._on_table_attach(util.portable_instancemethod(self._set_table)) + + def _set_table(self, column, table): + if self.inherit_schema: + self.schema = table.schema + + event.listen( + table, + "before_create", + util.portable_instancemethod( + self._on_table_create) + ) + event.listen( + table, + "after_drop", + util.portable_instancemethod(self._on_table_drop) + ) + if self.metadata is None: + # TODO: what's the difference between self.metadata + # and table.metadata here ? + event.listen( + table.metadata, + "before_create", + util.portable_instancemethod(self._on_metadata_create) + ) + event.listen( + table.metadata, + "after_drop", + util.portable_instancemethod(self._on_metadata_drop) + ) + + def copy(self, **kw): + return self.adapt(self.__class__) + + def adapt(self, impltype, **kw): + schema = kw.pop('schema', self.schema) + metadata = kw.pop('metadata', self.metadata) + return impltype(name=self.name, + schema=schema, + metadata=metadata, + inherit_schema=self.inherit_schema, + **kw + ) + + @property + def bind(self): + return self.metadata and self.metadata.bind or None + + def create(self, bind=None, checkfirst=False): + """Issue CREATE ddl for this type, if applicable.""" + + if bind is None: + bind = _bind_or_error(self) + t = self.dialect_impl(bind.dialect) + if t.__class__ is not self.__class__ and isinstance(t, SchemaType): + t.create(bind=bind, checkfirst=checkfirst) + + def drop(self, bind=None, checkfirst=False): + """Issue DROP ddl for this type, if applicable.""" + + if bind is None: + bind = _bind_or_error(self) + t = self.dialect_impl(bind.dialect) + if t.__class__ is not self.__class__ and isinstance(t, SchemaType): + t.drop(bind=bind, checkfirst=checkfirst) + + def _on_table_create(self, target, bind, **kw): + t = self.dialect_impl(bind.dialect) + if t.__class__ is not self.__class__ and isinstance(t, SchemaType): + t._on_table_create(target, bind, **kw) + + def _on_table_drop(self, target, bind, **kw): + t = self.dialect_impl(bind.dialect) + if t.__class__ is not self.__class__ and isinstance(t, SchemaType): + t._on_table_drop(target, bind, **kw) + + def _on_metadata_create(self, target, bind, **kw): + t = self.dialect_impl(bind.dialect) + if t.__class__ is not self.__class__ and isinstance(t, SchemaType): + t._on_metadata_create(target, bind, **kw) + + def _on_metadata_drop(self, target, bind, **kw): + t = self.dialect_impl(bind.dialect) + if t.__class__ is not self.__class__ and isinstance(t, SchemaType): + t._on_metadata_drop(target, bind, **kw) + +class Enum(String, SchemaType): + """Generic Enum Type. + + The Enum type provides a set of possible string values which the + column is constrained towards. + + By default, uses the backend's native ENUM type if available, + else uses VARCHAR + a CHECK constraint. + + .. seealso:: + + :class:`~.postgresql.ENUM` - PostgreSQL-specific type, + which has additional functionality. + + """ + + __visit_name__ = 'enum' + + def __init__(self, *enums, **kw): + """Construct an enum. + + Keyword arguments which don't apply to a specific backend are ignored + by that backend. + + :param \*enums: string or unicode enumeration labels. If unicode + labels are present, the `convert_unicode` flag is auto-enabled. + + :param convert_unicode: Enable unicode-aware bind parameter and + result-set processing for this Enum's data. This is set + automatically based on the presence of unicode label strings. + + :param metadata: Associate this type directly with a ``MetaData`` + object. For types that exist on the target database as an + independent schema construct (Postgresql), this type will be + created and dropped within ``create_all()`` and ``drop_all()`` + operations. If the type is not associated with any ``MetaData`` + object, it will associate itself with each ``Table`` in which it is + used, and will be created when any of those individual tables are + created, after a check is performed for it's existence. The type is + only dropped when ``drop_all()`` is called for that ``Table`` + object's metadata, however. + + :param name: The name of this type. This is required for Postgresql + and any future supported database which requires an explicitly + named type, or an explicitly named constraint in order to generate + the type and/or a table that uses it. + + :param native_enum: Use the database's native ENUM type when + available. Defaults to True. When False, uses VARCHAR + check + constraint for all backends. + + :param schema: Schema name of this type. For types that exist on the + target database as an independent schema construct (Postgresql), + this parameter specifies the named schema in which the type is + present. + + .. note:: + + The ``schema`` of the :class:`.Enum` type does not + by default make use of the ``schema`` established on the + owning :class:`.Table`. If this behavior is desired, + set the ``inherit_schema`` flag to ``True``. + + :param quote: Set explicit quoting preferences for the type's name. + + :param inherit_schema: When ``True``, the "schema" from the owning + :class:`.Table` will be copied to the "schema" attribute of this + :class:`.Enum`, replacing whatever value was passed for the + ``schema`` attribute. This also takes effect when using the + :meth:`.Table.tometadata` operation. + + .. versionadded:: 0.8 + + """ + self.enums = enums + self.native_enum = kw.pop('native_enum', True) + convert_unicode = kw.pop('convert_unicode', None) + if convert_unicode is None: + for e in enums: + if isinstance(e, util.text_type): + convert_unicode = True + break + else: + convert_unicode = False + + if self.enums: + length = max(len(x) for x in self.enums) + else: + length = 0 + String.__init__(self, + length=length, + convert_unicode=convert_unicode, + ) + SchemaType.__init__(self, **kw) + + def __repr__(self): + return util.generic_repr(self, + to_inspect=[Enum, SchemaType], + ) + + def _should_create_constraint(self, compiler): + return not self.native_enum or \ + not compiler.dialect.supports_native_enum + + @util.dependencies("sqlalchemy.sql.schema") + def _set_table(self, schema, column, table): + if self.native_enum: + SchemaType._set_table(self, column, table) + + e = schema.CheckConstraint( + type_coerce(column, self).in_(self.enums), + name=self.name, + _create_rule=util.portable_instancemethod( + self._should_create_constraint) + ) + table.append_constraint(e) + + def adapt(self, impltype, **kw): + schema = kw.pop('schema', self.schema) + metadata = kw.pop('metadata', self.metadata) + if issubclass(impltype, Enum): + return impltype(name=self.name, + schema=schema, + metadata=metadata, + convert_unicode=self.convert_unicode, + native_enum=self.native_enum, + inherit_schema=self.inherit_schema, + *self.enums, + **kw + ) + else: + return super(Enum, self).adapt(impltype, **kw) + + +class PickleType(TypeDecorator): + """Holds Python objects, which are serialized using pickle. + + PickleType builds upon the Binary type to apply Python's + ``pickle.dumps()`` to incoming objects, and ``pickle.loads()`` on + the way out, allowing any pickleable Python object to be stored as + a serialized binary field. + + To allow ORM change events to propagate for elements associated + with :class:`.PickleType`, see :ref:`mutable_toplevel`. + + """ + + impl = LargeBinary + + def __init__(self, protocol=pickle.HIGHEST_PROTOCOL, + pickler=None, comparator=None): + """ + Construct a PickleType. + + :param protocol: defaults to ``pickle.HIGHEST_PROTOCOL``. + + :param pickler: defaults to cPickle.pickle or pickle.pickle if + cPickle is not available. May be any object with + pickle-compatible ``dumps` and ``loads`` methods. + + :param comparator: a 2-arg callable predicate used + to compare values of this type. If left as ``None``, + the Python "equals" operator is used to compare values. + + """ + self.protocol = protocol + self.pickler = pickler or pickle + self.comparator = comparator + super(PickleType, self).__init__() + + def __reduce__(self): + return PickleType, (self.protocol, + None, + self.comparator) + + def bind_processor(self, dialect): + impl_processor = self.impl.bind_processor(dialect) + dumps = self.pickler.dumps + protocol = self.protocol + if impl_processor: + def process(value): + if value is not None: + value = dumps(value, protocol) + return impl_processor(value) + else: + def process(value): + if value is not None: + value = dumps(value, protocol) + return value + return process + + def result_processor(self, dialect, coltype): + impl_processor = self.impl.result_processor(dialect, coltype) + loads = self.pickler.loads + if impl_processor: + def process(value): + value = impl_processor(value) + if value is None: + return None + return loads(value) + else: + def process(value): + if value is None: + return None + return loads(value) + return process + + def compare_values(self, x, y): + if self.comparator: + return self.comparator(x, y) + else: + return x == y + + +class Boolean(TypeEngine, SchemaType): + """A bool datatype. + + Boolean typically uses BOOLEAN or SMALLINT on the DDL side, and on + the Python side deals in ``True`` or ``False``. + + """ + + __visit_name__ = 'boolean' + + def __init__(self, create_constraint=True, name=None): + """Construct a Boolean. + + :param create_constraint: defaults to True. If the boolean + is generated as an int/smallint, also create a CHECK constraint + on the table that ensures 1 or 0 as a value. + + :param name: if a CHECK constraint is generated, specify + the name of the constraint. + + """ + self.create_constraint = create_constraint + self.name = name + + def _should_create_constraint(self, compiler): + return not compiler.dialect.supports_native_boolean + + @util.dependencies("sqlalchemy.sql.schema") + def _set_table(self, schema, column, table): + if not self.create_constraint: + return + + e = schema.CheckConstraint( + type_coerce(column, self).in_([0, 1]), + name=self.name, + _create_rule=util.portable_instancemethod( + self._should_create_constraint) + ) + table.append_constraint(e) + + @property + def python_type(self): + return bool + + def bind_processor(self, dialect): + if dialect.supports_native_boolean: + return None + else: + return processors.boolean_to_int + + def result_processor(self, dialect, coltype): + if dialect.supports_native_boolean: + return None + else: + return processors.int_to_boolean + + +class Interval(_DateAffinity, TypeDecorator): + """A type for ``datetime.timedelta()`` objects. + + The Interval type deals with ``datetime.timedelta`` objects. In + PostgreSQL, the native ``INTERVAL`` type is used; for others, the + value is stored as a date which is relative to the "epoch" + (Jan. 1, 1970). + + Note that the ``Interval`` type does not currently provide date arithmetic + operations on platforms which do not support interval types natively. Such + operations usually require transformation of both sides of the expression + (such as, conversion of both sides into integer epoch values first) which + currently is a manual procedure (such as via + :attr:`~sqlalchemy.sql.expression.func`). + + """ + + impl = DateTime + epoch = dt.datetime.utcfromtimestamp(0) + + def __init__(self, native=True, + second_precision=None, + day_precision=None): + """Construct an Interval object. + + :param native: when True, use the actual + INTERVAL type provided by the database, if + supported (currently Postgresql, Oracle). + Otherwise, represent the interval data as + an epoch value regardless. + + :param second_precision: For native interval types + which support a "fractional seconds precision" parameter, + i.e. Oracle and Postgresql + + :param day_precision: for native interval types which + support a "day precision" parameter, i.e. Oracle. + + """ + super(Interval, self).__init__() + self.native = native + self.second_precision = second_precision + self.day_precision = day_precision + + def adapt(self, cls, **kw): + if self.native and hasattr(cls, '_adapt_from_generic_interval'): + return cls._adapt_from_generic_interval(self, **kw) + else: + return self.__class__( + native=self.native, + second_precision=self.second_precision, + day_precision=self.day_precision, + **kw) + + @property + def python_type(self): + return dt.timedelta + + def bind_processor(self, dialect): + impl_processor = self.impl.bind_processor(dialect) + epoch = self.epoch + if impl_processor: + def process(value): + if value is not None: + value = epoch + value + return impl_processor(value) + else: + def process(value): + if value is not None: + value = epoch + value + return value + return process + + def result_processor(self, dialect, coltype): + impl_processor = self.impl.result_processor(dialect, coltype) + epoch = self.epoch + if impl_processor: + def process(value): + value = impl_processor(value) + if value is None: + return None + return value - epoch + else: + def process(value): + if value is None: + return None + return value - epoch + return process + + @util.memoized_property + def _expression_adaptations(self): + return { + operators.add: { + Date: DateTime, + Interval: self.__class__, + DateTime: DateTime, + Time: Time, + }, + operators.sub: { + Interval: self.__class__ + }, + operators.mul: { + Numeric: self.__class__ + }, + operators.truediv: { + Numeric: self.__class__ + }, + operators.div: { + Numeric: self.__class__ + } + } + + @property + def _type_affinity(self): + return Interval + + def coerce_compared_value(self, op, value): + """See :meth:`.TypeEngine.coerce_compared_value` for a description.""" + + return self.impl.coerce_compared_value(op, value) + + +class REAL(Float): + """The SQL REAL type.""" + + __visit_name__ = 'REAL' + + +class FLOAT(Float): + """The SQL FLOAT type.""" + + __visit_name__ = 'FLOAT' + + +class NUMERIC(Numeric): + """The SQL NUMERIC type.""" + + __visit_name__ = 'NUMERIC' + + +class DECIMAL(Numeric): + """The SQL DECIMAL type.""" + + __visit_name__ = 'DECIMAL' + + +class INTEGER(Integer): + """The SQL INT or INTEGER type.""" + + __visit_name__ = 'INTEGER' +INT = INTEGER + + +class SMALLINT(SmallInteger): + """The SQL SMALLINT type.""" + + __visit_name__ = 'SMALLINT' + + +class BIGINT(BigInteger): + """The SQL BIGINT type.""" + + __visit_name__ = 'BIGINT' + + +class TIMESTAMP(DateTime): + """The SQL TIMESTAMP type.""" + + __visit_name__ = 'TIMESTAMP' + + def get_dbapi_type(self, dbapi): + return dbapi.TIMESTAMP + + +class DATETIME(DateTime): + """The SQL DATETIME type.""" + + __visit_name__ = 'DATETIME' + + +class DATE(Date): + """The SQL DATE type.""" + + __visit_name__ = 'DATE' + + +class TIME(Time): + """The SQL TIME type.""" + + __visit_name__ = 'TIME' + + +class TEXT(Text): + """The SQL TEXT type.""" + + __visit_name__ = 'TEXT' + + +class CLOB(Text): + """The CLOB type. + + This type is found in Oracle and Informix. + """ + + __visit_name__ = 'CLOB' + + +class VARCHAR(String): + """The SQL VARCHAR type.""" + + __visit_name__ = 'VARCHAR' + + +class NVARCHAR(Unicode): + """The SQL NVARCHAR type.""" + + __visit_name__ = 'NVARCHAR' + + +class CHAR(String): + """The SQL CHAR type.""" + + __visit_name__ = 'CHAR' + + +class NCHAR(Unicode): + """The SQL NCHAR type.""" + + __visit_name__ = 'NCHAR' + + +class BLOB(LargeBinary): + """The SQL BLOB type.""" + + __visit_name__ = 'BLOB' + + +class BINARY(_Binary): + """The SQL BINARY type.""" + + __visit_name__ = 'BINARY' + + +class VARBINARY(_Binary): + """The SQL VARBINARY type.""" + + __visit_name__ = 'VARBINARY' + + +class BOOLEAN(Boolean): + """The SQL BOOLEAN type.""" + + __visit_name__ = 'BOOLEAN' + +class NullType(TypeEngine): + """An unknown type. + + :class:`.NullType` is used as a default type for those cases where + a type cannot be determined, including: + + * During table reflection, when the type of a column is not recognized + by the :class:`.Dialect` + * When constructing SQL expressions using plain Python objects of + unknown types (e.g. ``somecolumn == my_special_object``) + * When a new :class:`.Column` is created, and the given type is passed + as ``None`` or is not passed at all. + + The :class:`.NullType` can be used within SQL expression invocation + without issue, it just has no behavior either at the expression construction + level or at the bind-parameter/result processing level. :class:`.NullType` + will result in a :exc:`.CompileError` if the compiler is asked to render + the type itself, such as if it is used in a :func:`.cast` operation + or within a schema creation operation such as that invoked by + :meth:`.MetaData.create_all` or the :class:`.CreateTable` construct. + + """ + __visit_name__ = 'null' + + _isnull = True + + def literal_processor(self, dialect): + def process(value): + return "NULL" + return process + + class Comparator(TypeEngine.Comparator): + def _adapt_expression(self, op, other_comparator): + if isinstance(other_comparator, NullType.Comparator) or \ + not operators.is_commutative(op): + return op, self.expr.type + else: + return other_comparator._adapt_expression(op, self) + comparator_factory = Comparator + + +NULLTYPE = NullType() +BOOLEANTYPE = Boolean() +STRINGTYPE = String() +INTEGERTYPE = Integer() + +_type_map = { + int: Integer(), + float: Numeric(), + bool: BOOLEANTYPE, + decimal.Decimal: Numeric(), + dt.date: Date(), + dt.datetime: DateTime(), + dt.time: Time(), + dt.timedelta: Interval(), + util.NoneType: NULLTYPE +} + +if util.py3k: + _type_map[bytes] = LargeBinary() + _type_map[str] = Unicode() +else: + _type_map[unicode] = Unicode() + _type_map[str] = String() + + +# back-assign to type_api +from . import type_api +type_api.BOOLEANTYPE = BOOLEANTYPE +type_api.STRINGTYPE = STRINGTYPE +type_api.INTEGERTYPE = INTEGERTYPE +type_api.NULLTYPE = NULLTYPE +type_api._type_map = _type_map + +# this one, there's all kinds of ways to play it, but at the EOD +# there's just a giant dependency cycle between the typing system and +# the expression element system, as you might expect. We can use +# importlaters or whatnot, but the typing system just necessarily has +# to have some kind of connection like this. right now we're injecting the +# _DefaultColumnComparator implementation into the TypeEngine.Comparator interface. +# Alternatively TypeEngine.Comparator could have an "impl" injected, though +# just injecting the base is simpler, error free, and more performant. +class Comparator(_DefaultColumnComparator): + BOOLEANTYPE = BOOLEANTYPE + +TypeEngine.Comparator.__bases__ = (Comparator, ) + TypeEngine.Comparator.__bases__ + diff --git a/libs/sqlalchemy/sql/type_api.py b/libs/sqlalchemy/sql/type_api.py new file mode 100644 index 00000000..c6aad92b --- /dev/null +++ b/libs/sqlalchemy/sql/type_api.py @@ -0,0 +1,1053 @@ +# sql/types_api.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +"""Base types API. + +""" + + +from .. import exc, util +from . import operators +from .visitors import Visitable + +# these are back-assigned by sqltypes. +BOOLEANTYPE = None +INTEGERTYPE = None +NULLTYPE = None +STRINGTYPE = None + +class TypeEngine(Visitable): + """Base for built-in types.""" + + _sqla_type = True + _isnull = False + + class Comparator(operators.ColumnOperators): + """Base class for custom comparison operations defined at the + type level. See :attr:`.TypeEngine.comparator_factory`. + + + """ + + def __init__(self, expr): + self.expr = expr + + def __reduce__(self): + return _reconstitute_comparator, (self.expr, ) + + + hashable = True + """Flag, if False, means values from this type aren't hashable. + + Used by the ORM when uniquing result lists. + + """ + + comparator_factory = Comparator + """A :class:`.TypeEngine.Comparator` class which will apply + to operations performed by owning :class:`.ColumnElement` objects. + + The :attr:`.comparator_factory` attribute is a hook consulted by + the core expression system when column and SQL expression operations + are performed. When a :class:`.TypeEngine.Comparator` class is + associated with this attribute, it allows custom re-definition of + all existing operators, as well as definition of new operators. + Existing operators include those provided by Python operator overloading + such as :meth:`.operators.ColumnOperators.__add__` and + :meth:`.operators.ColumnOperators.__eq__`, + those provided as standard + attributes of :class:`.operators.ColumnOperators` such as + :meth:`.operators.ColumnOperators.like` + and :meth:`.operators.ColumnOperators.in_`. + + Rudimentary usage of this hook is allowed through simple subclassing + of existing types, or alternatively by using :class:`.TypeDecorator`. + See the documentation section :ref:`types_operators` for examples. + + .. versionadded:: 0.8 The expression system was enhanced to support + customization of operators on a per-type level. + + """ + + def copy_value(self, value): + return value + + def literal_processor(self, dialect): + """Return a conversion function for processing literal values that are + to be rendered directly without using binds. + + This function is used when the compiler makes use of the + "literal_binds" flag, typically used in DDL generation as well + as in certain scenarios where backends don't accept bound parameters. + + .. versionadded:: 0.9.0 + + """ + return None + + def bind_processor(self, dialect): + """Return a conversion function for processing bind values. + + Returns a callable which will receive a bind parameter value + as the sole positional argument and will return a value to + send to the DB-API. + + If processing is not necessary, the method should return ``None``. + + :param dialect: Dialect instance in use. + + """ + return None + + def result_processor(self, dialect, coltype): + """Return a conversion function for processing result row values. + + Returns a callable which will receive a result row column + value as the sole positional argument and will return a value + to return to the user. + + If processing is not necessary, the method should return ``None``. + + :param dialect: Dialect instance in use. + + :param coltype: DBAPI coltype argument received in cursor.description. + + """ + return None + + def column_expression(self, colexpr): + """Given a SELECT column expression, return a wrapping SQL expression. + + This is typically a SQL function that wraps a column expression + as rendered in the columns clause of a SELECT statement. + It is used for special data types that require + columns to be wrapped in some special database function in order + to coerce the value before being sent back to the application. + It is the SQL analogue of the :meth:`.TypeEngine.result_processor` + method. + + The method is evaluated at statement compile time, as opposed + to statement construction time. + + See also: + + :ref:`types_sql_value_processing` + + """ + + return None + + @util.memoized_property + def _has_column_expression(self): + """memoized boolean, check if column_expression is implemented. + + Allows the method to be skipped for the vast majority of expression + types that don't use this feature. + + """ + + return self.__class__.column_expression.__code__ \ + is not TypeEngine.column_expression.__code__ + + def bind_expression(self, bindvalue): + """"Given a bind value (i.e. a :class:`.BindParameter` instance), + return a SQL expression in its place. + + This is typically a SQL function that wraps the existing bound + parameter within the statement. It is used for special data types + that require literals being wrapped in some special database function + in order to coerce an application-level value into a database-specific + format. It is the SQL analogue of the + :meth:`.TypeEngine.bind_processor` method. + + The method is evaluated at statement compile time, as opposed + to statement construction time. + + Note that this method, when implemented, should always return + the exact same structure, without any conditional logic, as it + may be used in an executemany() call against an arbitrary number + of bound parameter sets. + + See also: + + :ref:`types_sql_value_processing` + + """ + return None + + @util.memoized_property + def _has_bind_expression(self): + """memoized boolean, check if bind_expression is implemented. + + Allows the method to be skipped for the vast majority of expression + types that don't use this feature. + + """ + + return self.__class__.bind_expression.__code__ \ + is not TypeEngine.bind_expression.__code__ + + def compare_values(self, x, y): + """Compare two values for equality.""" + + return x == y + + def get_dbapi_type(self, dbapi): + """Return the corresponding type object from the underlying DB-API, if + any. + + This can be useful for calling ``setinputsizes()``, for example. + + """ + return None + + @property + def python_type(self): + """Return the Python type object expected to be returned + by instances of this type, if known. + + Basically, for those types which enforce a return type, + or are known across the board to do such for all common + DBAPIs (like ``int`` for example), will return that type. + + If a return type is not defined, raises + ``NotImplementedError``. + + Note that any type also accommodates NULL in SQL which + means you can also get back ``None`` from any type + in practice. + + """ + raise NotImplementedError() + + def with_variant(self, type_, dialect_name): + """Produce a new type object that will utilize the given + type when applied to the dialect of the given name. + + e.g.:: + + from sqlalchemy.types import String + from sqlalchemy.dialects import mysql + + s = String() + + s = s.with_variant(mysql.VARCHAR(collation='foo'), 'mysql') + + The construction of :meth:`.TypeEngine.with_variant` is always + from the "fallback" type to that which is dialect specific. + The returned type is an instance of :class:`.Variant`, which + itself provides a :meth:`~sqlalchemy.types.Variant.with_variant` + that can be called repeatedly. + + :param type_: a :class:`.TypeEngine` that will be selected + as a variant from the originating type, when a dialect + of the given name is in use. + :param dialect_name: base name of the dialect which uses + this type. (i.e. ``'postgresql'``, ``'mysql'``, etc.) + + .. versionadded:: 0.7.2 + + """ + return Variant(self, {dialect_name: type_}) + + + @util.memoized_property + def _type_affinity(self): + """Return a rudimental 'affinity' value expressing the general class + of type.""" + + typ = None + for t in self.__class__.__mro__: + if t in (TypeEngine, UserDefinedType): + return typ + elif issubclass(t, (TypeEngine, UserDefinedType)): + typ = t + else: + return self.__class__ + + def dialect_impl(self, dialect): + """Return a dialect-specific implementation for this + :class:`.TypeEngine`. + + """ + try: + return dialect._type_memos[self]['impl'] + except KeyError: + return self._dialect_info(dialect)['impl'] + + + def _cached_literal_processor(self, dialect): + """Return a dialect-specific literal processor for this type.""" + try: + return dialect._type_memos[self]['literal'] + except KeyError: + d = self._dialect_info(dialect) + d['literal'] = lp = d['impl'].literal_processor(dialect) + return lp + + def _cached_bind_processor(self, dialect): + """Return a dialect-specific bind processor for this type.""" + + try: + return dialect._type_memos[self]['bind'] + except KeyError: + d = self._dialect_info(dialect) + d['bind'] = bp = d['impl'].bind_processor(dialect) + return bp + + def _cached_result_processor(self, dialect, coltype): + """Return a dialect-specific result processor for this type.""" + + try: + return dialect._type_memos[self][coltype] + except KeyError: + d = self._dialect_info(dialect) + # key assumption: DBAPI type codes are + # constants. Else this dictionary would + # grow unbounded. + d[coltype] = rp = d['impl'].result_processor(dialect, coltype) + return rp + + def _dialect_info(self, dialect): + """Return a dialect-specific registry which + caches a dialect-specific implementation, bind processing + function, and one or more result processing functions.""" + + if self in dialect._type_memos: + return dialect._type_memos[self] + else: + impl = self._gen_dialect_impl(dialect) + if impl is self: + impl = self.adapt(type(self)) + # this can't be self, else we create a cycle + assert impl is not self + dialect._type_memos[self] = d = {'impl': impl} + return d + + def _gen_dialect_impl(self, dialect): + return dialect.type_descriptor(self) + + def adapt(self, cls, **kw): + """Produce an "adapted" form of this type, given an "impl" class + to work with. + + This method is used internally to associate generic + types with "implementation" types that are specific to a particular + dialect. + """ + return util.constructor_copy(self, cls, **kw) + + + def coerce_compared_value(self, op, value): + """Suggest a type for a 'coerced' Python value in an expression. + + Given an operator and value, gives the type a chance + to return a type which the value should be coerced into. + + The default behavior here is conservative; if the right-hand + side is already coerced into a SQL type based on its + Python type, it is usually left alone. + + End-user functionality extension here should generally be via + :class:`.TypeDecorator`, which provides more liberal behavior in that + it defaults to coercing the other side of the expression into this + type, thus applying special Python conversions above and beyond those + needed by the DBAPI to both ides. It also provides the public method + :meth:`.TypeDecorator.coerce_compared_value` which is intended for + end-user customization of this behavior. + + """ + _coerced_type = _type_map.get(type(value), NULLTYPE) + if _coerced_type is NULLTYPE or _coerced_type._type_affinity \ + is self._type_affinity: + return self + else: + return _coerced_type + + def _compare_type_affinity(self, other): + return self._type_affinity is other._type_affinity + + def compile(self, dialect=None): + """Produce a string-compiled form of this :class:`.TypeEngine`. + + When called with no arguments, uses a "default" dialect + to produce a string result. + + :param dialect: a :class:`.Dialect` instance. + + """ + # arg, return value is inconsistent with + # ClauseElement.compile()....this is a mistake. + + if not dialect: + dialect = self._default_dialect() + + return dialect.type_compiler.process(self) + + @util.dependencies("sqlalchemy.engine.default") + def _default_dialect(self, default): + if self.__class__.__module__.startswith("sqlalchemy.dialects"): + tokens = self.__class__.__module__.split(".")[0:3] + mod = ".".join(tokens) + return getattr(__import__(mod).dialects, tokens[-1]).dialect() + else: + return default.DefaultDialect() + + def __str__(self): + if util.py2k: + return unicode(self.compile()).\ + encode('ascii', 'backslashreplace') + else: + return str(self.compile()) + + def __repr__(self): + return util.generic_repr(self) + +class UserDefinedType(TypeEngine): + """Base for user defined types. + + This should be the base of new types. Note that + for most cases, :class:`.TypeDecorator` is probably + more appropriate:: + + import sqlalchemy.types as types + + class MyType(types.UserDefinedType): + def __init__(self, precision = 8): + self.precision = precision + + def get_col_spec(self): + return "MYTYPE(%s)" % self.precision + + def bind_processor(self, dialect): + def process(value): + return value + return process + + def result_processor(self, dialect, coltype): + def process(value): + return value + return process + + Once the type is made, it's immediately usable:: + + table = Table('foo', meta, + Column('id', Integer, primary_key=True), + Column('data', MyType(16)) + ) + + """ + __visit_name__ = "user_defined" + + + class Comparator(TypeEngine.Comparator): + def _adapt_expression(self, op, other_comparator): + if hasattr(self.type, 'adapt_operator'): + util.warn_deprecated( + "UserDefinedType.adapt_operator is deprecated. Create " + "a UserDefinedType.Comparator subclass instead which " + "generates the desired expression constructs, given a " + "particular operator." + ) + return self.type.adapt_operator(op), self.type + else: + return op, self.type + + comparator_factory = Comparator + + def coerce_compared_value(self, op, value): + """Suggest a type for a 'coerced' Python value in an expression. + + Default behavior for :class:`.UserDefinedType` is the + same as that of :class:`.TypeDecorator`; by default it returns + ``self``, assuming the compared value should be coerced into + the same type as this one. See + :meth:`.TypeDecorator.coerce_compared_value` for more detail. + + .. versionchanged:: 0.8 :meth:`.UserDefinedType.coerce_compared_value` + now returns ``self`` by default, rather than falling onto the + more fundamental behavior of + :meth:`.TypeEngine.coerce_compared_value`. + + """ + + return self + + +class TypeDecorator(TypeEngine): + """Allows the creation of types which add additional functionality + to an existing type. + + This method is preferred to direct subclassing of SQLAlchemy's + built-in types as it ensures that all required functionality of + the underlying type is kept in place. + + Typical usage:: + + import sqlalchemy.types as types + + class MyType(types.TypeDecorator): + '''Prefixes Unicode values with "PREFIX:" on the way in and + strips it off on the way out. + ''' + + impl = types.Unicode + + def process_bind_param(self, value, dialect): + return "PREFIX:" + value + + def process_result_value(self, value, dialect): + return value[7:] + + def copy(self): + return MyType(self.impl.length) + + The class-level "impl" attribute is required, and can reference any + TypeEngine class. Alternatively, the load_dialect_impl() method + can be used to provide different type classes based on the dialect + given; in this case, the "impl" variable can reference + ``TypeEngine`` as a placeholder. + + Types that receive a Python type that isn't similar to the ultimate type + used may want to define the :meth:`TypeDecorator.coerce_compared_value` + method. This is used to give the expression system a hint when coercing + Python objects into bind parameters within expressions. Consider this + expression:: + + mytable.c.somecol + datetime.date(2009, 5, 15) + + Above, if "somecol" is an ``Integer`` variant, it makes sense that + we're doing date arithmetic, where above is usually interpreted + by databases as adding a number of days to the given date. + The expression system does the right thing by not attempting to + coerce the "date()" value into an integer-oriented bind parameter. + + However, in the case of ``TypeDecorator``, we are usually changing an + incoming Python type to something new - ``TypeDecorator`` by default will + "coerce" the non-typed side to be the same type as itself. Such as below, + we define an "epoch" type that stores a date value as an integer:: + + class MyEpochType(types.TypeDecorator): + impl = types.Integer + + epoch = datetime.date(1970, 1, 1) + + def process_bind_param(self, value, dialect): + return (value - self.epoch).days + + def process_result_value(self, value, dialect): + return self.epoch + timedelta(days=value) + + Our expression of ``somecol + date`` with the above type will coerce the + "date" on the right side to also be treated as ``MyEpochType``. + + This behavior can be overridden via the + :meth:`~TypeDecorator.coerce_compared_value` method, which returns a type + that should be used for the value of the expression. Below we set it such + that an integer value will be treated as an ``Integer``, and any other + value is assumed to be a date and will be treated as a ``MyEpochType``:: + + def coerce_compared_value(self, op, value): + if isinstance(value, int): + return Integer() + else: + return self + + """ + + __visit_name__ = "type_decorator" + + def __init__(self, *args, **kwargs): + """Construct a :class:`.TypeDecorator`. + + Arguments sent here are passed to the constructor + of the class assigned to the ``impl`` class level attribute, + assuming the ``impl`` is a callable, and the resulting + object is assigned to the ``self.impl`` instance attribute + (thus overriding the class attribute of the same name). + + If the class level ``impl`` is not a callable (the unusual case), + it will be assigned to the same instance attribute 'as-is', + ignoring those arguments passed to the constructor. + + Subclasses can override this to customize the generation + of ``self.impl`` entirely. + + """ + + if not hasattr(self.__class__, 'impl'): + raise AssertionError("TypeDecorator implementations " + "require a class-level variable " + "'impl' which refers to the class of " + "type being decorated") + self.impl = to_instance(self.__class__.impl, *args, **kwargs) + + coerce_to_is_types = (util.NoneType, ) + """Specify those Python types which should be coerced at the expression + level to "IS " when compared using ``==`` (and same for + ``IS NOT`` in conjunction with ``!=``. + + For most SQLAlchemy types, this includes ``NoneType``, as well as ``bool``. + + :class:`.TypeDecorator` modifies this list to only include ``NoneType``, + as typedecorator implementations that deal with boolean types are common. + + Custom :class:`.TypeDecorator` classes can override this attribute to + return an empty tuple, in which case no values will be coerced to + constants. + + ..versionadded:: 0.8.2 + Added :attr:`.TypeDecorator.coerce_to_is_types` to allow for easier + control of ``__eq__()`` ``__ne__()`` operations. + + """ + + class Comparator(TypeEngine.Comparator): + + def operate(self, op, *other, **kwargs): + kwargs['_python_is_types'] = self.expr.type.coerce_to_is_types + return super(TypeDecorator.Comparator, self).operate( + op, *other, **kwargs) + + def reverse_operate(self, op, other, **kwargs): + kwargs['_python_is_types'] = self.expr.type.coerce_to_is_types + return super(TypeDecorator.Comparator, self).reverse_operate( + op, other, **kwargs) + + @property + def comparator_factory(self): + return type("TDComparator", + (TypeDecorator.Comparator, self.impl.comparator_factory), + {}) + + def _gen_dialect_impl(self, dialect): + """ + #todo + """ + adapted = dialect.type_descriptor(self) + if adapted is not self: + return adapted + + # otherwise adapt the impl type, link + # to a copy of this TypeDecorator and return + # that. + typedesc = self.load_dialect_impl(dialect).dialect_impl(dialect) + tt = self.copy() + if not isinstance(tt, self.__class__): + raise AssertionError('Type object %s does not properly ' + 'implement the copy() method, it must ' + 'return an object of type %s' % (self, + self.__class__)) + tt.impl = typedesc + return tt + + @property + def _type_affinity(self): + """ + #todo + """ + return self.impl._type_affinity + + def type_engine(self, dialect): + """Return a dialect-specific :class:`.TypeEngine` instance + for this :class:`.TypeDecorator`. + + In most cases this returns a dialect-adapted form of + the :class:`.TypeEngine` type represented by ``self.impl``. + Makes usage of :meth:`dialect_impl` but also traverses + into wrapped :class:`.TypeDecorator` instances. + Behavior can be customized here by overriding + :meth:`load_dialect_impl`. + + """ + adapted = dialect.type_descriptor(self) + if type(adapted) is not type(self): + return adapted + elif isinstance(self.impl, TypeDecorator): + return self.impl.type_engine(dialect) + else: + return self.load_dialect_impl(dialect) + + def load_dialect_impl(self, dialect): + """Return a :class:`.TypeEngine` object corresponding to a dialect. + + This is an end-user override hook that can be used to provide + differing types depending on the given dialect. It is used + by the :class:`.TypeDecorator` implementation of :meth:`type_engine` + to help determine what type should ultimately be returned + for a given :class:`.TypeDecorator`. + + By default returns ``self.impl``. + + """ + return self.impl + + def __getattr__(self, key): + """Proxy all other undefined accessors to the underlying + implementation.""" + return getattr(self.impl, key) + + def process_literal_param(self, value, dialect): + """Receive a literal parameter value to be rendered inline within + a statement. + + This method is used when the compiler renders a + literal value without using binds, typically within DDL + such as in the "server default" of a column or an expression + within a CHECK constraint. + + The returned string will be rendered into the output string. + + .. versionadded:: 0.9.0 + + """ + raise NotImplementedError() + + def process_bind_param(self, value, dialect): + """Receive a bound parameter value to be converted. + + Subclasses override this method to return the + value that should be passed along to the underlying + :class:`.TypeEngine` object, and from there to the + DBAPI ``execute()`` method. + + The operation could be anything desired to perform custom + behavior, such as transforming or serializing data. + This could also be used as a hook for validating logic. + + This operation should be designed with the reverse operation + in mind, which would be the process_result_value method of + this class. + + :param value: Data to operate upon, of any type expected by + this method in the subclass. Can be ``None``. + :param dialect: the :class:`.Dialect` in use. + + """ + + raise NotImplementedError() + + def process_result_value(self, value, dialect): + """Receive a result-row column value to be converted. + + Subclasses should implement this method to operate on data + fetched from the database. + + Subclasses override this method to return the + value that should be passed back to the application, + given a value that is already processed by + the underlying :class:`.TypeEngine` object, originally + from the DBAPI cursor method ``fetchone()`` or similar. + + The operation could be anything desired to perform custom + behavior, such as transforming or serializing data. + This could also be used as a hook for validating logic. + + :param value: Data to operate upon, of any type expected by + this method in the subclass. Can be ``None``. + :param dialect: the :class:`.Dialect` in use. + + This operation should be designed to be reversible by + the "process_bind_param" method of this class. + + """ + + raise NotImplementedError() + + @util.memoized_property + def _has_bind_processor(self): + """memoized boolean, check if process_bind_param is implemented. + + Allows the base process_bind_param to raise + NotImplementedError without needing to test an expensive + exception throw. + + """ + + return self.__class__.process_bind_param.__code__ \ + is not TypeDecorator.process_bind_param.__code__ + + @util.memoized_property + def _has_literal_processor(self): + """memoized boolean, check if process_literal_param is implemented. + + + """ + + return self.__class__.process_literal_param.__code__ \ + is not TypeDecorator.process_literal_param.__code__ + + def literal_processor(self, dialect): + """Provide a literal processing function for the given + :class:`.Dialect`. + + Subclasses here will typically override :meth:`.TypeDecorator.process_literal_param` + instead of this method directly. + + By default, this method makes use of :meth:`.TypeDecorator.process_bind_param` + if that method is implemented, where :meth:`.TypeDecorator.process_literal_param` + is not. The rationale here is that :class:`.TypeDecorator` typically deals + with Python conversions of data that are above the layer of database + presentation. With the value converted by :meth:`.TypeDecorator.process_bind_param`, + the underlying type will then handle whether it needs to be presented to the + DBAPI as a bound parameter or to the database as an inline SQL value. + + .. versionadded:: 0.9.0 + + """ + if self._has_literal_processor: + process_param = self.process_literal_param + elif self._has_bind_processor: + # the bind processor should normally be OK + # for TypeDecorator since it isn't doing DB-level + # handling, the handling here won't be different for bound vs. + # literals. + process_param = self.process_bind_param + else: + process_param = None + + if process_param: + impl_processor = self.impl.literal_processor(dialect) + if impl_processor: + def process(value): + return impl_processor(process_param(value, dialect)) + else: + def process(value): + return process_param(value, dialect) + + return process + else: + return self.impl.literal_processor(dialect) + + def bind_processor(self, dialect): + """Provide a bound value processing function for the + given :class:`.Dialect`. + + This is the method that fulfills the :class:`.TypeEngine` + contract for bound value conversion. :class:`.TypeDecorator` + will wrap a user-defined implementation of + :meth:`process_bind_param` here. + + User-defined code can override this method directly, + though its likely best to use :meth:`process_bind_param` so that + the processing provided by ``self.impl`` is maintained. + + :param dialect: Dialect instance in use. + + This method is the reverse counterpart to the + :meth:`result_processor` method of this class. + + """ + if self._has_bind_processor: + process_param = self.process_bind_param + impl_processor = self.impl.bind_processor(dialect) + if impl_processor: + def process(value): + return impl_processor(process_param(value, dialect)) + + else: + def process(value): + return process_param(value, dialect) + + return process + else: + return self.impl.bind_processor(dialect) + + @util.memoized_property + def _has_result_processor(self): + """memoized boolean, check if process_result_value is implemented. + + Allows the base process_result_value to raise + NotImplementedError without needing to test an expensive + exception throw. + + """ + return self.__class__.process_result_value.__code__ \ + is not TypeDecorator.process_result_value.__code__ + + def result_processor(self, dialect, coltype): + """Provide a result value processing function for the given + :class:`.Dialect`. + + This is the method that fulfills the :class:`.TypeEngine` + contract for result value conversion. :class:`.TypeDecorator` + will wrap a user-defined implementation of + :meth:`process_result_value` here. + + User-defined code can override this method directly, + though its likely best to use :meth:`process_result_value` so that + the processing provided by ``self.impl`` is maintained. + + :param dialect: Dialect instance in use. + :param coltype: An SQLAlchemy data type + + This method is the reverse counterpart to the + :meth:`bind_processor` method of this class. + + """ + if self._has_result_processor: + process_value = self.process_result_value + impl_processor = self.impl.result_processor(dialect, + coltype) + if impl_processor: + def process(value): + return process_value(impl_processor(value), dialect) + + else: + def process(value): + return process_value(value, dialect) + + return process + else: + return self.impl.result_processor(dialect, coltype) + + def coerce_compared_value(self, op, value): + """Suggest a type for a 'coerced' Python value in an expression. + + By default, returns self. This method is called by + the expression system when an object using this type is + on the left or right side of an expression against a plain Python + object which does not yet have a SQLAlchemy type assigned:: + + expr = table.c.somecolumn + 35 + + Where above, if ``somecolumn`` uses this type, this method will + be called with the value ``operator.add`` + and ``35``. The return value is whatever SQLAlchemy type should + be used for ``35`` for this particular operation. + + """ + return self + + def copy(self): + """Produce a copy of this :class:`.TypeDecorator` instance. + + This is a shallow copy and is provided to fulfill part of + the :class:`.TypeEngine` contract. It usually does not + need to be overridden unless the user-defined :class:`.TypeDecorator` + has local state that should be deep-copied. + + """ + + instance = self.__class__.__new__(self.__class__) + instance.__dict__.update(self.__dict__) + return instance + + def get_dbapi_type(self, dbapi): + """Return the DBAPI type object represented by this + :class:`.TypeDecorator`. + + By default this calls upon :meth:`.TypeEngine.get_dbapi_type` of the + underlying "impl". + """ + return self.impl.get_dbapi_type(dbapi) + + def compare_values(self, x, y): + """Given two values, compare them for equality. + + By default this calls upon :meth:`.TypeEngine.compare_values` + of the underlying "impl", which in turn usually + uses the Python equals operator ``==``. + + This function is used by the ORM to compare + an original-loaded value with an intercepted + "changed" value, to determine if a net change + has occurred. + + """ + return self.impl.compare_values(x, y) + + def __repr__(self): + return util.generic_repr(self, to_inspect=self.impl) + + +class Variant(TypeDecorator): + """A wrapping type that selects among a variety of + implementations based on dialect in use. + + The :class:`.Variant` type is typically constructed + using the :meth:`.TypeEngine.with_variant` method. + + .. versionadded:: 0.7.2 + + .. seealso:: :meth:`.TypeEngine.with_variant` for an example of use. + + """ + + def __init__(self, base, mapping): + """Construct a new :class:`.Variant`. + + :param base: the base 'fallback' type + :param mapping: dictionary of string dialect names to + :class:`.TypeEngine` instances. + + """ + self.impl = base + self.mapping = mapping + + def load_dialect_impl(self, dialect): + if dialect.name in self.mapping: + return self.mapping[dialect.name] + else: + return self.impl + + def with_variant(self, type_, dialect_name): + """Return a new :class:`.Variant` which adds the given + type + dialect name to the mapping, in addition to the + mapping present in this :class:`.Variant`. + + :param type_: a :class:`.TypeEngine` that will be selected + as a variant from the originating type, when a dialect + of the given name is in use. + :param dialect_name: base name of the dialect which uses + this type. (i.e. ``'postgresql'``, ``'mysql'``, etc.) + + """ + + if dialect_name in self.mapping: + raise exc.ArgumentError( + "Dialect '%s' is already present in " + "the mapping for this Variant" % dialect_name) + mapping = self.mapping.copy() + mapping[dialect_name] = type_ + return Variant(self.impl, mapping) + +def _reconstitute_comparator(expression): + return expression.comparator + + +def to_instance(typeobj, *arg, **kw): + if typeobj is None: + return NULLTYPE + + if util.callable(typeobj): + return typeobj(*arg, **kw) + else: + return typeobj + + +def adapt_type(typeobj, colspecs): + if isinstance(typeobj, type): + typeobj = typeobj() + for t in typeobj.__class__.__mro__[0:-1]: + try: + impltype = colspecs[t] + break + except KeyError: + pass + else: + # couldnt adapt - so just return the type itself + # (it may be a user-defined type) + return typeobj + # if we adapted the given generic type to a database-specific type, + # but it turns out the originally given "generic" type + # is actually a subclass of our resulting type, then we were already + # given a more specific type than that required; so use that. + if (issubclass(typeobj.__class__, impltype)): + return typeobj + return typeobj.adapt(impltype) + + diff --git a/libs/sqlalchemy/sql/util.py b/libs/sqlalchemy/sql/util.py index 0a00674c..50ce30aa 100644 --- a/libs/sqlalchemy/sql/util.py +++ b/libs/sqlalchemy/sql/util.py @@ -1,41 +1,33 @@ # sql/util.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from sqlalchemy import exc, schema, util, sql, types as sqltypes -from sqlalchemy.util import topological -from sqlalchemy.sql import expression, operators, visitors +"""High level utilities which build upon other modules here. + +""" + +from .. import exc, util +from .base import _from_objects, ColumnSet +from . import operators, visitors from itertools import chain from collections import deque -"""Utility functions that build upon SQL and Schema constructs.""" +from .elements import BindParameter, ColumnClause, ColumnElement, \ + Null, UnaryExpression, literal_column, Label +from .selectable import ScalarSelect, Join, FromClause, FromGrouping +from .schema import Column -def sort_tables(tables): - """sort a collection of Table objects in order of their foreign-key dependency.""" +join_condition = util.langhelpers.public_factory( + Join._join_condition, + ".sql.util.join_condition") - tables = list(tables) - tuples = [] - def visit_foreign_key(fkey): - if fkey.use_alter: - return - parent_table = fkey.column.table - if parent_table in tables: - child_table = fkey.parent.table - if parent_table is not child_table: - tuples.append((parent_table, child_table)) +# names that are still being imported from the outside +from .annotation import _shallow_annotate, _deep_annotate, _deep_deannotate +from .elements import _find_columns +from .ddl import sort_tables - for table in tables: - visitors.traverse(table, - {'schema_visitor':True}, - {'foreign_key':visit_foreign_key}) - - tuples.extend( - [parent, table] for parent in table._extra_dependencies - ) - - return list(topological.sort(tuples, tables)) def find_join_source(clauses, join_to): """Given a list of FROM clauses and a selectable, @@ -54,7 +46,7 @@ def find_join_source(clauses, join_to): """ - selectables = list(expression._from_objects(join_to)) + selectables = list(_from_objects(join_to)) for i, f in enumerate(clauses): for s in selectables: if f.is_derived_from(s): @@ -62,6 +54,67 @@ def find_join_source(clauses, join_to): else: return None, None + +def visit_binary_product(fn, expr): + """Produce a traversal of the given expression, delivering + column comparisons to the given function. + + The function is of the form:: + + def my_fn(binary, left, right) + + For each binary expression located which has a + comparison operator, the product of "left" and + "right" will be delivered to that function, + in terms of that binary. + + Hence an expression like:: + + and_( + (a + b) == q + func.sum(e + f), + j == r + ) + + would have the traversal:: + + a q + a e + a f + b q + b e + b f + j r + + That is, every combination of "left" and + "right" that doesn't further contain + a binary comparison is passed as pairs. + + """ + stack = [] + + def visit(element): + if isinstance(element, ScalarSelect): + # we dont want to dig into correlated subqueries, + # those are just column elements by themselves + yield element + elif element.__visit_name__ == 'binary' and \ + operators.is_comparison(element.operator): + stack.insert(0, element) + for l in visit(element.left): + for r in visit(element.right): + fn(stack[0], l, r) + stack.pop(0) + for elem in element.get_children(): + visit(elem) + else: + if isinstance(element, ColumnClause): + yield element + for elem in element.get_children(): + for e in visit(elem): + yield e + list(visit(expr)) + + def find_tables(clause, check_columns=False, include_aliases=False, include_joins=False, include_selects=False, include_crud=False): @@ -77,7 +130,7 @@ def find_tables(clause, check_columns=False, _visitors['join'] = tables.append if include_aliases: - _visitors['alias'] = tables.append + _visitors['alias'] = tables.append if include_crud: _visitors['insert'] = _visitors['update'] = \ @@ -90,15 +143,10 @@ def find_tables(clause, check_columns=False, _visitors['table'] = tables.append - visitors.traverse(clause, {'column_collections':False}, _visitors) + visitors.traverse(clause, {'column_collections': False}, _visitors) return tables -def find_columns(clause): - """locate Column objects within the given expression.""" - cols = util.column_set() - visitors.traverse(clause, {}, {'column':cols.add}) - return cols def unwrap_order_by(clause): """Break up an 'order by' expression into individual column-expressions, @@ -108,9 +156,9 @@ def unwrap_order_by(clause): stack = deque([clause]) while stack: t = stack.popleft() - if isinstance(t, expression.ColumnElement) and \ + if isinstance(t, ColumnElement) and \ ( - not isinstance(t, expression._UnaryExpression) or \ + not isinstance(t, UnaryExpression) or \ not operators.is_ordering_modifier(t.modifier) ): cols.add(t) @@ -119,6 +167,7 @@ def unwrap_order_by(clause): stack.append(c) return cols + def clause_is_present(clause, search): """Given a target clause and a second to search within, return True if the target is plainly present in the search without any @@ -128,15 +177,30 @@ def clause_is_present(clause, search): """ - stack = [search] + for elem in surface_selectables(search): + if clause == elem: # use == here so that Annotated's compare + return True + else: + return False + +def surface_selectables(clause): + stack = [clause] while stack: elem = stack.pop() - if clause is elem: - return True - elif isinstance(elem, expression.Join): + yield elem + if isinstance(elem, Join): stack.extend((elem.left, elem.right)) - return False + elif isinstance(elem, FromGrouping): + stack.append(elem.element) +def selectables_overlap(left, right): + """Return True if left/right have some overlapping selectable""" + + return bool( + set(surface_selectables(left)).intersection( + surface_selectables(right) + ) + ) def bind_values(clause): """Return an ordered list of "bound" values in the given clause. @@ -151,19 +215,22 @@ def bind_values(clause): """ v = [] + def visit_bindparam(bind): v.append(bind.effective_value) - visitors.traverse(clause, {}, {'bindparam':visit_bindparam}) + visitors.traverse(clause, {}, {'bindparam': visit_bindparam}) return v + def _quote_ddl_expr(element): - if isinstance(element, basestring): + if isinstance(element, util.string_types): element = element.replace("'", "''") return "'%s'" % element else: return repr(element) + class _repr_params(object): """A string view of bound parameters, truncating display to the given number of 'multi' parameter sets. @@ -177,289 +244,40 @@ class _repr_params(object): if isinstance(self.params, (list, tuple)) and \ len(self.params) > self.batches and \ isinstance(self.params[0], (list, dict, tuple)): + msg = " ... displaying %i of %i total bound parameter sets ... " return ' '.join(( repr(self.params[:self.batches - 2])[0:-1], - " ... displaying %i of %i total bound parameter sets ... " % (self.batches, len(self.params)), + msg % (self.batches, len(self.params)), repr(self.params[-2:])[1:] )) else: return repr(self.params) -def expression_as_ddl(clause): - """Given a SQL expression, convert for usage in DDL, such as - CREATE INDEX and CHECK CONSTRAINT. - Converts bind params into quoted literals, column identifiers - into detached column constructs so that the parent table - identifier is not included. - - """ - def repl(element): - if isinstance(element, expression._BindParamClause): - return expression.literal_column(_quote_ddl_expr(element.value)) - elif isinstance(element, expression.ColumnClause) and \ - element.table is not None: - return expression.column(element.name) - else: - return None - - return visitors.replacement_traverse(clause, {}, repl) def adapt_criterion_to_null(crit, nulls): - """given criterion containing bind params, convert selected elements to IS NULL.""" + """given criterion containing bind params, convert selected elements + to IS NULL. + + """ def visit_binary(binary): - if isinstance(binary.left, expression._BindParamClause) \ + if isinstance(binary.left, BindParameter) \ and binary.left._identifying_key in nulls: # reverse order if the NULL is on the left side binary.left = binary.right - binary.right = expression.null() + binary.right = Null() binary.operator = operators.is_ binary.negate = operators.isnot - elif isinstance(binary.right, expression._BindParamClause) \ + elif isinstance(binary.right, BindParameter) \ and binary.right._identifying_key in nulls: - binary.right = expression.null() + binary.right = Null() binary.operator = operators.is_ binary.negate = operators.isnot - return visitors.cloned_traverse(crit, {}, {'binary':visit_binary}) + return visitors.cloned_traverse(crit, {}, {'binary': visit_binary}) -def join_condition(a, b, ignore_nonexistent_tables=False, a_subset=None): - """create a join condition between two tables or selectables. - - e.g.:: - - join_condition(tablea, tableb) - - would produce an expression along the lines of:: - - tablea.c.id==tableb.c.tablea_id - - The join is determined based on the foreign key relationships - between the two selectables. If there are multiple ways - to join, or no way to join, an error is raised. - - :param ignore_nonexistent_tables: Deprecated - this - flag is no longer used. Only resolution errors regarding - the two given tables are propagated. - - :param a_subset: An optional expression that is a sub-component - of ``a``. An attempt will be made to join to just this sub-component - first before looking at the full ``a`` construct, and if found - will be successful even if there are other ways to join to ``a``. - This allows the "right side" of a join to be passed thereby - providing a "natural join". - - """ - crit = [] - constraints = set() - - for left in (a_subset, a): - if left is None: - continue - for fk in sorted( - b.foreign_keys, - key=lambda fk:fk.parent._creation_order): - try: - col = fk.get_referent(left) - except exc.NoReferenceError, nrte: - if nrte.table_name == left.name: - raise - else: - continue - - if col is not None: - crit.append(col == fk.parent) - constraints.add(fk.constraint) - if left is not b: - for fk in sorted( - left.foreign_keys, - key=lambda fk:fk.parent._creation_order): - try: - col = fk.get_referent(b) - except exc.NoReferenceError, nrte: - if nrte.table_name == b.name: - raise - else: - # this is totally covered. can't get - # coverage to mark it. - continue - - if col is not None: - crit.append(col == fk.parent) - constraints.add(fk.constraint) - if crit: - break - - if len(crit) == 0: - if isinstance(b, expression._FromGrouping): - hint = " Perhaps you meant to convert the right side to a "\ - "subquery using alias()?" - else: - hint = "" - raise exc.ArgumentError( - "Can't find any foreign key relationships " - "between '%s' and '%s'.%s" % (a.description, b.description, hint)) - elif len(constraints) > 1: - raise exc.ArgumentError( - "Can't determine join between '%s' and '%s'; " - "tables have more than one foreign key " - "constraint relationship between them. " - "Please specify the 'onclause' of this " - "join explicitly." % (a.description, b.description)) - elif len(crit) == 1: - return (crit[0]) - else: - return sql.and_(*crit) - - -class Annotated(object): - """clones a ClauseElement and applies an 'annotations' dictionary. - - Unlike regular clones, this clone also mimics __hash__() and - __cmp__() of the original element so that it takes its place - in hashed collections. - - A reference to the original element is maintained, for the important - reason of keeping its hash value current. When GC'ed, the - hash value may be reused, causing conflicts. - - """ - - def __new__(cls, *args): - if not args: - # clone constructor - return object.__new__(cls) - else: - element, values = args - # pull appropriate subclass from registry of annotated - # classes - try: - cls = annotated_classes[element.__class__] - except KeyError: - cls = annotated_classes[element.__class__] = type.__new__(type, - "Annotated%s" % element.__class__.__name__, - (Annotated, element.__class__), {}) - return object.__new__(cls) - - def __init__(self, element, values): - # force FromClause to generate their internal - # collections into __dict__ - if isinstance(element, expression.FromClause): - element.c - - self.__dict__ = element.__dict__.copy() - self.__element = element - self._annotations = values - - def _annotate(self, values): - _values = self._annotations.copy() - _values.update(values) - clone = self.__class__.__new__(self.__class__) - clone.__dict__ = self.__dict__.copy() - clone._annotations = _values - return clone - - def _deannotate(self): - return self.__element - - def _compiler_dispatch(self, visitor, **kw): - return self.__element.__class__._compiler_dispatch(self, visitor, **kw) - - @property - def _constructor(self): - return self.__element._constructor - - def _clone(self): - clone = self.__element._clone() - if clone is self.__element: - # detect immutable, don't change anything - return self - else: - # update the clone with any changes that have occurred - # to this object's __dict__. - clone.__dict__.update(self.__dict__) - return Annotated(clone, self._annotations) - - def __hash__(self): - return hash(self.__element) - - def __eq__(self, other): - if isinstance(self.__element, expression.ColumnOperators): - return self.__element.__class__.__eq__(self, other) - else: - return hash(other) == hash(self) - - -# hard-generate Annotated subclasses. this technique -# is used instead of on-the-fly types (i.e. type.__new__()) -# so that the resulting objects are pickleable. -annotated_classes = {} - -for cls in expression.__dict__.values() + [schema.Column, schema.Table]: - if isinstance(cls, type) and issubclass(cls, expression.ClauseElement): - exec "class Annotated%s(Annotated, cls):\n" \ - " pass" % (cls.__name__, ) in locals() - exec "annotated_classes[cls] = Annotated%s" % (cls.__name__) - -def _deep_annotate(element, annotations, exclude=None): - """Deep copy the given ClauseElement, annotating each element - with the given annotations dictionary. - - Elements within the exclude collection will be cloned but not annotated. - - """ - cloned = util.column_dict() - - def clone(elem): - # check if element is present in the exclude list. - # take into account proxying relationships. - if elem in cloned: - return cloned[elem] - elif exclude and \ - hasattr(elem, 'proxy_set') and \ - elem.proxy_set.intersection(exclude): - newelem = elem._clone() - elif annotations != elem._annotations: - newelem = elem._annotate(annotations) - else: - newelem = elem - newelem._copy_internals(clone=clone) - cloned[elem] = newelem - return newelem - - if element is not None: - element = clone(element) - return element - -def _deep_deannotate(element): - """Deep copy the given element, removing all annotations.""" - - cloned = util.column_dict() - - def clone(elem): - if elem not in cloned: - newelem = elem._deannotate() - newelem._copy_internals(clone=clone) - cloned[elem] = newelem - return cloned[elem] - - if element is not None: - element = clone(element) - return element - -def _shallow_annotate(element, annotations): - """Annotate the given ClauseElement and copy its internals so that - internal objects refer to the new annotated object. - - Basically used to apply a "dont traverse" annotation to a - selectable, without digging throughout the whole - structure wasting time. - """ - element = element._annotate(annotations) - element._copy_internals() - return element def splice_joins(left, right, stop_on=None): if left is None: @@ -471,7 +289,7 @@ def splice_joins(left, right, stop_on=None): ret = None while stack: (right, prevright) = stack.pop() - if isinstance(right, expression.Join) and right is not stop_on: + if isinstance(right, Join) and right is not stop_on: right = right._clone() right._reset_exported() right.onclause = adapter.traverse(right.onclause) @@ -485,18 +303,21 @@ def splice_joins(left, right, stop_on=None): return ret + def reduce_columns(columns, *clauses, **kw): - """given a list of columns, return a 'reduced' set based on natural equivalents. + """given a list of columns, return a 'reduced' set based on natural + equivalents. the set is reduced to the smallest list of columns which have no natural - equivalent present in the list. A "natural equivalent" means that two columns - will ultimately represent the same value because they are related by a foreign key. + equivalent present in the list. A "natural equivalent" means that two + columns will ultimately represent the same value because they are related + by a foreign key. \*clauses is an optional list of join clauses which will be traversed to further identify columns that are "equivalent". \**kw may specify 'ignore_nonexistent_tables' to ignore foreign keys - whose tables are not yet configured. + whose tables are not yet configured, or columns that aren't yet present. This function is primarily used to determine the most minimal "primary key" from a selectable, by reducing the set of primary key columns present @@ -504,6 +325,7 @@ def reduce_columns(columns, *clauses, **kw): """ ignore_nonexistent_tables = kw.pop('ignore_nonexistent_tables', False) + only_synonyms = kw.pop('only_synonyms', False) columns = util.ordered_column_set(columns) @@ -515,28 +337,44 @@ def reduce_columns(columns, *clauses, **kw): continue try: fk_col = fk.column - except exc.NoReferencedTableError: + except exc.NoReferencedColumnError: + # TODO: add specific coverage here + # to test/sql/test_selectable ReduceTest if ignore_nonexistent_tables: continue else: raise - if fk_col.shares_lineage(c): + except exc.NoReferencedTableError: + # TODO: add specific coverage here + # to test/sql/test_selectable ReduceTest + if ignore_nonexistent_tables: + continue + else: + raise + if fk_col.shares_lineage(c) and \ + (not only_synonyms or \ + c.name == col.name): omit.add(col) break if clauses: def visit_binary(binary): if binary.operator == operators.eq: - cols = util.column_set(chain(*[c.proxy_set for c in columns.difference(omit)])) + cols = util.column_set(chain(*[c.proxy_set + for c in columns.difference(omit)])) if binary.left in cols and binary.right in cols: - for c in columns: - if c.shares_lineage(binary.right): + for c in reversed(columns): + if c.shares_lineage(binary.right) and \ + (not only_synonyms or \ + c.name == binary.left.name): omit.add(c) break for clause in clauses: - visitors.traverse(clause, {}, {'binary':visit_binary}) + if clause is not None: + visitors.traverse(clause, {}, {'binary': visit_binary}) + + return ColumnSet(columns.difference(omit)) - return expression.ColumnSet(columns.difference(omit)) def criterion_as_pairs(expression, consider_as_foreign_keys=None, consider_as_referenced_keys=None, any_operator=False): @@ -547,84 +385,47 @@ def criterion_as_pairs(expression, consider_as_foreign_keys=None, "'consider_as_foreign_keys' or " "'consider_as_referenced_keys'") + def col_is(a, b): + #return a is b + return a.compare(b) + def visit_binary(binary): if not any_operator and binary.operator is not operators.eq: return - if not isinstance(binary.left, sql.ColumnElement) or \ - not isinstance(binary.right, sql.ColumnElement): + if not isinstance(binary.left, ColumnElement) or \ + not isinstance(binary.right, ColumnElement): return if consider_as_foreign_keys: if binary.left in consider_as_foreign_keys and \ - (binary.right is binary.left or + (col_is(binary.right, binary.left) or binary.right not in consider_as_foreign_keys): pairs.append((binary.right, binary.left)) elif binary.right in consider_as_foreign_keys and \ - (binary.left is binary.right or + (col_is(binary.left, binary.right) or binary.left not in consider_as_foreign_keys): pairs.append((binary.left, binary.right)) elif consider_as_referenced_keys: if binary.left in consider_as_referenced_keys and \ - (binary.right is binary.left or + (col_is(binary.right, binary.left) or binary.right not in consider_as_referenced_keys): pairs.append((binary.left, binary.right)) elif binary.right in consider_as_referenced_keys and \ - (binary.left is binary.right or + (col_is(binary.left, binary.right) or binary.left not in consider_as_referenced_keys): pairs.append((binary.right, binary.left)) else: - if isinstance(binary.left, schema.Column) and \ - isinstance(binary.right, schema.Column): + if isinstance(binary.left, Column) and \ + isinstance(binary.right, Column): if binary.left.references(binary.right): pairs.append((binary.right, binary.left)) elif binary.right.references(binary.left): pairs.append((binary.left, binary.right)) pairs = [] - visitors.traverse(expression, {}, {'binary':visit_binary}) + visitors.traverse(expression, {}, {'binary': visit_binary}) return pairs -def folded_equivalents(join, equivs=None): - """Return a list of uniquely named columns. - The column list of the given Join will be narrowed - down to a list of all equivalently-named, - equated columns folded into one column, where 'equated' means they are - equated to each other in the ON clause of this join. - - This function is used by Join.select(fold_equivalents=True). - - Deprecated. This function is used for a certain kind of - "polymorphic_union" which is designed to achieve joined - table inheritance where the base table has no "discriminator" - column; [ticket:1131] will provide a better way to - achieve this. - - """ - if equivs is None: - equivs = set() - def visit_binary(binary): - if binary.operator == operators.eq and binary.left.name == binary.right.name: - equivs.add(binary.right) - equivs.add(binary.left) - visitors.traverse(join.onclause, {}, {'binary':visit_binary}) - collist = [] - if isinstance(join.left, expression.Join): - left = folded_equivalents(join.left, equivs) - else: - left = list(join.left.columns) - if isinstance(join.right, expression.Join): - right = folded_equivalents(join.right, equivs) - else: - right = list(join.right.columns) - used = set() - for c in left + right: - if c in equivs: - if c.name not in used: - collist.append(c) - used.add(c.name) - else: - collist.append(c) - return collist class AliasedRow(object): """Wrap a RowProxy with a translation map. @@ -681,15 +482,27 @@ class ClauseAdapter(visitors.ReplacingCloningVisitor): s.c.col1 == table2.c.col1 """ - def __init__(self, selectable, equivalents=None, include=None, exclude=None, adapt_on_names=False): - self.__traverse_options__ = {'stop_on':[selectable]} + def __init__(self, selectable, equivalents=None, + include=None, exclude=None, + include_fn=None, exclude_fn=None, + adapt_on_names=False): + self.__traverse_options__ = {'stop_on': [selectable]} self.selectable = selectable - self.include = include - self.exclude = exclude + if include: + assert not include_fn + self.include_fn = lambda e: e in include + else: + self.include_fn = include_fn + if exclude: + assert not exclude_fn + self.exclude_fn = lambda e: e in exclude + else: + self.exclude_fn = exclude_fn self.equivalents = util.column_dict(equivalents or {}) self.adapt_on_names = adapt_on_names - def _corresponding_column(self, col, require_embedded, _seen=util.EMPTY_SET): + def _corresponding_column(self, col, require_embedded, + _seen=util.EMPTY_SET): newcol = self.selectable.corresponding_column( col, require_embedded=require_embedded) @@ -704,20 +517,20 @@ class ClauseAdapter(visitors.ReplacingCloningVisitor): newcol = self.selectable.c.get(col.name) return newcol + magic_flag = False def replace(self, col): - if isinstance(col, expression.FromClause): - if self.selectable.is_derived_from(col): - return self.selectable - - if not isinstance(col, expression.ColumnElement): + if not self.magic_flag and isinstance(col, FromClause) and \ + self.selectable.is_derived_from(col): + return self.selectable + elif not isinstance(col, ColumnElement): return None - - if self.include and col not in self.include: + elif self.include_fn and not self.include_fn(col): return None - elif self.exclude and col in self.exclude: + elif self.exclude_fn and self.exclude_fn(col): return None + else: + return self._corresponding_column(col, True) - return self._corresponding_column(col, True) class ColumnAdapter(ClauseAdapter): """Extends ClauseAdapter with extra utility functions. @@ -761,15 +574,14 @@ class ColumnAdapter(ClauseAdapter): c = self.adapt_clause(col) # anonymize labels in case they have a hardcoded name - if isinstance(c, expression._Label): + if isinstance(c, Label): c = c.label(None) - # adapt_required indicates that if we got the same column - # back which we put in (i.e. it passed through), - # it's not correct. this is used by eagerloading which - # knows that all columns and expressions need to be adapted - # to a result row, and a "passthrough" is definitely targeting - # the wrong column. + # adapt_required used by eager loading to indicate that + # we don't trust a result row column that is not translated. + # this is to prevent a column from being interpreted as that + # of the child row in a self-referential scenario, see + # inheritance/test_basic.py->EagerTargetingTest.test_adapt_stringency if self.adapt_required and c is col: return None @@ -786,3 +598,4 @@ class ColumnAdapter(ClauseAdapter): def __setstate__(self, state): self.__dict__.update(state) self.columns = util.PopulateDict(self._locate_col) + diff --git a/libs/sqlalchemy/sql/visitors.py b/libs/sqlalchemy/sql/visitors.py index d236063d..d9ad04fc 100644 --- a/libs/sqlalchemy/sql/visitors.py +++ b/libs/sqlalchemy/sql/visitors.py @@ -1,5 +1,5 @@ # sql/visitors.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -24,15 +24,17 @@ http://techspot.zzzeek.org/2008/01/23/expression-transformations/ """ from collections import deque -import re -from sqlalchemy import util +from .. import util import operator +from .. import exc __all__ = ['VisitableType', 'Visitable', 'ClauseVisitor', 'CloningVisitor', 'ReplacingCloningVisitor', 'iterate', 'iterate_depthfirst', 'traverse_using', 'traverse', + 'traverse_depthfirst', 'cloned_traverse', 'replacement_traverse'] + class VisitableType(type): """Metaclass which assigns a `_compiler_dispatch` method to classes having a `__visit_name__` attribute. @@ -43,16 +45,15 @@ class VisitableType(type): def _compiler_dispatch (self, visitor, **kw): '''Look for an attribute named "visit_" + self.__visit_name__ on the visitor, and call it with the same kw params.''' - return getattr(visitor, 'visit_%s' % self.__visit_name__)(self, **kw) + visit_attr = 'visit_%s' % self.__visit_name__ + return getattr(visitor, visit_attr)(self, **kw) Classes having no __visit_name__ attribute will remain unaffected. """ def __init__(cls, clsname, bases, clsdict): - if cls.__name__ == 'Visitable' or not hasattr(cls, '__visit_name__'): - super(VisitableType, cls).__init__(clsname, bases, clsdict) - return - - _generate_dispatch(cls) + if clsname != 'Visitable' and \ + hasattr(cls, '__visit_name__'): + _generate_dispatch(cls) super(VisitableType, cls).__init__(clsname, bases, clsdict) @@ -68,28 +69,40 @@ def _generate_dispatch(cls): # the string name of the class's __visit_name__ is known at # this early stage (import time) so it can be pre-constructed. getter = operator.attrgetter("visit_%s" % visit_name) + def _compiler_dispatch(self, visitor, **kw): - return getter(visitor)(self, **kw) + try: + meth = getter(visitor) + except AttributeError: + raise exc.UnsupportedCompilationError(visitor, cls) + else: + return meth(self, **kw) else: # The optimization opportunity is lost for this case because the # __visit_name__ is not yet a string. As a result, the visit # string has to be recalculated with each compilation. def _compiler_dispatch(self, visitor, **kw): - return getattr(visitor, 'visit_%s' % self.__visit_name__)(self, **kw) + visit_attr = 'visit_%s' % self.__visit_name__ + try: + meth = getattr(visitor, visit_attr) + except AttributeError: + raise exc.UnsupportedCompilationError(visitor, cls) + else: + return meth(self, **kw) - _compiler_dispatch.__doc__ = \ + _compiler_dispatch.__doc__ = \ """Look for an attribute named "visit_" + self.__visit_name__ on the visitor, and call it with the same kw params. """ cls._compiler_dispatch = _compiler_dispatch -class Visitable(object): + +class Visitable(util.with_metaclass(VisitableType, object)): """Base class for visitable objects, applies the ``VisitableType`` metaclass. """ - __metaclass__ = VisitableType class ClauseVisitor(object): """Base class for visitor objects which can traverse using @@ -106,8 +119,10 @@ class ClauseVisitor(object): return meth(obj, **kw) def iterate(self, obj): - """traverse the given expression structure, returning an iterator of all elements.""" + """traverse the given expression structure, returning an iterator + of all elements. + """ return iterate(obj, self.__traverse_options__) def traverse(self, obj): @@ -143,6 +158,7 @@ class ClauseVisitor(object): tail._next = visitor return self + class CloningVisitor(ClauseVisitor): """Base class for visitor objects which can traverse using the cloned_traverse() function. @@ -150,14 +166,18 @@ class CloningVisitor(ClauseVisitor): """ def copy_and_process(self, list_): - """Apply cloned traversal to the given list of elements, and return the new list.""" + """Apply cloned traversal to the given list of elements, and return + the new list. + """ return [self.traverse(x) for x in list_] def traverse(self, obj): """traverse and visit the given expression structure.""" - return cloned_traverse(obj, self.__traverse_options__, self._visitor_dict) + return cloned_traverse( + obj, self.__traverse_options__, self._visitor_dict) + class ReplacingCloningVisitor(CloningVisitor): """Base class for visitor objects which can traverse using @@ -184,6 +204,7 @@ class ReplacingCloningVisitor(CloningVisitor): return e return replacement_traverse(obj, self.__traverse_options__, replace) + def iterate(obj, opts): """traverse the given expression structure, returning an iterator. @@ -197,6 +218,7 @@ def iterate(obj, opts): for c in t.get_children(**opts): stack.append(c) + def iterate_depthfirst(obj, opts): """traverse the given expression structure, returning an iterator. @@ -212,43 +234,53 @@ def iterate_depthfirst(obj, opts): stack.append(c) return iter(traversal) -def traverse_using(iterator, obj, visitors): - """visit the given expression structure using the given iterator of objects.""" +def traverse_using(iterator, obj, visitors): + """visit the given expression structure using the given iterator of + objects. + + """ for target in iterator: meth = visitors.get(target.__visit_name__, None) if meth: meth(target) return obj -def traverse(obj, opts, visitors): - """traverse and visit the given expression structure using the default iterator.""" +def traverse(obj, opts, visitors): + """traverse and visit the given expression structure using the default + iterator. + + """ return traverse_using(iterate(obj, opts), obj, visitors) -def traverse_depthfirst(obj, opts, visitors): - """traverse and visit the given expression structure using the depth-first iterator.""" +def traverse_depthfirst(obj, opts, visitors): + """traverse and visit the given expression structure using the + depth-first iterator. + + """ return traverse_using(iterate_depthfirst(obj, opts), obj, visitors) + def cloned_traverse(obj, opts, visitors): """clone the given expression structure, allowing modifications by visitors.""" - cloned = util.column_dict() - stop_on = util.column_set(opts.get('stop_on', [])) + cloned = {} + stop_on = set(opts.get('stop_on', [])) def clone(elem): if elem in stop_on: return elem else: - if elem not in cloned: - cloned[elem] = newelem = elem._clone() + if id(elem) not in cloned: + cloned[id(elem)] = newelem = elem._clone() newelem._copy_internals(clone=clone) meth = visitors.get(newelem.__visit_name__, None) if meth: meth(newelem) - return cloned[elem] + return cloned[id(elem)] if obj is not None: obj = clone(obj) @@ -259,17 +291,17 @@ def replacement_traverse(obj, opts, replace): """clone the given expression structure, allowing element replacement by a given replacement function.""" - cloned = util.column_dict() - stop_on = util.column_set(opts.get('stop_on', [])) + cloned = {} + stop_on = set([id(x) for x in opts.get('stop_on', [])]) def clone(elem, **kw): - if elem in stop_on or \ + if id(elem) in stop_on or \ 'no_replacement_traverse' in elem._annotations: return elem else: newelem = replace(elem) if newelem is not None: - stop_on.add(newelem) + stop_on.add(id(newelem)) return newelem else: if elem not in cloned: diff --git a/libs/sqlalchemy/types.py b/libs/sqlalchemy/types.py index 5fe2ba20..3994bd4a 100644 --- a/libs/sqlalchemy/types.py +++ b/libs/sqlalchemy/types.py @@ -1,2406 +1,77 @@ -# sqlalchemy/types.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# types.py +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -"""defines genericized SQL types, each represented by a subclass of -:class:`~sqlalchemy.types.AbstractType`. Dialects define further subclasses of these -types. - -For more information see the SQLAlchemy documentation on types. +"""Compatiblity namespace for sqlalchemy.sql.types. """ -__all__ = [ 'TypeEngine', 'TypeDecorator', 'AbstractType', 'UserDefinedType', - 'INT', 'CHAR', 'VARCHAR', 'NCHAR', 'NVARCHAR','TEXT', 'Text', + +__all__ = ['TypeEngine', 'TypeDecorator', 'UserDefinedType', + 'INT', 'CHAR', 'VARCHAR', 'NCHAR', 'NVARCHAR', 'TEXT', 'Text', 'FLOAT', 'NUMERIC', 'REAL', 'DECIMAL', 'TIMESTAMP', 'DATETIME', - 'CLOB', 'BLOB', 'BINARY', 'VARBINARY', 'BOOLEAN', 'BIGINT', 'SMALLINT', - 'INTEGER', 'DATE', 'TIME', 'String', 'Integer', 'SmallInteger', - 'BigInteger', 'Numeric', 'Float', 'DateTime', 'Date', 'Time', - 'LargeBinary', 'Binary', 'Boolean', 'Unicode', 'Concatenable', - 'UnicodeText','PickleType', 'Interval', 'Enum', 'MutableType' ] - -import inspect -import datetime as dt -import codecs - -from sqlalchemy import exc, schema -from sqlalchemy.sql import expression, operators -from sqlalchemy.util import pickle -from sqlalchemy.util.compat import decimal -from sqlalchemy.sql.visitors import Visitable -from sqlalchemy import util -from sqlalchemy import processors, events, event -import collections -default = util.importlater("sqlalchemy.engine", "default") - -NoneType = type(None) -if util.jython: - import array - -class AbstractType(Visitable): - """Base for all types - not needed except for backwards - compatibility.""" - -class TypeEngine(AbstractType): - """Base for built-in types.""" - - def copy_value(self, value): - return value - - def bind_processor(self, dialect): - """Return a conversion function for processing bind values. - - Returns a callable which will receive a bind parameter value - as the sole positional argument and will return a value to - send to the DB-API. - - If processing is not necessary, the method should return ``None``. - - :param dialect: Dialect instance in use. - - """ - return None - - def result_processor(self, dialect, coltype): - """Return a conversion function for processing result row values. - - Returns a callable which will receive a result row column - value as the sole positional argument and will return a value - to return to the user. - - If processing is not necessary, the method should return ``None``. - - :param dialect: Dialect instance in use. - - :param coltype: DBAPI coltype argument received in cursor.description. - - """ - return None - - def compare_values(self, x, y): - """Compare two values for equality.""" - - return x == y - - def is_mutable(self): - """Return True if the target Python type is 'mutable'. - - This allows systems like the ORM to know if a column value can - be considered 'not changed' by comparing the identity of - objects alone. Values such as dicts, lists which - are serialized into strings are examples of "mutable" - column structures. - - .. note:: - - This functionality is now superseded by the - ``sqlalchemy.ext.mutable`` extension described in - :ref:`mutable_toplevel`. - - When this method is overridden, :meth:`copy_value` should - also be supplied. The :class:`.MutableType` mixin - is recommended as a helper. - - """ - return False - - def get_dbapi_type(self, dbapi): - """Return the corresponding type object from the underlying DB-API, if - any. - - This can be useful for calling ``setinputsizes()``, for example. - - """ - return None - - @property - def python_type(self): - """Return the Python type object expected to be returned - by instances of this type, if known. - - Basically, for those types which enforce a return type, - or are known across the board to do such for all common - DBAPIs (like ``int`` for example), will return that type. - - If a return type is not defined, raises - ``NotImplementedError``. - - Note that any type also accommodates NULL in SQL which - means you can also get back ``None`` from any type - in practice. - - """ - raise NotImplementedError() - - def with_variant(self, type_, dialect_name): - """Produce a new type object that will utilize the given - type when applied to the dialect of the given name. - - e.g.:: - - from sqlalchemy.types import String - from sqlalchemy.dialects import mysql - - s = String() - - s = s.with_variant(mysql.VARCHAR(collation='foo'), 'mysql') - - The construction of :meth:`.TypeEngine.with_variant` is always - from the "fallback" type to that which is dialect specific. - The returned type is an instance of :class:`.Variant`, which - itself provides a :meth:`~sqlalchemy.types.Variant.with_variant` that can - be called repeatedly. - - :param type_: a :class:`.TypeEngine` that will be selected - as a variant from the originating type, when a dialect - of the given name is in use. - :param dialect_name: base name of the dialect which uses - this type. (i.e. ``'postgresql'``, ``'mysql'``, etc.) - - .. versionadded:: 0.7.2 - - """ - return Variant(self, {dialect_name:type_}) - - def _adapt_expression(self, op, othertype): - """evaluate the return type of , - and apply any adaptations to the given operator. - - """ - return op, self - - @util.memoized_property - def _type_affinity(self): - """Return a rudimental 'affinity' value expressing the general class - of type.""" - - typ = None - for t in self.__class__.__mro__: - if t is TypeEngine or t is UserDefinedType: - return typ - elif issubclass(t, TypeEngine): - typ = t - else: - return self.__class__ - - def dialect_impl(self, dialect): - """Return a dialect-specific implementation for this :class:`.TypeEngine`.""" - - try: - return dialect._type_memos[self]['impl'] - except KeyError: - return self._dialect_info(dialect)['impl'] - - def _cached_bind_processor(self, dialect): - """Return a dialect-specific bind processor for this type.""" - - try: - return dialect._type_memos[self]['bind'] - except KeyError: - d = self._dialect_info(dialect) - d['bind'] = bp = d['impl'].bind_processor(dialect) - return bp - - def _cached_result_processor(self, dialect, coltype): - """Return a dialect-specific result processor for this type.""" - - try: - return dialect._type_memos[self][coltype] - except KeyError: - d = self._dialect_info(dialect) - # key assumption: DBAPI type codes are - # constants. Else this dictionary would - # grow unbounded. - d[coltype] = rp = d['impl'].result_processor(dialect, coltype) - return rp - - def _dialect_info(self, dialect): - """Return a dialect-specific registry which - caches a dialect-specific implementation, bind processing - function, and one or more result processing functions.""" - - if self in dialect._type_memos: - return dialect._type_memos[self] - else: - impl = self._gen_dialect_impl(dialect) - if impl is self: - impl = self.adapt(type(self)) - # this can't be self, else we create a cycle - assert impl is not self - dialect._type_memos[self] = d = {'impl':impl} - return d - - def _gen_dialect_impl(self, dialect): - return dialect.type_descriptor(self) - - def adapt(self, cls, **kw): - """Produce an "adapted" form of this type, given an "impl" class - to work with. - - This method is used internally to associate generic - types with "implementation" types that are specific to a particular - dialect. - """ - return util.constructor_copy(self, cls, **kw) - - def _coerce_compared_value(self, op, value): - """Suggest a type for a 'coerced' Python value in an expression. - - Given an operator and value, gives the type a chance - to return a type which the value should be coerced into. - - The default behavior here is conservative; if the right-hand - side is already coerced into a SQL type based on its - Python type, it is usually left alone. - - End-user functionality extension here should generally be via - :class:`.TypeDecorator`, which provides more liberal behavior in that - it defaults to coercing the other side of the expression into this - type, thus applying special Python conversions above and beyond those - needed by the DBAPI to both ides. It also provides the public method - :meth:`.TypeDecorator.coerce_compared_value` which is intended for - end-user customization of this behavior. - - """ - _coerced_type = _type_map.get(type(value), NULLTYPE) - if _coerced_type is NULLTYPE or _coerced_type._type_affinity \ - is self._type_affinity: - return self - else: - return _coerced_type - - def _compare_type_affinity(self, other): - return self._type_affinity is other._type_affinity - - def compile(self, dialect=None): - """Produce a string-compiled form of this :class:`.TypeEngine`. - - When called with no arguments, uses a "default" dialect - to produce a string result. - - :param dialect: a :class:`.Dialect` instance. - - """ - # arg, return value is inconsistent with - # ClauseElement.compile()....this is a mistake. - - if not dialect: - dialect = self._default_dialect - - return dialect.type_compiler.process(self) - - @property - def _default_dialect(self): - if self.__class__.__module__.startswith("sqlalchemy.dialects"): - tokens = self.__class__.__module__.split(".")[0:3] - mod = ".".join(tokens) - return getattr(__import__(mod).dialects, tokens[-1]).dialect() - else: - return default.DefaultDialect() - - def __str__(self): - # Py3K - #return unicode(self.compile()) - # Py2K - return unicode(self.compile()).\ - encode('ascii', 'backslashreplace') - # end Py2K - - def __init__(self, *args, **kwargs): - """Support implementations that were passing arguments""" - if args or kwargs: - util.warn_deprecated("Passing arguments to type object " - "constructor %s is deprecated" % self.__class__) - - def __repr__(self): - return util.generic_repr(self) - -class UserDefinedType(TypeEngine): - """Base for user defined types. - - This should be the base of new types. Note that - for most cases, :class:`.TypeDecorator` is probably - more appropriate:: - - import sqlalchemy.types as types - - class MyType(types.UserDefinedType): - def __init__(self, precision = 8): - self.precision = precision - - def get_col_spec(self): - return "MYTYPE(%s)" % self.precision - - def bind_processor(self, dialect): - def process(value): - return value - return process - - def result_processor(self, dialect, coltype): - def process(value): - return value - return process - - Once the type is made, it's immediately usable:: - - table = Table('foo', meta, - Column('id', Integer, primary_key=True), - Column('data', MyType(16)) - ) - - """ - __visit_name__ = "user_defined" - - def _adapt_expression(self, op, othertype): - """evaluate the return type of , - and apply any adaptations to the given operator. - - """ - return self.adapt_operator(op), self - - def adapt_operator(self, op): - """A hook which allows the given operator to be adapted - to something new. - - See also UserDefinedType._adapt_expression(), an as-yet- - semi-public method with greater capability in this regard. - - """ - return op - -class TypeDecorator(TypeEngine): - """Allows the creation of types which add additional functionality - to an existing type. - - This method is preferred to direct subclassing of SQLAlchemy's - built-in types as it ensures that all required functionality of - the underlying type is kept in place. - - Typical usage:: - - import sqlalchemy.types as types - - class MyType(types.TypeDecorator): - '''Prefixes Unicode values with "PREFIX:" on the way in and - strips it off on the way out. - ''' - - impl = types.Unicode - - def process_bind_param(self, value, dialect): - return "PREFIX:" + value - - def process_result_value(self, value, dialect): - return value[7:] - - def copy(self): - return MyType(self.impl.length) - - The class-level "impl" attribute is required, and can reference any - TypeEngine class. Alternatively, the load_dialect_impl() method - can be used to provide different type classes based on the dialect - given; in this case, the "impl" variable can reference - ``TypeEngine`` as a placeholder. - - Types that receive a Python type that isn't similar to the ultimate type - used may want to define the :meth:`TypeDecorator.coerce_compared_value` - method. This is used to give the expression system a hint when coercing - Python objects into bind parameters within expressions. Consider this - expression:: - - mytable.c.somecol + datetime.date(2009, 5, 15) - - Above, if "somecol" is an ``Integer`` variant, it makes sense that - we're doing date arithmetic, where above is usually interpreted - by databases as adding a number of days to the given date. - The expression system does the right thing by not attempting to - coerce the "date()" value into an integer-oriented bind parameter. - - However, in the case of ``TypeDecorator``, we are usually changing an - incoming Python type to something new - ``TypeDecorator`` by default will - "coerce" the non-typed side to be the same type as itself. Such as below, - we define an "epoch" type that stores a date value as an integer:: - - class MyEpochType(types.TypeDecorator): - impl = types.Integer - - epoch = datetime.date(1970, 1, 1) - - def process_bind_param(self, value, dialect): - return (value - self.epoch).days - - def process_result_value(self, value, dialect): - return self.epoch + timedelta(days=value) - - Our expression of ``somecol + date`` with the above type will coerce the - "date" on the right side to also be treated as ``MyEpochType``. - - This behavior can be overridden via the - :meth:`~TypeDecorator.coerce_compared_value` method, which returns a type - that should be used for the value of the expression. Below we set it such - that an integer value will be treated as an ``Integer``, and any other - value is assumed to be a date and will be treated as a ``MyEpochType``:: - - def coerce_compared_value(self, op, value): - if isinstance(value, int): - return Integer() - else: - return self - - """ - - __visit_name__ = "type_decorator" - - def __init__(self, *args, **kwargs): - """Construct a :class:`.TypeDecorator`. - - Arguments sent here are passed to the constructor - of the class assigned to the ``impl`` class level attribute, - assuming the ``impl`` is a callable, and the resulting - object is assigned to the ``self.impl`` instance attribute - (thus overriding the class attribute of the same name). - - If the class level ``impl`` is not a callable (the unusual case), - it will be assigned to the same instance attribute 'as-is', - ignoring those arguments passed to the constructor. - - Subclasses can override this to customize the generation - of ``self.impl`` entirely. - - """ - - if not hasattr(self.__class__, 'impl'): - raise AssertionError("TypeDecorator implementations " - "require a class-level variable " - "'impl' which refers to the class of " - "type being decorated") - self.impl = to_instance(self.__class__.impl, *args, **kwargs) - - - def _gen_dialect_impl(self, dialect): - """ - #todo - """ - adapted = dialect.type_descriptor(self) - if adapted is not self: - return adapted - - # otherwise adapt the impl type, link - # to a copy of this TypeDecorator and return - # that. - typedesc = self.load_dialect_impl(dialect).dialect_impl(dialect) - tt = self.copy() - if not isinstance(tt, self.__class__): - raise AssertionError('Type object %s does not properly ' - 'implement the copy() method, it must ' - 'return an object of type %s' % (self, - self.__class__)) - tt.impl = typedesc - return tt - - @property - def _type_affinity(self): - """ - #todo - """ - return self.impl._type_affinity - - def type_engine(self, dialect): - """Return a dialect-specific :class:`.TypeEngine` instance for this :class:`.TypeDecorator`. - - In most cases this returns a dialect-adapted form of - the :class:`.TypeEngine` type represented by ``self.impl``. - Makes usage of :meth:`dialect_impl` but also traverses - into wrapped :class:`.TypeDecorator` instances. - Behavior can be customized here by overriding :meth:`load_dialect_impl`. - - """ - adapted = dialect.type_descriptor(self) - if type(adapted) is not type(self): - return adapted - elif isinstance(self.impl, TypeDecorator): - return self.impl.type_engine(dialect) - else: - return self.load_dialect_impl(dialect) - - def load_dialect_impl(self, dialect): - """Return a :class:`.TypeEngine` object corresponding to a dialect. - - This is an end-user override hook that can be used to provide - differing types depending on the given dialect. It is used - by the :class:`.TypeDecorator` implementation of :meth:`type_engine` - to help determine what type should ultimately be returned - for a given :class:`.TypeDecorator`. - - By default returns ``self.impl``. - - """ - return self.impl - - def __getattr__(self, key): - """Proxy all other undefined accessors to the underlying - implementation.""" - return getattr(self.impl, key) - - def process_bind_param(self, value, dialect): - """Receive a bound parameter value to be converted. - - Subclasses override this method to return the - value that should be passed along to the underlying - :class:`.TypeEngine` object, and from there to the - DBAPI ``execute()`` method. - - The operation could be anything desired to perform custom - behavior, such as transforming or serializing data. - This could also be used as a hook for validating logic. - - This operation should be designed with the reverse operation - in mind, which would be the process_result_value method of - this class. - - :param value: Data to operate upon, of any type expected by - this method in the subclass. Can be ``None``. - :param dialect: the :class:`.Dialect` in use. - - """ - - raise NotImplementedError() - - def process_result_value(self, value, dialect): - """Receive a result-row column value to be converted. - - Subclasses should implement this method to operate on data - fetched from the database. - - Subclasses override this method to return the - value that should be passed back to the application, - given a value that is already processed by - the underlying :class:`.TypeEngine` object, originally - from the DBAPI cursor method ``fetchone()`` or similar. - - The operation could be anything desired to perform custom - behavior, such as transforming or serializing data. - This could also be used as a hook for validating logic. - - :param value: Data to operate upon, of any type expected by - this method in the subclass. Can be ``None``. - :param dialect: the :class:`.Dialect` in use. - - This operation should be designed to be reversible by - the "process_bind_param" method of this class. - - """ - - raise NotImplementedError() - - def bind_processor(self, dialect): - """Provide a bound value processing function for the - given :class:`.Dialect`. - - This is the method that fulfills the :class:`.TypeEngine` - contract for bound value conversion. :class:`.TypeDecorator` - will wrap a user-defined implementation of - :meth:`process_bind_param` here. - - User-defined code can override this method directly, - though its likely best to use :meth:`process_bind_param` so that - the processing provided by ``self.impl`` is maintained. - - :param dialect: Dialect instance in use. - - This method is the reverse counterpart to the - :meth:`result_processor` method of this class. - - """ - if self.__class__.process_bind_param.func_code \ - is not TypeDecorator.process_bind_param.func_code: - process_param = self.process_bind_param - impl_processor = self.impl.bind_processor(dialect) - if impl_processor: - def process(value): - return impl_processor(process_param(value, dialect)) - - else: - def process(value): - return process_param(value, dialect) - - return process - else: - return self.impl.bind_processor(dialect) - - def result_processor(self, dialect, coltype): - """Provide a result value processing function for the given :class:`.Dialect`. - - This is the method that fulfills the :class:`.TypeEngine` - contract for result value conversion. :class:`.TypeDecorator` - will wrap a user-defined implementation of - :meth:`process_result_value` here. - - User-defined code can override this method directly, - though its likely best to use :meth:`process_result_value` so that - the processing provided by ``self.impl`` is maintained. - - :param dialect: Dialect instance in use. - :param coltype: An SQLAlchemy data type - - This method is the reverse counterpart to the - :meth:`bind_processor` method of this class. - - """ - if self.__class__.process_result_value.func_code \ - is not TypeDecorator.process_result_value.func_code: - process_value = self.process_result_value - impl_processor = self.impl.result_processor(dialect, - coltype) - if impl_processor: - def process(value): - return process_value(impl_processor(value), dialect) - - else: - def process(value): - return process_value(value, dialect) - - return process - else: - return self.impl.result_processor(dialect, coltype) - - def coerce_compared_value(self, op, value): - """Suggest a type for a 'coerced' Python value in an expression. - - By default, returns self. This method is called by - the expression system when an object using this type is - on the left or right side of an expression against a plain Python - object which does not yet have a SQLAlchemy type assigned:: - - expr = table.c.somecolumn + 35 - - Where above, if ``somecolumn`` uses this type, this method will - be called with the value ``operator.add`` - and ``35``. The return value is whatever SQLAlchemy type should - be used for ``35`` for this particular operation. - - """ - return self - - def _coerce_compared_value(self, op, value): - """See :meth:`.TypeEngine._coerce_compared_value` for a description.""" - - return self.coerce_compared_value(op, value) - - def copy(self): - """Produce a copy of this :class:`.TypeDecorator` instance. - - This is a shallow copy and is provided to fulfill part of - the :class:`.TypeEngine` contract. It usually does not - need to be overridden unless the user-defined :class:`.TypeDecorator` - has local state that should be deep-copied. - - """ - - instance = self.__class__.__new__(self.__class__) - instance.__dict__.update(self.__dict__) - return instance - - def get_dbapi_type(self, dbapi): - """Return the DBAPI type object represented by this :class:`.TypeDecorator`. - - By default this calls upon :meth:`.TypeEngine.get_dbapi_type` of the - underlying "impl". - """ - return self.impl.get_dbapi_type(dbapi) - - def copy_value(self, value): - """Given a value, produce a copy of it. - - By default this calls upon :meth:`.TypeEngine.copy_value` - of the underlying "impl". - - :meth:`.copy_value` will return the object - itself, assuming "mutability" is not enabled. - Only the :class:`.MutableType` mixin provides a copy - function that actually produces a new object. - The copying function is used by the ORM when - "mutable" types are used, to memoize the original - version of an object as loaded from the database, - which is then compared to the possibly mutated - version to check for changes. - - Modern implementations should use the - ``sqlalchemy.ext.mutable`` extension described in - :ref:`mutable_toplevel` for intercepting in-place - changes to values. - - """ - return self.impl.copy_value(value) - - def compare_values(self, x, y): - """Given two values, compare them for equality. - - By default this calls upon :meth:`.TypeEngine.compare_values` - of the underlying "impl", which in turn usually - uses the Python equals operator ``==``. - - This function is used by the ORM to compare - an original-loaded value with an intercepted - "changed" value, to determine if a net change - has occurred. - - """ - return self.impl.compare_values(x, y) - - def is_mutable(self): - """Return True if the target Python type is 'mutable'. - - This allows systems like the ORM to know if a column value can - be considered 'not changed' by comparing the identity of - objects alone. Values such as dicts, lists which - are serialized into strings are examples of "mutable" - column structures. - - .. note:: - - This functionality is now superseded by the - ``sqlalchemy.ext.mutable`` extension described in - :ref:`mutable_toplevel`. - - """ - return self.impl.is_mutable() - - def _adapt_expression(self, op, othertype): - """ - #todo - """ - op, typ =self.impl._adapt_expression(op, othertype) - if typ is self.impl: - return op, self - else: - return op, typ - - def __repr__(self): - return util.generic_repr(self, to_inspect=self.impl) - -class Variant(TypeDecorator): - """A wrapping type that selects among a variety of - implementations based on dialect in use. - - The :class:`.Variant` type is typically constructed - using the :meth:`.TypeEngine.with_variant` method. - - .. versionadded:: 0.7.2 - - """ - - def __init__(self, base, mapping): - """Construct a new :class:`.Variant`. - - :param base: the base 'fallback' type - :param mapping: dictionary of string dialect names to :class:`.TypeEngine` - instances. - - """ - self.impl = base - self.mapping = mapping - - def load_dialect_impl(self, dialect): - if dialect.name in self.mapping: - return self.mapping[dialect.name] - else: - return self.impl - - def with_variant(self, type_, dialect_name): - """Return a new :class:`.Variant` which adds the given - type + dialect name to the mapping, in addition to the - mapping present in this :class:`.Variant`. - - :param type_: a :class:`.TypeEngine` that will be selected - as a variant from the originating type, when a dialect - of the given name is in use. - :param dialect_name: base name of the dialect which uses - this type. (i.e. ``'postgresql'``, ``'mysql'``, etc.) - - """ - - if dialect_name in self.mapping: - raise exc.ArgumentError( - "Dialect '%s' is already present in " - "the mapping for this Variant" % dialect_name) - mapping = self.mapping.copy() - mapping[dialect_name] = type_ - return Variant(self.impl, mapping) - -class MutableType(object): - """A mixin that marks a :class:`.TypeEngine` as representing - a mutable Python object type. This functionality is used - only by the ORM. - - .. versionchanged:: 0.7 - :class:`.MutableType` is superseded - by the ``sqlalchemy.ext.mutable`` extension described in - :ref:`mutable_toplevel`. This extension provides an event - driven approach to in-place mutation detection that does not - incur the severe performance penalty of the :class:`.MutableType` - approach. - - "mutable" means that changes can occur in place to a value - of this type. Examples includes Python lists, dictionaries, - and sets, as well as user-defined objects. The primary - need for identification of "mutable" types is by the ORM, - which applies special rules to such values in order to guarantee - that changes are detected. These rules may have a significant - performance impact, described below. - - A :class:`.MutableType` usually allows a flag called - ``mutable=False`` to enable/disable the "mutability" flag, - represented on this class by :meth:`is_mutable`. Examples - include :class:`.PickleType` and - :class:`~sqlalchemy.dialects.postgresql.base.ARRAY`. Setting - this flag to ``True`` enables mutability-specific behavior - by the ORM. - - The :meth:`copy_value` and :meth:`compare_values` functions - represent a copy and compare function for values of this - type - implementing subclasses should override these - appropriately. - - .. warning:: - - The usage of mutable types has significant performance - implications when using the ORM. In order to detect changes, the - ORM must create a copy of the value when it is first - accessed, so that changes to the current value can be compared - against the "clean" database-loaded value. Additionally, when the - ORM checks to see if any data requires flushing, it must scan - through all instances in the session which are known to have - "mutable" attributes and compare the current value of each - one to its "clean" - value. So for example, if the Session contains 6000 objects (a - fairly large amount) and autoflush is enabled, every individual - execution of :class:`.Query` will require a full scan of that subset of - the 6000 objects that have mutable attributes, possibly resulting - in tens of thousands of additional method calls for every query. - - .. versionchanged:: 0.7 - As of SQLAlchemy 0.7, the ``sqlalchemy.ext.mutable`` is provided - which allows an event driven approach to in-place - mutation detection. This approach should now be favored over - the usage of :class:`.MutableType` with ``mutable=True``. - ``sqlalchemy.ext.mutable`` is described in :ref:`mutable_toplevel`. - - """ - - def is_mutable(self): - """Return True if the target Python type is 'mutable'. - - For :class:`.MutableType`, this method is set to - return ``True``. - - """ - return True - - def copy_value(self, value): - """Unimplemented.""" - raise NotImplementedError() - - def compare_values(self, x, y): - """Compare *x* == *y*.""" - return x == y - -def to_instance(typeobj, *arg, **kw): - if typeobj is None: - return NULLTYPE - - if util.callable(typeobj): - return typeobj(*arg, **kw) - else: - return typeobj - -def adapt_type(typeobj, colspecs): - if isinstance(typeobj, type): - typeobj = typeobj() - for t in typeobj.__class__.__mro__[0:-1]: - try: - impltype = colspecs[t] - break - except KeyError: - pass - else: - # couldnt adapt - so just return the type itself - # (it may be a user-defined type) - return typeobj - # if we adapted the given generic type to a database-specific type, - # but it turns out the originally given "generic" type - # is actually a subclass of our resulting type, then we were already - # given a more specific type than that required; so use that. - if (issubclass(typeobj.__class__, impltype)): - return typeobj - return typeobj.adapt(impltype) - - - - - -class NullType(TypeEngine): - """An unknown type. - - NullTypes will stand in if :class:`~sqlalchemy.Table` reflection - encounters a column data type unknown to SQLAlchemy. The - resulting columns are nearly fully usable: the DB-API adapter will - handle all translation to and from the database data type. - - NullType does not have sufficient information to particpate in a - ``CREATE TABLE`` statement and will raise an exception if - encountered during a :meth:`~sqlalchemy.Table.create` operation. - - """ - __visit_name__ = 'null' - - def _adapt_expression(self, op, othertype): - if isinstance(othertype, NullType) or not operators.is_commutative(op): - return op, self - else: - return othertype._adapt_expression(op, self) - -NullTypeEngine = NullType - -class Concatenable(object): - """A mixin that marks a type as supporting 'concatenation', - typically strings.""" - - def _adapt_expression(self, op, othertype): - if op is operators.add and issubclass(othertype._type_affinity, - (Concatenable, NullType)): - return operators.concat_op, self - else: - return op, self - -class _DateAffinity(object): - """Mixin date/time specific expression adaptations. - - Rules are implemented within Date,Time,Interval,DateTime, Numeric, - Integer. Based on http://www.postgresql.org/docs/current/static - /functions-datetime.html. - - """ - - @property - def _expression_adaptations(self): - raise NotImplementedError() - - _blank_dict = util.immutabledict() - def _adapt_expression(self, op, othertype): - othertype = othertype._type_affinity - return op, \ - self._expression_adaptations.get(op, self._blank_dict).\ - get(othertype, NULLTYPE) - -class String(Concatenable, TypeEngine): - """The base for all string and character types. - - In SQL, corresponds to VARCHAR. Can also take Python unicode objects - and encode to the database's encoding in bind params (and the reverse for - result sets.) - - The `length` field is usually required when the `String` type is - used within a CREATE TABLE statement, as VARCHAR requires a length - on most databases. - - """ - - __visit_name__ = 'string' - - def __init__(self, length=None, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False - ): - """ - Create a string-holding type. - - :param length: optional, a length for the column for use in - DDL statements. May be safely omitted if no ``CREATE - TABLE`` will be issued. Certain databases may require a - ``length`` for use in DDL, and will raise an exception when - the ``CREATE TABLE`` DDL is issued if a ``VARCHAR`` - with no length is included. Whether the value is - interpreted as bytes or characters is database specific. - - :param convert_unicode: When set to ``True``, the - :class:`.String` type will assume that - input is to be passed as Python ``unicode`` objects, - and results returned as Python ``unicode`` objects. - If the DBAPI in use does not support Python unicode - (which is fewer and fewer these days), SQLAlchemy - will encode/decode the value, using the - value of the ``encoding`` parameter passed to - :func:`.create_engine` as the encoding. - - When using a DBAPI that natively supports Python - unicode objects, this flag generally does not - need to be set. For columns that are explicitly - intended to store non-ASCII data, the :class:`.Unicode` - or :class:`UnicodeText` - types should be used regardless, which feature - the same behavior of ``convert_unicode`` but - also indicate an underlying column type that - directly supports unicode, such as ``NVARCHAR``. - - For the extremely rare case that Python ``unicode`` - is to be encoded/decoded by SQLAlchemy on a backend - that does natively support Python ``unicode``, - the value ``force`` can be passed here which will - cause SQLAlchemy's encode/decode services to be - used unconditionally. - - :param assert_unicode: Deprecated. A warning is emitted - when a non-``unicode`` object is passed to the - :class:`.Unicode` subtype of :class:`.String`, - or the :class:`.UnicodeText` subtype of :class:`.Text`. - See :class:`.Unicode` for information on how to - control this warning. - - :param unicode_error: Optional, a method to use to handle Unicode - conversion errors. Behaves like the ``errors`` keyword argument to - the standard library's ``string.decode()`` functions. This flag - requires that ``convert_unicode`` is set to ``force`` - otherwise, - SQLAlchemy is not guaranteed to handle the task of unicode - conversion. Note that this flag adds significant performance - overhead to row-fetching operations for backends that already - return unicode objects natively (which most DBAPIs do). This - flag should only be used as a last resort for reading - strings from a column with varied or corrupted encodings. - - """ - if unicode_error is not None and convert_unicode != 'force': - raise exc.ArgumentError("convert_unicode must be 'force' " - "when unicode_error is set.") - - if assert_unicode: - util.warn_deprecated('assert_unicode is deprecated. ' - 'SQLAlchemy emits a warning in all ' - 'cases where it would otherwise like ' - 'to encode a Python unicode object ' - 'into a specific encoding but a plain ' - 'bytestring is received. This does ' - '*not* apply to DBAPIs that coerce ' - 'Unicode natively.') - self.length = length - self.convert_unicode = convert_unicode - self.unicode_error = unicode_error - self._warn_on_bytestring = _warn_on_bytestring - - def bind_processor(self, dialect): - if self.convert_unicode or dialect.convert_unicode: - if dialect.supports_unicode_binds and \ - self.convert_unicode != 'force': - if self._warn_on_bytestring: - def process(value): - # Py3K - #if isinstance(value, bytes): - # Py2K - if isinstance(value, str): - # end Py2K - util.warn("Unicode type received non-unicode bind " - "param value.") - return value - return process - else: - return None - else: - encoder = codecs.getencoder(dialect.encoding) - warn_on_bytestring = self._warn_on_bytestring - def process(value): - if isinstance(value, unicode): - return encoder(value, self.unicode_error)[0] - elif warn_on_bytestring and value is not None: - util.warn("Unicode type received non-unicode bind " - "param value") - return value - return process - else: - return None - - def result_processor(self, dialect, coltype): - wants_unicode = self.convert_unicode or dialect.convert_unicode - needs_convert = wants_unicode and \ - (dialect.returns_unicode_strings is not True or - self.convert_unicode == 'force') - - if needs_convert: - to_unicode = processors.to_unicode_processor_factory( - dialect.encoding, self.unicode_error) - - if dialect.returns_unicode_strings: - # we wouldn't be here unless convert_unicode='force' - # was specified, or the driver has erratic unicode-returning - # habits. since we will be getting back unicode - # in most cases, we check for it (decode will fail). - def process(value): - if isinstance(value, unicode): - return value - else: - return to_unicode(value) - return process - else: - # here, we assume that the object is not unicode, - # avoiding expensive isinstance() check. - return to_unicode - else: - return None - - @property - def python_type(self): - if self.convert_unicode: - return unicode - else: - return str - - def get_dbapi_type(self, dbapi): - return dbapi.STRING - -class Text(String): - """A variably sized string type. - - In SQL, usually corresponds to CLOB or TEXT. Can also take Python - unicode objects and encode to the database's encoding in bind - params (and the reverse for result sets.) - - """ - __visit_name__ = 'text' - -class Unicode(String): - """A variable length Unicode string type. - - The :class:`.Unicode` type is a :class:`.String` subclass - that assumes input and output as Python ``unicode`` data, - and in that regard is equivalent to the usage of the - ``convert_unicode`` flag with the :class:`.String` type. - However, unlike plain :class:`.String`, it also implies an - underlying column type that is explicitly supporting of non-ASCII - data, such as ``NVARCHAR`` on Oracle and SQL Server. - This can impact the output of ``CREATE TABLE`` statements - and ``CAST`` functions at the dialect level, and can - also affect the handling of bound parameters in some - specific DBAPI scenarios. - - The encoding used by the :class:`.Unicode` type is usually - determined by the DBAPI itself; most modern DBAPIs - feature support for Python ``unicode`` objects as bound - values and result set values, and the encoding should - be configured as detailed in the notes for the target - DBAPI in the :ref:`dialect_toplevel` section. - - For those DBAPIs which do not support, or are not configured - to accommodate Python ``unicode`` objects - directly, SQLAlchemy does the encoding and decoding - outside of the DBAPI. The encoding in this scenario - is determined by the ``encoding`` flag passed to - :func:`.create_engine`. - - When using the :class:`.Unicode` type, it is only appropriate - to pass Python ``unicode`` objects, and not plain ``str``. - If a plain ``str`` is passed under Python 2, a warning - is emitted. If you notice your application emitting these warnings but - you're not sure of the source of them, the Python - ``warnings`` filter, documented at - http://docs.python.org/library/warnings.html, - can be used to turn these warnings into exceptions - which will illustrate a stack trace:: - - import warnings - warnings.simplefilter('error') - - For an application that wishes to pass plain bytestrings - and Python ``unicode`` objects to the ``Unicode`` type - equally, the bytestrings must first be decoded into - unicode. The recipe at :ref:`coerce_to_unicode` illustrates - how this is done. - - See also: - - :class:`.UnicodeText` - unlengthed textual counterpart - to :class:`.Unicode`. - - """ - - __visit_name__ = 'unicode' - - def __init__(self, length=None, **kwargs): - """ - Create a :class:`.Unicode` object. - - Parameters are the same as that of :class:`.String`, - with the exception that ``convert_unicode`` - defaults to ``True``. - - """ - kwargs.setdefault('convert_unicode', True) - kwargs.setdefault('_warn_on_bytestring', True) - super(Unicode, self).__init__(length=length, **kwargs) - -class UnicodeText(Text): - """An unbounded-length Unicode string type. - - See :class:`.Unicode` for details on the unicode - behavior of this object. - - Like :class:`.Unicode`, usage the :class:`.UnicodeText` type implies a - unicode-capable type being used on the backend, such as - ``NCLOB``, ``NTEXT``. - - """ - - __visit_name__ = 'unicode_text' - - def __init__(self, length=None, **kwargs): - """ - Create a Unicode-converting Text type. - - Parameters are the same as that of :class:`.Text`, - with the exception that ``convert_unicode`` - defaults to ``True``. - - """ - kwargs.setdefault('convert_unicode', True) - kwargs.setdefault('_warn_on_bytestring', True) - super(UnicodeText, self).__init__(length=length, **kwargs) - - -class Integer(_DateAffinity, TypeEngine): - """A type for ``int`` integers.""" - - __visit_name__ = 'integer' - - def get_dbapi_type(self, dbapi): - return dbapi.NUMBER - - @property - def python_type(self): - return int - - @util.memoized_property - def _expression_adaptations(self): - # TODO: need a dictionary object that will - # handle operators generically here, this is incomplete - return { - operators.add:{ - Date:Date, - Integer:Integer, - Numeric:Numeric, - }, - operators.mul:{ - Interval:Interval, - Integer:Integer, - Numeric:Numeric, - }, - # Py2K - operators.div:{ - Integer:Integer, - Numeric:Numeric, - }, - # end Py2K - operators.truediv:{ - Integer:Integer, - Numeric:Numeric, - }, - operators.sub:{ - Integer:Integer, - Numeric:Numeric, - }, - } - -class SmallInteger(Integer): - """A type for smaller ``int`` integers. - - Typically generates a ``SMALLINT`` in DDL, and otherwise acts like - a normal :class:`.Integer` on the Python side. - - """ - - __visit_name__ = 'small_integer' - - -class BigInteger(Integer): - """A type for bigger ``int`` integers. - - Typically generates a ``BIGINT`` in DDL, and otherwise acts like - a normal :class:`.Integer` on the Python side. - - """ - - __visit_name__ = 'big_integer' - - -class Numeric(_DateAffinity, TypeEngine): - """A type for fixed precision numbers. - - Typically generates DECIMAL or NUMERIC. Returns - ``decimal.Decimal`` objects by default, applying - conversion as needed. - - .. note:: - - The `cdecimal `_ library - is a high performing alternative to Python's built-in - ``decimal.Decimal`` type, which performs very poorly in high volume - situations. SQLAlchemy 0.7 is tested against ``cdecimal`` and supports - it fully. The type is not necessarily supported by DBAPI - implementations however, most of which contain an import for plain - ``decimal`` in their source code, even though some such as psycopg2 - provide hooks for alternate adapters. SQLAlchemy imports ``decimal`` - globally as well. While the alternate ``Decimal`` class can be patched - into SQLA's ``decimal`` module, overall the most straightforward and - foolproof way to use "cdecimal" given current DBAPI and Python support - is to patch it directly into sys.modules before anything else is - imported:: - - import sys - import cdecimal - sys.modules["decimal"] = cdecimal - - While the global patch is a little ugly, it's particularly - important to use just one decimal library at a time since - Python Decimal and cdecimal Decimal objects - are not currently compatible *with each other*:: - - >>> import cdecimal - >>> import decimal - >>> decimal.Decimal("10") == cdecimal.Decimal("10") - False - - SQLAlchemy will provide more natural support of - cdecimal if and when it becomes a standard part of Python - installations and is supported by all DBAPIs. - - """ - - __visit_name__ = 'numeric' - - def __init__(self, precision=None, scale=None, asdecimal=True): - """ - Construct a Numeric. - - :param precision: the numeric precision for use in DDL ``CREATE - TABLE``. - - :param scale: the numeric scale for use in DDL ``CREATE TABLE``. - - :param asdecimal: default True. Return whether or not - values should be sent as Python Decimal objects, or - as floats. Different DBAPIs send one or the other based on - datatypes - the Numeric type will ensure that return values - are one or the other across DBAPIs consistently. - - When using the ``Numeric`` type, care should be taken to ensure - that the asdecimal setting is apppropriate for the DBAPI in use - - when Numeric applies a conversion from Decimal->float or float-> - Decimal, this conversion incurs an additional performance overhead - for all result columns received. - - DBAPIs that return Decimal natively (e.g. psycopg2) will have - better accuracy and higher performance with a setting of ``True``, - as the native translation to Decimal reduces the amount of floating- - point issues at play, and the Numeric type itself doesn't need - to apply any further conversions. However, another DBAPI which - returns floats natively *will* incur an additional conversion - overhead, and is still subject to floating point data loss - in - which case ``asdecimal=False`` will at least remove the extra - conversion overhead. - - """ - self.precision = precision - self.scale = scale - self.asdecimal = asdecimal - - def get_dbapi_type(self, dbapi): - return dbapi.NUMBER - - @property - def python_type(self): - if self.asdecimal: - return decimal.Decimal - else: - return float - - def bind_processor(self, dialect): - if dialect.supports_native_decimal: - return None - else: - return processors.to_float - - def result_processor(self, dialect, coltype): - if self.asdecimal: - if dialect.supports_native_decimal: - # we're a "numeric", DBAPI will give us Decimal directly - return None - else: - util.warn('Dialect %s+%s does *not* support Decimal ' - 'objects natively, and SQLAlchemy must ' - 'convert from floating point - rounding ' - 'errors and other issues may occur. Please ' - 'consider storing Decimal numbers as strings ' - 'or integers on this platform for lossless ' - 'storage.' % (dialect.name, dialect.driver)) - - # we're a "numeric", DBAPI returns floats, convert. - if self.scale is not None: - return processors.to_decimal_processor_factory( - decimal.Decimal, self.scale) - else: - return processors.to_decimal_processor_factory( - decimal.Decimal) - else: - if dialect.supports_native_decimal: - return processors.to_float - else: - return None - - @util.memoized_property - def _expression_adaptations(self): - return { - operators.mul:{ - Interval:Interval, - Numeric:Numeric, - Integer:Numeric, - }, - # Py2K - operators.div:{ - Numeric:Numeric, - Integer:Numeric, - }, - # end Py2K - operators.truediv:{ - Numeric:Numeric, - Integer:Numeric, - }, - operators.add:{ - Numeric:Numeric, - Integer:Numeric, - }, - operators.sub:{ - Numeric:Numeric, - Integer:Numeric, - } - } - -class Float(Numeric): - """A type for ``float`` numbers. - - Returns Python ``float`` objects by default, applying - conversion as needed. - - """ - - __visit_name__ = 'float' - - scale = None - - def __init__(self, precision=None, asdecimal=False, **kwargs): - """ - Construct a Float. - - :param precision: the numeric precision for use in DDL ``CREATE - TABLE``. - - :param asdecimal: the same flag as that of :class:`.Numeric`, but - defaults to ``False``. Note that setting this flag to ``True`` - results in floating point conversion. - - :param \**kwargs: deprecated. Additional arguments here are ignored - by the default :class:`.Float` type. For database specific - floats that support additional arguments, see that dialect's - documentation for details, such as :class:`sqlalchemy.dialects.mysql.FLOAT`. - - """ - self.precision = precision - self.asdecimal = asdecimal - if kwargs: - util.warn_deprecated("Additional keyword arguments " - "passed to Float ignored.") - - def result_processor(self, dialect, coltype): - if self.asdecimal: - return processors.to_decimal_processor_factory(decimal.Decimal) - else: - return None - - @util.memoized_property - def _expression_adaptations(self): - return { - operators.mul:{ - Interval:Interval, - Numeric:Float, - }, - # Py2K - operators.div:{ - Numeric:Float, - }, - # end Py2K - operators.truediv:{ - Numeric:Float, - }, - operators.add:{ - Numeric:Float, - }, - operators.sub:{ - Numeric:Float, - } - } - - -class DateTime(_DateAffinity, TypeEngine): - """A type for ``datetime.datetime()`` objects. - - Date and time types return objects from the Python ``datetime`` - module. Most DBAPIs have built in support for the datetime - module, with the noted exception of SQLite. In the case of - SQLite, date and time types are stored as strings which are then - converted back to datetime objects when rows are returned. - - """ - - __visit_name__ = 'datetime' - - def __init__(self, timezone=False): - """Construct a new :class:`.DateTime`. - - :param timezone: boolean. If True, and supported by the - backend, will produce 'TIMESTAMP WITH TIMEZONE'. For backends - that don't support timezone aware timestamps, has no - effect. - - """ - self.timezone = timezone - - def get_dbapi_type(self, dbapi): - return dbapi.DATETIME - - @property - def python_type(self): - return dt.datetime - - @util.memoized_property - def _expression_adaptations(self): - return { - operators.add:{ - Interval:DateTime, - }, - operators.sub:{ - Interval:DateTime, - DateTime:Interval, - }, - } - - -class Date(_DateAffinity,TypeEngine): - """A type for ``datetime.date()`` objects.""" - - __visit_name__ = 'date' - - def get_dbapi_type(self, dbapi): - return dbapi.DATETIME - - @property - def python_type(self): - return dt.date - - @util.memoized_property - def _expression_adaptations(self): - return { - operators.add:{ - Integer:Date, - Interval:DateTime, - Time:DateTime, - }, - operators.sub:{ - # date - integer = date - Integer:Date, - - # date - date = integer. - Date:Integer, - - Interval:DateTime, - - # date - datetime = interval, - # this one is not in the PG docs - # but works - DateTime:Interval, - }, - } - - -class Time(_DateAffinity,TypeEngine): - """A type for ``datetime.time()`` objects.""" - - __visit_name__ = 'time' - - def __init__(self, timezone=False): - self.timezone = timezone - - def get_dbapi_type(self, dbapi): - return dbapi.DATETIME - - @property - def python_type(self): - return dt.time - - @util.memoized_property - def _expression_adaptations(self): - return { - operators.add:{ - Date:DateTime, - Interval:Time - }, - operators.sub:{ - Time:Interval, - Interval:Time, - }, - } - - -class _Binary(TypeEngine): - """Define base behavior for binary types.""" - - def __init__(self, length=None): - self.length = length - - @property - def python_type(self): - # Py3K - #return bytes - # Py2K - return str - # end Py2K - - # Python 3 - sqlite3 doesn't need the `Binary` conversion - # here, though pg8000 does to indicate "bytea" - def bind_processor(self, dialect): - DBAPIBinary = dialect.dbapi.Binary - def process(value): - x = self - if value is not None: - return DBAPIBinary(value) - else: - return None - return process - - # Python 3 has native bytes() type - # both sqlite3 and pg8000 seem to return it - # (i.e. and not 'memoryview') - # Py2K - def result_processor(self, dialect, coltype): - if util.jython: - def process(value): - if value is not None: - if isinstance(value, array.array): - return value.tostring() - return str(value) - else: - return None - else: - process = processors.to_str - return process - # end Py2K - - def _coerce_compared_value(self, op, value): - """See :meth:`.TypeEngine._coerce_compared_value` for a description.""" - - if isinstance(value, basestring): - return self - else: - return super(_Binary, self)._coerce_compared_value(op, value) - - def get_dbapi_type(self, dbapi): - return dbapi.BINARY - -class LargeBinary(_Binary): - """A type for large binary byte data. - - The Binary type generates BLOB or BYTEA when tables are created, - and also converts incoming values using the ``Binary`` callable - provided by each DB-API. - - """ - - __visit_name__ = 'large_binary' - - def __init__(self, length=None): - """ - Construct a LargeBinary type. - - :param length: optional, a length for the column for use in - DDL statements, for those BLOB types that accept a length - (i.e. MySQL). It does *not* produce a small BINARY/VARBINARY - type - use the BINARY/VARBINARY types specifically for those. - May be safely omitted if no ``CREATE - TABLE`` will be issued. Certain databases may require a - *length* for use in DDL, and will raise an exception when - the ``CREATE TABLE`` DDL is issued. - - """ - _Binary.__init__(self, length=length) - -class Binary(LargeBinary): - """Deprecated. Renamed to LargeBinary.""" - - def __init__(self, *arg, **kw): - util.warn_deprecated('The Binary type has been renamed to ' - 'LargeBinary.') - LargeBinary.__init__(self, *arg, **kw) - -class SchemaType(events.SchemaEventTarget): - """Mark a type as possibly requiring schema-level DDL for usage. - - Supports types that must be explicitly created/dropped (i.e. PG ENUM type) - as well as types that are complimented by table or schema level - constraints, triggers, and other rules. - - :class:`.SchemaType` classes can also be targets for the - :meth:`.DDLEvents.before_parent_attach` and :meth:`.DDLEvents.after_parent_attach` - events, where the events fire off surrounding the association of - the type object with a parent :class:`.Column`. - - """ - - def __init__(self, **kw): - self.name = kw.pop('name', None) - self.quote = kw.pop('quote', None) - self.schema = kw.pop('schema', None) - self.metadata = kw.pop('metadata', None) - if self.metadata: - event.listen( - self.metadata, - "before_create", - util.portable_instancemethod(self._on_metadata_create) - ) - event.listen( - self.metadata, - "after_drop", - util.portable_instancemethod(self._on_metadata_drop) - ) - - def _set_parent(self, column): - column._on_table_attach(util.portable_instancemethod(self._set_table)) - - def _set_table(self, column, table): - event.listen( - table, - "before_create", - util.portable_instancemethod( - self._on_table_create) - ) - event.listen( - table, - "after_drop", - util.portable_instancemethod(self._on_table_drop) - ) - if self.metadata is None: - # TODO: what's the difference between self.metadata - # and table.metadata here ? - event.listen( - table.metadata, - "before_create", - util.portable_instancemethod(self._on_metadata_create) - ) - event.listen( - table.metadata, - "after_drop", - util.portable_instancemethod(self._on_metadata_drop) - ) - - @property - def bind(self): - return self.metadata and self.metadata.bind or None - - def create(self, bind=None, checkfirst=False): - """Issue CREATE ddl for this type, if applicable.""" - - if bind is None: - bind = schema._bind_or_error(self) - t = self.dialect_impl(bind.dialect) - if t.__class__ is not self.__class__ and isinstance(t, SchemaType): - t.create(bind=bind, checkfirst=checkfirst) - - def drop(self, bind=None, checkfirst=False): - """Issue DROP ddl for this type, if applicable.""" - - if bind is None: - bind = schema._bind_or_error(self) - t = self.dialect_impl(bind.dialect) - if t.__class__ is not self.__class__ and isinstance(t, SchemaType): - t.drop(bind=bind, checkfirst=checkfirst) - - def _on_table_create(self, target, bind, **kw): - t = self.dialect_impl(bind.dialect) - if t.__class__ is not self.__class__ and isinstance(t, SchemaType): - t._on_table_create(target, bind, **kw) - - def _on_table_drop(self, target, bind, **kw): - t = self.dialect_impl(bind.dialect) - if t.__class__ is not self.__class__ and isinstance(t, SchemaType): - t._on_table_drop(target, bind, **kw) - - def _on_metadata_create(self, target, bind, **kw): - t = self.dialect_impl(bind.dialect) - if t.__class__ is not self.__class__ and isinstance(t, SchemaType): - t._on_metadata_create(target, bind, **kw) - - def _on_metadata_drop(self, target, bind, **kw): - t = self.dialect_impl(bind.dialect) - if t.__class__ is not self.__class__ and isinstance(t, SchemaType): - t._on_metadata_drop(target, bind, **kw) - -class Enum(String, SchemaType): - """Generic Enum Type. - - The Enum type provides a set of possible string values which the - column is constrained towards. - - By default, uses the backend's native ENUM type if available, - else uses VARCHAR + a CHECK constraint. - - See also: - - :class:`~.postgresql.ENUM` - PostgreSQL-specific type, - which has additional functionality. - - """ - - __visit_name__ = 'enum' - - def __init__(self, *enums, **kw): - """Construct an enum. - - Keyword arguments which don't apply to a specific backend are ignored - by that backend. - - :param \*enums: string or unicode enumeration labels. If unicode - labels are present, the `convert_unicode` flag is auto-enabled. - - :param convert_unicode: Enable unicode-aware bind parameter and - result-set processing for this Enum's data. This is set - automatically based on the presence of unicode label strings. - - :param metadata: Associate this type directly with a ``MetaData`` - object. For types that exist on the target database as an - independent schema construct (Postgresql), this type will be - created and dropped within ``create_all()`` and ``drop_all()`` - operations. If the type is not associated with any ``MetaData`` - object, it will associate itself with each ``Table`` in which it is - used, and will be created when any of those individual tables are - created, after a check is performed for it's existence. The type is - only dropped when ``drop_all()`` is called for that ``Table`` - object's metadata, however. - - :param name: The name of this type. This is required for Postgresql - and any future supported database which requires an explicitly - named type, or an explicitly named constraint in order to generate - the type and/or a table that uses it. - - :param native_enum: Use the database's native ENUM type when - available. Defaults to True. When False, uses VARCHAR + check - constraint for all backends. - - :param schema: Schemaname of this type. For types that exist on the - target database as an independent schema construct (Postgresql), - this parameter specifies the named schema in which the type is - present. - - :param quote: Force quoting to be on or off on the type's name. If - left as the default of `None`, the usual schema-level "case - sensitive"/"reserved name" rules are used to determine if this - type's name should be quoted. - - """ - self.enums = enums - self.native_enum = kw.pop('native_enum', True) - convert_unicode = kw.pop('convert_unicode', None) - if convert_unicode is None: - for e in enums: - if isinstance(e, unicode): - convert_unicode = True - break - else: - convert_unicode = False - - if self.enums: - length = max(len(x) for x in self.enums) - else: - length = 0 - String.__init__(self, - length=length, - convert_unicode=convert_unicode, - ) - SchemaType.__init__(self, **kw) - - def __repr__(self): - return util.generic_repr(self, [ - ("native_enum", True), - ("name", None) - ]) - - def _should_create_constraint(self, compiler): - return not self.native_enum or \ - not compiler.dialect.supports_native_enum - - def _set_table(self, column, table): - if self.native_enum: - SchemaType._set_table(self, column, table) - - - e = schema.CheckConstraint( - column.in_(self.enums), - name=self.name, - _create_rule=util.portable_instancemethod( - self._should_create_constraint) - ) - table.append_constraint(e) - - def adapt(self, impltype, **kw): - if issubclass(impltype, Enum): - return impltype(name=self.name, - quote=self.quote, - schema=self.schema, - metadata=self.metadata, - convert_unicode=self.convert_unicode, - native_enum=self.native_enum, - *self.enums, - **kw - ) - else: - return super(Enum, self).adapt(impltype, **kw) - -class PickleType(MutableType, TypeDecorator): - """Holds Python objects, which are serialized using pickle. - - PickleType builds upon the Binary type to apply Python's - ``pickle.dumps()`` to incoming objects, and ``pickle.loads()`` on - the way out, allowing any pickleable Python object to be stored as - a serialized binary field. - - """ - - impl = LargeBinary - - def __init__(self, protocol=pickle.HIGHEST_PROTOCOL, - pickler=None, mutable=False, comparator=None): - """ - Construct a PickleType. - - :param protocol: defaults to ``pickle.HIGHEST_PROTOCOL``. - - :param pickler: defaults to cPickle.pickle or pickle.pickle if - cPickle is not available. May be any object with - pickle-compatible ``dumps` and ``loads`` methods. - - :param mutable: defaults to False; implements - :meth:`AbstractType.is_mutable`. When ``True``, incoming - objects will be compared against copies of themselves - using the Python "equals" operator, unless the - ``comparator`` argument is present. See - :class:`.MutableType` for details on "mutable" type - behavior. - - .. versionchanged:: 0.7.0 - Default changed from ``True``. - - .. note:: - - This functionality is now superseded by the - ``sqlalchemy.ext.mutable`` extension described in - :ref:`mutable_toplevel`. - - :param comparator: a 2-arg callable predicate used - to compare values of this type. If left as ``None``, - the Python "equals" operator is used to compare values. - - """ - self.protocol = protocol - self.pickler = pickler or pickle - self.mutable = mutable - self.comparator = comparator - super(PickleType, self).__init__() - - def __reduce__(self): - return PickleType, (self.protocol, - None, - self.mutable, - self.comparator) - - def bind_processor(self, dialect): - impl_processor = self.impl.bind_processor(dialect) - dumps = self.pickler.dumps - protocol = self.protocol - if impl_processor: - def process(value): - if value is not None: - value = dumps(value, protocol) - return impl_processor(value) - else: - def process(value): - if value is not None: - value = dumps(value, protocol) - return value - return process - - def result_processor(self, dialect, coltype): - impl_processor = self.impl.result_processor(dialect, coltype) - loads = self.pickler.loads - if impl_processor: - def process(value): - value = impl_processor(value) - if value is None: - return None - return loads(value) - else: - def process(value): - if value is None: - return None - return loads(value) - return process - - def copy_value(self, value): - if self.mutable: - return self.pickler.loads( - self.pickler.dumps(value, self.protocol)) - else: - return value - - def compare_values(self, x, y): - if self.comparator: - return self.comparator(x, y) - else: - return x == y - - def is_mutable(self): - """Return True if the target Python type is 'mutable'. - - When this method is overridden, :meth:`copy_value` should - also be supplied. The :class:`.MutableType` mixin - is recommended as a helper. - - """ - return self.mutable - - -class Boolean(TypeEngine, SchemaType): - """A bool datatype. - - Boolean typically uses BOOLEAN or SMALLINT on the DDL side, and on - the Python side deals in ``True`` or ``False``. - - """ - - __visit_name__ = 'boolean' - - def __init__(self, create_constraint=True, name=None): - """Construct a Boolean. - - :param create_constraint: defaults to True. If the boolean - is generated as an int/smallint, also create a CHECK constraint - on the table that ensures 1 or 0 as a value. - - :param name: if a CHECK constraint is generated, specify - the name of the constraint. - - """ - self.create_constraint = create_constraint - self.name = name - - def _should_create_constraint(self, compiler): - return not compiler.dialect.supports_native_boolean - - def _set_table(self, column, table): - if not self.create_constraint: - return - - e = schema.CheckConstraint( - column.in_([0, 1]), - name=self.name, - _create_rule=util.portable_instancemethod( - self._should_create_constraint) - ) - table.append_constraint(e) - - @property - def python_type(self): - return bool - - def bind_processor(self, dialect): - if dialect.supports_native_boolean: - return None - else: - return processors.boolean_to_int - - def result_processor(self, dialect, coltype): - if dialect.supports_native_boolean: - return None - else: - return processors.int_to_boolean - -class Interval(_DateAffinity, TypeDecorator): - """A type for ``datetime.timedelta()`` objects. - - The Interval type deals with ``datetime.timedelta`` objects. In - PostgreSQL, the native ``INTERVAL`` type is used; for others, the - value is stored as a date which is relative to the "epoch" - (Jan. 1, 1970). - - Note that the ``Interval`` type does not currently provide date arithmetic - operations on platforms which do not support interval types natively. Such - operations usually require transformation of both sides of the expression - (such as, conversion of both sides into integer epoch values first) which - currently is a manual procedure (such as via - :attr:`~sqlalchemy.sql.expression.func`). - - """ - - impl = DateTime - epoch = dt.datetime.utcfromtimestamp(0) - - def __init__(self, native=True, - second_precision=None, - day_precision=None): - """Construct an Interval object. - - :param native: when True, use the actual - INTERVAL type provided by the database, if - supported (currently Postgresql, Oracle). - Otherwise, represent the interval data as - an epoch value regardless. - - :param second_precision: For native interval types - which support a "fractional seconds precision" parameter, - i.e. Oracle and Postgresql - - :param day_precision: for native interval types which - support a "day precision" parameter, i.e. Oracle. - - """ - super(Interval, self).__init__() - self.native = native - self.second_precision = second_precision - self.day_precision = day_precision - - def adapt(self, cls, **kw): - if self.native and hasattr(cls, '_adapt_from_generic_interval'): - return cls._adapt_from_generic_interval(self, **kw) - else: - return self.__class__( - native=self.native, - second_precision=self.second_precision, - day_precision=self.day_precision, - **kw) - - @property - def python_type(self): - return dt.timedelta - - def bind_processor(self, dialect): - impl_processor = self.impl.bind_processor(dialect) - epoch = self.epoch - if impl_processor: - def process(value): - if value is not None: - value = epoch + value - return impl_processor(value) - else: - def process(value): - if value is not None: - value = epoch + value - return value - return process - - def result_processor(self, dialect, coltype): - impl_processor = self.impl.result_processor(dialect, coltype) - epoch = self.epoch - if impl_processor: - def process(value): - value = impl_processor(value) - if value is None: - return None - return value - epoch - else: - def process(value): - if value is None: - return None - return value - epoch - return process - - @util.memoized_property - def _expression_adaptations(self): - return { - operators.add:{ - Date:DateTime, - Interval:Interval, - DateTime:DateTime, - Time:Time, - }, - operators.sub:{ - Interval:Interval - }, - operators.mul:{ - Numeric:Interval - }, - operators.truediv: { - Numeric:Interval - }, - # Py2K - operators.div: { - Numeric:Interval - } - # end Py2K - } - - @property - def _type_affinity(self): - return Interval - - def _coerce_compared_value(self, op, value): - """See :meth:`.TypeEngine._coerce_compared_value` for a description.""" - - return self.impl._coerce_compared_value(op, value) - - -class REAL(Float): - """The SQL REAL type.""" - - __visit_name__ = 'REAL' - -class FLOAT(Float): - """The SQL FLOAT type.""" - - __visit_name__ = 'FLOAT' - -class NUMERIC(Numeric): - """The SQL NUMERIC type.""" - - __visit_name__ = 'NUMERIC' - - -class DECIMAL(Numeric): - """The SQL DECIMAL type.""" - - __visit_name__ = 'DECIMAL' - - -class INTEGER(Integer): - """The SQL INT or INTEGER type.""" - - __visit_name__ = 'INTEGER' -INT = INTEGER - - -class SMALLINT(SmallInteger): - """The SQL SMALLINT type.""" - - __visit_name__ = 'SMALLINT' - - -class BIGINT(BigInteger): - """The SQL BIGINT type.""" - - __visit_name__ = 'BIGINT' - -class TIMESTAMP(DateTime): - """The SQL TIMESTAMP type.""" - - __visit_name__ = 'TIMESTAMP' - - def get_dbapi_type(self, dbapi): - return dbapi.TIMESTAMP - -class DATETIME(DateTime): - """The SQL DATETIME type.""" - - __visit_name__ = 'DATETIME' - - -class DATE(Date): - """The SQL DATE type.""" - - __visit_name__ = 'DATE' - - -class TIME(Time): - """The SQL TIME type.""" - - __visit_name__ = 'TIME' - -class TEXT(Text): - """The SQL TEXT type.""" - - __visit_name__ = 'TEXT' - -class CLOB(Text): - """The CLOB type. - - This type is found in Oracle and Informix. - """ - - __visit_name__ = 'CLOB' - -class VARCHAR(String): - """The SQL VARCHAR type.""" - - __visit_name__ = 'VARCHAR' - -class NVARCHAR(Unicode): - """The SQL NVARCHAR type.""" - - __visit_name__ = 'NVARCHAR' - -class CHAR(String): - """The SQL CHAR type.""" - - __visit_name__ = 'CHAR' - - -class NCHAR(Unicode): - """The SQL NCHAR type.""" - - __visit_name__ = 'NCHAR' - - -class BLOB(LargeBinary): - """The SQL BLOB type.""" - - __visit_name__ = 'BLOB' - -class BINARY(_Binary): - """The SQL BINARY type.""" - - __visit_name__ = 'BINARY' - -class VARBINARY(_Binary): - """The SQL VARBINARY type.""" - - __visit_name__ = 'VARBINARY' - - -class BOOLEAN(Boolean): - """The SQL BOOLEAN type.""" - - __visit_name__ = 'BOOLEAN' - -NULLTYPE = NullType() -BOOLEANTYPE = Boolean() -STRINGTYPE = String() - -_type_map = { - str: String(), - # Py3K - #bytes : LargeBinary(), - # Py2K - unicode : Unicode(), - # end Py2K - int : Integer(), - float : Numeric(), - bool: BOOLEANTYPE, - decimal.Decimal : Numeric(), - dt.date : Date(), - dt.datetime : DateTime(), - dt.time : Time(), - dt.timedelta : Interval(), - NoneType: NULLTYPE -} + 'CLOB', 'BLOB', 'BINARY', 'VARBINARY', 'BOOLEAN', 'BIGINT', + 'SMALLINT', 'INTEGER', 'DATE', 'TIME', 'String', 'Integer', + 'SmallInteger', 'BigInteger', 'Numeric', 'Float', 'DateTime', + 'Date', 'Time', 'LargeBinary', 'Binary', 'Boolean', 'Unicode', + 'Concatenable', 'UnicodeText', 'PickleType', 'Interval', 'Enum'] + +from .sql.type_api import ( + adapt_type, + TypeEngine, + TypeDecorator, + Variant, + to_instance, + UserDefinedType +) +from .sql.sqltypes import ( + BIGINT, + BINARY, + BLOB, + BOOLEAN, + BigInteger, + Binary, + _Binary, + Boolean, + CHAR, + CLOB, + Concatenable, + DATE, + DATETIME, + DECIMAL, + Date, + DateTime, + Enum, + FLOAT, + Float, + INT, + INTEGER, + Integer, + Interval, + LargeBinary, + NCHAR, + NVARCHAR, + NullType, + NULLTYPE, + NUMERIC, + Numeric, + PickleType, + REAL, + SchemaType, + SMALLINT, + SmallInteger, + String, + STRINGTYPE, + TEXT, + TIME, + TIMESTAMP, + Text, + Time, + Unicode, + UnicodeText, + VARBINARY, + VARCHAR, + _type_map + ) diff --git a/libs/sqlalchemy/util/__init__.py b/libs/sqlalchemy/util/__init__.py index 8cb4c65b..eba64ed1 100644 --- a/libs/sqlalchemy/util/__init__.py +++ b/libs/sqlalchemy/util/__init__.py @@ -1,34 +1,45 @@ # util/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from compat import callable, cmp, reduce, defaultdict, py25_dict, \ - threading, py3k_warning, jython, pypy, cpython, win32, set_types, buffer, \ - pickle, update_wrapper, partial, md5_hex, decode_slice, dottedgetter,\ - parse_qsl, any, contextmanager, next +from .compat import callable, cmp, reduce, \ + threading, py3k, py33, py2k, jython, pypy, cpython, win32, \ + pickle, dottedgetter, parse_qsl, namedtuple, next, reraise, \ + raise_from_cause, text_type, string_types, int_types, binary_type, \ + quote_plus, with_metaclass, print_, itertools_filterfalse, u, ue, b,\ + unquote_plus, unquote, b64decode, b64encode, byte_buffer, itertools_filter,\ + iterbytes, StringIO, inspect_getargspec -from _collections import NamedTuple, ImmutableContainer, immutabledict, \ +from ._collections import KeyedTuple, ImmutableContainer, immutabledict, \ Properties, OrderedProperties, ImmutableProperties, OrderedDict, \ OrderedSet, IdentitySet, OrderedIdentitySet, column_set, \ column_dict, ordered_column_set, populate_column_dict, unique_list, \ UniqueAppender, PopulateDict, EMPTY_SET, to_list, to_set, \ - to_column_set, update_copy, flatten_iterator, WeakIdentityMapping, \ - LRUCache, ScopedRegistry, ThreadLocalRegistry + to_column_set, update_copy, flatten_iterator, \ + LRUCache, ScopedRegistry, ThreadLocalRegistry, WeakSequence, \ + coerce_generator_arg -from langhelpers import iterate_attributes, class_hierarchy, \ +from .langhelpers import iterate_attributes, class_hierarchy, \ portable_instancemethod, unbound_method_to_callable, \ getargspec_init, format_argspec_init, format_argspec_plus, \ get_func_kwargs, get_cls_kwargs, decorator, as_interface, \ - memoized_property, memoized_instancemethod, \ - reset_memoized, group_expirable_memoized_property, importlater, \ + memoized_property, memoized_instancemethod, md5_hex, \ + group_expirable_memoized_property, dependencies, decode_slice, \ monkeypatch_proxied_specials, asbool, bool_or_str, coerce_kw_type,\ duck_type_collection, assert_arg_type, symbol, dictlike_iteritems,\ classproperty, set_creation_order, warn_exception, warn, NoneType,\ constructor_copy, methods_equivalent, chop_traceback, asint,\ - generic_repr, counter + generic_repr, counter, PluginLoader, hybridmethod, safe_reraise,\ + get_callable_argspec, only_once -from deprecations import warn_deprecated, warn_pending_deprecation, \ - deprecated, pending_deprecation +from .deprecations import warn_deprecated, warn_pending_deprecation, \ + deprecated, pending_deprecation, inject_docstring_text +# things that used to be not always available, +# but are now as of current support Python versions +from collections import defaultdict +from functools import partial +from functools import update_wrapper +from contextlib import contextmanager diff --git a/libs/sqlalchemy/util/_collections.py b/libs/sqlalchemy/util/_collections.py index 5a09dca6..c0a24ba4 100644 --- a/libs/sqlalchemy/util/_collections.py +++ b/libs/sqlalchemy/util/_collections.py @@ -1,44 +1,111 @@ # util/_collections.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """Collection classes and helpers.""" -import sys -import itertools +from __future__ import absolute_import import weakref import operator -from langhelpers import symbol -from compat import time_func, threading +from .compat import threading, itertools_filterfalse +from . import py2k +import types EMPTY_SET = frozenset() -class NamedTuple(tuple): - """tuple() subclass that adds labeled names. +class KeyedTuple(tuple): + """``tuple`` subclass that adds labeled names. - Is also pickleable. + E.g.:: + + >>> k = KeyedTuple([1, 2, 3], labels=["one", "two", "three"]) + >>> k.one + 1 + >>> k.two + 2 + + Result rows returned by :class:`.Query` that contain multiple + ORM entities and/or column expressions make use of this + class to return rows. + + The :class:`.KeyedTuple` exhibits similar behavior to the + ``collections.namedtuple()`` construct provided in the Python + standard library, however is architected very differently. + Unlike ``collections.namedtuple()``, :class:`.KeyedTuple` is + does not rely on creation of custom subtypes in order to represent + a new series of keys, instead each :class:`.KeyedTuple` instance + receives its list of keys in place. The subtype approach + of ``collections.namedtuple()`` introduces significant complexity + and performance overhead, which is not necessary for the + :class:`.Query` object's use case. + + .. versionchanged:: 0.8 + Compatibility methods with ``collections.namedtuple()`` have been + added including :attr:`.KeyedTuple._fields` and + :meth:`.KeyedTuple._asdict`. + + .. seealso:: + + :ref:`ormtutorial_querying` """ def __new__(cls, vals, labels=None): t = tuple.__new__(cls, vals) + t._labels = [] if labels: t.__dict__.update(zip(labels, vals)) t._labels = labels return t def keys(self): + """Return a list of string key names for this :class:`.KeyedTuple`. + + .. seealso:: + + :attr:`.KeyedTuple._fields` + + """ + return [l for l in self._labels if l is not None] + @property + def _fields(self): + """Return a tuple of string key names for this :class:`.KeyedTuple`. + + This method provides compatibility with ``collections.namedtuple()``. + + .. versionadded:: 0.8 + + .. seealso:: + + :meth:`.KeyedTuple.keys` + + """ + return tuple(self.keys()) + + def _asdict(self): + """Return the contents of this :class:`.KeyedTuple` as a dictionary. + + This method provides compatibility with ``collections.namedtuple()``, + with the exception that the dictionary returned is **not** ordered. + + .. versionadded:: 0.8 + + """ + return dict((key, self.__dict__[key]) for key in self.keys()) + + class ImmutableContainer(object): def _immutable(self, *arg, **kw): raise TypeError("%s object is immutable" % self.__class__.__name__) __delitem__ = __setitem__ = __setattr__ = _immutable + class immutabledict(ImmutableContainer, dict): clear = pop = popitem = setdefault = \ @@ -66,6 +133,7 @@ class immutabledict(ImmutableContainer, dict): def __repr__(self): return "immutabledict(%s)" % dict.__repr__(self) + class Properties(object): """Provide a __getattr__/__setattr__ interface over a dict.""" @@ -76,7 +144,7 @@ class Properties(object): return len(self._data) def __iter__(self): - return self._data.itervalues() + return iter(list(self._data.values())) def __add__(self, other): return list(self) + list(other) @@ -123,7 +191,13 @@ class Properties(object): return default def keys(self): - return self._data.keys() + return list(self._data) + + def values(self): + return list(self._data.values()) + + def items(self): + return list(self._data.items()) def has_key(self, key): return key in self._data @@ -131,6 +205,7 @@ class Properties(object): def clear(self): self._data.clear() + class OrderedProperties(Properties): """Provide a __getattr__/__setattr__ interface with an OrderedDict as backing store.""" @@ -187,23 +262,55 @@ class OrderedDict(dict): def __iter__(self): return iter(self._list) - def values(self): - return [self[key] for key in self._list] - def itervalues(self): - return iter([self[key] for key in self._list]) + if py2k: + def values(self): + return [self[key] for key in self._list] - def keys(self): - return list(self._list) + def keys(self): + return self._list - def iterkeys(self): - return iter(self.keys()) + def itervalues(self): + return iter([self[key] for key in self._list]) - def items(self): - return [(key, self[key]) for key in self.keys()] + def iterkeys(self): + return iter(self) + + def iteritems(self): + return iter(self.items()) + + def items(self): + return [(key, self[key]) for key in self._list] + else: + def values(self): + #return (self[key] for key in self) + return (self[key] for key in self._list) + + def keys(self): + #return iter(self) + return iter(self._list) + + def items(self): + #return ((key, self[key]) for key in self) + return ((key, self[key]) for key in self._list) + + _debug_iter = False + if _debug_iter: + # normally disabled to reduce function call + # overhead + def __iter__(self): + len_ = len(self._list) + for item in self._list: + yield item + assert len_ == len(self._list), \ + "Dictionary changed size during iteration" + def values(self): + return (self[key] for key in self) + def keys(self): + return iter(self) + def items(self): + return ((key, self[key]) for key in self) - def iteritems(self): - return iter(self.items()) def __setitem__(self, key, object): if key not in self: @@ -231,6 +338,7 @@ class OrderedDict(dict): self._list.remove(item[0]) return item + class OrderedSet(set): def __init__(self, d=None): set.__init__(self) @@ -314,22 +422,22 @@ class OrderedSet(set): def intersection_update(self, other): other = set(other) set.intersection_update(self, other) - self._list = [ a for a in self._list if a in other] + self._list = [a for a in self._list if a in other] return self __iand__ = intersection_update def symmetric_difference_update(self, other): set.symmetric_difference_update(self, other) - self._list = [ a for a in self._list if a in self] - self._list += [ a for a in other._list if a in self] + self._list = [a for a in self._list if a in self] + self._list += [a for a in other._list if a in self] return self __ixor__ = symmetric_difference_update def difference_update(self, other): set.difference_update(self, other) - self._list = [ a for a in self._list if a in self] + self._list = [a for a in self._list if a in self] return self __isub__ = difference_update @@ -376,9 +484,6 @@ class IdentitySet(object): def clear(self): self._members.clear() - def __sub__(self, other): - return self.difference(other) - def __cmp__(self, other): raise TypeError('cannot compare sets using cmp()') @@ -399,8 +504,8 @@ class IdentitySet(object): if len(self) > len(other): return False - for m in itertools.ifilterfalse(other._members.__contains__, - self._members.iterkeys()): + for m in itertools_filterfalse(other._members.__contains__, + iter(self._members.keys())): return False return True @@ -420,8 +525,8 @@ class IdentitySet(object): if len(self) < len(other): return False - for m in itertools.ifilterfalse(self._members.__contains__, - other._members.iterkeys()): + for m in itertools_filterfalse(self._members.__contains__, + iter(other._members.keys())): return False return True @@ -438,8 +543,9 @@ class IdentitySet(object): def union(self, iterable): result = type(self)() # testlib.pragma exempt:__hash__ - result._members.update( - self._working_set(self._member_id_tuples()).union(_iter_id(iterable))) + members = self._member_id_tuples() + other = _iter_id(iterable) + result._members.update(self._working_set(members).union(other)) return result def __or__(self, other): @@ -459,8 +565,9 @@ class IdentitySet(object): def difference(self, iterable): result = type(self)() # testlib.pragma exempt:__hash__ - result._members.update( - self._working_set(self._member_id_tuples()).difference(_iter_id(iterable))) + members = self._member_id_tuples() + other = _iter_id(iterable) + result._members.update(self._working_set(members).difference(other)) return result def __sub__(self, other): @@ -480,8 +587,9 @@ class IdentitySet(object): def intersection(self, iterable): result = type(self)() # testlib.pragma exempt:__hash__ - result._members.update( - self._working_set(self._member_id_tuples()).intersection(_iter_id(iterable))) + members = self._member_id_tuples() + other = _iter_id(iterable) + result._members.update(self._working_set(members).intersection(other)) return result def __and__(self, other): @@ -501,12 +609,14 @@ class IdentitySet(object): def symmetric_difference(self, iterable): result = type(self)() # testlib.pragma exempt:__hash__ + members = self._member_id_tuples() + other = _iter_id(iterable) result._members.update( - self._working_set(self._member_id_tuples()).symmetric_difference(_iter_id(iterable))) + self._working_set(members).symmetric_difference(other)) return result def _member_id_tuples(self): - return ((id(v), v) for v in self._members.itervalues()) + return ((id(v), v) for v in self._members.values()) def __xor__(self, other): if not isinstance(other, IdentitySet): @@ -523,7 +633,7 @@ class IdentitySet(object): return self def copy(self): - return type(self)(self._members.itervalues()) + return type(self)(iter(self._members.values())) __copy__ = copy @@ -531,13 +641,41 @@ class IdentitySet(object): return len(self._members) def __iter__(self): - return self._members.itervalues() + return iter(self._members.values()) def __hash__(self): raise TypeError('set objects are unhashable') def __repr__(self): - return '%s(%r)' % (type(self).__name__, self._members.values()) + return '%s(%r)' % (type(self).__name__, list(self._members.values())) + + +class WeakSequence(object): + def __init__(self, __elements=()): + self._storage = [ + weakref.ref(element, self._remove) for element in __elements + ] + + def append(self, item): + self._storage.append(weakref.ref(item, self._remove)) + + def _remove(self, ref): + self._storage.remove(ref) + + def __len__(self): + return len(self._storage) + + def __iter__(self): + return (obj for obj in + (ref() for ref in self._storage) if obj is not None) + + def __getitem__(self, index): + try: + obj = self._storage[index] + except KeyError: + raise IndexError("Index %s out of range" % index) + else: + return obj() class OrderedIdentitySet(IdentitySet): @@ -556,37 +694,25 @@ class OrderedIdentitySet(IdentitySet): self.add(o) -if sys.version_info >= (2, 5): - class PopulateDict(dict): - """A dict which populates missing values via a creation function. +class PopulateDict(dict): + """A dict which populates missing values via a creation function. - Note the creation function takes a key, unlike - collections.defaultdict. + Note the creation function takes a key, unlike + collections.defaultdict. - """ + """ - def __init__(self, creator): - self.creator = creator + def __init__(self, creator): + self.creator = creator - def __missing__(self, key): - self[key] = val = self.creator(key) - return val -else: - class PopulateDict(dict): - """A dict which populates missing values via a creation function.""" + def __missing__(self, key): + self[key] = val = self.creator(key) + return val - def __init__(self, creator): - self.creator = creator - - def __getitem__(self, key): - try: - return dict.__getitem__(self, key) - except KeyError: - self[key] = value = self.creator(key) - return value - -# define collections that are capable of storing +# Define collections that are capable of storing # ColumnElement objects as hashable keys/elements. +# At this point, these are mostly historical, things +# used to be more complicated. column_set = set column_dict = dict ordered_column_set = OrderedSet @@ -603,6 +729,7 @@ def unique_list(seq, hashfunc=None): if hashfunc(x) not in seen and not seen.__setitem__(hashfunc(x), True)] + class UniqueAppender(object): """Appends items to a collection ensuring uniqueness. @@ -629,6 +756,12 @@ class UniqueAppender(object): def __iter__(self): return iter(self.data) +def coerce_generator_arg(arg): + if len(arg) == 1 and isinstance(arg[0], types.GeneratorType): + return list(arg[0]) + else: + return arg + def to_list(x, default=None): if x is None: return default @@ -637,6 +770,7 @@ def to_list(x, default=None): else: return x + def to_set(x): if x is None: return set() @@ -645,6 +779,7 @@ def to_set(x): else: return x + def to_column_set(x): if x is None: return column_set() @@ -653,6 +788,7 @@ def to_column_set(x): else: return x + def update_copy(d, _new=None, **kw): """Copy the given dict and update with the given values.""" @@ -662,104 +798,19 @@ def update_copy(d, _new=None, **kw): d.update(**kw) return d + def flatten_iterator(x): """Given an iterator of which further sub-elements may also be iterators, flatten the sub-elements into a single iterator. """ for elem in x: - if not isinstance(elem, basestring) and hasattr(elem, '__iter__'): + if not isinstance(elem, str) and hasattr(elem, '__iter__'): for y in flatten_iterator(elem): yield y else: yield elem -class WeakIdentityMapping(weakref.WeakKeyDictionary): - """A WeakKeyDictionary with an object identity index. - - Adds a .by_id dictionary to a regular WeakKeyDictionary. Trades - performance during mutation operations for accelerated lookups by id(). - - The usual cautions about weak dictionaries and iteration also apply to - this subclass. - - """ - _none = symbol('none') - - def __init__(self): - weakref.WeakKeyDictionary.__init__(self) - self.by_id = {} - self._weakrefs = {} - - def __setitem__(self, object, value): - oid = id(object) - self.by_id[oid] = value - if oid not in self._weakrefs: - self._weakrefs[oid] = self._ref(object) - weakref.WeakKeyDictionary.__setitem__(self, object, value) - - def __delitem__(self, object): - del self._weakrefs[id(object)] - del self.by_id[id(object)] - weakref.WeakKeyDictionary.__delitem__(self, object) - - def setdefault(self, object, default=None): - value = weakref.WeakKeyDictionary.setdefault(self, object, default) - oid = id(object) - if value is default: - self.by_id[oid] = default - if oid not in self._weakrefs: - self._weakrefs[oid] = self._ref(object) - return value - - def pop(self, object, default=_none): - if default is self._none: - value = weakref.WeakKeyDictionary.pop(self, object) - else: - value = weakref.WeakKeyDictionary.pop(self, object, default) - if id(object) in self.by_id: - del self._weakrefs[id(object)] - del self.by_id[id(object)] - return value - - def popitem(self): - item = weakref.WeakKeyDictionary.popitem(self) - oid = id(item[0]) - del self._weakrefs[oid] - del self.by_id[oid] - return item - - def clear(self): - # Py2K - # in 3k, MutableMapping calls popitem() - self._weakrefs.clear() - self.by_id.clear() - # end Py2K - weakref.WeakKeyDictionary.clear(self) - - def update(self, *a, **kw): - raise NotImplementedError - - def _cleanup(self, wr, key=None): - if key is None: - key = wr.key - try: - del self._weakrefs[key] - except (KeyError, AttributeError): # pragma: no cover - pass # pragma: no cover - try: - del self.by_id[key] - except (KeyError, AttributeError): # pragma: no cover - pass # pragma: no cover - - class _keyed_weakref(weakref.ref): - def __init__(self, object, callback): - weakref.ref.__init__(self, object, callback) - self.key = id(object) - - def _ref(self, object): - return self._keyed_weakref(object, self._cleanup) - class LRUCache(dict): """Dictionary with 'squishy' removal of least @@ -869,6 +920,7 @@ class ScopedRegistry(object): except KeyError: pass + class ThreadLocalRegistry(ScopedRegistry): """A :class:`.ScopedRegistry` that uses a ``threading.local()`` variable for storage. @@ -897,9 +949,9 @@ class ThreadLocalRegistry(ScopedRegistry): except AttributeError: pass + def _iter_id(iterable): """Generator: ((id(o), o) for o in iterable).""" for item in iterable: yield id(item), item - diff --git a/libs/sqlalchemy/util/compat.py b/libs/sqlalchemy/util/compat.py index 18ea2815..f1346406 100644 --- a/libs/sqlalchemy/util/compat.py +++ b/libs/sqlalchemy/util/compat.py @@ -1,5 +1,5 @@ # util/compat.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,45 +8,24 @@ import sys - try: import threading except ImportError: import dummy_threading as threading +py33 = sys.version_info >= (3, 3) py32 = sys.version_info >= (3, 2) -py3k_warning = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0) +py3k = sys.version_info >= (3, 0) +py2k = sys.version_info < (3, 0) jython = sys.platform.startswith('java') pypy = hasattr(sys, 'pypy_version_info') win32 = sys.platform.startswith('win') cpython = not pypy and not jython # TODO: something better for this ? -if py3k_warning: - set_types = set -elif sys.version_info < (2, 6): - import sets - set_types = set, sets.Set -else: - # 2.6 deprecates sets.Set, but we still need to be able to detect them - # in user code and as return values from DB-APIs - ignore = ('ignore', None, DeprecationWarning, None, 0) - import warnings - try: - warnings.filters.insert(0, ignore) - except Exception: - import sets - else: - import sets - warnings.filters.remove(ignore) +import collections +next = next - set_types = set, sets.Set - -if sys.version_info < (2, 6): - def next(iter): - return iter.next() -else: - next = next -if py3k_warning: +if py3k: import pickle else: try: @@ -54,146 +33,121 @@ else: except ImportError: import pickle -# a controversial feature, required by MySQLdb currently -def buffer(x): - return x +ArgSpec = collections.namedtuple("ArgSpec", + ["args", "varargs", "keywords", "defaults"]) -# Py2K -buffer = buffer -# end Py2K +if py3k: + import builtins -try: - from contextlib import contextmanager -except ImportError: - def contextmanager(fn): - return fn + from inspect import getfullargspec as inspect_getfullargspec + from urllib.parse import quote_plus, unquote_plus, parse_qsl, quote, unquote + import configparser + from io import StringIO -try: - from functools import update_wrapper -except ImportError: - def update_wrapper(wrapper, wrapped, - assigned=('__doc__', '__module__', '__name__'), - updated=('__dict__',)): - for attr in assigned: - setattr(wrapper, attr, getattr(wrapped, attr)) - for attr in updated: - getattr(wrapper, attr).update(getattr(wrapped, attr, ())) - return wrapper + from io import BytesIO as byte_buffer -try: - from functools import partial -except ImportError: - def partial(func, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*(args + fargs), **newkeywords) - return newfunc + def inspect_getargspec(func): + return ArgSpec( + *inspect_getfullargspec(func)[0:4] + ) + string_types = str, + binary_type = bytes + text_type = str + int_types = int, + iterbytes = iter -if sys.version_info < (2, 6): - # emits a nasty deprecation warning - # in newer pythons - from cgi import parse_qsl -else: - from urlparse import parse_qsl + def u(s): + return s -# Py3K -#from inspect import getfullargspec as inspect_getfullargspec -# Py2K -from inspect import getargspec as inspect_getfullargspec -# end Py2K + def ue(s): + return s + + def b(s): + return s.encode("latin-1") + + if py32: + callable = callable + else: + def callable(fn): + return hasattr(fn, '__call__') -if py3k_warning: - # they're bringing it back in 3.2. brilliant ! - def callable(fn): - return hasattr(fn, '__call__') def cmp(a, b): return (a > b) - (a < b) from functools import reduce + + print_ = getattr(builtins, "print") + + import_ = getattr(builtins, '__import__') + + import itertools + itertools_filterfalse = itertools.filterfalse + itertools_filter = filter + itertools_imap = map + + import base64 + def b64encode(x): + return base64.b64encode(x).decode('ascii') + def b64decode(x): + return base64.b64decode(x.encode('ascii')) + else: + from inspect import getargspec as inspect_getfullargspec + inspect_getargspec = inspect_getfullargspec + from urllib import quote_plus, unquote_plus, quote, unquote + from urlparse import parse_qsl + import ConfigParser as configparser + from StringIO import StringIO + from cStringIO import StringIO as byte_buffer + + string_types = basestring, + binary_type = str + text_type = unicode + int_types = int, long + def iterbytes(buf): + return (ord(byte) for byte in buf) + + def u(s): + # this differs from what six does, which doesn't support non-ASCII + # strings - we only use u() with + # literal source strings, and all our source files with non-ascii + # in them (all are tests) are utf-8 encoded. + return unicode(s, "utf-8") + + def ue(s): + return unicode(s, "unicode_escape") + + def b(s): + return s + + def import_(*args): + if len(args) == 4: + args = args[0:3] + ([str(arg) for arg in args[3]],) + return __import__(*args) + callable = callable cmp = cmp reduce = reduce -try: - from collections import defaultdict -except ImportError: - class defaultdict(dict): - def __init__(self, default_factory=None, *a, **kw): - if (default_factory is not None and - not hasattr(default_factory, '__call__')): - raise TypeError('first argument must be callable') - dict.__init__(self, *a, **kw) - self.default_factory = default_factory - def __getitem__(self, key): - try: - return dict.__getitem__(self, key) - except KeyError: - return self.__missing__(key) - def __missing__(self, key): - if self.default_factory is None: - raise KeyError(key) - self[key] = value = self.default_factory() - return value - def __reduce__(self): - if self.default_factory is None: - args = tuple() - else: - args = self.default_factory, - return type(self), args, None, None, self.iteritems() - def copy(self): - return self.__copy__() - def __copy__(self): - return type(self)(self.default_factory, self) - def __deepcopy__(self, memo): - import copy - return type(self)(self.default_factory, - copy.deepcopy(self.items())) - def __repr__(self): - return 'defaultdict(%s, %s)' % (self.default_factory, - dict.__repr__(self)) + import base64 + b64encode = base64.b64encode + b64decode = base64.b64decode + def print_(*args, **kwargs): + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + for arg in enumerate(args): + if not isinstance(arg, basestring): + arg = str(arg) + fp.write(arg) -# find or create a dict implementation that supports __missing__ -class _probe(dict): - def __missing__(self, key): - return 1 + import itertools + itertools_filterfalse = itertools.ifilterfalse + itertools_filter = itertools.ifilter + itertools_imap = itertools.imap -try: - try: - _probe()['missing'] - py25_dict = dict - except KeyError: - class py25_dict(dict): - def __getitem__(self, key): - try: - return dict.__getitem__(self, key) - except KeyError: - try: - missing = self.__missing__ - except AttributeError: - raise KeyError(key) - else: - return missing(key) -finally: - del _probe - - -try: - import hashlib - _md5 = hashlib.md5 -except ImportError: - import md5 - _md5 = md5.new - -def md5_hex(x): - # Py3K - #x = x.encode('utf-8') - m = _md5() - m.update(x) - return m.hexdigest() import time if win32 or jython: @@ -201,43 +155,61 @@ if win32 or jython: else: time_func = time.time -if sys.version_info >= (2, 5): - any = any +from collections import namedtuple +from operator import attrgetter as dottedgetter + + +if py3k: + def reraise(tp, value, tb=None, cause=None): + if cause is not None: + value.__cause__ = cause + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + def raise_from_cause(exception, exc_info=None): + if exc_info is None: + exc_info = sys.exc_info() + exc_type, exc_value, exc_tb = exc_info + reraise(type(exception), exception, tb=exc_tb, cause=exc_value) else: - def any(iterator): - for item in iterator: - if bool(item): - return True + exec("def reraise(tp, value, tb=None, cause=None):\n" + " raise tp, value, tb\n") + + def raise_from_cause(exception, exc_info=None): + # not as nice as that of Py3K, but at least preserves + # the code line where the issue occurred + if exc_info is None: + exc_info = sys.exc_info() + exc_type, exc_value, exc_tb = exc_info + reraise(type(exception), exception, tb=exc_tb) + +if py3k: + exec_ = getattr(builtins, 'exec') +else: + def exec_(func_text, globals_, lcl=None): + if lcl is None: + exec('exec func_text in globals_') else: - return False - -if sys.version_info >= (2, 5): - def decode_slice(slc): - """decode a slice object as sent to __getitem__. - - takes into account the 2.5 __index__() method, basically. - - """ - ret = [] - for x in slc.start, slc.stop, slc.step: - if hasattr(x, '__index__'): - x = x.__index__() - ret.append(x) - return tuple(ret) -else: - def decode_slice(slc): - return (slc.start, slc.stop, slc.step) - -if sys.version_info >= (2, 6): - from operator import attrgetter as dottedgetter -else: - def dottedgetter(attr): - def g(obj): - for name in attr.split("."): - obj = getattr(obj, name) - return obj - return g + exec('exec func_text in globals_, lcl') -import decimal +def with_metaclass(meta, *bases): + """Create a base class with a metaclass. + + Drops the middle class upon creation. + + Source: http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/ + + """ + + class metaclass(meta): + __call__ = type.__call__ + __init__ = type.__init__ + def __new__(cls, name, this_bases, d): + if this_bases is None: + return type.__new__(cls, name, (), d) + return meta(name, bases, d) + return metaclass('temporary_class', None, {}) + diff --git a/libs/sqlalchemy/util/deprecations.py b/libs/sqlalchemy/util/deprecations.py index 330d35db..c8854dc3 100644 --- a/libs/sqlalchemy/util/deprecations.py +++ b/libs/sqlalchemy/util/deprecations.py @@ -1,5 +1,5 @@ # util/deprecations.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,17 +7,20 @@ """Helpers related to deprecation of functions, methods, classes, other functionality.""" -from sqlalchemy import exc +from .. import exc import warnings import re -from langhelpers import decorator +from .langhelpers import decorator + def warn_deprecated(msg, stacklevel=3): warnings.warn(msg, exc.SADeprecationWarning, stacklevel=stacklevel) + def warn_pending_deprecation(msg, stacklevel=3): warnings.warn(msg, exc.SAPendingDeprecationWarning, stacklevel=stacklevel) + def deprecated(version, message=None, add_deprecation_to_docstring=True): """Decorates a function and issues a deprecation warning on use. @@ -47,6 +50,7 @@ def deprecated(version, message=None, add_deprecation_to_docstring=True): message % dict(func=fn.__name__), header) return decorate + def pending_deprecation(version, message=None, add_deprecation_to_docstring=True): """Decorates a function and issues a pending deprecation warning on use. @@ -80,6 +84,7 @@ def pending_deprecation(version, message=None, message % dict(func=fn.__name__), header) return decorate + def _sanitize_restructured_text(text): def repl(m): type_, name = m.group(1, 2) @@ -102,17 +107,37 @@ def _decorate_with_warning(func, wtype, message, docstring_header=None): doc = func.__doc__ is not None and func.__doc__ or '' if docstring_header is not None: docstring_header %= dict(func=func.__name__) - docs = doc and doc.expandtabs().split('\n') or [] - indent = '' - for line in docs[1:]: - text = line.lstrip() - if text: - indent = line[0:len(line) - len(text)] - break - point = min(len(docs), 1) - docs.insert(point, '\n' + indent + docstring_header.rstrip()) - doc = '\n'.join(docs) + + doc = inject_docstring_text(doc, docstring_header, 1) decorated = warned(func) decorated.__doc__ = doc return decorated + +import textwrap + +def _dedent_docstring(text): + split_text = text.split("\n", 1) + if len(split_text) == 1: + return text + else: + firstline, remaining = split_text + if not firstline.startswith(" "): + return firstline + "\n" + textwrap.dedent(remaining) + else: + return textwrap.dedent(text) + +def inject_docstring_text(doctext, injecttext, pos): + doctext = _dedent_docstring(doctext or "") + lines = doctext.split('\n') + injectlines = textwrap.dedent(injecttext).split("\n") + if injectlines[0]: + injectlines.insert(0, "") + + blanks = [num for num, line in enumerate(lines) if not line.strip()] + blanks.insert(0, 0) + + inject_pos = blanks[min(pos, len(blanks) - 1)] + + lines = lines[0:inject_pos] + injectlines + lines[inject_pos:] + return "\n".join(lines) diff --git a/libs/sqlalchemy/util/langhelpers.py b/libs/sqlalchemy/util/langhelpers.py index b7c5132d..b6ca7eb2 100644 --- a/libs/sqlalchemy/util/langhelpers.py +++ b/libs/sqlalchemy/util/langhelpers.py @@ -1,5 +1,5 @@ # util/langhelpers.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -15,15 +15,69 @@ import re import sys import types import warnings -from compat import update_wrapper, set_types, threading, callable, inspect_getfullargspec, py3k_warning -from sqlalchemy import exc +from functools import update_wrapper +from .. import exc +import hashlib +from . import compat +from . import _collections + +def md5_hex(x): + if compat.py3k: + x = x.encode('utf-8') + m = hashlib.md5() + m.update(x) + return m.hexdigest() + +class safe_reraise(object): + """Reraise an exception after invoking some + handler code. + + Stores the existing exception info before + invoking so that it is maintained across a potential + coroutine context switch. + + e.g.:: + + try: + sess.commit() + except: + with safe_reraise(): + sess.rollback() + + """ + + def __enter__(self): + self._exc_info = sys.exc_info() + + def __exit__(self, type_, value, traceback): + # see #2703 for notes + if type_ is None: + exc_type, exc_value, exc_tb = self._exc_info + self._exc_info = None # remove potential circular references + compat.reraise(exc_type, exc_value, exc_tb) + else: + self._exc_info = None # remove potential circular references + compat.reraise(type_, value, traceback) + +def decode_slice(slc): + """decode a slice object as sent to __getitem__. + + takes into account the 2.5 __index__() method, basically. + + """ + ret = [] + for x in slc.start, slc.stop, slc.step: + if hasattr(x, '__index__'): + x = x.__index__() + ret.append(x) + return tuple(ret) def _unique_symbols(used, *bases): used = set(used) for base in bases: pool = itertools.chain((base,), - itertools.imap(lambda i: base + str(i), - xrange(1000))) + compat.itertools_imap(lambda i: base + str(i), + range(1000))) for sym in pool: if sym not in used: used.add(sym) @@ -32,28 +86,115 @@ def _unique_symbols(used, *bases): else: raise NameError("exhausted namespace for symbol base %s" % base) + def decorator(target): """A signature-matching decorator factory.""" def decorate(fn): if not inspect.isfunction(fn): raise Exception("not a decoratable function") - spec = inspect_getfullargspec(fn) - names = tuple(spec[0]) + spec[1:3] + (fn.func_name,) + spec = compat.inspect_getfullargspec(fn) + names = tuple(spec[0]) + spec[1:3] + (fn.__name__,) targ_name, fn_name = _unique_symbols(names, 'target', 'fn') metadata = dict(target=targ_name, fn=fn_name) metadata.update(format_argspec_plus(spec, grouped=False)) - - code = 'lambda %(args)s: %(target)s(%(fn)s, %(apply_kw)s)' % ( - metadata) - decorated = eval(code, {targ_name:target, fn_name:fn}) - decorated.func_defaults = getattr(fn, 'im_func', fn).func_defaults + metadata['name'] = fn.__name__ + code = """\ +def %(name)s(%(args)s): + return %(target)s(%(fn)s, %(apply_kw)s) +""" % metadata + decorated = _exec_code_in_env(code, + {targ_name: target, fn_name: fn}, + fn.__name__) + decorated.__defaults__ = getattr(fn, 'im_func', fn).__defaults__ return update_wrapper(decorated, fn) return update_wrapper(decorate, target) +def _exec_code_in_env(code, env, fn_name): + exec(code, env) + return env[fn_name] -def get_cls_kwargs(cls): +def public_factory(target, location): + """Produce a wrapping function for the given cls or classmethod. + + Rationale here is so that the __init__ method of the + class can serve as documentation for the function. + + """ + if isinstance(target, type): + fn = target.__init__ + callable_ = target + doc = "Construct a new :class:`.%s` object. \n\n"\ + "This constructor is mirrored as a public API function; see :func:`~%s` "\ + "for a full usage and argument description." % ( + target.__name__, location, ) + else: + fn = callable_ = target + doc = "This function is mirrored; see :func:`~%s` "\ + "for a description of arguments." % location + + location_name = location.split(".")[-1] + spec = compat.inspect_getfullargspec(fn) + del spec[0][0] + metadata = format_argspec_plus(spec, grouped=False) + metadata['name'] = location_name + code = """\ +def %(name)s(%(args)s): + return cls(%(apply_kw)s) +""" % metadata + env = {'cls': callable_, 'symbol': symbol} + exec(code, env) + decorated = env[location_name] + decorated.__doc__ = fn.__doc__ + if compat.py2k or hasattr(fn, '__func__'): + fn.__func__.__doc__ = doc + else: + fn.__doc__ = doc + return decorated + + +class PluginLoader(object): + + def __init__(self, group, auto_fn=None): + self.group = group + self.impls = {} + self.auto_fn = auto_fn + + def load(self, name): + if name in self.impls: + return self.impls[name]() + + if self.auto_fn: + loader = self.auto_fn(name) + if loader: + self.impls[name] = loader + return loader() + + try: + import pkg_resources + except ImportError: + pass + else: + for impl in pkg_resources.iter_entry_points( + self.group, name): + self.impls[name] = impl.load + return impl.load() + + raise exc.ArgumentError( + "Can't load plugin: %s:%s" % + (self.group, name)) + + def register(self, name, modulepath, objname): + def load(): + mod = compat.import_(modulepath) + for token in modulepath.split(".")[1:]: + mod = getattr(mod, token) + return getattr(mod, objname) + self.impls[name] = load + + +def get_cls_kwargs(cls, _set=None): """Return the full set of inherited kwargs for the given `cls`. Probes a class's __init__ method, collecting all named arguments. If the @@ -65,48 +206,50 @@ def get_cls_kwargs(cls): No anonymous tuple arguments please ! """ + toplevel = _set == None + if toplevel: + _set = set() - for c in cls.__mro__: - if '__init__' in c.__dict__: - stack = set([c]) - break - else: - return [] + ctr = cls.__dict__.get('__init__', False) - args = set() - while stack: - class_ = stack.pop() - ctr = class_.__dict__.get('__init__', False) - if (not ctr or - not isinstance(ctr, types.FunctionType) or - not isinstance(ctr.func_code, types.CodeType)): - stack.update(class_.__bases__) - continue - - # this is shorthand for - # names, _, has_kw, _ = inspect.getargspec(ctr) + has_init = ctr and isinstance(ctr, types.FunctionType) and \ + isinstance(ctr.__code__, types.CodeType) + if has_init: names, has_kw = inspect_func_args(ctr) - args.update(names) - if has_kw: - stack.update(class_.__bases__) - args.discard('self') - return args + _set.update(names) + + if not has_kw and not toplevel: + return None + + if not has_init or has_kw: + for c in cls.__bases__: + if get_cls_kwargs(c, _set) is None: + break + + _set.discard('self') + return _set + + try: + # TODO: who doesn't have this constant? from inspect import CO_VARKEYWORDS + def inspect_func_args(fn): - co = fn.func_code + co = fn.__code__ nargs = co.co_argcount names = co.co_varnames args = list(names[:nargs]) has_kw = bool(co.co_flags & CO_VARKEYWORDS) return args, has_kw + except ImportError: def inspect_func_args(fn): names, _, has_kw, _ = inspect.getargspec(fn) return names, bool(has_kw) + def get_func_kwargs(func): """Return the set of legal kwargs for the given `func`. @@ -115,7 +258,22 @@ def get_func_kwargs(func): """ - return inspect.getargspec(func)[0] + return compat.inspect_getargspec(func)[0] + +def get_callable_argspec(fn, no_self=False): + if isinstance(fn, types.FunctionType): + return compat.inspect_getargspec(fn) + elif isinstance(fn, types.MethodType) and no_self: + spec = compat.inspect_getargspec(fn.__func__) + return compat.ArgSpec(spec.args[1:], spec.varargs, spec.keywords, spec.defaults) + elif hasattr(fn, '__func__'): + return compat.inspect_getargspec(fn.__func__) + elif hasattr(fn, '__call__') and \ + not hasattr(fn.__call__, '__call__'): # functools.partial does this; + # not much we can do + return get_callable_argspec(fn.__call__) + else: + raise ValueError("Can't inspect function: %s" % fn) def format_argspec_plus(fn, grouped=True): """Returns a dictionary of formatted, introspected function arguments. @@ -149,8 +307,8 @@ def format_argspec_plus(fn, grouped=True): 'apply_pos': '(self, a, b, c, **d)'} """ - if callable(fn): - spec = inspect_getfullargspec(fn) + if compat.callable(fn): + spec = compat.inspect_getfullargspec(fn) else: # we accept an existing argspec... spec = fn @@ -162,28 +320,29 @@ def format_argspec_plus(fn, grouped=True): else: self_arg = None - # Py3K - #apply_pos = inspect.formatargspec(spec[0], spec[1], spec[2], None, spec[4]) - #num_defaults = 0 - #if spec[3]: - # num_defaults += len(spec[3]) - #if spec[4]: - # num_defaults += len(spec[4]) - #name_args = spec[0] + spec[4] - # Py2K - apply_pos = inspect.formatargspec(spec[0], spec[1], spec[2]) - num_defaults = 0 - if spec[3]: - num_defaults += len(spec[3]) - name_args = spec[0] - # end Py2K + if compat.py3k: + apply_pos = inspect.formatargspec(spec[0], spec[1], + spec[2], None, spec[4]) + num_defaults = 0 + if spec[3]: + num_defaults += len(spec[3]) + if spec[4]: + num_defaults += len(spec[4]) + name_args = spec[0] + spec[4] + else: + apply_pos = inspect.formatargspec(spec[0], spec[1], spec[2]) + num_defaults = 0 + if spec[3]: + num_defaults += len(spec[3]) + name_args = spec[0] if num_defaults: - defaulted_vals = name_args[0-num_defaults:] + defaulted_vals = name_args[0 - num_defaults:] else: defaulted_vals = () - apply_kw = inspect.formatargspec(name_args, spec[1], spec[2], defaulted_vals, + apply_kw = inspect.formatargspec(name_args, spec[1], spec[2], + defaulted_vals, formatvalue=lambda x: '=' + x) if grouped: return dict(args=args, self_arg=self_arg, @@ -192,6 +351,7 @@ def format_argspec_plus(fn, grouped=True): return dict(args=args[1:-1], self_arg=self_arg, apply_pos=apply_pos[1:-1], apply_kw=apply_kw[1:-1]) + def format_argspec_init(method, grouped=True): """format_argspec_plus with considerations for typical __init__ methods @@ -205,7 +365,6 @@ def format_argspec_init(method, grouped=True): try: return format_argspec_plus(method, grouped=grouped) except TypeError: - self_arg = 'self' if method is object.__init__: args = grouped and '(self)' or 'self' else: @@ -213,6 +372,7 @@ def format_argspec_init(method, grouped=True): or 'self, *args, **kwargs') return dict(self_arg='self', args=args, apply_pos=args, apply_kw=args) + def getargspec_init(method): """inspect.getargspec with considerations for typical __init__ methods @@ -232,53 +392,84 @@ def getargspec_init(method): def unbound_method_to_callable(func_or_cls): - """Adjust the incoming callable such that a 'self' argument is not required.""" + """Adjust the incoming callable such that a 'self' argument is not + required. - if isinstance(func_or_cls, types.MethodType) and not func_or_cls.im_self: - return func_or_cls.im_func + """ + + if isinstance(func_or_cls, types.MethodType) and not func_or_cls.__self__: + return func_or_cls.__func__ else: return func_or_cls + def generic_repr(obj, additional_kw=(), to_inspect=None): """Produce a __repr__() based on direct association of the __init__() specification vs. same-named attributes present. """ if to_inspect is None: - to_inspect = obj - def genargs(): + to_inspect = [obj] + else: + to_inspect = _collections.to_list(to_inspect) + + missing = object() + + pos_args = [] + kw_args = _collections.OrderedDict() + vargs = None + for i, insp in enumerate(to_inspect): try: - (args, vargs, vkw, defaults) = inspect.getargspec(to_inspect.__init__) + (_args, _vargs, vkw, defaults) = \ + inspect.getargspec(insp.__init__) except TypeError: - return - - default_len = defaults and len(defaults) or 0 - - if not default_len: - for arg in args[1:]: - yield repr(getattr(obj, arg, None)) - if vargs is not None and hasattr(obj, vargs): - yield ', '.join(repr(val) for val in getattr(obj, vargs)) + continue else: - for arg in args[1:-default_len]: - yield repr(getattr(obj, arg, None)) - for (arg, defval) in zip(args[-default_len:], defaults): - try: - val = getattr(obj, arg, None) - if val != defval: - yield '%s=%r' % (arg, val) - except: - pass - if additional_kw: - for arg, defval in additional_kw: - try: - val = getattr(obj, arg, None) - if val != defval: - yield '%s=%r' % (arg, val) - except: - pass + default_len = defaults and len(defaults) or 0 + if i == 0: + if _vargs: + vargs = _vargs + if default_len: + pos_args.extend(_args[1:-default_len]) + else: + pos_args.extend(_args[1:]) + else: + kw_args.update([ + (arg, missing) for arg in _args[1:-default_len] + ]) + + if default_len: + kw_args.update([ + (arg, default) + for arg, default + in zip(_args[-default_len:], defaults) + ]) + output = [] + + output.extend(repr(getattr(obj, arg, None)) for arg in pos_args) + + if vargs is not None and hasattr(obj, vargs): + output.extend([repr(val) for val in getattr(obj, vargs)]) + + for arg, defval in kw_args.items(): + try: + val = getattr(obj, arg, missing) + if val is not missing and val != defval: + output.append('%s=%r' % (arg, val)) + except: + pass + + if additional_kw: + for arg, defval in additional_kw: + try: + val = getattr(obj, arg, missing) + if val is not missing and val != defval: + output.append('%s=%r' % (arg, val)) + except: + pass + + return "%s(%s)" % (obj.__class__.__name__, ", ".join(output)) - return "%s(%s)" % (obj.__class__.__name__, ", ".join(genargs())) class portable_instancemethod(object): """Turn an instancemethod into a (parent, name) pair @@ -286,12 +477,13 @@ class portable_instancemethod(object): """ def __init__(self, meth): - self.target = meth.im_self + self.target = meth.__self__ self.name = meth.__name__ def __call__(self, *arg, **kw): return getattr(self.target, self.name)(*arg, **kw) + def class_hierarchy(cls): """Return an unordered sequence of all classes related to cls. @@ -305,37 +497,39 @@ def class_hierarchy(cls): will not be descended. """ - # Py2K - if isinstance(cls, types.ClassType): - return list() - # end Py2K + if compat.py2k: + if isinstance(cls, types.ClassType): + return list() + hier = set([cls]) process = list(cls.__mro__) while process: c = process.pop() - # Py2K - if isinstance(c, types.ClassType): - continue - for b in (_ for _ in c.__bases__ - if _ not in hier and not isinstance(_, types.ClassType)): - # end Py2K - # Py3K - #for b in (_ for _ in c.__bases__ - # if _ not in hier): + if compat.py2k: + if isinstance(c, types.ClassType): + continue + bases = (_ for _ in c.__bases__ + if _ not in hier and not isinstance(_, types.ClassType)) + else: + bases = (_ for _ in c.__bases__ if _ not in hier) + + for b in bases: process.append(b) hier.add(b) - # Py3K - #if c.__module__ == 'builtins' or not hasattr(c, '__subclasses__'): - # continue - # Py2K - if c.__module__ == '__builtin__' or not hasattr(c, '__subclasses__'): - continue - # end Py2K + + if compat.py3k: + if c.__module__ == 'builtins' or not hasattr(c, '__subclasses__'): + continue + else: + if c.__module__ == '__builtin__' or not hasattr(c, '__subclasses__'): + continue + for s in [_ for _ in c.__subclasses__() if _ not in hier]: process.append(s) hier.add(s) return list(hier) + def iterate_attributes(cls): """iterate all the keys and attributes associated with a class, without using getattr(). @@ -351,6 +545,7 @@ def iterate_attributes(cls): yield (key, c.__dict__[key]) break + def monkeypatch_proxied_specials(into_cls, from_cls, skip=None, only=None, name='self.proxy', from_instance=None): """Automates delegation of __specials__ for a proxying type.""" @@ -364,6 +559,7 @@ def monkeypatch_proxied_specials(into_cls, from_cls, skip=None, only=None, dunders = [m for m in dir(from_cls) if (m.startswith('__') and m.endswith('__') and not hasattr(into_cls, m) and m not in skip)] + for method in dunders: try: fn = getattr(from_cls, method) @@ -384,9 +580,9 @@ def monkeypatch_proxied_specials(into_cls, from_cls, skip=None, only=None, "return %(name)s.%(method)s%(d_args)s" % locals()) env = from_instance is not None and {name: from_instance} or {} - exec py in env + compat.exec_(py, env) try: - env[method].func_defaults = fn.func_defaults + env[method].__defaults__ = fn.__defaults__ except AttributeError: pass setattr(into_cls, method, env[method]) @@ -395,11 +591,8 @@ def monkeypatch_proxied_specials(into_cls, from_cls, skip=None, only=None, def methods_equivalent(meth1, meth2): """Return True if the two methods are the same implementation.""" - # Py3K - #return getattr(meth1, '__func__', meth1) is getattr(meth2, '__func__', meth2) - # Py2K - return getattr(meth1, 'im_func', meth1) is getattr(meth2, 'im_func', meth2) - # end Py2K + return getattr(meth1, '__func__', meth1) is getattr(meth2, '__func__', meth2) + def as_interface(obj, cls=None, methods=None, required=None): """Ensure basic interface compliance for an instance or dict of callables. @@ -471,7 +664,7 @@ def as_interface(obj, cls=None, methods=None, required=None): for method, impl in dictlike_iteritems(obj): if method not in interface: raise TypeError("%r: unknown in this interface" % method) - if not callable(impl): + if not compat.callable(impl): raise TypeError("%r=%r is not callable" % (method, impl)) setattr(AnonymousInterface, method, staticmethod(impl)) found.add(method) @@ -497,7 +690,12 @@ class memoized_property(object): return result def _reset(self, obj): - obj.__dict__.pop(self.__name__, None) + memoized_property.reset(obj, self.__name__) + + @classmethod + def reset(cls, obj, name): + obj.__dict__.pop(name, None) + class memoized_instancemethod(object): """Decorate a method memoize its return value. @@ -515,6 +713,7 @@ class memoized_instancemethod(object): def __get__(self, obj, cls): if obj is None: return self + def oneshot(*args, **kw): result = self.fget(obj, *args, **kw) memo = lambda *a, **kw: result @@ -522,13 +721,11 @@ class memoized_instancemethod(object): memo.__doc__ = self.__doc__ obj.__dict__[self.__name__] = memo return result + oneshot.__name__ = self.__name__ oneshot.__doc__ = self.__doc__ return oneshot -def reset_memoized(instance, name): - instance.__dict__.pop(name, None) - class group_expirable_memoized_property(object): """A family of @memoized_properties that can be expired in tandem.""" @@ -552,86 +749,141 @@ class group_expirable_memoized_property(object): self.attributes.append(fn.__name__) return memoized_instancemethod(fn) -class importlater(object): - """Deferred import object. - e.g.:: - somesubmod = importlater("mypackage.somemodule", "somesubmod") +def dependency_for(modulename): + def decorate(obj): + # TODO: would be nice to improve on this import silliness, + # unfortunately importlib doesn't work that great either + tokens = modulename.split(".") + mod = compat.import_(".".join(tokens[0:-1]), globals(), locals(), tokens[-1]) + mod = getattr(mod, tokens[-1]) + setattr(mod, obj.__name__, obj) + return obj + return decorate - is equivalent to:: +class dependencies(object): + """Apply imported dependencies as arguments to a function. - from mypackage.somemodule import somesubmod + E.g.:: - except evaluted upon attribute access to "somesubmod". + @util.dependencies( + "sqlalchemy.sql.widget", + "sqlalchemy.engine.default" + ); + def some_func(self, widget, default, arg1, arg2, **kw): + # ... - importlater() currently requires that resolve_all() be - called, typically at the bottom of a package's __init__.py. - This is so that __import__ still called only at - module import time, and not potentially within - a non-main thread later on. + Rationale is so that the impact of a dependency cycle can be + associated directly with the few functions that cause the cycle, + and not pollute the module-level namespace. """ - _unresolved = set() + def __init__(self, *deps): + self.import_deps = [] + for dep in deps: + tokens = dep.split(".") + self.import_deps.append( + dependencies._importlater( + ".".join(tokens[0:-1]), + tokens[-1] + ) + ) - def __init__(self, path, addtl=None): - self._il_path = path - self._il_addtl = addtl - importlater._unresolved.add(self) + def __call__(self, fn): + import_deps = self.import_deps + spec = compat.inspect_getfullargspec(fn) + + spec_zero = list(spec[0]) + hasself = spec_zero[0] in ('self', 'cls') + + for i in range(len(import_deps)): + spec[0][i + (1 if hasself else 0)] = "import_deps[%r]" % i + + inner_spec = format_argspec_plus(spec, grouped=False) + + for impname in import_deps: + del spec_zero[1 if hasself else 0] + spec[0][:] = spec_zero + + outer_spec = format_argspec_plus(spec, grouped=False) + + code = 'lambda %(args)s: fn(%(apply_kw)s)' % { + "args": outer_spec['args'], + "apply_kw": inner_spec['apply_kw'] + } + + decorated = eval(code, locals()) + decorated.__defaults__ = getattr(fn, 'im_func', fn).__defaults__ + return update_wrapper(decorated, fn) @classmethod - def resolve_all(cls): - for m in list(importlater._unresolved): - m._resolve() + def resolve_all(cls, path): + for m in list(dependencies._unresolved): + if m._full_path.startswith(path): + m._resolve() - @property - def _full_path(self): - if self._il_addtl: + _unresolved = set() + _by_key = {} + + class _importlater(object): + _unresolved = set() + + _by_key = {} + + def __new__(cls, path, addtl): + key = path + "." + addtl + if key in dependencies._by_key: + return dependencies._by_key[key] + else: + dependencies._by_key[key] = imp = object.__new__(cls) + return imp + + def __init__(self, path, addtl): + self._il_path = path + self._il_addtl = addtl + dependencies._unresolved.add(self) + + + @property + def _full_path(self): return self._il_path + "." + self._il_addtl - else: - return self._il_path - @memoized_property - def module(self): - if self in importlater._unresolved: - raise ImportError( - "importlater.resolve_all() hasn't been called") + @memoized_property + def module(self): + if self in dependencies._unresolved: + raise ImportError( + "importlater.resolve_all() hasn't " + "been called (this is %s %s)" + % (self._il_path, self._il_addtl)) - m = self._initial_import - if self._il_addtl: - m = getattr(m, self._il_addtl) - else: - for token in self._il_path.split(".")[1:]: - m = getattr(m, token) - return m + return getattr(self._initial_import, self._il_addtl) - def _resolve(self): - importlater._unresolved.discard(self) - if self._il_addtl: - self._initial_import = __import__( + def _resolve(self): + dependencies._unresolved.discard(self) + self._initial_import = compat.import_( self._il_path, globals(), locals(), [self._il_addtl]) - else: - self._initial_import = __import__(self._il_path) - def __getattr__(self, key): - if key == 'module': - raise ImportError("Could not resolve module %s" - % self._full_path) - try: - attr = getattr(self.module, key) - except AttributeError: - raise AttributeError( - "Module %s has no attribute '%s'" % - (self._full_path, key) - ) - self.__dict__[key] = attr - return attr + def __getattr__(self, key): + if key == 'module': + raise ImportError("Could not resolve module %s" + % self._full_path) + try: + attr = getattr(self.module, key) + except AttributeError: + raise AttributeError( + "Module %s has no attribute '%s'" % + (self._full_path, key) + ) + self.__dict__[key] = attr + return attr + # from paste.deploy.converters def asbool(obj): - if isinstance(obj, (str, unicode)): + if isinstance(obj, compat.string_types): obj = obj.strip().lower() if obj in ['true', 'yes', 'on', 'y', 't', '1']: return True @@ -641,6 +893,7 @@ def asbool(obj): raise ValueError("String is not true/false: %r" % obj) return bool(obj) + def bool_or_str(*text): """Return a callable that will evaulate a string as boolean, or one of a set of "alternate" string values. @@ -653,6 +906,7 @@ def bool_or_str(*text): return asbool(obj) return bool_or_value + def asint(value): """Coerce to integer.""" @@ -689,19 +943,20 @@ def constructor_copy(obj, cls, **kw): def counter(): """Return a threadsafe counter function.""" - lock = threading.Lock() - counter = itertools.count(1L) + lock = compat.threading.Lock() + counter = itertools.count(1) # avoid the 2to3 "next" transformation... def _next(): lock.acquire() try: - return counter.next() + return next(counter) finally: lock.release() return _next + def duck_type_collection(specimen, default=None): """Given an instance or class, guess if it is or is acting as one of the basic collection types: list, set and dict. If the __emulates__ @@ -711,7 +966,7 @@ def duck_type_collection(specimen, default=None): if hasattr(specimen, '__emulates__'): # canonicalize set vs sets.Set to a standard: the builtin set if (specimen.__emulates__ is not None and - issubclass(specimen.__emulates__, set_types)): + issubclass(specimen.__emulates__, set)): return set else: return specimen.__emulates__ @@ -719,7 +974,7 @@ def duck_type_collection(specimen, default=None): isa = isinstance(specimen, type) and issubclass or isinstance if isa(specimen, list): return list - elif isa(specimen, set_types): + elif isa(specimen, set): return set elif isa(specimen, dict): return dict @@ -733,32 +988,32 @@ def duck_type_collection(specimen, default=None): else: return default + def assert_arg_type(arg, argtype, name): if isinstance(arg, argtype): return arg else: if isinstance(argtype, tuple): raise exc.ArgumentError( - "Argument '%s' is expected to be one of type %s, got '%s'" % - (name, ' or '.join("'%s'" % a for a in argtype), type(arg))) + "Argument '%s' is expected to be one of type %s, got '%s'" % + (name, ' or '.join("'%s'" % a for a in argtype), type(arg))) else: raise exc.ArgumentError( - "Argument '%s' is expected to be of type '%s', got '%s'" % - (name, argtype, type(arg))) + "Argument '%s' is expected to be of type '%s', got '%s'" % + (name, argtype, type(arg))) def dictlike_iteritems(dictlike): """Return a (key, value) iterator for almost any dict-like object.""" - # Py3K - #if hasattr(dictlike, 'items'): - # return dictlike.items() - # Py2K - if hasattr(dictlike, 'iteritems'): - return dictlike.iteritems() - elif hasattr(dictlike, 'items'): - return iter(dictlike.items()) - # end Py2K + if compat.py3k: + if hasattr(dictlike, 'items'): + return list(dictlike.items()) + else: + if hasattr(dictlike, 'iteritems'): + return dictlike.iteritems() + elif hasattr(dictlike, 'items'): + return iter(dictlike.items()) getter = getattr(dictlike, '__getitem__', getattr(dictlike, 'get', None)) if getter is None: @@ -796,17 +1051,38 @@ class classproperty(property): return desc.fget(cls) -class _symbol(object): - def __init__(self, name, doc=None): +class hybridmethod(object): + """Decorate a function as cls- or instance- level.""" + def __init__(self, func, expr=None): + self.func = func + + def __get__(self, instance, owner): + if instance is None: + return self.func.__get__(owner, owner.__class__) + else: + return self.func.__get__(instance, owner) + + +class _symbol(int): + def __new__(self, name, doc=None, canonical=None): """Construct a new named symbol.""" - assert isinstance(name, str) - self.name = name + assert isinstance(name, compat.string_types) + if canonical is None: + canonical = hash(name) + v = int.__new__(_symbol, canonical) + v.name = name if doc: - self.__doc__ = doc + v.__doc__ = doc + return v + def __reduce__(self): - return symbol, (self.name,) + return symbol, (self.name, "x", int(self)) + + def __str__(self): + return repr(self) + def __repr__(self): - return "" % self.name + return "symbol(%r)" % self.name _symbol.__name__ = 'symbol' @@ -833,20 +1109,22 @@ class symbol(object): """ symbols = {} - _lock = threading.Lock() + _lock = compat.threading.Lock() - def __new__(cls, name, doc=None): + def __new__(cls, name, doc=None, canonical=None): cls._lock.acquire() try: sym = cls.symbols.get(name) if sym is None: - cls.symbols[name] = sym = _symbol(name, doc) + cls.symbols[name] = sym = _symbol(name, doc, canonical) return sym finally: symbol._lock.release() _creation_order = 1 + + def set_creation_order(instance): """Assign a '_creation_order' sequence to the given instance. @@ -857,10 +1135,14 @@ def set_creation_order(instance): """ global _creation_order instance._creation_order = _creation_order - _creation_order +=1 + _creation_order += 1 + def warn_exception(func, *args, **kwargs): - """executes the given function, catches all exceptions and converts to a warning.""" + """executes the given function, catches all exceptions and converts to + a warning. + + """ try: return func(*args, **kwargs) except: @@ -881,13 +1163,28 @@ def warn(msg, stacklevel=3): be controlled. """ - if isinstance(msg, basestring): + if isinstance(msg, compat.string_types): warnings.warn(msg, exc.SAWarning, stacklevel=stacklevel) else: warnings.warn(msg, stacklevel=stacklevel) + +def only_once(fn): + """Decorate the given function to be a no-op after it is called exactly + once.""" + + once = [fn] + def go(*arg, **kw): + if once: + once_fn = once.pop() + return once_fn(*arg, **kw) + + return update_wrapper(go, fn) + + _SQLA_RE = re.compile(r'sqlalchemy/([a-z_]+/){0,2}[a-z_]+\.py') _UNITTEST_RE = re.compile(r'unit(?:2|test2?/)') + def chop_traceback(tb, exclude_prefix=_UNITTEST_RE, exclude_suffix=_SQLA_RE): """Chop extraneous lines off beginning and end of a traceback. @@ -906,6 +1203,6 @@ def chop_traceback(tb, exclude_prefix=_UNITTEST_RE, exclude_suffix=_SQLA_RE): start += 1 while start <= end and exclude_suffix.search(tb[end]): end -= 1 - return tb[start:end+1] + return tb[start:end + 1] NoneType = type(None) diff --git a/libs/sqlalchemy/util/queue.py b/libs/sqlalchemy/util/queue.py index acccf3c5..82ff55a5 100644 --- a/libs/sqlalchemy/util/queue.py +++ b/libs/sqlalchemy/util/queue.py @@ -1,48 +1,53 @@ # util/queue.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """An adaptation of Py2.3/2.4's Queue module which supports reentrant -behavior, using RLock instead of Lock for its mutex object. +behavior, using RLock instead of Lock for its mutex object. The +Queue object is used exclusively by the sqlalchemy.pool.QueuePool +class. This is to support the connection pool's usage of weakref callbacks to return connections to the underlying Queue, which can in extremely rare cases be invoked within the ``get()`` method of the Queue itself, producing a ``put()`` inside the ``get()`` and therefore a reentrant -condition.""" +condition. + +An additional change includes a special "abort" method which can be used +to immediately raise a special exception for threads that are blocking +on get(). This is to accommodate a rare race condition that can occur +within QueuePool. + +""" from collections import deque from time import time as _time -from sqlalchemy.util import threading -import sys - -if sys.version_info < (2, 6): - def notify_all(condition): - condition.notify() -else: - def notify_all(condition): - condition.notify_all() +from .compat import threading -__all__ = ['Empty', 'Full', 'Queue'] +__all__ = ['Empty', 'Full', 'Queue', 'SAAbort'] + class Empty(Exception): "Exception raised by Queue.get(block=0)/get_nowait()." pass + class Full(Exception): "Exception raised by Queue.put(block=0)/put_nowait()." pass + class SAAbort(Exception): "Special SQLA exception to abort waiting" def __init__(self, context): self.context = context + class Queue: def __init__(self, maxsize=0): """Initialize a queue object with a given maximum size. @@ -146,7 +151,6 @@ class Queue: return an item if one is immediately available, else raise the ``Empty`` exception (`timeout` is ignored in that case). """ - self.not_empty.acquire() try: if not block: @@ -154,7 +158,11 @@ class Queue: raise Empty elif timeout is None: while self._empty(): - self.not_empty.wait() + # wait for only half a second, then + # loop around, so that we can see a change in + # _sqla_abort_context in case we missed the notify_all() + # called by abort() + self.not_empty.wait(.5) if self._sqla_abort_context: raise SAAbort(self._sqla_abort_context) else: @@ -183,7 +191,10 @@ class Queue: if not self.not_full.acquire(False): return try: - notify_all(self.not_empty) + # note that this is now optional + # as the waiters in get() both loop around + # to check the _sqla_abort_context flag periodically + self.not_empty.notify_all() finally: self.not_full.release() diff --git a/libs/sqlalchemy/util/topological.py b/libs/sqlalchemy/util/topological.py index 86e42c1f..fe7e7689 100644 --- a/libs/sqlalchemy/util/topological.py +++ b/libs/sqlalchemy/util/topological.py @@ -1,17 +1,17 @@ # util/topological.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """Topological sorting algorithms.""" -from sqlalchemy.exc import CircularDependencyError -from sqlalchemy import util - +from ..exc import CircularDependencyError +from .. import util __all__ = ['sort', 'sort_as_subsets', 'find_cycles'] + def sort_as_subsets(tuples, allitems): edges = util.defaultdict(set) @@ -36,6 +36,7 @@ def sort_as_subsets(tuples, allitems): todo.difference_update(output) yield output + def sort(tuples, allitems): """sort the given list of items by dependency. @@ -46,8 +47,10 @@ def sort(tuples, allitems): for s in set_: yield s + def find_cycles(tuples, allitems): - # straight from gvr with some mods + # adapted from: + # http://neopythonic.blogspot.com/2009/01/detecting-cycles-in-directed-graph.html edges = util.defaultdict(set) for parent, child in tuples: @@ -84,6 +87,7 @@ def find_cycles(tuples, allitems): node = stack.pop() return output + def _gen_edges(edges): return set([ (right, left)