imdbPy update
This commit is contained in:
Ruud
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,508 @@
|
||||
"""
|
||||
parser.sql.alchemyadapter module (imdb.parser.sql package).
|
||||
|
||||
This module adapts the SQLAlchemy ORM to the internal mechanism.
|
||||
|
||||
Copyright 2008-2010 Davide Alberani <da@erlug.linux.it>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import logging
|
||||
from sqlalchemy import *
|
||||
from sqlalchemy import schema
|
||||
try: from sqlalchemy import exc # 0.5
|
||||
except ImportError: from sqlalchemy import exceptions as exc # 0.4
|
||||
|
||||
_alchemy_logger = logging.getLogger('imdbpy.parser.sql.alchemy')
|
||||
|
||||
try:
|
||||
import migrate.changeset
|
||||
HAS_MC = True
|
||||
except ImportError:
|
||||
HAS_MC = False
|
||||
_alchemy_logger.warn('Unable to import migrate.changeset: Foreign ' \
|
||||
'Keys will not be created.')
|
||||
|
||||
from imdb._exceptions import IMDbDataAccessError
|
||||
from dbschema import *
|
||||
|
||||
# Used to convert table and column names.
|
||||
re_upper = re.compile(r'([A-Z])')
|
||||
|
||||
# XXX: I'm not sure at all that this is the best method to connect
|
||||
# to the database and bind that connection to every table.
|
||||
metadata = MetaData()
|
||||
|
||||
# Maps our placeholders to SQLAlchemy's column types.
|
||||
MAP_COLS = {
|
||||
INTCOL: Integer,
|
||||
UNICODECOL: UnicodeText,
|
||||
STRINGCOL: String
|
||||
}
|
||||
|
||||
|
||||
class NotFoundError(IMDbDataAccessError):
|
||||
"""Exception raised when Table.get(id) returns no value."""
|
||||
pass
|
||||
|
||||
|
||||
def _renameTable(tname):
|
||||
"""Build the name of a table, as done by SQLObject."""
|
||||
tname = re_upper.sub(r'_\1', tname)
|
||||
if tname.startswith('_'):
|
||||
tname = tname[1:]
|
||||
return tname.lower()
|
||||
|
||||
def _renameColumn(cname):
|
||||
"""Build the name of a column, as done by SQLObject."""
|
||||
cname = cname.replace('ID', 'Id')
|
||||
return _renameTable(cname)
|
||||
|
||||
|
||||
class DNNameObj(object):
|
||||
"""Used to access table.sqlmeta.columns[column].dbName (a string)."""
|
||||
def __init__(self, dbName):
|
||||
self.dbName = dbName
|
||||
|
||||
def __repr__(self):
|
||||
return '<DNNameObj(dbName=%s) [id=%s]>' % (self.dbName, id(self))
|
||||
|
||||
|
||||
class DNNameDict(object):
|
||||
"""Used to access table.sqlmeta.columns (a dictionary)."""
|
||||
def __init__(self, colMap):
|
||||
self.colMap = colMap
|
||||
|
||||
def __getitem__(self, key):
|
||||
return DNNameObj(self.colMap[key])
|
||||
|
||||
def __repr__(self):
|
||||
return '<DNNameDict(colMap=%s) [id=%s]>' % (self.colMap, id(self))
|
||||
|
||||
|
||||
class SQLMetaAdapter(object):
|
||||
"""Used to access table.sqlmeta (an object with .table, .columns and
|
||||
.idName attributes)."""
|
||||
def __init__(self, table, colMap=None):
|
||||
self.table = table
|
||||
if colMap is None:
|
||||
colMap = {}
|
||||
self.colMap = colMap
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name == 'table':
|
||||
return getattr(self.table, name)
|
||||
if name == 'columns':
|
||||
return DNNameDict(self.colMap)
|
||||
if name == 'idName':
|
||||
return self.colMap.get('id', 'id')
|
||||
return None
|
||||
|
||||
def __repr__(self):
|
||||
return '<SQLMetaAdapter(table=%s, colMap=%s) [id=%s]>' % \
|
||||
(repr(self.table), repr(self.colMap), id(self))
|
||||
|
||||
|
||||
class QAdapter(object):
|
||||
"""Used to access table.q attribute (remapped to SQLAlchemy table.c)."""
|
||||
def __init__(self, table, colMap=None):
|
||||
self.table = table
|
||||
if colMap is None:
|
||||
colMap = {}
|
||||
self.colMap = colMap
|
||||
|
||||
def __getattr__(self, name):
|
||||
try: return getattr(self.table.c, self.colMap[name])
|
||||
except KeyError, e: raise AttributeError, "unable to get '%s'" % name
|
||||
|
||||
def __repr__(self):
|
||||
return '<QAdapter(table=%s, colMap=%s) [id=%s]>' % \
|
||||
(repr(self.table), repr(self.colMap), id(self))
|
||||
|
||||
|
||||
class RowAdapter(object):
|
||||
"""Adapter for a SQLAlchemy RowProxy object."""
|
||||
def __init__(self, row, table, colMap=None):
|
||||
self.row = row
|
||||
# FIXME: it's OBSCENE that 'table' should be passed from
|
||||
# TableAdapter through ResultAdapter only to land here,
|
||||
# where it's used to directly update a row item.
|
||||
self.table = table
|
||||
if colMap is None:
|
||||
colMap = {}
|
||||
self.colMap = colMap
|
||||
self.colMapKeys = colMap.keys()
|
||||
|
||||
def __getattr__(self, name):
|
||||
try: return getattr(self.row, self.colMap[name])
|
||||
except KeyError, e: raise AttributeError, "unable to get '%s'" % name
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
# FIXME: I can't even think about how much performances suffer,
|
||||
# for this horrible hack (and it's used so rarely...)
|
||||
# For sure something like a "property" to map column names
|
||||
# to getter/setter functions would be much better, but it's
|
||||
# not possible (or at least not easy) to build them for a
|
||||
# single instance.
|
||||
if name in self.__dict__.get('colMapKeys', ()):
|
||||
# Trying to update a value in the database.
|
||||
row = self.__dict__['row']
|
||||
table = self.__dict__['table']
|
||||
colMap = self.__dict__['colMap']
|
||||
params = {colMap[name]: value}
|
||||
table.update(table.c.id==row.id).execute(**params)
|
||||
# XXX: minor bug: after a value is assigned with the
|
||||
# 'rowAdapterInstance.colName = value' syntax, for some
|
||||
# reason rowAdapterInstance.colName still returns the
|
||||
# previous value (even if the database is updated).
|
||||
# Fix it? I'm not even sure it's ever used.
|
||||
return
|
||||
# For every other attribute.
|
||||
object.__setattr__(self, name, value)
|
||||
|
||||
def __repr__(self):
|
||||
return '<RowAdapter(row=%s, table=%s, colMap=%s) [id=%s]>' % \
|
||||
(repr(self.row), repr(self.table), repr(self.colMap), id(self))
|
||||
|
||||
|
||||
class ResultAdapter(object):
|
||||
"""Adapter for a SQLAlchemy ResultProxy object."""
|
||||
def __init__(self, result, table, colMap=None):
|
||||
self.result = result
|
||||
self.table = table
|
||||
if colMap is None:
|
||||
colMap = {}
|
||||
self.colMap = colMap
|
||||
|
||||
def count(self):
|
||||
return len(self)
|
||||
|
||||
def __len__(self):
|
||||
# FIXME: why sqlite returns -1? (that's wrooong!)
|
||||
if self.result.rowcount == -1:
|
||||
return 0
|
||||
return self.result.rowcount
|
||||
|
||||
def __getitem__(self, key):
|
||||
res = list(self.result)[key]
|
||||
if not isinstance(key, slice):
|
||||
# A single item.
|
||||
return RowAdapter(res, self.table, colMap=self.colMap)
|
||||
else:
|
||||
# A (possible empty) list of items.
|
||||
return [RowAdapter(x, self.table, colMap=self.colMap)
|
||||
for x in res]
|
||||
|
||||
def __iter__(self):
|
||||
for item in self.result:
|
||||
yield RowAdapter(item, self.table, colMap=self.colMap)
|
||||
|
||||
def __repr__(self):
|
||||
return '<ResultAdapter(result=%s, table=%s, colMap=%s) [id=%s]>' % \
|
||||
(repr(self.result), repr(self.table),
|
||||
repr(self.colMap), id(self))
|
||||
|
||||
|
||||
class TableAdapter(object):
|
||||
"""Adapter for a SQLAlchemy Table object, to mimic a SQLObject class."""
|
||||
def __init__(self, table, uri=None):
|
||||
"""Initialize a TableAdapter object."""
|
||||
self._imdbpySchema = table
|
||||
self._imdbpyName = table.name
|
||||
self.connectionURI = uri
|
||||
self.colMap = {}
|
||||
columns = []
|
||||
for col in table.cols:
|
||||
# Column's paramters.
|
||||
params = {'nullable': True}
|
||||
params.update(col.params)
|
||||
if col.name == 'id':
|
||||
params['primary_key'] = True
|
||||
if 'notNone' in params:
|
||||
params['nullable'] = not params['notNone']
|
||||
del params['notNone']
|
||||
cname = _renameColumn(col.name)
|
||||
self.colMap[col.name] = cname
|
||||
colClass = MAP_COLS[col.kind]
|
||||
colKindParams = {}
|
||||
if 'length' in params:
|
||||
colKindParams['length'] = params['length']
|
||||
del params['length']
|
||||
elif colClass is UnicodeText and col.index:
|
||||
# XXX: limit length for UNICODECOLs that will have an index.
|
||||
# this can result in name.name and title.title truncations!
|
||||
colClass = Unicode
|
||||
# Should work for most of the database servers.
|
||||
length = 511
|
||||
if self.connectionURI:
|
||||
if self.connectionURI.startswith('mysql'):
|
||||
# To stay compatible with MySQL 4.x.
|
||||
length = 255
|
||||
colKindParams['length'] = length
|
||||
elif self._imdbpyName == 'PersonInfo' and col.name == 'info':
|
||||
if self.connectionURI:
|
||||
if self.connectionURI.startswith('ibm'):
|
||||
# There are some entries longer than 32KB.
|
||||
colClass = CLOB
|
||||
# I really do hope that this space isn't wasted
|
||||
# for each other shorter entry... <g>
|
||||
colKindParams['length'] = 68*1024
|
||||
colKind = colClass(**colKindParams)
|
||||
if 'alternateID' in params:
|
||||
# There's no need to handle them here.
|
||||
del params['alternateID']
|
||||
# Create a column.
|
||||
colObj = Column(cname, colKind, **params)
|
||||
columns.append(colObj)
|
||||
self.tableName = _renameTable(table.name)
|
||||
# Create the table.
|
||||
self.table = Table(self.tableName, metadata, *columns)
|
||||
self._ta_insert = self.table.insert()
|
||||
self._ta_select = self.table.select
|
||||
# Adapters for special attributes.
|
||||
self.q = QAdapter(self.table, colMap=self.colMap)
|
||||
self.sqlmeta = SQLMetaAdapter(self.table, colMap=self.colMap)
|
||||
|
||||
def select(self, conditions=None):
|
||||
"""Return a list of results."""
|
||||
result = self._ta_select(conditions).execute()
|
||||
return ResultAdapter(result, self.table, colMap=self.colMap)
|
||||
|
||||
def get(self, theID):
|
||||
"""Get an object given its ID."""
|
||||
result = self.select(self.table.c.id == theID)
|
||||
#if not result:
|
||||
# raise NotFoundError, 'no data for ID %s' % theID
|
||||
# FIXME: isn't this a bit risky? We can't check len(result),
|
||||
# because sqlite returns -1...
|
||||
# What about converting it to a list and getting the first item?
|
||||
try:
|
||||
return result[0]
|
||||
except KeyError:
|
||||
raise NotFoundError, 'no data for ID %s' % theID
|
||||
|
||||
def dropTable(self, checkfirst=True):
|
||||
"""Drop the table."""
|
||||
dropParams = {'checkfirst': checkfirst}
|
||||
# Guess what? Another work-around for a ibm_db bug.
|
||||
if self.table.bind.engine.url.drivername.startswith('ibm_db'):
|
||||
del dropParams['checkfirst']
|
||||
try:
|
||||
self.table.drop(**dropParams)
|
||||
except exc.ProgrammingError:
|
||||
# As above: re-raise the exception, but only if it's not ibm_db.
|
||||
if not self.table.bind.engine.url.drivername.startswith('ibm_db'):
|
||||
raise
|
||||
|
||||
def createTable(self, checkfirst=True):
|
||||
"""Create the table."""
|
||||
self.table.create(checkfirst=checkfirst)
|
||||
# Create indexes for alternateID columns (other indexes will be
|
||||
# created later, at explicit request for performances reasons).
|
||||
for col in self._imdbpySchema.cols:
|
||||
if col.name == 'id':
|
||||
continue
|
||||
if col.params.get('alternateID', False):
|
||||
self._createIndex(col, checkfirst=checkfirst)
|
||||
|
||||
def _createIndex(self, col, checkfirst=True):
|
||||
"""Create an index for a given (schema) column."""
|
||||
# XXX: indexLen is ignored in SQLAlchemy, and that means that
|
||||
# indexes will be over the whole 255 chars strings...
|
||||
# NOTE: don't use a dot as a separator, or DB2 will do
|
||||
# nasty things.
|
||||
idx_name = '%s_%s' % (self.table.name, col.index or col.name)
|
||||
if checkfirst:
|
||||
for index in self.table.indexes:
|
||||
if index.name == idx_name:
|
||||
return
|
||||
idx = Index(idx_name, getattr(self.table.c, self.colMap[col.name]))
|
||||
# XXX: beware that exc.OperationalError can be raised, is some
|
||||
# strange circumstances; that's why the index name doesn't
|
||||
# follow the SQLObject convention, but includes the table name:
|
||||
# sqlite, for example, expects index names to be unique at
|
||||
# db-level.
|
||||
try:
|
||||
idx.create()
|
||||
except exc.OperationalError, e:
|
||||
_alchemy_logger.warn('Skipping creation of the %s.%s index: %s' %
|
||||
(self.sqlmeta.table, col.name, e))
|
||||
|
||||
def addIndexes(self, ifNotExists=True):
|
||||
"""Create all required indexes."""
|
||||
for col in self._imdbpySchema.cols:
|
||||
if col.index:
|
||||
self._createIndex(col, checkfirst=ifNotExists)
|
||||
|
||||
def addForeignKeys(self, mapTables, ifNotExists=True):
|
||||
"""Create all required foreign keys."""
|
||||
if not HAS_MC:
|
||||
return
|
||||
# It seems that there's no reason to prevent the creation of
|
||||
# indexes for columns with FK constrains: if there's already
|
||||
# an index, the FK index is not created.
|
||||
countCols = 0
|
||||
for col in self._imdbpySchema.cols:
|
||||
countCols += 1
|
||||
if not col.foreignKey:
|
||||
continue
|
||||
fks = col.foreignKey.split('.', 1)
|
||||
foreignTableName = fks[0]
|
||||
if len(fks) == 2:
|
||||
foreignColName = fks[1]
|
||||
else:
|
||||
foreignColName = 'id'
|
||||
foreignColName = mapTables[foreignTableName].colMap.get(
|
||||
foreignColName, foreignColName)
|
||||
thisColName = self.colMap.get(col.name, col.name)
|
||||
thisCol = self.table.columns[thisColName]
|
||||
foreignTable = mapTables[foreignTableName].table
|
||||
foreignCol = getattr(foreignTable.c, foreignColName)
|
||||
# Need to explicitly set an unique name, otherwise it will
|
||||
# explode, if two cols points to the same table.
|
||||
fkName = 'fk_%s_%s_%d' % (foreignTable.name, foreignColName,
|
||||
countCols)
|
||||
constrain = migrate.changeset.ForeignKeyConstraint([thisCol],
|
||||
[foreignCol],
|
||||
name=fkName)
|
||||
try:
|
||||
constrain.create()
|
||||
except exc.OperationalError:
|
||||
continue
|
||||
|
||||
def __call__(self, *args, **kwds):
|
||||
"""To insert a new row with the syntax: TableClass(key=value, ...)"""
|
||||
taArgs = {}
|
||||
for key, value in kwds.items():
|
||||
taArgs[self.colMap.get(key, key)] = value
|
||||
self._ta_insert.execute(*args, **taArgs)
|
||||
|
||||
def __repr__(self):
|
||||
return '<TableAdapter(table=%s) [id=%s]>' % (repr(self.table), id(self))
|
||||
|
||||
|
||||
# Module-level "cache" for SQLObject classes, to prevent
|
||||
# "Table 'tableName' is already defined for this MetaData instance" errors,
|
||||
# when two or more connections to the database are made.
|
||||
# XXX: is this the best way to act?
|
||||
TABLES_REPOSITORY = {}
|
||||
|
||||
def getDBTables(uri=None):
|
||||
"""Return a list of TableAdapter objects to be used to access the
|
||||
database through the SQLAlchemy ORM. The connection uri is optional, and
|
||||
can be used to tailor the db schema to specific needs."""
|
||||
DB_TABLES = []
|
||||
for table in DB_SCHEMA:
|
||||
if table.name in TABLES_REPOSITORY:
|
||||
DB_TABLES.append(TABLES_REPOSITORY[table.name])
|
||||
continue
|
||||
tableAdapter = TableAdapter(table, uri)
|
||||
DB_TABLES.append(tableAdapter)
|
||||
TABLES_REPOSITORY[table.name] = tableAdapter
|
||||
return DB_TABLES
|
||||
|
||||
|
||||
# Functions used to emulate SQLObject's logical operators.
|
||||
def AND(*params):
|
||||
"""Emulate SQLObject's AND."""
|
||||
return and_(*params)
|
||||
|
||||
def OR(*params):
|
||||
"""Emulate SQLObject's OR."""
|
||||
return or_(*params)
|
||||
|
||||
def IN(item, inList):
|
||||
"""Emulate SQLObject's IN."""
|
||||
if not isinstance(item, schema.Column):
|
||||
return OR(*[x == item for x in inList])
|
||||
else:
|
||||
return item.in_(inList)
|
||||
|
||||
def ISNULL(x):
|
||||
"""Emulate SQLObject's ISNULL."""
|
||||
# XXX: Should we use null()? Can null() be a global instance?
|
||||
# XXX: Is it safe to test None with the == operator, in this case?
|
||||
return x == None
|
||||
|
||||
def ISNOTNULL(x):
|
||||
"""Emulate SQLObject's ISNOTNULL."""
|
||||
return x != None
|
||||
|
||||
def CONTAINSSTRING(expr, pattern):
|
||||
"""Emulate SQLObject's CONTAINSSTRING."""
|
||||
return expr.like('%%%s%%' % pattern)
|
||||
|
||||
|
||||
def toUTF8(s):
|
||||
"""For some strange reason, sometimes SQLObject wants utf8 strings
|
||||
instead of unicode; with SQLAlchemy we just return the unicode text."""
|
||||
return s
|
||||
|
||||
|
||||
class _AlchemyConnection(object):
|
||||
"""A proxy for the connection object, required since _ConnectionFairy
|
||||
uses __slots__."""
|
||||
def __init__(self, conn):
|
||||
self.conn = conn
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.conn, name)
|
||||
|
||||
|
||||
def setConnection(uri, tables, encoding='utf8', debug=False):
|
||||
"""Set connection for every table."""
|
||||
# FIXME: why on earth MySQL requires an additional parameter,
|
||||
# is well beyond my understanding...
|
||||
if uri.startswith('mysql'):
|
||||
if '?' in uri:
|
||||
uri += '&'
|
||||
else:
|
||||
uri += '?'
|
||||
uri += 'charset=%s' % encoding
|
||||
params = {'encoding': encoding}
|
||||
if debug:
|
||||
params['echo'] = True
|
||||
if uri.startswith('ibm_db'):
|
||||
# Try to work-around a possible bug of the ibm_db DB2 driver.
|
||||
params['convert_unicode'] = True
|
||||
# XXX: is this the best way to connect?
|
||||
engine = create_engine(uri, **params)
|
||||
metadata.bind = engine
|
||||
eng_conn = engine.connect()
|
||||
if uri.startswith('sqlite'):
|
||||
major = sys.version_info[0]
|
||||
minor = sys.version_info[1]
|
||||
if major > 2 or (major == 2 and minor > 5):
|
||||
eng_conn.connection.connection.text_factory = str
|
||||
# XXX: OH MY, THAT'S A MESS!
|
||||
# We need to return a "connection" object, with the .dbName
|
||||
# attribute set to the db engine name (e.g. "mysql"), .paramstyle
|
||||
# set to the style of the paramters for query() calls, and the
|
||||
# .module attribute set to a module (?) with .OperationalError and
|
||||
# .IntegrityError attributes.
|
||||
# Another attribute of "connection" is the getConnection() function,
|
||||
# used to return an object with a .cursor() method.
|
||||
connection = _AlchemyConnection(eng_conn.connection)
|
||||
paramstyle = eng_conn.dialect.paramstyle
|
||||
connection.module = eng_conn.dialect.dbapi
|
||||
connection.paramstyle = paramstyle
|
||||
connection.getConnection = lambda: connection.connection
|
||||
connection.dbName = engine.url.drivername
|
||||
return connection
|
||||
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
/*
|
||||
* cutils.c module.
|
||||
*
|
||||
* Miscellaneous functions to speed up the IMDbPY package.
|
||||
*
|
||||
* Contents:
|
||||
* - pyratcliff():
|
||||
* Function that implements the Ratcliff-Obershelp comparison
|
||||
* amongst Python strings.
|
||||
*
|
||||
* - pysoundex():
|
||||
* Return a soundex code string, for the given string.
|
||||
*
|
||||
* Copyright 2004-2009 Davide Alberani <da@erlug.linux.it>
|
||||
* Released under the GPL license.
|
||||
*
|
||||
* NOTE: The Ratcliff-Obershelp part was heavily based on code from the
|
||||
* "simil" Python module.
|
||||
* The "simil" module is copyright of Luca Montecchiani <cbm64 _at_ inwind.it>
|
||||
* and can be found here: http://spazioinwind.libero.it/montecchiani/
|
||||
* It was released under the GPL license; original comments are leaved
|
||||
* below.
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
/*========== Ratcliff-Obershelp ==========*/
|
||||
/*****************************************************************************
|
||||
*
|
||||
* Stolen code from :
|
||||
*
|
||||
* [Python-Dev] Why is soundex marked obsolete?
|
||||
* by Eric S. Raymond [4]esr@thyrsus.com
|
||||
* on Sun, 14 Jan 2001 14:09:01 -0500
|
||||
*
|
||||
*****************************************************************************/
|
||||
|
||||
/*****************************************************************************
|
||||
*
|
||||
* Ratcliff-Obershelp common-subpattern similarity.
|
||||
*
|
||||
* This code first appeared in a letter to the editor in Doctor
|
||||
* Dobbs's Journal, 11/1988. The original article on the algorithm,
|
||||
* "Pattern Matching by Gestalt" by John Ratcliff, had appeared in the
|
||||
* July 1988 issue (#181) but the algorithm was presented in assembly.
|
||||
* The main drawback of the Ratcliff-Obershelp algorithm is the cost
|
||||
* of the pairwise comparisons. It is significantly more expensive
|
||||
* than stemming, Hamming distance, soundex, and the like.
|
||||
*
|
||||
* Running time quadratic in the data size, memory usage constant.
|
||||
*
|
||||
*****************************************************************************/
|
||||
|
||||
#include <Python.h>
|
||||
|
||||
#define DONTCOMPARE_NULL 0.0
|
||||
#define DONTCOMPARE_SAME 1.0
|
||||
#define COMPARE 2.0
|
||||
#define STRING_MAXLENDIFFER 0.7
|
||||
|
||||
/* As of 05 Mar 2008, the longest title is ~600 chars. */
|
||||
#define MXLINELEN 1023
|
||||
|
||||
#define MAX(a,b) ((a) > (b) ? (a) : (b))
|
||||
|
||||
|
||||
//*****************************************
|
||||
// preliminary check....
|
||||
//*****************************************
|
||||
static float
|
||||
strings_check(char const *s, char const *t)
|
||||
{
|
||||
float threshold; // lenght difference
|
||||
int s_len = strlen(s); // length of s
|
||||
int t_len = strlen(t); // length of t
|
||||
|
||||
// NULL strings ?
|
||||
if ((t_len * s_len) == 0)
|
||||
return (DONTCOMPARE_NULL);
|
||||
|
||||
// the same ?
|
||||
if (strcmp(s, t) == 0)
|
||||
return (DONTCOMPARE_SAME);
|
||||
|
||||
// string lenght difference threshold
|
||||
// we don't want to compare too different lenght strings ;)
|
||||
if (s_len < t_len)
|
||||
threshold = (float) s_len / (float) t_len;
|
||||
else
|
||||
threshold = (float) t_len / (float) s_len;
|
||||
if (threshold < STRING_MAXLENDIFFER)
|
||||
return (DONTCOMPARE_NULL);
|
||||
|
||||
// proceed
|
||||
return (COMPARE);
|
||||
}
|
||||
|
||||
|
||||
static int
|
||||
RatcliffObershelp(char *st1, char *end1, char *st2, char *end2)
|
||||
{
|
||||
register char *a1, *a2;
|
||||
char *b1, *b2;
|
||||
char *s1 = st1, *s2 = st2; /* initializations are just to pacify GCC */
|
||||
short max, i;
|
||||
|
||||
if (end1 <= st1 || end2 <= st2)
|
||||
return (0);
|
||||
if (end1 == st1 + 1 && end2 == st2 + 1)
|
||||
return (0);
|
||||
|
||||
max = 0;
|
||||
b1 = end1;
|
||||
b2 = end2;
|
||||
|
||||
for (a1 = st1; a1 < b1; a1++) {
|
||||
for (a2 = st2; a2 < b2; a2++) {
|
||||
if (*a1 == *a2) {
|
||||
/* determine length of common substring */
|
||||
for (i = 1; a1[i] && (a1[i] == a2[i]); i++)
|
||||
continue;
|
||||
if (i > max) {
|
||||
max = i;
|
||||
s1 = a1;
|
||||
s2 = a2;
|
||||
b1 = end1 - max;
|
||||
b2 = end2 - max;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!max)
|
||||
return (0);
|
||||
max += RatcliffObershelp(s1 + max, end1, s2 + max, end2); /* rhs */
|
||||
max += RatcliffObershelp(st1, s1, st2, s2); /* lhs */
|
||||
return max;
|
||||
}
|
||||
|
||||
|
||||
static float
|
||||
ratcliff(char *s1, char *s2)
|
||||
/* compute Ratcliff-Obershelp similarity of two strings */
|
||||
{
|
||||
int l1, l2;
|
||||
float res;
|
||||
|
||||
// preliminary tests
|
||||
res = strings_check(s1, s2);
|
||||
if (res != COMPARE)
|
||||
return(res);
|
||||
|
||||
l1 = strlen(s1);
|
||||
l2 = strlen(s2);
|
||||
|
||||
return 2.0 * RatcliffObershelp(s1, s1 + l1, s2, s2 + l2) / (l1 + l2);
|
||||
}
|
||||
|
||||
|
||||
/* Change a string to lowercase. */
|
||||
static void
|
||||
strtolower(char *s1)
|
||||
{
|
||||
int i;
|
||||
for (i=0; i < strlen(s1); i++) s1[i] = tolower(s1[i]);
|
||||
}
|
||||
|
||||
|
||||
/* Ratcliff-Obershelp for two python strings; returns a python float. */
|
||||
static PyObject*
|
||||
pyratcliff(PyObject *self, PyObject *pArgs)
|
||||
{
|
||||
char *s1 = NULL;
|
||||
char *s2 = NULL;
|
||||
PyObject *discard = NULL;
|
||||
char s1copy[MXLINELEN+1];
|
||||
char s2copy[MXLINELEN+1];
|
||||
|
||||
/* The optional PyObject parameter is here to be compatible
|
||||
* with the pure python implementation, which uses a
|
||||
* difflib.SequenceMatcher object. */
|
||||
if (!PyArg_ParseTuple(pArgs, "ss|O", &s1, &s2, &discard))
|
||||
return NULL;
|
||||
|
||||
strncpy(s1copy, s1, MXLINELEN);
|
||||
strncpy(s2copy, s2, MXLINELEN);
|
||||
/* Work on copies. */
|
||||
strtolower(s1copy);
|
||||
strtolower(s2copy);
|
||||
|
||||
return Py_BuildValue("f", ratcliff(s1copy, s2copy));
|
||||
}
|
||||
|
||||
|
||||
/*========== soundex ==========*/
|
||||
/* Max length of the soundex code to output (an uppercase char and
|
||||
* _at most_ 4 digits). */
|
||||
#define SOUNDEX_LEN 5
|
||||
|
||||
/* Group Number Lookup Table */
|
||||
static char soundTable[26] =
|
||||
{ 0 /* A */, '1' /* B */, '2' /* C */, '3' /* D */, 0 /* E */, '1' /* F */,
|
||||
'2' /* G */, 0 /* H */, 0 /* I */, '2' /* J */, '2' /* K */, '4' /* L */,
|
||||
'5' /* M */, '5' /* N */, 0 /* O */, '1' /* P */, '2' /* Q */, '6' /* R */,
|
||||
'2' /* S */, '3' /* T */, 0 /* U */, '1' /* V */, 0 /* W */, '2' /* X */,
|
||||
0 /* Y */, '2' /* Z */};
|
||||
|
||||
static PyObject*
|
||||
pysoundex(PyObject *self, PyObject *pArgs)
|
||||
{
|
||||
int i, j, n;
|
||||
char *s = NULL;
|
||||
char word[MXLINELEN+1];
|
||||
char soundCode[SOUNDEX_LEN+1];
|
||||
char c;
|
||||
|
||||
if (!PyArg_ParseTuple(pArgs, "s", &s))
|
||||
return NULL;
|
||||
|
||||
j = 0;
|
||||
n = strlen(s);
|
||||
|
||||
/* Convert to uppercase and exclude non-ascii chars. */
|
||||
for (i = 0; i < n; i++) {
|
||||
c = toupper(s[i]);
|
||||
if (c < 91 && c > 64) {
|
||||
word[j] = c;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
word[j] = '\0';
|
||||
|
||||
n = strlen(word);
|
||||
if (n == 0) {
|
||||
/* If the string is empty, returns None. */
|
||||
return Py_BuildValue("");
|
||||
}
|
||||
soundCode[0] = word[0];
|
||||
|
||||
/* Build the soundCode string. */
|
||||
j = 1;
|
||||
for (i = 1; j < SOUNDEX_LEN && i < n; i++) {
|
||||
c = soundTable[(word[i]-65)];
|
||||
/* Compact zeroes and equal consecutive digits ("12234112"->"123412") */
|
||||
if (c != 0 && c != soundCode[j-1]) {
|
||||
soundCode[j++] = c;
|
||||
}
|
||||
}
|
||||
soundCode[j] = '\0';
|
||||
|
||||
return Py_BuildValue("s", soundCode);
|
||||
}
|
||||
|
||||
|
||||
static PyMethodDef cutils_methods[] = {
|
||||
{"ratcliff", pyratcliff,
|
||||
METH_VARARGS, "Ratcliff-Obershelp similarity."},
|
||||
{"soundex", pysoundex,
|
||||
METH_VARARGS, "Soundex code for strings."},
|
||||
{NULL}
|
||||
};
|
||||
|
||||
|
||||
void
|
||||
initcutils(void)
|
||||
{
|
||||
Py_InitModule("cutils", cutils_methods);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,461 @@
|
||||
#-*- encoding: utf-8 -*-
|
||||
"""
|
||||
parser.sql.dbschema module (imdb.parser.sql package).
|
||||
|
||||
This module provides the schema used to describe the layout of the
|
||||
database used by the imdb.parser.sql package; functions to create/drop
|
||||
tables and indexes are also provided.
|
||||
|
||||
Copyright 2005-2010 Davide Alberani <da@erlug.linux.it>
|
||||
2006 Giuseppe "Cowo" Corbelli <cowo --> lugbs.linux.it>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
_dbschema_logger = logging.getLogger('imdbpy.parser.sql.dbschema')
|
||||
|
||||
|
||||
# Placeholders for column types.
|
||||
INTCOL = 1
|
||||
UNICODECOL = 2
|
||||
STRINGCOL = 3
|
||||
_strMap = {1: 'INTCOL', 2: 'UNICODECOL', 3: 'STRINGCOL'}
|
||||
|
||||
class DBCol(object):
|
||||
"""Define column objects."""
|
||||
def __init__(self, name, kind, **params):
|
||||
self.name = name
|
||||
self.kind = kind
|
||||
self.index = None
|
||||
self.indexLen = None
|
||||
# If not None, two notations are accepted: 'TableName'
|
||||
# and 'TableName.ColName'; in the first case, 'id' is assumed
|
||||
# as the name of the pointed column.
|
||||
self.foreignKey = None
|
||||
if 'index' in params:
|
||||
self.index = params['index']
|
||||
del params['index']
|
||||
if 'indexLen' in params:
|
||||
self.indexLen = params['indexLen']
|
||||
del params['indexLen']
|
||||
if 'foreignKey' in params:
|
||||
self.foreignKey = params['foreignKey']
|
||||
del params['foreignKey']
|
||||
self.params = params
|
||||
|
||||
def __str__(self):
|
||||
"""Class representation."""
|
||||
s = '<DBCol %s %s' % (self.name, _strMap[self.kind])
|
||||
if self.index:
|
||||
s += ' INDEX'
|
||||
if self.indexLen:
|
||||
s += '[:%d]' % self.indexLen
|
||||
if self.foreignKey:
|
||||
s += ' FOREIGN'
|
||||
if 'default' in self.params:
|
||||
val = self.params['default']
|
||||
if val is not None:
|
||||
val = '"%s"' % val
|
||||
s += ' DEFAULT=%s' % val
|
||||
for param in self.params:
|
||||
if param == 'default': continue
|
||||
s += ' %s' % param.upper()
|
||||
s += '>'
|
||||
return s
|
||||
|
||||
def __repr__(self):
|
||||
"""Class representation."""
|
||||
s = '<DBCol(name="%s", %s' % (self.name, _strMap[self.kind])
|
||||
if self.index:
|
||||
s += ', index="%s"' % self.index
|
||||
if self.indexLen:
|
||||
s += ', indexLen=%d' % self.indexLen
|
||||
if self.foreignKey:
|
||||
s += ', foreignKey="%s"' % self.foreignKey
|
||||
for param in self.params:
|
||||
val = self.params[param]
|
||||
if isinstance(val, (unicode, str)):
|
||||
val = u'"%s"' % val
|
||||
s += ', %s=%s' % (param, val)
|
||||
s += ')>'
|
||||
return s
|
||||
|
||||
|
||||
class DBTable(object):
|
||||
"""Define table objects."""
|
||||
def __init__(self, name, *cols, **kwds):
|
||||
self.name = name
|
||||
self.cols = cols
|
||||
# Default values.
|
||||
self.values = kwds.get('values', {})
|
||||
|
||||
def __str__(self):
|
||||
"""Class representation."""
|
||||
return '<DBTable %s (%d cols, %d values)>' % (self.name,
|
||||
len(self.cols), sum([len(v) for v in self.values.values()]))
|
||||
|
||||
def __repr__(self):
|
||||
"""Class representation."""
|
||||
s = '<DBTable(name="%s"' % self.name
|
||||
col_s = ', '.join([repr(col).rstrip('>').lstrip('<')
|
||||
for col in self.cols])
|
||||
if col_s:
|
||||
s += ', %s' % col_s
|
||||
if self.values:
|
||||
s += ', values=%s' % self.values
|
||||
s += ')>'
|
||||
return s
|
||||
|
||||
|
||||
# Default values to insert in some tables: {'column': (list, of, values, ...)}
|
||||
kindTypeDefs = {'kind': ('movie', 'tv series', 'tv movie', 'video movie',
|
||||
'tv mini series', 'video game', 'episode')}
|
||||
companyTypeDefs = {'kind': ('distributors', 'production companies',
|
||||
'special effects companies', 'miscellaneous companies')}
|
||||
infoTypeDefs = {'info': ('runtimes', 'color info', 'genres', 'languages',
|
||||
'certificates', 'sound mix', 'tech info', 'countries', 'taglines',
|
||||
'keywords', 'alternate versions', 'crazy credits', 'goofs',
|
||||
'soundtrack', 'quotes', 'release dates', 'trivia', 'locations',
|
||||
'mini biography', 'birth notes', 'birth date', 'height',
|
||||
'death date', 'spouse', 'other works', 'birth name',
|
||||
'salary history', 'nick names', 'books', 'agent address',
|
||||
'biographical movies', 'portrayed in', 'where now', 'trade mark',
|
||||
'interviews', 'article', 'magazine cover photo', 'pictorial',
|
||||
'death notes', 'LD disc format', 'LD year', 'LD digital sound',
|
||||
'LD official retail price', 'LD frequency response', 'LD pressing plant',
|
||||
'LD length', 'LD language', 'LD review', 'LD spaciality', 'LD release date',
|
||||
'LD production country', 'LD contrast', 'LD color rendition',
|
||||
'LD picture format', 'LD video noise', 'LD video artifacts',
|
||||
'LD release country', 'LD sharpness', 'LD dynamic range',
|
||||
'LD audio noise', 'LD color information', 'LD group genre',
|
||||
'LD quality program', 'LD close captions-teletext-ld-g',
|
||||
'LD category', 'LD analog left', 'LD certification',
|
||||
'LD audio quality', 'LD video quality', 'LD aspect ratio',
|
||||
'LD analog right', 'LD additional information',
|
||||
'LD number of chapter stops', 'LD dialogue intellegibility',
|
||||
'LD disc size', 'LD master format', 'LD subtitles',
|
||||
'LD status of availablility', 'LD quality of source',
|
||||
'LD number of sides', 'LD video standard', 'LD supplement',
|
||||
'LD original title', 'LD sound encoding', 'LD number', 'LD label',
|
||||
'LD catalog number', 'LD laserdisc title', 'screenplay-teleplay',
|
||||
'novel', 'adaption', 'book', 'production process protocol',
|
||||
'printed media reviews', 'essays', 'other literature', 'mpaa',
|
||||
'plot', 'votes distribution', 'votes', 'rating',
|
||||
'production dates', 'copyright holder', 'filming dates', 'budget',
|
||||
'weekend gross', 'gross', 'opening weekend', 'rentals',
|
||||
'admissions', 'studios', 'top 250 rank', 'bottom 10 rank')}
|
||||
compCastTypeDefs = {'kind': ('cast', 'crew', 'complete', 'complete+verified')}
|
||||
linkTypeDefs = {'link': ('follows', 'followed by', 'remake of', 'remade as',
|
||||
'references', 'referenced in', 'spoofs', 'spoofed in',
|
||||
'features', 'featured in', 'spin off from', 'spin off',
|
||||
'version of', 'similar to', 'edited into',
|
||||
'edited from', 'alternate language version of',
|
||||
'unknown link')}
|
||||
roleTypeDefs = {'role': ('actor', 'actress', 'producer', 'writer',
|
||||
'cinematographer', 'composer', 'costume designer',
|
||||
'director', 'editor', 'miscellaneous crew',
|
||||
'production designer', 'guest')}
|
||||
|
||||
# Schema of tables in our database.
|
||||
# XXX: Foreign keys can be used to create constrains between tables,
|
||||
# but they create indexes in the database, and this
|
||||
# means poor performances at insert-time.
|
||||
DB_SCHEMA = [
|
||||
DBTable('Name',
|
||||
# namePcodeCf is the soundex of the name in the canonical format.
|
||||
# namePcodeNf is the soundex of the name in the normal format, if
|
||||
# different from namePcodeCf.
|
||||
# surnamePcode is the soundex of the surname, if different from the
|
||||
# other two values.
|
||||
|
||||
# The 'id' column is simply skipped by SQLObject (it's a default);
|
||||
# the alternateID attribute here will be ignored by SQLAlchemy.
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('name', UNICODECOL, notNone=True, index='idx_name', indexLen=6),
|
||||
DBCol('imdbIndex', UNICODECOL, length=12, default=None),
|
||||
DBCol('imdbID', INTCOL, default=None),
|
||||
DBCol('namePcodeCf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodecf'),
|
||||
DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodenf'),
|
||||
DBCol('surnamePcode', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcode'),
|
||||
DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
|
||||
),
|
||||
|
||||
DBTable('CharName',
|
||||
# namePcodeNf is the soundex of the name in the normal format.
|
||||
# surnamePcode is the soundex of the surname, if different
|
||||
# from namePcodeNf.
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('name', UNICODECOL, notNone=True, index='idx_name', indexLen=6),
|
||||
DBCol('imdbIndex', UNICODECOL, length=12, default=None),
|
||||
DBCol('imdbID', INTCOL, default=None),
|
||||
DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodenf'),
|
||||
DBCol('surnamePcode', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcode'),
|
||||
DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
|
||||
),
|
||||
|
||||
DBTable('CompanyName',
|
||||
# namePcodeNf is the soundex of the name in the normal format.
|
||||
# namePcodeSf is the soundex of the name plus the country code.
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('name', UNICODECOL, notNone=True, index='idx_name', indexLen=6),
|
||||
DBCol('countryCode', UNICODECOL, length=255, default=None),
|
||||
DBCol('imdbID', INTCOL, default=None),
|
||||
DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodenf'),
|
||||
DBCol('namePcodeSf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodesf'),
|
||||
DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
|
||||
),
|
||||
|
||||
DBTable('KindType',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('kind', STRINGCOL, length=15, default=None, alternateID=True),
|
||||
values=kindTypeDefs
|
||||
),
|
||||
|
||||
DBTable('Title',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('title', UNICODECOL, notNone=True,
|
||||
index='idx_title', indexLen=10),
|
||||
DBCol('imdbIndex', UNICODECOL, length=12, default=None),
|
||||
DBCol('kindID', INTCOL, notNone=True, foreignKey='KindType'),
|
||||
DBCol('productionYear', INTCOL, default=None),
|
||||
DBCol('imdbID', INTCOL, default=None),
|
||||
DBCol('phoneticCode', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcode'),
|
||||
DBCol('episodeOfID', INTCOL, default=None, index='idx_epof',
|
||||
foreignKey='Title'),
|
||||
DBCol('seasonNr', INTCOL, default=None),
|
||||
DBCol('episodeNr', INTCOL, default=None),
|
||||
# Maximum observed length is 44; 49 can store 5 comma-separated
|
||||
# year-year pairs.
|
||||
DBCol('seriesYears', STRINGCOL, length=49, default=None),
|
||||
DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
|
||||
),
|
||||
|
||||
DBTable('CompanyType',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('kind', STRINGCOL, length=32, default=None, alternateID=True),
|
||||
values=companyTypeDefs
|
||||
),
|
||||
|
||||
DBTable('AkaName',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('personID', INTCOL, notNone=True, index='idx_person',
|
||||
foreignKey='Name'),
|
||||
DBCol('name', UNICODECOL, notNone=True),
|
||||
DBCol('imdbIndex', UNICODECOL, length=12, default=None),
|
||||
DBCol('namePcodeCf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodecf'),
|
||||
DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcodenf'),
|
||||
DBCol('surnamePcode', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcode'),
|
||||
DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
|
||||
),
|
||||
|
||||
DBTable('AkaTitle',
|
||||
# XXX: It's safer to set notNone to False, here.
|
||||
# alias for akas are stored completely in the AkaTitle table;
|
||||
# this means that episodes will set also a "tv series" alias name.
|
||||
# Reading the aka-title.list file it looks like there are
|
||||
# episode titles with aliases to different titles for both
|
||||
# the episode and the series title, while for just the series
|
||||
# there are no aliases.
|
||||
# E.g.:
|
||||
# aka title original title
|
||||
# "Series, The" (2005) {The Episode} "Other Title" (2005) {Other Title}
|
||||
# But there is no:
|
||||
# "Series, The" (2005) "Other Title" (2005)
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_movieid',
|
||||
foreignKey='Title'),
|
||||
DBCol('title', UNICODECOL, notNone=True),
|
||||
DBCol('imdbIndex', UNICODECOL, length=12, default=None),
|
||||
DBCol('kindID', INTCOL, notNone=True, foreignKey='KindType'),
|
||||
DBCol('productionYear', INTCOL, default=None),
|
||||
DBCol('phoneticCode', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcode'),
|
||||
DBCol('episodeOfID', INTCOL, default=None, index='idx_epof',
|
||||
foreignKey='AkaTitle'),
|
||||
DBCol('seasonNr', INTCOL, default=None),
|
||||
DBCol('episodeNr', INTCOL, default=None),
|
||||
DBCol('note', UNICODECOL, default=None),
|
||||
DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
|
||||
),
|
||||
|
||||
DBTable('RoleType',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('role', STRINGCOL, length=32, notNone=True, alternateID=True),
|
||||
values=roleTypeDefs
|
||||
),
|
||||
|
||||
DBTable('CastInfo',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('personID', INTCOL, notNone=True, index='idx_pid',
|
||||
foreignKey='Name'),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
|
||||
foreignKey='Title'),
|
||||
DBCol('personRoleID', INTCOL, default=None, index='idx_cid',
|
||||
foreignKey='CharName'),
|
||||
DBCol('note', UNICODECOL, default=None),
|
||||
DBCol('nrOrder', INTCOL, default=None),
|
||||
DBCol('roleID', INTCOL, notNone=True, foreignKey='RoleType')
|
||||
),
|
||||
|
||||
DBTable('CompCastType',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('kind', STRINGCOL, length=32, notNone=True, alternateID=True),
|
||||
values=compCastTypeDefs
|
||||
),
|
||||
|
||||
DBTable('CompleteCast',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, index='idx_mid', foreignKey='Title'),
|
||||
DBCol('subjectID', INTCOL, notNone=True, foreignKey='CompCastType'),
|
||||
DBCol('statusID', INTCOL, notNone=True, foreignKey='CompCastType')
|
||||
),
|
||||
|
||||
DBTable('InfoType',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('info', STRINGCOL, length=32, notNone=True, alternateID=True),
|
||||
values=infoTypeDefs
|
||||
),
|
||||
|
||||
DBTable('LinkType',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('link', STRINGCOL, length=32, notNone=True, alternateID=True),
|
||||
values=linkTypeDefs
|
||||
),
|
||||
|
||||
DBTable('Keyword',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
# XXX: can't use alternateID=True, because it would create
|
||||
# a UNIQUE index; unfortunately (at least with a common
|
||||
# collation like utf8_unicode_ci) MySQL will consider
|
||||
# some different keywords identical - like
|
||||
# "fiancée" and "fiancee".
|
||||
DBCol('keyword', UNICODECOL, length=255, notNone=True,
|
||||
index='idx_keyword', indexLen=5),
|
||||
DBCol('phoneticCode', STRINGCOL, length=5, default=None,
|
||||
index='idx_pcode')
|
||||
),
|
||||
|
||||
DBTable('MovieKeyword',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
|
||||
foreignKey='Title'),
|
||||
DBCol('keywordID', INTCOL, notNone=True, index='idx_keywordid',
|
||||
foreignKey='Keyword')
|
||||
),
|
||||
|
||||
DBTable('MovieLink',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
|
||||
foreignKey='Title'),
|
||||
DBCol('linkedMovieID', INTCOL, notNone=True, foreignKey='Title'),
|
||||
DBCol('linkTypeID', INTCOL, notNone=True, foreignKey='LinkType')
|
||||
),
|
||||
|
||||
DBTable('MovieInfo',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
|
||||
foreignKey='Title'),
|
||||
DBCol('infoTypeID', INTCOL, notNone=True, foreignKey='InfoType'),
|
||||
DBCol('info', UNICODECOL, notNone=True),
|
||||
DBCol('note', UNICODECOL, default=None)
|
||||
),
|
||||
|
||||
# This table is identical to MovieInfo, except that both 'infoTypeID'
|
||||
# and 'info' are indexed.
|
||||
DBTable('MovieInfoIdx',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
|
||||
foreignKey='Title'),
|
||||
DBCol('infoTypeID', INTCOL, notNone=True, index='idx_infotypeid',
|
||||
foreignKey='InfoType'),
|
||||
DBCol('info', UNICODECOL, notNone=True, index='idx_info', indexLen=10),
|
||||
DBCol('note', UNICODECOL, default=None)
|
||||
),
|
||||
|
||||
DBTable('MovieCompanies',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
|
||||
foreignKey='Title'),
|
||||
DBCol('companyID', INTCOL, notNone=True, index='idx_cid',
|
||||
foreignKey='CompanyName'),
|
||||
DBCol('companyTypeID', INTCOL, notNone=True, foreignKey='CompanyType'),
|
||||
DBCol('note', UNICODECOL, default=None)
|
||||
),
|
||||
|
||||
DBTable('PersonInfo',
|
||||
DBCol('id', INTCOL, notNone=True, alternateID=True),
|
||||
DBCol('personID', INTCOL, notNone=True, index='idx_pid',
|
||||
foreignKey='Name'),
|
||||
DBCol('infoTypeID', INTCOL, notNone=True, foreignKey='InfoType'),
|
||||
DBCol('info', UNICODECOL, notNone=True),
|
||||
DBCol('note', UNICODECOL, default=None)
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# Functions to manage tables.
|
||||
def dropTables(tables, ifExists=True):
|
||||
"""Drop the tables."""
|
||||
# In reverse order (useful to avoid errors about foreign keys).
|
||||
DB_TABLES_DROP = list(tables)
|
||||
DB_TABLES_DROP.reverse()
|
||||
for table in DB_TABLES_DROP:
|
||||
_dbschema_logger.info('dropping table %s', table._imdbpyName)
|
||||
table.dropTable(ifExists)
|
||||
|
||||
def createTables(tables, ifNotExists=True):
|
||||
"""Create the tables and insert default values."""
|
||||
for table in tables:
|
||||
# Create the table.
|
||||
_dbschema_logger.info('creating table %s', table._imdbpyName)
|
||||
table.createTable(ifNotExists)
|
||||
# Insert default values, if any.
|
||||
if table._imdbpySchema.values:
|
||||
_dbschema_logger.info('inserting values into table %s',
|
||||
table._imdbpyName)
|
||||
for key in table._imdbpySchema.values:
|
||||
for value in table._imdbpySchema.values[key]:
|
||||
table(**{key: unicode(value)})
|
||||
|
||||
def createIndexes(tables, ifNotExists=True):
|
||||
"""Create the indexes in the database."""
|
||||
for table in tables:
|
||||
_dbschema_logger.info('creating indexes for table %s',
|
||||
table._imdbpyName)
|
||||
table.addIndexes(ifNotExists)
|
||||
|
||||
def createForeignKeys(tables, ifNotExists=True):
|
||||
"""Create Foreign Keys."""
|
||||
mapTables = {}
|
||||
for table in tables:
|
||||
mapTables[table._imdbpyName] = table
|
||||
for table in tables:
|
||||
_dbschema_logger.info('creating foreign keys for table %s',
|
||||
table._imdbpyName)
|
||||
table.addForeignKeys(mapTables, ifNotExists)
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
"""
|
||||
parser.sql.objectadapter module (imdb.parser.sql package).
|
||||
|
||||
This module adapts the SQLObject ORM to the internal mechanism.
|
||||
|
||||
Copyright 2008-2010 Davide Alberani <da@erlug.linux.it>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
"""
|
||||
|
||||
import sys
|
||||
import logging
|
||||
|
||||
from sqlobject import *
|
||||
from sqlobject.sqlbuilder import ISNULL, ISNOTNULL, AND, OR, IN, CONTAINSSTRING
|
||||
|
||||
from dbschema import *
|
||||
|
||||
_object_logger = logging.getLogger('imdbpy.parser.sql.object')
|
||||
|
||||
|
||||
# Maps our placeholders to SQLAlchemy's column types.
|
||||
MAP_COLS = {
|
||||
INTCOL: IntCol,
|
||||
UNICODECOL: UnicodeCol,
|
||||
STRINGCOL: StringCol
|
||||
}
|
||||
|
||||
|
||||
# Exception raised when Table.get(id) returns no value.
|
||||
NotFoundError = SQLObjectNotFound
|
||||
|
||||
|
||||
# class method to be added to the SQLObject class.
|
||||
def addIndexes(cls, ifNotExists=True):
|
||||
"""Create all required indexes."""
|
||||
for col in cls._imdbpySchema.cols:
|
||||
if col.index:
|
||||
idxName = col.index
|
||||
colToIdx = col.name
|
||||
if col.indexLen:
|
||||
colToIdx = {'column': col.name, 'length': col.indexLen}
|
||||
if idxName in [i.name for i in cls.sqlmeta.indexes]:
|
||||
# Check if the index is already present.
|
||||
continue
|
||||
idx = DatabaseIndex(colToIdx, name=idxName)
|
||||
cls.sqlmeta.addIndex(idx)
|
||||
try:
|
||||
cls.createIndexes(ifNotExists)
|
||||
except dberrors.OperationalError, e:
|
||||
_object_logger.warn('Skipping creation of the %s.%s index: %s' %
|
||||
(cls.sqlmeta.table, col.name, e))
|
||||
addIndexes = classmethod(addIndexes)
|
||||
|
||||
|
||||
# Global repository for "fake" tables with Foreign Keys - need to
|
||||
# prevent troubles if addForeignKeys is called more than one time.
|
||||
FAKE_TABLES_REPOSITORY = {}
|
||||
|
||||
def _buildFakeFKTable(cls, fakeTableName):
|
||||
"""Return a "fake" table, with foreign keys where needed."""
|
||||
countCols = 0
|
||||
attrs = {}
|
||||
for col in cls._imdbpySchema.cols:
|
||||
countCols += 1
|
||||
if col.name == 'id':
|
||||
continue
|
||||
if not col.foreignKey:
|
||||
# A non-foreign key column - add it as usual.
|
||||
attrs[col.name] = MAP_COLS[col.kind](**col.params)
|
||||
continue
|
||||
# XXX: Foreign Keys pointing to TableName.ColName not yet supported.
|
||||
thisColName = col.name
|
||||
if thisColName.endswith('ID'):
|
||||
thisColName = thisColName[:-2]
|
||||
|
||||
fks = col.foreignKey.split('.', 1)
|
||||
foreignTableName = fks[0]
|
||||
if len(fks) == 2:
|
||||
foreignColName = fks[1]
|
||||
else:
|
||||
foreignColName = 'id'
|
||||
# Unused...
|
||||
#fkName = 'fk_%s_%s_%d' % (foreignTableName, foreignColName,
|
||||
# countCols)
|
||||
# Create a Foreign Key column, with the correct references.
|
||||
fk = ForeignKey(foreignTableName, name=thisColName, default=None)
|
||||
attrs[thisColName] = fk
|
||||
# Build a _NEW_ SQLObject subclass, with foreign keys, if needed.
|
||||
newcls = type(fakeTableName, (SQLObject,), attrs)
|
||||
return newcls
|
||||
|
||||
def addForeignKeys(cls, mapTables, ifNotExists=True):
|
||||
"""Create all required foreign keys."""
|
||||
# Do not even try, if there are no FK, in this table.
|
||||
if not filter(None, [col.foreignKey for col in cls._imdbpySchema.cols]):
|
||||
return
|
||||
fakeTableName = 'myfaketable%s' % cls.sqlmeta.table
|
||||
if fakeTableName in FAKE_TABLES_REPOSITORY:
|
||||
newcls = FAKE_TABLES_REPOSITORY[fakeTableName]
|
||||
else:
|
||||
newcls = _buildFakeFKTable(cls, fakeTableName)
|
||||
FAKE_TABLES_REPOSITORY[fakeTableName] = newcls
|
||||
# Connect the class with foreign keys.
|
||||
newcls.setConnection(cls._connection)
|
||||
for col in cls._imdbpySchema.cols:
|
||||
if col.name == 'id':
|
||||
continue
|
||||
if not col.foreignKey:
|
||||
continue
|
||||
# Get the SQL that _WOULD BE_ run, if we had to create
|
||||
# this "fake" table.
|
||||
fkQuery = newcls._connection.createReferenceConstraint(newcls,
|
||||
newcls.sqlmeta.columns[col.name])
|
||||
if not fkQuery:
|
||||
# Probably the db doesn't support foreign keys (SQLite).
|
||||
continue
|
||||
# Remove "myfaketable" to get references to _real_ tables.
|
||||
fkQuery = fkQuery.replace('myfaketable', '')
|
||||
# Execute the query.
|
||||
newcls._connection.query(fkQuery)
|
||||
# Disconnect it.
|
||||
newcls._connection.close()
|
||||
addForeignKeys = classmethod(addForeignKeys)
|
||||
|
||||
|
||||
# Module-level "cache" for SQLObject classes, to prevent
|
||||
# "class TheClass is already in the registry" errors, when
|
||||
# two or more connections to the database are made.
|
||||
# XXX: is this the best way to act?
|
||||
TABLES_REPOSITORY = {}
|
||||
|
||||
def getDBTables(uri=None):
|
||||
"""Return a list of classes to be used to access the database
|
||||
through the SQLObject ORM. The connection uri is optional, and
|
||||
can be used to tailor the db schema to specific needs."""
|
||||
DB_TABLES = []
|
||||
for table in DB_SCHEMA:
|
||||
if table.name in TABLES_REPOSITORY:
|
||||
DB_TABLES.append(TABLES_REPOSITORY[table.name])
|
||||
continue
|
||||
attrs = {'_imdbpyName': table.name, '_imdbpySchema': table,
|
||||
'addIndexes': addIndexes, 'addForeignKeys': addForeignKeys}
|
||||
for col in table.cols:
|
||||
if col.name == 'id':
|
||||
continue
|
||||
attrs[col.name] = MAP_COLS[col.kind](**col.params)
|
||||
# Create a subclass of SQLObject.
|
||||
# XXX: use a metaclass? I can't see any advantage.
|
||||
cls = type(table.name, (SQLObject,), attrs)
|
||||
DB_TABLES.append(cls)
|
||||
TABLES_REPOSITORY[table.name] = cls
|
||||
return DB_TABLES
|
||||
|
||||
|
||||
def toUTF8(s):
|
||||
"""For some strange reason, sometimes SQLObject wants utf8 strings
|
||||
instead of unicode."""
|
||||
return s.encode('utf_8')
|
||||
|
||||
|
||||
def setConnection(uri, tables, encoding='utf8', debug=False):
|
||||
"""Set connection for every table."""
|
||||
kw = {}
|
||||
# FIXME: it's absolutely unclear what we should do to correctly
|
||||
# support unicode in MySQL; with some versions of SQLObject,
|
||||
# it seems that setting use_unicode=1 is the _wrong_ thing to do.
|
||||
_uriLower = uri.lower()
|
||||
if _uriLower.startswith('mysql'):
|
||||
kw['use_unicode'] = 1
|
||||
#kw['sqlobject_encoding'] = encoding
|
||||
kw['charset'] = encoding
|
||||
conn = connectionForURI(uri, **kw)
|
||||
conn.debug = debug
|
||||
if uri.startswith('sqlite'):
|
||||
major = sys.version_info[0]
|
||||
minor = sys.version_info[1]
|
||||
if major > 2 or (major == 2 and minor > 5):
|
||||
conn.connection.connection.text_factory = str
|
||||
for table in tables:
|
||||
table.setConnection(conn)
|
||||
#table.sqlmeta.cacheValues = False
|
||||
# FIXME: is it safe to set table._cacheValue to False? Looks like
|
||||
# we can't retrieve correct values after an update (I think
|
||||
# it's never needed, but...) Anyway, these are set to False
|
||||
# for performance reason at insert time (see imdbpy2sql.py).
|
||||
table._cacheValue = False
|
||||
# Required by imdbpy2sql.py.
|
||||
conn.paramstyle = conn.module.paramstyle
|
||||
return conn
|
||||
|
||||
Reference in New Issue
Block a user