dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added pyDAL from repository, adapted web2py code

Browse Source
This commit is contained in:
gi0baro
2014-12-15 20:13:40 +01:00
parent 5ea654ed06
commit dcadcf0ffb
43 changed files with 87 additions and 11692 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "gluon/contrib/pydal"]
path = gluon/contrib/pydal
url = https://github.com/web2py/pydal.git

3
gluon/__init__.py
View File

@@ -11,8 +11,9 @@ Web2Py framework modules
"""
__all__ = ['A', 'B', 'BEAUTIFY', 'BODY', 'BR', 'CAT', 'CENTER', 'CLEANUP', 'CODE', 'CRYPT', 'DAL', 'DIV', 'EM', 'EMBED', 'FIELDSET', 'FORM', 'Field', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'HTTP', 'I', 'IFRAME', 'IMG', 'INPUT', 'IS_ALPHANUMERIC', 'IS_DATE', 'IS_DATETIME', 'IS_DATETIME_IN_RANGE', 'IS_DATE_IN_RANGE', 'IS_DECIMAL_IN_RANGE', 'IS_EMAIL', 'IS_LIST_OF_EMAILS', 'IS_EMPTY_OR', 'IS_EQUAL_TO', 'IS_EXPR', 'IS_FLOAT_IN_RANGE', 'IS_IMAGE', 'IS_JSON', 'IS_INT_IN_RANGE', 'IS_IN_DB', 'IS_IN_SET', 'IS_IPV4', 'IS_LENGTH', 'IS_LIST_OF', 'IS_LOWER', 'IS_MATCH', 'IS_NOT_EMPTY', 'IS_NOT_IN_DB', 'IS_NULL_OR', 'IS_SLUG', 'IS_STRONG', 'IS_TIME', 'IS_UPLOAD_FILENAME', 'IS_UPPER', 'IS_URL', 'LABEL', 'LEGEND', 'LI', 'LINK', 'LOAD', 'MARKMIN', 'MENU', 'META', 'OBJECT', 'OL', 'ON', 'OPTGROUP', 'OPTION', 'P', 'PRE', 'SCRIPT', 'SELECT', 'SPAN', 'SQLFORM', 'SQLTABLE', 'STRONG', 'STYLE', 'TABLE', 'TAG', 'TBODY', 'TD', 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'UL', 'URL', 'XHTML', 'XML', 'redirect', 'current', 'embed64']
from globals import current
from contrib import pydal
from html import *
from validators import *
from http import redirect, HTTP

6
gluon/compileapp.py
View File

@@ -26,7 +26,7 @@ from gluon.fileutils import mktree, listdir, read_file, write_file
from gluon.myregex import regex_expose, regex_longcomments
from gluon.languages import translator
from gluon.dal import DAL, Field
from gluon.dal.base import BaseAdapter
from gluon.pydal.base import BaseAdapter
from gluon.sqlhtml import SQLFORM, SQLTABLE
from gluon.cache import Cache
from gluon.globals import current, Response
@@ -407,7 +407,7 @@ def build_environment(request, response, session, store_current=True):
# Enable standard conditional models (i.e., /*.py, /[controller]/*.py, and
# /[controller]/[function]/*.py)
response.models_to_run = [
r'^\w+\.py$',
r'^\w+\.py$',
r'^%s/\w+\.py$' % request.controller,
r'^%s/%s/\w+\.py$' % (request.controller, request.function)
]
@@ -514,7 +514,7 @@ def compile_controllers(folder):
for function in exposed:
command = data + "\nresponse._vars=response._caller(%s)\n" % \
function
filename = pjoin(folder, 'compiled',
filename = pjoin(folder, 'compiled',
'controllers.%s.%s.py' % (fname[:-3],function))
write_file(filename, command)
save_pyc(filename)

5
gluon/contrib/__init__.py
View File

@@ -1 +1,6 @@
import os
import sys
sys.path.append(os.path.join(os.path.abspath(__file__), "pydal"))
import pydal

1
gluon/contrib/pydal Submodule

Submodule gluon/contrib/pydal added at d886553357

20
gluon/dal.py Normal file
View File

@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from gluon.pydal import DAL as pyDAL
from gluon.pydal import Field, SQLCustomType, geoPoint, geoLine, geoPolygon
from gluon import serializers as w2p_serializers
from gluon import validators as w2p_validators
from gluon.utils import web2py_uuid
from gluon import sqlhtml
class DAL(pyDAL):
serializers = w2p_serializers
validators = w2p_validators
uuid = web2py_uuid
representers = {
'rows_render': sqlhtml.represent,
'rows_xml': sqlhtml.SQLTABLE
}

4
gluon/dal/__init__.py
View File

@@ -1,4 +0,0 @@
from .base import DAL
from .objects import Field
from .helpers.classes import SQLCustomType
from .helpers.methods import geoPoint, geoLine, geoPolygon

21
gluon/dal/_compat.py
View File

@@ -1,21 +0,0 @@
import sys
import hashlib
import os
PY2 = sys.version_info[0] == 2
if PY2:
import cPickle as pickle
import cStringIO as StringIO
import copy_reg as copyreg
hashlib_md5 = hashlib.md5
else:
import pickle
from io import StringIO
import copyreg
hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
pjoin = os.path.join
exists = os.path.exists
ogetattr = object.__getattribute__
osetattr = object.__setattr__

13
gluon/dal/_globals.py
View File

@@ -1,13 +0,0 @@
import threading
import logging
GLOBAL_LOCKER = threading.RLock()
THREAD_LOCAL = threading.local()
LOGGER = logging.getLogger("web2py.dal")
DEFAULT = lambda: None
def IDENTITY(x): return x
def OR(a,b): return a|b
def AND(a,b): return a&b

313
gluon/dal/_load.py
View File

@@ -1,313 +0,0 @@
# -*- coding: utf-8 -*-
import decimal
import re
from ._globals import LOGGER
# verify presence of web2py modules
try:
from collections import OrderedDict
except:
from gluon.contrib.ordereddict import OrderedDict
try:
from gluon.utils import web2py_uuid
except (ImportError, SystemError):
import uuid
def web2py_uuid(): return str(uuid.uuid4())
try:
import portalocker
have_portalocker = True
except ImportError:
portalocker = None
have_portalocker = False
try:
from gluon import serializers
have_serializers = True
simplejson = None
except ImportError:
serializers = None
have_serializers = False
try:
import json as simplejson
except ImportError:
try:
import gluon.contrib.simplejson as simplejson
except ImportError:
simplejson = None
# list of drivers will be built on the fly
# and lists only what is available
DRIVERS = []
try:
from new import classobj
from google.appengine.ext import db as gae
from google.appengine.ext import ndb
from google.appengine.api import namespace_manager, rdbms
from google.appengine.api.datastore_types import Key ### for belongs on ID
from google.appengine.ext.db.polymodel import PolyModel
from google.appengine.ext.ndb.polymodel import PolyModel as NDBPolyModel
DRIVERS.append('google')
except ImportError:
classobj = None
gae = None
ndb = None
namespace_manager = rdbms = None
Key = None
PolyModel = NDBPolyModel = None
if not 'google' in DRIVERS:
try:
from pysqlite2 import dbapi2 as sqlite2
DRIVERS.append('sqlite2')
except ImportError:
LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
try:
from sqlite3 import dbapi2 as sqlite3
DRIVERS.append('sqlite3')
except ImportError:
LOGGER.debug('no SQLite drivers sqlite3')
try:
# first try contrib driver, then from site-packages (if installed)
try:
import gluon.contrib.pymysql as pymysql
# monkeypatch pymysql because they havent fixed the bug:
# https://github.com/petehunt/PyMySQL/issues/86
pymysql.ESCAPE_REGEX = re.compile("'")
pymysql.ESCAPE_MAP = {"'": "''"}
# end monkeypatch
except ImportError:
import pymysql
DRIVERS.append('pymysql')
except ImportError:
LOGGER.debug('no MySQL driver pymysql')
try:
import MySQLdb
DRIVERS.append('MySQLdb')
except ImportError:
LOGGER.debug('no MySQL driver MySQLDB')
try:
import mysql.connector as mysqlconnector
DRIVERS.append("mysqlconnector")
except ImportError:
LOGGER.debug("no driver mysql.connector")
try:
import psycopg2
from psycopg2.extensions import adapt as psycopg2_adapt
DRIVERS.append('psycopg2')
except ImportError:
psycopg2_adapt = None
LOGGER.debug('no PostgreSQL driver psycopg2')
try:
# first try contrib driver, then from site-packages (if installed)
try:
import gluon.contrib.pg8000.dbapi as pg8000
except ImportError:
import pg8000.dbapi as pg8000
DRIVERS.append('pg8000')
except ImportError:
LOGGER.debug('no PostgreSQL driver pg8000')
try:
import cx_Oracle
DRIVERS.append('cx_Oracle')
except ImportError:
cx_Oracle = None
LOGGER.debug('no Oracle driver cx_Oracle')
try:
try:
import pyodbc
except ImportError:
try:
import gluon.contrib.pypyodbc as pyodbc
except Exception, e:
raise ImportError(str(e))
DRIVERS.append('pyodbc')
#DRIVERS.append('DB2(pyodbc)')
#DRIVERS.append('Teradata(pyodbc)')
#DRIVERS.append('Ingres(pyodbc)')
except ImportError:
pyodbc = None
LOGGER.debug('no MSSQL/DB2/Teradata/Ingres driver pyodbc')
try:
import ibm_db_dbi
DRIVERS.append('ibm_db_dbi')
except ImportError:
LOGGER.debug('no DB2 driver ibm_db_dbi')
try:
import Sybase
DRIVERS.append('Sybase')
except ImportError:
LOGGER.debug('no Sybase driver')
try:
import kinterbasdb
DRIVERS.append('kinterbasdb')
#DRIVERS.append('Firebird(kinterbasdb)')
except ImportError:
LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
try:
import fdb
DRIVERS.append('fdb')
except ImportError:
LOGGER.debug('no Firebird driver fdb')
try:
import firebirdsql
DRIVERS.append('firebirdsql')
except ImportError:
LOGGER.debug('no Firebird driver firebirdsql')
try:
import informixdb
DRIVERS.append('informixdb')
LOGGER.warning('Informix support is experimental')
except ImportError:
LOGGER.debug('no Informix driver informixdb')
try:
import sapdb
DRIVERS.append('sapdb')
LOGGER.warning('SAPDB support is experimental')
except ImportError:
LOGGER.debug('no SAP driver sapdb')
try:
import cubriddb
DRIVERS.append('cubriddb')
LOGGER.warning('Cubrid support is experimental')
except ImportError:
LOGGER.debug('no Cubrid driver cubriddb')
try:
from com.ziclix.python.sql import zxJDBC
import java.sql
# Try sqlite jdbc driver from http://www.zentus.com/sqlitejdbc/
from org.sqlite import JDBC # required by java.sql; ensure we have it
zxJDBC_sqlite = java.sql.DriverManager
DRIVERS.append('zxJDBC')
#DRIVERS.append('SQLite(zxJDBC)')
LOGGER.warning('zxJDBC support is experimental')
is_jdbc = True
except ImportError:
LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
is_jdbc = False
try:
import couchdb
DRIVERS.append('couchdb')
except ImportError:
couchdb = None
LOGGER.debug('no Couchdb driver couchdb')
try:
import pymongo
DRIVERS.append('pymongo')
except:
LOGGER.debug('no MongoDB driver pymongo')
try:
import imaplib
DRIVERS.append('imaplib')
except:
LOGGER.debug('no IMAP driver imaplib')
GAEDecimalProperty = None
NDBDecimalProperty = None
else:
is_jdbc = False
class GAEDecimalProperty(gae.Property):
"""
GAE decimal implementation
"""
data_type = decimal.Decimal
def __init__(self, precision, scale, **kwargs):
super(GAEDecimalProperty, self).__init__(self, **kwargs)
d = '1.'
for x in range(scale):
d += '0'
self.round = decimal.Decimal(d)
def get_value_for_datastore(self, model_instance):
value = super(GAEDecimalProperty, self)\
.get_value_for_datastore(model_instance)
if value is None or value == '':
return None
else:
return str(value)
def make_value_from_datastore(self, value):
if value is None or value == '':
return None
else:
return decimal.Decimal(value).quantize(self.round)
def validate(self, value):
value = super(GAEDecimalProperty, self).validate(value)
if value is None or isinstance(value, decimal.Decimal):
return value
elif isinstance(value, basestring):
return decimal.Decimal(value)
raise gae.BadValueError("Property %s must be a Decimal or string."\
% self.name)
#TODO Needs more testing
class NDBDecimalProperty(ndb.StringProperty):
"""
NDB decimal implementation
"""
data_type = decimal.Decimal
def __init__(self, precision, scale, **kwargs):
d = '1.'
for x in range(scale):
d += '0'
self.round = decimal.Decimal(d)
def _to_base_type(self, value):
if value is None or value == '':
return None
else:
return str(value)
def _from_base_type(self, value):
if value is None or value == '':
return None
else:
return decimal.Decimal(value).quantize(self.round)
def _validate(self, value):
if value is None or isinstance(value, decimal.Decimal):
return value
elif isinstance(value, basestring):
return decimal.Decimal(value)
raise TypeError("Property %s must be a Decimal or string."\
% self._name)
psycopg2_adapt = None
cx_Oracle = None
pyodbc = None
couchdb = None
def get_driver(name):
return globals().get(name)

60
gluon/dal/adapters/__init__.py
View File

@@ -1,60 +0,0 @@
# -*- coding: utf-8 -*-
from .sqlite import SQLiteAdapter, SpatiaLiteAdapter, JDBCSQLiteAdapter
from .mysql import MySQLAdapter
from .postgres import PostgreSQLAdapter, NewPostgreSQLAdapter, JDBCPostgreSQLAdapter
from .oracle import OracleAdapter
from .mssql import MSSQLAdapter, MSSQL2Adapter, MSSQL3Adapter, MSSQL4Adapter, \
VerticaAdapter, SybaseAdapter
from .firebird import FireBirdAdapter
from .informix import InformixAdapter, InformixSEAdapter
from .db2 import DB2Adapter
from .teradata import TeradataAdapter
from .ingres import IngresAdapter, IngresUnicodeAdapter
from .sapdb import SAPDBAdapter
from .cubrid import CubridAdapter
from .google import GoogleDatastoreAdapter, GoogleSQLAdapter
from .couchdb import CouchDBAdapter
from .mongo import MongoDBAdapter
from .imap import IMAPAdapter
ADAPTERS = {
'sqlite': SQLiteAdapter,
'spatialite': SpatiaLiteAdapter,
'sqlite:memory': SQLiteAdapter,
'spatialite:memory': SpatiaLiteAdapter,
'mysql': MySQLAdapter,
'postgres': PostgreSQLAdapter,
'postgres:psycopg2': PostgreSQLAdapter,
'postgres:pg8000': PostgreSQLAdapter,
'postgres2:psycopg2': NewPostgreSQLAdapter,
'postgres2:pg8000': NewPostgreSQLAdapter,
'oracle': OracleAdapter,
'mssql': MSSQLAdapter,
'mssql2': MSSQL2Adapter,
'mssql3': MSSQL3Adapter,
'mssql4' : MSSQL4Adapter,
'vertica': VerticaAdapter,
'sybase': SybaseAdapter,
'db2:ibm_db_dbi': DB2Adapter,
'db2:pyodbc': DB2Adapter,
'teradata': TeradataAdapter,
'informix': InformixAdapter,
'informix-se': InformixSEAdapter,
'firebird': FireBirdAdapter,
'firebird_embedded': FireBirdAdapter,
'ingres': IngresAdapter,
'ingresu': IngresUnicodeAdapter,
'sapdb': SAPDBAdapter,
'cubrid': CubridAdapter,
'jdbc:sqlite': JDBCSQLiteAdapter,
'jdbc:sqlite:memory': JDBCSQLiteAdapter,
'jdbc:postgres': JDBCPostgreSQLAdapter,
'gae': GoogleDatastoreAdapter, # discouraged, for backward compatibility
'google:datastore': GoogleDatastoreAdapter,
'google:datastore+ndb': GoogleDatastoreAdapter,
'google:sql': GoogleSQLAdapter,
'couchdb': CouchDBAdapter,
'mongodb': MongoDBAdapter,
'imap': IMAPAdapter
}

1876
gluon/dal/adapters/base.py
View File

File diff suppressed because it is too large Load Diff

202
gluon/dal/adapters/couchdb.py
View File

@@ -1,202 +0,0 @@
# -*- coding: utf-8 -*-
import datetime
from .._globals import IDENTITY
from .._load import serializers, couchdb, web2py_uuid
from ..objects import Field, Query
from ..helpers.classes import SQLALL
from ..helpers.methods import uuid2int
from .base import BaseAdapter, NoSQLAdapter, SELECT_ARGS
class CouchDBAdapter(NoSQLAdapter):
drivers = ('couchdb',)
uploads_in_blob = True
types = {
'boolean': bool,
'string': str,
'text': str,
'json': str,
'password': str,
'blob': str,
'upload': str,
'integer': long,
'bigint': long,
'float': float,
'double': float,
'date': datetime.date,
'time': datetime.time,
'datetime': datetime.datetime,
'id': long,
'reference': long,
'list:string': list,
'list:integer': list,
'list:reference': list,
}
def file_exists(self, filename): pass
def file_open(self, filename, mode='rb', lock=True): pass
def file_close(self, fileobj): pass
def expand(self,expression,field_type=None):
if isinstance(expression,Field):
if expression.type=='id':
return "%s._id" % expression.tablename
return BaseAdapter.expand(self,expression,field_type)
def AND(self,first,second):
return '(%s && %s)' % (self.expand(first),self.expand(second))
def OR(self,first,second):
return '(%s || %s)' % (self.expand(first),self.expand(second))
def EQ(self,first,second):
if second is None:
return '(%s == null)' % self.expand(first)
return '(%s == %s)' % (self.expand(first),self.expand(second,first.type))
def NE(self,first,second):
if second is None:
return '(%s != null)' % self.expand(first)
return '(%s != %s)' % (self.expand(first),self.expand(second,first.type))
def COMMA(self,first,second):
return '%s + %s' % (self.expand(first),self.expand(second))
def represent(self, obj, fieldtype):
value = NoSQLAdapter.represent(self, obj, fieldtype)
if fieldtype=='id':
return repr(str(long(value)))
elif fieldtype in ('date','time','datetime','boolean'):
return serializers.json(value)
return repr(not isinstance(value,unicode) and value \
or value and value.encode('utf8'))
def __init__(self,db,uri='couchdb://127.0.0.1:5984',
pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.uri = uri
if do_connect: self.find_driver(adapter_args)
self.dbengine = 'couchdb'
self.folder = folder
db['_lastsql'] = ''
self.db_codec = 'UTF-8'
self._after_connection = after_connection
self.pool_size = pool_size
url='http://'+uri[10:]
def connector(url=url,driver_args=driver_args):
return self.driver.Server(url,**driver_args)
self.reconnect(connector,cursor=False)
def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
if migrate:
try:
self.connection.create(table._tablename)
except:
pass
def insert(self,table,fields):
id = uuid2int(web2py_uuid())
ctable = self.connection[table._tablename]
values = dict((k.name,self.represent(v,k.type)) for k,v in fields)
values['_id'] = str(id)
ctable.save(values)
return id
def _select(self,query,fields,attributes):
if not isinstance(query,Query):
raise SyntaxError("Not Supported")
for key in set(attributes.keys())-SELECT_ARGS:
raise SyntaxError('invalid select attribute: %s' % key)
new_fields=[]
for item in fields:
if isinstance(item,SQLALL):
new_fields += item._table
else:
new_fields.append(item)
def uid(fd):
return fd=='id' and '_id' or fd
def get(row,fd):
return fd=='id' and long(row['_id']) or row.get(fd,None)
fields = new_fields
tablename = self.get_table(query)
fieldnames = [f.name for f in (fields or self.db[tablename])]
colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
dict(t=tablename,
query=self.expand(query),
order='%s._id' % tablename,
fields=fields)
return fn, colnames
def select(self,query,fields,attributes):
if not isinstance(query,Query):
raise SyntaxError("Not Supported")
fn, colnames = self._select(query,fields,attributes)
tablename = colnames[0].split('.')[0]
ctable = self.connection[tablename]
rows = [cols['value'] for cols in ctable.query(fn)]
processor = attributes.get('processor',self.parse)
return processor(rows,fields,colnames,False)
def delete(self,tablename,query):
if not isinstance(query,Query):
raise SyntaxError("Not Supported")
if query.first.type=='id' and query.op==self.EQ:
id = query.second
tablename = query.first.tablename
assert(tablename == query.first.tablename)
ctable = self.connection[tablename]
try:
del ctable[str(id)]
return 1
except couchdb.http.ResourceNotFound:
return 0
else:
tablename = self.get_table(query)
rows = self.select(query,[self.db[tablename]._id],{})
ctable = self.connection[tablename]
for row in rows:
del ctable[str(row.id)]
return len(rows)
def update(self,tablename,query,fields):
if not isinstance(query,Query):
raise SyntaxError("Not Supported")
if query.first.type=='id' and query.op==self.EQ:
id = query.second
tablename = query.first.tablename
ctable = self.connection[tablename]
try:
doc = ctable[str(id)]
for key,value in fields:
doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
ctable.save(doc)
return 1
except couchdb.http.ResourceNotFound:
return 0
else:
tablename = self.get_table(query)
rows = self.select(query,[self.db[tablename]._id],{})
ctable = self.connection[tablename]
table = self.db[tablename]
for row in rows:
doc = ctable[str(row.id)]
for key,value in fields:
doc[key.name] = self.represent(value,table[key.name].type)
ctable.save(doc)
return len(rows)
def count(self,query,distinct=None):
if distinct:
raise RuntimeError("COUNT DISTINCT not supported")
if not isinstance(query,Query):
raise SyntaxError("Not Supported")
tablename = self.get_table(query)
rows = self.select(query,[self.db[tablename]._id],{})
return len(rows)

54
gluon/dal/adapters/cubrid.py
View File

@@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
import re
from .._globals import IDENTITY
from .mysql import MySQLAdapter
class CubridAdapter(MySQLAdapter):
drivers = ('cubriddb',)
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "cubrid"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError(
"Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
port = int(m.group('port') or '30000')
user = credential_decoder(user)
passwd = credential_decoder(password)
def connector(host=host,port=port,db=db,
user=user,passwd=passwd,driver_args=driver_args):
return self.driver.connect(host,port,db,user,passwd,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.execute('SET FOREIGN_KEY_CHECKS=1;')
self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")

105
gluon/dal/adapters/db2.py
View File

@@ -1,105 +0,0 @@
# -*- coding: utf-8 -*-
import base64
import datetime
from .._globals import IDENTITY
from .base import BaseAdapter
class DB2Adapter(BaseAdapter):
drivers = ('ibm_db_dbi', 'pyodbc')
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'CLOB',
'json': 'CLOB',
'password': 'VARCHAR(%(length)s)',
'blob': 'BLOB',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'REAL',
'double': 'DOUBLE',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'TIMESTAMP',
'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'CLOB',
'list:string': 'CLOB',
'list:reference': 'CLOB',
'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def LEFT_JOIN(self):
return 'LEFT OUTER JOIN'
def RANDOM(self):
return 'RAND()'
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def represent_exceptions(self, obj, fieldtype):
if fieldtype == 'blob':
obj = base64.b64encode(str(obj))
return "BLOB('%s')" % obj
elif fieldtype == 'datetime':
if isinstance(obj, datetime.datetime):
obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
elif isinstance(obj, datetime.date):
obj = obj.isoformat()[:10]+'-00.00.00'
return "'%s'" % obj
return None
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "db2"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://', 1)[1]
def connector(cnxn=ruri,driver_args=driver_args):
if self.driver_name == 'ibm_db_dbi':
vars = cnxn.split(";")
cnxn = {}
for var in vars:
v = var.split('=')
cnxn[v[0].lower()] = v[1]
return self.driver.connect(cnxn['dsn'], cnxn['uid'], cnxn['pwd'], **driver_args)
else:
return self.driver.connect(cnxn, **driver_args)
self.connector = connector
if do_connect: self.reconnect()
def execute(self,command,placeholders=None):
if command[-1:]==';':
command = command[:-1]
if placeholders:
return self.log_execute(command, placeholders)
return self.log_execute(command)
def lastrowid(self,table):
self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
return long(self.cursor.fetchone()[0])
def rowslice(self,rows,minimum=0,maximum=None):
if maximum is None:
return rows[minimum:]
return rows[minimum:maximum]

182
gluon/dal/adapters/firebird.py
View File

@@ -1,182 +0,0 @@
# -*- coding: utf-8 -*-
import re
from .._globals import IDENTITY
from ..objects import Expression
from .base import BaseAdapter
class FireBirdAdapter(BaseAdapter):
drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
commit_on_alter_table = False
support_distributed_transaction = True
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'BLOB SUB_TYPE 1',
'json': 'BLOB SUB_TYPE 1',
'password': 'VARCHAR(%(length)s)',
'blob': 'BLOB SUB_TYPE 0',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INTEGER',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'DOUBLE PRECISION',
'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'TIMESTAMP',
'id': 'INTEGER PRIMARY KEY',
'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'BLOB SUB_TYPE 1',
'list:string': 'BLOB SUB_TYPE 1',
'list:reference': 'BLOB SUB_TYPE 1',
'big-id': 'BIGINT PRIMARY KEY',
'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
}
def sequence_name(self,tablename):
return ('genid_' + self.QUOTE_TEMPLATE) % tablename
def trigger_name(self,tablename):
return 'trg_id_%s' % tablename
def RANDOM(self):
return 'RAND()'
def EPOCH(self, first):
return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
def NOT_NULL(self,default,field_type):
return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
def SUBSTRING(self,field,parameters):
return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
def LENGTH(self, first):
return "CHAR_LENGTH(%s)" % self.expand(first)
def CONTAINS(self,first,second,case_sensitive=False):
if first.type.startswith('list:'):
second = Expression(None,self.CONCAT('|',Expression(
None,self.REPLACE(second,('|','||'))),'|'))
return '(%s CONTAINING %s)' % (self.expand(first),
self.expand(second, 'string'))
def _drop(self,table,mode):
sequence_name = table._sequence_name
return ['DROP TABLE %s %s;' % (table.sqlsafe, mode), 'DROP GENERATOR %s;' % sequence_name]
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def _truncate(self,table,mode = ''):
return ['DELETE FROM %s;' % table._tablename,
'SET GENERATOR %s TO 0;' % table._sequence_name]
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "firebird"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
port = int(m.group('port') or 3050)
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
charset = m.group('charset') or 'UTF8'
driver_args.update(dsn='%s/%s:%s' % (host,port,db),
user = credential_decoder(user),
password = credential_decoder(password),
charset = charset)
def connector(driver_args=driver_args):
return self.driver.connect(**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def create_sequence_and_triggers(self, query, table, **args):
tablename = table._tablename
sequence_name = table._sequence_name
trigger_name = table._trigger_name
self.execute(query)
self.execute('create generator %s;' % sequence_name)
self.execute('set generator %s to 0;' % sequence_name)
self.execute('create trigger %s for %s active before insert position 0 as\nbegin\nif(new.id is null) then\nbegin\nnew.id = gen_id(%s, 1);\nend\nend;' % (trigger_name, tablename, sequence_name))
def lastrowid(self,table):
sequence_name = table._sequence_name
self.execute('SELECT gen_id(%s, 0) FROM rdb$database' % sequence_name)
return long(self.cursor.fetchone()[0])
class FireBirdEmbeddedAdapter(FireBirdAdapter):
drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "firebird"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError(
"Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
pathdb = m.group('path')
if not pathdb:
raise SyntaxError('Path required')
charset = m.group('charset')
if not charset:
charset = 'UTF8'
host = ''
driver_args.update(host=host,
database=pathdb,
user=credential_decoder(user),
password=credential_decoder(password),
charset=charset)
def connector(driver_args=driver_args):
return self.driver.connect(**driver_args)
self.connector = connector
if do_connect: self.reconnect()

621
gluon/dal/adapters/google.py
View File

@@ -1,621 +0,0 @@
# -*- coding: utf-8 -*-
import os
import re
from .._compat import pjoin
from .._globals import IDENTITY, LOGGER, THREAD_LOCAL
from .._load import classobj, gae, ndb, NDBDecimalProperty, GAEDecimalProperty, \
namespace_manager, Key, NDBPolyModel, PolyModel, rdbms, have_serializers, \
serializers, simplejson
from ..objects import Table, Field, Expression, Query
from ..helpers.classes import SQLCustomType, SQLALL, Reference, UseDatabaseStoredFile
from ..helpers.methods import use_common_filters, xorify
from .base import NoSQLAdapter
from .mysql import MySQLAdapter
class GoogleSQLAdapter(UseDatabaseStoredFile, MySQLAdapter):
uploads_in_blob = True
REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
def __init__(self, db, uri='google:sql://realm:domain/database',
pool_size=0, folder=None, db_codec='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "mysql"
self.uri = uri
self.pool_size = pool_size
self.db_codec = db_codec
self._after_connection = after_connection
if do_connect: self.find_driver(adapter_args, uri)
self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
os.sep+'applications'+os.sep,1)[1])
ruri = uri.split("://")[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
instance = credential_decoder(m.group('instance'))
self.dbstring = db = credential_decoder(m.group('db'))
driver_args['instance'] = instance
if not 'charset' in driver_args:
driver_args['charset'] = 'utf8'
self.createdb = createdb = adapter_args.get('createdb',True)
if not createdb:
driver_args['database'] = db
def connector(driver_args=driver_args):
return rdbms.connect(**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
if self.createdb:
# self.execute('DROP DATABASE %s' % self.dbstring)
self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
self.execute('USE %s' % self.dbstring)
self.execute("SET FOREIGN_KEY_CHECKS=1;")
self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
def execute(self, command, *a, **b):
return self.log_execute(command.decode('utf8'), *a, **b)
def find_driver(self,adapter_args,uri=None):
self.adapter_args = adapter_args
self.driver = "google"
class GAEF(object):
def __init__(self,name,op,value,apply):
self.name=name=='id' and '__key__' or name
self.op=op
self.value=value
self.apply=apply
def __repr__(self):
return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
class GoogleDatastoreAdapter(NoSQLAdapter):
"""
NDB:
You can enable NDB by using adapter_args::
db = DAL('google:datastore', adapter_args={'ndb_settings':ndb_settings, 'use_ndb':True})
ndb_settings is optional and can be used for per model caching settings.
It must be a dict in this form::
ndb_settings = {<table_name>:{<variable_name>:<variable_value>}}
See: https://developers.google.com/appengine/docs/python/ndb/cache
"""
MAX_FETCH_LIMIT = 1000000
uploads_in_blob = True
types = {}
# reconnect is not required for Datastore dbs
reconnect = lambda *args, **kwargs: None
def file_exists(self, filename): pass
def file_open(self, filename, mode='rb', lock=True): pass
def file_close(self, fileobj): pass
REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.use_ndb = adapter_args.get('use_ndb',uri.startswith('google:datastore+ndb'))
if self.use_ndb is True:
self.types.update({
'boolean': ndb.BooleanProperty,
'string': (lambda **kwargs: ndb.StringProperty(**kwargs)),
'text': ndb.TextProperty,
'json': ndb.TextProperty,
'password': ndb.StringProperty,
'blob': ndb.BlobProperty,
'upload': ndb.StringProperty,
'integer': ndb.IntegerProperty,
'bigint': ndb.IntegerProperty,
'float': ndb.FloatProperty,
'double': ndb.FloatProperty,
'decimal': NDBDecimalProperty,
'date': ndb.DateProperty,
'time': ndb.TimeProperty,
'datetime': ndb.DateTimeProperty,
'id': None,
'reference': ndb.IntegerProperty,
'list:string': (lambda **kwargs: ndb.StringProperty(repeated=True,default=None, **kwargs)),
'list:integer': (lambda **kwargs: ndb.IntegerProperty(repeated=True,default=None, **kwargs)),
'list:reference': (lambda **kwargs: ndb.IntegerProperty(repeated=True,default=None, **kwargs)),
})
else:
self.types.update({
'boolean': gae.BooleanProperty,
'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)),
'text': gae.TextProperty,
'json': gae.TextProperty,
'password': gae.StringProperty,
'blob': gae.BlobProperty,
'upload': gae.StringProperty,
'integer': gae.IntegerProperty,
'bigint': gae.IntegerProperty,
'float': gae.FloatProperty,
'double': gae.FloatProperty,
'decimal': GAEDecimalProperty,
'date': gae.DateProperty,
'time': gae.TimeProperty,
'datetime': gae.DateTimeProperty,
'id': None,
'reference': gae.IntegerProperty,
'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)),
'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
})
self.db = db
self.uri = uri
self.dbengine = 'google:datastore'
self.folder = folder
db['_lastsql'] = ''
self.db_codec = 'UTF-8'
self._after_connection = after_connection
self.pool_size = 0
match = self.REGEX_NAMESPACE.match(uri)
if match:
namespace_manager.set_namespace(match.group('namespace'))
self.keyfunc = (self.use_ndb and ndb.Key) or Key.from_path
self.ndb_settings = None
if 'ndb_settings' in adapter_args:
self.ndb_settings = adapter_args['ndb_settings']
def parse_id(self, value, field_type):
return value
def represent(self, obj, fieldtype):
if fieldtype == "json":
if have_serializers:
return serializers.json(obj)
elif simplejson:
return simplejson.dumps(obj)
else:
raise Exception("Could not dump json object (missing json library)")
else:
return NoSQLAdapter.represent(self, obj, fieldtype)
def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
myfields = {}
for field in table:
if isinstance(polymodel,Table) and field.name in polymodel.fields():
continue
attr = {}
if isinstance(field.custom_qualifier, dict):
#this is custom properties to add to the GAE field declartion
attr = field.custom_qualifier
field_type = field.type
if isinstance(field_type, SQLCustomType):
ftype = self.types[field_type.native or field_type.type](**attr)
elif isinstance(field_type, ((self.use_ndb and ndb.Property) or gae.Property)):
ftype = field_type
elif field_type.startswith('id'):
continue
elif field_type.startswith('decimal'):
precision, scale = field_type[7:].strip('()').split(',')
precision = int(precision)
scale = int(scale)
dec_cls = (self.use_ndb and NDBDecimalProperty) or GAEDecimalProperty
ftype = dec_cls(precision, scale, **attr)
elif field_type.startswith('reference'):
if field.notnull:
attr = dict(required=True)
ftype = self.types[field_type[:9]](**attr)
elif field_type.startswith('list:reference'):
if field.notnull:
attr['required'] = True
ftype = self.types[field_type[:14]](**attr)
elif field_type.startswith('list:'):
ftype = self.types[field_type](**attr)
elif not field_type in self.types\
or not self.types[field_type]:
raise SyntaxError('Field: unknown field type: %s' % field_type)
else:
ftype = self.types[field_type](**attr)
myfields[field.name] = ftype
if not polymodel:
model_cls = (self.use_ndb and ndb.Model) or gae.Model
table._tableobj = classobj(table._tablename, (model_cls, ), myfields)
if self.use_ndb:
# Set NDB caching variables
if self.ndb_settings and (table._tablename in self.ndb_settings):
for k, v in self.ndb_settings.iteritems():
setattr(table._tableobj, k, v)
elif polymodel==True:
pm_cls = (self.use_ndb and NDBPolyModel) or PolyModel
table._tableobj = classobj(table._tablename, (pm_cls, ), myfields)
elif isinstance(polymodel,Table):
table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
else:
raise SyntaxError("polymodel must be None, True, a table or a tablename")
return None
def expand(self,expression,field_type=None):
if isinstance(expression,Field):
if expression.type in ('text', 'blob', 'json'):
raise SyntaxError('AppEngine does not index by: %s' % expression.type)
return expression.name
elif isinstance(expression, (Expression, Query)):
if not expression.second is None:
return expression.op(expression.first, expression.second)
elif not expression.first is None:
return expression.op(expression.first)
else:
return expression.op()
elif field_type:
return self.represent(expression,field_type)
elif isinstance(expression,(list,tuple)):
return ','.join([self.represent(item,field_type) for item in expression])
else:
return str(expression)
### TODO from gql.py Expression
def AND(self,first,second):
a = self.expand(first)
b = self.expand(second)
if b[0].name=='__key__' and a[0].name!='__key__':
return b+a
return a+b
def EQ(self,first,second=None):
if isinstance(second, Key):
return [GAEF(first.name,'=',second,lambda a,b:a==b)]
return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
def NE(self,first,second=None):
if first.type != 'id':
return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
else:
if not second is None:
second = self.keyfunc(first._tablename, long(second))
return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
def LT(self,first,second=None):
if first.type != 'id':
return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
else:
second = self.keyfunc(first._tablename, long(second))
return [GAEF(first.name,'<',second,lambda a,b:a<b)]
def LE(self,first,second=None):
if first.type != 'id':
return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
else:
second = self.keyfunc(first._tablename, long(second))
return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
def GT(self,first,second=None):
if first.type != 'id' or second==0 or second == '0':
return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
else:
second = self.keyfunc(first._tablename, long(second))
return [GAEF(first.name,'>',second,lambda a,b:a>b)]
def GE(self,first,second=None):
if first.type != 'id':
return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
else:
second = self.keyfunc(first._tablename, long(second))
return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
def INVERT(self,first):
return '-%s' % first.name
def COMMA(self,first,second):
return '%s, %s' % (self.expand(first),self.expand(second))
def BELONGS(self,first,second=None):
if not isinstance(second,(list, tuple, set)):
raise SyntaxError("Not supported")
if not self.use_ndb:
if isinstance(second,set):
second = list(second)
if first.type == 'id':
second = [self.keyfunc(first._tablename, int(i)) for i in second]
return [GAEF(first.name,'in',second,lambda a,b:a in b)]
def CONTAINS(self,first,second,case_sensitive=False):
# silently ignoring: GAE can only do case sensitive matches!
if not first.type.startswith('list:'):
raise SyntaxError("Not supported")
return [GAEF(first.name,'=',self.expand(second,first.type[5:]),lambda a,b:b in a)]
def NOT(self,first):
nops = { self.EQ: self.NE,
self.NE: self.EQ,
self.LT: self.GE,
self.GT: self.LE,
self.LE: self.GT,
self.GE: self.LT}
if not isinstance(first,Query):
raise SyntaxError("Not suported")
nop = nops.get(first.op,None)
if not nop:
raise SyntaxError("Not suported %s" % first.op.__name__)
first.op = nop
return self.expand(first)
def truncate(self,table,mode):
self.db(self.db._adapter.id_query(table)).delete()
GAE_FILTER_OPTIONS = {
'=': lambda q, t, p, v: q.filter(getattr(t,p) == v),
'>': lambda q, t, p, v: q.filter(getattr(t,p) > v),
'<': lambda q, t, p, v: q.filter(getattr(t,p) < v),
'<=': lambda q, t, p, v: q.filter(getattr(t,p) <= v),
'>=': lambda q, t, p, v: q.filter(getattr(t,p) >= v),
'!=': lambda q, t, p, v: q.filter(getattr(t,p) != v),
'in': lambda q, t, p, v: q.filter(getattr(t,p).IN(v)),
}
def filter(self, query, tableobj, prop, op, value):
return self.GAE_FILTER_OPTIONS[op](query, tableobj, prop, value)
def select_raw(self,query,fields=None,attributes=None,count_only=False):
db = self.db
fields = fields or []
attributes = attributes or {}
args_get = attributes.get
new_fields = []
for item in fields:
if isinstance(item,SQLALL):
new_fields += item._table
else:
new_fields.append(item)
fields = new_fields
if query:
tablename = self.get_table(query)
elif fields:
tablename = fields[0].tablename
query = db._adapter.id_query(fields[0].table)
else:
raise SyntaxError("Unable to determine a tablename")
if query:
if use_common_filters(query):
query = self.common_filter(query,[tablename])
#tableobj is a GAE/NDB Model class (or subclass)
tableobj = db[tablename]._tableobj
filters = self.expand(query)
projection = None
if len(db[tablename].fields) == len(fields):
#getting all fields, not a projection query
projection = None
elif args_get('projection') == True:
projection = []
for f in fields:
if f.type in ['text', 'blob', 'json']:
raise SyntaxError(
"text and blob field types not allowed in projection queries")
else:
projection.append(f.name)
elif args_get('filterfields') is True:
projection = []
for f in fields:
projection.append(f.name)
# real projection's can't include 'id'.
# it will be added to the result later
query_projection = [
p for p in projection if \
p != db[tablename]._id.name] if projection and \
args_get('projection') == True\
else None
cursor = args_get('reusecursor')
cursor = cursor if isinstance(cursor, str) else None
if self.use_ndb:
qo = ndb.QueryOptions(projection=query_projection, cursor=cursor)
items = tableobj.query(default_options=qo)
else:
items = gae.Query(tableobj, projection=query_projection, cursor=cursor)
for filter in filters:
if (args_get('projection') == True and
filter.name in query_projection and
filter.op in ('=', '<=', '>=')):
raise SyntaxError("projection fields cannot have equality filters")
if filter.name=='__key__' and filter.op=='>' and filter.value==0:
continue
elif filter.name=='__key__' and filter.op=='=':
if filter.value==0:
items = []
elif isinstance(filter.value, (self.use_ndb and ndb.Key) or Key):
# key qeuries return a class instance,
# can't use projection
# extra values will be ignored in post-processing later
item = filter.value.get() if self.use_ndb else tableobj.get(filter.value)
items = [item] if item else []
else:
# key qeuries return a class instance,
# can't use projection
# extra values will be ignored in post-processing later
item = tableobj.get_by_id(filter.value)
items = [item] if item else []
elif isinstance(items,list): # i.e. there is a single record!
items = [i for i in items if filter.apply(
getattr(item,filter.name),filter.value)]
else:
if filter.name=='__key__' and filter.op != 'in':
items.order(tableobj._key) if self.use_ndb else items.order('__key__')
if self.use_ndb:
items = self.filter(items, tableobj, filter.name, filter.op, filter.value)
else:
items = items.filter('%s %s' % (filter.name,filter.op), filter.value)
if count_only:
items = [len(items) if isinstance(items,list) else items.count()]
elif not isinstance(items,list):
query = items
if args_get('left', None):
raise SyntaxError('Set: no left join in appengine')
if args_get('groupby', None):
raise SyntaxError('Set: no groupby in appengine')
orderby = args_get('orderby', False)
if orderby:
### THIS REALLY NEEDS IMPROVEMENT !!!
if isinstance(orderby, (list, tuple)):
orderby = xorify(orderby)
if isinstance(orderby,Expression):
orderby = self.expand(orderby)
orders = orderby.split(', ')
for order in orders:
if self.use_ndb:
#TODO There must be a better way
def make_order(o):
s = str(o)
desc = s[0] == '-'
s = (desc and s[1:]) or s
return (desc and -getattr(tableobj, s)) or getattr(tableobj, s)
_order = {'-id':-tableobj._key,'id':tableobj._key}.get(order)
if _order is None:
_order = make_order(order)
query = query.order(_order)
else:
order={'-id':'-__key__','id':'__key__'}.get(order,order)
query = query.order(order)
if args_get('limitby', None):
(lmin, lmax) = attributes['limitby']
limit, fetch_args = lmax-lmin, {'offset':lmin,'keys_only':True}
if self.use_ndb:
keys, cursor, more = query.fetch_page(limit,**fetch_args)
items = ndb.get_multi(keys)
else:
keys = query.fetch(limit, **fetch_args)
items = gae.get(keys)
cursor = query.cursor()
#cursor is only useful if there was a limit and we didn't return
# all results
if args_get('reusecursor'):
db['_lastcursor'] = cursor
else:
# if a limit is not specified, always return an iterator
rows = query
return (items, tablename, projection or db[tablename].fields)
def select(self,query,fields,attributes):
"""
This is the GAE version of select. Some notes to consider:
- db['_lastsql'] is not set because there is not SQL statement string
for a GAE query
- 'nativeRef' is a magical fieldname used for self references on GAE
- optional attribute 'projection' when set to True will trigger
use of the GAE projection queries. note that there are rules for
what is accepted imposed by GAE: each field must be indexed,
projection queries cannot contain blob or text fields, and you
cannot use == and also select that same field.
see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
- optional attribute 'filterfields' when set to True web2py will only
parse the explicitly listed fields into the Rows object, even though
all fields are returned in the query. This can be used to reduce
memory usage in cases where true projection queries are not
usable.
- optional attribute 'reusecursor' allows use of cursor with queries
that have the limitby attribute. Set the attribute to True for the
first query, set it to the value of db['_lastcursor'] to continue
a previous query. The user must save the cursor value between
requests, and the filters must be identical. It is up to the user
to follow google's limitations:
https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
"""
(items, tablename, fields) = self.select_raw(query,fields,attributes)
# self.db['_lastsql'] = self._select(query,fields,attributes)
rows = [[(t==self.db[tablename]._id.name and item) or \
(t=='nativeRef' and item) or getattr(item, t) \
for t in fields] for item in items]
colnames = ['%s.%s' % (tablename, t) for t in fields]
processor = attributes.get('processor',self.parse)
return processor(rows,fields,colnames,False)
def parse_list_integers(self, value, field_type):
return value[:] if self.use_ndb else value
def parse_list_strings(self, value, field_type):
return value[:] if self.use_ndb else value
def count(self,query,distinct=None,limit=None):
if distinct:
raise RuntimeError("COUNT DISTINCT not supported")
(items, tablename, fields) = self.select_raw(query,count_only=True)
return items[0]
def delete(self,tablename, query):
"""
This function was changed on 2010-05-04 because according to
http://code.google.com/p/googleappengine/issues/detail?id=3119
GAE no longer supports deleting more than 1000 records.
"""
# self.db['_lastsql'] = self._delete(tablename,query)
(items, tablename, fields) = self.select_raw(query)
# items can be one item or a query
if not isinstance(items,list):
#use a keys_only query to ensure that this runs as a datastore
# small operations
leftitems = items.fetch(1000, keys_only=True)
counter = 0
while len(leftitems):
counter += len(leftitems)
if self.use_ndb:
ndb.delete_multi(leftitems)
else:
gae.delete(leftitems)
leftitems = items.fetch(1000, keys_only=True)
else:
counter = len(items)
if self.use_ndb:
ndb.delete_multi([item.key for item in items])
else:
gae.delete(items)
return counter
def update(self,tablename,query,update_fields):
# self.db['_lastsql'] = self._update(tablename,query,update_fields)
(items, tablename, fields) = self.select_raw(query)
counter = 0
for item in items:
for field, value in update_fields:
setattr(item, field.name, self.represent(value,field.type))
item.put()
counter += 1
LOGGER.info(str(counter))
return counter
def insert(self,table,fields):
dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
# table._db['_lastsql'] = self._insert(table,fields)
tmp = table._tableobj(**dfields)
tmp.put()
key = tmp.key if self.use_ndb else tmp.key()
rid = Reference(key.id())
(rid._table, rid._record, rid._gaekey) = (table, None, key)
return rid
def bulk_insert(self,table,items):
parsed_items = []
for item in items:
dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
parsed_items.append(table._tableobj(**dfields))
if self.use_ndb:
ndb.put_multi(parsed_items)
else:
gae.put(parsed_items)
return True

1034
gluon/dal/adapters/imap.py
View File

File diff suppressed because it is too large Load Diff

134
gluon/dal/adapters/informix.py
View File

@@ -1,134 +0,0 @@
# -*- coding: utf-8 -*-
import datetime
import re
from .._globals import IDENTITY
from .base import BaseAdapter
class InformixAdapter(BaseAdapter):
drivers = ('informixdb',)
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'BLOB SUB_TYPE 1',
'json': 'BLOB SUB_TYPE 1',
'password': 'VARCHAR(%(length)s)',
'blob': 'BLOB SUB_TYPE 0',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INTEGER',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'DOUBLE PRECISION',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'CHAR(8)',
'datetime': 'DATETIME',
'id': 'SERIAL',
'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'BLOB SUB_TYPE 1',
'list:string': 'BLOB SUB_TYPE 1',
'list:reference': 'BLOB SUB_TYPE 1',
'big-id': 'BIGSERIAL',
'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': 'REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s CONSTRAINT FK_%(table_name)s_%(field_name)s',
'reference TFK': 'FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s CONSTRAINT TFK_%(table_name)s_%(field_name)s',
}
def RANDOM(self):
return 'Random()'
def NOT_NULL(self,default,field_type):
return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
fetch_amt = lmax - lmin
dbms_version = int(self.connection.dbms_version.split('.')[0])
if lmin and (dbms_version >= 10):
# Requires Informix 10.0+
sql_s += ' SKIP %d' % (lmin, )
if fetch_amt and (dbms_version >= 9):
# Requires Informix 9.0+
sql_s += ' FIRST %d' % (fetch_amt, )
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def represent_exceptions(self, obj, fieldtype):
if fieldtype == 'date':
if isinstance(obj, (datetime.date, datetime.datetime)):
obj = obj.isoformat()[:10]
else:
obj = str(obj)
return "to_date('%s','%%Y-%%m-%%d')" % obj
elif fieldtype == 'datetime':
if isinstance(obj, datetime.datetime):
obj = obj.isoformat()[:19].replace('T',' ')
elif isinstance(obj, datetime.date):
obj = obj.isoformat()[:10]+' 00:00:00'
else:
obj = str(obj)
return "to_date('%s','%%Y-%%m-%%d %%H:%%M:%%S')" % obj
return None
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "informix"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError(
"Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
user = credential_decoder(user)
password = credential_decoder(password)
dsn = '%s@%s' % (db,host)
driver_args.update(user=user,password=password,autocommit=True)
def connector(dsn=dsn,driver_args=driver_args):
return self.driver.connect(dsn,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def execute(self,command):
if command[-1:]==';':
command = command[:-1]
return self.log_execute(command)
def lastrowid(self,table):
return self.cursor.sqlerrd[1]
class InformixSEAdapter(InformixAdapter):
""" work in progress """
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
return 'SELECT %s %s FROM %s%s%s;' % \
(sql_s, sql_f, sql_t, sql_w, sql_o)
def rowslice(self,rows,minimum=0,maximum=None):
if maximum is None:
return rows[minimum:]
return rows[minimum:maximum]

147
gluon/dal/adapters/ingres.py
View File

@@ -1,147 +0,0 @@
# -*- coding: utf-8 -*-
from .._globals import IDENTITY
from .._load import pyodbc
from .base import BaseAdapter
# NOTE invalid database object name (ANSI-SQL wants
# this form of name to be a delimited identifier)
INGRES_SEQNAME='ii***lineitemsequence'
class IngresAdapter(BaseAdapter):
drivers = ('pyodbc',)
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'CLOB',
'json': 'CLOB',
'password': 'VARCHAR(%(length)s)', ## Not sure what this contains utf8 or nvarchar. Or even bytes?
'blob': 'BLOB',
'upload': 'VARCHAR(%(length)s)', ## FIXME utf8 or nvarchar... or blob? what is this type?
'integer': 'INTEGER4', # or int8...
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT8',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'ANSIDATE',
'time': 'TIME WITHOUT TIME ZONE',
'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'CLOB',
'list:string': 'CLOB',
'list:reference': 'CLOB',
'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO
}
def LEFT_JOIN(self):
return 'LEFT OUTER JOIN'
def RANDOM(self):
return 'RANDOM()'
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
fetch_amt = lmax - lmin
if fetch_amt:
sql_s += ' FIRST %d ' % (fetch_amt, )
if lmin:
# Requires Ingres 9.2+
sql_o += ' OFFSET %d' % (lmin, )
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "ingres"
self._driver = pyodbc
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
connstr = uri.split(':', 1)[1]
# Simple URI processing
connstr = connstr.lstrip()
while connstr.startswith('/'):
connstr = connstr[1:]
if '=' in connstr:
# Assume we have a regular ODBC connection string and just use it
ruri = connstr
else:
# Assume only (local) dbname is passed in with OS auth
database_name = connstr
default_driver_name = 'Ingres'
vnode = '(local)'
servertype = 'ingres'
ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name)
def connector(cnxn=ruri,driver_args=driver_args):
return self.driver.connect(cnxn,**driver_args)
self.connector = connector
# TODO if version is >= 10, set types['id'] to Identity column, see http://community.actian.com/wiki/Using_Ingres_Identity_Columns
if do_connect: self.reconnect()
def create_sequence_and_triggers(self, query, table, **args):
# post create table auto inc code (if needed)
# modify table to btree for performance....
# Older Ingres releases could use rule/trigger like Oracle above.
if hasattr(table,'_primarykey'):
modify_tbl_sql = 'modify %s to btree unique on %s' % \
(table._tablename,
', '.join(["'%s'" % x for x in table.primarykey]))
self.execute(modify_tbl_sql)
else:
tmp_seqname='%s_iisq' % table._tablename
query=query.replace(INGRES_SEQNAME, tmp_seqname)
self.execute('create sequence %s' % tmp_seqname)
self.execute(query)
self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
def lastrowid(self,table):
tmp_seqname='%s_iisq' % table
self.execute('select current value for %s' % tmp_seqname)
return long(self.cursor.fetchone()[0]) # don't really need int type cast here...
class IngresUnicodeAdapter(IngresAdapter):
drivers = ('pyodbc',)
types = {
'boolean': 'CHAR(1)',
'string': 'NVARCHAR(%(length)s)',
'text': 'NCLOB',
'json': 'NCLOB',
'password': 'NVARCHAR(%(length)s)', ## Not sure what this contains utf8 or nvarchar. Or even bytes?
'blob': 'BLOB',
'upload': 'VARCHAR(%(length)s)', ## FIXME utf8 or nvarchar... or blob? what is this type?
'integer': 'INTEGER4', # or int8...
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT8',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'ANSIDATE',
'time': 'TIME WITHOUT TIME ZONE',
'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'NCLOB',
'list:string': 'NCLOB',
'list:reference': 'NCLOB',
'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO
}

575
gluon/dal/adapters/mongo.py
View File

@@ -1,575 +0,0 @@
# -*- coding: utf-8 -*-
import datetime
import logging
import re
from .._globals import IDENTITY
from ..objects import Table, Query, Field, Expression
from ..helpers.classes import SQLALL
from ..helpers.methods import xorify
from .base import NoSQLAdapter
class MongoDBAdapter(NoSQLAdapter):
drivers = ('pymongo',)
driver_auto_json = ['loads','dumps']
uploads_in_blob = False
types = {
'boolean': bool,
'string': str,
'text': str,
'json': str,
'password': str,
'blob': str,
'upload': str,
'integer': long,
'bigint': long,
'float': float,
'double': float,
'date': datetime.date,
'time': datetime.time,
'datetime': datetime.datetime,
'id': long,
'reference': long,
'list:string': list,
'list:integer': list,
'list:reference': list,
}
error_messages = {"javascript_needed": "This must yet be replaced" +
" with javascript in order to work."}
def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
pool_size=0, folder=None, db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.uri = uri
if do_connect: self.find_driver(adapter_args)
import random
from bson.objectid import ObjectId
from bson.son import SON
import pymongo.uri_parser
m = pymongo.uri_parser.parse_uri(uri)
self.SON = SON
self.ObjectId = ObjectId
self.random = random
self.dbengine = 'mongodb'
self.folder = folder
db['_lastsql'] = ''
self.db_codec = 'UTF-8'
self._after_connection = after_connection
self.pool_size = pool_size
#this is the minimum amount of replicates that it should wait
# for on insert/update
self.minimumreplication = adapter_args.get('minimumreplication',0)
# by default all inserts and selects are performand asynchronous,
# but now the default is
# synchronous, except when overruled by either this default or
# function parameter
self.safe = adapter_args.get('safe',True)
# load user setting for uploads in blob storage
self.uploads_in_blob = adapter_args.get('uploads_in_blob', False)
if isinstance(m,tuple):
m = {"database" : m[1]}
if m.get('database') is None:
raise SyntaxError("Database is required!")
def connector(uri=self.uri,m=m):
# Connection() is deprecated
if hasattr(self.driver, "MongoClient"):
Connection = self.driver.MongoClient
else:
Connection = self.driver.Connection
return Connection(uri)[m.get('database')]
self.reconnect(connector,cursor=False)
def object_id(self, arg=None):
""" Convert input to a valid Mongodb ObjectId instance
self.object_id("<random>") -> ObjectId (not unique) instance """
if not arg:
arg = 0
if isinstance(arg, basestring):
# we assume an integer as default input
rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
if arg.isdigit() and (not rawhex):
arg = int(arg)
elif arg == "<random>":
arg = int("0x%sL" % \
"".join([self.random.choice("0123456789abcdef") \
for x in range(24)]), 0)
elif arg.isalnum():
if not arg.startswith("0x"):
arg = "0x%s" % arg
try:
arg = int(arg, 0)
except ValueError, e:
raise ValueError(
"invalid objectid argument string: %s" % e)
else:
raise ValueError("Invalid objectid argument string. " +
"Requires an integer or base 16 value")
elif isinstance(arg, self.ObjectId):
return arg
if not isinstance(arg, (int, long)):
raise TypeError("object_id argument must be of type " +
"ObjectId or an objectid representable integer")
hexvalue = hex(arg)[2:].rstrip('L').zfill(24)
return self.ObjectId(hexvalue)
def parse_reference(self, value, field_type):
# here we have to check for ObjectID before base parse
if isinstance(value, self.ObjectId):
value = long(str(value), 16)
return super(MongoDBAdapter,
self).parse_reference(value, field_type)
def parse_id(self, value, field_type):
if isinstance(value, self.ObjectId):
value = long(str(value), 16)
return super(MongoDBAdapter,
self).parse_id(value, field_type)
def represent(self, obj, fieldtype):
# the base adatpter does not support MongoDB ObjectId
if isinstance(obj, self.ObjectId):
value = obj
else:
value = NoSQLAdapter.represent(self, obj, fieldtype)
# reference types must be convert to ObjectID
if fieldtype =='date':
if value is None:
return value
# this piece of data can be stripped off based on the fieldtype
t = datetime.time(0, 0, 0)
# mongodb doesn't has a date object and so it must datetime,
# string or integer
return datetime.datetime.combine(value, t)
elif fieldtype == 'time':
if value is None:
return value
# this piece of data can be stripped of based on the fieldtype
d = datetime.date(2000, 1, 1)
# mongodb doesn't has a time object and so it must datetime,
# string or integer
return datetime.datetime.combine(d, value)
elif fieldtype == "blob":
if value is None:
return value
from bson import Binary
if not isinstance(value, Binary):
if not isinstance(value, basestring):
return Binary(str(value))
return Binary(value)
return value
elif (isinstance(fieldtype, basestring) and
fieldtype.startswith('list:')):
if fieldtype.startswith('list:reference'):
newval = []
for v in value:
newval.append(self.object_id(v))
return newval
return value
elif ((isinstance(fieldtype, basestring) and
fieldtype.startswith("reference")) or
(isinstance(fieldtype, Table)) or fieldtype=="id"):
value = self.object_id(value)
return value
def create_table(self, table, migrate=True, fake_migrate=False,
polymodel=None, isCapped=False):
if isCapped:
raise RuntimeError("Not implemented")
def count(self, query, distinct=None, snapshot=True):
if distinct:
raise RuntimeError("COUNT DISTINCT not supported")
if not isinstance(query,Query):
raise SyntaxError("Not Supported")
tablename = self.get_table(query)
return long(self.select(query,[self.db[tablename]._id], {},
count=True,snapshot=snapshot)['count'])
# Maybe it would be faster if we just implemented the pymongo
# .count() function which is probably quicker?
# therefor call __select() connection[table].find(query).count()
# Since this will probably reduce the return set?
def expand(self, expression, field_type=None):
if isinstance(expression, Query):
# any query using 'id':=
# set name as _id (as per pymongo/mongodb primary key)
# convert second arg to an objectid field
# (if its not already)
# if second arg is 0 convert to objectid
if isinstance(expression.first,Field) and \
((expression.first.type == 'id') or \
("reference" in expression.first.type)):
if expression.first.type == 'id':
expression.first.name = '_id'
# cast to Mongo ObjectId
if isinstance(expression.second, (tuple, list, set)):
expression.second = [self.object_id(item) for
item in expression.second]
else:
expression.second = self.object_id(expression.second)
result = expression.op(expression.first, expression.second)
if isinstance(expression, Field):
if expression.type=='id':
result = "_id"
else:
result = expression.name
elif isinstance(expression, (Expression, Query)):
if not expression.second is None:
result = expression.op(expression.first, expression.second)
elif not expression.first is None:
result = expression.op(expression.first)
elif not isinstance(expression.op, str):
result = expression.op()
else:
result = expression.op
elif field_type:
result = self.represent(expression,field_type)
elif isinstance(expression,(list,tuple)):
result = [self.represent(item,field_type) for
item in expression]
else:
result = expression
return result
def drop(self, table, mode=''):
ctable = self.connection[table._tablename]
ctable.drop()
def truncate(self, table, mode, safe=None):
if safe == None:
safe=self.safe
ctable = self.connection[table._tablename]
ctable.remove(None, safe=True)
def select(self, query, fields, attributes, count=False,
snapshot=False):
mongofields_dict = self.SON()
mongoqry_dict = {}
new_fields, mongosort_list = [], []
# try an orderby attribute
orderby = attributes.get('orderby', False)
limitby = attributes.get('limitby', False)
# distinct = attributes.get('distinct', False)
if 'for_update' in attributes:
logging.warn('mongodb does not support for_update')
for key in set(attributes.keys())-set(('limitby',
'orderby','for_update')):
if attributes[key] is not None:
logging.warn('select attribute not implemented: %s' % key)
if limitby:
limitby_skip, limitby_limit = limitby[0], int(limitby[1])
else:
limitby_skip = limitby_limit = 0
if orderby:
if isinstance(orderby, (list, tuple)):
orderby = xorify(orderby)
# !!!! need to add 'random'
for f in self.expand(orderby).split(','):
if f.startswith('-'):
mongosort_list.append((f[1:], -1))
else:
mongosort_list.append((f, 1))
for item in fields:
if isinstance(item, SQLALL):
new_fields += item._table
else:
new_fields.append(item)
fields = new_fields
if isinstance(query,Query):
tablename = self.get_table(query)
elif len(fields) != 0:
tablename = fields[0].tablename
else:
raise SyntaxError("The table name could not be found in " +
"the query nor from the select statement.")
mongoqry_dict = self.expand(query)
fields = fields or self.db[tablename]
for field in fields:
mongofields_dict[field.name] = 1
ctable = self.connection[tablename]
if count:
return {'count' : ctable.find(
mongoqry_dict, mongofields_dict,
skip=limitby_skip, limit=limitby_limit,
sort=mongosort_list, snapshot=snapshot).count()}
else:
# pymongo cursor object
mongo_list_dicts = ctable.find(mongoqry_dict,
mongofields_dict, skip=limitby_skip,
limit=limitby_limit, sort=mongosort_list,
snapshot=snapshot)
rows = []
# populate row in proper order
# Here we replace ._id with .id to follow the standard naming
colnames = []
newnames = []
for field in fields:
colname = str(field)
colnames.append(colname)
tablename, fieldname = colname.split(".")
if fieldname == "_id":
# Mongodb reserved uuid key
field.name = "id"
newnames.append(".".join((tablename, field.name)))
for record in mongo_list_dicts:
row=[]
for colname in colnames:
tablename, fieldname = colname.split(".")
# switch to Mongo _id uuids for retrieving
# record id's
if fieldname == "id": fieldname = "_id"
if fieldname in record:
value = record[fieldname]
else:
value = None
row.append(value)
rows.append(row)
processor = attributes.get('processor', self.parse)
result = processor(rows, fields, newnames, False)
return result
def insert(self, table, fields, safe=None):
"""Safe determines whether a asynchronous request is done or a
synchronous action is done
For safety, we use by default synchronous requests"""
values = dict()
if safe is None:
safe = self.safe
ctable = self.connection[table._tablename]
for k, v in fields:
if not k.name in ["id", "safe"]:
fieldname = k.name
fieldtype = table[k.name].type
values[fieldname] = self.represent(v, fieldtype)
ctable.insert(values, safe=safe)
return long(str(values['_id']), 16)
def update(self, tablename, query, fields, safe=None):
if safe == None:
safe = self.safe
# return amount of adjusted rows or zero, but no exceptions
# @ related not finding the result
if not isinstance(query, Query):
raise RuntimeError("Not implemented")
amount = self.count(query, False)
if not isinstance(query, Query):
raise SyntaxError("Not Supported")
filter = None
if query:
filter = self.expand(query)
# do not try to update id fields to avoid backend errors
modify = {'$set': dict((k.name, self.represent(v, k.type)) for
k, v in fields if (not k.name in ("_id", "id")))}
try:
result = self.connection[tablename].update(filter,
modify, multi=True, safe=safe)
if safe:
try:
# if result count is available fetch it
return result["n"]
except (KeyError, AttributeError, TypeError):
return amount
else:
return amount
except Exception, e:
# TODO Reverse update query to verifiy that the query succeded
raise RuntimeError("uncaught exception when updating rows: %s" % e)
def delete(self, tablename, query, safe=None):
if safe is None:
safe = self.safe
amount = 0
amount = self.count(query, False)
if not isinstance(query, Query):
raise RuntimeError("query type %s is not supported" % \
type(query))
filter = self.expand(query)
self.connection[tablename].remove(filter, safe=safe)
return amount
def bulk_insert(self, table, items):
return [self.insert(table,item) for item in items]
## OPERATORS
def INVERT(self, first):
#print "in invert first=%s" % first
return '-%s' % self.expand(first)
# TODO This will probably not work:(
def NOT(self, first):
return {'$not': self.expand(first)}
def AND(self,first,second):
# pymongo expects: .find({'$and': [{'x':'1'}, {'y':'2'}]})
return {'$and': [self.expand(first),self.expand(second)]}
def OR(self,first,second):
# pymongo expects: .find({'$or': [{'name':'1'}, {'name':'2'}]})
return {'$or': [self.expand(first),self.expand(second)]}
def BELONGS(self, first, second):
if isinstance(second, str):
return {self.expand(first) : {"$in" : [ second[:-1]]} }
elif second==[] or second==() or second==set():
return {1:0}
items = [self.expand(item, first.type) for item in second]
return {self.expand(first) : {"$in" : items} }
def EQ(self,first,second=None):
result = {}
result[self.expand(first)] = self.expand(second)
return result
def NE(self, first, second=None):
result = {}
result[self.expand(first)] = {'$ne': self.expand(second)}
return result
def LT(self,first,second=None):
if second is None:
raise RuntimeError("Cannot compare %s < None" % first)
result = {}
result[self.expand(first)] = {'$lt': self.expand(second)}
return result
def LE(self,first,second=None):
if second is None:
raise RuntimeError("Cannot compare %s <= None" % first)
result = {}
result[self.expand(first)] = {'$lte': self.expand(second)}
return result
def GT(self,first,second):
result = {}
result[self.expand(first)] = {'$gt': self.expand(second)}
return result
def GE(self,first,second=None):
if second is None:
raise RuntimeError("Cannot compare %s >= None" % first)
result = {}
result[self.expand(first)] = {'$gte': self.expand(second)}
return result
def ADD(self, first, second):
raise NotImplementedError(self.error_messages["javascript_needed"])
return '%s + %s' % (self.expand(first),
self.expand(second, first.type))
def SUB(self, first, second):
raise NotImplementedError(self.error_messages["javascript_needed"])
return '(%s - %s)' % (self.expand(first),
self.expand(second, first.type))
def MUL(self, first, second):
raise NotImplementedError(self.error_messages["javascript_needed"])
return '(%s * %s)' % (self.expand(first),
self.expand(second, first.type))
def DIV(self, first, second):
raise NotImplementedError(self.error_messages["javascript_needed"])
return '(%s / %s)' % (self.expand(first),
self.expand(second, first.type))
def MOD(self, first, second):
raise NotImplementedError(self.error_messages["javascript_needed"])
return '(%s %% %s)' % (self.expand(first),
self.expand(second, first.type))
def AS(self, first, second):
raise NotImplementedError(self.error_messages["javascript_needed"])
return '%s AS %s' % (self.expand(first), second)
# We could implement an option that simulates a full featured SQL
# database. But I think the option should be set explicit or
# implemented as another library.
def ON(self, first, second):
raise NotImplementedError("This is not possible in NoSQL" +
" but can be simulated with a wrapper.")
return '%s ON %s' % (self.expand(first), self.expand(second))
# BLOW ARE TWO IMPLEMENTATIONS OF THE SAME FUNCITONS
# WHICH ONE IS BEST?
def COMMA(self, first, second):
return '%s, %s' % (self.expand(first), self.expand(second))
def LIKE(self, first, second):
#escaping regex operators?
return {self.expand(first): ('%s' % \
self.expand(second, 'string').replace('%','/'))}
def ILIKE(self, first, second):
val = second if isinstance(second,self.ObjectId) else {
'$regex': second.replace('%', ''), '$options': 'i'}
return {self.expand(first): val}
def STARTSWITH(self, first, second):
#escaping regex operators?
return {self.expand(first): ('/^%s/' % \
self.expand(second, 'string'))}
def ENDSWITH(self, first, second):
#escaping regex operators?
return {self.expand(first): ('/%s^/' % \
self.expand(second, 'string'))}
def CONTAINS(self, first, second, case_sensitive=False):
# silently ignore, only case sensitive
# There is a technical difference, but mongodb doesn't support
# that, but the result will be the same
val = second if isinstance(second,self.ObjectId) else \
{'$regex':".*" + re.escape(self.expand(second, 'string')) + ".*"}
return {self.expand(first) : val}
def LIKE(self, first, second):
import re
return {self.expand(first): {'$regex': \
re.escape(self.expand(second,
'string')).replace('%','.*')}}
#TODO verify full compatibilty with official SQL Like operator
def STARTSWITH(self, first, second):
#TODO Solve almost the same problem as with endswith
import re
return {self.expand(first): {'$regex' : '^' +
re.escape(self.expand(second,
'string'))}}
#TODO verify full compatibilty with official SQL Like operator
def ENDSWITH(self, first, second):
#escaping regex operators?
#TODO if searched for a name like zsa_corbitt and the function
# is endswith('a') then this is also returned.
# Aldo it end with a t
import re
return {self.expand(first): {'$regex': \
re.escape(self.expand(second, 'string')) + '$'}}
#TODO verify full compatibilty with official oracle contains operator
def CONTAINS(self, first, second, case_sensitive=False):
# silently ignore, only case sensitive
#There is a technical difference, but mongodb doesn't support
# that, but the result will be the same
#TODO contains operators need to be transformed to Regex
return {self.expand(first) : {'$regex': \
".*" + re.escape(self.expand(second, 'string')) + ".*"}}

513
gluon/dal/adapters/mssql.py
View File

@@ -1,513 +0,0 @@
# -*- coding: utf-8 -*-
import re
import sys
from .._globals import IDENTITY, LOGGER
from ..helpers.methods import varquote_aux
from .base import BaseAdapter
class MSSQLAdapter(BaseAdapter):
drivers = ('pyodbc',)
T_SEP = 'T'
QUOTE_TEMPLATE = '"%s"'
types = {
'boolean': 'BIT',
'string': 'VARCHAR(%(length)s)',
'text': 'TEXT',
'json': 'TEXT',
'password': 'VARCHAR(%(length)s)',
'blob': 'IMAGE',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATETIME',
'time': 'CHAR(8)',
'datetime': 'DATETIME',
'id': 'INT IDENTITY PRIMARY KEY',
'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'TEXT',
'list:string': 'TEXT',
'list:reference': 'TEXT',
'geometry': 'geometry',
'geography': 'geography',
'big-id': 'BIGINT IDENTITY PRIMARY KEY',
'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def concat_add(self,tablename):
return '; ALTER TABLE %s ADD ' % tablename
def varquote(self,name):
return varquote_aux(name,'[%s]')
def EXTRACT(self,field,what):
return "DATEPART(%s,%s)" % (what, self.expand(field))
def LEFT_JOIN(self):
return 'LEFT OUTER JOIN'
def RANDOM(self):
return 'NEWID()'
def ALLOW_NULL(self):
return ' NULL'
def CAST(self, first, second):
return first # apparently no cast necessary in MSSQL
def SUBSTRING(self,field,parameters):
return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
def PRIMARY_KEY(self,key):
return 'PRIMARY KEY CLUSTERED (%s)' % key
def AGGREGATE(self, first, what):
if what == 'LENGTH':
what = 'LEN'
return "%s(%s)" % (what, self.expand(first))
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
sql_s += ' TOP %i' % lmax
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
TRUE = 1
FALSE = 0
REGEX_DSN = re.compile('^(?P<dsn>.+)$')
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, srid=4326,
after_connection=None):
self.db = db
self.dbengine = "mssql"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.srid = srid
self.find_or_make_work_folder()
# ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8
ruri = uri.split('://',1)[1]
if '@' not in ruri:
try:
m = self.REGEX_DSN.match(ruri)
if not m:
raise SyntaxError(
'Parsing uri string(%s) has no result' % self.uri)
dsn = m.group('dsn')
if not dsn:
raise SyntaxError('DSN required')
except SyntaxError:
e = sys.exc_info()[1]
LOGGER.error('NdGpatch error')
raise e
# was cnxn = 'DSN=%s' % dsn
cnxn = dsn
else:
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError(
"Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
port = m.group('port') or '1433'
# Parse the optional url name-value arg pairs after the '?'
# (in the form of arg1=value1&arg2=value2&...)
# Default values (drivers like FreeTDS insist on uppercase parameter keys)
argsdict = { 'DRIVER':'{SQL Server}' }
urlargs = m.group('urlargs') or ''
for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
% (host, port, db, user, password, urlargs)
def connector(cnxn=cnxn,driver_args=driver_args):
return self.driver.connect(cnxn,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def lastrowid(self,table):
#self.execute('SELECT @@IDENTITY;')
self.execute('SELECT SCOPE_IDENTITY();')
return long(self.cursor.fetchone()[0])
def rowslice(self,rows,minimum=0,maximum=None):
if maximum is None:
return rows[minimum:]
return rows[minimum:maximum]
def EPOCH(self, first):
return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
def CONCAT(self, *items):
return '(%s)' % ' + '.join(self.expand(x,'string') for x in items)
# GIS Spatial Extensions
# No STAsGeoJSON in MSSQL
def ST_ASTEXT(self, first):
return '%s.STAsText()' %(self.expand(first))
def ST_CONTAINS(self, first, second):
return '%s.STContains(%s)=1' %(self.expand(first), self.expand(second, first.type))
def ST_DISTANCE(self, first, second):
return '%s.STDistance(%s)' %(self.expand(first), self.expand(second, first.type))
def ST_EQUALS(self, first, second):
return '%s.STEquals(%s)=1' %(self.expand(first), self.expand(second, first.type))
def ST_INTERSECTS(self, first, second):
return '%s.STIntersects(%s)=1' %(self.expand(first), self.expand(second, first.type))
def ST_OVERLAPS(self, first, second):
return '%s.STOverlaps(%s)=1' %(self.expand(first), self.expand(second, first.type))
# no STSimplify in MSSQL
def ST_TOUCHES(self, first, second):
return '%s.STTouches(%s)=1' %(self.expand(first), self.expand(second, first.type))
def ST_WITHIN(self, first, second):
return '%s.STWithin(%s)=1' %(self.expand(first), self.expand(second, first.type))
def represent(self, obj, fieldtype):
field_is_type = fieldtype.startswith
if field_is_type('geometry'):
srid = 0 # MS SQL default srid for geometry
geotype, parms = fieldtype[:-1].split('(')
if parms:
srid = parms
return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
elif fieldtype == 'geography':
srid = 4326 # MS SQL default srid for geography
geotype, parms = fieldtype[:-1].split('(')
if parms:
srid = parms
return "geography::STGeomFromText('%s',%s)" %(obj, srid)
# else:
# raise SyntaxError('Invalid field type %s' %fieldtype)
return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
return BaseAdapter.represent(self, obj, fieldtype)
class MSSQL3Adapter(MSSQLAdapter):
"""Experimental support for pagination in MSSQL
Requires MSSQL >= 2005, uses `ROW_NUMBER()`
"""
types = {
'boolean': 'BIT',
'string': 'VARCHAR(%(length)s)',
'text': 'VARCHAR(MAX)',
'json': 'VARCHAR(MAX)',
'password': 'VARCHAR(%(length)s)',
'blob': 'IMAGE',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATETIME',
'time': 'TIME(7)',
'datetime': 'DATETIME',
'id': 'INT IDENTITY PRIMARY KEY',
'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'VARCHAR(MAX)',
'list:string': 'VARCHAR(MAX)',
'list:reference': 'VARCHAR(MAX)',
'geometry': 'geometry',
'geography': 'geography',
'big-id': 'BIGINT IDENTITY PRIMARY KEY',
'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
if lmin == 0:
sql_s += ' TOP %i' % lmax
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
lmin += 1
sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
sql_f_inner = [f for f in sql_f.split(',')]
sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
sql_f_iproxy = ', '.join(sql_f_iproxy)
sql_f_oproxy = ', '.join(sql_f_outer)
return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
def rowslice(self,rows,minimum=0,maximum=None):
return rows
class MSSQL4Adapter(MSSQLAdapter):
"""Support for "native" pagination
Requires MSSQL >= 2012, uses `OFFSET ... ROWS ... FETCH NEXT ... ROWS ONLY`
"""
types = {
'boolean': 'BIT',
'string': 'VARCHAR(%(length)s)',
'text': 'VARCHAR(MAX)',
'json': 'VARCHAR(MAX)',
'password': 'VARCHAR(%(length)s)',
'blob': 'IMAGE',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATETIME',
'time': 'TIME(7)',
'datetime': 'DATETIME',
'id': 'INT IDENTITY PRIMARY KEY',
'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'VARCHAR(MAX)',
'list:string': 'VARCHAR(MAX)',
'list:reference': 'VARCHAR(MAX)',
'geometry': 'geometry',
'geography': 'geography',
'big-id': 'BIGINT IDENTITY PRIMARY KEY',
'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
if lmin == 0:
#top is still slightly faster, especially because
#web2py's default to fetch references is to not specify
#an orderby clause
sql_s += ' TOP %i' % lmax
else:
if not sql_o:
#if there is no orderby, we can't use the brand new statements
#that being said, developer chose its own poison, so be it random
sql_o += ' ORDER BY %s' % self.RANDOM()
sql_o += ' OFFSET %i ROWS FETCH NEXT %i ROWS ONLY' % (lmin, lmax - lmin)
return 'SELECT %s %s FROM %s%s%s;' % \
(sql_s, sql_f, sql_t, sql_w, sql_o)
def rowslice(self,rows,minimum=0,maximum=None):
return rows
class MSSQL2Adapter(MSSQLAdapter):
drivers = ('pyodbc',)
types = {
'boolean': 'CHAR(1)',
'string': 'NVARCHAR(%(length)s)',
'text': 'NTEXT',
'json': 'NTEXT',
'password': 'NVARCHAR(%(length)s)',
'blob': 'IMAGE',
'upload': 'NVARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATETIME',
'time': 'CHAR(8)',
'datetime': 'DATETIME',
'id': 'INT IDENTITY PRIMARY KEY',
'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'NTEXT',
'list:string': 'NTEXT',
'list:reference': 'NTEXT',
'big-id': 'BIGINT IDENTITY PRIMARY KEY',
'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def represent(self, obj, fieldtype):
value = BaseAdapter.represent(self, obj, fieldtype)
if fieldtype in ('string','text', 'json') and value[:1]=="'":
value = 'N'+value
return value
def execute(self,a):
return self.log_execute(a.decode('utf8'))
class VerticaAdapter(MSSQLAdapter):
drivers = ('pyodbc',)
T_SEP = ' '
types = {
'boolean': 'BOOLEAN',
'string': 'VARCHAR(%(length)s)',
'text': 'BYTEA',
'json': 'VARCHAR(%(length)s)',
'password': 'VARCHAR(%(length)s)',
'blob': 'BYTEA',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'DOUBLE PRECISION',
'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'DATETIME',
'id': 'IDENTITY',
'reference': 'INT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'BYTEA',
'list:string': 'BYTEA',
'list:reference': 'BYTEA',
'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
}
def EXTRACT(self, first, what):
return "DATE_PART('%s', TIMESTAMP %s)" % (what, self.expand(first))
def _truncate(self, table, mode=''):
tablename = table._tablename
return ['TRUNCATE %s %s;' % (tablename, mode or '')]
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
return 'SELECT %s %s FROM %s%s%s;' % \
(sql_s, sql_f, sql_t, sql_w, sql_o)
def lastrowid(self,table):
self.execute('SELECT LAST_INSERT_ID();')
return long(self.cursor.fetchone()[0])
def execute(self, a):
return self.log_execute(a)
class SybaseAdapter(MSSQLAdapter):
drivers = ('Sybase',)
types = {
'boolean': 'BIT',
'string': 'CHAR VARYING(%(length)s)',
'text': 'TEXT',
'json': 'TEXT',
'password': 'CHAR VARYING(%(length)s)',
'blob': 'IMAGE',
'upload': 'CHAR VARYING(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATETIME',
'time': 'CHAR(8)',
'datetime': 'DATETIME',
'id': 'INT IDENTITY PRIMARY KEY',
'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'TEXT',
'list:string': 'TEXT',
'list:reference': 'TEXT',
'geometry': 'geometry',
'geography': 'geography',
'big-id': 'BIGINT IDENTITY PRIMARY KEY',
'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, srid=4326,
after_connection=None):
self.db = db
self.dbengine = "sybase"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.srid = srid
self.find_or_make_work_folder()
# ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8
ruri = uri.split('://',1)[1]
if '@' not in ruri:
try:
m = self.REGEX_DSN.match(ruri)
if not m:
raise SyntaxError(
'Parsing uri string(%s) has no result' % self.uri)
dsn = m.group('dsn')
if not dsn:
raise SyntaxError('DSN required')
except SyntaxError:
e = sys.exc_info()[1]
LOGGER.error('NdGpatch error')
raise e
else:
m = self.REGEX_URI.match(uri)
if not m:
raise SyntaxError(
"Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
port = m.group('port') or '1433'
dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
driver_args.update(user = credential_decoder(user),
password = credential_decoder(password))
def connector(dsn=dsn,driver_args=driver_args):
return self.driver.connect(dsn,**driver_args)
self.connector = connector
if do_connect: self.reconnect()

140
gluon/dal/adapters/mysql.py
View File

@@ -1,140 +0,0 @@
# -*- coding: utf-8 -*-
import re
from .._globals import IDENTITY
from ..helpers.methods import varquote_aux
from .base import BaseAdapter
class MySQLAdapter(BaseAdapter):
drivers = ('MySQLdb','pymysql', 'mysqlconnector')
commit_on_alter_table = True
support_distributed_transaction = True
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'LONGTEXT',
'json': 'LONGTEXT',
'password': 'VARCHAR(%(length)s)',
'blob': 'LONGBLOB',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'DOUBLE',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'DATETIME',
'id': 'INT AUTO_INCREMENT NOT NULL',
'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'LONGTEXT',
'list:string': 'LONGTEXT',
'list:reference': 'LONGTEXT',
'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT `FK_%(constraint_name)s` FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
}
QUOTE_TEMPLATE = "`%s`"
def varquote(self,name):
return varquote_aux(name,'`%s`')
def RANDOM(self):
return 'RAND()'
def SUBSTRING(self,field,parameters):
return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
parameters[0], parameters[1])
def EPOCH(self, first):
return "UNIX_TIMESTAMP(%s)" % self.expand(first)
def CONCAT(self, *items):
return 'CONCAT(%s)' % ','.join(self.expand(x,'string') for x in items)
def REGEXP(self,first,second):
return '(%s REGEXP %s)' % (self.expand(first),
self.expand(second,'string'))
def CAST(self, first, second):
if second=='LONGTEXT': second = 'CHAR'
return 'CAST(%s AS %s)' % (first, second)
def _drop(self,table,mode):
# breaks db integrity but without this mysql does not drop table
table_rname = table.sqlsafe
return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table_rname,
'SET FOREIGN_KEY_CHECKS=1;']
def _insert_empty(self, table):
return 'INSERT INTO %s VALUES (DEFAULT);' % (table.sqlsafe)
def distributed_transaction_begin(self,key):
self.execute('XA START;')
def prepare(self,key):
self.execute("XA END;")
self.execute("XA PREPARE;")
def commit_prepared(self,key):
self.execute("XA COMMIT;")
def rollback_prepared(self,key):
self.execute("XA ROLLBACK;")
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "mysql"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError(
"Invalid URI string in DAL: %s" % self.uri)
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
port = int(m.group('port') or '3306')
charset = m.group('charset') or 'utf8'
driver_args.update(db=db,
user=credential_decoder(user),
passwd=credential_decoder(password),
host=host,
port=port,
charset=charset)
def connector(driver_args=driver_args):
return self.driver.connect(**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.execute('SET FOREIGN_KEY_CHECKS=1;')
self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
def lastrowid(self,table):
self.execute('select last_insert_id();')
return int(self.cursor.fetchone()[0])

191
gluon/dal/adapters/oracle.py
View File

@@ -1,191 +0,0 @@
# -*- coding: utf-8 -*-
import base64
import datetime
import re
from .._globals import IDENTITY
from .._load import cx_Oracle
from .base import BaseAdapter
class OracleAdapter(BaseAdapter):
drivers = ('cx_Oracle',)
commit_on_alter_table = False
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR2(%(length)s)',
'text': 'CLOB',
'json': 'CLOB',
'password': 'VARCHAR2(%(length)s)',
'blob': 'CLOB',
'upload': 'VARCHAR2(%(length)s)',
'integer': 'INT',
'bigint': 'NUMBER',
'float': 'FLOAT',
'double': 'BINARY_DOUBLE',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'CHAR(8)',
'datetime': 'DATE',
'id': 'NUMBER PRIMARY KEY',
'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'CLOB',
'list:string': 'CLOB',
'list:reference': 'CLOB',
'big-id': 'NUMBER PRIMARY KEY',
'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def trigger_name(self,tablename):
return '%s_trigger' % tablename
def LEFT_JOIN(self):
return 'LEFT OUTER JOIN'
def RANDOM(self):
return 'dbms_random.value'
def NOT_NULL(self,default,field_type):
return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
def REGEXP(self, first, second):
return 'REGEXP_LIKE(%s, %s)' % (self.expand(first),
self.expand(second, 'string'))
def _drop(self,table,mode):
sequence_name = table._sequence_name
return ['DROP TABLE %s %s;' % (table.sqlsafe, mode), 'DROP SEQUENCE %s;' % sequence_name]
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
if len(sql_w) > 1:
sql_w_row = sql_w + ' AND w_row > %i' % lmin
else:
sql_w_row = 'WHERE w_row > %i' % lmin
return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def constraint_name(self, tablename, fieldname):
constraint_name = BaseAdapter.constraint_name(self, tablename, fieldname)
if len(constraint_name)>30:
constraint_name = '%s_%s__constraint' % (tablename[:10], fieldname[:7])
return constraint_name
def represent_exceptions(self, obj, fieldtype):
if fieldtype == 'blob':
obj = base64.b64encode(str(obj))
return ":CLOB('%s')" % obj
elif fieldtype == 'date':
if isinstance(obj, (datetime.date, datetime.datetime)):
obj = obj.isoformat()[:10]
else:
obj = str(obj)
return "to_date('%s','yyyy-mm-dd')" % obj
elif fieldtype == 'datetime':
if isinstance(obj, datetime.datetime):
obj = obj.isoformat()[:19].replace('T',' ')
elif isinstance(obj, datetime.date):
obj = obj.isoformat()[:10]+' 00:00:00'
else:
obj = str(obj)
return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
return None
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "oracle"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
if not 'threaded' in driver_args:
driver_args['threaded']=True
def connector(uri=ruri,driver_args=driver_args):
return self.driver.connect(uri,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
def execute(self, command, args=None):
args = args or []
i = 1
while True:
m = self.oracle_fix.match(command)
if not m:
break
command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
args.append(m.group('clob')[6:-2].replace("''", "'"))
i += 1
if command[-1:]==';':
command = command[:-1]
return self.log_execute(command, args)
def create_sequence_and_triggers(self, query, table, **args):
tablename = table._rname or table._tablename
id_name = table._id.name
sequence_name = table._sequence_name
trigger_name = table._trigger_name
self.execute(query)
self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
self.execute("""
CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
DECLARE
curr_val NUMBER;
diff_val NUMBER;
PRAGMA autonomous_transaction;
BEGIN
IF :NEW.%(id)s IS NOT NULL THEN
EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
diff_val := :NEW.%(id)s - curr_val - 1;
IF diff_val != 0 THEN
EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
END IF;
END IF;
SELECT %(sequence_name)s.nextval INTO :NEW.%(id)s FROM DUAL;
END;
""" % dict(trigger_name=trigger_name, tablename=tablename,
sequence_name=sequence_name,id=id_name))
def lastrowid(self,table):
sequence_name = table._sequence_name
self.execute('SELECT %s.currval FROM dual;' % sequence_name)
return long(self.cursor.fetchone()[0])
#def parse_value(self, value, field_type, blob_decode=True):
# if blob_decode and isinstance(value, cx_Oracle.LOB):
# try:
# value = value.read()
# except self.driver.ProgrammingError:
# # After a subsequent fetch the LOB value is not valid anymore
# pass
# return BaseAdapter.parse_value(self, value, field_type, blob_decode)
def _fetchall(self):
if any(x[1]==cx_Oracle.LOB for x in self.cursor.description):
return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
for c in r]) for r in self.cursor]
else:
return self.cursor.fetchall()
def sqlsafe_table(self, tablename, ot=None):
if ot is not None:
return (self.QUOTE_TEMPLATE + ' ' \
+ self.QUOTE_TEMPLATE) % (ot, tablename)
return self.QUOTE_TEMPLATE % tablename

420
gluon/dal/adapters/postgres.py
View File

@@ -1,420 +0,0 @@
# -*- coding: utf-8 -*-
import re
from .._load import psycopg2_adapt
from .._globals import IDENTITY, LOGGER
from ..helpers.methods import varquote_aux
from .base import BaseAdapter
class PostgreSQLAdapter(BaseAdapter):
drivers = ('psycopg2','pg8000')
QUOTE_TEMPLATE = '"%s"'
support_distributed_transaction = True
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'TEXT',
'json': 'TEXT',
'password': 'VARCHAR(%(length)s)',
'blob': 'BYTEA',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INTEGER',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT8',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'TIMESTAMP',
'id': 'SERIAL PRIMARY KEY',
'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'TEXT',
'list:string': 'TEXT',
'list:reference': 'TEXT',
'geometry': 'GEOMETRY',
'geography': 'GEOGRAPHY',
'big-id': 'BIGSERIAL PRIMARY KEY',
'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference FK': ', CONSTRAINT "FK_%(constraint_name)s" FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'reference TFK': ' CONSTRAINT "FK_%(foreign_table)s_PK" FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
}
def varquote(self,name):
return varquote_aux(name,'"%s"')
def adapt(self,obj):
if self.driver_name == 'psycopg2':
return psycopg2_adapt(obj).getquoted()
elif self.driver_name == 'pg8000':
return "'%s'" % str(obj).replace("%","%%").replace("'","''")
else:
return "'%s'" % str(obj).replace("'","''")
def sequence_name(self,table):
return self.QUOTE_TEMPLATE % (table + '_id_seq')
def RANDOM(self):
return 'RANDOM()'
def ADD(self, first, second):
t = first.type
if t in ('text','string','password', 'json', 'upload','blob'):
return '(%s || %s)' % (self.expand(first), self.expand(second, t))
else:
return '(%s + %s)' % (self.expand(first), self.expand(second, t))
def distributed_transaction_begin(self,key):
return
def prepare(self,key):
self.execute("PREPARE TRANSACTION '%s';" % key)
def commit_prepared(self,key):
self.execute("COMMIT PREPARED '%s';" % key)
def rollback_prepared(self,key):
self.execute("ROLLBACK PREPARED '%s';" % key)
def create_sequence_and_triggers(self, query, table, **args):
# following lines should only be executed if table._sequence_name does not exist
# self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
# self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
# % (table._tablename, table._fieldname, table._sequence_name))
self.execute(query)
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, srid=4326,
after_connection=None):
self.db = db
self.dbengine = "postgres"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.srid = srid
self.find_or_make_work_folder()
self._last_insert = None # for INSERT ... RETURNING ID
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError("Invalid URI string in DAL")
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
port = m.group('port') or '5432'
sslmode = m.group('sslmode')
if sslmode:
msg = ("dbname='%s' user='%s' host='%s' "
"port=%s password='%s' sslmode='%s'") \
% (db, user, host, port, password, sslmode)
else:
msg = ("dbname='%s' user='%s' host='%s' "
"port=%s password='%s'") \
% (db, user, host, port, password)
# choose diver according uri
if self.driver:
self.__version__ = "%s %s" % (self.driver.__name__,
self.driver.__version__)
else:
self.__version__ = None
def connector(msg=msg,driver_args=driver_args):
return self.driver.connect(msg,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.connection.set_client_encoding('UTF8')
self.execute("SET standard_conforming_strings=on;")
self.try_json()
def _insert(self, table, fields):
table_rname = table.sqlsafe
if fields:
keys = ','.join(f.sqlsafe_name for f, v in fields)
values = ','.join(self.expand(v, f.type) for f, v in fields)
if table._id:
self._last_insert = (table._id, 1)
return 'INSERT INTO %s(%s) VALUES (%s) RETURNING %s;' % (
table_rname, keys, values, table._id.name)
else:
self._last_insert = None
return 'INSERT INTO %s(%s) VALUES (%s);' % (table_rname, keys, values)
else:
self._last_insert
return self._insert_empty(table)
def lastrowid(self, table=None):
if self._last_insert:
return int(self.cursor.fetchone()[0])
else:
self.execute("select lastval()")
return int(self.cursor.fetchone()[0])
def try_json(self):
# check JSON data type support
# (to be added to after_connection)
# until pg8000 supports json, leave this commented
#if self.driver_name == "pg8000":
# supports_json = self.connection.server_version >= "9.2.0"
if (self.driver_name == "psycopg2" and
self.driver.__version__ >= "2.0.12"):
supports_json = self.connection.server_version >= 90200
elif self.driver_name == "zxJDBC":
supports_json = self.connection.dbversion >= "9.2.0"
else:
supports_json = None
if supports_json:
self.types["json"] = "JSON"
if (self.driver_name == "psycopg2" and
self.driver.__version__ >= '2.5.0'):
self.driver_auto_json = ['loads']
else:
LOGGER.debug("Your database version does not support the JSON"
" data type (using TEXT instead)")
def LIKE(self,first,second):
args = (self.expand(first), self.expand(second,'string'))
if not first.type in ('string', 'text', 'json'):
return '(%s LIKE %s)' % (
self.CAST(args[0], 'CHAR(%s)' % first.length), args[1])
else:
return '(%s LIKE %s)' % args
def ILIKE(self,first,second):
args = (self.expand(first), self.expand(second,'string'))
if not first.type in ('string', 'text', 'json'):
return '(%s LIKE %s)' % (
self.CAST(args[0], 'CHAR(%s)' % first.length), args[1])
else:
return '(%s ILIKE %s)' % args
def REGEXP(self,first,second):
return '(%s ~ %s)' % (self.expand(first),
self.expand(second,'string'))
# GIS functions
def ST_ASGEOJSON(self, first, second):
"""
http://postgis.org/docs/ST_AsGeoJSON.html
"""
return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
self.expand(first), second['precision'], second['options'])
def ST_ASTEXT(self, first):
"""
http://postgis.org/docs/ST_AsText.html
"""
return 'ST_AsText(%s)' %(self.expand(first))
def ST_X(self, first):
"""
http://postgis.org/docs/ST_X.html
"""
return 'ST_X(%s)' %(self.expand(first))
def ST_Y(self, first):
"""
http://postgis.org/docs/ST_Y.html
"""
return 'ST_Y(%s)' %(self.expand(first))
def ST_CONTAINS(self, first, second):
"""
http://postgis.org/docs/ST_Contains.html
"""
return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_DISTANCE(self, first, second):
"""
http://postgis.org/docs/ST_Distance.html
"""
return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_EQUALS(self, first, second):
"""
http://postgis.org/docs/ST_Equals.html
"""
return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_INTERSECTS(self, first, second):
"""
http://postgis.org/docs/ST_Intersects.html
"""
return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_OVERLAPS(self, first, second):
"""
http://postgis.org/docs/ST_Overlaps.html
"""
return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_SIMPLIFY(self, first, second):
"""
http://postgis.org/docs/ST_Simplify.html
"""
return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
def ST_TOUCHES(self, first, second):
"""
http://postgis.org/docs/ST_Touches.html
"""
return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_WITHIN(self, first, second):
"""
http://postgis.org/docs/ST_Within.html
"""
return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
def ST_DWITHIN(self, first, (second, third)):
"""
http://postgis.org/docs/ST_DWithin.html
"""
return 'ST_DWithin(%s,%s,%s)' %(self.expand(first),
self.expand(second, first.type),
self.expand(third, 'double'))
def represent(self, obj, fieldtype):
field_is_type = fieldtype.startswith
if field_is_type('geo'):
srid = 4326 # postGIS default srid for geometry
geotype, parms = fieldtype[:-1].split('(')
parms = parms.split(',')
if len(parms) >= 2:
schema, srid = parms[:2]
if field_is_type('geometry'):
value = "ST_GeomFromText('%s',%s)" %(obj, srid)
elif field_is_type('geography'):
value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
# else:
# raise SyntaxError('Invalid field type %s' %fieldtype)
return value
return BaseAdapter.represent(self, obj, fieldtype)
def _drop(self, table, mode='restrict'):
if mode not in ['restrict', 'cascade', '']:
raise ValueError('Invalid mode: %s' % mode)
return ['DROP TABLE ' + table.sqlsafe + ' ' + str(mode) + ';']
class NewPostgreSQLAdapter(PostgreSQLAdapter):
drivers = ('psycopg2','pg8000')
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'TEXT',
'json': 'TEXT',
'password': 'VARCHAR(%(length)s)',
'blob': 'BYTEA',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INTEGER',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'FLOAT8',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'TIMESTAMP',
'id': 'SERIAL PRIMARY KEY',
'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'BIGINT[]',
'list:string': 'TEXT[]',
'list:reference': 'BIGINT[]',
'geometry': 'GEOMETRY',
'geography': 'GEOGRAPHY',
'big-id': 'BIGSERIAL PRIMARY KEY',
'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
}
def parse_list_integers(self, value, field_type):
return value
def parse_list_references(self, value, field_type):
return [self.parse_reference(r, field_type[5:]) for r in value]
def parse_list_strings(self, value, field_type):
return value
def represent(self, obj, fieldtype):
field_is_type = fieldtype.startswith
if field_is_type('list:'):
if not obj:
obj = []
elif not isinstance(obj, (list, tuple)):
obj = [obj]
if field_is_type('list:string'):
obj = map(str,obj)
else:
obj = map(int,obj)
return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
return BaseAdapter.represent(self, obj, fieldtype)
class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
drivers = ('zxJDBC',)
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None ):
self.db = db
self.dbengine = "postgres"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError("Invalid URI string in DAL")
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
port = m.group('port') or '5432'
msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
def connector(msg=msg,driver_args=driver_args):
return self.driver.connect(*msg,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.connection.set_client_encoding('UTF8')
self.execute('BEGIN;')
self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
self.try_json()

97
gluon/dal/adapters/sapdb.py
View File

@@ -1,97 +0,0 @@
# -*- coding: utf-8 -*-
import re
from .._globals import IDENTITY
from .base import BaseAdapter
class SAPDBAdapter(BaseAdapter):
drivers = ('sapdb',)
support_distributed_transaction = False
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'LONG',
'json': 'LONG',
'password': 'VARCHAR(%(length)s)',
'blob': 'LONG',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'FLOAT',
'double': 'DOUBLE PRECISION',
'decimal': 'FIXED(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'TIMESTAMP',
'id': 'INT PRIMARY KEY',
'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
'list:integer': 'LONG',
'list:string': 'LONG',
'list:reference': 'LONG',
'big-id': 'BIGINT PRIMARY KEY',
'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
}
def sequence_name(self,table):
return (self.QUOTE_TEMPLATE + '_id_Seq') % table
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
if len(sql_w) > 1:
sql_w_row = sql_w + ' AND w_row > %i' % lmin
else:
sql_w_row = 'WHERE w_row > %i' % lmin
return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def create_sequence_and_triggers(self, query, table, **args):
# following lines should only be executed if table._sequence_name does not exist
self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
% (table._tablename, table._id.name, table._sequence_name))
self.execute(query)
REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "sapdb"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://',1)[1]
m = self.REGEX_URI.match(ruri)
if not m:
raise SyntaxError("Invalid URI string in DAL")
user = credential_decoder(m.group('user'))
if not user:
raise SyntaxError('User required')
password = credential_decoder(m.group('password'))
if not password:
password = ''
host = m.group('host')
if not host:
raise SyntaxError('Host name required')
db = m.group('db')
if not db:
raise SyntaxError('Database name required')
def connector(user=user, password=password, database=db,
host=host, driver_args=driver_args):
return self.driver.Connection(user, password, database,
host, **driver_args)
self.connector = connector
if do_connect: self.reconnect()
def lastrowid(self,table):
self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
return long(self.cursor.fetchone()[0])

280
gluon/dal/adapters/sqlite.py
View File

@@ -1,280 +0,0 @@
# -*- coding: utf-8 -*-
import copy
import datetime
import locale
import platform
import re
import sys
import time
from .._compat import PY2, pjoin
from .._globals import IDENTITY
from .base import BaseAdapter
class SQLiteAdapter(BaseAdapter):
drivers = ('sqlite2','sqlite3')
can_select_for_update = None # support ourselves with BEGIN TRANSACTION
def EXTRACT(self,field,what):
return "web2py_extract('%s',%s)" % (what, self.expand(field))
@staticmethod
def web2py_extract(lookup, s):
table = {
'year': (0, 4),
'month': (5, 7),
'day': (8, 10),
'hour': (11, 13),
'minute': (14, 16),
'second': (17, 19),
}
try:
if lookup != 'epoch':
(i, j) = table[lookup]
return int(s[i:j])
else:
return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
except:
return None
@staticmethod
def web2py_regexp(expression, item):
return re.compile(expression).search(item) is not None
def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "sqlite"
self.uri = uri
self.adapter_args = adapter_args
if do_connect: self.find_driver(adapter_args)
self.pool_size = 0
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
path_encoding = sys.getfilesystemencoding() \
or locale.getdefaultlocale()[1] or 'utf8'
if uri.startswith('sqlite:memory'):
self.dbpath = ':memory:'
else:
self.dbpath = uri.split('://',1)[1]
if self.dbpath[0] != '/':
if PY2:
self.dbpath = pjoin(
self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
else:
self.dbpath = pjoin(self.folder, self.dbpath)
if not 'check_same_thread' in driver_args:
driver_args['check_same_thread'] = False
if not 'detect_types' in driver_args and do_connect:
driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
def connector(dbpath=self.dbpath, driver_args=driver_args):
return self.driver.Connection(dbpath, **driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.connection.create_function('web2py_extract', 2,
SQLiteAdapter.web2py_extract)
self.connection.create_function("REGEXP", 2,
SQLiteAdapter.web2py_regexp)
if self.adapter_args.get('foreign_keys',True):
self.execute('PRAGMA foreign_keys=ON;')
def _truncate(self, table, mode=''):
tablename = table._tablename
return ['DELETE FROM %s;' % tablename,
"DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
def lastrowid(self, table):
return self.cursor.lastrowid
def REGEXP(self,first,second):
return '(%s REGEXP %s)' % (self.expand(first),
self.expand(second,'string'))
def delete(self, tablename, query):
# SQLite requires its own delete to handle CASCADE
db = self.db
table = db[tablename]
deleted = [x[table._id.name] for x in db(query).select(table._id)]
counter = super(SQLiteAdapter, self).delete(tablename, query)
if counter:
for field in table._referenced_by:
if field.type == 'reference '+ tablename \
and field.ondelete == 'CASCADE':
db(field.belongs(deleted)).delete()
return counter
def select(self, query, fields, attributes):
"""
Simulate `SELECT ... FOR UPDATE` with `BEGIN IMMEDIATE TRANSACTION`.
Note that the entire database, rather than one record, is locked
(it will be locked eventually anyway by the following UPDATE).
"""
if attributes.get('for_update', False) and not 'cache' in attributes:
self.execute('BEGIN IMMEDIATE TRANSACTION;')
return super(SQLiteAdapter, self).select(query, fields, attributes)
SPATIALLIBS = {
'Windows':'libspatialite',
'Linux':'libspatialite.so',
'Darwin':'libspatialite.dylib'
}
class SpatiaLiteAdapter(SQLiteAdapter):
drivers = ('sqlite3','sqlite2')
types = copy.copy(BaseAdapter.types)
types.update(geometry='GEOMETRY')
def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, srid=4326, after_connection=None):
self.db = db
self.dbengine = "spatialite"
self.uri = uri
if do_connect: self.find_driver(adapter_args)
self.pool_size = 0
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
self.srid = srid
path_encoding = sys.getfilesystemencoding() \
or locale.getdefaultlocale()[1] or 'utf8'
if uri.startswith('spatialite:memory'):
self.dbpath = ':memory:'
else:
self.dbpath = uri.split('://',1)[1]
if self.dbpath[0] != '/':
self.dbpath = pjoin(
self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
if not 'check_same_thread' in driver_args:
driver_args['check_same_thread'] = False
if not 'detect_types' in driver_args and do_connect:
driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
def connector(dbpath=self.dbpath, driver_args=driver_args):
return self.driver.Connection(dbpath, **driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
self.connection.enable_load_extension(True)
# for Windows, rename libspatialite-2.dll to libspatialite.dll
# Linux uses libspatialite.so
# Mac OS X uses libspatialite.dylib
libspatialite = SPATIALLIBS[platform.system()]
self.execute(r'SELECT load_extension("%s");' % libspatialite)
self.connection.create_function('web2py_extract', 2,
SQLiteAdapter.web2py_extract)
self.connection.create_function("REGEXP", 2,
SQLiteAdapter.web2py_regexp)
# GIS functions
def ST_ASGEOJSON(self, first, second):
return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
second['precision'], second['options'])
def ST_ASTEXT(self, first):
return 'AsText(%s)' %(self.expand(first))
def ST_CONTAINS(self, first, second):
return 'Contains(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def ST_DISTANCE(self, first, second):
return 'Distance(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def ST_EQUALS(self, first, second):
return 'Equals(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def ST_INTERSECTS(self, first, second):
return 'Intersects(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def ST_OVERLAPS(self, first, second):
return 'Overlaps(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def ST_SIMPLIFY(self, first, second):
return 'Simplify(%s,%s)' %(self.expand(first),
self.expand(second, 'double'))
def ST_TOUCHES(self, first, second):
return 'Touches(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def ST_WITHIN(self, first, second):
return 'Within(%s,%s)' %(self.expand(first),
self.expand(second, first.type))
def represent(self, obj, fieldtype):
field_is_type = fieldtype.startswith
if field_is_type('geo'):
srid = 4326 # Spatialite default srid for geometry
geotype, parms = fieldtype[:-1].split('(')
parms = parms.split(',')
if len(parms) >= 2:
schema, srid = parms[:2]
# if field_is_type('geometry'):
value = "ST_GeomFromText('%s',%s)" %(obj, srid)
# elif field_is_type('geography'):
# value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
# else:
# raise SyntaxError, 'Invalid field type %s' %fieldtype
return value
return BaseAdapter.represent(self, obj, fieldtype)
class JDBCSQLiteAdapter(SQLiteAdapter):
drivers = ('zxJDBC_sqlite',)
def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "sqlite"
self.uri = uri
if do_connect: self.find_driver(adapter_args)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
path_encoding = sys.getfilesystemencoding() \
or locale.getdefaultlocale()[1] or 'utf8'
if uri.startswith('sqlite:memory'):
self.dbpath = ':memory:'
else:
self.dbpath = uri.split('://',1)[1]
if self.dbpath[0] != '/':
self.dbpath = pjoin(
self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
def connector(dbpath=self.dbpath,driver_args=driver_args):
return self.driver.connect(
self.driver.getConnection('jdbc:sqlite:'+dbpath),
**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def after_connection(self):
# FIXME http://www.zentus.com/sqlitejdbc/custom_functions.html for UDFs
self.connection.create_function('web2py_extract', 2,
SQLiteAdapter.web2py_extract)
def execute(self, a):
return self.log_execute(a)

76
gluon/dal/adapters/teradata.py
View File

@@ -1,76 +0,0 @@
# -*- coding: utf-8 -*-
from .._globals import IDENTITY
from ..connection import ConnectionPool
from .base import BaseAdapter
class TeradataAdapter(BaseAdapter):
drivers = ('pyodbc',)
types = {
'boolean': 'CHAR(1)',
'string': 'VARCHAR(%(length)s)',
'text': 'VARCHAR(2000)',
'json': 'VARCHAR(4000)',
'password': 'VARCHAR(%(length)s)',
'blob': 'BLOB',
'upload': 'VARCHAR(%(length)s)',
'integer': 'INT',
'bigint': 'BIGINT',
'float': 'REAL',
'double': 'DOUBLE',
'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
'date': 'DATE',
'time': 'TIME',
'datetime': 'TIMESTAMP',
# Modified Constraint syntax for Teradata.
# Teradata does not support ON DELETE.
'id': 'INT GENERATED ALWAYS AS IDENTITY', # Teradata Specific
'reference': 'INT',
'list:integer': 'VARCHAR(4000)',
'list:string': 'VARCHAR(4000)',
'list:reference': 'VARCHAR(4000)',
'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY', # Teradata Specific
'big-reference': 'BIGINT',
'reference FK': ' REFERENCES %(foreign_key)s',
'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
}
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None):
self.db = db
self.dbengine = "teradata"
self.uri = uri
if do_connect: self.find_driver(adapter_args,uri)
self.pool_size = pool_size
self.folder = folder
self.db_codec = db_codec
self._after_connection = after_connection
self.find_or_make_work_folder()
ruri = uri.split('://', 1)[1]
def connector(cnxn=ruri,driver_args=driver_args):
return self.driver.connect(cnxn,**driver_args)
self.connector = connector
if do_connect: self.reconnect()
def close(self,action='commit',really=True):
# Teradata does not implicitly close off the cursor
# leading to SQL_ACTIVE_STATEMENTS limit errors
self.cursor.close()
ConnectionPool.close(self, action, really)
def LEFT_JOIN(self):
return 'LEFT OUTER JOIN'
# Similar to MSSQL, Teradata can't specify a range (for Pageby)
def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
if limitby:
(lmin, lmax) = limitby
sql_s += ' TOP %i' % lmax
return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
def _truncate(self, table, mode=''):
tablename = table._tablename
return ['DELETE FROM %s ALL;' % (tablename)]

1095
gluon/dal/base.py
View File

File diff suppressed because it is too large Load Diff

116
gluon/dal/connection.py
View File

@@ -1,116 +0,0 @@
# -*- coding: utf-8 -*-
import os
from ._compat import exists
from ._globals import GLOBAL_LOCKER, THREAD_LOCAL
from .helpers.classes import UseDatabaseStoredFile
class ConnectionPool(object):
POOLS = {}
check_active_connection = True
@staticmethod
def set_folder(folder):
THREAD_LOCAL.folder = folder
# ## this allows gluon to commit/rollback all dbs in this thread
def close(self,action='commit',really=True):
if action:
if callable(action):
action(self)
else:
getattr(self, action)()
# ## if you want pools, recycle this connection
if self.pool_size:
GLOBAL_LOCKER.acquire()
pool = ConnectionPool.POOLS[self.uri]
if len(pool) < self.pool_size:
pool.append(self.connection)
really = False
GLOBAL_LOCKER.release()
if really:
self.close_connection()
self.connection = None
@staticmethod
def close_all_instances(action):
""" to close cleanly databases in a multithreaded environment """
dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
for db_uid, db_group in dbs:
for db in db_group:
if hasattr(db,'_adapter'):
db._adapter.close(action)
getattr(THREAD_LOCAL,'db_instances',{}).clear()
getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
if callable(action):
action(None)
return
def find_or_make_work_folder(self):
#this actually does not make the folder. it has to be there
self.folder = getattr(THREAD_LOCAL,'folder','')
if (os.path.isabs(self.folder) and
isinstance(self, UseDatabaseStoredFile) and
self.folder.startswith(os.getcwd())):
self.folder = os.path.relpath(self.folder, os.getcwd())
# Creating the folder if it does not exist
if False and self.folder and not exists(self.folder):
os.mkdir(self.folder)
def after_connection_hook(self):
"""Hook for the after_connection parameter"""
if callable(self._after_connection):
self._after_connection(self)
self.after_connection()
def after_connection(self):
#this it is supposed to be overloaded by adapters
pass
def reconnect(self, f=None, cursor=True):
"""
Defines: `self.connection` and `self.cursor`
(if cursor is True)
if `self.pool_size>0` it will try pull the connection from the pool
if the connection is not active (closed by db server) it will loop
if not `self.pool_size` or no active connections in pool makes a new one
"""
if getattr(self,'connection', None) is not None:
return
if f is None:
f = self.connector
# if not hasattr(self, "driver") or self.driver is None:
# LOGGER.debug("Skipping connection since there's no driver")
# return
if not self.pool_size:
self.connection = f()
self.cursor = cursor and self.connection.cursor()
else:
uri = self.uri
POOLS = ConnectionPool.POOLS
while True:
GLOBAL_LOCKER.acquire()
if not uri in POOLS:
POOLS[uri] = []
if POOLS[uri]:
self.connection = POOLS[uri].pop()
GLOBAL_LOCKER.release()
self.cursor = cursor and self.connection.cursor()
try:
if self.cursor and self.check_active_connection:
self.execute('SELECT 1;')
break
except:
pass
else:
GLOBAL_LOCKER.release()
self.connection = f()
self.cursor = cursor and self.connection.cursor()
break
self.after_connection_hook()

0
gluon/dal/helpers/__init__.py
View File

298
gluon/dal/helpers/classes.py
View File

@@ -1,298 +0,0 @@
# -*- coding: utf-8 -*-
import copy
import marshal
import struct
import traceback
from .._compat import exists, copyreg
from .._globals import LOGGER
class Reference(long):
def __allocate(self):
if not self._record:
self._record = self._table[long(self)]
if not self._record:
raise RuntimeError(
"Using a recursive select but encountered a broken reference: %s %d"%(self._table, long(self)))
def __getattr__(self, key):
if key == 'id':
return long(self)
if key in self._table:
self.__allocate()
if self._record:
return self._record.get(key,None) # to deal with case self.update_record()
else:
return None
def get(self, key, default=None):
return self.__getattr__(key, default)
def __setattr__(self, key, value):
if key.startswith('_'):
long.__setattr__(self, key, value)
return
self.__allocate()
self._record[key] = value
def __getitem__(self, key):
if key == 'id':
return long(self)
self.__allocate()
return self._record.get(key, None)
def __setitem__(self,key,value):
self.__allocate()
self._record[key] = value
def Reference_unpickler(data):
return marshal.loads(data)
def Reference_pickler(data):
try:
marshal_dump = marshal.dumps(long(data))
except AttributeError:
marshal_dump = 'i%s' % struct.pack('<i', long(data))
return (Reference_unpickler, (marshal_dump,))
copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
class SQLCallableList(list):
def __call__(self):
return copy.copy(self)
class SQLALL(object):
"""
Helper class providing a comma-separated string having all the field names
(prefixed by table name and '.')
normally only called from within gluon.dal
"""
def __init__(self, table):
self._table = table
def __str__(self):
return ', '.join([str(field) for field in self._table])
class SQLCustomType(object):
"""
Allows defining of custom SQL types
Args:
type: the web2py type (default = 'string')
native: the backend type
encoder: how to encode the value to store it in the backend
decoder: how to decode the value retrieved from the backend
validator: what validators to use ( default = None, will use the
default validator for type)
Example::
Define as:
decimal = SQLCustomType(
type ='double',
native ='integer',
encoder =(lambda x: int(float(x) * 100)),
decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
)
db.define_table(
'example',
Field('value', type=decimal)
)
"""
def __init__(
self,
type='string',
native=None,
encoder=None,
decoder=None,
validator=None,
_class=None,
):
self.type = type
self.native = native
self.encoder = encoder or (lambda x: x)
self.decoder = decoder or (lambda x: x)
self.validator = validator
self._class = _class or type
def startswith(self, text=None):
try:
return self.type.startswith(self, text)
except TypeError:
return False
def endswith(self, text=None):
try:
return self.type.endswith(self, text)
except TypeError:
return False
def __getslice__(self, a=0, b=100):
return None
def __getitem__(self, i):
return None
def __str__(self):
return self._class
class RecordUpdater(object):
def __init__(self, colset, table, id):
self.colset, self.db, self.tablename, self.id = \
colset, table._db, table._tablename, id
def __call__(self, **fields):
colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
table = db[tablename]
newfields = fields or dict(colset)
for fieldname in newfields.keys():
if not fieldname in table.fields or table[fieldname].type=='id':
del newfields[fieldname]
table._db(table._id==id,ignore_common_filters=True).update(**newfields)
colset.update(newfields)
return colset
class RecordDeleter(object):
def __init__(self, table, id):
self.db, self.tablename, self.id = table._db, table._tablename, id
def __call__(self):
return self.db(self.db[self.tablename]._id==self.id).delete()
class MethodAdder(object):
def __init__(self,table):
self.table = table
def __call__(self):
return self.register()
def __getattr__(self,method_name):
return self.register(method_name)
def register(self,method_name=None):
def _decorated(f):
instance = self.table
import types
method = types.MethodType(f, instance, instance.__class__)
name = method_name or f.func_name
setattr(instance, name, method)
return f
return _decorated
class DatabaseStoredFile:
web2py_filesystems = set()
def escape(self,obj):
return self.db._adapter.escape(obj)
@staticmethod
def try_create_web2py_filesystem(db):
if not db._uri in DatabaseStoredFile.web2py_filesystems:
if db._adapter.dbengine == 'mysql':
sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
elif db._adapter.dbengine in ('postgres', 'sqlite'):
sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
db.executesql(sql)
DatabaseStoredFile.web2py_filesystems.add(db._uri)
def __init__(self,db,filename,mode):
if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'):
raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now")
self.db = db
self.filename = filename
self.mode = mode
DatabaseStoredFile.try_create_web2py_filesystem(db)
self.p=0
self.data = ''
if mode in ('r','rw','a'):
query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
% filename
rows = self.db.executesql(query)
if rows:
self.data = rows[0][0]
elif exists(filename):
datafile = open(filename, 'r')
try:
self.data = datafile.read()
finally:
datafile.close()
elif mode in ('r','rw'):
raise RuntimeError("File %s does not exist" % filename)
def read(self, bytes):
data = self.data[self.p:self.p+bytes]
self.p += len(data)
return data
def readline(self):
i = self.data.find('\n',self.p)+1
if i>0:
data, self.p = self.data[self.p:i], i
else:
data, self.p = self.data[self.p:], len(self.data)
return data
def write(self,data):
self.data += data
def close_connection(self):
if self.db is not None:
self.db.executesql(
"DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
% (self.filename, self.data.replace("'","''"))
self.db.executesql(query)
self.db.commit()
self.db = None
def close(self):
self.close_connection()
@staticmethod
def exists(db, filename):
if exists(filename):
return True
DatabaseStoredFile.try_create_web2py_filesystem(db)
query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
try:
if db.executesql(query):
return True
except Exception, e:
if not (db._adapter.isOperationalError(e) or
db._adapter.isProgrammingError(e)):
raise
# no web2py_filesystem found?
tb = traceback.format_exc()
LOGGER.error("Could not retrieve %s\n%s" % (filename, tb))
return False
class UseDatabaseStoredFile:
def file_exists(self, filename):
return DatabaseStoredFile.exists(self.db,filename)
def file_open(self, filename, mode='rb', lock=True):
return DatabaseStoredFile(self.db,filename,mode)
def file_close(self, fileobj):
fileobj.close_connection()
def file_delete(self,filename):
query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
self.db.executesql(query)
self.db.commit()

342
gluon/dal/helpers/methods.py
View File

@@ -1,342 +0,0 @@
# -*- coding: utf-8 -*-
import uuid
import re
from .regex import REGEX_NOPASSWD, REGEX_UNPACK, REGEX_CONST_STRING, REGEX_W
from .classes import SQLCustomType
#from ..objects import Field, Table
PLURALIZE_RULES = [
(re.compile('child$'), re.compile('child$'), 'children'),
(re.compile('oot$'), re.compile('oot$'), 'eet'),
(re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
(re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
(re.compile('sis$'), re.compile('sis$'), 'ses'),
(re.compile('man$'), re.compile('man$'), 'men'),
(re.compile('ife$'), re.compile('ife$'), 'ives'),
(re.compile('eau$'), re.compile('eau$'), 'eaux'),
(re.compile('lf$'), re.compile('lf$'), 'lves'),
(re.compile('[sxz]$'), re.compile('$'), 'es'),
(re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
(re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
(re.compile('$'), re.compile('$'), 's'),
]
def pluralize(singular, rules=PLURALIZE_RULES):
for line in rules:
re_search, re_sub, replace = line
plural = re_search.search(singular) and re_sub.sub(replace, singular)
if plural: return plural
def hide_password(uri):
if isinstance(uri,(list,tuple)):
return [hide_password(item) for item in uri]
return REGEX_NOPASSWD.sub('******',uri)
def cleanup(text):
"""
Validates that the given text is clean: only contains [0-9a-zA-Z_]
"""
#if not REGEX_ALPHANUMERIC.match(text):
# raise SyntaxError('invalid table or field name: %s' % text)
return text
def list_represent(x,r=None):
return ', '.join(str(y) for y in x or [])
def xorify(orderby):
if not orderby:
return None
orderby2 = orderby[0]
for item in orderby[1:]:
orderby2 = orderby2 | item
return orderby2
def use_common_filters(query):
return (query and hasattr(query,'ignore_common_filters') and \
not query.ignore_common_filters)
def bar_escape(item):
return str(item).replace('|', '||')
def bar_encode(items):
return '|%s|' % '|'.join(bar_escape(item) for item in items if str(item).strip())
def bar_decode_integer(value):
if not hasattr(value,'split') and hasattr(value,'read'):
value = value.read()
return [long(x) for x in value.split('|') if x.strip()]
def bar_decode_string(value):
return [x.replace('||', '|') for x in
REGEX_UNPACK.split(value[1:-1]) if x.strip()]
def archive_record(qset, fs, archive_table, current_record):
tablenames = qset.db._adapter.tables(qset.query)
if len(tablenames) != 1:
raise RuntimeError("cannot update join")
for row in qset.select():
fields = archive_table._filter_fields(row)
fields[current_record] = row.id
archive_table.insert(**fields)
return False
def smart_query(fields,text):
from ..objects import Field, Table
if not isinstance(fields,(list,tuple)):
fields = [fields]
new_fields = []
for field in fields:
if isinstance(field,Field):
new_fields.append(field)
elif isinstance(field,Table):
for ofield in field:
new_fields.append(ofield)
else:
raise RuntimeError("fields must be a list of fields")
fields = new_fields
field_map = {}
for field in fields:
n = field.name.lower()
if not n in field_map:
field_map[n] = field
n = str(field).lower()
if not n in field_map:
field_map[n] = field
constants = {}
i = 0
while True:
m = REGEX_CONST_STRING.search(text)
if not m: break
text = text[:m.start()]+('#%i' % i)+text[m.end():]
constants[str(i)] = m.group()[1:-1]
i+=1
text = re.sub('\s+',' ',text).lower()
for a,b in [('&','and'),
('|','or'),
('~','not'),
('==','='),
('<','<'),
('>','>'),
('<=','<='),
('>=','>='),
('<>','!='),
('=<','<='),
('=>','>='),
('=','='),
(' less or equal than ','<='),
(' greater or equal than ','>='),
(' equal or less than ','<='),
(' equal or greater than ','>='),
(' less or equal ','<='),
(' greater or equal ','>='),
(' equal or less ','<='),
(' equal or greater ','>='),
(' not equal to ','!='),
(' not equal ','!='),
(' equal to ','='),
(' equal ','='),
(' equals ','='),
(' less than ','<'),
(' greater than ','>'),
(' starts with ','startswith'),
(' ends with ','endswith'),
(' not in ' , 'notbelongs'),
(' in ' , 'belongs'),
(' is ','=')]:
if a[0]==' ':
text = text.replace(' is'+a,' %s ' % b)
text = text.replace(a,' %s ' % b)
text = re.sub('\s+',' ',text).lower()
text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
query = field = neg = op = logic = None
for item in text.split():
if field is None:
if item == 'not':
neg = True
elif not neg and not logic and item in ('and','or'):
logic = item
elif item in field_map:
field = field_map[item]
else:
raise RuntimeError("Invalid syntax")
elif not field is None and op is None:
op = item
elif not op is None:
if item.startswith('#'):
if not item[1:] in constants:
raise RuntimeError("Invalid syntax")
value = constants[item[1:]]
else:
value = item
if field.type in ('text', 'string', 'json'):
if op == '=': op = 'like'
if op == '=': new_query = field==value
elif op == '<': new_query = field<value
elif op == '>': new_query = field>value
elif op == '<=': new_query = field<=value
elif op == '>=': new_query = field>=value
elif op == '!=': new_query = field!=value
elif op == 'belongs': new_query = field.belongs(value.split(','))
elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
elif field.type in ('text', 'string', 'json'):
if op == 'contains': new_query = field.contains(value)
elif op == 'like': new_query = field.ilike(value)
elif op == 'startswith': new_query = field.startswith(value)
elif op == 'endswith': new_query = field.endswith(value)
else: raise RuntimeError("Invalid operation")
elif field._db._adapter.dbengine=='google:datastore' and \
field.type in ('list:integer', 'list:string', 'list:reference'):
if op == 'contains': new_query = field.contains(value)
else: raise RuntimeError("Invalid operation")
else: raise RuntimeError("Invalid operation")
if neg: new_query = ~new_query
if query is None:
query = new_query
elif logic == 'and':
query &= new_query
elif logic == 'or':
query |= new_query
field = op = neg = logic = None
return query
def sqlhtml_validators(field):
"""
Field type validation, using web2py's validators mechanism.
makes sure the content of a field is in line with the declared
fieldtype
"""
db = field.db
try:
from gluon import validators
except ImportError:
return []
field_type, field_length = field.type, field.length
if isinstance(field_type, SQLCustomType):
if hasattr(field_type, 'validator'):
return field_type.validator
else:
field_type = field_type.type
elif not isinstance(field_type,str):
return []
requires=[]
def ff(r,id):
row=r(id)
if not row:
return str(id)
elif hasattr(r, '_format') and isinstance(r._format,str):
return r._format % row
elif hasattr(r, '_format') and callable(r._format):
return r._format(row)
else:
return str(id)
if field_type in (('string', 'text', 'password')):
requires.append(validators.IS_LENGTH(field_length))
elif field_type == 'json':
requires.append(validators.IS_EMPTY_OR(validators.IS_JSON()))
elif field_type == 'double' or field_type == 'float':
requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
elif field_type == 'integer':
requires.append(validators.IS_INT_IN_RANGE(-2**31, 2**31))
elif field_type == 'bigint':
requires.append(validators.IS_INT_IN_RANGE(-2**63, 2**63))
elif field_type.startswith('decimal'):
requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
elif field_type == 'date':
requires.append(validators.IS_DATE())
elif field_type == 'time':
requires.append(validators.IS_TIME())
elif field_type == 'datetime':
requires.append(validators.IS_DATETIME())
elif db and field_type.startswith('reference') and \
field_type.find('.') < 0 and \
field_type[10:] in db.tables:
referenced = db[field_type[10:]]
def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
field.represent = field.represent or repr_ref
if hasattr(referenced, '_format') and referenced._format:
requires = validators.IS_IN_DB(db,referenced._id,
referenced._format)
if field.unique:
requires._and = validators.IS_NOT_IN_DB(db,field)
if field.tablename == field_type[10:]:
return validators.IS_EMPTY_OR(requires)
return requires
elif db and field_type.startswith('list:reference') and \
field_type.find('.') < 0 and \
field_type[15:] in db.tables:
referenced = db[field_type[15:]]
def list_ref_repr(ids, row=None, r=referenced, f=ff):
if not ids:
return None
from ..adapters.google import GoogleDatastoreAdapter
refs = None
db, id = r._db, r._id
if isinstance(db._adapter, GoogleDatastoreAdapter):
def count(values): return db(id.belongs(values)).select(id)
rx = range(0, len(ids), 30)
refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
else:
refs = db(id.belongs(ids)).select(id)
return (refs and ', '.join(f(r,x.id) for x in refs) or '')
field.represent = field.represent or list_ref_repr
if hasattr(referenced, '_format') and referenced._format:
requires = validators.IS_IN_DB(db,referenced._id,
referenced._format,multiple=True)
else:
requires = validators.IS_IN_DB(db,referenced._id,
multiple=True)
if field.unique:
requires._and = validators.IS_NOT_IN_DB(db,field)
if not field.notnull:
requires = validators.IS_EMPTY_OR(requires)
return requires
elif field_type.startswith('list:'):
def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
field.represent = field.represent or repr_list
if field.unique:
requires.append(validators.IS_NOT_IN_DB(db, field))
sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
if field.notnull and not field_type[:2] in sff:
requires.append(validators.IS_NOT_EMPTY())
elif not field.notnull and field_type[:2] in sff and requires:
requires[0] = validators.IS_EMPTY_OR(requires[0])
return requires
def varquote_aux(name,quotestr='%s'):
return name if REGEX_W.match(name) else quotestr % name
def uuid2int(uuidv):
return uuid.UUID(uuidv).int
def int2uuid(n):
return str(uuid.UUID(int=n))
# Geodal utils
def geoPoint(x, y):
return "POINT (%f %f)" % (x, y)
def geoLine(*line):
return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
def geoPolygon(*line):
return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)

22
gluon/dal/helpers/regex.py
View File

@@ -1,22 +0,0 @@
# -*- coding: utf-8 -*-
import re
REGEX_TYPE = re.compile('^([\w\_\:]+)')
REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
REGEX_W = re.compile('^\w+$')
REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.([^.]+)$')
REGEX_NO_GREEDY_ENTITY_NAME = r'(.+?)'
REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)(\.(?P<name>\w+))?\.\w+$')
REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
REGEX_QUOTES = re.compile("'[^']*'")
REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
REGEX_NOPASSWD = re.compile('\/\/[\w\.\-]+[\:\/](.+)(?=@)') # was '(?<=[\:\/])([^:@/]+)(?=@.+)'

2704
gluon/dal/objects.py
View File

File diff suppressed because it is too large Load Diff

6
gluon/main.py
View File

@@ -93,7 +93,7 @@ from gluon.globals import Request, Response, Session
from gluon.compileapp import build_environment, run_models_in, \
run_controller_in, run_view_in
from gluon.contenttype import contenttype
from gluon.dal.base import BaseAdapter
from gluon.pydal.base import BaseAdapter
from gluon.validators import CRYPT
from gluon.html import URL, xmlescape
from gluon.utils import is_valid_ip_address, getipaddrinfo
@@ -365,8 +365,8 @@ def wsgibase(environ, responder):
client = client,
folder = abspath('applications', app) + os.sep,
ajax = x_req_with == 'xmlhttprequest',
cid = env.http_web2py_component_element,
is_local = (env.remote_addr in local_hosts and
cid = env.http_web2py_component_element,
is_local = (env.remote_addr in local_hosts and
client == env.remote_addr),
is_shell = cmd_opts and cmd_opts.shell,
is_sheduler = cmd_opts and cmd_opts.scheduler,

4
gluon/scheduler.py
View File

@@ -598,7 +598,7 @@ class Scheduler(MetaScheduler):
def define_tables(self, db, migrate):
"""Defines Scheduler tables structure"""
from gluon.dal.base import DEFAULT
from gluon.pydal.base import DEFAULT
logger.debug('defining tables (migrate=%s)', migrate)
now = self.now
db.define_table(
@@ -1316,7 +1316,7 @@ class Scheduler(MetaScheduler):
have all fields == None
"""
from gluon.dal.objects import Query
from gluon.pydal.objects import Query
sr, st = self.db.scheduler_run, self.db.scheduler_task
if isinstance(ref, (int, long)):
q = st.id == ref

4
gluon/shell.py
View File

@@ -28,7 +28,7 @@ from gluon.restricted import RestrictedError
from gluon.globals import Request, Response, Session
from gluon.storage import Storage, List
from gluon.admin import w2p_unpack
from gluon.dal.base import BaseAdapter
from gluon.pydal.base import BaseAdapter
logger = logging.getLogger("web2py")
@@ -129,7 +129,7 @@ def env(
request.function)
if global_settings.cmd_options:
ip = global_settings.cmd_options.ip
port = global_settings.cmd_options.port
port = global_settings.cmd_options.port
else:
ip, port = '127.0.0.1', '8000'
request.env.http_host = '%s:%s' % (ip,port)

7
gluon/sql.py
View File

@@ -13,9 +13,10 @@ Just for backward compatibility
__all__ = ['DAL', 'Field', 'DRIVERS']
from dal import DAL, Field, SQLCustomType
from dal.adapters.base import BaseAdapter, DRIVERS
from dal.objects import Table, Query, Set, Expression, Row, Rows
from dal.helpers.classes import SQLALL
from gluon.pydal.base import BaseAdapter
from gluon.pydal.drivers import DRIVERS
from gluon.pydal.objects import Table, Query, Set, Expression, Row, Rows
from gluon.pydal.helpers.classes import SQLALL
SQLDB = DAL
GQLDB = DAL

11
gluon/sqlhtml.py
View File

@@ -26,12 +26,11 @@ from gluon.html import XML, SPAN, TAG, A, DIV, CAT, UL, LI, TEXTAREA, BR, IMG
from gluon.html import FORM, INPUT, LABEL, OPTION, SELECT, COL, COLGROUP
from gluon.html import TABLE, THEAD, TBODY, TR, TD, TH, STYLE, SCRIPT
from gluon.html import URL, FIELDSET, P, DEFAULT_PASSWORD_DISPLAY
from gluon.dal import DAL, Field
from gluon.dal.base import DEFAULT
from gluon.dal.objects import Table, Row, Expression
from gluon.dal.adapters.base import CALLABLETYPES
from gluon.dal.helpers.methods import smart_query, bar_encode, sqlhtml_validators
from gluon.dal.helpers.classes import Reference, SQLCustomType
from gluon.pydal.base import DEFAULT
from gluon.pydal.objects import Table, Row, Expression
from gluon.pydal.adapters.base import CALLABLETYPES
from gluon.pydal.helpers.methods import smart_query, bar_encode, sqlhtml_validators
from gluon.pydal.helpers.classes import Reference, SQLCustomType
from gluon.storage import Storage
from gluon.utils import md5_hash
from gluon.validators import IS_EMPTY_OR, IS_NOT_EMPTY, IS_LIST_OF, IS_DATE

64
gluon/tools.py
View File

@@ -43,7 +43,7 @@ from gluon import *
from gluon.contrib.autolinks import expand_one
from gluon.contrib.markmin.markmin2html import \
replace_at_urls, replace_autolinks, replace_components
from gluon.dal.objects import Table, Row, Set, Query
from gluon.pydal.objects import Table, Row, Set, Query
import gluon.serializers as serializers
@@ -172,7 +172,7 @@ class Mail(object):
chain of email certificate. It can be a
string containing the certs to. (PEM format)
x509_nocerts : if True then no attached certificate in mail
x509_crypt_certfiles: the certificates file or strings to encrypt
x509_crypt_certfiles: the certificates file or strings to encrypt
the messages with can be a file name /
string or a list of file names /
strings (PEM format)
@@ -340,7 +340,7 @@ class Mail(object):
from_address: address to appear in the 'From:' header, this is not
the envelope sender. If not specified the sender will be used
cipher_type :
cipher_type :
gpg - need a python-pyme package and gpgme lib
x509 - smime
gpg_home : you can set a GNUPGHOME environment variable
@@ -359,8 +359,8 @@ class Mail(object):
chain of email certificate. It can be a
string containing the certs to. (PEM format)
x509_nocerts : if True then no attached certificate in mail
x509_crypt_certfiles: the certificates file or strings to encrypt
the messages with can be a file name / string or
x509_crypt_certfiles: the certificates file or strings to encrypt
the messages with can be a file name / string or
a list of file names / strings (PEM format)
Examples:
Send plain text message to single address::
@@ -1346,7 +1346,7 @@ class Auth(object):
reset_password_onvalidation = [],
reset_password_onaccept = [],
hmac_key = hmac_key,
formstyle = current.response.formstyle,
formstyle = current.response.formstyle,
)
settings.lock_keys = True
@@ -2340,7 +2340,7 @@ class Auth(object):
items = snext.split('/')
if '//' in snext and items[2] != request.env.http_host:
snext = None
if snext:
session._auth_next = snext
elif session._auth_next:
@@ -2428,20 +2428,20 @@ class Auth(object):
separator=settings.label_separator,
extra_fields = extra_fields,
)
captcha = settings.login_captcha or \
(settings.login_captcha != False and settings.captcha)
if captcha:
addrow(form, captcha.label, captcha, captcha.comment,
settings.formstyle, 'captcha__row')
accepted_form = False
if form.accepts(request, session if self.csrf_prevention else None,
formname='login', dbio=False,
onvalidation=onvalidation,
hideerror=settings.hideerror):
accepted_form = True
# check for username in db
entered_username = form.vars[username]
@@ -2459,7 +2459,7 @@ class Auth(object):
elif temp_user.registration_key in ('disabled', 'blocked'):
response.flash = self.messages.login_disabled
return form
elif (not temp_user.registration_key is None
elif (not temp_user.registration_key is None
and temp_user.registration_key.strip()):
response.flash = \
self.messages.registration_verifying
@@ -2507,11 +2507,11 @@ class Auth(object):
redirect(
self.url(args=request.args, vars=request.get_vars),
client_side=settings.client_side)
else: # use a central authentication server
cas = settings.login_form
cas_user = cas.get_user()
if cas_user:
cas_user[passfield] = None
user = self.get_or_create_user(
@@ -2527,7 +2527,7 @@ class Auth(object):
# Extra login logic for two-factor authentication
#################################################
# If the 'user' variable has a value, this means that the first
# If the 'user' variable has a value, this means that the first
# authentication step was successful (i.e. user provided correct
# username and password at the first challenge).
# Check if this user is signed up for two-factor authentication
@@ -2540,7 +2540,7 @@ class Auth(object):
if session.auth_two_factor_enabled:
form = SQLFORM.factory(
Field('authentication_code',
required=True,
required=True,
comment='This code was emailed to you and is required for login.'),
hidden=dict(_next=next),
formstyle=settings.formstyle,
@@ -2559,8 +2559,8 @@ class Auth(object):
session.auth_two_factor_tries_left = 3 # Allow user to try up to 4 times
# TODO: Add some error checking to handle cases where email cannot be sent
self.settings.mailer.send(
to=user.email,
subject="Two-step Login Authentication Code",
to=user.email,
subject="Two-step Login Authentication Code",
message="Your temporary login code is {0}".format(session.auth_two_factor))
if form.accepts(request, session if self.csrf_prevention else None,
formname='login', dbio=False,
@@ -2575,15 +2575,15 @@ class Auth(object):
# normal.
if user is None or user == session.auth_two_factor_user:
user = session.auth_two_factor_user
# For security, because the username stored in the
# For security, because the username stored in the
# session somehow does not match the just validated
# user. Should not be possible without session stealing
# which is hard with SSL.
elif user != session.auth_two_factor_user:
user = None
# Either way, the user and code associated with this session should
# be removed. This handles cases where the session login may have
# expired but browser window is open, so the old session key and
# be removed. This handles cases where the session login may have
# expired but browser window is open, so the old session key and
# session usernamem will still exist
self._reset_two_factor_auth(session)
else:
@@ -2631,11 +2631,11 @@ class Auth(object):
"""
Logouts and redirects to login
"""
# Clear out 2-step authentication information if user logs
# out. This information is also cleared on successful login.
self._reset_two_factor_auth(current.session)
if next is DEFAULT:
next = self.get_vars_next() or self.settings.logout_next
if onlogout is DEFAULT:
@@ -2786,7 +2786,7 @@ class Auth(object):
else:
next = replace_id(next, form)
redirect(next, client_side=self.settings.client_side)
return form
def is_logged_in(self):
@@ -3086,7 +3086,7 @@ class Auth(object):
if log is DEFAULT:
log = self.messages['reset_password_log']
userfield = self.settings.login_userfield or 'username' \
if 'username' in table_user.fields else 'email'
if 'username' in table_user.fields else 'email'
if userfield=='email':
table_user.email.requires = [
IS_EMAIL(error_message=self.messages.invalid_email),
@@ -3192,11 +3192,11 @@ class Auth(object):
log = self.messages['change_password_log']
passfield = self.settings.password_field
requires = table_user[passfield].requires
if not isinstance(requires,(list, tuple)):
if not isinstance(requires,(list, tuple)):
requires = [requires]
requires = filter(lambda t:isinstance(t,CRYPT), requires)
if requires:
requires[0].min_length = 0
requires[0].min_length = 0
form = SQLFORM.factory(
Field('old_password', 'password', requires=requires,
label=self.messages.old_password),
@@ -3748,7 +3748,7 @@ class Auth(object):
archive_current=False,
fields=None):
"""
If you have a table (db.mytable) that needs full revision history you
If you have a table (db.mytable) that needs full revision history you
can just do::
form=crud.update(db.mytable,myrecord,onaccept=auth.archive)
@@ -5366,8 +5366,8 @@ class Expose(object):
base = base or os.path.join(current.request.folder, 'static')
basename = basename or current.request.function
self.basename = basename
if current.request.raw_args:
if current.request.raw_args:
self.args = [arg for arg in current.request.raw_args.split('/') if arg]
else:
self.args = [arg for arg in current.request.args if args]
@@ -5691,8 +5691,8 @@ class Wiki(object):
def automenu(self):
"""adds the menu if not present"""
if (not self.wiki_menu_items and
self.settings.controller and
if (not self.wiki_menu_items and
self.settings.controller and
self.settings.function):
self.wiki_menu_items = self.menu(self.settings.controller,
self.settings.function)

8
gluon/validators.py
View File

@@ -22,7 +22,7 @@ import decimal
import unicodedata
from cStringIO import StringIO
from gluon.utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE
from gluon.dal.objects import FieldVirtual, FieldMethod
from gluon.pydal.objects import FieldVirtual, FieldMethod
regex_isint = re.compile('^[+-]?\d+$')
@@ -506,7 +506,7 @@ class IS_IN_DB(Validator):
sort=False,
_and=None,
):
from dal.objects import Table
from pydal.objects import Table
if isinstance(field, Table):
field = field._id
@@ -603,7 +603,7 @@ class IS_IN_DB(Validator):
if not [v for v in values if not v in self.theset]:
return (values, None)
else:
from dal.adapters import GoogleDatastoreAdapter
from gluon.pydal.adapters import GoogleDatastoreAdapter
def count(values, s=self.dbset, f=field):
return s(f.belongs(map(int, values))).count()
@@ -648,7 +648,7 @@ class IS_NOT_IN_DB(Validator):
ignore_common_filters=False,
):
from dal.objects import Table
from pydal.objects import Table
if isinstance(field, Table):
field = field._id

2
gluon/widget.py
View File

@@ -1086,7 +1086,7 @@ def start(cron=True):
print ProgramAuthor
print ProgramVersion
from dal.adapters.base import DRIVERS
from gluon.pydal.drivers import DRIVERS
if not options.nobanner:
print 'Database drivers available: %s' % ', '.join(DRIVERS)