Added modular DAL, updated gluon imports to new structure

gluon/dal/adapters/__init__.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+from .sqlite import SQLiteAdapter, SpatiaLiteAdapter, JDBCSQLiteAdapter
+from .mysql import MySQLAdapter
+from .postgre import PostgreSQLAdapter, NewPostgreSQLAdapter, JDBCPostgreSQLAdapter
+from .oracle import OracleAdapter
+from .mssql import MSSQLAdapter, MSSQL2Adapter, MSSQL3Adapter, MSSQL4Adapter, \
+    VerticaAdapter, SybaseAdapter
+from .firebird import FireBirdAdapter
+from .informix import InformixAdapter, InformixSEAdapter
+from .db2 import DB2Adapter
+from .teradata import TeradataAdapter
+from .ingres import IngresAdapter, IngresUnicodeAdapter
+from .sapdb import SAPDBAdapter
+from .cubrid import CubridAdapter
+from .google import GoogleDatastoreAdapter, GoogleSQLAdapter
+from .couchdb import CouchDBAdapter
+from .mongo import MongoDBAdapter
+from .imap import IMAPAdapter
+
+
+ADAPTERS = {
+    'sqlite': SQLiteAdapter,
+    'spatialite': SpatiaLiteAdapter,
+    'sqlite:memory': SQLiteAdapter,
+    'spatialite:memory': SpatiaLiteAdapter,
+    'mysql': MySQLAdapter,
+    'postgres': PostgreSQLAdapter,
+    'postgres:psycopg2': PostgreSQLAdapter,
+    'postgres:pg8000': PostgreSQLAdapter,
+    'postgres2:psycopg2': NewPostgreSQLAdapter,
+    'postgres2:pg8000': NewPostgreSQLAdapter,
+    'oracle': OracleAdapter,
+    'mssql': MSSQLAdapter,
+    'mssql2': MSSQL2Adapter,
+    'mssql3': MSSQL3Adapter,
+    'mssql4' : MSSQL4Adapter,
+    'vertica': VerticaAdapter,
+    'sybase': SybaseAdapter,
+    'db2': DB2Adapter,
+    'teradata': TeradataAdapter,
+    'informix': InformixAdapter,
+    'informix-se': InformixSEAdapter,
+    'firebird': FireBirdAdapter,
+    'firebird_embedded': FireBirdAdapter,
+    'ingres': IngresAdapter,
+    'ingresu': IngresUnicodeAdapter,
+    'sapdb': SAPDBAdapter,
+    'cubrid': CubridAdapter,
+    'jdbc:sqlite': JDBCSQLiteAdapter,
+    'jdbc:sqlite:memory': JDBCSQLiteAdapter,
+    'jdbc:postgres': JDBCPostgreSQLAdapter,
+    'gae': GoogleDatastoreAdapter, # discouraged, for backward compatibility
+    'google:datastore': GoogleDatastoreAdapter,
+    'google:datastore+ndb': GoogleDatastoreAdapter,
+    'google:sql': GoogleSQLAdapter,
+    'couchdb': CouchDBAdapter,
+    'mongodb': MongoDBAdapter,
+    'imap': IMAPAdapter
+}

gluon/dal/adapters/couchdb.py

@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+import datetime
+
+from .._globals import IDENTITY
+from .._load import serializers, couchdb, web2py_uuid
+from ..objects import Field, Query
+from ..helpers.classes import SQLALL
+from ..helpers.methods import uuid2int
+from .base import BaseAdapter, NoSQLAdapter, SELECT_ARGS
+
+
+class CouchDBAdapter(NoSQLAdapter):
+    drivers = ('couchdb',)
+
+    uploads_in_blob = True
+    types = {
+            'boolean': bool,
+            'string': str,
+            'text': str,
+            'json': str,
+            'password': str,
+            'blob': str,
+            'upload': str,
+            'integer': long,
+            'bigint': long,
+            'float': float,
+            'double': float,
+            'date': datetime.date,
+            'time': datetime.time,
+            'datetime': datetime.datetime,
+            'id': long,
+            'reference': long,
+            'list:string': list,
+            'list:integer': list,
+            'list:reference': list,
+        }
+
+    def file_exists(self, filename): pass
+    def file_open(self, filename, mode='rb', lock=True): pass
+    def file_close(self, fileobj): pass
+
+    def expand(self,expression,field_type=None):
+        if isinstance(expression,Field):
+            if expression.type=='id':
+                return "%s._id" % expression.tablename
+        return BaseAdapter.expand(self,expression,field_type)
+
+    def AND(self,first,second):
+        return '(%s && %s)' % (self.expand(first),self.expand(second))
+
+    def OR(self,first,second):
+        return '(%s || %s)' % (self.expand(first),self.expand(second))
+
+    def EQ(self,first,second):
+        if second is None:
+            return '(%s == null)' % self.expand(first)
+        return '(%s == %s)' % (self.expand(first),self.expand(second,first.type))
+
+    def NE(self,first,second):
+        if second is None:
+            return '(%s != null)' % self.expand(first)
+        return '(%s != %s)' % (self.expand(first),self.expand(second,first.type))
+
+    def COMMA(self,first,second):
+        return '%s + %s' % (self.expand(first),self.expand(second))
+
+    def represent(self, obj, fieldtype):
+        value = NoSQLAdapter.represent(self, obj, fieldtype)
+        if fieldtype=='id':
+            return repr(str(long(value)))
+        elif fieldtype in ('date','time','datetime','boolean'):
+            return serializers.json(value)
+        return repr(not isinstance(value,unicode) and value \
+                        or value and value.encode('utf8'))
+
+    def __init__(self,db,uri='couchdb://127.0.0.1:5984',
+                 pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args)
+        self.dbengine = 'couchdb'
+        self.folder = folder
+        db['_lastsql'] = ''
+        self.db_codec = 'UTF-8'
+        self._after_connection = after_connection
+        self.pool_size = pool_size
+
+        url='http://'+uri[10:]
+        def connector(url=url,driver_args=driver_args):
+            return self.driver.Server(url,**driver_args)
+        self.reconnect(connector,cursor=False)
+
+    def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
+        if migrate:
+            try:
+                self.connection.create(table._tablename)
+            except:
+                pass
+
+    def insert(self,table,fields):
+        id = uuid2int(web2py_uuid())
+        ctable = self.connection[table._tablename]
+        values = dict((k.name,self.represent(v,k.type)) for k,v in fields)
+        values['_id'] = str(id)
+        ctable.save(values)
+        return id
+
+    def _select(self,query,fields,attributes):
+        if not isinstance(query,Query):
+            raise SyntaxError("Not Supported")
+        for key in set(attributes.keys())-SELECT_ARGS:
+            raise SyntaxError('invalid select attribute: %s' % key)
+        new_fields=[]
+        for item in fields:
+            if isinstance(item,SQLALL):
+                new_fields += item._table
+            else:
+                new_fields.append(item)
+        def uid(fd):
+            return fd=='id' and '_id' or fd
+        def get(row,fd):
+            return fd=='id' and long(row['_id']) or row.get(fd,None)
+        fields = new_fields
+        tablename = self.get_table(query)
+        fieldnames = [f.name for f in (fields or self.db[tablename])]
+        colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
+        fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
+        fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
+            dict(t=tablename,
+                 query=self.expand(query),
+                 order='%s._id' % tablename,
+                 fields=fields)
+        return fn, colnames
+
+    def select(self,query,fields,attributes):
+        if not isinstance(query,Query):
+            raise SyntaxError("Not Supported")
+        fn, colnames = self._select(query,fields,attributes)
+        tablename = colnames[0].split('.')[0]
+        ctable = self.connection[tablename]
+        rows = [cols['value'] for cols in ctable.query(fn)]
+        processor = attributes.get('processor',self.parse)
+        return processor(rows,fields,colnames,False)
+
+    def delete(self,tablename,query):
+        if not isinstance(query,Query):
+            raise SyntaxError("Not Supported")
+        if query.first.type=='id' and query.op==self.EQ:
+            id = query.second
+            tablename = query.first.tablename
+            assert(tablename == query.first.tablename)
+            ctable = self.connection[tablename]
+            try:
+                del ctable[str(id)]
+                return 1
+            except couchdb.http.ResourceNotFound:
+                return 0
+        else:
+            tablename = self.get_table(query)
+            rows = self.select(query,[self.db[tablename]._id],{})
+            ctable = self.connection[tablename]
+            for row in rows:
+                del ctable[str(row.id)]
+            return len(rows)
+
+    def update(self,tablename,query,fields):
+        if not isinstance(query,Query):
+            raise SyntaxError("Not Supported")
+        if query.first.type=='id' and query.op==self.EQ:
+            id = query.second
+            tablename = query.first.tablename
+            ctable = self.connection[tablename]
+            try:
+                doc = ctable[str(id)]
+                for key,value in fields:
+                    doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
+                ctable.save(doc)
+                return 1
+            except couchdb.http.ResourceNotFound:
+                return 0
+        else:
+            tablename = self.get_table(query)
+            rows = self.select(query,[self.db[tablename]._id],{})
+            ctable = self.connection[tablename]
+            table = self.db[tablename]
+            for row in rows:
+                doc = ctable[str(row.id)]
+                for key,value in fields:
+                    doc[key.name] = self.represent(value,table[key.name].type)
+                ctable.save(doc)
+            return len(rows)
+
+    def count(self,query,distinct=None):
+        if distinct:
+            raise RuntimeError("COUNT DISTINCT not supported")
+        if not isinstance(query,Query):
+            raise SyntaxError("Not Supported")
+        tablename = self.get_table(query)
+        rows = self.select(query,[self.db[tablename]._id],{})
+        return len(rows)

gluon/dal/adapters/cubrid.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+import re
+
+from .._globals import IDENTITY
+from .mysql import MySQLAdapter
+
+
+class CubridAdapter(MySQLAdapter):
+    drivers = ('cubriddb',)
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
+
+    def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "cubrid"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError(
+                "Invalid URI string in DAL: %s" % self.uri)
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        port = int(m.group('port') or '30000')
+        user = credential_decoder(user)
+        passwd = credential_decoder(password)
+        def connector(host=host,port=port,db=db,
+                    user=user,passwd=passwd,driver_args=driver_args):
+            return self.driver.connect(host,port,db,user,passwd,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.execute('SET FOREIGN_KEY_CHECKS=1;')
+        self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
+

gluon/dal/adapters/db2.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+import base64
+import datetime
+
+from .._globals import IDENTITY
+from .base import BaseAdapter
+
+
+class DB2Adapter(BaseAdapter):
+    drivers = ('pyodbc',)
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'CLOB',
+        'json': 'CLOB',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BLOB',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'REAL',
+        'double': 'DOUBLE',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'TIMESTAMP',
+        'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
+        'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'CLOB',
+        'list:string': 'CLOB',
+        'list:reference': 'CLOB',
+        'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
+        'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+        }
+
+    def LEFT_JOIN(self):
+        return 'LEFT OUTER JOIN'
+
+    def RANDOM(self):
+        return 'RAND()'
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def represent_exceptions(self, obj, fieldtype):
+        if fieldtype == 'blob':
+            obj = base64.b64encode(str(obj))
+            return "BLOB('%s')" % obj
+        elif fieldtype == 'datetime':
+            if isinstance(obj, datetime.datetime):
+                obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
+            elif isinstance(obj, datetime.date):
+                obj = obj.isoformat()[:10]+'-00.00.00'
+            return "'%s'" % obj
+        return None
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "db2"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://', 1)[1]
+        def connector(cnxn=ruri,driver_args=driver_args):
+            return self.driver.connect(cnxn,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def execute(self,command):
+        if command[-1:]==';':
+            command = command[:-1]
+        return self.log_execute(command)
+
+    def lastrowid(self,table):
+        self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
+        return long(self.cursor.fetchone()[0])
+
+    def rowslice(self,rows,minimum=0,maximum=None):
+        if maximum is None:
+            return rows[minimum:]
+        return rows[minimum:maximum]

gluon/dal/adapters/firebird.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+import re
+
+from .._globals import IDENTITY
+from ..objects import Expression
+from .base import BaseAdapter
+
+
+class FireBirdAdapter(BaseAdapter):
+    drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
+
+    commit_on_alter_table = False
+    support_distributed_transaction = True
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'BLOB SUB_TYPE 1',
+        'json': 'BLOB SUB_TYPE 1',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BLOB SUB_TYPE 0',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INTEGER',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'DOUBLE PRECISION',
+        'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'TIMESTAMP',
+        'id': 'INTEGER PRIMARY KEY',
+        'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'BLOB SUB_TYPE 1',
+        'list:string': 'BLOB SUB_TYPE 1',
+        'list:reference': 'BLOB SUB_TYPE 1',
+        'big-id': 'BIGINT PRIMARY KEY',
+        'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        }
+
+    def sequence_name(self,tablename):
+        return ('genid_' + self.QUOTE_TEMPLATE) % tablename
+
+    def trigger_name(self,tablename):
+        return 'trg_id_%s' % tablename
+
+    def RANDOM(self):
+        return 'RAND()'
+
+    def EPOCH(self, first):
+        return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
+
+    def NOT_NULL(self,default,field_type):
+        return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
+
+    def SUBSTRING(self,field,parameters):
+        return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
+
+    def LENGTH(self, first):
+        return "CHAR_LENGTH(%s)" % self.expand(first)
+
+    def CONTAINS(self,first,second,case_sensitive=False):
+        if first.type.startswith('list:'):
+            second = Expression(None,self.CONCAT('|',Expression(
+                        None,self.REPLACE(second,('|','||'))),'|'))
+        return '(%s CONTAINING %s)' % (self.expand(first),
+                                       self.expand(second, 'string'))
+
+    def _drop(self,table,mode):
+        sequence_name = table._sequence_name
+        return ['DROP TABLE %s %s;' % (table.sqlsafe, mode), 'DROP GENERATOR %s;' % sequence_name]
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def _truncate(self,table,mode = ''):
+        return ['DELETE FROM %s;' % table._tablename,
+                'SET GENERATOR %s TO 0;' % table._sequence_name]
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "firebird"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        port = int(m.group('port') or 3050)
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        charset = m.group('charset') or 'UTF8'
+        driver_args.update(dsn='%s/%s:%s' % (host,port,db),
+                           user = credential_decoder(user),
+                           password = credential_decoder(password),
+                           charset = charset)
+
+        def connector(driver_args=driver_args):
+            return self.driver.connect(**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def create_sequence_and_triggers(self, query, table, **args):
+        tablename = table._tablename
+        sequence_name = table._sequence_name
+        trigger_name = table._trigger_name
+        self.execute(query)
+        self.execute('create generator %s;' % sequence_name)
+        self.execute('set generator %s to 0;' % sequence_name)
+        self.execute('create trigger %s for %s active before insert position 0 as\nbegin\nif(new.id is null) then\nbegin\nnew.id = gen_id(%s, 1);\nend\nend;' % (trigger_name, tablename, sequence_name))
+
+    def lastrowid(self,table):
+        sequence_name = table._sequence_name
+        self.execute('SELECT gen_id(%s, 0) FROM rdb$database' % sequence_name)
+        return long(self.cursor.fetchone()[0])
+
+
+class FireBirdEmbeddedAdapter(FireBirdAdapter):
+    drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "firebird"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError(
+                "Invalid URI string in DAL: %s" % self.uri)
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        pathdb = m.group('path')
+        if not pathdb:
+            raise SyntaxError('Path required')
+        charset = m.group('charset')
+        if not charset:
+            charset = 'UTF8'
+        host = ''
+        driver_args.update(host=host,
+                           database=pathdb,
+                           user=credential_decoder(user),
+                           password=credential_decoder(password),
+                           charset=charset)
+
+        def connector(driver_args=driver_args):
+            return self.driver.connect(**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()

gluon/dal/adapters/google.py

@@ -0,0 +1,607 @@
+# -*- coding: utf-8 -*-
+import os
+import re
+
+from .._compat import pjoin
+from .._globals import IDENTITY, LOGGER, THREAD_LOCAL
+from .._load import classobj, gae, ndb, NDBDecimalProperty, GAEDecimalProperty, \
+    namespace_manager, Key, NDBPolyModel, PolyModel, rdbms
+from ..objects import Table, Field, Expression, Query
+from ..helpers.classes import SQLCustomType, SQLALL, Reference, UseDatabaseStoredFile
+from ..helpers.methods import use_common_filters, xorify
+from .base import NoSQLAdapter
+from .mysql import MySQLAdapter
+
+
+class GoogleSQLAdapter(UseDatabaseStoredFile, MySQLAdapter):
+    uploads_in_blob = True
+
+    REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
+
+    def __init__(self, db, uri='google:sql://realm:domain/database',
+                 pool_size=0, folder=None, db_codec='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+
+        self.db = db
+        self.dbengine = "mysql"
+        self.uri = uri
+        self.pool_size = pool_size
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        if do_connect: self.find_driver(adapter_args, uri)
+        self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
+                os.sep+'applications'+os.sep,1)[1])
+        ruri = uri.split("://")[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
+        instance = credential_decoder(m.group('instance'))
+        self.dbstring = db = credential_decoder(m.group('db'))
+        driver_args['instance'] = instance
+        if not 'charset' in driver_args:
+            driver_args['charset'] = 'utf8'
+        self.createdb = createdb = adapter_args.get('createdb',True)
+        if not createdb:
+            driver_args['database'] = db
+        def connector(driver_args=driver_args):
+            return rdbms.connect(**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        if self.createdb:
+            # self.execute('DROP DATABASE %s' % self.dbstring)
+            self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
+            self.execute('USE %s' % self.dbstring)
+        self.execute("SET FOREIGN_KEY_CHECKS=1;")
+        self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
+
+    def execute(self, command, *a, **b):
+        return self.log_execute(command.decode('utf8'), *a, **b)
+
+    def find_driver(self,adapter_args,uri=None):
+        self.adapter_args = adapter_args
+        self.driver = "google"
+
+
+class GAEF(object):
+    def __init__(self,name,op,value,apply):
+        self.name=name=='id' and '__key__' or name
+        self.op=op
+        self.value=value
+        self.apply=apply
+    def __repr__(self):
+        return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
+
+
+class GoogleDatastoreAdapter(NoSQLAdapter):
+    """
+    NDB:
+
+    You can enable NDB by using adapter_args::
+
+        db = DAL('google:datastore', adapter_args={'ndb_settings':ndb_settings, 'use_ndb':True})
+
+    ndb_settings is optional and can be used for per model caching settings.
+    It must be a dict in this form::
+
+        ndb_settings = {<table_name>:{<variable_name>:<variable_value>}}
+
+    See: https://developers.google.com/appengine/docs/python/ndb/cache
+    """
+
+    MAX_FETCH_LIMIT = 1000000
+    uploads_in_blob = True
+    types = {}
+    # reconnect is not required for Datastore dbs
+    reconnect = lambda *args, **kwargs: None
+
+    def file_exists(self, filename): pass
+    def file_open(self, filename, mode='rb', lock=True): pass
+    def file_close(self, fileobj): pass
+
+    REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.use_ndb = adapter_args.get('use_ndb',uri.startswith('google:datastore+ndb'))
+        if self.use_ndb is True:
+            self.types.update({
+                'boolean': ndb.BooleanProperty,
+                'string': (lambda **kwargs: ndb.StringProperty(**kwargs)),
+                'text': ndb.TextProperty,
+                'json': ndb.TextProperty,
+                'password': ndb.StringProperty,
+                'blob': ndb.BlobProperty,
+                'upload': ndb.StringProperty,
+                'integer': ndb.IntegerProperty,
+                'bigint': ndb.IntegerProperty,
+                'float': ndb.FloatProperty,
+                'double': ndb.FloatProperty,
+                'decimal': NDBDecimalProperty,
+                'date': ndb.DateProperty,
+                'time': ndb.TimeProperty,
+                'datetime': ndb.DateTimeProperty,
+                'id': None,
+                'reference': ndb.IntegerProperty,
+                'list:string': (lambda **kwargs: ndb.StringProperty(repeated=True,default=None, **kwargs)),
+                'list:integer': (lambda **kwargs: ndb.IntegerProperty(repeated=True,default=None, **kwargs)),
+                'list:reference': (lambda **kwargs: ndb.IntegerProperty(repeated=True,default=None, **kwargs)),
+                })
+        else:
+            self.types.update({
+                'boolean': gae.BooleanProperty,
+                'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)),
+                'text': gae.TextProperty,
+                'json': gae.TextProperty,
+                'password': gae.StringProperty,
+                'blob': gae.BlobProperty,
+                'upload': gae.StringProperty,
+                'integer': gae.IntegerProperty,
+                'bigint': gae.IntegerProperty,
+                'float': gae.FloatProperty,
+                'double': gae.FloatProperty,
+                'decimal': GAEDecimalProperty,
+                'date': gae.DateProperty,
+                'time': gae.TimeProperty,
+                'datetime': gae.DateTimeProperty,
+                'id': None,
+                'reference': gae.IntegerProperty,
+                'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)),
+                'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
+                'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
+                })
+        self.db = db
+        self.uri = uri
+        self.dbengine = 'google:datastore'
+        self.folder = folder
+        db['_lastsql'] = ''
+        self.db_codec = 'UTF-8'
+        self._after_connection = after_connection
+        self.pool_size = 0
+        match = self.REGEX_NAMESPACE.match(uri)
+        if match:
+            namespace_manager.set_namespace(match.group('namespace'))
+        self.keyfunc = (self.use_ndb and ndb.Key) or Key.from_path
+
+        self.ndb_settings = None
+        if 'ndb_settings' in adapter_args:
+            self.ndb_settings = adapter_args['ndb_settings']
+
+    def parse_id(self, value, field_type):
+        return value
+
+    def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
+        myfields = {}
+        for field in table:
+            if isinstance(polymodel,Table) and field.name in polymodel.fields():
+                continue
+            attr = {}
+            if isinstance(field.custom_qualifier, dict):
+                #this is custom properties to add to the GAE field declartion
+                attr = field.custom_qualifier
+            field_type = field.type
+            if isinstance(field_type, SQLCustomType):
+                ftype = self.types[field_type.native or field_type.type](**attr)
+            elif isinstance(field_type, ((self.use_ndb and ndb.Property) or gae.Property)):
+                ftype = field_type
+            elif field_type.startswith('id'):
+                continue
+            elif field_type.startswith('decimal'):
+                precision, scale = field_type[7:].strip('()').split(',')
+                precision = int(precision)
+                scale = int(scale)
+                dec_cls = (self.use_ndb and NDBDecimalProperty) or GAEDecimalProperty
+                ftype = dec_cls(precision, scale, **attr)
+            elif field_type.startswith('reference'):
+                if field.notnull:
+                    attr = dict(required=True)
+                ftype = self.types[field_type[:9]](**attr)
+            elif field_type.startswith('list:reference'):
+                if field.notnull:
+                    attr['required'] = True
+                ftype = self.types[field_type[:14]](**attr)
+            elif field_type.startswith('list:'):
+                ftype = self.types[field_type](**attr)
+            elif not field_type in self.types\
+                 or not self.types[field_type]:
+                raise SyntaxError('Field: unknown field type: %s' % field_type)
+            else:
+                ftype = self.types[field_type](**attr)
+            myfields[field.name] = ftype
+        if not polymodel:
+            model_cls = (self.use_ndb and ndb.Model) or gae.Model
+            table._tableobj =  classobj(table._tablename, (model_cls, ), myfields)
+            if self.use_ndb:
+                # Set NDB caching variables
+                if self.ndb_settings and (table._tablename in self.ndb_settings):
+                    for k, v in self.ndb_settings.iteritems():
+                        setattr(table._tableobj, k, v)
+        elif polymodel==True:
+            pm_cls = (self.use_ndb and NDBPolyModel) or PolyModel
+            table._tableobj = classobj(table._tablename, (pm_cls, ), myfields)
+        elif isinstance(polymodel,Table):
+            table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
+        else:
+            raise SyntaxError("polymodel must be None, True, a table or a tablename")
+        return None
+
+    def expand(self,expression,field_type=None):
+        if isinstance(expression,Field):
+            if expression.type in ('text', 'blob', 'json'):
+                raise SyntaxError('AppEngine does not index by: %s' % expression.type)
+            return expression.name
+        elif isinstance(expression, (Expression, Query)):
+            if not expression.second is None:
+                return expression.op(expression.first, expression.second)
+            elif not expression.first is None:
+                return expression.op(expression.first)
+            else:
+                return expression.op()
+        elif field_type:
+                return self.represent(expression,field_type)
+        elif isinstance(expression,(list,tuple)):
+            return ','.join([self.represent(item,field_type) for item in expression])
+        else:
+            return str(expression)
+
+    ### TODO from gql.py Expression
+    def AND(self,first,second):
+        a = self.expand(first)
+        b = self.expand(second)
+        if b[0].name=='__key__' and a[0].name!='__key__':
+            return b+a
+        return a+b
+
+    def EQ(self,first,second=None):
+        if isinstance(second, Key):
+            return [GAEF(first.name,'=',second,lambda a,b:a==b)]
+        return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
+
+    def NE(self,first,second=None):
+        if first.type != 'id':
+            return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
+        else:
+            if not second is None:
+                second = self.keyfunc(first._tablename, long(second))
+            return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
+
+    def LT(self,first,second=None):
+        if first.type != 'id':
+            return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
+        else:
+            second = self.keyfunc(first._tablename, long(second))
+            return [GAEF(first.name,'<',second,lambda a,b:a<b)]
+
+    def LE(self,first,second=None):
+        if first.type != 'id':
+            return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
+        else:
+            second = self.keyfunc(first._tablename, long(second))
+            return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
+
+    def GT(self,first,second=None):
+        if first.type != 'id' or second==0 or second == '0':
+            return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
+        else:
+            second = self.keyfunc(first._tablename, long(second))
+            return [GAEF(first.name,'>',second,lambda a,b:a>b)]
+
+    def GE(self,first,second=None):
+        if first.type != 'id':
+            return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
+        else:
+            second = self.keyfunc(first._tablename, long(second))
+            return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
+
+    def INVERT(self,first):
+        return '-%s' % first.name
+
+    def COMMA(self,first,second):
+        return '%s, %s' % (self.expand(first),self.expand(second))
+
+    def BELONGS(self,first,second=None):
+        if not isinstance(second,(list, tuple, set)):
+            raise SyntaxError("Not supported")
+        if not self.use_ndb:
+            if isinstance(second,set):
+                second = list(second)
+        if first.type == 'id':
+            second = [self.keyfunc(first._tablename, int(i)) for i in second]
+        return [GAEF(first.name,'in',second,lambda a,b:a in b)]
+
+    def CONTAINS(self,first,second,case_sensitive=False):
+        # silently ignoring: GAE can only do case sensitive matches!
+        if not first.type.startswith('list:'):
+            raise SyntaxError("Not supported")
+        return [GAEF(first.name,'=',self.expand(second,first.type[5:]),lambda a,b:b in a)]
+
+    def NOT(self,first):
+        nops = { self.EQ: self.NE,
+                 self.NE: self.EQ,
+                 self.LT: self.GE,
+                 self.GT: self.LE,
+                 self.LE: self.GT,
+                 self.GE: self.LT}
+        if not isinstance(first,Query):
+            raise SyntaxError("Not suported")
+        nop = nops.get(first.op,None)
+        if not nop:
+            raise SyntaxError("Not suported %s" % first.op.__name__)
+        first.op = nop
+        return self.expand(first)
+
+    def truncate(self,table,mode):
+        self.db(self.db._adapter.id_query(table)).delete()
+
+    GAE_FILTER_OPTIONS = {
+        '=': lambda q, t, p, v: q.filter(getattr(t,p) == v),
+        '>': lambda q, t, p, v: q.filter(getattr(t,p) > v),
+        '<': lambda q, t, p, v: q.filter(getattr(t,p) < v),
+        '<=': lambda q, t, p, v: q.filter(getattr(t,p) <= v),
+        '>=': lambda q, t, p, v: q.filter(getattr(t,p) >= v),
+        '!=': lambda q, t, p, v: q.filter(getattr(t,p) != v),
+        'in': lambda q, t, p, v: q.filter(getattr(t,p).IN(v)),
+        }
+
+    def filter(self, query, tableobj, prop, op, value):
+        return self.GAE_FILTER_OPTIONS[op](query, tableobj, prop, value)
+
+    def select_raw(self,query,fields=None,attributes=None,count_only=False):
+        db = self.db
+        fields = fields or []
+        attributes = attributes or {}
+        args_get = attributes.get
+        new_fields = []
+
+        for item in fields:
+            if isinstance(item,SQLALL):
+                new_fields += item._table
+            else:
+                new_fields.append(item)
+
+        fields = new_fields
+        if query:
+            tablename = self.get_table(query)
+        elif fields:
+            tablename = fields[0].tablename
+            query = db._adapter.id_query(fields[0].table)
+        else:
+            raise SyntaxError("Unable to determine a tablename")
+
+        if query:
+            if use_common_filters(query):
+                query = self.common_filter(query,[tablename])
+
+        #tableobj is a GAE/NDB Model class (or subclass)
+        tableobj = db[tablename]._tableobj
+        filters = self.expand(query)
+
+        projection = None
+        if len(db[tablename].fields) == len(fields):
+            #getting all fields, not a projection query
+            projection = None
+        elif args_get('projection') == True:
+            projection = []
+            for f in fields:
+                if f.type in ['text', 'blob', 'json']:
+                    raise SyntaxError(
+                        "text and blob field types not allowed in projection queries")
+                else:
+                    projection.append(f.name)
+
+        elif args_get('filterfields') is True:
+            projection = []
+            for f in fields:
+                projection.append(f.name)
+
+        # real projection's can't include 'id'.
+        # it will be added to the result later
+        query_projection = [
+            p for p in projection if \
+                p != db[tablename]._id.name] if projection and \
+                args_get('projection') == True\
+                else None
+
+        cursor = args_get('reusecursor')
+        cursor = cursor if isinstance(cursor, str) else None
+        if self.use_ndb:
+            qo = ndb.QueryOptions(projection=query_projection, cursor=cursor)
+            items = tableobj.query(default_options=qo)
+        else:
+            items = gae.Query(tableobj, projection=query_projection, cursor=cursor)
+
+        for filter in filters:
+            if (args_get('projection') == True and
+                filter.name in query_projection and
+                filter.op in ('=', '<=', '>=')):
+                raise SyntaxError("projection fields cannot have equality filters")
+            if filter.name=='__key__' and filter.op=='>' and filter.value==0:
+                continue
+            elif filter.name=='__key__' and filter.op=='=':
+                if filter.value==0:
+                    items = []
+                elif isinstance(filter.value, (self.use_ndb and ndb.Key) or Key):
+                    # key qeuries return a class instance,
+                    # can't use projection
+                    # extra values will be ignored in post-processing later
+                    item = filter.value.get() if self.use_ndb else tableobj.get(filter.value)
+                    items = [item] if item else []
+                else:
+                    # key qeuries return a class instance,
+                    # can't use projection
+                    # extra values will be ignored in post-processing later
+                    item = tableobj.get_by_id(filter.value)
+                    items = [item] if item else []
+            elif isinstance(items,list): # i.e. there is a single record!
+                items = [i for i in items if filter.apply(
+                        getattr(item,filter.name),filter.value)]
+            else:
+                if filter.name=='__key__' and filter.op != 'in':
+                    items.order(tableobj._key) if self.use_ndb else items.order('__key__')
+                if self.use_ndb:
+                    items = self.filter(items, tableobj, filter.name, filter.op, filter.value)
+                else:
+                    items = items.filter('%s %s' % (filter.name,filter.op), filter.value)
+
+        if count_only:
+            items = [len(items) if isinstance(items,list) else items.count()]
+        elif not isinstance(items,list):
+            query = items
+            if args_get('left', None):
+                raise SyntaxError('Set: no left join in appengine')
+            if args_get('groupby', None):
+                raise SyntaxError('Set: no groupby in appengine')
+            orderby = args_get('orderby', False)
+            if orderby:
+                ### THIS REALLY NEEDS IMPROVEMENT !!!
+                if isinstance(orderby, (list, tuple)):
+                    orderby = xorify(orderby)
+                if isinstance(orderby,Expression):
+                    orderby = self.expand(orderby)
+                orders = orderby.split(', ')
+                for order in orders:
+                    if self.use_ndb:
+                        #TODO There must be a better way
+                        def make_order(o):
+                            s = str(o)
+                            desc = s[0] == '-'
+                            s = (desc and s[1:]) or s
+                            return  (desc and  -getattr(tableobj, s)) or getattr(tableobj, s)
+                        _order = {'-id':-tableobj._key,'id':tableobj._key}.get(order)
+                        if _order is None:
+                            _order = make_order(order)
+                        query = query.order(_order)
+                    else:
+                        order={'-id':'-__key__','id':'__key__'}.get(order,order)
+                        query = query.order(order)
+
+            if args_get('limitby', None):
+                (lmin, lmax) = attributes['limitby']
+                limit, fetch_args = lmax-lmin, {'offset':lmin,'keys_only':True}
+
+                if self.use_ndb:
+                    keys, cursor, more = query.fetch_page(limit,**fetch_args)
+                    items = ndb.get_multi(keys)
+                else:
+                    keys =  query.fetch(limit, **fetch_args)
+                    items = gae.get(keys)
+                    cursor = query.cursor()
+                #cursor is only useful if there was a limit and we didn't return
+                # all results
+                if args_get('reusecursor'):
+                    db['_lastcursor'] = cursor
+            else:
+                # if a limit is not specified, always return an iterator
+                rows = query
+
+        return (items, tablename, projection or db[tablename].fields)
+
+    def select(self,query,fields,attributes):
+        """
+        This is the GAE version of select. Some notes to consider:
+            - db['_lastsql'] is not set because there is not SQL statement string
+            for a GAE query
+            - 'nativeRef' is a magical fieldname used for self references on GAE
+            - optional attribute 'projection' when set to True will trigger
+            use of the GAE projection queries.  note that there are rules for
+            what is accepted imposed by GAE: each field must be indexed,
+            projection queries cannot contain blob or text fields, and you
+            cannot use == and also select that same field.  see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
+            - optional attribute 'filterfields' when set to True web2py will only
+            parse the explicitly listed fields into the Rows object, even though
+            all fields are returned in the query.  This can be used to reduce
+            memory usage in cases where true projection queries are not
+            usable.
+            - optional attribute 'reusecursor' allows use of cursor with queries
+            that have the limitby attribute.  Set the attribute to True for the
+            first query, set it to the value of db['_lastcursor'] to continue
+            a previous query.  The user must save the cursor value between
+            requests, and the filters must be identical.  It is up to the user
+            to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
+        """
+
+        (items, tablename, fields) = self.select_raw(query,fields,attributes)
+        # self.db['_lastsql'] = self._select(query,fields,attributes)
+        rows = [[(t==self.db[tablename]._id.name and item) or \
+                 (t=='nativeRef' and item) or getattr(item, t) \
+                     for t in fields] for item in items]
+        colnames = ['%s.%s' % (tablename, t) for t in fields]
+        processor = attributes.get('processor',self.parse)
+        return processor(rows,fields,colnames,False)
+
+    def parse_list_integers(self, value, field_type):
+        return value[:] if self.use_ndb else value
+
+    def parse_list_strings(self, value, field_type):
+        return value[:] if self.use_ndb else value
+
+    def count(self,query,distinct=None,limit=None):
+        if distinct:
+            raise RuntimeError("COUNT DISTINCT not supported")
+        (items, tablename, fields) = self.select_raw(query,count_only=True)
+        return items[0]
+
+    def delete(self,tablename, query):
+        """
+        This function was changed on 2010-05-04 because according to
+        http://code.google.com/p/googleappengine/issues/detail?id=3119
+        GAE no longer supports deleting more than 1000 records.
+        """
+        # self.db['_lastsql'] = self._delete(tablename,query)
+        (items, tablename, fields) = self.select_raw(query)
+        # items can be one item or a query
+        if not isinstance(items,list):
+            #use a keys_only query to ensure that this runs as a datastore
+            # small operations
+            leftitems = items.fetch(1000, keys_only=True)
+            counter = 0
+            while len(leftitems):
+                counter += len(leftitems)
+                if self.use_ndb:
+                    ndb.delete_multi(leftitems)
+                else:
+                    gae.delete(leftitems)
+                leftitems = items.fetch(1000, keys_only=True)
+        else:
+            counter = len(items)
+            if self.use_ndb:
+                ndb.delete_multi([item.key for item in items])
+            else:
+                gae.delete(items)
+        return counter
+
+    def update(self,tablename,query,update_fields):
+        # self.db['_lastsql'] = self._update(tablename,query,update_fields)
+        (items, tablename, fields) = self.select_raw(query)
+        counter = 0
+        for item in items:
+            for field, value in update_fields:
+                setattr(item, field.name, self.represent(value,field.type))
+            item.put()
+            counter += 1
+        LOGGER.info(str(counter))
+        return counter
+
+    def insert(self,table,fields):
+        dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
+        # table._db['_lastsql'] = self._insert(table,fields)
+        tmp = table._tableobj(**dfields)
+        tmp.put()
+        key = tmp.key if self.use_ndb else tmp.key()
+        rid = Reference(key.id())
+        (rid._table, rid._record, rid._gaekey) = (table, None, key)
+        return rid
+
+    def bulk_insert(self,table,items):
+        parsed_items = []
+        for item in items:
+            dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
+            parsed_items.append(table._tableobj(**dfields))
+        if self.use_ndb:
+            ndb.put_multi(parsed_items)
+        else:
+            gae.put(parsed_items)
+        return True

gluon/dal/adapters/informix.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+import datetime
+import re
+
+from .._globals import IDENTITY
+from .base import BaseAdapter
+
+
+class InformixAdapter(BaseAdapter):
+    drivers = ('informixdb',)
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'BLOB SUB_TYPE 1',
+        'json': 'BLOB SUB_TYPE 1',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BLOB SUB_TYPE 0',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INTEGER',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'DOUBLE PRECISION',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'CHAR(8)',
+        'datetime': 'DATETIME',
+        'id': 'SERIAL',
+        'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'BLOB SUB_TYPE 1',
+        'list:string': 'BLOB SUB_TYPE 1',
+        'list:reference': 'BLOB SUB_TYPE 1',
+        'big-id': 'BIGSERIAL',
+        'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': 'REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s CONSTRAINT FK_%(table_name)s_%(field_name)s',
+        'reference TFK': 'FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s CONSTRAINT TFK_%(table_name)s_%(field_name)s',
+        }
+
+    def RANDOM(self):
+        return 'Random()'
+
+    def NOT_NULL(self,default,field_type):
+        return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            fetch_amt = lmax - lmin
+            dbms_version = int(self.connection.dbms_version.split('.')[0])
+            if lmin and (dbms_version >= 10):
+                # Requires Informix 10.0+
+                sql_s += ' SKIP %d' % (lmin, )
+            if fetch_amt and (dbms_version >= 9):
+                # Requires Informix 9.0+
+                sql_s += ' FIRST %d' % (fetch_amt, )
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def represent_exceptions(self, obj, fieldtype):
+        if fieldtype == 'date':
+            if isinstance(obj, (datetime.date, datetime.datetime)):
+                obj = obj.isoformat()[:10]
+            else:
+                obj = str(obj)
+            return "to_date('%s','%%Y-%%m-%%d')" % obj
+        elif fieldtype == 'datetime':
+            if isinstance(obj, datetime.datetime):
+                obj = obj.isoformat()[:19].replace('T',' ')
+            elif isinstance(obj, datetime.date):
+                obj = obj.isoformat()[:10]+' 00:00:00'
+            else:
+                obj = str(obj)
+            return "to_date('%s','%%Y-%%m-%%d %%H:%%M:%%S')" % obj
+        return None
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "informix"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError(
+                "Invalid URI string in DAL: %s" % self.uri)
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        user = credential_decoder(user)
+        password = credential_decoder(password)
+        dsn = '%s@%s' % (db,host)
+        driver_args.update(user=user,password=password,autocommit=True)
+        def connector(dsn=dsn,driver_args=driver_args):
+            return self.driver.connect(dsn,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def execute(self,command):
+        if command[-1:]==';':
+            command = command[:-1]
+        return self.log_execute(command)
+
+    def lastrowid(self,table):
+        return self.cursor.sqlerrd[1]
+
+
+class InformixSEAdapter(InformixAdapter):
+    """ work in progress """
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        return 'SELECT %s %s FROM %s%s%s;' % \
+            (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def rowslice(self,rows,minimum=0,maximum=None):
+        if maximum is None:
+            return rows[minimum:]
+        return rows[minimum:maximum]

gluon/dal/adapters/ingres.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+
+from .._globals import IDENTITY
+from .._load import pyodbc
+from .base import BaseAdapter
+
+# NOTE invalid database object name (ANSI-SQL wants 
+#      this form of name to be a delimited identifier)
+INGRES_SEQNAME='ii***lineitemsequence'
+
+
+class IngresAdapter(BaseAdapter):
+    drivers = ('pyodbc',)
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'CLOB',
+        'json': 'CLOB',
+        'password': 'VARCHAR(%(length)s)',  ## Not sure what this contains utf8 or nvarchar. Or even bytes?
+        'blob': 'BLOB',
+        'upload': 'VARCHAR(%(length)s)',  ## FIXME utf8 or nvarchar... or blob? what is this type?
+        'integer': 'INTEGER4', # or int8...
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT8',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'ANSIDATE',
+        'time': 'TIME WITHOUT TIME ZONE',
+        'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
+        'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
+        'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'CLOB',
+        'list:string': 'CLOB',
+        'list:reference': 'CLOB',
+        'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
+        'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO
+        }
+
+    def LEFT_JOIN(self):
+        return 'LEFT OUTER JOIN'
+
+    def RANDOM(self):
+        return 'RANDOM()'
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            fetch_amt = lmax - lmin
+            if fetch_amt:
+                sql_s += ' FIRST %d ' % (fetch_amt, )
+            if lmin:
+                # Requires Ingres 9.2+
+                sql_o += ' OFFSET %d' % (lmin, )
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "ingres"
+        self._driver = pyodbc
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        connstr = uri.split(':', 1)[1]
+        # Simple URI processing
+        connstr = connstr.lstrip()
+        while connstr.startswith('/'):
+            connstr = connstr[1:]
+        if '=' in connstr:
+            # Assume we have a regular ODBC connection string and just use it
+            ruri  = connstr
+        else:
+            # Assume only (local) dbname is passed in with OS auth
+            database_name = connstr
+            default_driver_name = 'Ingres'
+            vnode = '(local)'
+            servertype = 'ingres'
+            ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name)
+        def connector(cnxn=ruri,driver_args=driver_args):
+            return self.driver.connect(cnxn,**driver_args)
+
+        self.connector = connector
+
+        # TODO if version is >= 10, set types['id'] to Identity column, see http://community.actian.com/wiki/Using_Ingres_Identity_Columns
+        if do_connect: self.reconnect()
+
+    def create_sequence_and_triggers(self, query, table, **args):
+        # post create table auto inc code (if needed)
+        # modify table to btree for performance....
+        # Older Ingres releases could use rule/trigger like Oracle above.
+        if hasattr(table,'_primarykey'):
+            modify_tbl_sql = 'modify %s to btree unique on %s' % \
+                (table._tablename,
+                 ', '.join(["'%s'" % x for x in table.primarykey]))
+            self.execute(modify_tbl_sql)
+        else:
+            tmp_seqname='%s_iisq' % table._tablename
+            query=query.replace(INGRES_SEQNAME, tmp_seqname)
+            self.execute('create sequence %s' % tmp_seqname)
+            self.execute(query)
+            self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
+
+
+    def lastrowid(self,table):
+        tmp_seqname='%s_iisq' % table
+        self.execute('select current value for %s' % tmp_seqname)
+        return long(self.cursor.fetchone()[0]) # don't really need int type cast here...
+
+
+class IngresUnicodeAdapter(IngresAdapter):
+
+    drivers = ('pyodbc',)
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'NVARCHAR(%(length)s)',
+        'text': 'NCLOB',
+        'json': 'NCLOB',
+        'password': 'NVARCHAR(%(length)s)',  ## Not sure what this contains utf8 or nvarchar. Or even bytes?
+        'blob': 'BLOB',
+        'upload': 'VARCHAR(%(length)s)',  ## FIXME utf8 or nvarchar... or blob? what is this type?
+        'integer': 'INTEGER4', # or int8...
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT8',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'ANSIDATE',
+        'time': 'TIME WITHOUT TIME ZONE',
+        'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
+        'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
+        'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'NCLOB',
+        'list:string': 'NCLOB',
+        'list:reference': 'NCLOB',
+        'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
+        'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO
+        }

gluon/dal/adapters/mongo.py

@@ -0,0 +1,575 @@
+# -*- coding: utf-8 -*-
+import datetime
+import logging
+import re
+
+from .._globals import IDENTITY
+from ..objects import Table, Query, Field, Expression
+from ..helpers.classes import SQLALL
+from ..helpers.methods import xorify
+from .base import NoSQLAdapter
+
+class MongoDBAdapter(NoSQLAdapter):
+    native_json = True
+    drivers = ('pymongo',)
+
+    uploads_in_blob = False
+
+    types = {
+        'boolean': bool,
+        'string': str,
+        'text': str,
+        'json': str,
+        'password': str,
+        'blob': str,
+        'upload': str,
+        'integer': long,
+        'bigint': long,
+        'float': float,
+        'double': float,
+        'date': datetime.date,
+        'time': datetime.time,
+        'datetime': datetime.datetime,
+        'id': long,
+        'reference': long,
+        'list:string': list,
+        'list:integer': list,
+        'list:reference': list,
+        }
+
+    error_messages = {"javascript_needed": "This must yet be replaced" +
+                      " with javascript in order to work."}
+
+    def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
+                 pool_size=0, folder=None, db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+
+        self.db = db
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args)
+        import random
+        from bson.objectid import ObjectId
+        from bson.son import SON
+        import pymongo.uri_parser
+
+        m = pymongo.uri_parser.parse_uri(uri)
+
+        self.SON = SON
+        self.ObjectId = ObjectId
+        self.random = random
+
+        self.dbengine = 'mongodb'
+        self.folder = folder
+        db['_lastsql'] = ''
+        self.db_codec = 'UTF-8'
+        self._after_connection = after_connection
+        self.pool_size = pool_size
+        #this is the minimum amount of replicates that it should wait
+        # for on insert/update
+        self.minimumreplication = adapter_args.get('minimumreplication',0)
+        # by default all inserts and selects are performand asynchronous,
+        # but now the default is
+        # synchronous, except when overruled by either this default or
+        # function parameter
+        self.safe = adapter_args.get('safe',True)
+        # load user setting for uploads in blob storage
+        self.uploads_in_blob = adapter_args.get('uploads_in_blob', False)
+
+        if isinstance(m,tuple):
+            m = {"database" : m[1]}
+        if m.get('database') is None:
+            raise SyntaxError("Database is required!")
+
+        def connector(uri=self.uri,m=m):
+            # Connection() is deprecated
+            if hasattr(self.driver, "MongoClient"):
+                Connection = self.driver.MongoClient
+            else:
+                Connection = self.driver.Connection
+            return Connection(uri)[m.get('database')]
+
+        self.reconnect(connector,cursor=False)
+
+    def object_id(self, arg=None):
+        """ Convert input to a valid Mongodb ObjectId instance
+
+        self.object_id("<random>") -> ObjectId (not unique) instance """
+        if not arg:
+            arg = 0
+        if isinstance(arg, basestring):
+            # we assume an integer as default input
+            rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
+            if arg.isdigit() and (not rawhex):
+                arg = int(arg)
+            elif arg == "<random>":
+                arg = int("0x%sL" % \
+                "".join([self.random.choice("0123456789abcdef") \
+                for x in range(24)]), 0)
+            elif arg.isalnum():
+                if not arg.startswith("0x"):
+                    arg = "0x%s" % arg
+                try:
+                    arg = int(arg, 0)
+                except ValueError, e:
+                    raise ValueError(
+                            "invalid objectid argument string: %s" % e)
+            else:
+                raise ValueError("Invalid objectid argument string. " +
+                                 "Requires an integer or base 16 value")
+        elif isinstance(arg, self.ObjectId):
+            return arg
+
+        if not isinstance(arg, (int, long)):
+            raise TypeError("object_id argument must be of type " +
+                            "ObjectId or an objectid representable integer")
+        hexvalue = hex(arg)[2:].rstrip('L').zfill(24)
+        return self.ObjectId(hexvalue)
+
+    def parse_reference(self, value, field_type):
+        # here we have to check for ObjectID before base parse
+        if isinstance(value, self.ObjectId):
+            value = long(str(value), 16)
+        return super(MongoDBAdapter,
+                     self).parse_reference(value, field_type)
+
+    def parse_id(self, value, field_type):
+        if isinstance(value, self.ObjectId):
+            value = long(str(value), 16)
+        return super(MongoDBAdapter,
+                     self).parse_id(value, field_type)
+
+    def represent(self, obj, fieldtype):
+        # the base adatpter does not support MongoDB ObjectId
+        if isinstance(obj, self.ObjectId):
+            value = obj
+        else:
+            value = NoSQLAdapter.represent(self, obj, fieldtype)
+        # reference types must be convert to ObjectID
+        if fieldtype  =='date':
+            if value is None:
+                return value
+            # this piece of data can be stripped off based on the fieldtype
+            t = datetime.time(0, 0, 0)
+            # mongodb doesn't has a date object and so it must datetime,
+            # string or integer
+            return datetime.datetime.combine(value, t)
+        elif fieldtype == 'time':
+            if value is None:
+                return value
+            # this piece of data can be stripped of based on the fieldtype
+            d = datetime.date(2000, 1, 1)
+            # mongodb doesn't has a  time object and so it must datetime,
+            # string or integer
+            return datetime.datetime.combine(d, value)
+        elif fieldtype == "blob":
+            if value is None:
+                return value
+            from bson import Binary
+            if not isinstance(value, Binary):
+                if not isinstance(value, basestring):
+                    return Binary(str(value))
+                return Binary(value)
+            return value
+        elif (isinstance(fieldtype, basestring) and
+              fieldtype.startswith('list:')):
+            if fieldtype.startswith('list:reference'):
+                newval = []
+                for v in value:
+                    newval.append(self.object_id(v))
+                return newval
+            return value
+        elif ((isinstance(fieldtype, basestring) and
+               fieldtype.startswith("reference")) or
+               (isinstance(fieldtype, Table)) or fieldtype=="id"):
+            value = self.object_id(value)
+        return value
+
+    def create_table(self, table, migrate=True, fake_migrate=False,
+                     polymodel=None, isCapped=False):
+        if isCapped:
+            raise RuntimeError("Not implemented")
+
+    def count(self, query, distinct=None, snapshot=True):
+        if distinct:
+            raise RuntimeError("COUNT DISTINCT not supported")
+        if not isinstance(query,Query):
+            raise SyntaxError("Not Supported")
+        tablename = self.get_table(query)
+        return long(self.select(query,[self.db[tablename]._id], {},
+                                count=True,snapshot=snapshot)['count'])
+        # Maybe it would be faster if we just implemented the pymongo
+        # .count() function which is probably quicker?
+        # therefor call __select() connection[table].find(query).count()
+        # Since this will probably reduce the return set?
+
+    def expand(self, expression, field_type=None):
+        if isinstance(expression, Query):
+            # any query using 'id':=
+            # set name as _id (as per pymongo/mongodb primary key)
+            # convert second arg to an objectid field
+            # (if its not already)
+            # if second arg is 0 convert to objectid
+            if isinstance(expression.first,Field) and \
+                    ((expression.first.type == 'id') or \
+                    ("reference" in expression.first.type)):
+                if expression.first.type == 'id':
+                    expression.first.name = '_id'
+                # cast to Mongo ObjectId
+                if isinstance(expression.second, (tuple, list, set)):
+                    expression.second = [self.object_id(item) for
+                                         item in expression.second]
+                else:
+                    expression.second = self.object_id(expression.second)
+                result = expression.op(expression.first, expression.second)
+
+        if isinstance(expression, Field):
+            if expression.type=='id':
+                result = "_id"
+            else:
+                result =  expression.name
+        elif isinstance(expression, (Expression, Query)):
+            if not expression.second is None:
+                result = expression.op(expression.first, expression.second)
+            elif not expression.first is None:
+                result = expression.op(expression.first)
+            elif not isinstance(expression.op, str):
+                result = expression.op()
+            else:
+                result = expression.op
+        elif field_type:
+            result = self.represent(expression,field_type)
+        elif isinstance(expression,(list,tuple)):
+            result = [self.represent(item,field_type) for
+                      item in expression]
+        else:
+            result = expression
+        return result
+
+    def drop(self, table, mode=''):
+        ctable = self.connection[table._tablename]
+        ctable.drop()
+
+    def truncate(self, table, mode, safe=None):
+        if safe == None:
+            safe=self.safe
+        ctable = self.connection[table._tablename]
+        ctable.remove(None, safe=True)
+
+    def select(self, query, fields, attributes, count=False,
+               snapshot=False):
+        mongofields_dict = self.SON()
+        mongoqry_dict = {}
+        new_fields, mongosort_list = [], []
+        # try an orderby attribute
+        orderby = attributes.get('orderby', False)
+        limitby = attributes.get('limitby', False)
+        # distinct = attributes.get('distinct', False)
+        if 'for_update' in attributes:
+            logging.warn('mongodb does not support for_update')
+        for key in set(attributes.keys())-set(('limitby',
+                                               'orderby','for_update')):
+            if attributes[key] is not None:
+                logging.warn('select attribute not implemented: %s' % key)
+        if limitby:
+            limitby_skip, limitby_limit = limitby[0], int(limitby[1])
+        else:
+            limitby_skip = limitby_limit = 0
+        if orderby:
+            if isinstance(orderby, (list, tuple)):
+                orderby = xorify(orderby)
+            # !!!! need to add 'random'
+            for f in self.expand(orderby).split(','):
+                if f.startswith('-'):
+                    mongosort_list.append((f[1:], -1))
+                else:
+                    mongosort_list.append((f, 1))
+        for item in fields:
+            if isinstance(item, SQLALL):
+                new_fields += item._table
+            else:
+                new_fields.append(item)
+        fields = new_fields
+        if isinstance(query,Query):
+            tablename = self.get_table(query)
+        elif len(fields) != 0:
+            tablename = fields[0].tablename
+        else:
+            raise SyntaxError("The table name could not be found in " +
+                              "the query nor from the select statement.")
+        mongoqry_dict = self.expand(query)
+        fields = fields or self.db[tablename]
+        for field in fields:
+            mongofields_dict[field.name] = 1
+        ctable = self.connection[tablename]
+        if count:
+            return {'count' : ctable.find(
+                    mongoqry_dict, mongofields_dict,
+                    skip=limitby_skip, limit=limitby_limit,
+                    sort=mongosort_list, snapshot=snapshot).count()}
+        else:
+            # pymongo cursor object
+            mongo_list_dicts = ctable.find(mongoqry_dict,
+                                mongofields_dict, skip=limitby_skip,
+                                limit=limitby_limit, sort=mongosort_list,
+                                snapshot=snapshot)
+        rows = []
+        # populate row in proper order
+        # Here we replace ._id with .id to follow the standard naming
+        colnames = []
+        newnames = []
+        for field in fields:
+            colname = str(field)
+            colnames.append(colname)
+            tablename, fieldname = colname.split(".")
+            if fieldname == "_id":
+                # Mongodb reserved uuid key
+                field.name = "id"
+            newnames.append(".".join((tablename, field.name)))
+
+        for record in mongo_list_dicts:
+            row=[]
+            for colname in colnames:
+                tablename, fieldname = colname.split(".")
+                # switch to Mongo _id uuids for retrieving
+                # record id's
+                if fieldname == "id": fieldname = "_id"
+                if fieldname in record:
+                    value = record[fieldname]
+                else:
+                    value = None
+                row.append(value)
+            rows.append(row)
+        processor = attributes.get('processor', self.parse)
+        result = processor(rows, fields, newnames, False)
+        return result
+
+    def insert(self, table, fields, safe=None):
+        """Safe determines whether a asynchronous request is done or a
+        synchronous action is done
+        For safety, we use by default synchronous requests"""
+
+        values = dict()
+        if safe is None:
+            safe = self.safe
+        ctable = self.connection[table._tablename]
+        for k, v in fields:
+            if not k.name in ["id", "safe"]:
+                fieldname = k.name
+                fieldtype = table[k.name].type
+                values[fieldname] = self.represent(v, fieldtype)
+
+        ctable.insert(values, safe=safe)
+        return long(str(values['_id']), 16)
+
+    def update(self, tablename, query, fields, safe=None):
+        if safe == None:
+            safe = self.safe
+        # return amount of adjusted rows or zero, but no exceptions
+        # @ related not finding the result
+        if not isinstance(query, Query):
+            raise RuntimeError("Not implemented")
+        amount = self.count(query, False)
+        if not isinstance(query, Query):
+            raise SyntaxError("Not Supported")
+        filter = None
+        if query:
+            filter = self.expand(query)
+        # do not try to update id fields to avoid backend errors
+        modify = {'$set': dict((k.name, self.represent(v, k.type)) for
+                  k, v in fields if (not k.name in ("_id", "id")))}
+        try:
+            result = self.connection[tablename].update(filter,
+                       modify, multi=True, safe=safe)
+            if safe:
+                try:
+                    # if result count is available fetch it
+                    return result["n"]
+                except (KeyError, AttributeError, TypeError):
+                    return amount
+            else:
+                return amount
+        except Exception, e:
+            # TODO Reverse update query to verifiy that the query succeded
+            raise RuntimeError("uncaught exception when updating rows: %s" % e)
+
+    def delete(self, tablename, query, safe=None):
+        if safe is None:
+            safe = self.safe
+        amount = 0
+        amount = self.count(query, False)
+        if not isinstance(query, Query):
+            raise RuntimeError("query type %s is not supported" % \
+                               type(query))
+        filter = self.expand(query)
+        self.connection[tablename].remove(filter, safe=safe)
+        return amount
+
+    def bulk_insert(self, table, items):
+        return [self.insert(table,item) for item in items]
+
+    ## OPERATORS
+    def INVERT(self, first):
+        #print "in invert first=%s" % first
+        return '-%s' % self.expand(first)
+
+    # TODO This will probably not work:(
+    def NOT(self, first):
+        return {'$not': self.expand(first)}
+
+    def AND(self,first,second):
+        # pymongo expects: .find({'$and': [{'x':'1'}, {'y':'2'}]})
+        return {'$and': [self.expand(first),self.expand(second)]}
+
+    def OR(self,first,second):
+        # pymongo expects: .find({'$or': [{'name':'1'}, {'name':'2'}]})
+        return {'$or': [self.expand(first),self.expand(second)]}
+
+    def BELONGS(self, first, second):
+        if isinstance(second, str):
+            return {self.expand(first) : {"$in" : [ second[:-1]]} }
+        elif second==[] or second==() or second==set():
+            return {1:0}
+        items = [self.expand(item, first.type) for item in second]
+        return {self.expand(first) : {"$in" : items} }
+
+    def EQ(self,first,second=None):
+        result = {}
+        result[self.expand(first)] = self.expand(second)
+        return result
+
+    def NE(self, first, second=None):
+        result = {}
+        result[self.expand(first)] = {'$ne': self.expand(second)}
+        return result
+
+    def LT(self,first,second=None):
+        if second is None:
+            raise RuntimeError("Cannot compare %s < None" % first)
+        result = {}
+        result[self.expand(first)] = {'$lt': self.expand(second)}
+        return result
+
+    def LE(self,first,second=None):
+        if second is None:
+            raise RuntimeError("Cannot compare %s <= None" % first)
+        result = {}
+        result[self.expand(first)] = {'$lte': self.expand(second)}
+        return result
+
+    def GT(self,first,second):
+        result = {}
+        result[self.expand(first)] = {'$gt': self.expand(second)}
+        return result
+
+    def GE(self,first,second=None):
+        if second is None:
+            raise RuntimeError("Cannot compare %s >= None" % first)
+        result = {}
+        result[self.expand(first)] = {'$gte': self.expand(second)}
+        return result
+
+    def ADD(self, first, second):
+        raise NotImplementedError(self.error_messages["javascript_needed"])
+        return '%s + %s' % (self.expand(first),
+                            self.expand(second, first.type))
+
+    def SUB(self, first, second):
+        raise NotImplementedError(self.error_messages["javascript_needed"])
+        return '(%s - %s)' % (self.expand(first),
+                              self.expand(second, first.type))
+
+    def MUL(self, first, second):
+        raise NotImplementedError(self.error_messages["javascript_needed"])
+        return '(%s * %s)' % (self.expand(first),
+                              self.expand(second, first.type))
+
+    def DIV(self, first, second):
+        raise NotImplementedError(self.error_messages["javascript_needed"])
+        return '(%s / %s)' % (self.expand(first),
+                              self.expand(second, first.type))
+
+    def MOD(self, first, second):
+        raise NotImplementedError(self.error_messages["javascript_needed"])
+        return '(%s %% %s)' % (self.expand(first),
+                               self.expand(second, first.type))
+
+    def AS(self, first, second):
+        raise NotImplementedError(self.error_messages["javascript_needed"])
+        return '%s AS %s' % (self.expand(first), second)
+
+    # We could implement an option that simulates a full featured SQL
+    # database. But I think the option should be set explicit or
+    # implemented as another library.
+    def ON(self, first, second):
+        raise NotImplementedError("This is not possible in NoSQL" +
+                                  " but can be simulated with a wrapper.")
+        return '%s ON %s' % (self.expand(first), self.expand(second))
+
+    # BLOW ARE TWO IMPLEMENTATIONS OF THE SAME FUNCITONS
+    # WHICH ONE IS BEST?
+
+    def COMMA(self, first, second):
+        return '%s, %s' % (self.expand(first), self.expand(second))
+
+    def LIKE(self, first, second):
+        #escaping regex operators?
+        return {self.expand(first): ('%s' % \
+                self.expand(second, 'string').replace('%','/'))}
+
+    def ILIKE(self, first, second):
+        val = second if isinstance(second,self.ObjectId) else {
+            '$regex': second.replace('%', ''), '$options': 'i'}
+        return {self.expand(first): val}
+
+    def STARTSWITH(self, first, second):
+        #escaping regex operators?
+        return {self.expand(first): ('/^%s/' % \
+        self.expand(second, 'string'))}
+
+    def ENDSWITH(self, first, second):
+        #escaping regex operators?
+        return {self.expand(first): ('/%s^/' % \
+        self.expand(second, 'string'))}
+
+    def CONTAINS(self, first, second, case_sensitive=False):
+        # silently ignore, only case sensitive
+        # There is a technical difference, but mongodb doesn't support
+        # that, but the result will be the same
+        val = second if isinstance(second,self.ObjectId) else \
+            {'$regex':".*" + re.escape(self.expand(second, 'string')) + ".*"}
+        return {self.expand(first) : val}
+
+    def LIKE(self, first, second):
+        import re
+        return {self.expand(first): {'$regex': \
+                re.escape(self.expand(second,
+                                      'string')).replace('%','.*')}}
+
+    #TODO verify full compatibilty with official SQL Like operator
+    def STARTSWITH(self, first, second):
+        #TODO  Solve almost the same problem as with endswith
+        import re
+        return {self.expand(first): {'$regex' : '^' +
+                                     re.escape(self.expand(second,
+                                                           'string'))}}
+
+    #TODO verify full compatibilty with official SQL Like operator
+    def ENDSWITH(self, first, second):
+        #escaping regex operators?
+        #TODO if searched for a name like zsa_corbitt and the function
+        # is endswith('a') then this is also returned.
+        # Aldo it end with a t
+        import re
+        return {self.expand(first): {'$regex': \
+        re.escape(self.expand(second, 'string')) + '$'}}
+
+    #TODO verify full compatibilty with official oracle contains operator
+    def CONTAINS(self, first, second, case_sensitive=False):
+        # silently ignore, only case sensitive
+        #There is a technical difference, but mongodb doesn't support
+        # that, but the result will be the same
+        #TODO contains operators need to be transformed to Regex
+        return {self.expand(first) : {'$regex': \
+        ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
+

gluon/dal/adapters/mssql.py

@@ -0,0 +1,483 @@
+# -*- coding: utf-8 -*-
+import re
+import sys
+
+from .._globals import IDENTITY, LOGGER
+from ..helpers.methods import varquote_aux
+from .base import BaseAdapter
+
+
+class MSSQLAdapter(BaseAdapter):
+    drivers = ('pyodbc',)
+    T_SEP = 'T'
+
+    QUOTE_TEMPLATE = '"%s"'
+
+    types = {
+        'boolean': 'BIT',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'TEXT',
+        'json': 'TEXT',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'IMAGE',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATETIME',
+        'time': 'CHAR(8)',
+        'datetime': 'DATETIME',
+        'id': 'INT IDENTITY PRIMARY KEY',
+        'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'TEXT',
+        'list:string': 'TEXT',
+        'list:reference': 'TEXT',
+        'geometry': 'geometry',
+        'geography': 'geography',
+        'big-id': 'BIGINT IDENTITY PRIMARY KEY',
+        'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+        }
+
+    def concat_add(self,tablename):
+        return '; ALTER TABLE %s ADD ' % tablename
+
+    def varquote(self,name):
+        return varquote_aux(name,'[%s]')
+
+    def EXTRACT(self,field,what):
+        return "DATEPART(%s,%s)" % (what, self.expand(field))
+
+    def LEFT_JOIN(self):
+        return 'LEFT OUTER JOIN'
+
+    def RANDOM(self):
+        return 'NEWID()'
+
+    def ALLOW_NULL(self):
+        return ' NULL'
+
+    def CAST(self, first, second):
+        return first # apparently no cast necessary in MSSQL
+
+    def SUBSTRING(self,field,parameters):
+        return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
+
+    def PRIMARY_KEY(self,key):
+        return 'PRIMARY KEY CLUSTERED (%s)' % key
+
+    def AGGREGATE(self, first, what):
+        if what == 'LENGTH':
+            what = 'LEN'
+        return "%s(%s)" % (what, self.expand(first))
+
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            sql_s += ' TOP %i' % lmax
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    TRUE = 1
+    FALSE = 0
+
+    REGEX_DSN = re.compile('^(?P<dsn>.+)$')
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
+    REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, srid=4326,
+                 after_connection=None):
+        self.db = db
+        self.dbengine = "mssql"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.srid = srid
+        self.find_or_make_work_folder()
+        # ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8
+        ruri = uri.split('://',1)[1]
+        if '@' not in ruri:
+            try:
+                m = self.REGEX_DSN.match(ruri)
+                if not m:
+                    raise SyntaxError(
+                        'Parsing uri string(%s) has no result' % self.uri)
+                dsn = m.group('dsn')
+                if not dsn:
+                    raise SyntaxError('DSN required')
+            except SyntaxError:
+                e = sys.exc_info()[1]
+                LOGGER.error('NdGpatch error')
+                raise e
+            # was cnxn = 'DSN=%s' % dsn
+            cnxn = dsn
+        else:
+            m = self.REGEX_URI.match(ruri)
+            if not m:
+                raise SyntaxError(
+                    "Invalid URI string in DAL: %s" % self.uri)
+            user = credential_decoder(m.group('user'))
+            if not user:
+                raise SyntaxError('User required')
+            password = credential_decoder(m.group('password'))
+            if not password:
+                password = ''
+            host = m.group('host')
+            if not host:
+                raise SyntaxError('Host name required')
+            db = m.group('db')
+            if not db:
+                raise SyntaxError('Database name required')
+            port = m.group('port') or '1433'
+            # Parse the optional url name-value arg pairs after the '?'
+            # (in the form of arg1=value1&arg2=value2&...)
+            # Default values (drivers like FreeTDS insist on uppercase parameter keys)
+            argsdict = { 'DRIVER':'{SQL Server}' }
+            urlargs = m.group('urlargs') or ''
+            for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
+                argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
+            urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
+            cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
+                % (host, port, db, user, password, urlargs)
+        def connector(cnxn=cnxn,driver_args=driver_args):
+            return self.driver.connect(cnxn,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def lastrowid(self,table):
+        #self.execute('SELECT @@IDENTITY;')
+        self.execute('SELECT SCOPE_IDENTITY();')
+        return long(self.cursor.fetchone()[0])
+
+    def rowslice(self,rows,minimum=0,maximum=None):
+        if maximum is None:
+            return rows[minimum:]
+        return rows[minimum:maximum]
+
+    def EPOCH(self, first):
+        return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
+
+    def CONCAT(self, *items):
+        return '(%s)' % ' + '.join(self.expand(x,'string') for x in items)
+
+    # GIS Spatial Extensions
+
+    # No STAsGeoJSON in MSSQL
+
+    def ST_ASTEXT(self, first):
+        return '%s.STAsText()' %(self.expand(first))
+
+    def ST_CONTAINS(self, first, second):
+        return '%s.STContains(%s)=1' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_DISTANCE(self, first, second):
+        return '%s.STDistance(%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_EQUALS(self, first, second):
+        return '%s.STEquals(%s)=1' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_INTERSECTS(self, first, second):
+        return '%s.STIntersects(%s)=1' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_OVERLAPS(self, first, second):
+        return '%s.STOverlaps(%s)=1' %(self.expand(first), self.expand(second, first.type))
+
+    # no STSimplify in MSSQL
+
+    def ST_TOUCHES(self, first, second):
+        return '%s.STTouches(%s)=1' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_WITHIN(self, first, second):
+        return '%s.STWithin(%s)=1' %(self.expand(first), self.expand(second, first.type))
+
+    def represent(self, obj, fieldtype):
+        field_is_type = fieldtype.startswith
+        if field_is_type('geometry'):
+            srid = 0 # MS SQL default srid for geometry
+            geotype, parms = fieldtype[:-1].split('(')
+            if parms:
+                srid = parms
+            return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
+        elif fieldtype == 'geography':
+            srid = 4326 # MS SQL default srid for geography
+            geotype, parms = fieldtype[:-1].split('(')
+            if parms:
+                srid = parms
+            return "geography::STGeomFromText('%s',%s)" %(obj, srid)
+#             else:
+#                 raise SyntaxError('Invalid field type %s' %fieldtype)
+            return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
+        return BaseAdapter.represent(self, obj, fieldtype)
+
+
+class MSSQL3Adapter(MSSQLAdapter):
+    """Experimental support for pagination in MSSQL
+
+    Requires MSSQL >= 2005, uses `ROW_NUMBER()`
+    """
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            if lmin == 0:
+                sql_s += ' TOP %i' % lmax
+                return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+            lmin += 1
+            sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
+            sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
+            sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
+            sql_f_inner = [f for f in sql_f.split(',')]
+            sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
+            sql_f_iproxy = ', '.join(sql_f_iproxy)
+            sql_f_oproxy = ', '.join(sql_f_outer)
+            return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
+    def rowslice(self,rows,minimum=0,maximum=None):
+        return rows
+
+
+class MSSQL4Adapter(MSSQLAdapter):
+    """Support for "native" pagination
+
+    Requires MSSQL >= 2012, uses `OFFSET ... ROWS ... FETCH NEXT ... ROWS ONLY`
+    """
+    
+    types = {
+    'boolean': 'BIT',
+    'string': 'VARCHAR(%(length)s)',
+    'text': 'VARCHAR(MAX)',
+    'json': 'VARCHAR(MAX)',
+    'password': 'VARCHAR(%(length)s)',
+    'blob': 'IMAGE',
+    'upload': 'VARCHAR(%(length)s)',
+    'integer': 'INT',
+    'bigint': 'BIGINT',
+    'float': 'FLOAT',
+    'double': 'FLOAT',
+    'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+    'date': 'DATETIME',
+    'time': 'TIME(7)',
+    'datetime': 'DATETIME',
+    'id': 'INT IDENTITY PRIMARY KEY',
+    'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+    'list:integer': 'VARCHAR(MAX)',
+    'list:string': 'VARCHAR(MAX)',
+    'list:reference': 'VARCHAR(MAX)',
+    'geometry': 'geometry',
+    'geography': 'geography',
+    'big-id': 'BIGINT IDENTITY PRIMARY KEY',
+    'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+    'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+    'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+     }
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            if lmin == 0:
+                #top is still slightly faster, especially because
+                #web2py's default to fetch references is to not specify
+                #an orderby clause
+                sql_s += ' TOP %i' % lmax
+            else:
+                if not sql_o:
+                    #if there is no orderby, we can't use the brand new statements
+                    #that being said, developer chose its own poison, so be it random
+                    sql_o += ' ORDER BY %s' % self.RANDOM()
+                sql_o += ' OFFSET %i ROWS FETCH NEXT %i ROWS ONLY' % (lmin, lmax - lmin)
+        return 'SELECT %s %s FROM %s%s%s;' % \
+                (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def rowslice(self,rows,minimum=0,maximum=None):
+        return rows
+
+
+class MSSQL2Adapter(MSSQLAdapter):
+    drivers = ('pyodbc',)
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'NVARCHAR(%(length)s)',
+        'text': 'NTEXT',
+        'json': 'NTEXT',
+        'password': 'NVARCHAR(%(length)s)',
+        'blob': 'IMAGE',
+        'upload': 'NVARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATETIME',
+        'time': 'CHAR(8)',
+        'datetime': 'DATETIME',
+        'id': 'INT IDENTITY PRIMARY KEY',
+        'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'NTEXT',
+        'list:string': 'NTEXT',
+        'list:reference': 'NTEXT',
+        'big-id': 'BIGINT IDENTITY PRIMARY KEY',
+        'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+        }
+
+    def represent(self, obj, fieldtype):
+        value = BaseAdapter.represent(self, obj, fieldtype)
+        if fieldtype in ('string','text', 'json') and value[:1]=="'":
+            value = 'N'+value
+        return value
+
+    def execute(self,a):
+        return self.log_execute(a.decode('utf8'))
+
+
+class VerticaAdapter(MSSQLAdapter):
+    drivers = ('pyodbc',)
+    T_SEP = ' '
+
+    types = {
+        'boolean': 'BOOLEAN',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'BYTEA',
+        'json': 'VARCHAR(%(length)s)',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BYTEA',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'DOUBLE PRECISION',
+        'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'DATETIME',
+        'id': 'IDENTITY',
+        'reference': 'INT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'BYTEA',
+        'list:string': 'BYTEA',
+        'list:reference': 'BYTEA',
+        'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        }
+
+
+    def EXTRACT(self, first, what):
+        return "DATE_PART('%s', TIMESTAMP %s)" % (what, self.expand(first))
+
+    def _truncate(self, table, mode=''):
+        tablename = table._tablename
+        return ['TRUNCATE %s %s;' % (tablename, mode or '')]
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
+        return 'SELECT %s %s FROM %s%s%s;' % \
+            (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def lastrowid(self,table):
+        self.execute('SELECT LAST_INSERT_ID();')
+        return long(self.cursor.fetchone()[0])
+
+    def execute(self, a):
+        return self.log_execute(a)
+
+
+class SybaseAdapter(MSSQLAdapter):
+    drivers = ('Sybase',)
+
+    types = {
+        'boolean': 'BIT',
+        'string': 'CHAR VARYING(%(length)s)',
+        'text': 'TEXT',
+        'json': 'TEXT',
+        'password': 'CHAR VARYING(%(length)s)',
+        'blob': 'IMAGE',
+        'upload': 'CHAR VARYING(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATETIME',
+        'time': 'CHAR(8)',
+        'datetime': 'DATETIME',
+        'id': 'INT IDENTITY PRIMARY KEY',
+        'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'TEXT',
+        'list:string': 'TEXT',
+        'list:reference': 'TEXT',
+        'geometry': 'geometry',
+        'geography': 'geography',
+        'big-id': 'BIGINT IDENTITY PRIMARY KEY',
+        'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+        }
+
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, srid=4326,
+                 after_connection=None):
+        self.db = db
+        self.dbengine = "sybase"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.srid = srid
+        self.find_or_make_work_folder()
+        # ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8
+        ruri = uri.split('://',1)[1]
+        if '@' not in ruri:
+            try:
+                m = self.REGEX_DSN.match(ruri)
+                if not m:
+                    raise SyntaxError(
+                        'Parsing uri string(%s) has no result' % self.uri)
+                dsn = m.group('dsn')
+                if not dsn:
+                    raise SyntaxError('DSN required')
+            except SyntaxError:
+                e = sys.exc_info()[1]
+                LOGGER.error('NdGpatch error')
+                raise e
+        else:
+            m = self.REGEX_URI.match(uri)
+            if not m:
+                raise SyntaxError(
+                    "Invalid URI string in DAL: %s" % self.uri)
+            user = credential_decoder(m.group('user'))
+            if not user:
+                raise SyntaxError('User required')
+            password = credential_decoder(m.group('password'))
+            if not password:
+                password = ''
+            host = m.group('host')
+            if not host:
+                raise SyntaxError('Host name required')
+            db = m.group('db')
+            if not db:
+                raise SyntaxError('Database name required')
+            port = m.group('port') or '1433'
+
+            dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
+
+            driver_args.update(user = credential_decoder(user),
+                               password = credential_decoder(password))
+
+        def connector(dsn=dsn,driver_args=driver_args):
+            return self.driver.connect(dsn,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+

gluon/dal/adapters/mysql.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+import re
+
+from .._globals import IDENTITY
+from ..helpers.methods import varquote_aux
+from .base import BaseAdapter
+
+
+class MySQLAdapter(BaseAdapter):
+    drivers = ('MySQLdb','pymysql', 'mysqlconnector')
+
+    commit_on_alter_table = True
+    support_distributed_transaction = True
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'LONGTEXT',
+        'json': 'LONGTEXT',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'LONGBLOB',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'DOUBLE',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'DATETIME',
+        'id': 'INT AUTO_INCREMENT NOT NULL',
+        'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'LONGTEXT',
+        'list:string': 'LONGTEXT',
+        'list:reference': 'LONGTEXT',
+        'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
+        'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT  `FK_%(constraint_name)s` FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        }
+
+    QUOTE_TEMPLATE = "`%s`"
+
+    def varquote(self,name):
+        return varquote_aux(name,'`%s`')
+
+    def RANDOM(self):
+        return 'RAND()'
+
+    def SUBSTRING(self,field,parameters):
+        return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
+                                        parameters[0], parameters[1])
+
+    def EPOCH(self, first):
+        return "UNIX_TIMESTAMP(%s)" % self.expand(first)
+
+    def CONCAT(self, *items):
+        return 'CONCAT(%s)' % ','.join(self.expand(x,'string') for x in items)
+
+    def REGEXP(self,first,second):
+        return '(%s REGEXP %s)' % (self.expand(first),
+                                   self.expand(second,'string'))
+
+    def CAST(self, first, second):
+        if second=='LONGTEXT': second = 'CHAR'
+        return 'CAST(%s AS %s)' % (first, second)
+
+    def _drop(self,table,mode):
+        # breaks db integrity but without this mysql does not drop table
+        table_rname = table.sqlsafe
+        return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table_rname,
+                'SET FOREIGN_KEY_CHECKS=1;']
+
+    def _insert_empty(self, table):
+        return 'INSERT INTO %s VALUES (DEFAULT);' % (table.sqlsafe)
+
+    def distributed_transaction_begin(self,key):
+        self.execute('XA START;')
+
+    def prepare(self,key):
+        self.execute("XA END;")
+        self.execute("XA PREPARE;")
+
+    def commit_prepared(self,key):
+        self.execute("XA COMMIT;")
+
+    def rollback_prepared(self,key):
+        self.execute("XA ROLLBACK;")
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "mysql"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError(
+                "Invalid URI string in DAL: %s" % self.uri)
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        port = int(m.group('port') or '3306')
+        charset = m.group('charset') or 'utf8'
+        driver_args.update(db=db,
+                           user=credential_decoder(user),
+                           passwd=credential_decoder(password),
+                           host=host,
+                           port=port,
+                           charset=charset)
+
+
+        def connector(driver_args=driver_args):
+            return self.driver.connect(**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.execute('SET FOREIGN_KEY_CHECKS=1;')
+        self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
+
+    def lastrowid(self,table):
+        self.execute('select last_insert_id();')
+        return int(self.cursor.fetchone()[0])

gluon/dal/adapters/oracle.py

@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+import base64
+import datetime
+import re
+
+from .._globals import IDENTITY
+from .._load import cx_Oracle
+from .base import BaseAdapter
+
+class OracleAdapter(BaseAdapter):
+    drivers = ('cx_Oracle',)
+
+    commit_on_alter_table = False
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR2(%(length)s)',
+        'text': 'CLOB',
+        'json': 'CLOB',
+        'password': 'VARCHAR2(%(length)s)',
+        'blob': 'CLOB',
+        'upload': 'VARCHAR2(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'NUMBER',
+        'float': 'FLOAT',
+        'double': 'BINARY_DOUBLE',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'CHAR(8)',
+        'datetime': 'DATE',
+        'id': 'NUMBER PRIMARY KEY',
+        'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'CLOB',
+        'list:string': 'CLOB',
+        'list:reference': 'CLOB',
+        'big-id': 'NUMBER PRIMARY KEY',
+        'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+        }
+
+
+    def trigger_name(self,tablename):
+        return '%s_trigger' % tablename
+
+    def LEFT_JOIN(self):
+        return 'LEFT OUTER JOIN'
+
+    def RANDOM(self):
+        return 'dbms_random.value'
+
+    def NOT_NULL(self,default,field_type):
+        return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
+
+    def REGEXP(self, first, second):
+        return 'REGEXP_LIKE(%s, %s)' % (self.expand(first),
+                                        self.expand(second, 'string'))
+
+    def _drop(self,table,mode):
+        sequence_name = table._sequence_name
+        return ['DROP TABLE %s %s;' % (table.sqlsafe, mode), 'DROP SEQUENCE %s;' % sequence_name]
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            if len(sql_w) > 1:
+                sql_w_row = sql_w + ' AND w_row > %i' % lmin
+            else:
+                sql_w_row = 'WHERE w_row > %i' % lmin
+            return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def constraint_name(self, tablename, fieldname):
+        constraint_name = BaseAdapter.constraint_name(self, tablename, fieldname)
+        if len(constraint_name)>30:
+            constraint_name = '%s_%s__constraint' % (tablename[:10], fieldname[:7])
+        return constraint_name
+
+    def represent_exceptions(self, obj, fieldtype):
+        if fieldtype == 'blob':
+            obj = base64.b64encode(str(obj))
+            return ":CLOB('%s')" % obj
+        elif fieldtype == 'date':
+            if isinstance(obj, (datetime.date, datetime.datetime)):
+                obj = obj.isoformat()[:10]
+            else:
+                obj = str(obj)
+            return "to_date('%s','yyyy-mm-dd')" % obj
+        elif fieldtype == 'datetime':
+            if isinstance(obj, datetime.datetime):
+                obj = obj.isoformat()[:19].replace('T',' ')
+            elif isinstance(obj, datetime.date):
+                obj = obj.isoformat()[:10]+' 00:00:00'
+            else:
+                obj = str(obj)
+            return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
+        return None
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "oracle"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        if not 'threaded' in driver_args:
+            driver_args['threaded']=True
+        def connector(uri=ruri,driver_args=driver_args):
+            return self.driver.connect(uri,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
+        self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
+
+    oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
+
+    def execute(self, command, args=None):
+        args = args or []
+        i = 1
+        while True:
+            m = self.oracle_fix.match(command)
+            if not m:
+                break
+            command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
+            args.append(m.group('clob')[6:-2].replace("''", "'"))
+            i += 1
+        if command[-1:]==';':
+            command = command[:-1]
+        return self.log_execute(command, args)
+
+    def create_sequence_and_triggers(self, query, table, **args):
+        tablename = table._tablename
+        id_name = table._id.name
+        sequence_name = table._sequence_name
+        trigger_name = table._trigger_name
+        self.execute(query)
+        self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
+        self.execute("""
+            CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
+            DECLARE
+                curr_val NUMBER;
+                diff_val NUMBER;
+                PRAGMA autonomous_transaction;
+            BEGIN
+                IF :NEW.%(id)s IS NOT NULL THEN
+                    EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
+                    diff_val := :NEW.%(id)s - curr_val - 1;
+                    IF diff_val != 0 THEN
+                      EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
+                      EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
+                      EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
+                    END IF;
+                END IF;
+                SELECT %(sequence_name)s.nextval INTO :NEW.%(id)s FROM DUAL;
+            END;
+        """ % dict(trigger_name=trigger_name, tablename=tablename,
+                   sequence_name=sequence_name,id=id_name))
+
+    def lastrowid(self,table):
+        sequence_name = table._sequence_name
+        self.execute('SELECT %s.currval FROM dual;' % sequence_name)
+        return long(self.cursor.fetchone()[0])
+
+    #def parse_value(self, value, field_type, blob_decode=True):
+    #    if blob_decode and isinstance(value, cx_Oracle.LOB):
+    #        try:
+    #            value = value.read()
+    #        except self.driver.ProgrammingError:
+    #            # After a subsequent fetch the LOB value is not valid anymore
+    #            pass
+    #    return BaseAdapter.parse_value(self, value, field_type, blob_decode)
+
+    def _fetchall(self):
+        if any(x[1]==cx_Oracle.LOB for x in self.cursor.description):
+            return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
+                               for c in r]) for r in self.cursor]
+        else:
+            return self.cursor.fetchall()
+
+    def sqlsafe_table(self, tablename, ot=None):
+        if ot is not None:
+            return (self.QUOTE_TEMPLATE + ' ' \
+                    + self.QUOTE_TEMPLATE) % (ot, tablename)
+        return self.QUOTE_TEMPLATE % tablename

gluon/dal/adapters/postgre.py

@@ -0,0 +1,420 @@
+# -*- coding: utf-8 -*-
+import re
+
+from .._load import psycopg2_adapt
+from .._globals import IDENTITY, LOGGER
+from ..helpers.methods import varquote_aux
+from .base import BaseAdapter
+
+
+class PostgreSQLAdapter(BaseAdapter):
+    drivers = ('psycopg2','pg8000')
+
+    QUOTE_TEMPLATE = '"%s"'
+
+    support_distributed_transaction = True
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'TEXT',
+        'json': 'TEXT',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BYTEA',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INTEGER',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT8',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'TIMESTAMP',
+        'id': 'SERIAL PRIMARY KEY',
+        'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'TEXT',
+        'list:string': 'TEXT',
+        'list:reference': 'TEXT',
+        'geometry': 'GEOMETRY',
+        'geography': 'GEOGRAPHY',
+        'big-id': 'BIGSERIAL PRIMARY KEY',
+        'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference FK': ', CONSTRAINT  "FK_%(constraint_name)s" FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'reference TFK': ' CONSTRAINT  "FK_%(foreign_table)s_PK" FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
+
+        }
+
+
+    def varquote(self,name):
+        return varquote_aux(name,'"%s"')
+
+    def adapt(self,obj):
+        if self.driver_name == 'psycopg2':
+            return psycopg2_adapt(obj).getquoted()
+        elif self.driver_name == 'pg8000':
+            return "'%s'" % str(obj).replace("%","%%").replace("'","''")
+        else:
+            return "'%s'" % str(obj).replace("'","''")
+
+    def sequence_name(self,table):
+        return self.QUOTE_TEMPLATE % (table + '_id_seq')
+
+    def RANDOM(self):
+        return 'RANDOM()'
+
+    def ADD(self, first, second):
+        t = first.type
+        if t in ('text','string','password', 'json', 'upload','blob'):
+            return '(%s || %s)' % (self.expand(first), self.expand(second, t))
+        else:
+            return '(%s + %s)' % (self.expand(first), self.expand(second, t))
+
+    def distributed_transaction_begin(self,key):
+        return
+
+    def prepare(self,key):
+        self.execute("PREPARE TRANSACTION '%s';" % key)
+
+    def commit_prepared(self,key):
+        self.execute("COMMIT PREPARED '%s';" % key)
+
+    def rollback_prepared(self,key):
+        self.execute("ROLLBACK PREPARED '%s';" % key)
+
+    def create_sequence_and_triggers(self, query, table, **args):
+        # following lines should only be executed if table._sequence_name does not exist
+        # self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
+        # self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
+        #              % (table._tablename, table._fieldname, table._sequence_name))
+        self.execute(query)
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, srid=4326,
+                 after_connection=None):
+        self.db = db
+        self.dbengine = "postgres"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.srid = srid
+        self.find_or_make_work_folder()
+        self._last_insert = None # for INSERT ... RETURNING ID
+        
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError("Invalid URI string in DAL")
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        port = m.group('port') or '5432'
+        sslmode = m.group('sslmode')
+        if sslmode:
+            msg = ("dbname='%s' user='%s' host='%s' "
+                   "port=%s password='%s' sslmode='%s'") \
+                   % (db, user, host, port, password, sslmode)
+        else:
+            msg = ("dbname='%s' user='%s' host='%s' "
+                   "port=%s password='%s'") \
+                   % (db, user, host, port, password)
+        # choose diver according uri
+        if self.driver:
+            self.__version__ = "%s %s" % (self.driver.__name__,
+                                          self.driver.__version__)
+        else:
+            self.__version__ = None
+        def connector(msg=msg,driver_args=driver_args):
+            return self.driver.connect(msg,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.connection.set_client_encoding('UTF8')
+        self.execute("SET standard_conforming_strings=on;")
+        self.try_json()
+
+    def _insert(self, table, fields):
+        table_rname = table.sqlsafe
+        if fields:
+            keys = ','.join(f.sqlsafe_name for f, v in fields)
+            values = ','.join(self.expand(v, f.type) for f, v in fields)
+            if table._id:
+                self._last_insert = (table._id, 1)
+                return 'INSERT INTO %s(%s) VALUES (%s) RETURNING %s;' % (
+                    table_rname, keys, values, table._id.name)
+            else:
+                self._last_insert = None
+                return 'INSERT INTO %s(%s) VALUES (%s);' % (table_rname, keys, values)
+        else:
+            self._last_insert
+            return self._insert_empty(table)
+
+    def lastrowid(self, table=None):
+        if self._last_insert:
+            return int(self.cursor.fetchone()[0])
+        else:
+            self.execute("select lastval()")
+            return int(self.cursor.fetchone()[0])
+
+    def try_json(self):
+        # check JSON data type support
+        # (to be added to after_connection)
+        if self.driver_name == "pg8000":
+            supports_json = self.connection.server_version >= "9.2.0"
+        elif (self.driver_name == "psycopg2") and \
+             (self.driver.__version__ >= "2.0.12"):
+            supports_json = self.connection.server_version >= 90200
+        elif self.driver_name == "zxJDBC":
+            supports_json = self.connection.dbversion >= "9.2.0"
+        else: supports_json = None
+        if supports_json:
+            self.types["json"] = "JSON"
+            self.native_json = True
+        else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
+
+    def LIKE(self,first,second):
+        args = (self.expand(first), self.expand(second,'string'))
+        if not first.type in ('string', 'text', 'json'):
+            return '(%s LIKE %s)' % (
+                self.CAST(args[0], 'CHAR(%s)' % first.length), args[1])
+        else:
+            return '(%s LIKE %s)' % args
+
+    def ILIKE(self,first,second):
+        args = (self.expand(first), self.expand(second,'string'))
+        if not first.type in ('string', 'text', 'json'):
+            return '(%s LIKE %s)' % (
+                self.CAST(args[0], 'CHAR(%s)' % first.length), args[1])
+        else:
+            return '(%s ILIKE %s)' % args
+
+    def REGEXP(self,first,second):
+        return '(%s ~ %s)' % (self.expand(first),
+                              self.expand(second,'string'))
+
+    def STARTSWITH(self,first,second):
+        return '(%s LIKE %s)' % (self.expand(first),
+                                  self.expand(second+'%','string'))
+
+    def ENDSWITH(self,first,second):
+        return '(%s LIKE %s)' % (self.expand(first),
+                                  self.expand('%'+second,'string'))
+
+    # GIS functions
+
+    def ST_ASGEOJSON(self, first, second):
+        """
+        http://postgis.org/docs/ST_AsGeoJSON.html
+        """
+        return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
+            self.expand(first), second['precision'], second['options'])
+
+    def ST_ASTEXT(self, first):
+        """
+        http://postgis.org/docs/ST_AsText.html
+        """
+        return 'ST_AsText(%s)' %(self.expand(first))
+
+    def ST_X(self, first):
+        """
+        http://postgis.org/docs/ST_X.html
+        """
+        return 'ST_X(%s)' %(self.expand(first))
+
+    def ST_Y(self, first):
+        """
+        http://postgis.org/docs/ST_Y.html
+        """
+        return 'ST_Y(%s)' %(self.expand(first))
+
+    def ST_CONTAINS(self, first, second):
+        """
+        http://postgis.org/docs/ST_Contains.html
+        """
+        return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_DISTANCE(self, first, second):
+        """
+        http://postgis.org/docs/ST_Distance.html
+        """
+        return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_EQUALS(self, first, second):
+        """
+        http://postgis.org/docs/ST_Equals.html
+        """
+        return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_INTERSECTS(self, first, second):
+        """
+        http://postgis.org/docs/ST_Intersects.html
+        """
+        return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_OVERLAPS(self, first, second):
+        """
+        http://postgis.org/docs/ST_Overlaps.html
+        """
+        return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_SIMPLIFY(self, first, second):
+        """
+        http://postgis.org/docs/ST_Simplify.html
+        """
+        return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
+
+    def ST_TOUCHES(self, first, second):
+        """
+        http://postgis.org/docs/ST_Touches.html
+        """
+        return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_WITHIN(self, first, second):
+        """
+        http://postgis.org/docs/ST_Within.html
+        """
+        return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
+
+    def ST_DWITHIN(self, first, (second, third)):
+        """
+        http://postgis.org/docs/ST_DWithin.html
+        """
+        return 'ST_DWithin(%s,%s,%s)' %(self.expand(first),
+                                        self.expand(second, first.type),
+                                        self.expand(third, 'double'))
+
+    def represent(self, obj, fieldtype):
+        field_is_type = fieldtype.startswith
+        if field_is_type('geo'):
+            srid = 4326 # postGIS default srid for geometry
+            geotype, parms = fieldtype[:-1].split('(')
+            parms = parms.split(',')
+            if len(parms) >= 2:
+                schema, srid = parms[:2]
+            if field_is_type('geometry'):
+                value = "ST_GeomFromText('%s',%s)" %(obj, srid)
+            elif field_is_type('geography'):
+                value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
+#             else:
+#                 raise SyntaxError('Invalid field type %s' %fieldtype)
+            return value
+        return BaseAdapter.represent(self, obj, fieldtype)
+
+    def _drop(self, table, mode='restrict'):
+        if mode not in ['restrict', 'cascade', '']:
+            raise ValueError('Invalid mode: %s' % mode)
+        return ['DROP TABLE ' + table.sqlsafe + ' ' + str(mode) + ';']
+
+class NewPostgreSQLAdapter(PostgreSQLAdapter):
+    drivers = ('psycopg2','pg8000')
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'TEXT',
+        'json': 'TEXT',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BYTEA',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INTEGER',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'FLOAT8',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'TIMESTAMP',
+        'id': 'SERIAL PRIMARY KEY',
+        'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'BIGINT[]',
+        'list:string': 'TEXT[]',
+        'list:reference': 'BIGINT[]',
+        'geometry': 'GEOMETRY',
+        'geography': 'GEOGRAPHY',
+        'big-id': 'BIGSERIAL PRIMARY KEY',
+        'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        }
+
+    def parse_list_integers(self, value, field_type):
+        return value
+
+    def parse_list_references(self, value, field_type):
+        return [self.parse_reference(r, field_type[5:]) for r in value]
+
+    def parse_list_strings(self, value, field_type):
+        return value
+
+    def represent(self, obj, fieldtype):
+        field_is_type = fieldtype.startswith
+        if field_is_type('list:'):
+            if not obj:
+                obj = []
+            elif not isinstance(obj, (list, tuple)):
+                obj = [obj]
+            if field_is_type('list:string'):
+                obj = map(str,obj)
+            else:
+                obj = map(int,obj)
+            return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
+        return BaseAdapter.represent(self, obj, fieldtype)
+
+
+class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
+    drivers = ('zxJDBC',)
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None    ):
+        self.db = db
+        self.dbengine = "postgres"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError("Invalid URI string in DAL")
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        port = m.group('port') or '5432'
+        msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
+        def connector(msg=msg,driver_args=driver_args):
+            return self.driver.connect(*msg,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.connection.set_client_encoding('UTF8')
+        self.execute('BEGIN;')
+        self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
+        self.try_json()

gluon/dal/adapters/sapdb.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+import re
+
+from .._globals import IDENTITY
+from .base import BaseAdapter
+
+
+class SAPDBAdapter(BaseAdapter):
+    drivers = ('sapdb',)
+
+    support_distributed_transaction = False
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'LONG',
+        'json': 'LONG',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'LONG',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'FLOAT',
+        'double': 'DOUBLE PRECISION',
+        'decimal': 'FIXED(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'TIMESTAMP',
+        'id': 'INT PRIMARY KEY',
+        'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        'list:integer': 'LONG',
+        'list:string': 'LONG',
+        'list:reference': 'LONG',
+        'big-id': 'BIGINT PRIMARY KEY',
+        'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
+        }
+
+    def sequence_name(self,table):
+        return (self.QUOTE_TEMPLATE + '_id_Seq') % table
+
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            if len(sql_w) > 1:
+                sql_w_row = sql_w + ' AND w_row > %i' % lmin
+            else:
+                sql_w_row = 'WHERE w_row > %i' % lmin
+            return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def create_sequence_and_triggers(self, query, table, **args):
+        # following lines should only be executed if table._sequence_name does not exist
+        self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
+        self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
+                         % (table._tablename, table._id.name, table._sequence_name))
+        self.execute(query)
+
+    REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
+
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "sapdb"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://',1)[1]
+        m = self.REGEX_URI.match(ruri)
+        if not m:
+            raise SyntaxError("Invalid URI string in DAL")
+        user = credential_decoder(m.group('user'))
+        if not user:
+            raise SyntaxError('User required')
+        password = credential_decoder(m.group('password'))
+        if not password:
+            password = ''
+        host = m.group('host')
+        if not host:
+            raise SyntaxError('Host name required')
+        db = m.group('db')
+        if not db:
+            raise SyntaxError('Database name required')
+        def connector(user=user, password=password, database=db,
+                    host=host, driver_args=driver_args):
+            return self.driver.Connection(user, password, database,
+                                          host, **driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def lastrowid(self,table):
+        self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
+        return long(self.cursor.fetchone()[0])

gluon/dal/adapters/sqlite.py

@@ -0,0 +1,280 @@
+# -*- coding: utf-8 -*-
+import copy
+import datetime
+import locale
+import platform
+import re
+import sys
+import time
+
+from .._compat import PY2, pjoin
+from .._globals import IDENTITY
+from .base import BaseAdapter
+
+
+class SQLiteAdapter(BaseAdapter):
+    drivers = ('sqlite2','sqlite3')
+
+    can_select_for_update = None    # support ourselves with BEGIN TRANSACTION
+
+    def EXTRACT(self,field,what):
+        return "web2py_extract('%s',%s)" % (what, self.expand(field))
+
+    @staticmethod
+    def web2py_extract(lookup, s):
+        table = {
+            'year': (0, 4),
+            'month': (5, 7),
+            'day': (8, 10),
+            'hour': (11, 13),
+            'minute': (14, 16),
+            'second': (17, 19),
+            }
+        try:
+            if lookup != 'epoch':
+                (i, j) = table[lookup]
+                return int(s[i:j])
+            else:
+                return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
+        except:
+            return None
+
+    @staticmethod
+    def web2py_regexp(expression, item):
+        return re.compile(expression).search(item) is not None
+
+    def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "sqlite"
+        self.uri = uri
+        self.adapter_args = adapter_args
+        if do_connect: self.find_driver(adapter_args)
+        self.pool_size = 0
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        path_encoding = sys.getfilesystemencoding() \
+            or locale.getdefaultlocale()[1] or 'utf8'
+        if uri.startswith('sqlite:memory'):
+            self.dbpath = ':memory:'
+        else:
+            self.dbpath = uri.split('://',1)[1]
+            if self.dbpath[0] != '/':
+                if PY2:
+                    self.dbpath = pjoin(
+                        self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
+                else:
+                    self.dbpath = pjoin(self.folder, self.dbpath)
+        if not 'check_same_thread' in driver_args:
+            driver_args['check_same_thread'] = False
+        if not 'detect_types' in driver_args and do_connect:
+            driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
+        def connector(dbpath=self.dbpath, driver_args=driver_args):
+            return self.driver.Connection(dbpath, **driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.connection.create_function('web2py_extract', 2,
+                                        SQLiteAdapter.web2py_extract)
+        self.connection.create_function("REGEXP", 2,
+                                        SQLiteAdapter.web2py_regexp)
+
+        if self.adapter_args.get('foreign_keys',True):
+            self.execute('PRAGMA foreign_keys=ON;')
+
+    def _truncate(self, table, mode=''):
+        tablename = table._tablename
+        return ['DELETE FROM %s;' % tablename,
+                "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
+
+    def lastrowid(self, table):
+        return self.cursor.lastrowid
+
+    def REGEXP(self,first,second):
+        return '(%s REGEXP %s)' % (self.expand(first),
+                                   self.expand(second,'string'))
+    
+    def delete(self, tablename, query):
+        # SQLite requires its own delete to handle CASCADE
+        db = self.db
+        table = db[tablename]
+        deleted = [x[table._id.name] for x in db(query).select(table._id)]
+
+        counter = super(SQLiteAdapter, self).delete(tablename, query)
+
+        if counter:
+            for field in table._referenced_by:
+                if field.type == 'reference '+ tablename \
+                        and field.ondelete == 'CASCADE':
+                    db(field.belongs(deleted)).delete()
+
+        return counter
+
+    def select(self, query, fields, attributes):
+        """
+        Simulate `SELECT ... FOR UPDATE` with `BEGIN IMMEDIATE TRANSACTION`.
+        Note that the entire database, rather than one record, is locked
+        (it will be locked eventually anyway by the following UPDATE).
+        """
+        if attributes.get('for_update', False) and not 'cache' in attributes:
+            self.execute('BEGIN IMMEDIATE TRANSACTION;')
+        return super(SQLiteAdapter, self).select(query, fields, attributes)
+
+
+SPATIALLIBS = {
+    'Windows':'libspatialite',
+    'Linux':'libspatialite.so',
+    'Darwin':'libspatialite.dylib'
+    }
+
+class SpatiaLiteAdapter(SQLiteAdapter):
+    drivers = ('sqlite3','sqlite2')
+
+    types = copy.copy(BaseAdapter.types)
+    types.update(geometry='GEOMETRY')
+
+    def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
+        self.db = db
+        self.dbengine = "spatialite"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args)
+        self.pool_size = 0
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        self.srid = srid
+        path_encoding = sys.getfilesystemencoding() \
+            or locale.getdefaultlocale()[1] or 'utf8'
+        if uri.startswith('spatialite:memory'):
+            self.dbpath = ':memory:'
+        else:
+            self.dbpath = uri.split('://',1)[1]
+            if self.dbpath[0] != '/':
+                self.dbpath = pjoin(
+                    self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
+        if not 'check_same_thread' in driver_args:
+            driver_args['check_same_thread'] = False
+        if not 'detect_types' in driver_args and do_connect:
+            driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
+        def connector(dbpath=self.dbpath, driver_args=driver_args):
+            return self.driver.Connection(dbpath, **driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        self.connection.enable_load_extension(True)
+        # for Windows, rename libspatialite-2.dll to libspatialite.dll
+        # Linux uses libspatialite.so
+        # Mac OS X uses libspatialite.dylib
+        libspatialite = SPATIALLIBS[platform.system()]
+        self.execute(r'SELECT load_extension("%s");' % libspatialite)
+
+        self.connection.create_function('web2py_extract', 2,
+                                        SQLiteAdapter.web2py_extract)
+        self.connection.create_function("REGEXP", 2,
+                                        SQLiteAdapter.web2py_regexp)
+
+    # GIS functions
+
+    def ST_ASGEOJSON(self, first, second):
+        return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
+            second['precision'], second['options'])
+
+    def ST_ASTEXT(self, first):
+        return 'AsText(%s)' %(self.expand(first))
+
+    def ST_CONTAINS(self, first, second):
+        return 'Contains(%s,%s)' %(self.expand(first),
+                                   self.expand(second, first.type))
+
+    def ST_DISTANCE(self, first, second):
+        return 'Distance(%s,%s)' %(self.expand(first),
+                                   self.expand(second, first.type))
+
+    def ST_EQUALS(self, first, second):
+        return 'Equals(%s,%s)' %(self.expand(first),
+                                 self.expand(second, first.type))
+
+    def ST_INTERSECTS(self, first, second):
+        return 'Intersects(%s,%s)' %(self.expand(first),
+                                     self.expand(second, first.type))
+
+    def ST_OVERLAPS(self, first, second):
+        return 'Overlaps(%s,%s)' %(self.expand(first),
+                                   self.expand(second, first.type))
+
+    def ST_SIMPLIFY(self, first, second):
+        return 'Simplify(%s,%s)' %(self.expand(first),
+                                   self.expand(second, 'double'))
+
+    def ST_TOUCHES(self, first, second):
+        return 'Touches(%s,%s)' %(self.expand(first),
+                                  self.expand(second, first.type))
+
+    def ST_WITHIN(self, first, second):
+        return 'Within(%s,%s)' %(self.expand(first),
+                                 self.expand(second, first.type))
+
+    def represent(self, obj, fieldtype):
+        field_is_type = fieldtype.startswith
+        if field_is_type('geo'):
+            srid = 4326 # Spatialite default srid for geometry
+            geotype, parms = fieldtype[:-1].split('(')
+            parms = parms.split(',')
+            if len(parms) >= 2:
+                schema, srid = parms[:2]
+#             if field_is_type('geometry'):
+            value = "ST_GeomFromText('%s',%s)" %(obj, srid)
+#             elif field_is_type('geography'):
+#                 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
+#             else:
+#                 raise SyntaxError, 'Invalid field type %s' %fieldtype
+            return value
+        return BaseAdapter.represent(self, obj, fieldtype)
+
+
+class JDBCSQLiteAdapter(SQLiteAdapter):
+    drivers = ('zxJDBC_sqlite',)
+
+    def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "sqlite"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        path_encoding = sys.getfilesystemencoding() \
+            or locale.getdefaultlocale()[1] or 'utf8'
+        if uri.startswith('sqlite:memory'):
+            self.dbpath = ':memory:'
+        else:
+            self.dbpath = uri.split('://',1)[1]
+            if self.dbpath[0] != '/':
+                self.dbpath = pjoin(
+                    self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
+        def connector(dbpath=self.dbpath,driver_args=driver_args):
+            return self.driver.connect(
+                self.driver.getConnection('jdbc:sqlite:'+dbpath),
+                **driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def after_connection(self):
+        # FIXME http://www.zentus.com/sqlitejdbc/custom_functions.html for UDFs
+        self.connection.create_function('web2py_extract', 2,
+                                        SQLiteAdapter.web2py_extract)
+
+    def execute(self, a):
+        return self.log_execute(a)

gluon/dal/adapters/teradata.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+from .._globals import IDENTITY
+from ..connection import ConnectionPool
+from .base import BaseAdapter
+
+
+class TeradataAdapter(BaseAdapter):
+    drivers = ('pyodbc',)
+
+    types = {
+        'boolean': 'CHAR(1)',
+        'string': 'VARCHAR(%(length)s)',
+        'text': 'VARCHAR(2000)',
+        'json': 'VARCHAR(4000)',
+        'password': 'VARCHAR(%(length)s)',
+        'blob': 'BLOB',
+        'upload': 'VARCHAR(%(length)s)',
+        'integer': 'INT',
+        'bigint': 'BIGINT',
+        'float': 'REAL',
+        'double': 'DOUBLE',
+        'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
+        'date': 'DATE',
+        'time': 'TIME',
+        'datetime': 'TIMESTAMP',
+        # Modified Constraint syntax for Teradata.
+        # Teradata does not support ON DELETE.
+        'id': 'INT GENERATED ALWAYS AS IDENTITY',  # Teradata Specific
+        'reference': 'INT',
+        'list:integer': 'VARCHAR(4000)',
+        'list:string': 'VARCHAR(4000)',
+        'list:reference': 'VARCHAR(4000)',
+        'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',  # Teradata Specific
+        'big-reference': 'BIGINT',
+        'reference FK': ' REFERENCES %(foreign_key)s',
+        'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
+        }
+
+    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
+                 credential_decoder=IDENTITY, driver_args={},
+                 adapter_args={}, do_connect=True, after_connection=None):
+        self.db = db
+        self.dbengine = "teradata"
+        self.uri = uri
+        if do_connect: self.find_driver(adapter_args,uri)
+        self.pool_size = pool_size
+        self.folder = folder
+        self.db_codec = db_codec
+        self._after_connection = after_connection
+        self.find_or_make_work_folder()
+        ruri = uri.split('://', 1)[1]
+        def connector(cnxn=ruri,driver_args=driver_args):
+            return self.driver.connect(cnxn,**driver_args)
+        self.connector = connector
+        if do_connect: self.reconnect()
+
+    def close(self,action='commit',really=True):
+        # Teradata does not implicitly close off the cursor
+        # leading to SQL_ACTIVE_STATEMENTS limit errors
+        self.cursor.close()
+        ConnectionPool.close(self, action, really)
+
+    def LEFT_JOIN(self):
+        return 'LEFT OUTER JOIN'
+
+    # Similar to MSSQL, Teradata can't specify a range (for Pageby)
+    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
+        if limitby:
+            (lmin, lmax) = limitby
+            sql_s += ' TOP %i' % lmax
+        return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
+
+    def _truncate(self, table, mode=''):
+        tablename = table._tablename
+        return ['DELETE FROM %s ALL;' % (tablename)]