1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Thanks to
10 * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
11 * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
12 * Denes
13 * Chris Clark
14 * clach05
15 * Denes Lengyel
16 * and many others who have contributed to current and previous versions
17
18 This file contains the DAL support for many relational databases,
19 including:
20 - SQLite & SpatiaLite
21 - MySQL
22 - Postgres
23 - Firebird
24 - Oracle
25 - MS SQL
26 - DB2
27 - Interbase
28 - Ingres
29 - Informix (9+ and SE)
30 - SapDB (experimental)
31 - Cubrid (experimental)
32 - CouchDB (experimental)
33 - MongoDB (in progress)
34 - Google:nosql
35 - Google:sql
36 - Teradata
37 - IMAP (experimental)
38
39 Example of usage:
40
41 >>> # from dal import DAL, Field
42
43 ### create DAL connection (and create DB if it doesn't exist)
44 >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
45 ... folder=None)
46
47 ### define a table 'person' (create/alter as necessary)
48 >>> person = db.define_table('person',Field('name','string'))
49
50 ### insert a record
51 >>> id = person.insert(name='James')
52
53 ### retrieve it by id
54 >>> james = person(id)
55
56 ### retrieve it by name
57 >>> james = person(name='James')
58
59 ### retrieve it by arbitrary query
60 >>> query = (person.name=='James') & (person.name.startswith('J'))
61 >>> james = db(query).select(person.ALL)[0]
62
63 ### update one record
64 >>> james.update_record(name='Jim')
65 <Row {'id': 1, 'name': 'Jim'}>
66
67 ### update multiple records by query
68 >>> db(person.name.like('J%')).update(name='James')
69 1
70
71 ### delete records by query
72 >>> db(person.name.lower() == 'jim').delete()
73 0
74
75 ### retrieve multiple records (rows)
76 >>> people = db(person).select(orderby=person.name,
77 ... groupby=person.name, limitby=(0,100))
78
79 ### further filter them
80 >>> james = people.find(lambda row: row.name == 'James').first()
81 >>> print james.id, james.name
82 1 James
83
84 ### check aggregates
85 >>> counter = person.id.count()
86 >>> print db(person).select(counter).first()(counter)
87 1
88
89 ### delete one record
90 >>> james.delete_record()
91 1
92
93 ### delete (drop) entire database table
94 >>> person.drop()
95
96 Supported field types:
97 id string text boolean integer double decimal password upload
98 blob time date datetime
99
100 Supported DAL URI strings:
101 'sqlite://test.db'
102 'spatialite://test.db'
103 'sqlite:memory'
104 'spatialite:memory'
105 'jdbc:sqlite://test.db'
106 'mysql://root:none@localhost/test'
107 'postgres://mdipierro:password@localhost/test'
108 'postgres:psycopg2://mdipierro:password@localhost/test'
109 'postgres:pg8000://mdipierro:password@localhost/test'
110 'jdbc:postgres://mdipierro:none@localhost/test'
111 'mssql://web2py:none@A64X2/web2py_test'
112 'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
113 'oracle://username:password@database'
114 'firebird://user:password@server:3050/database'
115 'db2://DSN=dsn;UID=user;PWD=pass'
116 'firebird://username:password@hostname/database'
117 'firebird_embedded://username:password@c://path'
118 'informix://user:password@server:3050/database'
119 'informixu://user:password@server:3050/database' # unicode informix
120 'google:datastore' # for google app engine datastore
121 'google:sql' # for google app engine with sql (mysql compatible)
122 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
123 'imap://user:password@server:port' # experimental
124
125 For more info:
126 help(DAL)
127 help(Field)
128 """
129
130
131
132
133
134 __all__ = ['DAL', 'Field']
135
136 MAXCHARLENGTH = 2**15
137 DEFAULTLENGTH = {'string':512,
138 'password':512,
139 'upload':512,
140 'text':2**15,
141 'blob':2**31}
142 TIMINGSSIZE = 100
143 SPATIALLIBS = {
144 'Windows':'libspatialite',
145 'Linux':'libspatialite.so',
146 'Darwin':'libspatialite.dylib'
147 }
148
149 import re
150 import sys
151 import locale
152 import os
153 import types
154 import datetime
155 import threading
156 import time
157 import csv
158 import cgi
159 import copy
160 import socket
161 import logging
162 import base64
163 import shutil
164 import marshal
165 import decimal
166 import struct
167 import urllib
168 import hashlib
169 import uuid
170 import glob
171 import traceback
172 import platform
173
174 PYTHON_VERSION = sys.version_info[0]
175 if PYTHON_VERSION == 2:
176 import cPickle as pickle
177 import cStringIO as StringIO
178 import copy_reg as copyreg
179 hashlib_md5 = hashlib.md5
180 bytes, unicode = str, unicode
181 else:
182 import pickle
183 from io import StringIO as StringIO
184 import copyreg
185 long = int
186 hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
187 bytes, unicode = bytes, str
188
189 CALLABLETYPES = (types.LambdaType, types.FunctionType,
190 types.BuiltinFunctionType,
191 types.MethodType, types.BuiltinMethodType)
192
193 TABLE_ARGS = set(
194 ('migrate','primarykey','fake_migrate','format','redefine',
195 'singular','plural','trigger_name','sequence_name',
196 'common_filter','polymodel','table_class','on_define',))
197
198 SELECT_ARGS = set(
199 ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
200 'distinct', 'having', 'join','for_update', 'processor','cacheable'))
201
202 ogetattr = object.__getattribute__
203 osetattr = object.__setattr__
204 exists = os.path.exists
205 pjoin = os.path.join
206
207
208
209
210 try:
211 from utils import web2py_uuid
212 except (ImportError, SystemError):
213 import uuid
215
216 try:
217 import portalocker
218 have_portalocker = True
219 except ImportError:
220 have_portalocker = False
221
222 try:
223 import serializers
224 have_serializers = True
225 except ImportError:
226 have_serializers = False
227
228 try:
229 import validators
230 have_validators = True
231 except (ImportError, SyntaxError):
232 have_validators = False
233
234 LOGGER = logging.getLogger("web2py.dal")
235 DEFAULT = lambda:0
236
237 GLOBAL_LOCKER = threading.RLock()
238 THREAD_LOCAL = threading.local()
239
240
241
242
243 REGEX_TYPE = re.compile('^([\w\_\:]+)')
244 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
245 REGEX_W = re.compile('^\w+$')
246 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$')
247 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)\.(?P<name>\w+)\.\w+$')
248 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
249 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
250 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
251 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
252 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
253 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
254 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
255 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
256 REGEX_QUOTES = re.compile("'[^']*'")
257 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
258 REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
259 REGEX_NOPASSWD = re.compile('(?<=\:)([^:@/]+)(?=@.+)')
260
261
262
263 DRIVERS = []
264
265 try:
266 from new import classobj
267 from google.appengine.ext import db as gae
268 from google.appengine.api import namespace_manager, rdbms
269 from google.appengine.api.datastore_types import Key
270 from google.appengine.ext.db.polymodel import PolyModel
271 DRIVERS.append('google')
272 except ImportError:
273 pass
274
275 if not 'google' in DRIVERS:
276
277 try:
278 from pysqlite2 import dbapi2 as sqlite2
279 DRIVERS.append('SQLite(sqlite2)')
280 except ImportError:
281 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
282
283 try:
284 from sqlite3 import dbapi2 as sqlite3
285 DRIVERS.append('SQLite(sqlite3)')
286 except ImportError:
287 LOGGER.debug('no SQLite drivers sqlite3')
288
289 try:
290
291 try:
292 import contrib.pymysql as pymysql
293
294
295 pymysql.ESCAPE_REGEX = re.compile("'")
296 pymysql.ESCAPE_MAP = {"'": "''"}
297
298 except ImportError:
299 import pymysql
300 DRIVERS.append('MySQL(pymysql)')
301 except ImportError:
302 LOGGER.debug('no MySQL driver pymysql')
303
304 try:
305 import MySQLdb
306 DRIVERS.append('MySQL(MySQLdb)')
307 except ImportError:
308 LOGGER.debug('no MySQL driver MySQLDB')
309
310
311 try:
312 import psycopg2
313 from psycopg2.extensions import adapt as psycopg2_adapt
314 DRIVERS.append('PostgreSQL(psycopg2)')
315 except ImportError:
316 LOGGER.debug('no PostgreSQL driver psycopg2')
317
318 try:
319
320 try:
321 import contrib.pg8000.dbapi as pg8000
322 except ImportError:
323 import pg8000.dbapi as pg8000
324 DRIVERS.append('PostgreSQL(pg8000)')
325 except ImportError:
326 LOGGER.debug('no PostgreSQL driver pg8000')
327
328 try:
329 import cx_Oracle
330 DRIVERS.append('Oracle(cx_Oracle)')
331 except ImportError:
332 LOGGER.debug('no Oracle driver cx_Oracle')
333
334 try:
335 import pyodbc
336 DRIVERS.append('MSSQL(pyodbc)')
337 DRIVERS.append('DB2(pyodbc)')
338 DRIVERS.append('Teradata(pyodbc)')
339 except ImportError:
340 LOGGER.debug('no MSSQL/DB2/Teradata driver pyodbc')
341
342 try:
343 import Sybase
344 DRIVERS.append('Sybase(Sybase)')
345 except ImportError:
346 LOGGER.debug('no Sybase driver')
347
348 try:
349 import kinterbasdb
350 DRIVERS.append('Interbase(kinterbasdb)')
351 DRIVERS.append('Firebird(kinterbasdb)')
352 except ImportError:
353 LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
354
355 try:
356 import fdb
357 DRIVERS.append('Firbird(fdb)')
358 except ImportError:
359 LOGGER.debug('no Firebird driver fdb')
360
361 try:
362 import firebirdsql
363 DRIVERS.append('Firebird(firebirdsql)')
364 except ImportError:
365 LOGGER.debug('no Firebird driver firebirdsql')
366
367 try:
368 import informixdb
369 DRIVERS.append('Informix(informixdb)')
370 LOGGER.warning('Informix support is experimental')
371 except ImportError:
372 LOGGER.debug('no Informix driver informixdb')
373
374 try:
375 import sapdb
376 DRIVERS.append('SQL(sapdb)')
377 LOGGER.warning('SAPDB support is experimental')
378 except ImportError:
379 LOGGER.debug('no SAP driver sapdb')
380
381 try:
382 import cubriddb
383 DRIVERS.append('Cubrid(cubriddb)')
384 LOGGER.warning('Cubrid support is experimental')
385 except ImportError:
386 LOGGER.debug('no Cubrid driver cubriddb')
387
388 try:
389 from com.ziclix.python.sql import zxJDBC
390 import java.sql
391
392 from org.sqlite import JDBC
393 zxJDBC_sqlite = java.sql.DriverManager
394 DRIVERS.append('PostgreSQL(zxJDBC)')
395 DRIVERS.append('SQLite(zxJDBC)')
396 LOGGER.warning('zxJDBC support is experimental')
397 is_jdbc = True
398 except ImportError:
399 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
400 is_jdbc = False
401
402 try:
403 import ingresdbi
404 DRIVERS.append('Ingres(ingresdbi)')
405 except ImportError:
406 LOGGER.debug('no Ingres driver ingresdbi')
407
408
409 try:
410 import couchdb
411 DRIVERS.append('CouchDB(couchdb)')
412 except ImportError:
413 LOGGER.debug('no Couchdb driver couchdb')
414
415 try:
416 import pymongo
417 DRIVERS.append('MongoDB(pymongo)')
418 except:
419 LOGGER.debug('no MongoDB driver pymongo')
420
421 try:
422 import imaplib
423 DRIVERS.append('IMAP(imaplib)')
424 except:
425 LOGGER.debug('no IMAP driver imaplib')
426
427 PLURALIZE_RULES = [
428 (re.compile('child$'), re.compile('child$'), 'children'),
429 (re.compile('oot$'), re.compile('oot$'), 'eet'),
430 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
431 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
432 (re.compile('sis$'), re.compile('sis$'), 'ses'),
433 (re.compile('man$'), re.compile('man$'), 'men'),
434 (re.compile('ife$'), re.compile('ife$'), 'ives'),
435 (re.compile('eau$'), re.compile('eau$'), 'eaux'),
436 (re.compile('lf$'), re.compile('lf$'), 'lves'),
437 (re.compile('[sxz]$'), re.compile('$'), 'es'),
438 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
439 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
440 (re.compile('$'), re.compile('$'), 's'),
441 ]
448
451
454
457
459
462
463 if 'google' in DRIVERS:
464
465 is_jdbc = False
468 """
469 GAE decimal implementation
470 """
471 data_type = decimal.Decimal
472
473 - def __init__(self, precision, scale, **kwargs):
474 super(GAEDecimalProperty, self).__init__(self, **kwargs)
475 d = '1.'
476 for x in range(scale):
477 d += '0'
478 self.round = decimal.Decimal(d)
479
487
489 if value is None or value == '':
490 return None
491 else:
492 return decimal.Decimal(value).quantize(self.round)
493
495 value = super(GAEDecimalProperty, self).validate(value)
496 if value is None or isinstance(value, decimal.Decimal):
497 return value
498 elif isinstance(value, basestring):
499 return decimal.Decimal(value)
500 raise gae.BadValueError("Property %s must be a Decimal or string."\
501 % self.name)
502
508
509 POOLS = {}
510 check_active_connection = True
511
512 @staticmethod
515
516
517
518 - def close(self,action='commit',really=True):
535
536 @staticmethod
538 """ to close cleanly databases in a multithreaded environment """
539 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
540 for db_uid, db_group in dbs:
541 for db in db_group:
542 if hasattr(db,'_adapter'):
543 db._adapter.close(action)
544 getattr(THREAD_LOCAL,'db_instances',{}).clear()
545 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
546 if callable(action):
547 action(None)
548 return
549
551 """ this actually does not make the folder. it has to be there """
552 self.folder = getattr(THREAD_LOCAL,'folder','')
553
554
555 if False and self.folder and not exists(self.folder):
556 os.mkdir(self.folder)
557
559 """ this it is suppoed to be overloaded by adtapters"""
560 pass
561
563 """
564 this function defines: self.connection and self.cursor
565 (iff cursor is True)
566 if self.pool_size>0 it will try pull the connection from the pool
567 if the connection is not active (closed by db server) it will loop
568 if not self.pool_size or no active connections in pool makes a new one
569 """
570 if getattr(self,'connection',None) != None:
571 return
572 if f is None:
573 f = self.connector
574
575 if not self.pool_size:
576 self.connection = f()
577 self.cursor = cursor and self.connection.cursor()
578 else:
579 uri = self.uri
580 POOLS = ConnectionPool.POOLS
581 while True:
582 GLOBAL_LOCKER.acquire()
583 if not uri in POOLS:
584 POOLS[uri] = []
585 if POOLS[uri]:
586 self.connection = POOLS[uri].pop()
587 GLOBAL_LOCKER.release()
588 self.cursor = cursor and self.connection.cursor()
589 try:
590 if self.cursor and self.check_active_connection:
591 self.execute('SELECT 1;')
592 break
593 except:
594 pass
595 else:
596 GLOBAL_LOCKER.release()
597 self.connection = f()
598 self.cursor = cursor and self.connection.cursor()
599 break
600 self.after_connection()
601
608 driver = None
609 driver_name = None
610 drivers = ()
611 connection = None
612 maxcharlength = MAXCHARLENGTH
613 commit_on_alter_table = False
614 support_distributed_transaction = False
615 uploads_in_blob = False
616 can_select_for_update = True
617
618 TRUE = 'T'
619 FALSE = 'F'
620 types = {
621 'boolean': 'CHAR(1)',
622 'string': 'CHAR(%(length)s)',
623 'text': 'TEXT',
624 'password': 'CHAR(%(length)s)',
625 'blob': 'BLOB',
626 'upload': 'CHAR(%(length)s)',
627 'integer': 'INTEGER',
628 'bigint': 'INTEGER',
629 'float':'DOUBLE',
630 'double': 'DOUBLE',
631 'decimal': 'DOUBLE',
632 'date': 'DATE',
633 'time': 'TIME',
634 'datetime': 'TIMESTAMP',
635 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
636 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
637 'list:integer': 'TEXT',
638 'list:string': 'TEXT',
639 'list:reference': 'TEXT',
640
641 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT',
642 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
643 }
644
646 return table._id != None
647
649 return "'%s'" % obj.replace("'", "''")
650
652 if isinstance(obj,(int,float)):
653 return str(obj)
654 return self.adapt(str(obj))
655
657 return self.driver.IntegrityError
658
660 return self.driver.OperationalError
661
663 """
664 to be used ONLY for files that on GAE may not be on filesystem
665 """
666 return exists(filename)
667
668 - def file_open(self, filename, mode='rb', lock=True):
669 """
670 to be used ONLY for files that on GAE may not be on filesystem
671 """
672 if have_portalocker and lock:
673 fileobj = portalocker.LockedFile(filename,mode)
674 else:
675 fileobj = open(filename,mode)
676 return fileobj
677
679 """
680 to be used ONLY for files that on GAE may not be on filesystem
681 """
682 if fileobj:
683 fileobj.close()
684
687
689 if getattr(self,'driver',None) != None:
690 return
691 drivers_available = [driver for driver in self.drivers
692 if driver in globals()]
693 if uri:
694 items = uri.split('://',1)[0].split(':')
695 request_driver = items[1] if len(items)>1 else None
696 else:
697 request_driver = None
698 request_driver = request_driver or adapter_args.get('driver')
699 if request_driver:
700 if request_driver in drivers_available:
701 self.driver_name = request_driver
702 self.driver = globals().get(request_driver)
703 else:
704 raise RuntimeError("driver %s not available" % request_driver)
705 elif drivers_available:
706 self.driver_name = drivers_available[0]
707 self.driver = globals().get(self.driver_name)
708 else:
709 raise RuntimeError("no driver available %s" % str(self.drivers))
710
711
712 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
713 credential_decoder=IDENTITY, driver_args={},
714 adapter_args={},do_connect=True):
715 self.db = db
716 self.dbengine = "None"
717 self.uri = uri
718 self.pool_size = pool_size
719 self.folder = folder
720 self.db_codec = db_codec
721 class Dummy(object):
722 lastrowid = 1
723 def __getattr__(self, value):
724 return lambda *a, **b: []
725 self.connection = Dummy()
726 self.cursor = Dummy()
727
729 return '%s_sequence' % tablename
730
732 return '%s_sequence' % tablename
733
736
737 - def create_table(self, table,
738 migrate=True,
739 fake_migrate=False,
740 polymodel=None):
741 db = table._db
742 fields = []
743
744 postcreation_fields = []
745 sql_fields = {}
746 sql_fields_aux = {}
747 TFK = {}
748 tablename = table._tablename
749 sortable = 0
750 types = self.types
751 for field in table:
752 sortable += 1
753 field_name = field.name
754 field_type = field.type
755 if isinstance(field_type,SQLCustomType):
756 ftype = field_type.native or field_type.type
757 elif field_type.startswith('reference'):
758 referenced = field_type[10:].strip()
759 if referenced == '.':
760 referenced = tablename
761 constraint_name = self.constraint_name(tablename, field_name)
762 if not '.' in referenced \
763 and referenced != tablename \
764 and hasattr(table,'_primarykey'):
765 ftype = types['integer']
766 else:
767 if hasattr(table,'_primarykey'):
768 rtablename,rfieldname = referenced.split('.')
769 rtable = db[rtablename]
770 rfield = rtable[rfieldname]
771
772 if rfieldname in hasattr(rtable,'_primarykey') or \
773 rfield.unique:
774 ftype = types[rfield.type[:9]] % \
775 dict(length=rfield.length)
776
777 if not rfield.unique and len(rtable._primarykey)>1:
778
779 if rtablename not in TFK:
780 TFK[rtablename] = {}
781 TFK[rtablename][rfieldname] = field_name
782 else:
783 ftype = ftype + \
784 types['reference FK'] % dict(
785 constraint_name = constraint_name,
786 foreign_key = '%s (%s)' % (rtablename,
787 rfieldname),
788 on_delete_action=field.ondelete)
789 else:
790
791 if referenced in db:
792 id_fieldname = db[referenced]._id.name
793 elif referenced == tablename:
794 id_fieldname = table._id.name
795 else:
796 id_fieldname = 'id'
797 ftype = types[field_type[:9]] % dict(
798 index_name = field_name+'__idx',
799 field_name = field_name,
800 constraint_name = constraint_name,
801 foreign_key = '%s (%s)' % (referenced,
802 id_fieldname),
803 on_delete_action=field.ondelete)
804 elif field_type.startswith('list:reference'):
805 ftype = types[field_type[:14]]
806 elif field_type.startswith('decimal'):
807 precision, scale = map(int,field_type[8:-1].split(','))
808 ftype = types[field_type[:7]] % \
809 dict(precision=precision,scale=scale)
810 elif field_type.startswith('geo'):
811 if not hasattr(self,'srid'):
812 raise RuntimeError('Adapter does not support geometry')
813 srid = self.srid
814 geotype, parms = field_type[:-1].split('(')
815 if not geotype in types:
816 raise SyntaxError(
817 'Field: unknown field type: %s for %s' \
818 % (field_type, field_name))
819 ftype = types[geotype]
820 if self.dbengine == 'postgres' and geotype == 'geometry':
821
822 dimension = 2
823 parms = parms.split(',')
824 if len(parms) == 3:
825 schema, srid, dimension = parms
826 elif len(parms) == 2:
827 schema, srid = parms
828 else:
829 schema = parms[0]
830 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
831 ftype = ftype % dict(schema=schema,
832 tablename=tablename,
833 fieldname=field_name, srid=srid,
834 dimension=dimension)
835 postcreation_fields.append(ftype)
836 elif not field_type in types:
837 raise SyntaxError('Field: unknown field type: %s for %s' % \
838 (field_type, field_name))
839 else:
840 ftype = types[field_type]\
841 % dict(length=field.length)
842 if not field_type.startswith('id') and \
843 not field_type.startswith('reference'):
844 if field.notnull:
845 ftype += ' NOT NULL'
846 else:
847 ftype += self.ALLOW_NULL()
848 if field.unique:
849 ftype += ' UNIQUE'
850 if field.custom_qualifier:
851 ftype += ' %s' % field.custom_qualifier
852
853
854 sql_fields[field_name] = dict(
855 length=field.length,
856 unique=field.unique,
857 notnull=field.notnull,
858 sortable=sortable,
859 type=str(field_type),
860 sql=ftype)
861
862 if field.notnull and not field.default is None:
863
864
865
866
867
868
869 not_null = self.NOT_NULL(field.default, field_type)
870 ftype = ftype.replace('NOT NULL', not_null)
871 sql_fields_aux[field_name] = dict(sql=ftype)
872
873
874 if not (self.dbengine == 'postgres' and \
875 field_type.startswith('geom')):
876 fields.append('%s %s' % (field_name, ftype))
877 other = ';'
878
879
880 if self.dbengine == 'mysql':
881 if not hasattr(table, "_primarykey"):
882 fields.append('PRIMARY KEY(%s)' % table._id.name)
883 other = ' ENGINE=InnoDB CHARACTER SET utf8;'
884
885 fields = ',\n '.join(fields)
886 for rtablename in TFK:
887 rfields = TFK[rtablename]
888 pkeys = db[rtablename]._primarykey
889 fkeys = [ rfields[k] for k in pkeys ]
890 fields = fields + ',\n ' + \
891 types['reference TFK'] % dict(
892 table_name = tablename,
893 field_name=', '.join(fkeys),
894 foreign_table = rtablename,
895 foreign_key = ', '.join(pkeys),
896 on_delete_action = field.ondelete)
897
898 if hasattr(table,'_primarykey'):
899 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \
900 (tablename, fields,
901 self.PRIMARY_KEY(', '.join(table._primarykey)),other)
902 else:
903 query = "CREATE TABLE %s(\n %s\n)%s" % \
904 (tablename, fields, other)
905
906 if self.uri.startswith('sqlite:///') \
907 or self.uri.startswith('spatialite:///'):
908 path_encoding = sys.getfilesystemencoding() \
909 or locale.getdefaultlocale()[1] or 'utf8'
910 dbpath = self.uri[9:self.uri.rfind('/')]\
911 .decode('utf8').encode(path_encoding)
912 else:
913 dbpath = self.folder
914
915 if not migrate:
916 return query
917 elif self.uri.startswith('sqlite:memory')\
918 or self.uri.startswith('spatialite:memory'):
919 table._dbt = None
920 elif isinstance(migrate, str):
921 table._dbt = pjoin(dbpath, migrate)
922 else:
923 table._dbt = pjoin(
924 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
925
926 if table._dbt:
927 table._loggername = pjoin(dbpath, 'sql.log')
928 logfile = self.file_open(table._loggername, 'a')
929 else:
930 logfile = None
931 if not table._dbt or not self.file_exists(table._dbt):
932 if table._dbt:
933 logfile.write('timestamp: %s\n'
934 % datetime.datetime.today().isoformat())
935 logfile.write(query + '\n')
936 if not fake_migrate:
937 self.create_sequence_and_triggers(query,table)
938 table._db.commit()
939
940
941 for query in postcreation_fields:
942 self.execute(query)
943 table._db.commit()
944 if table._dbt:
945 tfile = self.file_open(table._dbt, 'w')
946 pickle.dump(sql_fields, tfile)
947 self.file_close(tfile)
948 if fake_migrate:
949 logfile.write('faked!\n')
950 else:
951 logfile.write('success!\n')
952 else:
953 tfile = self.file_open(table._dbt, 'r')
954 try:
955 sql_fields_old = pickle.load(tfile)
956 except EOFError:
957 self.file_close(tfile)
958 self.file_close(logfile)
959 raise RuntimeError('File %s appears corrupted' % table._dbt)
960 self.file_close(tfile)
961 if sql_fields != sql_fields_old:
962 self.migrate_table(table,
963 sql_fields, sql_fields_old,
964 sql_fields_aux, logfile,
965 fake_migrate=fake_migrate)
966 self.file_close(logfile)
967 return query
968
969 - def migrate_table(
970 self,
971 table,
972 sql_fields,
973 sql_fields_old,
974 sql_fields_aux,
975 logfile,
976 fake_migrate=False,
977 ):
978 db = table._db
979 db._migrated.append(table._tablename)
980 tablename = table._tablename
981 def fix(item):
982 k,v=item
983 if not isinstance(v,dict):
984 v=dict(type='unkown',sql=v)
985 return k.lower(),v
986
987
988 sql_fields = dict(map(fix,sql_fields.iteritems()))
989 sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
990 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
991 if db._debug:
992 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields))
993
994 keys = sql_fields.keys()
995 for key in sql_fields_old:
996 if not key in keys:
997 keys.append(key)
998 new_add = self.concat_add(tablename)
999
1000 metadata_change = False
1001 sql_fields_current = copy.copy(sql_fields_old)
1002 for key in keys:
1003 query = None
1004 if not key in sql_fields_old:
1005 sql_fields_current[key] = sql_fields[key]
1006 if self.dbengine in ('postgres',) and \
1007 sql_fields[key]['type'].startswith('geometry'):
1008
1009 query = [ sql_fields[key]['sql'] ]
1010 else:
1011 query = ['ALTER TABLE %s ADD %s %s;' % \
1012 (tablename, key,
1013 sql_fields_aux[key]['sql'].replace(', ', new_add))]
1014 metadata_change = True
1015 elif self.dbengine in ('sqlite', 'spatialite'):
1016 if key in sql_fields:
1017 sql_fields_current[key] = sql_fields[key]
1018 metadata_change = True
1019 elif not key in sql_fields:
1020 del sql_fields_current[key]
1021 ftype = sql_fields_old[key]['type']
1022 if self.dbengine in ('postgres',) \
1023 and ftype.startswith('geometry'):
1024 geotype, parms = ftype[:-1].split('(')
1025 schema = parms.split(',')[0]
1026 query = [ "SELECT DropGeometryColumn ('%(schema)s', '%(table)s', '%(field)s');" % dict(schema=schema, table=tablename, field=key,) ]
1027 elif not self.dbengine in ('firebird',):
1028 query = ['ALTER TABLE %s DROP COLUMN %s;'
1029 % (tablename, key)]
1030 else:
1031 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
1032 metadata_change = True
1033 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
1034 and not (key in table.fields and
1035 isinstance(table[key].type, SQLCustomType)) \
1036 and not sql_fields[key]['type'].startswith('reference')\
1037 and not sql_fields[key]['type'].startswith('double')\
1038 and not sql_fields[key]['type'].startswith('id'):
1039 sql_fields_current[key] = sql_fields[key]
1040 t = tablename
1041 tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
1042 if not self.dbengine in ('firebird',):
1043 query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
1044 'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
1045 'ALTER TABLE %s DROP COLUMN %s;' % (t, key),
1046 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1047 'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
1048 'ALTER TABLE %s DROP COLUMN %s__tmp;' % (t, key)]
1049 else:
1050 query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
1051 'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
1052 'ALTER TABLE %s DROP %s;' % (t, key),
1053 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1054 'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
1055 'ALTER TABLE %s DROP %s__tmp;' % (t, key)]
1056 metadata_change = True
1057 elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
1058 sql_fields_current[key] = sql_fields[key]
1059 metadata_change = True
1060
1061 if query:
1062 logfile.write('timestamp: %s\n'
1063 % datetime.datetime.today().isoformat())
1064 db['_lastsql'] = '\n'.join(query)
1065 for sub_query in query:
1066 logfile.write(sub_query + '\n')
1067 if not fake_migrate:
1068 self.execute(sub_query)
1069
1070
1071
1072 if db._adapter.commit_on_alter_table:
1073 db.commit()
1074 tfile = self.file_open(table._dbt, 'w')
1075 pickle.dump(sql_fields_current, tfile)
1076 self.file_close(tfile)
1077 logfile.write('success!\n')
1078 else:
1079 logfile.write('faked!\n')
1080 elif metadata_change:
1081 tfile = self.file_open(table._dbt, 'w')
1082 pickle.dump(sql_fields_current, tfile)
1083 self.file_close(tfile)
1084
1085 if metadata_change and \
1086 not (query and self.dbengine in ('mysql','oracle','firebird')):
1087 db.commit()
1088 tfile = self.file_open(table._dbt, 'w')
1089 pickle.dump(sql_fields_current, tfile)
1090 self.file_close(tfile)
1091
1092 - def LOWER(self, first):
1094
1095 - def UPPER(self, first):
1097
1098 - def COUNT(self, first, distinct=None):
1099 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \
1100 % self.expand(first)
1101
1103 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1104
1105 - def EPOCH(self, first):
1107
1110
1113
1116
1119
1120 - def NOT_NULL(self, default, field_type):
1121 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1122
1124 expressions = [self.expand(first)]+[self.expand(e) for e in second]
1125 return 'COALESCE(%s)' % ','.join(expressions)
1126
1129
1130 - def RAW(self, first):
1132
1135
1137 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1138
1140 return 'PRIMARY KEY(%s)' % key
1141
1142 - def _drop(self, table, mode):
1143 return ['DROP TABLE %s;' % table]
1144
1145 - def drop(self, table, mode=''):
1161
1162 - def _insert(self, table, fields):
1166
1167 - def insert(self, table, fields):
1168 query = self._insert(table,fields)
1169 try:
1170 self.execute(query)
1171 except Exception:
1172 e = sys.exc_info()[1]
1173 if isinstance(e,self.integrity_error_class()):
1174 return None
1175 raise e
1176 if hasattr(table,'_primarykey'):
1177 return dict([(k[0].name, k[1]) for k in fields \
1178 if k[0].name in table._primarykey])
1179 id = self.lastrowid(table)
1180 if not isinstance(id,int):
1181 return id
1182 rid = Reference(id)
1183 (rid._table, rid._record) = (table, None)
1184 return rid
1185
1187 return [self.insert(table,item) for item in items]
1188
1189 - def NOT(self, first):
1191
1192 - def AND(self, first, second):
1194
1195 - def OR(self, first, second):
1197
1198 - def BELONGS(self, first, second):
1199 if isinstance(second, str):
1200 return '(%s IN (%s))' % (self.expand(first), second[:-1])
1201 elif not second:
1202 return '(1=0)'
1203 items = ','.join(self.expand(item, first.type) for item in second)
1204 return '(%s IN (%s))' % (self.expand(first), items)
1205
1206 - def REGEXP(self, first, second):
1207 "regular expression operator"
1208 raise NotImplementedError
1209
1210 - def LIKE(self, first, second):
1211 "case sensitive like operator"
1212 raise NotImplementedError
1213
1214 - def ILIKE(self, first, second):
1215 "case in-sensitive like operator"
1216 return '(%s LIKE %s)' % (self.expand(first),
1217 self.expand(second, 'string'))
1218
1220 return '(%s LIKE %s)' % (self.expand(first),
1221 self.expand(second+'%', 'string'))
1222
1224 return '(%s LIKE %s)' % (self.expand(first),
1225 self.expand('%'+second, 'string'))
1226
1233
1234 - def EQ(self, first, second=None):
1239
1240 - def NE(self, first, second=None):
1245
1246 - def LT(self,first,second=None):
1247 if second is None:
1248 raise RuntimeError("Cannot compare %s < None" % first)
1249 return '(%s < %s)' % (self.expand(first),
1250 self.expand(second,first.type))
1251
1252 - def LE(self,first,second=None):
1253 if second is None:
1254 raise RuntimeError("Cannot compare %s <= None" % first)
1255 return '(%s <= %s)' % (self.expand(first),
1256 self.expand(second,first.type))
1257
1258 - def GT(self,first,second=None):
1259 if second is None:
1260 raise RuntimeError("Cannot compare %s > None" % first)
1261 return '(%s > %s)' % (self.expand(first),
1262 self.expand(second,first.type))
1263
1264 - def GE(self,first,second=None):
1265 if second is None:
1266 raise RuntimeError("Cannot compare %s >= None" % first)
1267 return '(%s >= %s)' % (self.expand(first),
1268 self.expand(second,first.type))
1269
1270 - def ADD(self, first, second):
1273
1274 - def SUB(self, first, second):
1277
1278 - def MUL(self, first, second):
1281
1282 - def DIV(self, first, second):
1285
1286 - def MOD(self, first, second):
1289
1290 - def AS(self, first, second):
1292
1293 - def ON(self, first, second):
1295
1298
1299 - def COMMA(self, first, second):
1301
1302 - def expand(self, expression, field_type=None):
1303 if isinstance(expression, Field):
1304 return '%s.%s' % (expression.tablename, expression.name)
1305 elif isinstance(expression, (Expression, Query)):
1306 first = expression.first
1307 second = expression.second
1308 op = expression.op
1309 if not second is None:
1310 return op(first, second)
1311 elif not first is None:
1312 return op(first)
1313 elif isinstance(op, str):
1314 if op.endswith(';'):
1315 op=op[:-1]
1316 return '(%s)' % op
1317 else:
1318 return op()
1319 elif field_type:
1320 return str(self.represent(expression,field_type))
1321 elif isinstance(expression,(list,tuple)):
1322 return ','.join(self.represent(item,field_type) \
1323 for item in expression)
1324 elif isinstance(expression, bool):
1325 return '1' if expression else '0'
1326 else:
1327 return str(expression)
1328
1329 - def alias(self, table, alias):
1330 """
1331 Given a table object, makes a new table object
1332 with alias name.
1333 """
1334 other = copy.copy(table)
1335 other['_ot'] = other._tablename
1336 other['ALL'] = SQLALL(other)
1337 other['_tablename'] = alias
1338 for fieldname in other.fields:
1339 other[fieldname] = copy.copy(other[fieldname])
1340 other[fieldname]._tablename = alias
1341 other[fieldname].tablename = alias
1342 other[fieldname].table = other
1343 table._db[alias] = other
1344 return other
1345
1347 tablename = table._tablename
1348 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1349
1351
1352 if table._dbt:
1353 logfile = self.file_open(table._loggername, 'a')
1354 else:
1355 class Logfile(object):
1356 def write(self, value):
1357 pass
1358 def close(self):
1359 pass
1360 logfile = Logfile()
1361
1362 try:
1363 queries = table._db._adapter._truncate(table, mode)
1364 for query in queries:
1365 logfile.write(query + '\n')
1366 self.execute(query)
1367 table._db.commit()
1368 logfile.write('success!\n')
1369 finally:
1370 logfile.close()
1371
1372 - def _update(self, tablename, query, fields):
1373 if query:
1374 if use_common_filters(query):
1375 query = self.common_filter(query, [tablename])
1376 sql_w = ' WHERE ' + self.expand(query)
1377 else:
1378 sql_w = ''
1379 sql_v = ','.join(['%s=%s' % (field.name,
1380 self.expand(value, field.type)) \
1381 for (field, value) in fields])
1382 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1383
1384 - def update(self, tablename, query, fields):
1385 sql = self._update(tablename, query, fields)
1386 self.execute(sql)
1387 try:
1388 return self.cursor.rowcount
1389 except:
1390 return None
1391
1392 - def _delete(self, tablename, query):
1393 if query:
1394 if use_common_filters(query):
1395 query = self.common_filter(query, [tablename])
1396 sql_w = ' WHERE ' + self.expand(query)
1397 else:
1398 sql_w = ''
1399 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1400
1401 - def delete(self, tablename, query):
1402 sql = self._delete(tablename, query)
1403
1404 db = self.db
1405 table = db[tablename]
1406 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by:
1407 deleted = [x[table._id.name] for x in db(query).select(table._id)]
1408
1409 self.execute(sql)
1410 try:
1411 counter = self.cursor.rowcount
1412 except:
1413 counter = None
1414
1415 if self.dbengine in ('sqlite', 'spatialite') and counter:
1416 for field in table._referenced_by:
1417 if field.type=='reference '+table._tablename \
1418 and field.ondelete=='CASCADE':
1419 db(field.belongs(deleted)).delete()
1420
1421 return counter
1422
1424 tablenames = self.tables(query)
1425 if len(tablenames)==1:
1426 return tablenames[0]
1427 elif len(tablenames)<1:
1428 raise RuntimeError("No table selected")
1429 else:
1430 raise RuntimeError("Too many tables selected")
1431
1433 db = self.db
1434 new_fields = []
1435 append = new_fields.append
1436 for item in fields:
1437 if isinstance(item,SQLALL):
1438 new_fields += item._table
1439 elif isinstance(item,str):
1440 if REGEX_TABLE_DOT_FIELD.match(item):
1441 tablename,fieldname = item.split('.')
1442 append(db[tablename][fieldname])
1443 else:
1444 append(Expression(db,lambda:item))
1445 else:
1446 append(item)
1447
1448 if not new_fields:
1449 for table in tablenames:
1450 for field in db[table]:
1451 append(field)
1452 return new_fields
1453
1454 - def _select(self, query, fields, attributes):
1455 tables = self.tables
1456 for key in set(attributes.keys())-SELECT_ARGS:
1457 raise SyntaxError('invalid select attribute: %s' % key)
1458 args_get = attributes.get
1459 tablenames = tables(query)
1460 for field in fields:
1461 if isinstance(field, basestring) \
1462 and REGEX_TABLE_DOT_FIELD.match(field):
1463 tn,fn = field.split('.')
1464 field = self.db[tn][fn]
1465 for tablename in tables(field):
1466 if not tablename in tablenames:
1467 tablenames.append(tablename)
1468
1469 if use_common_filters(query):
1470 query = self.common_filter(query,tablenames)
1471
1472 if len(tablenames) < 1:
1473 raise SyntaxError('Set: no tables selected')
1474 self._colnames = map(self.expand, fields)
1475 def geoexpand(field):
1476 if isinstance(field.type,str) and field.type.startswith('geometry'):
1477 field = field.st_astext()
1478 return self.expand(field)
1479 sql_f = ', '.join(map(geoexpand, fields))
1480 if query:
1481 sql_w = ' WHERE ' + self.expand(query)
1482 else:
1483 sql_w = ''
1484 sql_o = ''
1485 sql_s = ''
1486 left = args_get('left', False)
1487 inner_join = args_get('join', False)
1488 distinct = args_get('distinct', False)
1489 groupby = args_get('groupby', False)
1490 orderby = args_get('orderby', False)
1491 having = args_get('having', False)
1492 limitby = args_get('limitby', False)
1493 for_update = args_get('for_update', False)
1494 if self.can_select_for_update is False and for_update is True:
1495 raise SyntaxError('invalid select attribute: for_update')
1496 if distinct is True:
1497 sql_s += 'DISTINCT'
1498 elif distinct:
1499 sql_s += 'DISTINCT ON (%s)' % distinct
1500 if inner_join:
1501 icommand = self.JOIN()
1502 if not isinstance(inner_join, (tuple, list)):
1503 inner_join = [inner_join]
1504 ijoint = [t._tablename for t in inner_join
1505 if not isinstance(t,Expression)]
1506 ijoinon = [t for t in inner_join if isinstance(t, Expression)]
1507 itables_to_merge={}
1508 [itables_to_merge.update(
1509 dict.fromkeys(tables(t))) for t in ijoinon]
1510 ijoinont = [t.first._tablename for t in ijoinon]
1511 [itables_to_merge.pop(t) for t in ijoinont
1512 if t in itables_to_merge]
1513 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys()
1514 iexcluded = [t for t in tablenames
1515 if not t in iimportant_tablenames]
1516 if left:
1517 join = attributes['left']
1518 command = self.LEFT_JOIN()
1519 if not isinstance(join, (tuple, list)):
1520 join = [join]
1521 joint = [t._tablename for t in join
1522 if not isinstance(t, Expression)]
1523 joinon = [t for t in join if isinstance(t, Expression)]
1524
1525 tables_to_merge={}
1526 [tables_to_merge.update(
1527 dict.fromkeys(tables(t))) for t in joinon]
1528 joinont = [t.first._tablename for t in joinon]
1529 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
1530 important_tablenames = joint + joinont + tables_to_merge.keys()
1531 excluded = [t for t in tablenames
1532 if not t in important_tablenames ]
1533 def alias(t):
1534 return str(self.db[t])
1535 if inner_join and not left:
1536 sql_t = ', '.join([alias(t) for t in iexcluded + \
1537 itables_to_merge.keys()])
1538 for t in ijoinon:
1539 sql_t += ' %s %s' % (icommand, str(t))
1540 elif not inner_join and left:
1541 sql_t = ', '.join([alias(t) for t in excluded + \
1542 tables_to_merge.keys()])
1543 if joint:
1544 sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
1545 for t in joinon:
1546 sql_t += ' %s %s' % (command, str(t))
1547 elif inner_join and left:
1548 all_tables_in_query = set(important_tablenames + \
1549 iimportant_tablenames + \
1550 tablenames)
1551 tables_in_joinon = set(joinont + ijoinont)
1552 tables_not_in_joinon = \
1553 all_tables_in_query.difference(tables_in_joinon)
1554 sql_t = ','.join([alias(t) for t in tables_not_in_joinon])
1555 for t in ijoinon:
1556 sql_t += ' %s %s' % (icommand, str(t))
1557 if joint:
1558 sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
1559 for t in joinon:
1560 sql_t += ' %s %s' % (command, str(t))
1561 else:
1562 sql_t = ', '.join(alias(t) for t in tablenames)
1563 if groupby:
1564 if isinstance(groupby, (list, tuple)):
1565 groupby = xorify(groupby)
1566 sql_o += ' GROUP BY %s' % self.expand(groupby)
1567 if having:
1568 sql_o += ' HAVING %s' % attributes['having']
1569 if orderby:
1570 if isinstance(orderby, (list, tuple)):
1571 orderby = xorify(orderby)
1572 if str(orderby) == '<random>':
1573 sql_o += ' ORDER BY %s' % self.RANDOM()
1574 else:
1575 sql_o += ' ORDER BY %s' % self.expand(orderby)
1576 if limitby:
1577 if not orderby and tablenames:
1578 sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in (hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey or [self.db[t]._id.name])])
1579
1580 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
1581 if for_update and self.can_select_for_update is True:
1582 sql = sql.rstrip(';') + ' FOR UPDATE;'
1583 return sql
1584
1585 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1586 if limitby:
1587 (lmin, lmax) = limitby
1588 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
1589 return 'SELECT %s %s FROM %s%s%s;' % \
1590 (sql_s, sql_f, sql_t, sql_w, sql_o)
1591
1593 return self.cursor.fetchall()
1594
1596 args_get = attributes.get
1597 cache = args_get('cache',None)
1598 if not cache:
1599 self.execute(sql)
1600 rows = self._fetchall()
1601 else:
1602 (cache_model, time_expire) = cache
1603 key = self.uri + '/' + sql + '/rows'
1604 if len(key)>200: key = hashlib_md5(key).hexdigest()
1605 def _select_aux2():
1606 self.execute(sql)
1607 return self._fetchall()
1608 rows = cache_model(key,_select_aux2,time_expire)
1609 if isinstance(rows,tuple):
1610 rows = list(rows)
1611 limitby = args_get('limitby', None) or (0,)
1612 rows = self.rowslice(rows,limitby[0],None)
1613 processor = args_get('processor',self.parse)
1614 cacheable = args_get('cacheable',False)
1615 return processor(rows,fields,self._colnames,cacheable=cacheable)
1616
1617 - def select(self, query, fields, attributes):
1618 """
1619 Always returns a Rows object, possibly empty.
1620 """
1621 sql = self._select(query, fields, attributes)
1622 cache = attributes.get('cache', None)
1623 if cache and attributes.get('cacheable',False):
1624 del attributes['cache']
1625 (cache_model, time_expire) = cache
1626 key = self.uri + '/' + sql
1627 if len(key)>200: key = hashlib_md5(key).hexdigest()
1628 args = (sql,fields,attributes)
1629 return cache_model(
1630 key,
1631 lambda self=self,args=args:self._select_aux(*args),
1632 time_expire)
1633 else:
1634 return self._select_aux(sql,fields,attributes)
1635
1636 - def _count(self, query, distinct=None):
1637 tablenames = self.tables(query)
1638 if query:
1639 if use_common_filters(query):
1640 query = self.common_filter(query, tablenames)
1641 sql_w = ' WHERE ' + self.expand(query)
1642 else:
1643 sql_w = ''
1644 sql_t = ','.join(tablenames)
1645 if distinct:
1646 if isinstance(distinct,(list, tuple)):
1647 distinct = xorify(distinct)
1648 sql_d = self.expand(distinct)
1649 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \
1650 (sql_d, sql_t, sql_w)
1651 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1652
1653 - def count(self, query, distinct=None):
1654 self.execute(self._count(query, distinct))
1655 return self.cursor.fetchone()[0]
1656
1668
1671
1674
1677
1680
1683
1686
1689
1692
1694 return '%s_%s__constraint' % (table,fieldname)
1695
1698
1700 if not self.connection: return None
1701 command = a[0]
1702 if self.db._debug:
1703 LOGGER.debug('SQL: %s' % command)
1704 self.db._lastsql = command
1705 t0 = time.time()
1706 ret = self.cursor.execute(*a, **b)
1707 self.db._timings.append((command,time.time()-t0))
1708 del self.db._timings[:-TIMINGSSIZE]
1709 return ret
1710
1713
1715 field_is_type = fieldtype.startswith
1716 if isinstance(obj, CALLABLETYPES):
1717 obj = obj()
1718 if isinstance(fieldtype, SQLCustomType):
1719 value = fieldtype.encoder(obj)
1720 if fieldtype.type in ('string','text'):
1721 return self.adapt(value)
1722 return value
1723 if isinstance(obj, (Expression, Field)):
1724 return str(obj)
1725 if field_is_type('list:'):
1726 if not obj:
1727 obj = []
1728 elif not isinstance(obj, (list, tuple)):
1729 obj = [obj]
1730 if field_is_type('list:string'):
1731 obj = map(str,obj)
1732 else:
1733 obj = map(int,obj)
1734 if isinstance(obj, (list, tuple)):
1735 obj = bar_encode(obj)
1736 if obj is None:
1737 return 'NULL'
1738 if obj == '' and not fieldtype[:2] in ['st', 'te', 'pa', 'up']:
1739 return 'NULL'
1740 r = self.represent_exceptions(obj, fieldtype)
1741 if not r is None:
1742 return r
1743 if fieldtype == 'boolean':
1744 if obj and not str(obj)[:1].upper() in '0F':
1745 return self.smart_adapt(self.TRUE)
1746 else:
1747 return self.smart_adapt(self.FALSE)
1748 if fieldtype == 'id' or fieldtype == 'integer':
1749 return str(int(obj))
1750 if field_is_type('decimal'):
1751 return str(obj)
1752 elif field_is_type('reference'):
1753 if fieldtype.find('.')>0:
1754 return repr(obj)
1755 elif isinstance(obj, (Row, Reference)):
1756 return str(obj['id'])
1757 return str(int(obj))
1758 elif fieldtype == 'double':
1759 return repr(float(obj))
1760 if isinstance(obj, unicode):
1761 obj = obj.encode(self.db_codec)
1762 if fieldtype == 'blob':
1763 obj = base64.b64encode(str(obj))
1764 elif fieldtype == 'date':
1765 if isinstance(obj, (datetime.date, datetime.datetime)):
1766 obj = obj.isoformat()[:10]
1767 else:
1768 obj = str(obj)
1769 elif fieldtype == 'datetime':
1770 if isinstance(obj, datetime.datetime):
1771 obj = obj.isoformat()[:19].replace('T',' ')
1772 elif isinstance(obj, datetime.date):
1773 obj = obj.isoformat()[:10]+' 00:00:00'
1774 else:
1775 obj = str(obj)
1776 elif fieldtype == 'time':
1777 if isinstance(obj, datetime.time):
1778 obj = obj.isoformat()[:10]
1779 else:
1780 obj = str(obj)
1781 if not isinstance(obj,bytes):
1782 obj = bytes(obj)
1783 try:
1784 obj.decode(self.db_codec)
1785 except:
1786 obj = obj.decode('latin1').encode(self.db_codec)
1787 return self.adapt(obj)
1788
1791
1794
1797
1798 - def rowslice(self, rows, minimum=0, maximum=None):
1799 """
1800 By default this function does nothing;
1801 overload when db does not do slicing.
1802 """
1803 return rows
1804
1805 - def parse_value(self, value, field_type, blob_decode=True):
1806 if field_type != 'blob' and isinstance(value, str):
1807 try:
1808 value = value.decode(self.db._db_codec)
1809 except Exception:
1810 pass
1811 if isinstance(value, unicode):
1812 value = value.encode('utf-8')
1813 if isinstance(field_type, SQLCustomType):
1814 value = field_type.decoder(value)
1815 if not isinstance(field_type, str) or value is None:
1816 return value
1817 elif field_type in ('string', 'text', 'password', 'upload', 'dict'):
1818 return value
1819 elif field_type.startswith('geo'):
1820 return value
1821 elif field_type == 'blob' and not blob_decode:
1822 return value
1823 else:
1824 key = REGEX_TYPE.match(field_type).group(0)
1825 return self.parsemap[key](value,field_type)
1826
1828 referee = field_type[10:].strip()
1829 if not '.' in referee:
1830 value = Reference(value)
1831 value._table, value._record = self.db[referee], None
1832 return value
1833
1835 return value == True or str(value)[:1].lower() == 't'
1836
1838 if not isinstance(value, (datetime.date,datetime.datetime)):
1839 (y, m, d) = map(int, str(value)[:10].strip().split('-'))
1840 value = datetime.date(y, m, d)
1841 return value
1842
1844 if not isinstance(value, datetime.time):
1845 time_items = map(int,str(value)[:8].strip().split(':')[:3])
1846 if len(time_items) == 3:
1847 (h, mi, s) = time_items
1848 else:
1849 (h, mi, s) = time_items + [0]
1850 value = datetime.time(h, mi, s)
1851 return value
1852
1854 if not isinstance(value, datetime.datetime):
1855 value = str(value)
1856 date_part,time_part,timezone = value[:10],value[11:19],value[19:]
1857 if '+' in timezone:
1858 ms,tz = timezone.split('+')
1859 h,m = tz.split(':')
1860 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m))
1861 elif '-' in timezone:
1862 ms,tz = timezone.split('-')
1863 h,m = tz.split(':')
1864 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m))
1865 else:
1866 dt = None
1867 (y, m, d) = map(int,date_part.split('-'))
1868 time_parts = time_part and time_part.split(':')[:3] or (0,0,0)
1869 while len(time_parts)<3: time_parts.append(0)
1870 time_items = map(int,time_parts)
1871 (h, mi, s) = time_items
1872 value = datetime.datetime(y, m, d, h, mi, s)
1873 if dt:
1874 value = value + dt
1875 return value
1876
1878 return base64.b64decode(str(value))
1879
1881 decimals = int(field_type[8:-1].split(',')[-1])
1882 if self.dbengine in ('sqlite', 'spatialite'):
1883 value = ('%.' + str(decimals) + 'f') % value
1884 if not isinstance(value, decimal.Decimal):
1885 value = decimal.Decimal(str(value))
1886 return value
1887
1892
1897
1902
1903 - def parse_id(self, value, field_type):
1905
1908
1911
1913 self.parsemap = {
1914 'id':self.parse_id,
1915 'integer':self.parse_integer,
1916 'bigint':self.parse_integer,
1917 'float':self.parse_double,
1918 'double':self.parse_double,
1919 'reference':self.parse_reference,
1920 'boolean':self.parse_boolean,
1921 'date':self.parse_date,
1922 'time':self.parse_time,
1923 'datetime':self.parse_datetime,
1924 'blob':self.parse_blob,
1925 'decimal':self.parse_decimal,
1926 'list:integer':self.parse_list_integers,
1927 'list:reference':self.parse_list_references,
1928 'list:string':self.parse_list_strings,
1929 }
1930
1931 - def parse(self, rows, fields, colnames, blob_decode=True,
1932 cacheable = False):
1933 self.build_parsemap()
1934 db = self.db
1935 virtualtables = []
1936 new_rows = []
1937 tmps = []
1938 for colname in colnames:
1939 if not REGEX_TABLE_DOT_FIELD.match(colname):
1940 tmps.append(None)
1941 else:
1942 (tablename, fieldname) = colname.split('.')
1943 table = db[tablename]
1944 field = table[fieldname]
1945 ft = field.type
1946 tmps.append((tablename,fieldname,table,field,ft))
1947 for (i,row) in enumerate(rows):
1948 new_row = Row()
1949 for (j,colname) in enumerate(colnames):
1950 value = row[j]
1951 tmp = tmps[j]
1952 if tmp:
1953 (tablename,fieldname,table,field,ft) = tmp
1954 if tablename in new_row:
1955 colset = new_row[tablename]
1956 else:
1957 colset = new_row[tablename] = Row()
1958 if tablename not in virtualtables:
1959 virtualtables.append(tablename)
1960 value = self.parse_value(value,ft,blob_decode)
1961 if field.filter_out:
1962 value = field.filter_out(value)
1963 colset[fieldname] = value
1964
1965
1966 if ft=='id' and fieldname!='id' and \
1967 not 'id' in table.fields:
1968 colset['id'] = value
1969
1970 if ft == 'id' and not cacheable:
1971
1972
1973
1974 if isinstance(self, GoogleDatastoreAdapter):
1975 id = value.key().id_or_name()
1976 colset[fieldname] = id
1977 colset.gae_item = value
1978 else:
1979 id = value
1980 colset.update_record = RecordUpdater(colset,table,id)
1981 colset.delete_record = RecordDeleter(table,id)
1982 for rfield in table._referenced_by:
1983 referee_link = db._referee_name and \
1984 db._referee_name % dict(
1985 table=rfield.tablename,field=rfield.name)
1986 if referee_link and not referee_link in colset:
1987 colset[referee_link] = LazySet(rfield,id)
1988 else:
1989 if not '_extra' in new_row:
1990 new_row['_extra'] = Row()
1991 new_row['_extra'][colname] = \
1992 self.parse_value(value,
1993 fields[j].type,blob_decode)
1994 new_column_name = \
1995 REGEX_SELECT_AS_PARSER.search(colname)
1996 if not new_column_name is None:
1997 column_name = new_column_name.groups(0)
1998 setattr(new_row,column_name[0],value)
1999 new_rows.append(new_row)
2000 rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
2001
2002 for tablename in virtualtables:
2003
2004 table = db[tablename]
2005 fields_virtual = [(f,v) for (f,v) in table.iteritems()
2006 if isinstance(v,FieldVirtual)]
2007 fields_lazy = [(f,v) for (f,v) in table.iteritems()
2008 if isinstance(v,FieldMethod)]
2009 if fields_virtual or fields_lazy:
2010 for row in rowsobj.records:
2011 box = row[tablename]
2012 for f,v in fields_virtual:
2013 box[f] = v.f(row)
2014 for f,v in fields_lazy:
2015 box[f] = (v.handler or VirtualCommand)(v.f,row)
2016
2017
2018 for item in table.virtualfields:
2019 try:
2020 rowsobj = rowsobj.setvirtualfields(**{tablename:item})
2021 except (KeyError, AttributeError):
2022
2023 pass
2024 return rowsobj
2025
2027 tenant_fieldname = self.db._request_tenant
2028
2029 for tablename in tablenames:
2030 table = self.db[tablename]
2031
2032
2033 if table._common_filter != None:
2034 query = query & table._common_filter(query)
2035
2036
2037 if tenant_fieldname in table:
2038 default = table[tenant_fieldname].default
2039 if not default is None:
2040 newquery = table[tenant_fieldname] == default
2041 if query is None:
2042 query = newquery
2043 else:
2044 query = query & newquery
2045 return query
2046
2047 - def CASE(self,query,t,f):
2048 def represent(x):
2049 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'}
2050 if x is None: return 'NULL'
2051 elif isinstance(x,Expression): return str(x)
2052 else: return self.represent(x,types.get(type(x),'string'))
2053 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \
2054 (self.expand(query),represent(t),represent(f)))
2055
2061 drivers = ('sqlite2','sqlite3')
2062
2063 can_select_for_update = None
2064
2066 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2067
2068 @staticmethod
2070 table = {
2071 'year': (0, 4),
2072 'month': (5, 7),
2073 'day': (8, 10),
2074 'hour': (11, 13),
2075 'minute': (14, 16),
2076 'second': (17, 19),
2077 }
2078 try:
2079 if lookup != 'epoch':
2080 (i, j) = table[lookup]
2081 return int(s[i:j])
2082 else:
2083 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
2084 except:
2085 return None
2086
2087 @staticmethod
2089 return re.compile(expression).search(item) is not None
2090
2091 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2092 credential_decoder=IDENTITY, driver_args={},
2093 adapter_args={}, do_connect=True):
2094 self.db = db
2095 self.dbengine = "sqlite"
2096 self.uri = uri
2097 if do_connect: self.find_driver(adapter_args)
2098 self.pool_size = 0
2099 self.folder = folder
2100 self.db_codec = db_codec
2101 self.find_or_make_work_folder()
2102 path_encoding = sys.getfilesystemencoding() \
2103 or locale.getdefaultlocale()[1] or 'utf8'
2104 if uri.startswith('sqlite:memory'):
2105 dbpath = ':memory:'
2106 else:
2107 dbpath = uri.split('://',1)[1]
2108 if dbpath[0] != '/':
2109 if PYTHON_VERSION == 2:
2110 dbpath = pjoin(
2111 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2112 else:
2113 dbpath = pjoin(self.folder, dbpath)
2114 if not 'check_same_thread' in driver_args:
2115 driver_args['check_same_thread'] = False
2116 if not 'detect_types' in driver_args and do_connect:
2117 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2118 def connector(dbpath=dbpath, driver_args=driver_args):
2119 return self.driver.Connection(dbpath, **driver_args)
2120 self.connector = connector
2121 if do_connect: self.reconnect()
2122
2128
2130 tablename = table._tablename
2131 return ['DELETE FROM %s;' % tablename,
2132 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2133
2136
2137 - def REGEXP(self,first,second):
2138 return '(%s REGEXP %s)' % (self.expand(first),
2139 self.expand(second,'string'))
2140
2141 - def select(self, query, fields, attributes):
2142 """
2143 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
2144 Note that the entire database, rather than one record, is locked
2145 (it will be locked eventually anyway by the following UPDATE).
2146 """
2147 if attributes.get('for_update', False) and not 'cache' in attributes:
2148 self.execute('BEGIN IMMEDIATE TRANSACTION;')
2149 return super(SQLiteAdapter, self).select(query, fields, attributes)
2150
2152 drivers = ('sqlite3','sqlite2')
2153
2154 types = copy.copy(BaseAdapter.types)
2155 types.update(geometry='GEOMETRY')
2156
2157 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2158 credential_decoder=IDENTITY, driver_args={},
2159 adapter_args={}, do_connect=True, srid=4326):
2160 self.db = db
2161 self.dbengine = "spatialite"
2162 self.uri = uri
2163 if do_connect: self.find_driver(adapter_args)
2164 self.pool_size = 0
2165 self.folder = folder
2166 self.db_codec = db_codec
2167 self.find_or_make_work_folder()
2168 self.srid = srid
2169 path_encoding = sys.getfilesystemencoding() \
2170 or locale.getdefaultlocale()[1] or 'utf8'
2171 if uri.startswith('spatialite:memory'):
2172 dbpath = ':memory:'
2173 else:
2174 dbpath = uri.split('://',1)[1]
2175 if dbpath[0] != '/':
2176 dbpath = pjoin(
2177 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2178 if not 'check_same_thread' in driver_args:
2179 driver_args['check_same_thread'] = False
2180 if not 'detect_types' in driver_args and do_connect:
2181 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2182 def connector(dbpath=dbpath, driver_args=driver_args):
2183 return self.driver.Connection(dbpath, **driver_args)
2184 self.connector = connector
2185 if do_connect: self.reconnect()
2186
2199
2200
2201
2203 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
2204 second['precision'], second['options'])
2205
2206 - def ST_ASTEXT(self, first):
2207 return 'AsText(%s)' %(self.expand(first))
2208
2212
2216
2220
2224
2228
2230 return 'Simplify(%s,%s)' %(self.expand(first),
2231 self.expand(second, 'double'))
2232
2236
2240
2242 field_is_type = fieldtype.startswith
2243 if field_is_type('geo'):
2244 srid = 4326
2245 geotype, parms = fieldtype[:-1].split('(')
2246 parms = parms.split(',')
2247 if len(parms) >= 2:
2248 schema, srid = parms[:2]
2249
2250 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2251
2252
2253
2254
2255 return value
2256 return BaseAdapter.represent(self, obj, fieldtype)
2257
2260 drivers = ('zxJDBC_sqlite',)
2261
2262 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
2263 credential_decoder=IDENTITY, driver_args={},
2264 adapter_args={}, do_connect=True):
2265 self.db = db
2266 self.dbengine = "sqlite"
2267 self.uri = uri
2268 if do_connect: self.find_driver(adapter_args)
2269 self.pool_size = pool_size
2270 self.folder = folder
2271 self.db_codec = db_codec
2272 self.find_or_make_work_folder()
2273 path_encoding = sys.getfilesystemencoding() \
2274 or locale.getdefaultlocale()[1] or 'utf8'
2275 if uri.startswith('sqlite:memory'):
2276 dbpath = ':memory:'
2277 else:
2278 dbpath = uri.split('://',1)[1]
2279 if dbpath[0] != '/':
2280 dbpath = pjoin(
2281 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2282 def connector(dbpath=dbpath,driver_args=driver_args):
2283 return self.driver.connect(
2284 self.driver.getConnection('jdbc:sqlite:'+dbpath),
2285 **driver_args)
2286 self.connector = connector
2287 if do_connect: self.reconnect()
2288
2293
2296
2299 drivers = ('MySQLdb','pymysql')
2300
2301 maxcharlength = 255
2302 commit_on_alter_table = True
2303 support_distributed_transaction = True
2304 types = {
2305 'boolean': 'CHAR(1)',
2306 'string': 'VARCHAR(%(length)s)',
2307 'text': 'LONGTEXT',
2308 'password': 'VARCHAR(%(length)s)',
2309 'blob': 'LONGBLOB',
2310 'upload': 'VARCHAR(%(length)s)',
2311 'integer': 'INT',
2312 'bigint': 'BIGINT',
2313 'float': 'FLOAT',
2314 'double': 'DOUBLE',
2315 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2316 'date': 'DATE',
2317 'time': 'TIME',
2318 'datetime': 'DATETIME',
2319 'id': 'INT AUTO_INCREMENT NOT NULL',
2320 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2321 'list:integer': 'LONGTEXT',
2322 'list:string': 'LONGTEXT',
2323 'list:reference': 'LONGTEXT',
2324 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
2325 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2326 }
2327
2330
2333
2335 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
2336 parameters[0], parameters[1])
2337
2338 - def EPOCH(self, first):
2340
2341 - def _drop(self,table,mode):
2342
2343 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,
2344 'SET FOREIGN_KEY_CHECKS=1;']
2345
2348
2352
2355
2358
2359 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
2360
2361 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2362 credential_decoder=IDENTITY, driver_args={},
2363 adapter_args={}, do_connect=True):
2364 self.db = db
2365 self.dbengine = "mysql"
2366 self.uri = uri
2367 if do_connect: self.find_driver(adapter_args,uri)
2368 self.pool_size = pool_size
2369 self.folder = folder
2370 self.db_codec = db_codec
2371 self.find_or_make_work_folder()
2372 ruri = uri.split('://',1)[1]
2373 m = self.REGEX_URI.match(ruri)
2374 if not m:
2375 raise SyntaxError(
2376 "Invalid URI string in DAL: %s" % self.uri)
2377 user = credential_decoder(m.group('user'))
2378 if not user:
2379 raise SyntaxError('User required')
2380 password = credential_decoder(m.group('password'))
2381 if not password:
2382 password = ''
2383 host = m.group('host')
2384 if not host:
2385 raise SyntaxError('Host name required')
2386 db = m.group('db')
2387 if not db:
2388 raise SyntaxError('Database name required')
2389 port = int(m.group('port') or '3306')
2390 charset = m.group('charset') or 'utf8'
2391 driver_args.update(db=db,
2392 user=credential_decoder(user),
2393 passwd=credential_decoder(password),
2394 host=host,
2395 port=port,
2396 charset=charset)
2397
2398
2399 def connector(driver_args=driver_args):
2400 return self.driver.connect(**driver_args)
2401 self.connector = connector
2402 if do_connect: self.reconnect()
2403
2405 self.execute('SET FOREIGN_KEY_CHECKS=1;')
2406 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2407
2409 self.execute('select last_insert_id();')
2410 return int(self.cursor.fetchone()[0])
2411
2412 -class PostgreSQLAdapter(BaseAdapter):
2413 drivers = ('psycopg2','pg8000')
2414
2415 support_distributed_transaction = True
2416 types = {
2417 'boolean': 'CHAR(1)',
2418 'string': 'VARCHAR(%(length)s)',
2419 'text': 'TEXT',
2420 'password': 'VARCHAR(%(length)s)',
2421 'blob': 'BYTEA',
2422 'upload': 'VARCHAR(%(length)s)',
2423 'integer': 'INTEGER',
2424 'bigint': 'BIGINT',
2425 'float': 'FLOAT',
2426 'double': 'FLOAT8',
2427 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2428 'date': 'DATE',
2429 'time': 'TIME',
2430 'datetime': 'TIMESTAMP',
2431 'id': 'SERIAL PRIMARY KEY',
2432 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2433 'list:integer': 'TEXT',
2434 'list:string': 'TEXT',
2435 'list:reference': 'TEXT',
2436 'geometry': 'GEOMETRY',
2437 'geography': 'GEOGRAPHY',
2438 'big-id': 'BIGSERIAL PRIMARY KEY',
2439 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2440 }
2441
2442 - def varquote(self,name):
2443 return varquote_aux(name,'"%s"')
2444
2445 - def adapt(self,obj):
2446 if self.driver_name == 'psycopg2':
2447 return psycopg2_adapt(obj).getquoted()
2448 elif self.driver_name == 'pg8000':
2449 return "'%s'" % str(obj).replace("%","%%").replace("'","''")
2450 else:
2451 return "'%s'" % str(obj).replace("'","''")
2452
2453 - def sequence_name(self,table):
2454 return '%s_id_Seq' % table
2455
2458
2459 - def ADD(self, first, second):
2460 t = first.type
2461 if t in ('text','string','password','upload','blob'):
2462 return '(%s || %s)' % (self.expand(first), self.expand(second, t))
2463 else:
2464 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2465
2468
2469 - def prepare(self,key):
2470 self.execute("PREPARE TRANSACTION '%s';" % key)
2471
2472 - def commit_prepared(self,key):
2473 self.execute("COMMIT PREPARED '%s';" % key)
2474
2475 - def rollback_prepared(self,key):
2476 self.execute("ROLLBACK PREPARED '%s';" % key)
2477
2478 - def create_sequence_and_triggers(self, query, table, **args):
2479
2480
2481
2482
2483 self.execute(query)
2484
2485 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
2486
2487 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2488 credential_decoder=IDENTITY, driver_args={},
2489 adapter_args={}, do_connect=True, srid=4326):
2490 self.db = db
2491 self.dbengine = "postgres"
2492 self.uri = uri
2493 if do_connect: self.find_driver(adapter_args,uri)
2494 self.pool_size = pool_size
2495 self.folder = folder
2496 self.db_codec = db_codec
2497 self.srid = srid
2498 self.find_or_make_work_folder()
2499 ruri = uri.split('://',1)[1]
2500 m = self.REGEX_URI.match(ruri)
2501 if not m:
2502 raise SyntaxError("Invalid URI string in DAL")
2503 user = credential_decoder(m.group('user'))
2504 if not user:
2505 raise SyntaxError('User required')
2506 password = credential_decoder(m.group('password'))
2507 if not password:
2508 password = ''
2509 host = m.group('host')
2510 if not host:
2511 raise SyntaxError('Host name required')
2512 db = m.group('db')
2513 if not db:
2514 raise SyntaxError('Database name required')
2515 port = m.group('port') or '5432'
2516 sslmode = m.group('sslmode')
2517 if sslmode:
2518 msg = ("dbname='%s' user='%s' host='%s' "
2519 "port=%s password='%s' sslmode='%s'") \
2520 % (db, user, host, port, password, sslmode)
2521 else:
2522 msg = ("dbname='%s' user='%s' host='%s' "
2523 "port=%s password='%s'") \
2524 % (db, user, host, port, password)
2525
2526 self.__version__ = "%s %s" % (self.driver.__name__, self.driver.__version__)
2527 def connector(msg=msg,driver_args=driver_args):
2528 return self.driver.connect(msg,**driver_args)
2529 self.connector = connector
2530 if do_connect: self.reconnect()
2531
2532 - def after_connection(self):
2533 self.connection.set_client_encoding('UTF8')
2534 self.execute("SET standard_conforming_strings=on;")
2535
2536 - def lastrowid(self,table):
2537 self.execute("select currval('%s')" % table._sequence_name)
2538 return int(self.cursor.fetchone()[0])
2539
2540
2541 - def LIKE(self,first,second):
2542 return '(%s LIKE %s)' % (self.expand(first),
2543 self.expand(second,'string'))
2544
2545 - def ILIKE(self,first,second):
2546 return '(%s ILIKE %s)' % (self.expand(first),
2547 self.expand(second,'string'))
2548
2549 - def REGEXP(self,first,second):
2550 return '(%s ~ %s)' % (self.expand(first),
2551 self.expand(second,'string'))
2552
2553 - def STARTSWITH(self,first,second):
2554 return '(%s ILIKE %s)' % (self.expand(first),
2555 self.expand(second+'%','string'))
2556
2557 - def ENDSWITH(self,first,second):
2558 return '(%s ILIKE %s)' % (self.expand(first),
2559 self.expand('%'+second,'string'))
2560
2561 - def CONTAINS(self,first,second):
2562 if first.type in ('string','text'):
2563 key = '%'+str(second).replace('%','%%')+'%'
2564 elif first.type.startswith('list:'):
2565 key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
2566 return '(%s ILIKE %s)' % (self.expand(first),self.expand(key,'string'))
2567
2568
2569
2570 - def ST_ASGEOJSON(self, first, second):
2571 """
2572 http://postgis.org/docs/ST_AsGeoJSON.html
2573 """
2574 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
2575 self.expand(first), second['precision'], second['options'])
2576
2577 - def ST_ASTEXT(self, first):
2578 """
2579 http://postgis.org/docs/ST_AsText.html
2580 """
2581 return 'ST_AsText(%s)' %(self.expand(first))
2582
2583
2584
2585
2586
2587
2588
2589
2590 - def ST_CONTAINS(self, first, second):
2591 """
2592 http://postgis.org/docs/ST_Contains.html
2593 """
2594 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2595
2596 - def ST_DISTANCE(self, first, second):
2597 """
2598 http://postgis.org/docs/ST_Distance.html
2599 """
2600 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2601
2602 - def ST_EQUALS(self, first, second):
2603 """
2604 http://postgis.org/docs/ST_Equals.html
2605 """
2606 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2607
2608 - def ST_INTERSECTS(self, first, second):
2609 """
2610 http://postgis.org/docs/ST_Intersects.html
2611 """
2612 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2613
2614 - def ST_OVERLAPS(self, first, second):
2615 """
2616 http://postgis.org/docs/ST_Overlaps.html
2617 """
2618 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2619
2620 - def ST_SIMPLIFY(self, first, second):
2621 """
2622 http://postgis.org/docs/ST_Simplify.html
2623 """
2624 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2625
2626 - def ST_TOUCHES(self, first, second):
2627 """
2628 http://postgis.org/docs/ST_Touches.html
2629 """
2630 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2631
2632 - def ST_WITHIN(self, first, second):
2633 """
2634 http://postgis.org/docs/ST_Within.html
2635 """
2636 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2637
2638 - def represent(self, obj, fieldtype):
2639 field_is_type = fieldtype.startswith
2640 if field_is_type('geo'):
2641 srid = 4326
2642 geotype, parms = fieldtype[:-1].split('(')
2643 parms = parms.split(',')
2644 if len(parms) >= 2:
2645 schema, srid = parms[:2]
2646 if field_is_type('geometry'):
2647 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2648 elif field_is_type('geography'):
2649 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
2650
2651
2652 return value
2653 return BaseAdapter.represent(self, obj, fieldtype)
2654
2655 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2656 drivers = ('psycopg2','pg8000')
2657
2658 types = {
2659 'boolean': 'CHAR(1)',
2660 'string': 'VARCHAR(%(length)s)',
2661 'text': 'TEXT',
2662 'password': 'VARCHAR(%(length)s)',
2663 'blob': 'BYTEA',
2664 'upload': 'VARCHAR(%(length)s)',
2665 'integer': 'INTEGER',
2666 'bigint': 'BIGINT',
2667 'float': 'FLOAT',
2668 'double': 'FLOAT8',
2669 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2670 'date': 'DATE',
2671 'time': 'TIME',
2672 'datetime': 'TIMESTAMP',
2673 'id': 'SERIAL PRIMARY KEY',
2674 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2675 'list:integer': 'BIGINT[]',
2676 'list:string': 'TEXT[]',
2677 'list:reference': 'BIGINT[]',
2678 'geometry': 'GEOMETRY',
2679 'geography': 'GEOGRAPHY',
2680 'big-id': 'BIGSERIAL PRIMARY KEY',
2681 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2682 }
2683
2684 - def parse_list_integers(self, value, field_type):
2686
2687 - def parse_list_references(self, value, field_type):
2688 return [self.parse_reference(r, field_type[5:]) for r in value]
2689
2690 - def parse_list_strings(self, value, field_type):
2692
2693 - def represent(self, obj, fieldtype):
2694 field_is_type = fieldtype.startswith
2695 if field_is_type('list:'):
2696 if not obj:
2697 obj = []
2698 elif not isinstance(obj, (list, tuple)):
2699 obj = [obj]
2700 if field_is_type('list:string'):
2701 obj = map(str,obj)
2702 else:
2703 obj = map(int,obj)
2704 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
2705 return BaseAdapter.represent(self, obj, fieldtype)
2706
2707
2708 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
2709 drivers = ('zxJDBC',)
2710
2711 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
2712
2713 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2714 credential_decoder=IDENTITY, driver_args={},
2715 adapter_args={}, do_connect=True):
2716 self.db = db
2717 self.dbengine = "postgres"
2718 self.uri = uri
2719 if do_connect: self.find_driver(adapter_args,uri)
2720 self.pool_size = pool_size
2721 self.folder = folder
2722 self.db_codec = db_codec
2723 self.find_or_make_work_folder()
2724 ruri = uri.split('://',1)[1]
2725 m = self.REGEX_URI.match(ruri)
2726 if not m:
2727 raise SyntaxError("Invalid URI string in DAL")
2728 user = credential_decoder(m.group('user'))
2729 if not user:
2730 raise SyntaxError('User required')
2731 password = credential_decoder(m.group('password'))
2732 if not password:
2733 password = ''
2734 host = m.group('host')
2735 if not host:
2736 raise SyntaxError('Host name required')
2737 db = m.group('db')
2738 if not db:
2739 raise SyntaxError('Database name required')
2740 port = m.group('port') or '5432'
2741 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
2742 def connector(msg=msg,driver_args=driver_args):
2743 return self.driver.connect(*msg,**driver_args)
2744 self.connector = connector
2745 if do_connect: self.reconnect()
2746
2747 - def after_connection(self):
2748 self.connection.set_client_encoding('UTF8')
2749 self.execute('BEGIN;')
2750 self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
2751
2754 drivers = ('cx_Oracle',)
2755
2756 commit_on_alter_table = False
2757 types = {
2758 'boolean': 'CHAR(1)',
2759 'string': 'VARCHAR2(%(length)s)',
2760 'text': 'CLOB',
2761 'password': 'VARCHAR2(%(length)s)',
2762 'blob': 'CLOB',
2763 'upload': 'VARCHAR2(%(length)s)',
2764 'integer': 'INT',
2765 'bigint': 'NUMBER',
2766 'float': 'FLOAT',
2767 'double': 'BINARY_DOUBLE',
2768 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2769 'date': 'DATE',
2770 'time': 'CHAR(8)',
2771 'datetime': 'DATE',
2772 'id': 'NUMBER PRIMARY KEY',
2773 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2774 'list:integer': 'CLOB',
2775 'list:string': 'CLOB',
2776 'list:reference': 'CLOB',
2777 'big-id': 'NUMBER PRIMARY KEY',
2778 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2779 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2780 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2781 }
2782
2784 return '%s_sequence' % tablename
2785
2787 return '%s_trigger' % tablename
2788
2790 return 'LEFT OUTER JOIN'
2791
2793 return 'dbms_random.value'
2794
2795 - def NOT_NULL(self,default,field_type):
2796 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
2797
2798 - def _drop(self,table,mode):
2801
2802 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
2803 if limitby:
2804 (lmin, lmax) = limitby
2805 if len(sql_w) > 1:
2806 sql_w_row = sql_w + ' AND w_row > %i' % lmin
2807 else:
2808 sql_w_row = 'WHERE w_row > %i' % lmin
2809 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
2810 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
2811
2817
2819 if fieldtype == 'blob':
2820 obj = base64.b64encode(str(obj))
2821 return ":CLOB('%s')" % obj
2822 elif fieldtype == 'date':
2823 if isinstance(obj, (datetime.date, datetime.datetime)):
2824 obj = obj.isoformat()[:10]
2825 else:
2826 obj = str(obj)
2827 return "to_date('%s','yyyy-mm-dd')" % obj
2828 elif fieldtype == 'datetime':
2829 if isinstance(obj, datetime.datetime):
2830 obj = obj.isoformat()[:19].replace('T',' ')
2831 elif isinstance(obj, datetime.date):
2832 obj = obj.isoformat()[:10]+' 00:00:00'
2833 else:
2834 obj = str(obj)
2835 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
2836 return None
2837
2838 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2839 credential_decoder=IDENTITY, driver_args={},
2840 adapter_args={}, do_connect=True):
2841 self.db = db
2842 self.dbengine = "oracle"
2843 self.uri = uri
2844 if do_connect: self.find_driver(adapter_args,uri)
2845 self.pool_size = pool_size
2846 self.folder = folder
2847 self.db_codec = db_codec
2848 self.find_or_make_work_folder()
2849 ruri = uri.split('://',1)[1]
2850 if not 'threaded' in driver_args:
2851 driver_args['threaded']=True
2852 def connector(uri=ruri,driver_args=driver_args):
2853 return self.driver.connect(uri,**driver_args)
2854 self.connector = connector
2855 if do_connect: self.reconnect()
2856
2858 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
2859 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
2860
2861 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
2862
2863 - def execute(self, command, args=None):
2864 args = args or []
2865 i = 1
2866 while True:
2867 m = self.oracle_fix.match(command)
2868 if not m:
2869 break
2870 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
2871 args.append(m.group('clob')[6:-2].replace("''", "'"))
2872 i += 1
2873 if command[-1:]==';':
2874 command = command[:-1]
2875 return self.log_execute(command, args)
2876
2878 tablename = table._tablename
2879 sequence_name = table._sequence_name
2880 trigger_name = table._trigger_name
2881 self.execute(query)
2882 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
2883 self.execute("""
2884 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
2885 DECLARE
2886 curr_val NUMBER;
2887 diff_val NUMBER;
2888 PRAGMA autonomous_transaction;
2889 BEGIN
2890 IF :NEW.id IS NOT NULL THEN
2891 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
2892 diff_val := :NEW.id - curr_val - 1;
2893 IF diff_val != 0 THEN
2894 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
2895 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
2896 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
2897 END IF;
2898 END IF;
2899 SELECT %(sequence_name)s.nextval INTO :NEW.id FROM DUAL;
2900 END;
2901 """ % dict(trigger_name=trigger_name, tablename=tablename, sequence_name=sequence_name))
2902
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2918 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description):
2919 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
2920 for c in r]) for r in self.cursor]
2921 else:
2922 return self.cursor.fetchall()
2923
2925 drivers = ('pyodbc',)
2926
2927 types = {
2928 'boolean': 'BIT',
2929 'string': 'VARCHAR(%(length)s)',
2930 'text': 'TEXT',
2931 'password': 'VARCHAR(%(length)s)',
2932 'blob': 'IMAGE',
2933 'upload': 'VARCHAR(%(length)s)',
2934 'integer': 'INT',
2935 'bigint': 'BIGINT',
2936 'float': 'FLOAT',
2937 'double': 'FLOAT',
2938 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2939 'date': 'DATETIME',
2940 'time': 'CHAR(8)',
2941 'datetime': 'DATETIME',
2942 'id': 'INT IDENTITY PRIMARY KEY',
2943 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2944 'list:integer': 'TEXT',
2945 'list:string': 'TEXT',
2946 'list:reference': 'TEXT',
2947 'geometry': 'geometry',
2948 'geography': 'geography',
2949 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
2950 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2951 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2952 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2953 }
2954
2956 return '; ALTER TABLE %s ADD ' % tablename
2957
2960
2962 return "DATEPART(%s,%s)" % (what, self.expand(field))
2963
2965 return 'LEFT OUTER JOIN'
2966
2969
2972
2974 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
2975
2977 return 'PRIMARY KEY CLUSTERED (%s)' % key
2978
2980 if what == 'LENGTH':
2981 what = 'LEN'
2982 return "%s(%s)" % (what, self.expand(first))
2983
2984
2985 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
2986 if limitby:
2987 (lmin, lmax) = limitby
2988 sql_s += ' TOP %i' % lmax
2989 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
2990
2991 TRUE = 1
2992 FALSE = 0
2993
2994 REGEX_DSN = re.compile('^(?P<dsn>.+)$')
2995 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
2996 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
2997
2998 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2999 credential_decoder=IDENTITY, driver_args={},
3000 adapter_args={}, do_connect=True, srid=4326):
3001 self.db = db
3002 self.dbengine = "mssql"
3003 self.uri = uri
3004 if do_connect: self.find_driver(adapter_args,uri)
3005 self.pool_size = pool_size
3006 self.folder = folder
3007 self.db_codec = db_codec
3008 self.srid = srid
3009 self.find_or_make_work_folder()
3010
3011 ruri = uri.split('://',1)[1]
3012 if '@' not in ruri:
3013 try:
3014 m = self.REGEX_DSN.match(ruri)
3015 if not m:
3016 raise SyntaxError(
3017 'Parsing uri string(%s) has no result' % self.uri)
3018 dsn = m.group('dsn')
3019 if not dsn:
3020 raise SyntaxError('DSN required')
3021 except SyntaxError:
3022 e = sys.exc_info()[1]
3023 LOGGER.error('NdGpatch error')
3024 raise e
3025
3026 cnxn = dsn
3027 else:
3028 m = self.REGEX_URI.match(ruri)
3029 if not m:
3030 raise SyntaxError(
3031 "Invalid URI string in DAL: %s" % self.uri)
3032 user = credential_decoder(m.group('user'))
3033 if not user:
3034 raise SyntaxError('User required')
3035 password = credential_decoder(m.group('password'))
3036 if not password:
3037 password = ''
3038 host = m.group('host')
3039 if not host:
3040 raise SyntaxError('Host name required')
3041 db = m.group('db')
3042 if not db:
3043 raise SyntaxError('Database name required')
3044 port = m.group('port') or '1433'
3045
3046
3047
3048 argsdict = { 'DRIVER':'{SQL Server}' }
3049 urlargs = m.group('urlargs') or ''
3050 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
3051 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
3052 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
3053 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
3054 % (host, port, db, user, password, urlargs)
3055 def connector(cnxn=cnxn,driver_args=driver_args):
3056 return self.driver.connect(cnxn,**driver_args)
3057 self.connector = connector
3058 if do_connect: self.reconnect()
3059
3061
3062 self.execute('SELECT SCOPE_IDENTITY();')
3063 return int(self.cursor.fetchone()[0])
3064
3066 return pyodbc.IntegrityError
3067
3068 - def rowslice(self,rows,minimum=0,maximum=None):
3069 if maximum is None:
3070 return rows[minimum:]
3071 return rows[minimum:maximum]
3072
3073 - def EPOCH(self, first):
3074 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3075
3076
3077
3078
3079
3080 - def ST_ASTEXT(self, first):
3081 return '%s.STAsText()' %(self.expand(first))
3082
3085
3088
3091
3094
3097
3098
3099
3102
3105
3107 field_is_type = fieldtype.startswith
3108 if field_is_type('geometry'):
3109 srid = 0
3110 geotype, parms = fieldtype[:-1].split('(')
3111 if parms:
3112 srid = parms
3113 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3114 elif fieldtype == 'geography':
3115 srid = 4326
3116 geotype, parms = fieldtype[:-1].split('(')
3117 if parms:
3118 srid = parms
3119 return "geography::STGeomFromText('%s',%s)" %(obj, srid)
3120
3121
3122 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3123 return BaseAdapter.represent(self, obj, fieldtype)
3124
3127 """ experimental support for pagination in MSSQL"""
3128 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3129 if limitby:
3130 (lmin, lmax) = limitby
3131 return 'SELECT %s FROM (SELECT %s ROW_NUMBER() over (order by id) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_f,sql_s,sql_f,sql_t,sql_w,sql_o,lmin,lmax)
3132 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3133 - def rowslice(self,rows,minimum=0,maximum=None):
3135
3138 drivers = ('pyodbc',)
3139
3140 types = {
3141 'boolean': 'CHAR(1)',
3142 'string': 'NVARCHAR(%(length)s)',
3143 'text': 'NTEXT',
3144 'password': 'NVARCHAR(%(length)s)',
3145 'blob': 'IMAGE',
3146 'upload': 'NVARCHAR(%(length)s)',
3147 'integer': 'INT',
3148 'bigint': 'BIGINT',
3149 'float': 'FLOAT',
3150 'double': 'FLOAT',
3151 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3152 'date': 'DATETIME',
3153 'time': 'CHAR(8)',
3154 'datetime': 'DATETIME',
3155 'id': 'INT IDENTITY PRIMARY KEY',
3156 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3157 'list:integer': 'NTEXT',
3158 'list:string': 'NTEXT',
3159 'list:reference': 'NTEXT',
3160 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3161 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3162 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3163 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3164 }
3165
3167 value = BaseAdapter.represent(self, obj, fieldtype)
3168 if fieldtype in ('string','text') and value[:1]=="'":
3169 value = 'N'+value
3170 return value
3171
3174
3176 drivers = ('Sybase',)
3177
3178 types = {
3179 'boolean': 'BIT',
3180 'string': 'CHAR VARYING(%(length)s)',
3181 'text': 'TEXT',
3182 'password': 'CHAR VARYING(%(length)s)',
3183 'blob': 'IMAGE',
3184 'upload': 'CHAR VARYING(%(length)s)',
3185 'integer': 'INT',
3186 'bigint': 'BIGINT',
3187 'float': 'FLOAT',
3188 'double': 'FLOAT',
3189 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3190 'date': 'DATETIME',
3191 'time': 'CHAR(8)',
3192 'datetime': 'DATETIME',
3193 'id': 'INT IDENTITY PRIMARY KEY',
3194 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3195 'list:integer': 'TEXT',
3196 'list:string': 'TEXT',
3197 'list:reference': 'TEXT',
3198 'geometry': 'geometry',
3199 'geography': 'geography',
3200 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3201 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3202 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3203 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3204 }
3205
3206
3207 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3208 credential_decoder=IDENTITY, driver_args={},
3209 adapter_args={}, do_connect=True, srid=4326):
3210 self.db = db
3211 self.dbengine = "sybase"
3212 self.uri = uri
3213 if do_connect: self.find_driver(adapter_args,uri)
3214 self.pool_size = pool_size
3215 self.folder = folder
3216 self.db_codec = db_codec
3217 self.srid = srid
3218 self.find_or_make_work_folder()
3219
3220 ruri = uri.split('://',1)[1]
3221 if '@' not in ruri:
3222 try:
3223 m = self.REGEX_DSN.match(ruri)
3224 if not m:
3225 raise SyntaxError(
3226 'Parsing uri string(%s) has no result' % self.uri)
3227 dsn = m.group('dsn')
3228 if not dsn:
3229 raise SyntaxError('DSN required')
3230 except SyntaxError:
3231 e = sys.exc_info()[1]
3232 LOGGER.error('NdGpatch error')
3233 raise e
3234 else:
3235 m = self.REGEX_URI.match(uri)
3236 if not m:
3237 raise SyntaxError(
3238 "Invalid URI string in DAL: %s" % self.uri)
3239 user = credential_decoder(m.group('user'))
3240 if not user:
3241 raise SyntaxError('User required')
3242 password = credential_decoder(m.group('password'))
3243 if not password:
3244 password = ''
3245 host = m.group('host')
3246 if not host:
3247 raise SyntaxError('Host name required')
3248 db = m.group('db')
3249 if not db:
3250 raise SyntaxError('Database name required')
3251 port = m.group('port') or '1433'
3252
3253 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
3254
3255 driver_args.update(user = credential_decoder(user),
3256 password = credential_decoder(password))
3257
3258 def connector(dsn=dsn,driver_args=driver_args):
3259 return self.driver.connect(dsn,**driver_args)
3260 self.connector = connector
3261 if do_connect: self.reconnect()
3262
3265
3268 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3269
3270 commit_on_alter_table = False
3271 support_distributed_transaction = True
3272 types = {
3273 'boolean': 'CHAR(1)',
3274 'string': 'VARCHAR(%(length)s)',
3275 'text': 'BLOB SUB_TYPE 1',
3276 'password': 'VARCHAR(%(length)s)',
3277 'blob': 'BLOB SUB_TYPE 0',
3278 'upload': 'VARCHAR(%(length)s)',
3279 'integer': 'INTEGER',
3280 'bigint': 'BIGINT',
3281 'float': 'FLOAT',
3282 'double': 'DOUBLE PRECISION',
3283 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3284 'date': 'DATE',
3285 'time': 'TIME',
3286 'datetime': 'TIMESTAMP',
3287 'id': 'INTEGER PRIMARY KEY',
3288 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3289 'list:integer': 'BLOB SUB_TYPE 1',
3290 'list:string': 'BLOB SUB_TYPE 1',
3291 'list:reference': 'BLOB SUB_TYPE 1',
3292 'big-id': 'BIGINT PRIMARY KEY',
3293 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3294 }
3295
3297 return 'genid_%s' % tablename
3298
3300 return 'trg_id_%s' % tablename
3301
3304
3305 - def EPOCH(self, first):
3306 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3307
3308 - def NOT_NULL(self,default,field_type):
3309 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3310
3312 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3313
3321
3322 - def _drop(self,table,mode):
3325
3326 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3327 if limitby:
3328 (lmin, lmax) = limitby
3329 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
3330 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3331
3333 return ['DELETE FROM %s;' % table._tablename,
3334 'SET GENERATOR %s TO 0;' % table._sequence_name]
3335
3336 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
3337
3338 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3339 credential_decoder=IDENTITY, driver_args={},
3340 adapter_args={}, do_connect=True):
3341 self.db = db
3342 self.dbengine = "firebird"
3343 self.uri = uri
3344 if do_connect: self.find_driver(adapter_args,uri)
3345 self.pool_size = pool_size
3346 self.folder = folder
3347 self.db_codec = db_codec
3348 self.find_or_make_work_folder()
3349 ruri = uri.split('://',1)[1]
3350 m = self.REGEX_URI.match(ruri)
3351 if not m:
3352 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
3353 user = credential_decoder(m.group('user'))
3354 if not user:
3355 raise SyntaxError('User required')
3356 password = credential_decoder(m.group('password'))
3357 if not password:
3358 password = ''
3359 host = m.group('host')
3360 if not host:
3361 raise SyntaxError('Host name required')
3362 port = int(m.group('port') or 3050)
3363 db = m.group('db')
3364 if not db:
3365 raise SyntaxError('Database name required')
3366 charset = m.group('charset') or 'UTF8'
3367 driver_args.update(dsn='%s/%s:%s' % (host,port,db),
3368 user = credential_decoder(user),
3369 password = credential_decoder(password),
3370 charset = charset)
3371
3372 def connector(driver_args=driver_args):
3373 return self.driver.connect(**driver_args)
3374 self.connector = connector
3375 if do_connect: self.reconnect()
3376
3385
3390
3393 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3394
3395 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
3396
3397 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3398 credential_decoder=IDENTITY, driver_args={},
3399 adapter_args={}, do_connect=True):
3400 self.db = db
3401 self.dbengine = "firebird"
3402 self.uri = uri
3403 if do_connect: self.find_driver(adapter_args,uri)
3404 self.pool_size = pool_size
3405 self.folder = folder
3406 self.db_codec = db_codec
3407 self.find_or_make_work_folder()
3408 ruri = uri.split('://',1)[1]
3409 m = self.REGEX_URI.match(ruri)
3410 if not m:
3411 raise SyntaxError(
3412 "Invalid URI string in DAL: %s" % self.uri)
3413 user = credential_decoder(m.group('user'))
3414 if not user:
3415 raise SyntaxError('User required')
3416 password = credential_decoder(m.group('password'))
3417 if not password:
3418 password = ''
3419 pathdb = m.group('path')
3420 if not pathdb:
3421 raise SyntaxError('Path required')
3422 charset = m.group('charset')
3423 if not charset:
3424 charset = 'UTF8'
3425 host = ''
3426 driver_args.update(host=host,
3427 database=pathdb,
3428 user=credential_decoder(user),
3429 password=credential_decoder(password),
3430 charset=charset)
3431
3432 def connector(driver_args=driver_args):
3433 return self.driver.connect(**driver_args)
3434 self.connector = connector
3435 if do_connect: self.reconnect()
3436
3540
3545
3548
3551
3563
3565 drivers = ('pyodbc',)
3566
3567 types = {
3568 'boolean': 'CHAR(1)',
3569 'string': 'VARCHAR(%(length)s)',
3570 'text': 'CLOB',
3571 'password': 'VARCHAR(%(length)s)',
3572 'blob': 'BLOB',
3573 'upload': 'VARCHAR(%(length)s)',
3574 'integer': 'INT',
3575 'bigint': 'BIGINT',
3576 'float': 'REAL',
3577 'double': 'DOUBLE',
3578 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3579 'date': 'DATE',
3580 'time': 'TIME',
3581 'datetime': 'TIMESTAMP',
3582 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3583 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3584 'list:integer': 'CLOB',
3585 'list:string': 'CLOB',
3586 'list:reference': 'CLOB',
3587 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3588 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3589 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3590 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3591 }
3592
3594 return 'LEFT OUTER JOIN'
3595
3598
3599 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3600 if limitby:
3601 (lmin, lmax) = limitby
3602 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
3603 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3604
3606 if fieldtype == 'blob':
3607 obj = base64.b64encode(str(obj))
3608 return "BLOB('%s')" % obj
3609 elif fieldtype == 'datetime':
3610 if isinstance(obj, datetime.datetime):
3611 obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
3612 elif isinstance(obj, datetime.date):
3613 obj = obj.isoformat()[:10]+'-00.00.00'
3614 return "'%s'" % obj
3615 return None
3616
3617 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3618 credential_decoder=IDENTITY, driver_args={},
3619 adapter_args={}, do_connect=True):
3620 self.db = db
3621 self.dbengine = "db2"
3622 self.uri = uri
3623 if do_connect: self.find_driver(adapter_args,uri)
3624 self.pool_size = pool_size
3625 self.folder = folder
3626 self.db_codec = db_codec
3627 self.find_or_make_work_folder()
3628 ruri = uri.split('://', 1)[1]
3629 def connector(cnxn=ruri,driver_args=driver_args):
3630 return self.driver.connect(cnxn,**driver_args)
3631 self.connector = connector
3632 if do_connect: self.reconnect()
3633
3635 if command[-1:]==';':
3636 command = command[:-1]
3637 return self.log_execute(command)
3638
3640 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
3641 return int(self.cursor.fetchone()[0])
3642
3643 - def rowslice(self,rows,minimum=0,maximum=None):
3644 if maximum is None:
3645 return rows[minimum:]
3646 return rows[minimum:maximum]
3647
3650 drivers = ('pyodbc',)
3651
3652 types = {
3653 'boolean': 'CHAR(1)',
3654 'string': 'VARCHAR(%(length)s)',
3655 'text': 'CLOB',
3656 'password': 'VARCHAR(%(length)s)',
3657 'blob': 'BLOB',
3658 'upload': 'VARCHAR(%(length)s)',
3659 'integer': 'INT',
3660 'bigint': 'BIGINT',
3661 'float': 'REAL',
3662 'double': 'DOUBLE',
3663 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3664 'date': 'DATE',
3665 'time': 'TIME',
3666 'datetime': 'TIMESTAMP',
3667
3668
3669 'id': 'INT GENERATED ALWAYS AS IDENTITY',
3670 'reference': 'INT',
3671 'list:integer': 'CLOB',
3672 'list:string': 'CLOB',
3673 'list:reference': 'CLOB',
3674 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',
3675 'big-reference': 'BIGINT',
3676 'reference FK': ' REFERENCES %(foreign_key)s',
3677 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
3678 }
3679
3680 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3681 credential_decoder=IDENTITY, driver_args={},
3682 adapter_args={}, do_connect=True):
3683 self.db = db
3684 self.dbengine = "teradata"
3685 self.uri = uri
3686 if do_connect: self.find_driver(adapter_args,uri)
3687 self.pool_size = pool_size
3688 self.folder = folder
3689 self.db_codec = db_codec
3690 self.find_or_make_work_folder()
3691 ruri = uri.split('://', 1)[1]
3692 def connector(cnxn=ruri,driver_args=driver_args):
3693 return self.driver.connect(cnxn,**driver_args)
3694 self.connector = connector
3695 if do_connect: self.reconnect()
3696
3698 return 'LEFT OUTER JOIN'
3699
3700
3701 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3702 if limitby:
3703 (lmin, lmax) = limitby
3704 sql_s += ' TOP %i' % lmax
3705 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3706
3708 tablename = table._tablename
3709 return ['DELETE FROM %s ALL;' % (tablename)]
3710
3711 INGRES_SEQNAME='ii***lineitemsequence'
3716 drivers = ('ingresdbi',)
3717
3718 types = {
3719 'boolean': 'CHAR(1)',
3720 'string': 'VARCHAR(%(length)s)',
3721 'text': 'CLOB',
3722 'password': 'VARCHAR(%(length)s)',
3723 'blob': 'BLOB',
3724 'upload': 'VARCHAR(%(length)s)',
3725 'integer': 'INTEGER4',
3726 'bigint': 'BIGINT',
3727 'float': 'FLOAT',
3728 'double': 'FLOAT8',
3729 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3730 'date': 'ANSIDATE',
3731 'time': 'TIME WITHOUT TIME ZONE',
3732 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
3733 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
3734 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3735 'list:integer': 'CLOB',
3736 'list:string': 'CLOB',
3737 'list:reference': 'CLOB',
3738 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
3739 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3740 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3741 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3742 }
3743
3745 return 'LEFT OUTER JOIN'
3746
3749
3750 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3751 if limitby:
3752 (lmin, lmax) = limitby
3753 fetch_amt = lmax - lmin
3754 if fetch_amt:
3755 sql_s += ' FIRST %d ' % (fetch_amt, )
3756 if lmin:
3757
3758 sql_o += ' OFFSET %d' % (lmin, )
3759 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3760
3761 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3762 credential_decoder=IDENTITY, driver_args={},
3763 adapter_args={}, do_connect=True):
3764 self.db = db
3765 self.dbengine = "ingres"
3766 self.uri = uri
3767 if do_connect: self.find_driver(adapter_args,uri)
3768 self.pool_size = pool_size
3769 self.folder = folder
3770 self.db_codec = db_codec
3771 self.find_or_make_work_folder()
3772 connstr = self._uri.split(':', 1)[1]
3773
3774 connstr = connstr.lstrip()
3775 while connstr.startswith('/'):
3776 connstr = connstr[1:]
3777 database_name=connstr
3778 vnode = '(local)'
3779 servertype = 'ingres'
3780 trace = (0, None)
3781 driver_args.update(database=database_name,
3782 vnode=vnode,
3783 servertype=servertype,
3784 trace=trace)
3785 def connector(driver_args=driver_args):
3786 return self.driver.connect(**driver_args)
3787 self.connector = connector
3788 if do_connect: self.reconnect()
3789
3791
3792
3793
3794 if hasattr(table,'_primarykey'):
3795 modify_tbl_sql = 'modify %s to btree unique on %s' % \
3796 (table._tablename,
3797 ', '.join(["'%s'" % x for x in table.primarykey]))
3798 self.execute(modify_tbl_sql)
3799 else:
3800 tmp_seqname='%s_iisq' % table._tablename
3801 query=query.replace(INGRES_SEQNAME, tmp_seqname)
3802 self.execute('create sequence %s' % tmp_seqname)
3803 self.execute(query)
3804 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
3805
3806
3808 tmp_seqname='%s_iisq' % table
3809 self.execute('select current value for %s' % tmp_seqname)
3810 return int(self.cursor.fetchone()[0])
3811
3813 return ingresdbi.IntegrityError
3814
3817
3818 drivers = ('ingresdbi',)
3819
3820 types = {
3821 'boolean': 'CHAR(1)',
3822 'string': 'NVARCHAR(%(length)s)',
3823 'text': 'NCLOB',
3824 'password': 'NVARCHAR(%(length)s)',
3825 'blob': 'BLOB',
3826 'upload': 'VARCHAR(%(length)s)',
3827 'integer': 'INTEGER4',
3828 'bigint': 'BIGINT',
3829 'float': 'FLOAT',
3830 'double': 'FLOAT8',
3831 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3832 'date': 'ANSIDATE',
3833 'time': 'TIME WITHOUT TIME ZONE',
3834 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
3835 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
3836 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3837 'list:integer': 'NCLOB',
3838 'list:string': 'NCLOB',
3839 'list:reference': 'NCLOB',
3840 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
3841 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3842 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3843 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3844 }
3845
3847 drivers = ('sapdb',)
3848
3849 support_distributed_transaction = False
3850 types = {
3851 'boolean': 'CHAR(1)',
3852 'string': 'VARCHAR(%(length)s)',
3853 'text': 'LONG',
3854 'password': 'VARCHAR(%(length)s)',
3855 'blob': 'LONG',
3856 'upload': 'VARCHAR(%(length)s)',
3857 'integer': 'INT',
3858 'bigint': 'BIGINT',
3859 'float': 'FLOAT',
3860 'double': 'DOUBLE PRECISION',
3861 'decimal': 'FIXED(%(precision)s,%(scale)s)',
3862 'date': 'DATE',
3863 'time': 'TIME',
3864 'datetime': 'TIMESTAMP',
3865 'id': 'INT PRIMARY KEY',
3866 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3867 'list:integer': 'LONG',
3868 'list:string': 'LONG',
3869 'list:reference': 'LONG',
3870 'big-id': 'BIGINT PRIMARY KEY',
3871 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3872 }
3873
3875 return '%s_id_Seq' % table
3876
3877 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3878 if limitby:
3879 (lmin, lmax) = limitby
3880 if len(sql_w) > 1:
3881 sql_w_row = sql_w + ' AND w_row > %i' % lmin
3882 else:
3883 sql_w_row = 'WHERE w_row > %i' % lmin
3884 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
3885 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3886
3888
3889 self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
3890 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
3891 % (table._tablename, table._id.name, table._sequence_name))
3892 self.execute(query)
3893
3894 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
3895
3896
3897 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3898 credential_decoder=IDENTITY, driver_args={},
3899 adapter_args={}, do_connect=True):
3900 self.db = db
3901 self.dbengine = "sapdb"
3902 self.uri = uri
3903 if do_connect: self.find_driver(adapter_args,uri)
3904 self.pool_size = pool_size
3905 self.folder = folder
3906 self.db_codec = db_codec
3907 self.find_or_make_work_folder()
3908 ruri = uri.split('://',1)[1]
3909 m = self.REGEX_URI.match(ruri)
3910 if not m:
3911 raise SyntaxError("Invalid URI string in DAL")
3912 user = credential_decoder(m.group('user'))
3913 if not user:
3914 raise SyntaxError('User required')
3915 password = credential_decoder(m.group('password'))
3916 if not password:
3917 password = ''
3918 host = m.group('host')
3919 if not host:
3920 raise SyntaxError('Host name required')
3921 db = m.group('db')
3922 if not db:
3923 raise SyntaxError('Database name required')
3924 def connector(user=user, password=password, database=db,
3925 host=host, driver_args=driver_args):
3926 return self.driver.Connection(user, password, database,
3927 host, **driver_args)
3928 self.connector = connector
3929 if do_connect: self.reconnect()
3930
3932 self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
3933 return int(self.cursor.fetchone()[0])
3934
3936 drivers = ('cubriddb',)
3937
3938 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
3939
3940 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
3941 credential_decoder=IDENTITY, driver_args={},
3942 adapter_args={}, do_connect=True):
3943 self.db = db
3944 self.dbengine = "cubrid"
3945 self.uri = uri
3946 if do_connect: self.find_driver(adapter_args,uri)
3947 self.pool_size = pool_size
3948 self.folder = folder
3949 self.db_codec = db_codec
3950 self.find_or_make_work_folder()
3951 ruri = uri.split('://',1)[1]
3952 m = self.REGEX_URI.match(ruri)
3953 if not m:
3954 raise SyntaxError(
3955 "Invalid URI string in DAL: %s" % self.uri)
3956 user = credential_decoder(m.group('user'))
3957 if not user:
3958 raise SyntaxError('User required')
3959 password = credential_decoder(m.group('password'))
3960 if not password:
3961 password = ''
3962 host = m.group('host')
3963 if not host:
3964 raise SyntaxError('Host name required')
3965 db = m.group('db')
3966 if not db:
3967 raise SyntaxError('Database name required')
3968 port = int(m.group('port') or '30000')
3969 charset = m.group('charset') or 'utf8'
3970 user = credential_decoder(user)
3971 passwd = credential_decoder(password)
3972 def connector(host=host,port=port,db=db,
3973 user=user,passwd=password,driver_args=driver_args):
3974 return self.driver.connect(host,port,db,user,passwd,**driver_args)
3975 self.connector = connector
3976 if do_connect: self.reconnect()
3977
3979 self.execute('SET FOREIGN_KEY_CHECKS=1;')
3980 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
3981
3986
3987 web2py_filesystem = False
3988
3990 return self.db._adapter.escape(obj)
3991
3993 if not db._adapter.dbengine in ('mysql', 'postgres'):
3994 raise RuntimeError("only MySQL/Postgres can store metadata .table files in database for now")
3995 self.db = db
3996 self.filename = filename
3997 self.mode = mode
3998 if not self.web2py_filesystem:
3999 if db._adapter.dbengine == 'mysql':
4000 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
4001 elif db._adapter.dbengine == 'postgres':
4002 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
4003 self.db.executesql(sql)
4004 DatabaseStoredFile.web2py_filesystem = True
4005 self.p=0
4006 self.data = ''
4007 if mode in ('r','rw','a'):
4008 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
4009 % filename
4010 rows = self.db.executesql(query)
4011 if rows:
4012 self.data = rows[0][0]
4013 elif exists(filename):
4014 datafile = open(filename, 'r')
4015 try:
4016 self.data = datafile.read()
4017 finally:
4018 datafile.close()
4019 elif mode in ('r','rw'):
4020 raise RuntimeError("File %s does not exist" % filename)
4021
4022 - def read(self, bytes):
4023 data = self.data[self.p:self.p+bytes]
4024 self.p += len(data)
4025 return data
4026
4028 i = self.data.find('\n',self.p)+1
4029 if i>0:
4030 data, self.p = self.data[self.p:i], i
4031 else:
4032 data, self.p = self.data[self.p:], len(self.data)
4033 return data
4034
4037
4039 if self.db is not None:
4040 self.db.executesql(
4041 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
4042 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
4043 % (self.filename, self.data.replace("'","''"))
4044 self.db.executesql(query)
4045 self.db.commit()
4046 self.db = None
4047
4050
4051 @staticmethod
4053 if exists(filename):
4054 return True
4055 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
4056 if db.executesql(query):
4057 return True
4058 return False
4059
4062
4065
4066 - def file_open(self, filename, mode='rb', lock=True):
4068
4071
4073 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
4074 self.db.executesql(query)
4075 self.db.commit()
4076
4078 uploads_in_blob = True
4079
4080 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
4081
4082 - def __init__(self, db, uri='google:sql://realm:domain/database',
4083 pool_size=0, folder=None, db_codec='UTF-8',
4084 credential_decoder=IDENTITY, driver_args={},
4085 adapter_args={}, do_connect=True):
4086
4087 self.db = db
4088 self.dbengine = "mysql"
4089 self.uri = uri
4090 self.pool_size = pool_size
4091 self.db_codec = db_codec
4092 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
4093 os.sep+'applications'+os.sep,1)[1])
4094 ruri = uri.split("://")[1]
4095 m = self.REGEX_URI.match(ruri)
4096 if not m:
4097 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
4098 instance = credential_decoder(m.group('instance'))
4099 self.dbstring = db = credential_decoder(m.group('db'))
4100 driver_args['instance'] = instance
4101 self.createdb = createdb = adapter_args.get('createdb',True)
4102 if not createdb:
4103 driver_args['database'] = db
4104 def connector(driver_args=driver_args):
4105 return rdbms.connect(**driver_args)
4106 self.connector = connector
4107 if do_connect: self.reconnect()
4108
4110 if self.createdb:
4111
4112 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
4113 self.execute('USE %s' % self.dbstring)
4114 self.execute("SET FOREIGN_KEY_CHECKS=1;")
4115 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4116
4117 - def execute(self, command, *a, **b):
4119
4121 can_select_for_update = False
4122
4123 @staticmethod
4125 if isinstance(obj, str):
4126 return obj.decode('utf8')
4127 elif not isinstance(obj, unicode):
4128 return unicode(obj)
4129 return obj
4130
4132 return table._id > 0
4133
4135 field_is_type = fieldtype.startswith
4136 if isinstance(obj, CALLABLETYPES):
4137 obj = obj()
4138 if isinstance(fieldtype, SQLCustomType):
4139 return fieldtype.encoder(obj)
4140 if isinstance(obj, (Expression, Field)):
4141 raise SyntaxError("non supported on GAE")
4142 if self.dbengine == 'google:datastore':
4143 if isinstance(fieldtype, gae.Property):
4144 return obj
4145 is_string = isinstance(fieldtype,str)
4146 is_list = is_string and field_is_type('list:')
4147 if is_list:
4148 if not obj:
4149 obj = []
4150 if not isinstance(obj, (list, tuple)):
4151 obj = [obj]
4152 if obj == '' and not \
4153 (is_string and fieldtype[:2] in ['st','te','pa','up']):
4154 return None
4155 if not obj is None:
4156 if isinstance(obj, list) and not is_list:
4157 obj = [self.represent(o, fieldtype) for o in obj]
4158 elif fieldtype in ('integer','bigint','id'):
4159 obj = long(obj)
4160 elif fieldtype == 'double':
4161 obj = float(obj)
4162 elif is_string and field_is_type('reference'):
4163 if isinstance(obj, (Row, Reference)):
4164 obj = obj['id']
4165 obj = long(obj)
4166 elif fieldtype == 'boolean':
4167 if obj and not str(obj)[0].upper() in '0F':
4168 obj = True
4169 else:
4170 obj = False
4171 elif fieldtype == 'date':
4172 if not isinstance(obj, datetime.date):
4173 (y, m, d) = map(int,str(obj).strip().split('-'))
4174 obj = datetime.date(y, m, d)
4175 elif isinstance(obj,datetime.datetime):
4176 (y, m, d) = (obj.year, obj.month, obj.day)
4177 obj = datetime.date(y, m, d)
4178 elif fieldtype == 'time':
4179 if not isinstance(obj, datetime.time):
4180 time_items = map(int,str(obj).strip().split(':')[:3])
4181 if len(time_items) == 3:
4182 (h, mi, s) = time_items
4183 else:
4184 (h, mi, s) = time_items + [0]
4185 obj = datetime.time(h, mi, s)
4186 elif fieldtype == 'datetime':
4187 if not isinstance(obj, datetime.datetime):
4188 (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
4189 time_items = map(int,str(obj)[11:].strip().split(':')[:3])
4190 while len(time_items)<3:
4191 time_items.append(0)
4192 (h, mi, s) = time_items
4193 obj = datetime.datetime(y, m, d, h, mi, s)
4194 elif fieldtype == 'blob':
4195 pass
4196 elif is_string and field_is_type('list:string'):
4197 return map(self.to_unicode,obj)
4198 elif is_list:
4199 return map(int,obj)
4200 else:
4201 obj = self.to_unicode(obj)
4202 return obj
4203
4205 return 'insert %s in %s' % (fields, table)
4206
4207 - def _count(self,query,distinct=None):
4208 return 'count %s' % repr(query)
4209
4210 - def _select(self,query,fields,attributes):
4211 return 'select %s where %s' % (repr(fields), repr(query))
4212
4213 - def _delete(self,tablename, query):
4214 return 'delete %s where %s' % (repr(tablename),repr(query))
4215
4216 - def _update(self,tablename,query,fields):
4217 return 'update %s (%s) where %s' % (repr(tablename),
4218 repr(fields),repr(query))
4219
4221 """
4222 remember: no transactions on many NoSQL
4223 """
4224 pass
4225
4227 """
4228 remember: no transactions on many NoSQL
4229 """
4230 pass
4231
4233 """
4234 remember: no transactions on many NoSQL
4235 """
4236 pass
4237
4238
4239
4240 - def OR(self,first,second): raise SyntaxError("Not supported")
4241 - def AND(self,first,second): raise SyntaxError("Not supported")
4242 - def AS(self,first,second): raise SyntaxError("Not supported")
4243 - def ON(self,first,second): raise SyntaxError("Not supported")
4244 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4245 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4246 - def ADD(self,first,second): raise SyntaxError("Not supported")
4247 - def SUB(self,first,second): raise SyntaxError("Not supported")
4248 - def MUL(self,first,second): raise SyntaxError("Not supported")
4249 - def DIV(self,first,second): raise SyntaxError("Not supported")
4250 - def LOWER(self,first): raise SyntaxError("Not supported")
4251 - def UPPER(self,first): raise SyntaxError("Not supported")
4253 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4254 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4255 - def RANDOM(self): raise SyntaxError("Not supported")
4256 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4257 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4258 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4259 - def drop(self,table,mode): raise SyntaxError("Not supported")
4260 - def alias(self,table,alias): raise SyntaxError("Not supported")
4261 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4263 - def prepare(self,key): raise SyntaxError("Not supported")
4266 - def concat_add(self,table): raise SyntaxError("Not supported")
4267 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4269 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4270 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4272 - def lastrowid(self,table): raise SyntaxError("Not supported")
4274 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4275
4276
4277 -class GAEF(object):
4278 - def __init__(self,name,op,value,apply):
4279 self.name=name=='id' and '__key__' or name
4280 self.op=op
4281 self.value=value
4282 self.apply=apply
4284 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4287 uploads_in_blob = True
4288 types = {}
4289
4291 - def file_open(self, filename, mode='rb', lock=True): pass
4293
4294 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
4295
4296 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4297 credential_decoder=IDENTITY, driver_args={},
4298 adapter_args={}, do_connect=True):
4299 self.types.update({
4300 'boolean': gae.BooleanProperty,
4301 'string': (lambda: gae.StringProperty(multiline=True)),
4302 'text': gae.TextProperty,
4303 'password': gae.StringProperty,
4304 'blob': gae.BlobProperty,
4305 'upload': gae.StringProperty,
4306 'integer': gae.IntegerProperty,
4307 'bigint': gae.IntegerProperty,
4308 'float': gae.FloatProperty,
4309 'double': gae.FloatProperty,
4310 'decimal': GAEDecimalProperty,
4311 'date': gae.DateProperty,
4312 'time': gae.TimeProperty,
4313 'datetime': gae.DateTimeProperty,
4314 'id': None,
4315 'reference': gae.IntegerProperty,
4316 'list:string': (lambda: gae.StringListProperty(default=None)),
4317 'list:integer': (lambda: gae.ListProperty(int,default=None)),
4318 'list:reference': (lambda: gae.ListProperty(int,default=None)),
4319 })
4320 self.db = db
4321 self.uri = uri
4322 self.dbengine = 'google:datastore'
4323 self.folder = folder
4324 db['_lastsql'] = ''
4325 self.db_codec = 'UTF-8'
4326 self.pool_size = 0
4327 match = self.REGEX_NAMESPACE.match(uri)
4328 if match:
4329 namespace_manager.set_namespace(match.group('namespace'))
4330
4331 - def parse_id(self, value, field_type):
4333
4334 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4335 myfields = {}
4336 for field in table:
4337 if isinstance(polymodel,Table) and field.name in polymodel.fields():
4338 continue
4339 attr = {}
4340 field_type = field.type
4341 if isinstance(field_type, SQLCustomType):
4342 ftype = self.types[field_type.native or field_type.type](**attr)
4343 elif isinstance(field_type, gae.Property):
4344 ftype = field_type
4345 elif field_type.startswith('id'):
4346 continue
4347 elif field_type.startswith('decimal'):
4348 precision, scale = field_type[7:].strip('()').split(',')
4349 precision = int(precision)
4350 scale = int(scale)
4351 ftype = GAEDecimalProperty(precision, scale, **attr)
4352 elif field_type.startswith('reference'):
4353 if field.notnull:
4354 attr = dict(required=True)
4355 referenced = field_type[10:].strip()
4356 ftype = self.types[field_type[:9]](referenced)
4357 elif field_type.startswith('list:reference'):
4358 if field.notnull:
4359 attr = dict(required=True)
4360 referenced = field_type[15:].strip()
4361 ftype = self.types[field_type[:14]](**attr)
4362 elif field_type.startswith('list:'):
4363 ftype = self.types[field_type](**attr)
4364 elif not field_type in self.types\
4365 or not self.types[field_type]:
4366 raise SyntaxError('Field: unknown field type: %s' % field_type)
4367 else:
4368 ftype = self.types[field_type](**attr)
4369 myfields[field.name] = ftype
4370 if not polymodel:
4371 table._tableobj = classobj(table._tablename, (gae.Model, ), myfields)
4372 elif polymodel==True:
4373 table._tableobj = classobj(table._tablename, (PolyModel, ), myfields)
4374 elif isinstance(polymodel,Table):
4375 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
4376 else:
4377 raise SyntaxError("polymodel must be None, True, a table or a tablename")
4378 return None
4379
4380 - def expand(self,expression,field_type=None):
4381 if isinstance(expression,Field):
4382 if expression.type in ('text','blob'):
4383 raise SyntaxError('AppEngine does not index by: %s' % expression.type)
4384 return expression.name
4385 elif isinstance(expression, (Expression, Query)):
4386 if not expression.second is None:
4387 return expression.op(expression.first, expression.second)
4388 elif not expression.first is None:
4389 return expression.op(expression.first)
4390 else:
4391 return expression.op()
4392 elif field_type:
4393 return self.represent(expression,field_type)
4394 elif isinstance(expression,(list,tuple)):
4395 return ','.join([self.represent(item,field_type) for item in expression])
4396 else:
4397 return str(expression)
4398
4399
4400 - def AND(self,first,second):
4406
4407 - def EQ(self,first,second=None):
4408 if isinstance(second, Key):
4409 return [GAEF(first.name,'=',second,lambda a,b:a==b)]
4410 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4411
4412 - def NE(self,first,second=None):
4413 if first.type != 'id':
4414 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
4415 else:
4416 if not second is None:
4417 second = Key.from_path(first._tablename, long(second))
4418 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4419
4420 - def LT(self,first,second=None):
4421 if first.type != 'id':
4422 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
4423 else:
4424 second = Key.from_path(first._tablename, long(second))
4425 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4426
4427 - def LE(self,first,second=None):
4428 if first.type != 'id':
4429 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
4430 else:
4431 second = Key.from_path(first._tablename, long(second))
4432 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4433
4434 - def GT(self,first,second=None):
4435 if first.type != 'id' or second==0 or second == '0':
4436 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
4437 else:
4438 second = Key.from_path(first._tablename, long(second))
4439 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4440
4441 - def GE(self,first,second=None):
4442 if first.type != 'id':
4443 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
4444 else:
4445 second = Key.from_path(first._tablename, long(second))
4446 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4447
4450
4451 - def COMMA(self,first,second):
4453
4454 - def BELONGS(self,first,second=None):
4455 if not isinstance(second,(list, tuple)):
4456 raise SyntaxError("Not supported")
4457 if first.type != 'id':
4458 return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
4459 else:
4460 second = [Key.from_path(first._tablename, int(i)) for i in second]
4461 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4462
4467
4468 - def NOT(self,first):
4469 nops = { self.EQ: self.NE,
4470 self.NE: self.EQ,
4471 self.LT: self.GE,
4472 self.GT: self.LE,
4473 self.LE: self.GT,
4474 self.GE: self.LT}
4475 if not isinstance(first,Query):
4476 raise SyntaxError("Not suported")
4477 nop = nops.get(first.op,None)
4478 if not nop:
4479 raise SyntaxError("Not suported %s" % first.op.__name__)
4480 first.op = nop
4481 return self.expand(first)
4482
4485
4486 - def select_raw(self,query,fields=None,attributes=None):
4487 db = self.db
4488 fields = fields or []
4489 attributes = attributes or {}
4490 args_get = attributes.get
4491 new_fields = []
4492 for item in fields:
4493 if isinstance(item,SQLALL):
4494 new_fields += item._table
4495 else:
4496 new_fields.append(item)
4497 fields = new_fields
4498 if query:
4499 tablename = self.get_table(query)
4500 elif fields:
4501 tablename = fields[0].tablename
4502 query = db._adapter.id_query(fields[0].table)
4503 else:
4504 raise SyntaxError("Unable to determine a tablename")
4505
4506 if query:
4507 if use_common_filters(query):
4508 query = self.common_filter(query,[tablename])
4509
4510
4511 tableobj = db[tablename]._tableobj
4512 filters = self.expand(query)
4513
4514 projection = None
4515 if len(db[tablename].fields) == len(fields):
4516
4517 projection = None
4518 elif args_get('projection') == True:
4519 projection = []
4520 for f in fields:
4521 if f.type in ['text', 'blob']:
4522 raise SyntaxError(
4523 "text and blob field types not allowed in projection queries")
4524 else:
4525 projection.append(f.name)
4526
4527
4528
4529 query_projection = [
4530 p for p in projection if \
4531 p != db[tablename]._id.name] if projection \
4532 else None
4533
4534 cursor = None
4535 if isinstance(args_get('reusecursor'), str):
4536 cursor = args_get('reusecursor')
4537 items = gae.Query(tableobj, projection=query_projection,
4538 cursor=cursor)
4539
4540 for filter in filters:
4541 if args_get('projection') == True and \
4542 filter.name in query_projection and \
4543 filter.op in ['=', '<=', '>=']:
4544 raise SyntaxError(
4545 "projection fields cannot have equality filters")
4546 if filter.name=='__key__' and filter.op=='>' and filter.value==0:
4547 continue
4548 elif filter.name=='__key__' and filter.op=='=':
4549 if filter.value==0:
4550 items = []
4551 elif isinstance(filter.value, Key):
4552
4553
4554
4555 item = tableobj.get(filter.value)
4556 items = (item and [item]) or []
4557 else:
4558
4559
4560
4561 item = tableobj.get_by_id(filter.value)
4562 items = (item and [item]) or []
4563 elif isinstance(items,list):
4564 items = [i for i in items if filter.apply(
4565 getattr(item,filter.name),filter.value)]
4566 else:
4567 if filter.name=='__key__' and filter.op != 'in':
4568 items.order('__key__')
4569 items = items.filter('%s %s' % (filter.name,filter.op),
4570 filter.value)
4571 if not isinstance(items,list):
4572 if args_get('left', None):
4573 raise SyntaxError('Set: no left join in appengine')
4574 if args_get('groupby', None):
4575 raise SyntaxError('Set: no groupby in appengine')
4576 orderby = args_get('orderby', False)
4577 if orderby:
4578
4579 if isinstance(orderby, (list, tuple)):
4580 orderby = xorify(orderby)
4581 if isinstance(orderby,Expression):
4582 orderby = self.expand(orderby)
4583 orders = orderby.split(', ')
4584 for order in orders:
4585 order={'-id':'-__key__','id':'__key__'}.get(order,order)
4586 items = items.order(order)
4587 if args_get('limitby', None):
4588 (lmin, lmax) = attributes['limitby']
4589 (limit, offset) = (lmax - lmin, lmin)
4590 rows = items.fetch(limit,offset=offset)
4591
4592
4593 if args_get('reusecursor'):
4594 db['_lastcursor'] = items.cursor()
4595 items = rows
4596 return (items, tablename, projection or db[tablename].fields)
4597
4598 - def select(self,query,fields,attributes):
4599 """
4600 This is the GAE version of select. some notes to consider:
4601 - db['_lastsql'] is not set because there is not SQL statement string
4602 for a GAE query
4603 - 'nativeRef' is a magical fieldname used for self references on GAE
4604 - optional attribute 'projection' when set to True will trigger
4605 use of the GAE projection queries. note that there are rules for
4606 what is accepted imposed by GAE: each field must be indexed,
4607 projection queries cannot contain blob or text fields, and you
4608 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
4609 - optional attribute 'reusecursor' allows use of cursor with queries
4610 that have the limitby attribute. Set the attribute to True for the
4611 first query, set it to the value of db['_lastcursor'] to continue
4612 a previous query. The user must save the cursor value between
4613 requests, and the filters must be identical. It is up to the user
4614 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
4615 """
4616
4617 (items, tablename, fields) = self.select_raw(query,fields,attributes)
4618
4619 rows = [[(t==self.db[tablename]._id.name and item) or \
4620 (t=='nativeRef' and item) or getattr(item, t) \
4621 for t in fields] for item in items]
4622 colnames = ['%s.%s' % (tablename, t) for t in fields]
4623 processor = attributes.get('processor',self.parse)
4624 return processor(rows,fields,colnames,False)
4625
4626 - def count(self,query,distinct=None,limit=None):
4627 if distinct:
4628 raise RuntimeError("COUNT DISTINCT not supported")
4629 (items, tablename, fields) = self.select_raw(query)
4630
4631 try:
4632 return len(items)
4633 except TypeError:
4634 return items.count(limit=limit)
4635
4636 - def delete(self,tablename, query):
4637 """
4638 This function was changed on 2010-05-04 because according to
4639 http://code.google.com/p/googleappengine/issues/detail?id=3119
4640 GAE no longer supports deleting more than 1000 records.
4641 """
4642
4643 (items, tablename, fields) = self.select_raw(query)
4644
4645 if not isinstance(items,list):
4646
4647
4648 leftitems = items.fetch(1000, keys_only=True)
4649 counter = 0
4650 while len(leftitems):
4651 counter += len(leftitems)
4652 gae.delete(leftitems)
4653 leftitems = items.fetch(1000, keys_only=True)
4654 else:
4655 counter = len(items)
4656 gae.delete(items)
4657 return counter
4658
4659 - def update(self,tablename,query,update_fields):
4660
4661 (items, tablename, fields) = self.select_raw(query)
4662 counter = 0
4663 for item in items:
4664 for field, value in update_fields:
4665 setattr(item, field.name, self.represent(value,field.type))
4666 item.put()
4667 counter += 1
4668 LOGGER.info(str(counter))
4669 return counter
4670
4671 - def insert(self,table,fields):
4672 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
4673
4674 tmp = table._tableobj(**dfields)
4675 tmp.put()
4676 rid = Reference(tmp.key().id())
4677 (rid._table, rid._record, rid._gaekey) = (table, None, tmp.key())
4678 return rid
4679
4681 parsed_items = []
4682 for item in items:
4683 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
4684 parsed_items.append(table._tableobj(**dfields))
4685 gae.put(parsed_items)
4686 return True
4687
4689 return uuid.UUID(uuidv).int
4690
4692 return str(uuid.UUID(int=n))
4693
4695 drivers = ('couchdb',)
4696
4697 uploads_in_blob = True
4698 types = {
4699 'boolean': bool,
4700 'string': str,
4701 'text': str,
4702 'password': str,
4703 'blob': str,
4704 'upload': str,
4705 'integer': long,
4706 'bigint': long,
4707 'float': float,
4708 'double': float,
4709 'date': datetime.date,
4710 'time': datetime.time,
4711 'datetime': datetime.datetime,
4712 'id': long,
4713 'reference': long,
4714 'list:string': list,
4715 'list:integer': list,
4716 'list:reference': list,
4717 }
4718
4720 - def file_open(self, filename, mode='rb', lock=True): pass
4722
4723 - def expand(self,expression,field_type=None):
4724 if isinstance(expression,Field):
4725 if expression.type=='id':
4726 return "%s._id" % expression.tablename
4727 return BaseAdapter.expand(self,expression,field_type)
4728
4729 - def AND(self,first,second):
4731
4732 - def OR(self,first,second):
4734
4735 - def EQ(self,first,second):
4739
4740 - def NE(self,first,second):
4744
4745 - def COMMA(self,first,second):
4747
4749 value = NoSQLAdapter.represent(self, obj, fieldtype)
4750 if fieldtype=='id':
4751 return repr(str(int(value)))
4752 elif fieldtype in ('date','time','datetime','boolean'):
4753 return serializers.json(value)
4754 return repr(not isinstance(value,unicode) and value \
4755 or value and value.encode('utf8'))
4756
4757 - def __init__(self,db,uri='couchdb://127.0.0.1:5984',
4758 pool_size=0,folder=None,db_codec ='UTF-8',
4759 credential_decoder=IDENTITY, driver_args={},
4760 adapter_args={}, do_connect=True):
4761 self.db = db
4762 self.uri = uri
4763 if do_connect: self.find_driver(adapter_args)
4764 self.dbengine = 'couchdb'
4765 self.folder = folder
4766 db['_lastsql'] = ''
4767 self.db_codec = 'UTF-8'
4768 self.pool_size = pool_size
4769
4770 url='http://'+uri[10:]
4771 def connector(url=url,driver_args=driver_args):
4772 return self.driver.Server(url,**driver_args)
4773 self.reconnect(connector,cursor=False)
4774
4775 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
4776 if migrate:
4777 try:
4778 self.connection.create(table._tablename)
4779 except:
4780 pass
4781
4782 - def insert(self,table,fields):
4789
4790 - def _select(self,query,fields,attributes):
4791 if not isinstance(query,Query):
4792 raise SyntaxError("Not Supported")
4793 for key in set(attributes.keys())-SELECT_ARGS:
4794 raise SyntaxError('invalid select attribute: %s' % key)
4795 new_fields=[]
4796 for item in fields:
4797 if isinstance(item,SQLALL):
4798 new_fields += item._table
4799 else:
4800 new_fields.append(item)
4801 def uid(fd):
4802 return fd=='id' and '_id' or fd
4803 def get(row,fd):
4804 return fd=='id' and int(row['_id']) or row.get(fd,None)
4805 fields = new_fields
4806 tablename = self.get_table(query)
4807 fieldnames = [f.name for f in (fields or self.db[tablename])]
4808 colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
4809 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
4810 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
4811 dict(t=tablename,
4812 query=self.expand(query),
4813 order='%s._id' % tablename,
4814 fields=fields)
4815 return fn, colnames
4816
4817 - def select(self,query,fields,attributes):
4818 if not isinstance(query,Query):
4819 raise SyntaxError("Not Supported")
4820 fn, colnames = self._select(query,fields,attributes)
4821 tablename = colnames[0].split('.')[0]
4822 ctable = self.connection[tablename]
4823 rows = [cols['value'] for cols in ctable.query(fn)]
4824 processor = attributes.get('processor',self.parse)
4825 return processor(rows,fields,colnames,False)
4826
4827 - def delete(self,tablename,query):
4828 if not isinstance(query,Query):
4829 raise SyntaxError("Not Supported")
4830 if query.first.type=='id' and query.op==self.EQ:
4831 id = query.second
4832 tablename = query.first.tablename
4833 assert(tablename == query.first.tablename)
4834 ctable = self.connection[tablename]
4835 try:
4836 del ctable[str(id)]
4837 return 1
4838 except couchdb.http.ResourceNotFound:
4839 return 0
4840 else:
4841 tablename = self.get_table(query)
4842 rows = self.select(query,[self.db[tablename]._id],{})
4843 ctable = self.connection[tablename]
4844 for row in rows:
4845 del ctable[str(row.id)]
4846 return len(rows)
4847
4848 - def update(self,tablename,query,fields):
4849 if not isinstance(query,Query):
4850 raise SyntaxError("Not Supported")
4851 if query.first.type=='id' and query.op==self.EQ:
4852 id = query.second
4853 tablename = query.first.tablename
4854 ctable = self.connection[tablename]
4855 try:
4856 doc = ctable[str(id)]
4857 for key,value in fields:
4858 doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
4859 ctable.save(doc)
4860 return 1
4861 except couchdb.http.ResourceNotFound:
4862 return 0
4863 else:
4864 tablename = self.get_table(query)
4865 rows = self.select(query,[self.db[tablename]._id],{})
4866 ctable = self.connection[tablename]
4867 table = self.db[tablename]
4868 for row in rows:
4869 doc = ctable[str(row.id)]
4870 for key,value in fields:
4871 doc[key.name] = self.represent(value,table[key.name].type)
4872 ctable.save(doc)
4873 return len(rows)
4874
4875 - def count(self,query,distinct=None):
4876 if distinct:
4877 raise RuntimeError("COUNT DISTINCT not supported")
4878 if not isinstance(query,Query):
4879 raise SyntaxError("Not Supported")
4880 tablename = self.get_table(query)
4881 rows = self.select(query,[self.db[tablename]._id],{})
4882 return len(rows)
4883
4885 """
4886 validates that the given text is clean: only contains [0-9a-zA-Z_]
4887 """
4888 if not REGEX_ALPHANUMERIC.match(text):
4889 raise SyntaxError('invalid table or field name: %s' % text)
4890 return text
4891
4893 drivers = ('pymongo',)
4894
4895 uploads_in_blob = True
4896
4897 types = {
4898 'boolean': bool,
4899 'string': str,
4900 'text': str,
4901 'password': str,
4902 'blob': str,
4903 'upload': str,
4904 'integer': long,
4905 'bigint': long,
4906 'float': float,
4907 'double': float,
4908 'date': datetime.date,
4909 'time': datetime.time,
4910 'datetime': datetime.datetime,
4911 'id': long,
4912 'reference': long,
4913 'list:string': list,
4914 'list:integer': list,
4915 'list:reference': list,
4916 }
4917
4918 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
4919 pool_size=0,folder=None,db_codec ='UTF-8',
4920 credential_decoder=IDENTITY, driver_args={},
4921 adapter_args={}, do_connect=True):
4922 self.db = db
4923 self.uri = uri
4924 if do_connect: self.find_driver(adapter_args)
4925
4926 m=None
4927 try:
4928
4929 import pymongo.uri_parser
4930 m = pymongo.uri_parser.parse_uri(uri)
4931 except ImportError:
4932 try:
4933
4934 import pymongo.connection
4935 m = pymongo.connection._parse_uri(uri)
4936 except ImportError:
4937 raise ImportError("Uriparser for mongodb is not available")
4938 except:
4939 raise SyntaxError("This type of uri is not supported by the mongodb uri parser")
4940 self.dbengine = 'mongodb'
4941 self.folder = folder
4942 db['_lastsql'] = ''
4943 self.db_codec = 'UTF-8'
4944 self.pool_size = pool_size
4945
4946 self.minimumreplication = adapter_args.get('minimumreplication',0)
4947
4948
4949 self.safe = adapter_args.get('safe',True)
4950
4951
4952 if isinstance(m,tuple):
4953 m = {"database" : m[1]}
4954 if m.get('database')==None:
4955 raise SyntaxError("Database is required!")
4956 def connector(uri=self.uri,m=m):
4957 try:
4958 return self.driver.Connection(uri)[m.get('database')]
4959 except self.driver.errors.ConnectionFailure:
4960 inst = sys.exc_info()[1]
4961 raise SyntaxError("The connection to " + uri + " could not be made")
4962 except Exception:
4963 inst = sys.exc_info()[1]
4964 if inst == "cannot specify database without a username and password":
4965 raise SyntaxError("You are probebly running version 1.1 of pymongo which contains a bug which requires authentication. Update your pymongo.")
4966 else:
4967 raise SyntaxError("This is not an official Mongodb uri (http://www.mongodb.org/display/DOCS/Connections) Error : %s" % inst)
4968 self.reconnect(connector,cursor=False)
4969
4971 value = NoSQLAdapter.represent(self, obj, fieldtype)
4972 if fieldtype =='date':
4973 if value == None:
4974 return value
4975 t = datetime.time(0, 0, 0)
4976 return datetime.datetime.combine(value, t)
4977 elif fieldtype == 'time':
4978 if value == None:
4979 return value
4980 d = datetime.date(2000, 1, 1)
4981 return datetime.datetime.combine(d, value)
4982 elif fieldtype == 'list:string' or fieldtype == 'list:integer' or fieldtype == 'list:reference':
4983 return value
4984 return value
4985
4986
4987
4988 - def insert(self,table,fields,safe=None):
4995
4996 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None, isCapped=False):
4997 if isCapped:
4998 raise RuntimeError("Not implemented")
4999 else:
5000 pass
5001
5002 - def count(self,query,distinct=None,snapshot=True):
5003 if distinct:
5004 raise RuntimeError("COUNT DISTINCT not supported")
5005 if not isinstance(query,Query):
5006 raise SyntaxError("Not Supported")
5007 tablename = self.get_table(query)
5008 return int(self.select(query,[self.db[tablename]._id],{},count=True,snapshot=snapshot)['count'])
5009
5010
5011
5012 - def expand(self, expression, field_type=None):
5013 try:
5014 from pymongo.objectid import ObjectId
5015 except ImportError:
5016 from bson.objectid import ObjectId
5017
5018
5019
5020 if isinstance(expression, Query):
5021
5022
5023
5024
5025
5026 if isinstance(expression.first,Field) and \
5027 expression.first.type == 'id':
5028 expression.first.name = '_id'
5029 if expression.second != 0 and \
5030 not isinstance(expression.second,ObjectId):
5031 if isinstance(expression.second,int):
5032 try:
5033
5034
5035
5036
5037 expression.second = ObjectId(("%X" % expression.second))
5038 except:
5039 raise SyntaxError('The second argument must by an integer that can represent an objectid.')
5040 else:
5041 try:
5042
5043 expression.second = ObjectId(expression.second)
5044 except:
5045 raise SyntaxError('second argument must be of type ObjectId or an objectid representable integer')
5046 elif expression.second == 0:
5047 expression.second = ObjectId('000000000000000000000000')
5048 return expression.op(expression.first, expression.second)
5049 if isinstance(expression, Field):
5050 if expression.type=='id':
5051 return "_id"
5052 else:
5053 return expression.name
5054
5055 elif isinstance(expression, (Expression, Query)):
5056 if not expression.second is None:
5057 return expression.op(expression.first, expression.second)
5058 elif not expression.first is None:
5059 return expression.op(expression.first)
5060 elif not isinstance(expression.op, str):
5061 return expression.op()
5062 else:
5063 return expression.op
5064 elif field_type:
5065 return str(self.represent(expression,field_type))
5066 elif isinstance(expression,(list,tuple)):
5067 return ','.join(self.represent(item,field_type) for item in expression)
5068 else:
5069 return expression
5070
5071 - def _select(self,query,fields,attributes):
5072 try:
5073 from bson.son import SON
5074 except ImportError:
5075 from pymongo.son import SON
5076
5077 if 'for_update' in attributes:
5078 logging.warn('mongodb does not support for_update')
5079 for key in set(attributes.keys())-set(('limitby','orderby','for_update')):
5080 if attributes[key]!=None:
5081 raise SyntaxError('invalid select attribute: %s' % key)
5082
5083 new_fields=[]
5084 mongosort_list = []
5085
5086
5087 orderby = attributes.get('orderby', False)
5088 limitby = attributes.get('limitby', False)
5089
5090 if orderby:
5091
5092 if isinstance(orderby, (list, tuple)):
5093 orderby = xorify(orderby)
5094
5095
5096 for f in self.expand(orderby).split(','):
5097 if f.startswith('-'):
5098 mongosort_list.append((f[1:],-1))
5099 else:
5100 mongosort_list.append((f,1))
5101
5102 if limitby:
5103 limitby_skip, limitby_limit = limitby
5104 else:
5105 limitby_skip = limitby_limit = 0
5106
5107 mongofields_dict = SON()
5108 mongoqry_dict = {}
5109 for item in fields:
5110 if isinstance(item,SQLALL):
5111 new_fields += item._table
5112 else:
5113 new_fields.append(item)
5114 fields = new_fields
5115 if isinstance(query,Query):
5116 tablename = self.get_table(query)
5117 elif len(fields) != 0:
5118 tablename = fields[0].tablename
5119 else:
5120 raise SyntaxError("The table name could not be found in the query nor from the select statement.")
5121 mongoqry_dict = self.expand(query)
5122 fields = fields or self.db[tablename]
5123 for field in fields:
5124 mongofields_dict[field.name] = 1
5125 return tablename, mongoqry_dict, mongofields_dict, \
5126 mongosort_list, limitby_limit, limitby_skip
5127
5128
5129
5130 - def select(self,query,fields,attributes,count=False,snapshot=False):
5131 try:
5132 from pymongo.objectid import ObjectId
5133 except ImportError:
5134 from bson.objectid import ObjectId
5135 tablename, mongoqry_dict, mongofields_dict, \
5136 mongosort_list, limitby_limit, limitby_skip = \
5137 self._select(query,fields,attributes)
5138 ctable = self.connection[tablename]
5139 if count:
5140 return {'count' : ctable.find(
5141 mongoqry_dict, mongofields_dict,
5142 skip=limitby_skip, limit=limitby_limit,
5143 sort=mongosort_list, snapshot=snapshot).count()}
5144 else:
5145 mongo_list_dicts = ctable.find(
5146 mongoqry_dict, mongofields_dict,
5147 skip=limitby_skip, limit=limitby_limit,
5148 sort=mongosort_list, snapshot=snapshot)
5149
5150 rows = []
5151
5152 colnames = [str(field) for field in fields]
5153 for k,record in enumerate(mongo_list_dicts):
5154 row=[]
5155 for fullcolname in colnames:
5156 colname = fullcolname.split('.')[1]
5157 column = '_id' if colname=='id' else colname
5158 if column in record:
5159 if column == '_id' and isinstance(
5160 record[column],ObjectId):
5161 value = int(str(record[column]),16)
5162 elif column != '_id':
5163 value = record[column]
5164 else:
5165 value = None
5166 else:
5167 value = None
5168 row.append(value)
5169 rows.append(row)
5170 processor = attributes.get('processor',self.parse)
5171 return processor(rows,fields,colnames,False)
5172
5176
5177 - def drop(self, table, mode=''):
5180
5181
5182 - def truncate(self,table,mode,safe=None):
5183 if safe==None:
5184 safe=self.safe
5185 ctable = self.connection[table._tablename]
5186 ctable.remove(None, safe=True)
5187
5188
5189 - def oupdate(self,tablename,query,fields):
5190 if not isinstance(query,Query):
5191 raise SyntaxError("Not Supported")
5192 filter = None
5193 if query:
5194 filter = self.expand(query)
5195 f_v = []
5196
5197
5198 modify = { '$set' : dict(((k.name,self.represent(v,k.type)) for k,v in fields)) }
5199 return modify,filter
5200
5201
5202
5203
5204
5205 - def update(self,tablename,query,fields,safe=None):
5206 if safe==None:
5207 safe=self.safe
5208
5209 if not isinstance(query,Query):
5210 raise RuntimeError("Not implemented")
5211 amount = self.count(query,False)
5212 modify,filter = self.oupdate(tablename,query,fields)
5213 try:
5214 if safe:
5215 return self.connection[tablename].update(filter,modify,multi=True,safe=safe).n
5216 else:
5217 amount =self.count(query)
5218 self.connection[tablename].update(filter,modify,multi=True,safe=safe)
5219 return amount
5220 except:
5221
5222 return 0
5223 """
5224 An special update operator that enables the update of specific field
5225 return a dict
5226 """
5227
5228
5229
5230
5231 - def _update(self,tablename,query,fields):
5233
5235 return [self.insert(table,item) for item in items]
5236
5237
5238 - def NOT(self, first):
5239 result = {}
5240 result["$not"] = self.expand(first)
5241 return result
5242
5243 - def AND(self,first,second):
5248
5249 - def OR(self,first,second):
5250
5251 result = {}
5252 f = self.expand(first)
5253 s = self.expand(second)
5254 result['$or'] = [f,s]
5255 return result
5256
5257 - def BELONGS(self, first, second):
5258 if isinstance(second, str):
5259 return {self.expand(first) : {"$in" : [ second[:-1]]} }
5260 elif second==[] or second==():
5261 return {1:0}
5262 items = [self.expand(item, first.type) for item in second]
5263 return {self.expand(first) : {"$in" : items} }
5264
5265 - def EQ(self,first,second):
5266 result = {}
5267
5268
5269
5270 result[self.expand(first)] = self.expand(second)
5271 return result
5272
5273 - def NE(self, first, second=None):
5274 result = {}
5275 result[self.expand(first)] = {'$ne': self.expand(second)}
5276 return result
5277
5278 - def LT(self,first,second=None):
5279 if second is None:
5280 raise RuntimeError("Cannot compare %s < None" % first)
5281 result = {}
5282 result[self.expand(first)] = {'$lt': self.expand(second)}
5283 return result
5284
5285 - def LE(self,first,second=None):
5286 if second is None:
5287 raise RuntimeError("Cannot compare %s <= None" % first)
5288 result = {}
5289 result[self.expand(first)] = {'$lte': self.expand(second)}
5290 return result
5291
5292 - def GT(self,first,second):
5293 result = {}
5294 result[self.expand(first)] = {'$gt': self.expand(second)}
5295 return result
5296
5297 - def GE(self,first,second=None):
5298 if second is None:
5299 raise RuntimeError("Cannot compare %s >= None" % first)
5300 result = {}
5301 result[self.expand(first)] = {'$gte': self.expand(second)}
5302 return result
5303
5304 - def ADD(self, first, second):
5305 raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry")
5306 return '%s + %s' % (self.expand(first), self.expand(second, first.type))
5307
5308 - def SUB(self, first, second):
5309 raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry")
5310 return '(%s - %s)' % (self.expand(first), self.expand(second, first.type))
5311
5312 - def MUL(self, first, second):
5313 raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry")
5314 return '(%s * %s)' % (self.expand(first), self.expand(second, first.type))
5315
5316 - def DIV(self, first, second):
5317 raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry")
5318 return '(%s / %s)' % (self.expand(first), self.expand(second, first.type))
5319
5320 - def MOD(self, first, second):
5321 raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry")
5322 return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type))
5323
5324 - def AS(self, first, second):
5325 raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry")
5326 return '%s AS %s' % (self.expand(first), second)
5327
5328
5329 - def ON(self, first, second):
5330 raise NotImplementedError("This is not possible in NoSQL, but can be simulated with a wrapper.")
5331 return '%s ON %s' % (self.expand(first), self.expand(second))
5332
5333
5334
5335
5336
5337
5338 - def COMMA(self, first, second):
5340
5341 - def LIKE(self, first, second):
5344
5348
5352
5356
5357 - def LIKE(self, first, second):
5360
5361
5366
5367
5369
5370
5371 import re
5372 return {self.expand(first) : {'$regex' : re.escape(self.expand(second, 'string')) + '$'}}
5373
5374
5379
5385 drivers = ('imaplib',)
5386
5387 """ IMAP server adapter
5388
5389 This class is intended as an interface with
5390 email IMAP servers to perform simple queries in the
5391 web2py DAL query syntax, so email read, search and
5392 other related IMAP mail services (as those implemented
5393 by brands like Google(r), and Yahoo!(r)
5394 can be managed from web2py applications.
5395
5396 The code uses examples by Yuji Tomita on this post:
5397 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
5398 and is based in docs for Python imaplib, python email
5399 and email IETF's (i.e. RFC2060 and RFC3501)
5400
5401 This adapter was tested with a small set of operations with Gmail(r). Other
5402 services requests could raise command syntax and response data issues.
5403
5404 It creates its table and field names "statically",
5405 meaning that the developer should leave the table and field
5406 definitions to the DAL instance by calling the adapter's
5407 .define_tables() method. The tables are defined with the
5408 IMAP server mailbox list information.
5409
5410 .define_tables() returns a dictionary mapping dal tablenames
5411 to the server mailbox names with the following structure:
5412
5413 {<tablename>: str <server mailbox name>}
5414
5415 Here is a list of supported fields:
5416
5417 Field Type Description
5418 ################################################################
5419 uid string
5420 answered boolean Flag
5421 created date
5422 content list:string A list of text or html parts
5423 to string
5424 cc string
5425 bcc string
5426 size integer the amount of octets of the message*
5427 deleted boolean Flag
5428 draft boolean Flag
5429 flagged boolean Flag
5430 sender string
5431 recent boolean Flag
5432 seen boolean Flag
5433 subject string
5434 mime string The mime header declaration
5435 email string The complete RFC822 message**
5436 attachments list:string Each non text decoded part as string
5437
5438 *At the application side it is measured as the length of the RFC822
5439 message string
5440
5441 WARNING: As row id's are mapped to email sequence numbers,
5442 make sure your imap client web2py app does not delete messages
5443 during select or update actions, to prevent
5444 updating or deleting different messages.
5445 Sequence numbers change whenever the mailbox is updated.
5446 To avoid this sequence numbers issues, it is recommended the use
5447 of uid fields in query references (although the update and delete
5448 in separate actions rule still applies).
5449
5450 # This is the code recommended to start imap support
5451 # at the app's model:
5452
5453 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
5454 imapdb.define_tables()
5455
5456 Here is an (incomplete) list of possible imap commands:
5457
5458 # Count today's unseen messages
5459 # smaller than 6000 octets from the
5460 # inbox mailbox
5461
5462 q = imapdb.INBOX.seen == False
5463 q &= imapdb.INBOX.created == datetime.date.today()
5464 q &= imapdb.INBOX.size < 6000
5465 unread = imapdb(q).count()
5466
5467 # Fetch last query messages
5468 rows = imapdb(q).select()
5469
5470 # it is also possible to filter query select results with limitby and
5471 # sequences of mailbox fields
5472
5473 set.select(<fields sequence>, limitby=(<int>, <int>))
5474
5475 # Mark last query messages as seen
5476 messages = [row.uid for row in rows]
5477 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
5478
5479 # Delete messages in the imap database that have mails from mr. Gumby
5480
5481 deleted = 0
5482 for mailbox in imapdb.tables
5483 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
5484
5485 # It is possible also to mark messages for deletion instead of ereasing them
5486 # directly with set.update(deleted=True)
5487
5488
5489 # This object give access
5490 # to the adapter auto mailbox
5491 # mapped names (which native
5492 # mailbox has what table name)
5493
5494 db.mailboxes <dict> # tablename, server native name pairs
5495
5496 # To retrieve a table native mailbox name use:
5497 db.<table>.mailbox
5498
5499 """
5500
5501 types = {
5502 'string': str,
5503 'text': str,
5504 'date': datetime.date,
5505 'datetime': datetime.datetime,
5506 'id': long,
5507 'boolean': bool,
5508 'integer': int,
5509 'bigint': long,
5510 'blob': str,
5511 'list:string': str,
5512 }
5513
5514 dbengine = 'imap'
5515
5516 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')
5517
5518 - def __init__(self,
5519 db,
5520 uri,
5521 pool_size=0,
5522 folder=None,
5523 db_codec ='UTF-8',
5524 credential_decoder=IDENTITY,
5525 driver_args={},
5526 adapter_args={}, do_connect=True):
5527
5528
5529
5530
5531 self.db = db
5532 self.uri = uri
5533 if do_connect: self.find_driver(adapter_args)
5534 self.pool_size=pool_size
5535 self.folder = folder
5536 self.db_codec = db_codec
5537 self.credential_decoder = credential_decoder
5538 self.driver_args = driver_args
5539 self.adapter_args = adapter_args
5540 self.mailbox_size = None
5541 self.charset = sys.getfilesystemencoding()
5542
5543 self.imap4 = None
5544 uri = uri.split("://")[1]
5545
5546 """ MESSAGE is an identifier for sequence number"""
5547
5548 self.flags = ['\\Deleted', '\\Draft', '\\Flagged',
5549 '\\Recent', '\\Seen', '\\Answered']
5550 self.search_fields = {
5551 'id': 'MESSAGE', 'created': 'DATE',
5552 'uid': 'UID', 'sender': 'FROM',
5553 'to': 'TO', 'cc': 'CC',
5554 'bcc': 'BCC', 'content': 'TEXT',
5555 'size': 'SIZE', 'deleted': '\\Deleted',
5556 'draft': '\\Draft', 'flagged': '\\Flagged',
5557 'recent': '\\Recent', 'seen': '\\Seen',
5558 'subject': 'SUBJECT', 'answered': '\\Answered',
5559 'mime': None, 'email': None,
5560 'attachments': None
5561 }
5562
5563 db['_lastsql'] = ''
5564
5565 m = self.REGEX_URI.match(uri)
5566 user = m.group('user')
5567 password = m.group('password')
5568 host = m.group('host')
5569 port = int(m.group('port'))
5570 over_ssl = False
5571 if port==993:
5572 over_ssl = True
5573
5574 driver_args.update(host=host,port=port, password=password, user=user)
5575 def connector(driver_args=driver_args):
5576
5577
5578 if over_ssl:
5579 self.imap4 = self.driver.IMAP4_SSL
5580 else:
5581 self.imap4 = self.driver.IMAP4
5582 connection = self.imap4(driver_args["host"], driver_args["port"])
5583 data = connection.login(driver_args["user"], driver_args["password"])
5584
5585
5586 connection.mailbox_names = None
5587
5588
5589 connection.cursor = lambda : True
5590
5591 return connection
5592
5593 self.db.define_tables = self.define_tables
5594 self.connector = connector
5595 if do_connect: self.reconnect()
5596
5641
5643 last_message = None
5644
5645
5646 if not isinstance(self.connection.mailbox_names, dict):
5647 self.get_mailboxes()
5648 try:
5649 result = self.connection.select(self.connection.mailbox_names[tablename])
5650 last_message = int(result[1][0])
5651 except (IndexError, ValueError, TypeError, KeyError):
5652 e = sys.exc_info()[1]
5653 LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e))
5654 return last_message
5655
5657 if not isinstance(self.connection.mailbox_names, dict):
5658 self.get_mailboxes()
5659
5660
5661 last_message = self.get_last_message(tablename)
5662 result, data = self.connection.uid("search", None, "(ALL)")
5663 uid_list = data[0].strip().split()
5664 if len(uid_list) <= 0:
5665 return None
5666 else:
5667 return (uid_list[0], uid_list[-1])
5668
5670 if add is None:
5671 add = datetime.timedelta()
5672 """ Convert a date object to a string
5673 with d-Mon-Y style for IMAP or the inverse
5674 case
5675
5676 add <timedelta> adds to the date object
5677 """
5678 months = [None, "Jan","Feb","Mar","Apr","May","Jun",
5679 "Jul", "Aug","Sep","Oct","Nov","Dec"]
5680 if isinstance(date, basestring):
5681
5682 try:
5683 dayname, datestring = date.split(",")
5684 except (ValueError):
5685 LOGGER.debug("Could not parse date text: %s" % date)
5686 return None
5687 date_list = datestring.strip().split()
5688 year = int(date_list[2])
5689 month = months.index(date_list[1])
5690 day = int(date_list[0])
5691 hms = map(int, date_list[3].split(":"))
5692 return datetime.datetime(year, month, day,
5693 hms[0], hms[1], hms[2]) + add
5694 elif isinstance(date, (datetime.datetime, datetime.date)):
5695 return (date + add).strftime("%d-%b-%Y")
5696
5697 else:
5698 return None
5699
5700 - def encode_text(self, text, charset, errors="replace"):
5701 """ convert text for mail to unicode"""
5702 if text is None:
5703 text = ""
5704 else:
5705 if isinstance(text, str):
5706 if charset is not None:
5707 text = unicode(text, charset, errors)
5708 else:
5709 text = unicode(text, "utf-8", errors)
5710 else:
5711 raise Exception("Unsupported mail text type %s" % type(text))
5712 return text.encode("utf-8")
5713
5715 charset = message.get_content_charset()
5716 return charset
5717
5721
5723 """ Query the mail database for mailbox names """
5724 mailboxes_list = self.connection.list()
5725 self.connection.mailbox_names = dict()
5726 mailboxes = list()
5727 x = 0
5728 for item in mailboxes_list[1]:
5729 x = x + 1
5730 item = item.strip()
5731 if not "NOSELECT" in item.upper():
5732 sub_items = item.split("\"")
5733 sub_items = [sub_item for sub_item in sub_items \
5734 if len(sub_item.strip()) > 0]
5735 mailbox = sub_items[len(sub_items) - 1]
5736
5737
5738 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
5739 mailboxes.append(mailbox_name)
5740 self.connection.mailbox_names[mailbox_name] = mailbox
5741
5742 return mailboxes
5743
5745 nofield = True
5746 tablename = None
5747 attr = query
5748 while nofield:
5749 if hasattr(attr, "first"):
5750 attr = attr.first
5751 if isinstance(attr, Field):
5752 return attr.tablename
5753 elif isinstance(attr, Query):
5754 pass
5755 else:
5756 return None
5757 else:
5758 return None
5759 return tablename
5760
5762 if self.search_fields.get(flag, None) in self.flags:
5763 return True
5764 else:
5765 return False
5766
5768 """
5769 Auto create common IMAP fileds
5770
5771 This function creates fields definitions "statically"
5772 meaning that custom fields as in other adapters should
5773 not be supported and definitions handled on a service/mode
5774 basis (local syntax for Gmail(r), Ymail(r)
5775
5776 Returns a dictionary with tablename, server native mailbox name
5777 pairs.
5778 """
5779 if not isinstance(self.connection.mailbox_names, dict):
5780 self.get_mailboxes()
5781 mailboxes = self.connection.mailbox_names.keys()
5782 for mailbox_name in mailboxes:
5783 self.db.define_table("%s" % mailbox_name,
5784 Field("uid", "string", writable=False),
5785 Field("answered", "boolean"),
5786 Field("created", "datetime", writable=False),
5787 Field("content", "list:string", writable=False),
5788 Field("to", "string", writable=False),
5789 Field("cc", "string", writable=False),
5790 Field("bcc", "string", writable=False),
5791 Field("size", "integer", writable=False),
5792 Field("deleted", "boolean"),
5793 Field("draft", "boolean"),
5794 Field("flagged", "boolean"),
5795 Field("sender", "string", writable=False),
5796 Field("recent", "boolean", writable=False),
5797 Field("seen", "boolean"),
5798 Field("subject", "string", writable=False),
5799 Field("mime", "string", writable=False),
5800 Field("email", "string", writable=False, readable=False),
5801 Field("attachments", "list:string", writable=False, readable=False),
5802 )
5803
5804
5805
5806 self.db[mailbox_name].mailbox = \
5807 self.connection.mailbox_names[mailbox_name]
5808
5809
5810 self.db.mailboxes = self.connection.mailbox_names
5811 return self.db.mailboxes
5812
5814
5815 LOGGER.debug("Create table feature is not implemented for %s" % type(self))
5816
5817 - def _select(self,query,fields,attributes):
5818 """ Search and Fetch records and return web2py
5819 rows
5820 """
5821
5822 if use_common_filters(query):
5823 query = self.common_filter(query, [self.get_query_mailbox(query),])
5824
5825
5826 import email
5827 import email.header
5828 decode_header = email.header.decode_header
5829
5830
5831 tablename = None
5832 fetch_results = list()
5833 if isinstance(query, (Expression, Query)):
5834 tablename = self.get_table(query)
5835 mailbox = self.connection.mailbox_names.get(tablename, None)
5836 if isinstance(query, Expression):
5837 pass
5838 elif isinstance(query, Query):
5839 if mailbox is not None:
5840
5841 selected = self.connection.select(mailbox, True)
5842 self.mailbox_size = int(selected[1][0])
5843 search_query = "(%s)" % str(query).strip()
5844 search_result = self.connection.uid("search", None, search_query)
5845
5846 if search_result[0] == "OK":
5847
5848
5849
5850
5851
5852 limitby = attributes.get('limitby', None)
5853 messages_set = search_result[1][0].split()
5854
5855 messages_set.reverse()
5856 if limitby is not None:
5857
5858 messages_set = messages_set[int(limitby[0]):int(limitby[1])]
5859
5860
5861
5862 imap_fields = "(RFC822)"
5863 if len(messages_set) > 0:
5864
5865
5866
5867
5868 for uid in messages_set:
5869
5870 typ, data = self.connection.uid("fetch", uid, imap_fields)
5871 if typ == "OK":
5872 fr = {"message": int(data[0][0].split()[0]),
5873 "uid": int(uid),
5874 "email": email.message_from_string(data[0][1]),
5875 "raw_message": data[0][1]
5876 }
5877 fr["multipart"] = fr["email"].is_multipart()
5878
5879 ftyp, fdata = self.connection.uid("fetch", uid, "(FLAGS)")
5880 if ftyp == "OK":
5881 fr["flags"] = self.driver.ParseFlags(fdata[0])
5882 fetch_results.append(fr)
5883 else:
5884
5885 pass
5886 else:
5887
5888 pass
5889
5890 elif isinstance(query, basestring):
5891
5892 pass
5893 else:
5894 pass
5895
5896 imapqry_dict = {}
5897 imapfields_dict = {}
5898
5899 if len(fields) == 1 and isinstance(fields[0], SQLALL):
5900 allfields = True
5901 elif len(fields) == 0:
5902 allfields = True
5903 else:
5904 allfields = False
5905 if allfields:
5906 fieldnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
5907 else:
5908 fieldnames = ["%s.%s" % (tablename, field.name) for field in fields]
5909
5910 for k in fieldnames:
5911 imapfields_dict[k] = k
5912
5913 imapqry_list = list()
5914 imapqry_array = list()
5915 for fr in fetch_results:
5916 attachments = []
5917 content = []
5918 size = 0
5919 n = int(fr["message"])
5920 item_dict = dict()
5921 message = fr["email"]
5922 uid = fr["uid"]
5923 charset = self.get_charset(message)
5924 flags = fr["flags"]
5925 raw_message = fr["raw_message"]
5926
5927
5928
5929
5930
5931
5932
5933
5934 if "%s.id" % tablename in fieldnames:
5935 item_dict["%s.id" % tablename] = n
5936 if "%s.created" % tablename in fieldnames:
5937 item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
5938 if "%s.uid" % tablename in fieldnames:
5939 item_dict["%s.uid" % tablename] = uid
5940 if "%s.sender" % tablename in fieldnames:
5941
5942
5943
5944
5945
5946
5947 item_dict["%s.sender" % tablename] = self.encode_text(message["From"], charset)
5948 if "%s.to" % tablename in fieldnames:
5949 item_dict["%s.to" % tablename] = self.encode_text(message["To"], charset)
5950 if "%s.cc" % tablename in fieldnames:
5951 if "Cc" in message.keys():
5952 item_dict["%s.cc" % tablename] = self.encode_text(message["Cc"], charset)
5953 else:
5954 item_dict["%s.cc" % tablename] = ""
5955 if "%s.bcc" % tablename in fieldnames:
5956 if "Bcc" in message.keys():
5957 item_dict["%s.bcc" % tablename] = self.encode_text(message["Bcc"], charset)
5958 else:
5959 item_dict["%s.bcc" % tablename] = ""
5960 if "%s.deleted" % tablename in fieldnames:
5961 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
5962 if "%s.draft" % tablename in fieldnames:
5963 item_dict["%s.draft" % tablename] = "\\Draft" in flags
5964 if "%s.flagged" % tablename in fieldnames:
5965 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
5966 if "%s.recent" % tablename in fieldnames:
5967 item_dict["%s.recent" % tablename] = "\\Recent" in flags
5968 if "%s.seen" % tablename in fieldnames:
5969 item_dict["%s.seen" % tablename] = "\\Seen" in flags
5970 if "%s.subject" % tablename in fieldnames:
5971 subject = message["Subject"]
5972 decoded_subject = decode_header(subject)
5973 text = decoded_subject[0][0]
5974 encoding = decoded_subject[0][1]
5975 if encoding in (None, ""):
5976 encoding = charset
5977 item_dict["%s.subject" % tablename] = self.encode_text(text, encoding)
5978 if "%s.answered" % tablename in fieldnames:
5979 item_dict["%s.answered" % tablename] = "\\Answered" in flags
5980 if "%s.mime" % tablename in fieldnames:
5981 item_dict["%s.mime" % tablename] = message.get_content_type()
5982
5983
5984
5985
5986
5987 if "%s.email" % tablename in fieldnames:
5988 item_dict["%s.email" % tablename] = self.encode_text(raw_message, charset)
5989
5990
5991
5992
5993
5994 for part in message.walk():
5995 if "%s.attachments" % tablename in fieldnames:
5996 if not "text" in part.get_content_maintype():
5997 attachments.append(part.get_payload(decode=True))
5998 if "%s.content" % tablename in fieldnames:
5999 if "text" in part.get_content_maintype():
6000 payload = self.encode_text(part.get_payload(decode=True), charset)
6001 content.append(payload)
6002 if "%s.size" % tablename in fieldnames:
6003 if part is not None:
6004 size += len(str(part))
6005
6006 item_dict["%s.content" % tablename] = bar_encode(content)
6007 item_dict["%s.attachments" % tablename] = bar_encode(attachments)
6008 item_dict["%s.size" % tablename] = size
6009
6010 imapqry_list.append(item_dict)
6011
6012
6013
6014 for item_dict in imapqry_list:
6015 imapqry_array_item = list()
6016 for fieldname in fieldnames:
6017 imapqry_array_item.append(item_dict[fieldname])
6018 imapqry_array.append(imapqry_array_item)
6019
6020 return tablename, imapqry_array, fieldnames
6021
6022 - def select(self,query,fields,attributes):
6023 tablename, imapqry_array , fieldnames = self._select(query,fields,attributes)
6024
6025 colnames = fieldnames
6026 processor = attributes.get('processor',self.parse)
6027 return processor(imapqry_array, fields, colnames)
6028
6029 - def update(self, tablename, query, fields):
6030 if use_common_filters(query):
6031 query = self.common_filter(query, [tablename,])
6032
6033 mark = []
6034 unmark = []
6035 rowcount = 0
6036 query = str(query)
6037 if query:
6038 for item in fields:
6039 field = item[0]
6040 name = field.name
6041 value = item[1]
6042 if self.is_flag(name):
6043 flag = self.search_fields[name]
6044 if (value is not None) and (flag != "\\Recent"):
6045 if value:
6046 mark.append(flag)
6047 else:
6048 unmark.append(flag)
6049
6050 result, data = self.connection.select(
6051 self.connection.mailbox_names[tablename])
6052 string_query = "(%s)" % query
6053 result, data = self.connection.search(None, string_query)
6054 store_list = [item.strip() for item in data[0].split()
6055 if item.strip().isdigit()]
6056
6057 for number in store_list:
6058 result = None
6059 if len(mark) > 0:
6060 result, data = self.connection.store(
6061 number, "+FLAGS", "(%s)" % " ".join(mark))
6062 if len(unmark) > 0:
6063 result, data = self.connection.store(
6064 number, "-FLAGS", "(%s)" % " ".join(unmark))
6065 if result == "OK":
6066 rowcount += 1
6067 return rowcount
6068
6069 - def count(self,query,distinct=None):
6081
6082 - def delete(self, tablename, query):
6083 counter = 0
6084 if query:
6085 if use_common_filters(query):
6086 query = self.common_filter(query, [tablename,])
6087 result, data = self.connection.select(self.connection.mailbox_names[tablename])
6088 string_query = "(%s)" % query
6089 result, data = self.connection.search(None, string_query)
6090 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
6091 for number in store_list:
6092 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
6093 if result == "OK":
6094 counter += 1
6095 if counter > 0:
6096 result, data = self.connection.expunge()
6097 return counter
6098
6099 - def BELONGS(self, first, second):
6100 result = None
6101 name = self.search_fields[first.name]
6102 if name == "MESSAGE":
6103 values = [str(val) for val in second if str(val).isdigit()]
6104 result = "%s" % ",".join(values).strip()
6105
6106 elif name == "UID":
6107 values = [str(val) for val in second if str(val).isdigit()]
6108 result = "UID %s" % ",".join(values).strip()
6109
6110 else:
6111 raise Exception("Operation not supported")
6112
6113 return result
6114
6116 result = None
6117 name = self.search_fields[first.name]
6118
6119 if name in ("FROM", "TO", "SUBJECT", "TEXT"):
6120 result = "%s \"%s\"" % (name, self.expand(second))
6121 else:
6122 if first.name in ("cc", "bcc"):
6123 result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
6124 elif first.name == "mime":
6125 result = "HEADER Content-Type \"%s\"" % self.expand(second)
6126 else:
6127 raise Exception("Operation not supported")
6128 return result
6129
6130 - def GT(self, first, second):
6131 result = None
6132 name = self.search_fields[first.name]
6133 if name == "MESSAGE":
6134 last_message = self.get_last_message(first.tablename)
6135 result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
6136 elif name == "UID":
6137
6138
6139
6140 try:
6141 pedestal, threshold = self.get_uid_bounds(first.tablename)
6142 except TypeError:
6143 e = sys.exc_info()[1]
6144 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6145 return ""
6146 try:
6147 lower_limit = int(self.expand(second)) + 1
6148 except (ValueError, TypeError):
6149 e = sys.exc_info()[1]
6150 raise Exception("Operation not supported (non integer UID)")
6151 result = "UID %s:%s" % (lower_limit, threshold)
6152 elif name == "DATE":
6153 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
6154 elif name == "SIZE":
6155 result = "LARGER %s" % self.expand(second)
6156 else:
6157 raise Exception("Operation not supported")
6158 return result
6159
6160 - def GE(self, first, second):
6161 result = None
6162 name = self.search_fields[first.name]
6163 if name == "MESSAGE":
6164 last_message = self.get_last_message(first.tablename)
6165 result = "%s:%s" % (self.expand(second), last_message)
6166 elif name == "UID":
6167
6168
6169
6170 try:
6171 pedestal, threshold = self.get_uid_bounds(first.tablename)
6172 except TypeError:
6173 e = sys.exc_info()[1]
6174 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6175 return ""
6176 lower_limit = self.expand(second)
6177 result = "UID %s:%s" % (lower_limit, threshold)
6178 elif name == "DATE":
6179 result = "SINCE %s" % self.convert_date(second)
6180 else:
6181 raise Exception("Operation not supported")
6182 return result
6183
6184 - def LT(self, first, second):
6185 result = None
6186 name = self.search_fields[first.name]
6187 if name == "MESSAGE":
6188 result = "%s:%s" % (1, int(self.expand(second)) - 1)
6189 elif name == "UID":
6190 try:
6191 pedestal, threshold = self.get_uid_bounds(first.tablename)
6192 except TypeError:
6193 e = sys.exc_info()[1]
6194 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6195 return ""
6196 try:
6197 upper_limit = int(self.expand(second)) - 1
6198 except (ValueError, TypeError):
6199 e = sys.exc_info()[1]
6200 raise Exception("Operation not supported (non integer UID)")
6201 result = "UID %s:%s" % (pedestal, upper_limit)
6202 elif name == "DATE":
6203 result = "BEFORE %s" % self.convert_date(second)
6204 elif name == "SIZE":
6205 result = "SMALLER %s" % self.expand(second)
6206 else:
6207 raise Exception("Operation not supported")
6208 return result
6209
6210 - def LE(self, first, second):
6211 result = None
6212 name = self.search_fields[first.name]
6213 if name == "MESSAGE":
6214 result = "%s:%s" % (1, self.expand(second))
6215 elif name == "UID":
6216 try:
6217 pedestal, threshold = self.get_uid_bounds(first.tablename)
6218 except TypeError:
6219 e = sys.exc_info()[1]
6220 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6221 return ""
6222 upper_limit = int(self.expand(second))
6223 result = "UID %s:%s" % (pedestal, upper_limit)
6224 elif name == "DATE":
6225 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
6226 else:
6227 raise Exception("Operation not supported")
6228 return result
6229
6230 - def NE(self, first, second=None):
6231 if (second is None) and isinstance(first, Field):
6232
6233 if first.type == "id":
6234 return self.GE(first, 1)
6235 result = self.NOT(self.EQ(first, second))
6236 result = result.replace("NOT NOT", "").strip()
6237 return result
6238
6239 - def EQ(self,first,second):
6240 name = self.search_fields[first.name]
6241 result = None
6242 if name is not None:
6243 if name == "MESSAGE":
6244
6245 result = "%s" % self.expand(second)
6246 elif name == "UID":
6247 result = "UID %s" % self.expand(second)
6248 elif name == "DATE":
6249 result = "ON %s" % self.convert_date(second)
6250
6251 elif name in self.flags:
6252 if second:
6253 result = "%s" % (name.upper()[1:])
6254 else:
6255 result = "NOT %s" % (name.upper()[1:])
6256 else:
6257 raise Exception("Operation not supported")
6258 else:
6259 raise Exception("Operation not supported")
6260 return result
6261
6262 - def AND(self, first, second):
6265
6266 - def OR(self, first, second):
6269
6270 - def NOT(self, first):
6271 result = "NOT %s" % self.expand(first)
6272 return result
6273
6274
6275
6276
6277
6278 ADAPTERS = {
6279 'sqlite': SQLiteAdapter,
6280 'spatialite': SpatiaLiteAdapter,
6281 'sqlite:memory': SQLiteAdapter,
6282 'spatialite:memory': SpatiaLiteAdapter,
6283 'mysql': MySQLAdapter,
6284 'postgres': PostgreSQLAdapter,
6285 'postgres:psycopg2': PostgreSQLAdapter,
6286 'postgres:pg8000': PostgreSQLAdapter,
6287 'postgres2:psycopg2': NewPostgreSQLAdapter,
6288 'postgres2:pg8000': NewPostgreSQLAdapter,
6289 'oracle': OracleAdapter,
6290 'mssql': MSSQLAdapter,
6291 'mssql2': MSSQL2Adapter,
6292 'sybase': SybaseAdapter,
6293 'db2': DB2Adapter,
6294 'teradata': TeradataAdapter,
6295 'informix': InformixAdapter,
6296 'informix-se': InformixSEAdapter,
6297 'firebird': FireBirdAdapter,
6298 'firebird_embedded': FireBirdAdapter,
6299 'ingres': IngresAdapter,
6300 'ingresu': IngresUnicodeAdapter,
6301 'sapdb': SAPDBAdapter,
6302 'cubrid': CubridAdapter,
6303 'jdbc:sqlite': JDBCSQLiteAdapter,
6304 'jdbc:sqlite:memory': JDBCSQLiteAdapter,
6305 'jdbc:postgres': JDBCPostgreSQLAdapter,
6306 'gae': GoogleDatastoreAdapter,
6307 'google:datastore': GoogleDatastoreAdapter,
6308 'google:sql': GoogleSQLAdapter,
6309 'couchdb': CouchDBAdapter,
6310 'mongodb': MongoDBAdapter,
6311 'imap': IMAPAdapter
6312 }
6316 """
6317 Field type validation, using web2py's validators mechanism.
6318
6319 makes sure the content of a field is in line with the declared
6320 fieldtype
6321 """
6322 db = field.db
6323 if not have_validators:
6324 return []
6325 field_type, field_length = field.type, field.length
6326 if isinstance(field_type, SQLCustomType):
6327 if hasattr(field_type, 'validator'):
6328 return field_type.validator
6329 else:
6330 field_type = field_type.type
6331 elif not isinstance(field_type,str):
6332 return []
6333 requires=[]
6334 def ff(r,id):
6335 row=r(id)
6336 if not row:
6337 return id
6338 elif hasattr(r, '_format') and isinstance(r._format,str):
6339 return r._format % row
6340 elif hasattr(r, '_format') and callable(r._format):
6341 return r._format(row)
6342 else:
6343 return id
6344 if field_type == 'string':
6345 requires.append(validators.IS_LENGTH(field_length))
6346 elif field_type == 'text':
6347 requires.append(validators.IS_LENGTH(field_length))
6348 elif field_type == 'password':
6349 requires.append(validators.IS_LENGTH(field_length))
6350 elif field_type == 'double' or field_type == 'float':
6351 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
6352 elif field_type in ('integer','bigint'):
6353 requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100))
6354 elif field_type.startswith('decimal'):
6355 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
6356 elif field_type == 'date':
6357 requires.append(validators.IS_DATE())
6358 elif field_type == 'time':
6359 requires.append(validators.IS_TIME())
6360 elif field_type == 'datetime':
6361 requires.append(validators.IS_DATETIME())
6362 elif db and field_type.startswith('reference') and \
6363 field_type.find('.') < 0 and \
6364 field_type[10:] in db.tables:
6365 referenced = db[field_type[10:]]
6366 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
6367 field.represent = field.represent or repr_ref
6368 if hasattr(referenced, '_format') and referenced._format:
6369 requires = validators.IS_IN_DB(db,referenced._id,
6370 referenced._format)
6371 if field.unique:
6372 requires._and = validators.IS_NOT_IN_DB(db,field)
6373 if field.tablename == field_type[10:]:
6374 return validators.IS_EMPTY_OR(requires)
6375 return requires
6376 elif db and field_type.startswith('list:reference') and \
6377 field_type.find('.') < 0 and \
6378 field_type[15:] in db.tables:
6379 referenced = db[field_type[15:]]
6380 def list_ref_repr(ids, row=None, r=referenced, f=ff):
6381 if not ids:
6382 return None
6383 refs = None
6384 db, id = r._db, r._id
6385 if isinstance(db._adapter, GoogleDatastoreAdapter):
6386 def count(values): return db(id.belongs(values)).select(id)
6387 rx = range(0, len(ids), 30)
6388 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
6389 else:
6390 refs = db(id.belongs(ids)).select(id)
6391 return (refs and ', '.join(str(f(r,x.id)) for x in refs) or '')
6392 field.represent = field.represent or list_ref_repr
6393 if hasattr(referenced, '_format') and referenced._format:
6394 requires = validators.IS_IN_DB(db,referenced._id,
6395 referenced._format,multiple=True)
6396 else:
6397 requires = validators.IS_IN_DB(db,referenced._id,
6398 multiple=True)
6399 if field.unique:
6400 requires._and = validators.IS_NOT_IN_DB(db,field)
6401 return requires
6402 elif field_type.startswith('list:'):
6403 def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
6404 field.represent = field.represent or repr_list
6405 if field.unique:
6406 requires.insert(0,validators.IS_NOT_IN_DB(db,field))
6407 sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
6408 if field.notnull and not field_type[:2] in sff:
6409 requires.insert(0, validators.IS_NOT_EMPTY())
6410 elif not field.notnull and field_type[:2] in sff and requires:
6411 requires[-1] = validators.IS_EMPTY_OR(requires[-1])
6412 return requires
6413
6416 return str(item).replace('|', '||')
6417
6420
6422 if not hasattr(value,'split') and hasattr(value,'read'):
6423 value = value.read()
6424 return [int(x) for x in value.split('|') if x.strip()]
6425
6429
6430
6431 -class Row(object):
6432
6433 """
6434 a dictionary that lets you do d['a'] as well as d.a
6435 this is only used to store a Row
6436 """
6437
6439 self.__dict__.update(*args,**kwargs)
6440
6442 key=str(key)
6443 m = REGEX_TABLE_DOT_FIELD.match(key)
6444 if key in self.get('_extra',{}):
6445 return self._extra[key]
6446 elif m:
6447 try:
6448 return ogetattr(self, m.group(1))[m.group(2)]
6449 except (KeyError,AttributeError,TypeError):
6450 key = m.group(2)
6451 return ogetattr(self, key)
6452
6454 setattr(self, str(key), value)
6455
6456 __call__ = __getitem__
6457
6458 - def get(self,key,default=None):
6459 return self.__dict__.get(key,default)
6460
6462 return key in self.__dict__
6463
6464 has_key = __contains__
6465
6467 return len(self.__dict__)>0
6468
6469 - def update(self, *args, **kwargs):
6470 self.__dict__.update(*args, **kwargs)
6471
6473 return self.__dict__.keys()
6474
6476 return self.__dict__.items()
6477
6479 return self.__dict__.values()
6480
6483
6486
6488
6489 return '<Row %s>' % self.as_dict()
6490
6492 return '<Row %s>' % self.as_dict()
6493
6495 return object.__getattribute__(self,'id')
6496
6498 try:
6499 return self.as_dict() == other.as_dict()
6500 except AttributeError:
6501 return False
6502
6504 return not (self == other)
6505
6507 return Row(dict(self))
6508
6509 - def as_dict(self, datetime_to_str=False, custom_types=None):
6510 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict]
6511 if isinstance(custom_types,(list,tuple,set)):
6512 SERIALIZABLE_TYPES += custom_types
6513 elif custom_types:
6514 SERIALIZABLE_TYPES.append(custom_types)
6515 d = dict(self)
6516 for k in copy.copy(d.keys()):
6517 v=d[k]
6518 if d[k] is None:
6519 continue
6520 elif isinstance(v,Row):
6521 d[k]=v.as_dict()
6522 elif isinstance(v,Reference):
6523 d[k]=int(v)
6524 elif isinstance(v,decimal.Decimal):
6525 d[k]=float(v)
6526 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
6527 if datetime_to_str:
6528 d[k] = v.isoformat().replace('T',' ')[:19]
6529 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)):
6530 del d[k]
6531 return d
6532
6541
6543 if not isinstance(fields,(list,tuple)):
6544 fields = [fields]
6545 new_fields = []
6546 for field in fields:
6547 if isinstance(field,Field):
6548 new_fields.append(field)
6549 elif isinstance(field,Table):
6550 for ofield in field:
6551 new_fields.append(ofield)
6552 else:
6553 raise RuntimeError("fields must be a list of fields")
6554 fields = new_fields
6555 field_map = {}
6556 for field in fields:
6557 n = field.name.lower()
6558 if not n in field_map:
6559 field_map[n] = field
6560 n = str(field).lower()
6561 if not n in field_map:
6562 field_map[n] = field
6563 constants = {}
6564 i = 0
6565 while True:
6566 m = REGEX_CONST_STRING.search(text)
6567 if not m: break
6568 text = text[:m.start()]+('#%i' % i)+text[m.end():]
6569 constants[str(i)] = m.group()[1:-1]
6570 i+=1
6571 text = re.sub('\s+',' ',text).lower()
6572 for a,b in [('&','and'),
6573 ('|','or'),
6574 ('~','not'),
6575 ('==','='),
6576 ('<','<'),
6577 ('>','>'),
6578 ('<=','<='),
6579 ('>=','>='),
6580 ('<>','!='),
6581 ('=<','<='),
6582 ('=>','>='),
6583 ('=','='),
6584 (' less or equal than ','<='),
6585 (' greater or equal than ','>='),
6586 (' equal or less than ','<='),
6587 (' equal or greater than ','>='),
6588 (' less or equal ','<='),
6589 (' greater or equal ','>='),
6590 (' equal or less ','<='),
6591 (' equal or greater ','>='),
6592 (' not equal to ','!='),
6593 (' not equal ','!='),
6594 (' equal to ','='),
6595 (' equal ','='),
6596 (' equals ','='),
6597 (' less than ','<'),
6598 (' greater than ','>'),
6599 (' starts with ','startswith'),
6600 (' ends with ','endswith'),
6601 (' not in ' , 'notbelongs'),
6602 (' in ' , 'belongs'),
6603 (' is ','=')]:
6604 if a[0]==' ':
6605 text = text.replace(' is'+a,' %s ' % b)
6606 text = text.replace(a,' %s ' % b)
6607 text = re.sub('\s+',' ',text).lower()
6608 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
6609 query = field = neg = op = logic = None
6610 for item in text.split():
6611 if field is None:
6612 if item == 'not':
6613 neg = True
6614 elif not neg and not logic and item in ('and','or'):
6615 logic = item
6616 elif item in field_map:
6617 field = field_map[item]
6618 else:
6619 raise RuntimeError("Invalid syntax")
6620 elif not field is None and op is None:
6621 op = item
6622 elif not op is None:
6623 if item.startswith('#'):
6624 if not item[1:] in constants:
6625 raise RuntimeError("Invalid syntax")
6626 value = constants[item[1:]]
6627 else:
6628 value = item
6629 if field.type in ('text','string'):
6630 if op == '=': op = 'like'
6631 if op == '=': new_query = field==value
6632 elif op == '<': new_query = field<value
6633 elif op == '>': new_query = field>value
6634 elif op == '<=': new_query = field<=value
6635 elif op == '>=': new_query = field>=value
6636 elif op == '!=': new_query = field!=value
6637 elif op == 'belongs': new_query = field.belongs(value.split(','))
6638 elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
6639 elif field.type in ('text','string'):
6640 if op == 'contains': new_query = field.contains(value)
6641 elif op == 'like': new_query = field.like(value)
6642 elif op == 'startswith': new_query = field.startswith(value)
6643 elif op == 'endswith': new_query = field.endswith(value)
6644 else: raise RuntimeError("Invalid operation")
6645 elif field._db._adapter.dbengine=='google:datastore' and \
6646 field.type in ('list:integer', 'list:string', 'list:reference'):
6647 if op == 'contains': new_query = field.contains(value)
6648 else: raise RuntimeError("Invalid operation")
6649 else: raise RuntimeError("Invalid operation")
6650 if neg: new_query = ~new_query
6651 if query is None:
6652 query = new_query
6653 elif logic == 'and':
6654 query &= new_query
6655 elif logic == 'or':
6656 query |= new_query
6657 field = op = neg = logic = None
6658 return query
6659
6661
6662 """
6663 an instance of this class represents a database connection
6664
6665 Example::
6666
6667 db = DAL('sqlite://test.db')
6668 db.define_table('tablename', Field('fieldname1'),
6669 Field('fieldname2'))
6670 """
6671
6672 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
6699
6700 @staticmethod
6702 """
6703 # ## this allows gluon to set a folder for this thread
6704 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
6705 """
6706 BaseAdapter.set_folder(folder)
6707
6708 @staticmethod
6710 """
6711 Returns a dictionary with uri as key with timings and defined tables
6712 {'sqlite://storage.sqlite': {
6713 'dbstats': [(select auth_user.email from auth_user, 0.02009)],
6714 'dbtables': {
6715 'defined': ['auth_cas', 'auth_event', 'auth_group',
6716 'auth_membership', 'auth_permission', 'auth_user'],
6717 'lazy': '[]'
6718 }
6719 }
6720 }
6721 """
6722 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
6723 infos = {}
6724 for db_uid, db_group in dbs:
6725 for db in db_group:
6726 if not db._uri:
6727 continue
6728 k = REGEX_NOPASSWD.sub('******',db._uri)
6729 infos[k] = dict(dbstats = [(row[0], row[1]) for row in db._timings],
6730 dbtables = {'defined':
6731 sorted(list(set(db.tables) -
6732 set(db._LAZY_TABLES.keys()))),
6733 'lazy': sorted(db._LAZY_TABLES.keys())}
6734 )
6735 return infos
6736
6737 @staticmethod
6750
6751 @staticmethod
6773
6774 - def __init__(self, uri='sqlite://dummy.db',
6775 pool_size=0, folder=None,
6776 db_codec='UTF-8', check_reserved=None,
6777 migrate=True, fake_migrate=False,
6778 migrate_enabled=True, fake_migrate_all=False,
6779 decode_credentials=False, driver_args=None,
6780 adapter_args=None, attempts=5, auto_import=False,
6781 bigint_id=False,debug=False,lazy_tables=False,
6782 db_uid=None, do_connect=True):
6783 """
6784 Creates a new Database Abstraction Layer instance.
6785
6786 Keyword arguments:
6787
6788 :uri: string that contains information for connecting to a database.
6789 (default: 'sqlite://dummy.db')
6790 :pool_size: How many open connections to make to the database object.
6791 :folder: <please update me>
6792 :db_codec: string encoding of the database (default: 'UTF-8')
6793 :check_reserved: list of adapters to check tablenames and column names
6794 against sql reserved keywords. (Default None)
6795
6796 * 'common' List of sql keywords that are common to all database types
6797 such as "SELECT, INSERT". (recommended)
6798 * 'all' Checks against all known SQL keywords. (not recommended)
6799 <adaptername> Checks against the specific adapters list of keywords
6800 (recommended)
6801 * '<adaptername>_nonreserved' Checks against the specific adapters
6802 list of nonreserved keywords. (if available)
6803 :migrate (defaults to True) sets default migrate behavior for all tables
6804 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
6805 :migrate_enabled (defaults to True). If set to False disables ALL migrations
6806 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
6807 :attempts (defaults to 5). Number of times to attempt connecting
6808 """
6809
6810 if uri == '<zombie>' and db_uid is not None: return
6811
6812 if not decode_credentials:
6813 credential_decoder = lambda cred: cred
6814 else:
6815 credential_decoder = lambda cred: urllib.unquote(cred)
6816 if folder:
6817 self.set_folder(folder)
6818 self._uri = uri
6819 self._pool_size = pool_size
6820 self._db_codec = db_codec
6821 self._lastsql = ''
6822 self._timings = []
6823 self._pending_references = {}
6824 self._request_tenant = 'request_tenant'
6825 self._common_fields = []
6826 self._referee_name = '%(table)s'
6827 self._bigint_id = bigint_id
6828 self._debug = debug
6829 self._migrated = []
6830 self._LAZY_TABLES = {}
6831 self._lazy_tables = lazy_tables
6832 if not str(attempts).isdigit() or attempts < 0:
6833 attempts = 5
6834 if uri:
6835 uris = isinstance(uri,(list,tuple)) and uri or [uri]
6836 error = ''
6837 connected = False
6838 for k in range(attempts):
6839 for uri in uris:
6840 try:
6841 if is_jdbc and not uri.startswith('jdbc:'):
6842 uri = 'jdbc:'+uri
6843 self._dbname = REGEX_DBNAME.match(uri).group()
6844 if not self._dbname in ADAPTERS:
6845 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname)
6846
6847
6848 kwargs = dict(db=self,uri=uri,
6849 pool_size=pool_size,
6850 folder=folder,
6851 db_codec=db_codec,
6852 credential_decoder=credential_decoder,
6853 driver_args=driver_args or {},
6854 adapter_args=adapter_args or {},
6855 do_connect=do_connect)
6856 self._adapter = ADAPTERS[self._dbname](**kwargs)
6857 types = ADAPTERS[self._dbname].types
6858
6859 self._adapter.types = copy.copy(types)
6860 if bigint_id:
6861 if 'big-id' in types and 'reference' in types:
6862 self._adapter.types['id'] = types['big-id']
6863 self._adapter.types['reference'] = types['big-reference']
6864 connected = True
6865 break
6866 except SyntaxError:
6867 raise
6868 except Exception:
6869 tb = traceback.format_exc()
6870 sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
6871 if connected:
6872 break
6873 else:
6874 time.sleep(1)
6875 if not connected:
6876 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb))
6877 else:
6878 self._adapter = BaseAdapter(db=self,pool_size=0,
6879 uri='None',folder=folder,
6880 db_codec=db_codec)
6881 migrate = fake_migrate = False
6882 adapter = self._adapter
6883 self._uri_hash = hashlib_md5(adapter.uri).hexdigest()
6884 self._tables = SQLCallableList()
6885 self.check_reserved = check_reserved
6886 if self.check_reserved:
6887 from reserved_sql_keywords import ADAPTERS as RSK
6888 self.RSK = RSK
6889 self._migrate = migrate
6890 self._fake_migrate = fake_migrate
6891 self._migrate_enabled = migrate_enabled
6892 self._fake_migrate_all = fake_migrate_all
6893 if auto_import:
6894 self.import_table_definitions(adapter.folder)
6895
6896 @property
6899
6901 pattern = pjoin(path,self._uri_hash+'_*.table')
6902 for filename in glob.glob(pattern):
6903 tfile = self._adapter.file_open(filename, 'r')
6904 try:
6905 sql_fields = pickle.load(tfile)
6906 name = filename[len(pattern)-7:-6]
6907 mf = [(value['sortable'],
6908 Field(key,
6909 type=value['type'],
6910 length=value.get('length',None),
6911 notnull=value.get('notnull',False),
6912 unique=value.get('unique',False))) \
6913 for key, value in sql_fields.iteritems()]
6914 mf.sort(lambda a,b: cmp(a[0],b[0]))
6915 self.define_table(name,*[item[1] for item in mf],
6916 **dict(migrate=migrate,fake_migrate=fake_migrate))
6917 finally:
6918 self._adapter.file_close(tfile)
6919
6921 """
6922 Validates ``name`` against SQL keywords
6923 Uses self.check_reserve which is a list of
6924 operators to use.
6925 self.check_reserved
6926 ['common', 'postgres', 'mysql']
6927 self.check_reserved
6928 ['all']
6929 """
6930 for backend in self.check_reserved:
6931 if name.upper() in self.RSK[backend]:
6932 raise SyntaxError(
6933 'invalid table/column name "%s" is a "%s" reserved SQL keyword' % (name, backend.upper()))
6934
6935 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
6936 """
6937 EXAMPLE:
6938
6939 db.define_table('person',Field('name'),Field('info'))
6940 db.define_table('pet',Field('owner',db.person),Field('name'),Field('info'))
6941
6942 @request.restful()
6943 def index():
6944 def GET(*args,**vars):
6945 patterns = [
6946 "/friends[person]",
6947 "/{friend.name.startswith}",
6948 "/{friend.name}/:field",
6949 "/{friend.name}/pets[pet.owner]",
6950 "/{friend.name}/pet[pet.owner]/{pet.name}",
6951 "/{friend.name}/pet[pet.owner]/{pet.name}/:field"
6952 ]
6953 parser = db.parse_as_rest(patterns,args,vars)
6954 if parser.status == 200:
6955 return dict(content=parser.response)
6956 else:
6957 raise HTTP(parser.status,parser.error)
6958 def POST(table_name,**vars):
6959 if table_name == 'person':
6960 return db.person.validate_and_insert(**vars)
6961 elif table_name == 'pet':
6962 return db.pet.validate_and_insert(**vars)
6963 else:
6964 raise HTTP(400)
6965 return locals()
6966 """
6967
6968 db = self
6969 re1 = REGEX_SEARCH_PATTERN
6970 re2 = REGEX_SQUARE_BRACKETS
6971
6972 def auto_table(table,base='',depth=0):
6973 patterns = []
6974 for field in db[table].fields:
6975 if base:
6976 tag = '%s/%s' % (base,field.replace('_','-'))
6977 else:
6978 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
6979 f = db[table][field]
6980 if not f.readable: continue
6981 if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
6982 tag += '/{%s.%s}' % (table,field)
6983 patterns.append(tag)
6984 patterns.append(tag+'/:field')
6985 elif f.type.startswith('boolean'):
6986 tag += '/{%s.%s}' % (table,field)
6987 patterns.append(tag)
6988 patterns.append(tag+'/:field')
6989 elif f.type in ('float','double','integer','bigint'):
6990 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
6991 patterns.append(tag)
6992 patterns.append(tag+'/:field')
6993 elif f.type.startswith('list:'):
6994 tag += '/{%s.%s.contains}' % (table,field)
6995 patterns.append(tag)
6996 patterns.append(tag+'/:field')
6997 elif f.type in ('date','datetime'):
6998 tag+= '/{%s.%s.year}' % (table,field)
6999 patterns.append(tag)
7000 patterns.append(tag+'/:field')
7001 tag+='/{%s.%s.month}' % (table,field)
7002 patterns.append(tag)
7003 patterns.append(tag+'/:field')
7004 tag+='/{%s.%s.day}' % (table,field)
7005 patterns.append(tag)
7006 patterns.append(tag+'/:field')
7007 if f.type in ('datetime','time'):
7008 tag+= '/{%s.%s.hour}' % (table,field)
7009 patterns.append(tag)
7010 patterns.append(tag+'/:field')
7011 tag+='/{%s.%s.minute}' % (table,field)
7012 patterns.append(tag)
7013 patterns.append(tag+'/:field')
7014 tag+='/{%s.%s.second}' % (table,field)
7015 patterns.append(tag)
7016 patterns.append(tag+'/:field')
7017 if depth>0:
7018 for f in db[table]._referenced_by:
7019 tag+='/%s[%s.%s]' % (table,f.tablename,f.name)
7020 patterns.append(tag)
7021 patterns += auto_table(table,base=tag,depth=depth-1)
7022 return patterns
7023
7024 if patterns==DEFAULT:
7025 patterns=[]
7026 for table in db.tables:
7027 if not table.startswith('auth_'):
7028 patterns.append('/%s[%s]' % (table,table))
7029 patterns += auto_table(table,base='',depth=1)
7030 else:
7031 i = 0
7032 while i<len(patterns):
7033 pattern = patterns[i]
7034 tokens = pattern.split('/')
7035 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
7036 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
7037 '/'.join(tokens[:-1]))
7038 patterns = patterns[:i]+new_patterns+patterns[i+1:]
7039 i += len(new_patterns)
7040 else:
7041 i += 1
7042 if '/'.join(args) == 'patterns':
7043 return Row({'status':200,'pattern':'list',
7044 'error':None,'response':patterns})
7045 for pattern in patterns:
7046 otable=table=None
7047 if not isinstance(queries,dict):
7048 dbset=db(queries)
7049 i=0
7050 tags = pattern[1:].split('/')
7051 if len(tags)!=len(args):
7052 continue
7053 for tag in tags:
7054 if re1.match(tag):
7055
7056 tokens = tag[1:-1].split('.')
7057 table, field = tokens[0], tokens[1]
7058 if not otable or table == otable:
7059 if len(tokens)==2 or tokens[2]=='eq':
7060 query = db[table][field]==args[i]
7061 elif tokens[2]=='ne':
7062 query = db[table][field]!=args[i]
7063 elif tokens[2]=='lt':
7064 query = db[table][field]<args[i]
7065 elif tokens[2]=='gt':
7066 query = db[table][field]>args[i]
7067 elif tokens[2]=='ge':
7068 query = db[table][field]>=args[i]
7069 elif tokens[2]=='le':
7070 query = db[table][field]<=args[i]
7071 elif tokens[2]=='year':
7072 query = db[table][field].year()==args[i]
7073 elif tokens[2]=='month':
7074 query = db[table][field].month()==args[i]
7075 elif tokens[2]=='day':
7076 query = db[table][field].day()==args[i]
7077 elif tokens[2]=='hour':
7078 query = db[table][field].hour()==args[i]
7079 elif tokens[2]=='minute':
7080 query = db[table][field].minutes()==args[i]
7081 elif tokens[2]=='second':
7082 query = db[table][field].seconds()==args[i]
7083 elif tokens[2]=='startswith':
7084 query = db[table][field].startswith(args[i])
7085 elif tokens[2]=='contains':
7086 query = db[table][field].contains(args[i])
7087 else:
7088 raise RuntimeError("invalid pattern: %s" % pattern)
7089 if len(tokens)==4 and tokens[3]=='not':
7090 query = ~query
7091 elif len(tokens)>=4:
7092 raise RuntimeError("invalid pattern: %s" % pattern)
7093 if not otable and isinstance(queries,dict):
7094 dbset = db(queries[table])
7095 dbset=dbset(query)
7096 else:
7097 raise RuntimeError("missing relation in pattern: %s" % pattern)
7098 elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
7099 ref = tag[tag.find('[')+1:-1]
7100 if '.' in ref and otable:
7101 table,field = ref.split('.')
7102
7103 if nested_select:
7104 try:
7105 dbset=db(db[table][field].belongs(dbset._select(db[otable]._id)))
7106 except ValueError:
7107 return Row({'status':400,'pattern':pattern,
7108 'error':'invalid path','response':None})
7109 else:
7110 items = [item.id for item in dbset.select(db[otable]._id)]
7111 dbset=db(db[table][field].belongs(items))
7112 else:
7113 table = ref
7114 if not otable and isinstance(queries,dict):
7115 dbset = db(queries[table])
7116 dbset=dbset(db[table])
7117 elif tag==':field' and table:
7118
7119 field = args[i]
7120 if not field in db[table]: break
7121
7122 if not db[table][field].readable:
7123 return Row({'status':418,'pattern':pattern,
7124 'error':'I\'m a teapot','response':None})
7125 try:
7126 item = dbset.select(db[table][field],limitby=(0,1)).first()
7127 except ValueError:
7128 return Row({'status':400,'pattern':pattern,
7129 'error':'invalid path','response':None})
7130 if not item:
7131 return Row({'status':404,'pattern':pattern,
7132 'error':'record not found','response':None})
7133 else:
7134 return Row({'status':200,'response':item[field],
7135 'pattern':pattern})
7136 elif tag != args[i]:
7137 break
7138 otable = table
7139 i += 1
7140 if i==len(tags) and table:
7141 ofields = vars.get('order',db[table]._id.name).split('|')
7142 try:
7143 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
7144 except (KeyError, AttributeError):
7145 return Row({'status':400,'error':'invalid orderby','response':None})
7146 fields = [field for field in db[table] if field.readable]
7147 count = dbset.count()
7148 try:
7149 offset = int(vars.get('offset',None) or 0)
7150 limits = (offset,int(vars.get('limit',None) or 1000)+offset)
7151 except ValueError:
7152 Row({'status':400,'error':'invalid limits','response':None})
7153 if count > limits[1]-limits[0]:
7154 Row({'status':400,'error':'too many records','response':None})
7155 try:
7156 response = dbset.select(limitby=limits,orderby=orderby,*fields)
7157 except ValueError:
7158 return Row({'status':400,'pattern':pattern,
7159 'error':'invalid path','response':None})
7160 return Row({'status':200,'response':response,
7161 'pattern':pattern,'count':count})
7162 return Row({'status':400,'error':'no matching pattern','response':None})
7163
7164 - def define_table(
7165 self,
7166 tablename,
7167 *fields,
7168 **args
7169 ):
7170 if not isinstance(tablename,str):
7171 raise SyntaxError("missing table name")
7172 elif hasattr(self,tablename) or tablename in self.tables:
7173 if not args.get('redefine',False):
7174 raise SyntaxError('table already defined: %s' % tablename)
7175 elif tablename.startswith('_') or hasattr(self,tablename) or \
7176 REGEX_PYTHON_KEYWORDS.match(tablename):
7177 raise SyntaxError('invalid table name: %s' % tablename)
7178 elif self.check_reserved:
7179 self.check_reserved_keyword(tablename)
7180 else:
7181 invalid_args = set(args)-TABLE_ARGS
7182 if invalid_args:
7183 raise SyntaxError('invalid table "%s" attributes: %s' \
7184 % (tablename,invalid_args))
7185 if self._lazy_tables and not tablename in self._LAZY_TABLES:
7186 self._LAZY_TABLES[tablename] = (tablename,fields,args)
7187 table = None
7188 else:
7189 table = self.lazy_define_table(tablename,*fields,**args)
7190 if not tablename in self.tables:
7191 self.tables.append(tablename)
7192 return table
7193
7194 - def lazy_define_table(
7195 self,
7196 tablename,
7197 *fields,
7198 **args
7199 ):
7200 args_get = args.get
7201 common_fields = self._common_fields
7202 if common_fields:
7203 fields = list(fields) + list(common_fields)
7204
7205 table_class = args_get('table_class',Table)
7206 table = table_class(self, tablename, *fields, **args)
7207 table._actual = True
7208 self[tablename] = table
7209
7210 table._create_references()
7211 for field in table:
7212 if field.requires == DEFAULT:
7213 field.requires = sqlhtml_validators(field)
7214
7215 migrate = self._migrate_enabled and args_get('migrate',self._migrate)
7216 if migrate and not self._uri in (None,'None') \
7217 or self._adapter.dbengine=='google:datastore':
7218 fake_migrate = self._fake_migrate_all or \
7219 args_get('fake_migrate',self._fake_migrate)
7220 polymodel = args_get('polymodel',None)
7221 try:
7222 GLOBAL_LOCKER.acquire()
7223 self._adapter.create_table(table,migrate=migrate,
7224 fake_migrate=fake_migrate,
7225 polymodel=polymodel)
7226 finally:
7227 GLOBAL_LOCKER.release()
7228 else:
7229 table._dbt = None
7230 on_define = args_get('on_define',None)
7231 if on_define: on_define(table)
7232 return table
7233
7235 try:
7236 return tablename in self.tables
7237 except AttributeError:
7238
7239 return False
7240
7241 has_key = __contains__
7242
7243 - def get(self,key,default=None):
7244 return self.__dict__.get(key,default)
7245
7247 for tablename in self.tables:
7248 yield self[tablename]
7249
7252
7254 if ogetattr(self,'_lazy_tables') and \
7255 key in ogetattr(self,'_LAZY_TABLES'):
7256 tablename, fields, args = self._LAZY_TABLES.pop(key)
7257 return self.lazy_define_table(tablename,*fields,**args)
7258 return ogetattr(self, key)
7259
7261 osetattr(self, str(key), value)
7262
7264 if key[:1]!='_' and key in self:
7265 raise SyntaxError(
7266 'Object %s exists and cannot be redefined' % key)
7267 osetattr(self,key,value)
7268
7269 __delitem__ = object.__delattr__
7270
7272 if hasattr(self,'_uri'):
7273 return '<DAL uri="%s">' % hide_password(str(self._uri))
7274 else:
7275 return '<DAL db_uid="%s">' % self._db_uid
7276
7279
7280 - def __call__(self, query=None, ignore_common_filters=None):
7281 if isinstance(query,Table):
7282 query = self._adapter.id_query(query)
7283 elif isinstance(query,Field):
7284 query = query!=None
7285 return Set(self, query, ignore_common_filters=ignore_common_filters)
7286
7289
7292
7294 self._adapter.close()
7295 if self._db_uid in THREAD_LOCAL.db_instances:
7296 db_group = THREAD_LOCAL.db_instances[self._db_uid]
7297 db_group.remove(self)
7298 if not db_group:
7299 del THREAD_LOCAL.db_instances[self._db_uid]
7300
7301 - def executesql(self, query, placeholders=None, as_dict=False,
7302 fields=None, colnames=None):
7303 """
7304 placeholders is optional and will always be None.
7305 If using raw SQL with placeholders, placeholders may be
7306 a sequence of values to be substituted in
7307 or, (if supported by the DB driver), a dictionary with keys
7308 matching named placeholders in your SQL.
7309
7310 Added 2009-12-05 "as_dict" optional argument. Will always be
7311 None when using DAL. If using raw SQL can be set to True
7312 and the results cursor returned by the DB driver will be
7313 converted to a sequence of dictionaries keyed with the db
7314 field names. Tested with SQLite but should work with any database
7315 since the cursor.description used to get field names is part of the
7316 Python dbi 2.0 specs. Results returned with as_dict=True are
7317 the same as those returned when applying .to_list() to a DAL query.
7318
7319 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
7320
7321 Added 2012-08-24 "fields" and "colnames" optional arguments. If either
7322 is provided, the results cursor returned by the DB driver will be
7323 converted to a DAL Rows object using the db._adapter.parse() method.
7324
7325 The "fields" argument is a list of DAL Field objects that match the
7326 fields returned from the DB. The Field objects should be part of one or
7327 more Table objects defined on the DAL object. The "fields" list can
7328 include one or more DAL Table objects in addition to or instead of
7329 including Field objects, or it can be just a single table (not in a
7330 list). In that case, the Field objects will be extracted from the
7331 table(s).
7332
7333 Instead of specifying the "fields" argument, the "colnames" argument
7334 can be specified as a list of field names in tablename.fieldname format.
7335 Again, these should represent tables and fields defined on the DAL
7336 object.
7337
7338 It is also possible to specify both "fields" and the associated
7339 "colnames". In that case, "fields" can also include DAL Expression
7340 objects in addition to Field objects. For Field objects in "fields",
7341 the associated "colnames" must still be in tablename.fieldname format.
7342 For Expression objects in "fields", the associated "colnames" can
7343 be any arbitrary labels.
7344
7345 Note, the DAL Table objects referred to by "fields" or "colnames" can
7346 be dummy tables and do not have to represent any real tables in the
7347 database. Also, note that the "fields" and "colnames" must be in the
7348 same order as the fields in the results cursor returned from the DB.
7349 """
7350 adapter = self._adapter
7351 if placeholders:
7352 adapter.execute(query, placeholders)
7353 else:
7354 adapter.execute(query)
7355 if as_dict:
7356 if not hasattr(adapter.cursor,'description'):
7357 raise RuntimeError("database does not support executesql(...,as_dict=True)")
7358
7359
7360
7361 columns = adapter.cursor.description
7362
7363 fields = [f[0] for f in columns]
7364
7365 data = adapter._fetchall()
7366
7367
7368 return [dict(zip(fields,row)) for row in data]
7369 try:
7370 data = adapter._fetchall()
7371 except:
7372 return None
7373 if fields or colnames:
7374 fields = [] if fields is None else fields
7375 if not isinstance(fields, list):
7376 fields = [fields]
7377 extracted_fields = []
7378 for field in fields:
7379 if isinstance(field, Table):
7380 extracted_fields.extend([f for f in field])
7381 else:
7382 extracted_fields.append(field)
7383 if not colnames:
7384 colnames = ['%s.%s' % (f.tablename, f.name)
7385 for f in extracted_fields]
7386 data = adapter.parse(
7387 data, fields=extracted_fields, colnames=colnames)
7388 return data
7389
7391 for table in self:
7392 table._referenced_by = [field for field in table._referenced_by
7393 if not field.table==thistable]
7394
7396 step = int(kwargs.get('max_fetch_rows,',500))
7397 write_colnames = kwargs['write_colnames'] = \
7398 kwargs.get("write_colnames", True)
7399 for table in self.tables:
7400 ofile.write('TABLE %s\r\n' % table)
7401 query = self._adapter.id_query(self[table])
7402 nrows = self(query).count()
7403 kwargs['write_colnames'] = write_colnames
7404 for k in range(0,nrows,step):
7405 self(query).select(limitby=(k,k+step)).export_to_csv_file(
7406 ofile, *args, **kwargs)
7407 kwargs['write_colnames'] = False
7408 ofile.write('\r\n\r\n')
7409 ofile.write('END')
7410
7411 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
7412 unique='uuid', *args, **kwargs):
7413
7414 id_offset = {}
7415 for line in ifile:
7416 line = line.strip()
7417 if not line:
7418 continue
7419 elif line == 'END':
7420 return
7421 elif not line.startswith('TABLE ') or not line[6:] in self.tables:
7422 raise SyntaxError('invalid file format')
7423 else:
7424 tablename = line[6:]
7425 self[tablename].import_from_csv_file(
7426 ifile, id_map, null, unique, id_offset, *args, **kwargs)
7427
7429 return DAL('<zombie>',db_uid=db_uid)
7430
7433
7434 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
7437 """
7438 Helper class providing a comma-separated string having all the field names
7439 (prefixed by table name and '.')
7440
7441 normally only called from within gluon.sql
7442 """
7443
7446
7448 return ', '.join([str(field) for field in self._table])
7449
7452
7454 if not self._record:
7455 self._record = self._table[int(self)]
7456 if not self._record:
7457 raise RuntimeError(
7458 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, int(self)))
7459
7461 if key == 'id':
7462 return int(self)
7463 self.__allocate()
7464 return self._record.get(key, None)
7465
7466 - def get(self, key, default=None):
7468
7475
7477 if key == 'id':
7478 return int(self)
7479 self.__allocate()
7480 return self._record.get(key, None)
7481
7483 self.__allocate()
7484 self._record[key] = value
7485
7488 return marshal.loads(data)
7489
7491 try:
7492 marshal_dump = marshal.dumps(int(data))
7493 except AttributeError:
7494 marshal_dump = 'i%s' % struct.pack('<i', int(data))
7495 return (Reference_unpickler, (marshal_dump,))
7496
7497 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
7498
7499
7500 -class Table(object):
7501
7502 """
7503 an instance of this class represents a database table
7504
7505 Example::
7506
7507 db = DAL(...)
7508 db.define_table('users', Field('name'))
7509 db.users.insert(name='me') # print db.users._insert(...) to see SQL
7510 db.users.drop()
7511 """
7512
7513 - def __init__(
7514 self,
7515 db,
7516 tablename,
7517 *fields,
7518 **args
7519 ):
7520 """
7521 Initializes the table and performs checking on the provided fields.
7522
7523 Each table will have automatically an 'id'.
7524
7525 If a field is of type Table, the fields (excluding 'id') from that table
7526 will be used instead.
7527
7528 :raises SyntaxError: when a supplied field is of incorrect type.
7529 """
7530 self._actual = False
7531 self._tablename = tablename
7532 self._sequence_name = args.get('sequence_name',None) or \
7533 db and db._adapter.sequence_name(tablename)
7534 self._trigger_name = args.get('trigger_name',None) or \
7535 db and db._adapter.trigger_name(tablename)
7536 self._common_filter = args.get('common_filter', None)
7537 self._format = args.get('format',None)
7538 self._singular = args.get(
7539 'singular',tablename.replace('_',' ').capitalize())
7540 self._plural = args.get(
7541 'plural',pluralize(self._singular.lower()).capitalize())
7542
7543 if 'primarykey' in args and args['primarykey']:
7544 self._primarykey = args.get('primarykey', None)
7545
7546 self._before_insert = []
7547 self._before_update = [Set.delete_uploaded_files]
7548 self._before_delete = [Set.delete_uploaded_files]
7549 self._after_insert = []
7550 self._after_update = []
7551 self._after_delete = []
7552
7553 fieldnames,newfields=set(),[]
7554 if hasattr(self,'_primarykey'):
7555 if not isinstance(self._primarykey,list):
7556 raise SyntaxError(
7557 "primarykey must be a list of fields from table '%s'" \
7558 % tablename)
7559 if len(self._primarykey)==1:
7560 self._id = [f for f in fields if isinstance(f,Field) \
7561 and f.name==self._primarykey[0]][0]
7562 elif not [f for f in fields if isinstance(f,Field) and f.type=='id']:
7563 field = Field('id', 'id')
7564 newfields.append(field)
7565 fieldnames.add('id')
7566 self._id = field
7567 virtual_fields = []
7568 for field in fields:
7569 if isinstance(field, (FieldMethod, FieldVirtual)):
7570 virtual_fields.append(field)
7571 elif isinstance(field, Field) and not field.name in fieldnames:
7572 if field.db is not None:
7573 field = copy.copy(field)
7574 newfields.append(field)
7575 fieldnames.add(field.name)
7576 if field.type=='id':
7577 self._id = field
7578 elif isinstance(field, Table):
7579 table = field
7580 for field in table:
7581 if not field.name in fieldnames and not field.type=='id':
7582 t2 = not table._actual and self._tablename
7583 field = field.clone(point_self_references_to=t2)
7584 newfields.append(field)
7585 fieldnames.add(field.name)
7586 elif not isinstance(field, (Field, Table)):
7587 raise SyntaxError(
7588 'define_table argument is not a Field or Table: %s' % field)
7589 fields = newfields
7590 self._db = db
7591 tablename = tablename
7592 self._fields = SQLCallableList()
7593 self.virtualfields = []
7594 fields = list(fields)
7595
7596 if db and db._adapter.uploads_in_blob==True:
7597 uploadfields = [f.name for f in fields if f.type=='blob']
7598 for field in fields:
7599 fn = field.uploadfield
7600 if isinstance(field, Field) and field.type == 'upload'\
7601 and fn is True:
7602 fn = field.uploadfield = '%s_blob' % field.name
7603 if isinstance(fn,str) and not fn in uploadfields:
7604 fields.append(Field(fn,'blob',default='',
7605 writable=False,readable=False))
7606
7607 lower_fieldnames = set()
7608 reserved = dir(Table) + ['fields']
7609 for field in fields:
7610 field_name = field.name
7611 if db and db.check_reserved:
7612 db.check_reserved_keyword(field_name)
7613 elif field_name in reserved:
7614 raise SyntaxError("field name %s not allowed" % field_name)
7615
7616 if field_name.lower() in lower_fieldnames:
7617 raise SyntaxError("duplicate field %s in table %s" \
7618 % (field_name, tablename))
7619 else:
7620 lower_fieldnames.add(field_name.lower())
7621
7622 self.fields.append(field_name)
7623 self[field_name] = field
7624 if field.type == 'id':
7625 self['id'] = field
7626 field.tablename = field._tablename = tablename
7627 field.table = field._table = self
7628 field.db = field._db = db
7629 if db and not field.type in ('text','blob') and \
7630 db._adapter.maxcharlength < field.length:
7631 field.length = db._adapter.maxcharlength
7632 self.ALL = SQLALL(self)
7633
7634 if hasattr(self,'_primarykey'):
7635 for k in self._primarykey:
7636 if k not in self.fields:
7637 raise SyntaxError(
7638 "primarykey must be a list of fields from table '%s " % tablename)
7639 else:
7640 self[k].notnull = True
7641 for field in virtual_fields:
7642 self[field.name] = field
7643
7644 @property
7647
7648 - def update(self,*args,**kwargs):
7649 raise RuntimeError("Syntax Not Supported")
7650
7651 - def _enable_record_versioning(self,
7652 archive_db=None,
7653 archive_name = '%(tablename)s_archive',
7654 current_record = 'current_record',
7655 is_active = 'is_active'):
7656 archive_db = archive_db or self._db
7657 archive_name = archive_name % dict(tablename=self._tablename)
7658 if archive_name in archive_db.tables():
7659 return
7660 fieldnames = self.fields()
7661 field_type = self if archive_db is self._db else 'bigint'
7662 archive_db.define_table(
7663 archive_name,
7664 Field(current_record,field_type),
7665 *[field.clone(unique=False) for field in self])
7666 self._before_update.append(
7667 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record:
7668 archive_record(qset,fs,db[an],cn))
7669 if is_active and is_active in fieldnames:
7670 self._before_delete.append(
7671 lambda qset: qset.update(is_active=False))
7672 newquery = lambda query, t=self: t.is_active == True
7673 query = self._common_filter
7674 if query:
7675 newquery = query & newquery
7676 self._common_filter = newquery
7677
7685
7687 db = self._db
7688 pr = db._pending_references
7689 self._referenced_by = []
7690 for field in self:
7691 fieldname = field.name
7692 field_type = field.type
7693 if isinstance(field_type,str) and field_type[:10] == 'reference ':
7694 ref = field_type[10:].strip()
7695 if not ref.split():
7696 raise SyntaxError('Table: reference to nothing: %s' %ref)
7697 refs = ref.split('.')
7698 rtablename = refs[0]
7699 if not rtablename in db:
7700 pr[rtablename] = pr.get(rtablename,[]) + [field]
7701 continue
7702 rtable = db[rtablename]
7703 if len(refs)==2:
7704 rfieldname = refs[1]
7705 if not hasattr(rtable,'_primarykey'):
7706 raise SyntaxError(
7707 'keyed tables can only reference other keyed tables (for now)')
7708 if rfieldname not in rtable.fields:
7709 raise SyntaxError(
7710 "invalid field '%s' for referenced table '%s' in table '%s'" \
7711 % (rfieldname, rtablename, self._tablename))
7712 rtable._referenced_by.append(field)
7713 for referee in pr.get(self._tablename,[]):
7714 self._referenced_by.append(referee)
7715
7717 return dict([(k, v) for (k, v) in record.iteritems() if k
7718 in self.fields and (self[k].type!='id' or id)])
7719
7721 """ for keyed table only """
7722 query = None
7723 for k,v in key.iteritems():
7724 if k in self._primarykey:
7725 if query:
7726 query = query & (self[k] == v)
7727 else:
7728 query = (self[k] == v)
7729 else:
7730 raise SyntaxError(
7731 'Field %s is not part of the primary key of %s' % \
7732 (k,self._tablename))
7733 return query
7734
7736 if not key:
7737 return None
7738 elif isinstance(key, dict):
7739 """ for keyed table """
7740 query = self._build_query(key)
7741 rows = self._db(query).select()
7742 if rows:
7743 return rows[0]
7744 return None
7745 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key):
7746 return self._db(self._id == key).select(limitby=(0,1)).first()
7747 elif key:
7748 return ogetattr(self, str(key))
7749
7751 for_update = kwargs.get('_for_update',False)
7752 if '_for_update' in kwargs: del kwargs['_for_update']
7753
7754 orderby = kwargs.get('_orderby',None)
7755 if '_orderby' in kwargs: del kwargs['_orderby']
7756
7757 if not key is DEFAULT:
7758 if isinstance(key, Query):
7759 record = self._db(key).select(
7760 limitby=(0,1),for_update=for_update, orderby=orderby).first()
7761 elif not str(key).isdigit():
7762 record = None
7763 else:
7764 record = self._db(self._id == key).select(
7765 limitby=(0,1),for_update=for_update, orderby=orderby).first()
7766 if record:
7767 for k,v in kwargs.iteritems():
7768 if record[k]!=v: return None
7769 return record
7770 elif kwargs:
7771 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
7772 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby).first()
7773 else:
7774 return None
7775
7777 if isinstance(key, dict) and isinstance(value, dict):
7778 """ option for keyed table """
7779 if set(key.keys()) == set(self._primarykey):
7780 value = self._filter_fields(value)
7781 kv = {}
7782 kv.update(value)
7783 kv.update(key)
7784 if not self.insert(**kv):
7785 query = self._build_query(key)
7786 self._db(query).update(**self._filter_fields(value))
7787 else:
7788 raise SyntaxError(
7789 'key must have all fields from primary key: %s'%\
7790 (self._primarykey))
7791 elif str(key).isdigit():
7792 if key == 0:
7793 self.insert(**self._filter_fields(value))
7794 elif self._db(self._id == key)\
7795 .update(**self._filter_fields(value)) is None:
7796 raise SyntaxError('No such record: %s' % key)
7797 else:
7798 if isinstance(key, dict):
7799 raise SyntaxError(
7800 'value must be a dictionary: %s' % value)
7801 osetattr(self, str(key), value)
7802
7803 __getattr__ = __getitem__
7804
7806 if key[:1]!='_' and key in self:
7807 raise SyntaxError('Object exists and cannot be redefined: %s' % key)
7808 osetattr(self,key,value)
7809
7811 if isinstance(key, dict):
7812 query = self._build_query(key)
7813 if not self._db(query).delete():
7814 raise SyntaxError('No such record: %s' % key)
7815 elif not str(key).isdigit() or \
7816 not self._db(self._id == key).delete():
7817 raise SyntaxError('No such record: %s' % key)
7818
7820 return hasattr(self,key)
7821
7822 has_key = __contains__
7823
7825 return self.__dict__.items()
7826
7828 for fieldname in self.fields:
7829 yield self[fieldname]
7830
7833
7834
7836 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
7837
7839 if hasattr(self,'_ot') and self._ot is not None:
7840 if 'Oracle' in str(type(self._db._adapter)):
7841 return '%s %s' % (self._ot, self._tablename)
7842 return '%s AS %s' % (self._ot, self._tablename)
7843 return self._tablename
7844
7845 - def _drop(self, mode = ''):
7846 return self._db._adapter._drop(self, mode)
7847
7848 - def drop(self, mode = ''):
7849 return self._db._adapter.drop(self,mode)
7850
7851 - def _listify(self,fields,update=False):
7852 new_fields = {}
7853
7854
7855 for name in fields:
7856 if not name in self.fields:
7857 if name != 'id':
7858 raise SyntaxError(
7859 'Field %s does not belong to the table' % name)
7860 else:
7861 field = self[name]
7862 value = fields[name]
7863 if field.filter_in:
7864 value = field.filter_in(value)
7865 new_fields[name] = (field,value)
7866
7867
7868 to_compute = []
7869 for ofield in self:
7870 name = ofield.name
7871 if not name in new_fields:
7872
7873 if ofield.compute:
7874 to_compute.append((name,ofield))
7875
7876 elif not update and not ofield.default is None:
7877 value = ofield.default
7878 fields[name] = value
7879 new_fields[name] = (ofield,value)
7880
7881 elif update and not ofield.update is None:
7882 value = ofield.update
7883 fields[name] = value
7884 new_fields[name] = (ofield,value)
7885
7886 elif not update and ofield.required:
7887 raise RuntimeError(
7888 'Table: missing required field: %s' % name)
7889
7890 if to_compute:
7891 row = Row(fields)
7892 for name,ofield in to_compute:
7893
7894 try:
7895 new_fields[name] = (ofield,ofield.compute(row))
7896 except (KeyError, AttributeError):
7897
7898 if ofield.required:
7899 raise SyntaxError('unable to comput field: %s' % name)
7900 return new_fields.values()
7901
7903 for field in self:
7904 if field.type=='upload' and field.name in fields:
7905 value = fields[field.name]
7906 if value and not isinstance(value,str):
7907 if hasattr(value,'file') and hasattr(value,'filename'):
7908 new_name = field.store(value.file,filename=value.filename)
7909 elif hasattr(value,'read') and hasattr(value,'name'):
7910 new_name = field.store(value,filename=value.name)
7911 else:
7912 raise RuntimeError("Unable to handle upload")
7913 fields[field.name] = new_name
7914
7917
7926
7942
7944 if _key is DEFAULT:
7945 record = self(**values)
7946 elif isinstance(_key,dict):
7947 record = self(**_key)
7948 else:
7949 record = self(_key)
7950 if record:
7951 record.update_record(**values)
7952 newid = None
7953 else:
7954 newid = self.insert(**values)
7955 return newid
7956
7958 """
7959 here items is a list of dictionaries
7960 """
7961 items = [self._listify(item) for item in items]
7962 if any(f(item) for item in items for f in self._before_insert):return 0
7963 ret = self._db._adapter.bulk_insert(self,items)
7964 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert]
7965 return ret
7966
7968 return self._db._adapter._truncate(self, mode)
7969
7971 return self._db._adapter.truncate(self, mode)
7972
7973 - def import_from_csv_file(
7974 self,
7975 csvfile,
7976 id_map=None,
7977 null='<NULL>',
7978 unique='uuid',
7979 id_offset=None,
7980 *args, **kwargs
7981 ):
7982 """
7983 Import records from csv file.
7984 Column headers must have same names as table fields.
7985 Field 'id' is ignored.
7986 If column names read 'table.file' the 'table.' prefix is ignored.
7987 'unique' argument is a field which must be unique
7988 (typically a uuid field)
7989 'restore' argument is default False;
7990 if set True will remove old values in table first.
7991 'id_map' ff set to None will not map ids.
7992 The import will keep the id numbers in the restored table.
7993 This assumes that there is an field of type id that
7994 is integer and in incrementing order.
7995 Will keep the id numbers in restored table.
7996 """
7997
7998 delimiter = kwargs.get('delimiter', ',')
7999 quotechar = kwargs.get('quotechar', '"')
8000 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
8001 restore = kwargs.get('restore', False)
8002 if restore:
8003 self._db[self].truncate()
8004
8005 reader = csv.reader(csvfile, delimiter=delimiter,
8006 quotechar=quotechar, quoting=quoting)
8007 colnames = None
8008 if isinstance(id_map, dict):
8009 if not self._tablename in id_map:
8010 id_map[self._tablename] = {}
8011 id_map_self = id_map[self._tablename]
8012
8013 def fix(field, value, id_map, id_offset):
8014 list_reference_s='list:reference'
8015 if value == null:
8016 value = None
8017 elif field.type=='blob':
8018 value = base64.b64decode(value)
8019 elif field.type=='double' or field.type=='float':
8020 if not value.strip():
8021 value = None
8022 else:
8023 value = float(value)
8024 elif field.type in ('integer','bigint'):
8025 if not value.strip():
8026 value = None
8027 else:
8028 value = int(value)
8029 elif field.type.startswith('list:string'):
8030 value = bar_decode_string(value)
8031 elif field.type.startswith(list_reference_s):
8032 ref_table = field.type[len(list_reference_s):].strip()
8033 if id_map is not None:
8034 value = [id_map[ref_table][int(v)] \
8035 for v in bar_decode_string(value)]
8036 else:
8037 value = [v for v in bar_decode_string(value)]
8038 elif field.type.startswith('list:'):
8039 value = bar_decode_integer(value)
8040 elif id_map and field.type.startswith('reference'):
8041 try:
8042 value = id_map[field.type[9:].strip()][int(value)]
8043 except KeyError:
8044 pass
8045 elif id_offset and field.type.startswith('reference'):
8046 try:
8047 value = id_offset[field.type[9:].strip()]+int(value)
8048 except KeyError:
8049 pass
8050 return (field.name, value)
8051
8052 def is_id(colname):
8053 if colname in self:
8054 return self[colname].type == 'id'
8055 else:
8056 return False
8057
8058 first = True
8059 unique_idx = None
8060 for line in reader:
8061 if not line:
8062 break
8063 if not colnames:
8064 colnames = [x.split('.',1)[-1] for x in line][:len(line)]
8065 cols, cid = [], None
8066 for i,colname in enumerate(colnames):
8067 if is_id(colname):
8068 cid = i
8069 else:
8070 cols.append(i)
8071 if colname == unique:
8072 unique_idx = i
8073 else:
8074 items = [fix(self[colnames[i]], line[i], id_map, id_offset) \
8075 for i in cols if colnames[i] in self.fields]
8076
8077 if not id_map and cid is not None and id_offset is not None and not unique_idx:
8078 csv_id = int(line[cid])
8079 curr_id = self.insert(**dict(items))
8080 if first:
8081 first = False
8082
8083
8084
8085 if curr_id>csv_id:
8086 id_offset[self._tablename] = curr_id-csv_id
8087 else:
8088 id_offset[self._tablename] = 0
8089
8090 while curr_id<csv_id+id_offset[self._tablename]:
8091 self._db(self._db[self][colnames[cid]] == curr_id).delete()
8092 curr_id = self.insert(**dict(items))
8093
8094
8095 elif not unique_idx:
8096 new_id = self.insert(**dict(items))
8097 else:
8098 unique_value = line[unique_idx]
8099 query = self._db[self][unique] == unique_value
8100 record = self._db(query).select().first()
8101 if record:
8102 record.update_record(**dict(items))
8103 new_id = record[self._id.name]
8104 else:
8105 new_id = self.insert(**dict(items))
8106 if id_map and cid is not None:
8107 id_map_self[int(line[cid])] = new_id
8108
8111
8112 - def on(self, query):
8113 return Expression(self._db,self._db._adapter.ON,self,query)
8114
8116 tablenames = qset.db._adapter.tables(qset.query)
8117 if len(tablenames)!=1: raise RuntimeError("cannot update join")
8118 table = qset.db[tablenames[0]]
8119 for row in qset.select():
8120 fields = archive_table._filter_fields(row)
8121 fields[current_record] = row.id
8122 archive_table.insert(**fields)
8123 return False
8124
8128
8129 - def __init__(
8130 self,
8131 db,
8132 op,
8133 first=None,
8134 second=None,
8135 type=None,
8136 ):
8137
8138 self.db = db
8139 self.op = op
8140 self.first = first
8141 self.second = second
8142 self._table = getattr(first,'_table',None)
8143
8144 if not type and first and hasattr(first,'type'):
8145 self.type = first.type
8146 else:
8147 self.type = type
8148
8152
8156
8160
8164
8168
8172
8176
8180
8184
8188
8192
8196
8200
8204
8208
8212
8216
8218 db = self.db
8219 if start < 0:
8220 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
8221 else:
8222 pos0 = start + 1
8223
8224 if stop < 0:
8225 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
8226 elif stop == sys.maxint:
8227 length = self.len()
8228 else:
8229 length = '(%s - %s)' % (stop + 1, pos0)
8230 return Expression(db,db._adapter.SUBSTRING,
8231 self, (pos0, length), self.type)
8232
8234 return self[i:i + 1]
8235
8237 return self.db._adapter.expand(self,self.type)
8238
8240 db = self.db
8241 return Expression(db,db._adapter.COMMA,self,other,self.type)
8242
8248
8252
8254 db = self.db
8255 if self.type in ('integer','bigint'):
8256 result_type = 'integer'
8257 elif self.type in ['date','time','datetime','double','float']:
8258 result_type = 'double'
8259 else:
8260 raise SyntaxError("subtraction operation not supported for type")
8261 return Expression(db,db._adapter.SUB,self,other,result_type)
8262
8266
8270
8274
8278
8282
8286
8290
8294
8298
8299 - def like(self, value, case_sensitive=False):
8300 db = self.db
8301 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE
8302 return Query(db, op, self, value)
8303
8307
8309 """
8310 Accepts the following inputs:
8311 field.belongs(1,2)
8312 field.belongs((1,2))
8313 field.belongs(query)
8314
8315 Does NOT accept:
8316 field.belongs(1)
8317 """
8318 db = self.db
8319 if len(value) == 1:
8320 value = value[0]
8321 if isinstance(value,Query):
8322 value = db(value)._select(value.first._table._id)
8323 return Query(db, db._adapter.BELONGS, self, value)
8324
8326 db = self.db
8327 if not self.type in ('string', 'text'):
8328 raise SyntaxError("startswith used with incompatible field type")
8329 return Query(db, db._adapter.STARTSWITH, self, value)
8330
8332 db = self.db
8333 if not self.type in ('string', 'text'):
8334 raise SyntaxError("endswith used with incompatible field type")
8335 return Query(db, db._adapter.ENDSWITH, self, value)
8336
8338 db = self.db
8339 if isinstance(value,(list, tuple)):
8340 subqueries = [self.contains(str(v).strip()) for v in value if str(v).strip()]
8341 if not subqueries:
8342 return self.contains('')
8343 else:
8344 return reduce(all and AND or OR,subqueries)
8345 if not self.type in ('string', 'text') and not self.type.startswith('list:'):
8346 raise SyntaxError("contains used with incompatible field type")
8347 return Query(db, db._adapter.CONTAINS, self, value)
8348
8352
8353
8354
8355 - def st_asgeojson(self, precision=15, options=0, version=1):
8359
8360 - def st_astext(self):
8361 db = self.db
8362 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
8363
8367
8371
8375
8379
8383
8387
8391
8395
8399
8404 """
8405 allows defining of custom SQL types
8406
8407 Example::
8408
8409 decimal = SQLCustomType(
8410 type ='double',
8411 native ='integer',
8412 encoder =(lambda x: int(float(x) * 100)),
8413 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
8414 )
8415
8416 db.define_table(
8417 'example',
8418 Field('value', type=decimal)
8419 )
8420
8421 :param type: the web2py type (default = 'string')
8422 :param native: the backend type
8423 :param encoder: how to encode the value to store it in the backend
8424 :param decoder: how to decode the value retrieved from the backend
8425 :param validator: what validators to use ( default = None, will use the
8426 default validator for type)
8427 """
8428
8429 - def __init__(
8430 self,
8431 type='string',
8432 native=None,
8433 encoder=None,
8434 decoder=None,
8435 validator=None,
8436 _class=None,
8437 ):
8438
8439 self.type = type
8440 self.native = native
8441 self.encoder = encoder or (lambda x: x)
8442 self.decoder = decoder or (lambda x: x)
8443 self.validator = validator
8444 self._class = _class or type
8445
8447 try:
8448 return self.type.startswith(self, text)
8449 except TypeError:
8450 return False
8451
8454
8457
8460
8465
8467 - def __init__(self, name, f=None, handler=None):
8473 return ', '.join(str(y) for y in x or [])
8474
8475 -class Field(Expression):
8476
8477 Virtual = FieldVirtual
8478 Method = FieldMethod
8479 Lazy = FieldMethod
8480
8481 """
8482 an instance of this class represents a database field
8483
8484 example::
8485
8486 a = Field(name, 'string', length=32, default=None, required=False,
8487 requires=IS_NOT_EMPTY(), ondelete='CASCADE',
8488 notnull=False, unique=False,
8489 uploadfield=True, widget=None, label=None, comment=None,
8490 uploadfield=True, # True means store on disk,
8491 # 'a_field_name' means store in this field in db
8492 # False means file content will be discarded.
8493 writable=True, readable=True, update=None, authorize=None,
8494 autodelete=False, represent=None, uploadfolder=None,
8495 uploadseparate=False # upload to separate directories by uuid_keys
8496 # first 2 character and tablename.fieldname
8497 # False - old behavior
8498 # True - put uploaded file in
8499 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
8500 # directory)
8501 uploadfs=None # a pyfilesystem where to store upload
8502
8503 to be used as argument of DAL.define_table
8504
8505 allowed field types:
8506 string, boolean, integer, double, text, blob,
8507 date, time, datetime, upload, password
8508
8509 strings must have a length of Adapter.maxcharlength by default (512 or 255 for mysql)
8510 fields should have a default or they will be required in SQLFORMs
8511 the requires argument is used to validate the field input in SQLFORMs
8512
8513 """
8514
8515 - def __init__(
8516 self,
8517 fieldname,
8518 type='string',
8519 length=None,
8520 default=DEFAULT,
8521 required=False,
8522 requires=DEFAULT,
8523 ondelete='CASCADE',
8524 notnull=False,
8525 unique=False,
8526 uploadfield=True,
8527 widget=None,
8528 label=None,
8529 comment=None,
8530 writable=True,
8531 readable=True,
8532 update=None,
8533 authorize=None,
8534 autodelete=False,
8535 represent=None,
8536 uploadfolder=None,
8537 uploadseparate=False,
8538 uploadfs=None,
8539 compute=None,
8540 custom_store=None,
8541 custom_retrieve=None,
8542 custom_retrieve_file_properties=None,
8543 custom_delete=None,
8544 filter_in = None,
8545 filter_out = None,
8546 custom_qualifier = None,
8547 map_none = None,
8548 ):
8549 self._db = self.db = None
8550 self.op = None
8551 self.first = None
8552 self.second = None
8553 self.name = fieldname = cleanup(fieldname)
8554 if not isinstance(fieldname,str) or hasattr(Table,fieldname) or \
8555 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
8556 raise SyntaxError('Field: invalid field name: %s' % fieldname)
8557 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
8558 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512)
8559 self.default = default if default!=DEFAULT else (update or None)
8560 self.required = required
8561 self.ondelete = ondelete.upper()
8562 self.notnull = notnull
8563 self.unique = unique
8564 self.uploadfield = uploadfield
8565 self.uploadfolder = uploadfolder
8566 self.uploadseparate = uploadseparate
8567 self.uploadfs = uploadfs
8568 self.widget = widget
8569 self.comment = comment
8570 self.writable = writable
8571 self.readable = readable
8572 self.update = update
8573 self.authorize = authorize
8574 self.autodelete = autodelete
8575 self.represent = list_represent if \
8576 represent==None and type in ('list:integer','list:string') else represent
8577 self.compute = compute
8578 self.isattachment = True
8579 self.custom_store = custom_store
8580 self.custom_retrieve = custom_retrieve
8581 self.custom_retrieve_file_properties = custom_retrieve_file_properties
8582 self.custom_delete = custom_delete
8583 self.filter_in = filter_in
8584 self.filter_out = filter_out
8585 self.custom_qualifier = custom_qualifier
8586 self.label = label if label!=None else fieldname.replace('_',' ').title()
8587 self.requires = requires if requires!=None else []
8588 self.map_none = map_none
8589
8591 self.__dict__.update(*args,**attributes)
8592
8593 - def clone(self,point_self_references_to=False,**args):
8594 field = copy.copy(self)
8595 if point_self_references_to and \
8596 field.type == 'reference %s'+field._tablename:
8597 field.type = 'reference %s' % point_self_references_to
8598 field.__dict__.update(args)
8599 return field
8600
8601 - def store(self, file, filename=None, path=None):
8602 if self.custom_store:
8603 return self.custom_store(file,filename,path)
8604 if isinstance(file, cgi.FieldStorage):
8605 filename = filename or file.filename
8606 file = file.file
8607 elif not filename:
8608 filename = file.name
8609 filename = os.path.basename(filename.replace('/', os.sep)\
8610 .replace('\\', os.sep))
8611 m = REGEX_STORE_PATTERN.search(filename)
8612 extension = m and m.group('e') or 'txt'
8613 uuid_key = web2py_uuid().replace('-', '')[-16:]
8614 encoded_filename = base64.b16encode(filename).lower()
8615 newfilename = '%s.%s.%s.%s' % \
8616 (self._tablename, self.name, uuid_key, encoded_filename)
8617 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
8618 self_uploadfield = self.uploadfield
8619 if isinstance(self_uploadfield,Field):
8620 blob_uploadfield_name = self_uploadfield.uploadfield
8621 keys={self_uploadfield.name: newfilename,
8622 blob_uploadfield_name: file.read()}
8623 self_uploadfield.table.insert(**keys)
8624 elif self_uploadfield == True:
8625 if path:
8626 pass
8627 elif self.uploadfolder:
8628 path = self.uploadfolder
8629 elif self.db._adapter.folder:
8630 path = pjoin(self.db._adapter.folder, '..', 'uploads')
8631 else:
8632 raise RuntimeError(
8633 "you must specify a Field(...,uploadfolder=...)")
8634 if self.uploadseparate:
8635 if self.uploadfs:
8636 raise RuntimeError("not supported")
8637 path = pjoin(path,"%s.%s" %(self._tablename, self.name),
8638 uuid_key[:2])
8639 if not exists(path):
8640 os.makedirs(path)
8641 pathfilename = pjoin(path, newfilename)
8642 if self.uploadfs:
8643 dest_file = self.uploadfs.open(newfilename, 'wb')
8644 else:
8645 dest_file = open(pathfilename, 'wb')
8646 try:
8647 shutil.copyfileobj(file, dest_file)
8648 except IOError:
8649 raise IOError(
8650 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename)
8651 dest_file.close()
8652 return newfilename
8653
8655 self_uploadfield = self.uploadfield
8656 if self.custom_retrieve:
8657 return self.custom_retrieve(name, path)
8658 import http
8659 if self.authorize or isinstance(self_uploadfield, str):
8660 row = self.db(self == name).select().first()
8661 if not row:
8662 raise http.HTTP(404)
8663 if self.authorize and not self.authorize(row):
8664 raise http.HTTP(403)
8665 m = REGEX_UPLOAD_PATTERN.match(name)
8666 if not m or not self.isattachment:
8667 raise TypeError('Can\'t retrieve %s' % name)
8668 file_properties = self.retrieve_file_properties(name,path)
8669 filename = file_properties['filename']
8670 if isinstance(self_uploadfield, str):
8671 stream = StringIO.StringIO(row[self_uploadfield] or '')
8672 elif isinstance(self_uploadfield,Field):
8673 blob_uploadfield_name = self_uploadfield.uploadfield
8674 query = self_uploadfield == name
8675 data = self_uploadfield.table(query)[blob_uploadfield_name]
8676 stream = StringIO.StringIO(data)
8677 elif self.uploadfs:
8678
8679 stream = self.uploadfs.open(name, 'rb')
8680 else:
8681
8682 stream = open(pjoin(file_properties['path'], name), 'rb')
8683 return (filename, stream)
8684
8686 self_uploadfield = self.uploadfield
8687 if self.custom_retrieve_file_properties:
8688 return self.custom_retrieve_file_properties(name, path)
8689 try:
8690 m = REGEX_UPLOAD_PATTERN.match(name)
8691 if not m or not self.isattachment:
8692 raise TypeError('Can\'t retrieve %s file properties' % name)
8693 filename = base64.b16decode(m.group('name'), True)
8694 filename = REGEX_CLEANUP_FN.sub('_', filename)
8695 except (TypeError, AttributeError):
8696 filename = name
8697 if isinstance(self_uploadfield, str):
8698 return dict(path=None,filename=filename)
8699 elif isinstance(self_uploadfield,Field):
8700 return dict(path=None,filename=filename)
8701 else:
8702
8703 if path:
8704 pass
8705 elif self.uploadfolder:
8706 path = self.uploadfolder
8707 else:
8708 path = pjoin(self.db._adapter.folder, '..', 'uploads')
8709 if self.uploadseparate:
8710 t = m.group('table')
8711 f = m.group('field')
8712 u = m.group('uuidkey')
8713 path = pjoin(path,"%s.%s" % (t,f),u[:2])
8714 return dict(path=path,filename=filename)
8715
8716
8732
8744
8745 - def count(self, distinct=None):
8747
8750
8752 try:
8753 return '%s.%s' % (self.tablename, self.name)
8754 except:
8755 return '<no table>.%s' % self.name
8756
8757
8758 -class Query(object):
8759
8760 """
8761 a query object necessary to define a set.
8762 it can be stored or can be passed to DAL.__call__() to obtain a Set
8763
8764 Example::
8765
8766 query = db.users.name=='Max'
8767 set = db(query)
8768 records = set.select()
8769
8770 """
8771
8772 - def __init__(
8773 self,
8774 db,
8775 op,
8776 first=None,
8777 second=None,
8778 ignore_common_filters = False,
8779 ):
8780 self.db = self._db = db
8781 self.op = op
8782 self.first = first
8783 self.second = second
8784 self.ignore_common_filters = ignore_common_filters
8785
8788
8790 return self.db._adapter.expand(self)
8791
8794
8796 return Query(self.db,self.db._adapter.OR,self,other)
8797
8799 if self.op==self.db._adapter.NOT:
8800 return self.first
8801 return Query(self.db,self.db._adapter.NOT,self)
8802
8803 - def case(self,t=1,f=0):
8804 return self.db._adapter.CASE(self,t,f)
8805
8806
8807
8808 -def xorify(orderby):
8809 if not orderby:
8810 return None
8811 orderby2 = orderby[0]
8812 for item in orderby[1:]:
8813 orderby2 = orderby2 | item
8814 return orderby2
8815
8817 return (query and hasattr(query,'ignore_common_filters') and \
8818 not query.ignore_common_filters)
8819
8821
8822 """
8823 a Set represents a set of records in the database,
8824 the records are identified by the query=Query(...) object.
8825 normally the Set is generated by DAL.__call__(Query(...))
8826
8827 given a set, for example
8828 set = db(db.users.name=='Max')
8829 you can:
8830 set.update(db.users.name='Massimo')
8831 set.delete() # all elements in the set
8832 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
8833 and take subsets:
8834 subset = set(db.users.id<5)
8835 """
8836
8837 - def __init__(self, db, query, ignore_common_filters = None):
8838 self.db = db
8839 self._db = db
8840 if not ignore_common_filters is None and \
8841 use_common_filters(query) == ignore_common_filters:
8842 query = copy.copy(query)
8843 query.ignore_common_filters = ignore_common_filters
8844 self.query = query
8845
8848
8849 - def __call__(self, query, ignore_common_filters=False):
8850 if isinstance(query,Table):
8851 query = self.db._adapter.id_query(query)
8852 elif isinstance(query,str):
8853 query = Expression(self.db,query)
8854 elif isinstance(query,Field):
8855 query = query!=None
8856 if self.query:
8857 return Set(self.db, self.query & query,
8858 ignore_common_filters = ignore_common_filters)
8859 else:
8860 return Set(self.db, query,
8861 ignore_common_filters = ignore_common_filters)
8862
8863 - def _count(self,distinct=None):
8864 return self.db._adapter._count(self.query,distinct)
8865
8866 - def _select(self, *fields, **attributes):
8867 adapter = self.db._adapter
8868 tablenames = adapter.tables(self.query,
8869 attributes.get('join',None),
8870 attributes.get('left',None),
8871 attributes.get('orderby',None),
8872 attributes.get('groupby',None))
8873 fields = adapter.expand_all(fields, tablenames)
8874 return adapter._select(self.query,fields,attributes)
8875
8877 db = self.db
8878 tablename = db._adapter.get_table(self.query)
8879 return db._adapter._delete(tablename,self.query)
8880
8881 - def _update(self, **update_fields):
8886
8888 return not self.select(limitby=(0,1))
8889
8890 - def count(self,distinct=None, cache=None):
8891 db = self.db
8892 if cache:
8893 cache_model, time_expire = cache
8894 sql = self._count(distinct=distinct)
8895 key = db._uri + '/' + sql
8896 if len(key)>200: key = hashlib_md5(key).hexdigest()
8897 return cache_model(
8898 key,
8899 (lambda self=self,distinct=distinct: \
8900 db._adapter.count(self.query,distinct)),
8901 time_expire)
8902 return db._adapter.count(self.query,distinct)
8903
8904 - def select(self, *fields, **attributes):
8905 adapter = self.db._adapter
8906 tablenames = adapter.tables(self.query,
8907 attributes.get('join',None),
8908 attributes.get('left',None),
8909 attributes.get('orderby',None),
8910 attributes.get('groupby',None))
8911 fields = adapter.expand_all(fields, tablenames)
8912 return adapter.select(self.query,fields,attributes)
8913
8916
8918 db = self.db
8919 tablename = db._adapter.get_table(self.query)
8920 table = db[tablename]
8921 if any(f(self) for f in table._before_delete): return 0
8922 ret = db._adapter.delete(tablename,self.query)
8923 ret and [f(self) for f in table._after_delete]
8924 return ret
8925
8926 - def update(self, **update_fields):
8927 db = self.db
8928 tablename = db._adapter.get_table(self.query)
8929 table = db[tablename]
8930 table._attempt_upload(update_fields)
8931 if any(f(self,update_fields) for f in table._before_update):
8932 return 0
8933 fields = table._listify(update_fields,update=True)
8934 if not fields:
8935 raise SyntaxError("No fields to update")
8936 ret = db._adapter.update(tablename,self.query,fields)
8937 ret and [f(self,update_fields) for f in table._after_update]
8938 return ret
8939
8941 """
8942 same as update but does not call table._before_update and _after_update
8943 """
8944 tablename = self.db._adapter.get_table(self.query)
8945 table = self.db[tablename]
8946 fields = table._listify(update_fields,update=True)
8947 if not fields: raise SyntaxError("No fields to update")
8948 ret = self.db._adapter.update(tablename,self.query,fields)
8949 return ret
8950
8952 tablename = self.db._adapter.get_table(self.query)
8953 response = Row()
8954 response.errors = Row()
8955 new_fields = copy.copy(update_fields)
8956 for key,value in update_fields.iteritems():
8957 value,error = self.db[tablename][key].validate(value)
8958 if error:
8959 response.errors[key] = error
8960 else:
8961 new_fields[key] = value
8962 table = self.db[tablename]
8963 if response.errors:
8964 response.updated = None
8965 else:
8966 if not any(f(self,new_fields) for f in table._before_update):
8967 fields = table._listify(new_fields,update=True)
8968 if not fields: raise SyntaxError("No fields to update")
8969 ret = self.db._adapter.update(tablename,self.query,fields)
8970 ret and [f(self,new_fields) for f in table._after_update]
8971 else:
8972 ret = 0
8973 response.updated = ret
8974 return response
8975
8977 table = self.db[self.db._adapter.tables(self.query)[0]]
8978
8979 if upload_fields:
8980 fields = upload_fields.keys()
8981 else:
8982 fields = table.fields
8983 fields = [f for f in fields if table[f].type == 'upload'
8984 and table[f].uploadfield == True
8985 and table[f].autodelete]
8986 if not fields:
8987 return False
8988 for record in self.select(*[table[f] for f in fields]):
8989 for fieldname in fields:
8990 field = table[fieldname]
8991 oldname = record.get(fieldname, None)
8992 if not oldname:
8993 continue
8994 if upload_fields and oldname == upload_fields[fieldname]:
8995 continue
8996 if field.custom_delete:
8997 field.custom_delete(oldname)
8998 else:
8999 uploadfolder = field.uploadfolder
9000 if not uploadfolder:
9001 uploadfolder = pjoin(
9002 self.db._adapter.folder, '..', 'uploads')
9003 if field.uploadseparate:
9004 items = oldname.split('.')
9005 uploadfolder = pjoin(
9006 uploadfolder,
9007 "%s.%s" % (items[0], items[1]),
9008 items[2][:2])
9009 oldpath = pjoin(uploadfolder, oldname)
9010 if exists(oldpath):
9011 os.unlink(oldpath)
9012 return False
9013
9015 - def __init__(self, colset, table, id):
9016 self.colset, self.db, self.tablename, self.id = \
9017 colset, table._db, table._tablename, id
9018
9020 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
9021 table = db[tablename]
9022 newfields = fields or dict(colset)
9023 for fieldname in newfields.keys():
9024 if not fieldname in table.fields or table[fieldname].type=='id':
9025 del newfields[fieldname]
9026 table._db(table._id==id,ignore_common_filters=True).update(**newfields)
9027 colset.update(newfields)
9028 return colset
9032 self.db, self.tablename, self.id = table._db, table._tablename, id
9034 return self.db(self.db[self.tablename]._id==self.id).delete()
9038 self.db, self.tablename, self.fieldname, self.id = \
9039 field.db, field._tablename, field.name, id
9041 query = self.db[self.tablename][self.fieldname]==self.id
9042 return Set(self.db,query)
9045 - def __call__(self, query, ignore_common_filters=False):
9046 return self._getset()(query, ignore_common_filters)
9047 - def _count(self,distinct=None):
9049 - def _select(self, *fields, **attributes):
9053 - def _update(self, **update_fields):
9057 - def count(self,distinct=None, cache=None):
9059 - def select(self, *fields, **attributes):
9065 - def update(self, **update_fields):
9076 self.method=method
9077 self.row=row
9079 return self.method(self.row,*args,**kwargs)
9082 f.__lazy__ = True
9083 return f
9084
9085 -class Rows(object):
9086
9087 """
9088 A wrapper for the return value of a select. It basically represents a table.
9089 It has an iterator and each row is represented as a dictionary.
9090 """
9091
9092
9093
9094 - def __init__(
9095 self,
9096 db=None,
9097 records=[],
9098 colnames=[],
9099 compact=True,
9100 rawrows=None
9101 ):
9102 self.db = db
9103 self.records = records
9104 self.colnames = colnames
9105 self.compact = compact
9106 self.response = rawrows
9107
9109 return '<Rows (%s)>' % len(self.records)
9110
9112 """
9113 db.define_table('x',Field('number','integer'))
9114 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
9115
9116 from gluon.dal import lazy_virtualfield
9117
9118 class MyVirtualFields(object):
9119 # normal virtual field (backward compatible, discouraged)
9120 def normal_shift(self): return self.x.number+1
9121 # lazy virtual field (because of @staticmethod)
9122 @lazy_virtualfield
9123 def lazy_shift(instance,row,delta=4): return row.x.number+delta
9124 db.x.virtualfields.append(MyVirtualFields())
9125
9126 for row in db(db.x).select():
9127 print row.number, row.normal_shift, row.lazy_shift(delta=7)
9128 """
9129 if not keyed_virtualfields:
9130 return self
9131 for row in self.records:
9132 for (tablename,virtualfields) in keyed_virtualfields.iteritems():
9133 attributes = dir(virtualfields)
9134 if not tablename in row:
9135 box = row[tablename] = Row()
9136 else:
9137 box = row[tablename]
9138 updated = False
9139 for attribute in attributes:
9140 if attribute[0] != '_':
9141 method = getattr(virtualfields,attribute)
9142 if hasattr(method,'__lazy__'):
9143 box[attribute]=VirtualCommand(method,row)
9144 elif type(method)==types.MethodType:
9145 if not updated:
9146 virtualfields.__dict__.update(row)
9147 updated = True
9148 box[attribute]=method()
9149 return self
9150
9152 if self.colnames!=other.colnames:
9153 raise Exception('Cannot & incompatible Rows objects')
9154 records = self.records+other.records
9155 return Rows(self.db,records,self.colnames)
9156
9158 if self.colnames!=other.colnames:
9159 raise Exception('Cannot | incompatible Rows objects')
9160 records = self.records
9161 records += [record for record in other.records \
9162 if not record in records]
9163 return Rows(self.db,records,self.colnames)
9164
9166 if len(self.records):
9167 return 1
9168 return 0
9169
9171 return len(self.records)
9172
9174 return Rows(self.db,self.records[a:b],self.colnames)
9175
9177 row = self.records[i]
9178 keys = row.keys()
9179 if self.compact and len(keys) == 1 and keys[0] != '_extra':
9180 return row[row.keys()[0]]
9181 return row
9182
9184 """
9185 iterator over records
9186 """
9187
9188 for i in xrange(len(self)):
9189 yield self[i]
9190
9192 """
9193 serializes the table into a csv file
9194 """
9195
9196 s = StringIO.StringIO()
9197 self.export_to_csv_file(s)
9198 return s.getvalue()
9199
9201 if not self.records:
9202 return None
9203 return self[0]
9204
9206 if not self.records:
9207 return None
9208 return self[-1]
9209
9210 - def find(self,f,limitby=None):
9211 """
9212 returns a new Rows object, a subset of the original object,
9213 filtered by the function f
9214 """
9215 if not self:
9216 return Rows(self.db, [], self.colnames)
9217 records = []
9218 if limitby:
9219 a,b = limitby
9220 else:
9221 a,b = 0,len(self)
9222 k = 0
9223 for row in self:
9224 if f(row):
9225 if a<=k: records.append(row)
9226 k += 1
9227 if k==b: break
9228 return Rows(self.db, records, self.colnames)
9229
9231 """
9232 removes elements from the calling Rows object, filtered by the function f,
9233 and returns a new Rows object containing the removed elements
9234 """
9235 if not self.records:
9236 return Rows(self.db, [], self.colnames)
9237 removed = []
9238 i=0
9239 while i<len(self):
9240 row = self[i]
9241 if f(row):
9242 removed.append(self.records[i])
9243 del self.records[i]
9244 else:
9245 i += 1
9246 return Rows(self.db, removed, self.colnames)
9247
9248 - def sort(self, f, reverse=False):
9249 """
9250 returns a list of sorted elements (not sorted in place)
9251 """
9252 rows = Rows(self.db,[],self.colnames,compact=False)
9253 rows.records = sorted(self,key=f,reverse=reverse)
9254 return rows
9255
9256
9258 """
9259 regroups the rows, by one of the fields
9260 """
9261 if not self.records:
9262 return {}
9263 key = str(field)
9264 grouped_row_group = dict()
9265
9266 for row in self:
9267 value = row[key]
9268 if not value in grouped_row_group:
9269 grouped_row_group[value] = [row]
9270 else:
9271 grouped_row_group[value].append(row)
9272 return grouped_row_group
9273
9274 - def as_list(self,
9275 compact=True,
9276 storage_to_dict=True,
9277 datetime_to_str=True,
9278 custom_types=None):
9279 """
9280 returns the data as a list or dictionary.
9281 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
9282 :param datetime_to_str: convert datetime fields as strings (default True)
9283 """
9284 (oc, self.compact) = (self.compact, compact)
9285 if storage_to_dict:
9286 items = [item.as_dict(datetime_to_str, custom_types) for item in self]
9287 else:
9288 items = [item for item in self]
9289 self.compact = compact
9290 return items
9291
9292
9293 - def as_dict(self,
9294 key='id',
9295 compact=True,
9296 storage_to_dict=True,
9297 datetime_to_str=True,
9298 custom_types=None):
9299 """
9300 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
9301
9302 :param key: the name of the field to be used as dict key, normally the id
9303 :param compact: ? (default True)
9304 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
9305 :param datetime_to_str: convert datetime fields as strings (default True)
9306 """
9307 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types)
9308 if isinstance(key,str) and key.count('.')==1:
9309 (table, field) = key.split('.')
9310 return dict([(r[table][field],r) for r in rows])
9311 elif isinstance(key,str):
9312 return dict([(r[key],r) for r in rows])
9313 else:
9314 return dict([(key(r),r) for r in rows])
9315
9317 """
9318 export data to csv, the first line contains the column names
9319
9320 :param ofile: where the csv must be exported to
9321 :param null: how null values must be represented (default '<NULL>')
9322 :param delimiter: delimiter to separate values (default ',')
9323 :param quotechar: character to use to quote string values (default '"')
9324 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
9325 :param represent: use the fields .represent value (default False)
9326 :param colnames: list of column names to use (default self.colnames)
9327 This will only work when exporting rows objects!!!!
9328 DO NOT use this with db.export_to_csv()
9329 """
9330 delimiter = kwargs.get('delimiter', ',')
9331 quotechar = kwargs.get('quotechar', '"')
9332 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
9333 represent = kwargs.get('represent', False)
9334 writer = csv.writer(ofile, delimiter=delimiter,
9335 quotechar=quotechar, quoting=quoting)
9336 colnames = kwargs.get('colnames', self.colnames)
9337 write_colnames = kwargs.get('write_colnames',True)
9338
9339 if write_colnames:
9340 writer.writerow(colnames)
9341
9342 def none_exception(value):
9343 """
9344 returns a cleaned up value that can be used for csv export:
9345 - unicode text is encoded as such
9346 - None values are replaced with the given representation (default <NULL>)
9347 """
9348 if value is None:
9349 return null
9350 elif isinstance(value, unicode):
9351 return value.encode('utf8')
9352 elif isinstance(value,Reference):
9353 return int(value)
9354 elif hasattr(value, 'isoformat'):
9355 return value.isoformat()[:19].replace('T', ' ')
9356 elif isinstance(value, (list,tuple)):
9357 return bar_encode(value)
9358 return value
9359
9360 for record in self:
9361 row = []
9362 for col in colnames:
9363 if not REGEX_TABLE_DOT_FIELD.match(col):
9364 row.append(record._extra[col])
9365 else:
9366 (t, f) = col.split('.')
9367 field = self.db[t][f]
9368 if isinstance(record.get(t, None), (Row,dict)):
9369 value = record[t][f]
9370 else:
9371 value = record[f]
9372 if field.type=='blob' and not value is None:
9373 value = base64.b64encode(value)
9374 elif represent and field.represent:
9375 value = field.represent(value)
9376 row.append(none_exception(value))
9377 writer.writerow(row)
9378
9379 - def xml(self,strict=False,row_name='row',rows_name='rows'):
9380 """
9381 serializes the table using sqlhtml.SQLTABLE (if present)
9382 """
9383 if strict:
9384 ncols = len(self.colnames)
9385 def f(row,field,indent=' '):
9386 if isinstance(row,Row):
9387 spc = indent+' \n'
9388 items = [f(row[x],x,indent+' ') for x in row]
9389 return '%s<%s>\n%s\n%s</%s>' % (
9390 indent,
9391 field,
9392 spc.join(item for item in items if item),
9393 indent,
9394 field)
9395 elif not callable(row):
9396 if REGEX_ALPHANUMERIC.match(field):
9397 return '%s<%s>%s</%s>' % (indent,field,row,field)
9398 else:
9399 return '%s<extra name="%s">%s</extra>' % \
9400 (indent,field,row)
9401 else:
9402 return None
9403 return '<%s>\n%s\n</%s>' % (
9404 rows_name,
9405 '\n'.join(f(row,row_name) for row in self),
9406 rows_name)
9407 import sqlhtml
9408 return sqlhtml.SQLTABLE(self).xml()
9409
9410 - def json(self, mode='object', default=None):
9411 """
9412 serializes the table to a JSON list of objects
9413 """
9414 mode = mode.lower()
9415 if not mode in ['object', 'array']:
9416 raise SyntaxError('Invalid JSON serialization mode: %s' % mode)
9417
9418 def inner_loop(record, col):
9419 (t, f) = col.split('.')
9420 res = None
9421 if not REGEX_TABLE_DOT_FIELD.match(col):
9422 key = col
9423 res = record._extra[col]
9424 else:
9425 key = f
9426 if isinstance(record.get(t, None), Row):
9427 res = record[t][f]
9428 else:
9429 res = record[f]
9430 if mode == 'object':
9431 return (key, res)
9432 else:
9433 return res
9434
9435 if mode == 'object':
9436 items = [dict([inner_loop(record, col) for col in
9437 self.colnames]) for record in self]
9438 else:
9439 items = [[inner_loop(record, col) for col in self.colnames]
9440 for record in self]
9441 if have_serializers:
9442 return serializers.json(items,default=default or serializers.custom_json)
9443 else:
9444 try:
9445 import json as simplejson
9446 except ImportError:
9447 import gluon.contrib.simplejson as simplejson
9448 return simplejson.dumps(items)
9449
9450
9451
9452
9453
9454
9455 -def test_all():
9456 """
9457
9458 >>> if len(sys.argv)<2: db = DAL(\"sqlite://test.db\")
9459 >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
9460 >>> tmp = db.define_table('users',\
9461 Field('stringf', 'string', length=32, required=True),\
9462 Field('booleanf', 'boolean', default=False),\
9463 Field('passwordf', 'password', notnull=True),\
9464 Field('uploadf', 'upload'),\
9465 Field('blobf', 'blob'),\
9466 Field('integerf', 'integer', unique=True),\
9467 Field('doublef', 'double', unique=True,notnull=True),\
9468 Field('datef', 'date', default=datetime.date.today()),\
9469 Field('timef', 'time'),\
9470 Field('datetimef', 'datetime'),\
9471 migrate='test_user.table')
9472
9473 Insert a field
9474
9475 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
9476 uploadf=None, integerf=5, doublef=3.14,\
9477 datef=datetime.date(2001, 1, 1),\
9478 timef=datetime.time(12, 30, 15),\
9479 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
9480 1
9481
9482 Drop the table
9483
9484 >>> db.users.drop()
9485
9486 Examples of insert, select, update, delete
9487
9488 >>> tmp = db.define_table('person',\
9489 Field('name'),\
9490 Field('birth','date'),\
9491 migrate='test_person.table')
9492 >>> person_id = db.person.insert(name=\"Marco\",birth='2005-06-22')
9493 >>> person_id = db.person.insert(name=\"Massimo\",birth='1971-12-21')
9494
9495 commented len(db().select(db.person.ALL))
9496 commented 2
9497
9498 >>> me = db(db.person.id==person_id).select()[0] # test select
9499 >>> me.name
9500 'Massimo'
9501 >>> db.person[2].name
9502 'Massimo'
9503 >>> db.person(2).name
9504 'Massimo'
9505 >>> db.person(name='Massimo').name
9506 'Massimo'
9507 >>> db.person(db.person.name=='Massimo').name
9508 'Massimo'
9509 >>> row = db.person[2]
9510 >>> row.name == row['name'] == row['person.name'] == row('person.name')
9511 True
9512 >>> db(db.person.name=='Massimo').update(name='massimo') # test update
9513 1
9514 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
9515 1
9516
9517 Update a single record
9518
9519 >>> me.update_record(name=\"Max\")
9520 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}>
9521 >>> me.name
9522 'Max'
9523
9524 Examples of complex search conditions
9525
9526 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
9527 1
9528 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
9529 1
9530 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
9531 1
9532 >>> me = db(db.person.id==person_id).select(db.person.name)[0]
9533 >>> me.name
9534 'Max'
9535
9536 Examples of search conditions using extract from date/datetime/time
9537
9538 >>> len(db(db.person.birth.month()==12).select())
9539 1
9540 >>> len(db(db.person.birth.year()>1900).select())
9541 1
9542
9543 Example of usage of NULL
9544
9545 >>> len(db(db.person.birth==None).select()) ### test NULL
9546 0
9547 >>> len(db(db.person.birth!=None).select()) ### test NULL
9548 1
9549
9550 Examples of search conditions using lower, upper, and like
9551
9552 >>> len(db(db.person.name.upper()=='MAX').select())
9553 1
9554 >>> len(db(db.person.name.like('%ax')).select())
9555 1
9556 >>> len(db(db.person.name.upper().like('%AX')).select())
9557 1
9558 >>> len(db(~db.person.name.upper().like('%AX')).select())
9559 0
9560
9561 orderby, groupby and limitby
9562
9563 >>> people = db().select(db.person.name, orderby=db.person.name)
9564 >>> order = db.person.name|~db.person.birth
9565 >>> people = db().select(db.person.name, orderby=order)
9566
9567 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
9568
9569 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
9570
9571 Example of one 2 many relation
9572
9573 >>> tmp = db.define_table('dog',\
9574 Field('name'),\
9575 Field('birth','date'),\
9576 Field('owner',db.person),\
9577 migrate='test_dog.table')
9578 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
9579 1
9580
9581 A simple JOIN
9582
9583 >>> len(db(db.dog.owner==db.person.id).select())
9584 1
9585
9586 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
9587 1
9588
9589 Drop tables
9590
9591 >>> db.dog.drop()
9592 >>> db.person.drop()
9593
9594 Example of many 2 many relation and Set
9595
9596 >>> tmp = db.define_table('author', Field('name'),\
9597 migrate='test_author.table')
9598 >>> tmp = db.define_table('paper', Field('title'),\
9599 migrate='test_paper.table')
9600 >>> tmp = db.define_table('authorship',\
9601 Field('author_id', db.author),\
9602 Field('paper_id', db.paper),\
9603 migrate='test_authorship.table')
9604 >>> aid = db.author.insert(name='Massimo')
9605 >>> pid = db.paper.insert(title='QCD')
9606 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
9607
9608 Define a Set
9609
9610 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
9611 >>> rows = authored_papers.select(db.author.name, db.paper.title)
9612 >>> for row in rows: print row.author.name, row.paper.title
9613 Massimo QCD
9614
9615 Example of search condition using belongs
9616
9617 >>> set = (1, 2, 3)
9618 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
9619 >>> print rows[0].title
9620 QCD
9621
9622 Example of search condition using nested select
9623
9624 >>> nested_select = db()._select(db.authorship.paper_id)
9625 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
9626 >>> print rows[0].title
9627 QCD
9628
9629 Example of expressions
9630
9631 >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
9632 >>> db(mynumber).delete()
9633 0
9634 >>> for i in range(10): tmp = mynumber.insert(x=i)
9635 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum())
9636 45
9637
9638 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
9639 5
9640
9641 Output in csv
9642
9643 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
9644 author.name,paper.title\r
9645 Massimo,QCD
9646
9647 Delete all leftover tables
9648
9649 >>> DAL.distributed_transaction_commit(db)
9650
9651 >>> db.mynumber.drop()
9652 >>> db.authorship.drop()
9653 >>> db.author.drop()
9654 >>> db.paper.drop()
9655 """
9656
9657
9658
9659
9660 SQLField = Field
9661 SQLTable = Table
9662 SQLXorable = Expression
9663 SQLQuery = Query
9664 SQLSet = Set
9665 SQLRows = Rows
9666 SQLStorage = Row
9667 SQLDB = DAL
9668 GQLDB = DAL
9669 DAL.Field = Field
9670 DAL.Table = Table
9671
9672
9673
9674
9675
9676 -def geoPoint(*line):
9677 return "POINT (%f %f)" % (x,y)
9678
9680 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
9681
9683 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
9684
9685
9686
9687
9688
9689 if __name__ == '__main__':
9690 import doctest
9691 doctest.testmod()
9692