diff --git a/VERSION b/VERSION index 7b48a4bd..e88b45a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.00.1 (2012-08-28 22:14:54) rc4 +Version 2.00.1 (2012-08-29 07:27:23) rc4 diff --git a/gluon/dal.py b/gluon/dal.py index fbde461a..5f9b5b0e 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -39,7 +39,7 @@ Example of usage: >>> # from dal import DAL, Field ### create DAL connection (and create DB if it doesn't exist) ->>> db = DAL(('mysql://a:b@localhost/x', 'sqlite://storage.sqlite'), +>>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'), ... folder=None) ### define a table 'person' (create/alter as necessary) @@ -174,7 +174,21 @@ import platform CALLABLETYPES = (types.LambdaType, types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.BuiltinMethodType) -TABLE_ARGS = ('migrate','primarykey','fake_migrate','format','singular','plural','trigger_name','sequence_name','common_filter','polymodel','table_class','on_define') + +TABLE_ARGS = set( + ('migrate','primarykey','fake_migrate','format', + 'singular','plural','trigger_name','sequence_name', + 'common_filter','polymodel','table_class','on_define')) + +SELECT_ARGS = set( + ('orderby', 'groupby', 'limitby','required', 'cache', 'left', + 'distinct', 'having', 'join','for_update', 'processor','cacheable')) + +ogetattr = object.__getattribute__ +osetattr = object.__setattr__ +exists = os.path.exists +pjoin = os.path.join + ################################################################################### # following checks allow the use of dal without web2py, as a standalone module @@ -492,7 +506,7 @@ class ConnectionPool(object): self.folder = thread.folder = '' # Creating the folder if it does not exist - if False and self.folder and not os.path.exists(self.folder): + if False and self.folder and not exists(self.folder): os.mkdir(self.folder) def after_connection(self): @@ -513,6 +527,7 @@ class ConnectionPool(object): if the connection is not active (closed by db server) it will loop if not self.pool_size or no active connections in pool makes a new one """ + pools = ConnectionPool.pools self._connection_function = f if not self.pool_size: self.connection = f() @@ -521,10 +536,10 @@ class ConnectionPool(object): uri = self.uri while True: sql_locker.acquire() - if not uri in ConnectionPool.pools: - ConnectionPool.pools[uri] = [] - if ConnectionPool.pools[uri]: - self.connection = ConnectionPool.pools[uri].pop() + if not uri in pools: + pools[uri] = [] + if pools[uri]: + self.connection = pools[uri].pop() sql_locker.release() self.cursor = cursor and self.connection.cursor() try: @@ -593,7 +608,7 @@ class BaseAdapter(ConnectionPool): """ to be used ONLY for files that on GAE may not be on filesystem """ - return os.path.exists(filename) + return exists(filename) def file_open(self, filename, mode='rb', lock=True): """ @@ -652,37 +667,41 @@ class BaseAdapter(ConnectionPool): TFK = {} tablename = table._tablename sortable = 0 + types = self.types for field in table: sortable += 1 - k = field.name - if isinstance(field.type,SQLCustomType): - ftype = field.type.native or field.type.type - elif field.type.startswith('reference'): - referenced = field.type[10:].strip() + field_name = field.name + field_type = field.type + if isinstance(field_type,SQLCustomType): + ftype = field_type.native or field_type.type + elif field_type.startswith('reference'): + referenced = field_type[10:].strip() if referenced == '.': referenced = tablename - constraint_name = self.constraint_name(tablename, field.name) + constraint_name = self.constraint_name(tablename, field_name) if not '.' in referenced \ and referenced != tablename \ and hasattr(table,'_primarykey'): - ftype = self.types['integer'] + ftype = types['integer'] else: if hasattr(table,'_primarykey'): rtablename,rfieldname = referenced.split('.') rtable = table._db[rtablename] rfield = rtable[rfieldname] # must be PK reference or unique - if rfieldname in hasattr(rtable,'_primarykey') or rfield.unique: - ftype = self.types[rfield.type[:9]] % \ + if rfieldname in hasattr(rtable,'_primarykey') or \ + rfield.unique: + ftype = types[rfield.type[:9]] % \ dict(length=rfield.length) # multicolumn primary key reference? if not rfield.unique and len(rtable._primarykey)>1: # then it has to be a table level FK if rtablename not in TFK: TFK[rtablename] = {} - TFK[rtablename][rfieldname] = field.name + TFK[rtablename][rfieldname] = field_name else: - ftype = ftype + self.types['reference FK'] % dict( + ftype = ftype + \ + types['reference FK'] % dict( constraint_name = constraint_name, # should be quoted foreign_key = '%s (%s)' % (rtablename, rfieldname), @@ -691,26 +710,27 @@ class BaseAdapter(ConnectionPool): # make a guess here for circular references id_fieldname = referenced in table._db \ and table._db[referenced]._id.name or 'id' - ftype = self.types[field.type[:9]] % dict( - index_name = field.name+'__idx', - field_name = field.name, + ftype = types[field_type[:9]] % dict( + index_name = field_name+'__idx', + field_name = field_name, constraint_name = constraint_name, foreign_key = '%s (%s)' % (referenced, id_fieldname), on_delete_action=field.ondelete) - elif field.type.startswith('list:reference'): - ftype = self.types[field.type[:14]] - elif field.type.startswith('decimal'): - precision, scale = map(int,field.type[8:-1].split(',')) - ftype = self.types[field.type[:7]] % \ + elif field_type.startswith('list:reference'): + ftype = types[field_type[:14]] + elif field_type.startswith('decimal'): + precision, scale = map(int,field_type[8:-1].split(',')) + ftype = types[field_type[:7]] % \ dict(precision=precision,scale=scale) - elif field.type.startswith('geo'): + elif field_type.startswith('geo'): srid = self.srid - geotype, parms = field.type[:-1].split('(') - if not geotype in self.types: - raise SyntaxError, 'Field: unknown field type: %s for %s' \ - % (field.type, field.name) - ftype = self.types[geotype] + geotype, parms = field_type[:-1].split('(') + if not geotype in types: + raise SyntaxError, \ + 'Field: unknown field type: %s for %s' \ + % (field_type, field_name) + ftype = types[geotype] if self.dbengine == 'postgres' and geotype == 'geometry': # parameters: schema, srid, dimension dimension = 2 # GIS.dimension ??? @@ -721,19 +741,20 @@ class BaseAdapter(ConnectionPool): schema, srid = parms else: schema = parms[0] - ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % self.types[geotype] - ftype = ftype % dict(schema=schema, tablename=tablename, - fieldname=field.name, srid=srid, + ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype] + ftype = ftype % dict(schema=schema, + tablename=tablename, + fieldname=field_name, srid=srid, dimension=dimension) postcreation_fields.append(ftype) - elif not field.type in self.types: + elif not field_type in types: raise SyntaxError, 'Field: unknown field type: %s for %s' % \ - (field.type, field.name) + (field_type, field_name) else: - ftype = self.types[field.type]\ + ftype = types[field_type]\ % dict(length=field.length) - if not field.type.startswith('id') and \ - not field.type.startswith('reference'): + if not field_type.startswith('id') and \ + not field_type.startswith('reference'): if field.notnull: ftype += ' NOT NULL' else: @@ -744,9 +765,10 @@ class BaseAdapter(ConnectionPool): ftype += ' %s' % field.custom_qualifier # add to list of fields - sql_fields[field.name] = dict(sortable=sortable, - type=str(field.type), - sql=ftype) + sql_fields[field_name] = dict( + sortable=sortable, + type=str(field_type), + sql=ftype) if isinstance(field.default,(str,int,float)): # Caveat: sql_fields and sql_fields_aux @@ -755,13 +777,14 @@ class BaseAdapter(ConnectionPool): # is used for create tables. # The reason is that we do not want to trigger # a migration simply because a default value changes. - not_null = self.NOT_NULL(field.default, field.type) + not_null = self.NOT_NULL(field.default, field_type) ftype = ftype.replace('NOT NULL', not_null) - sql_fields_aux[field.name] = dict(sql=ftype) + sql_fields_aux[field_name] = dict(sql=ftype) # Postgres - PostGIS: # geometry fields are added after the table has been created, not now - if not (self.dbengine == 'postgres' and field.type.startswith('geom')): - fields.append('%s %s' % (field.name, ftype)) + if not (self.dbengine == 'postgres' and \ + field_type.startswith('geom')): + fields.append('%s %s' % (field_name, ftype)) other = ';' # backend-specific extensions to fields @@ -776,7 +799,7 @@ class BaseAdapter(ConnectionPool): pkeys = table._db[rtablename]._primarykey fkeys = [ rfields[k] for k in pkeys ] fields = fields + ',\n ' + \ - self.types['reference TFK'] % dict( + types['reference TFK'] % dict( table_name = tablename, field_name=', '.join(fkeys), foreign_table = rtablename, @@ -806,12 +829,13 @@ class BaseAdapter(ConnectionPool): or self.uri.startswith('spatialite:memory'): table._dbt = None elif isinstance(migrate, str): - table._dbt = os.path.join(dbpath, migrate) + table._dbt = pjoin(dbpath, migrate) else: - table._dbt = os.path.join(dbpath, '%s_%s.table' \ - % (table._db._uri_hash, tablename)) + table._dbt = pjoin( + dbpath, '%s_%s.table' % (table._db._uri_hash, tablename)) + if table._dbt: - table._loggername = os.path.join(dbpath, 'sql.log') + table._loggername = pjoin(dbpath, 'sql.log') logfile = self.file_open(table._loggername, 'a') else: logfile = None @@ -870,9 +894,9 @@ class BaseAdapter(ConnectionPool): v=dict(type='unkown',sql=v) return k.lower(),v ### make sure all field names are lower case to avoid conflicts - sql_fields = dict(map(fix,sql_fields.items())) - sql_fields_old = dict(map(fix,sql_fields_old.items())) - sql_fields_aux = dict(map(fix,sql_fields_aux.items())) + sql_fields = dict(map(fix,sql_fields.iteritems())) + sql_fields_old = dict(map(fix,sql_fields_old.iteritems())) + sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems())) keys = sql_fields.keys() for key in sql_fields_old: @@ -1025,6 +1049,7 @@ class BaseAdapter(ConnectionPool): return ['DROP TABLE %s;' % table] def drop(self, table, mode=''): + db = table._db if table._dbt: logfile = self.file_open(table._loggername, 'a') queries = self._drop(table, mode) @@ -1032,10 +1057,10 @@ class BaseAdapter(ConnectionPool): if table._dbt: logfile.write(query + '\n') self.execute(query) - table._db.commit() - del table._db[table._tablename] - del table._db.tables[table._db.tables.index(table._tablename)] - table._db._update_referenced_by(table._tablename) + db.commit() + del db[table._tablename] + del db.tables[db.tables.index(table._tablename)] + db._remove_references_to(table) if table._dbt: self.file_delete(table._dbt) logfile.write('success!\n') @@ -1183,16 +1208,19 @@ class BaseAdapter(ConnectionPool): if isinstance(expression, Field): return '%s.%s' % (expression.tablename, expression.name) elif isinstance(expression, (Expression, Query)): - if not expression.second is None: - return expression.op(expression.first, expression.second) - elif not expression.first is None: - return expression.op(expression.first) - elif not isinstance(expression.op, str): - return expression.op() - else: - op = expression.op - if op.endswith(';'): op=op[:-1] + first = expression.first + second = expression.second + op = expression.op + if not second is None: + return op(first, second) + elif not first is None: + return op(first) + elif isinstance(op, str): + if op.endswith(';'): + op=op[:-1] return '(%s)' % op + else: + return op() elif field_type: return str(self.represent(expression,field_type)) elif isinstance(expression,(list,tuple)): @@ -1290,11 +1318,10 @@ class BaseAdapter(ConnectionPool): counter = None ### special code to handle CASCADE in SQLite & SpatiaLite if self.dbengine in ('sqlite', 'spatialite') and counter: - for tablename,fieldname in table._referenced_by: - f = db[tablename][fieldname] - if f.type=='reference '+table._tablename \ - and f.ondelete=='CASCADE': - db(db[tablename][fieldname].belongs(deleted)).delete() + for field in table._referenced_by: + if field.type=='reference '+table._tablename \ + and field.ondelete=='CASCADE': + db(field.belongs(deleted)).delete() ### end special code to handle CASCADE in SQLite & SpatiaLite return counter @@ -1308,39 +1335,39 @@ class BaseAdapter(ConnectionPool): raise RuntimeError, "Too many tables selected" def expand_all(self, fields, tablenames): + db = self.db new_fields = [] + append = new_fields.append for item in fields: if isinstance(item,SQLALL): new_fields += item._table elif isinstance(item,str): if regex_table_field.match(item): tablename,fieldname = item.split('.') - new_fields.append(self.db[tablename][fieldname]) + append(db[tablename][fieldname]) else: - new_fields.append(Expression(self.db,lambda:item)) + append(Expression(db,lambda:item)) else: - new_fields.append(item) + append(item) # ## if no fields specified take them all from the requested tables if not new_fields: for table in tablenames: - for field in self.db[table]: - new_fields.append(field) + for field in db[table]: + append(field) return new_fields def _select(self, query, fields, attributes): - for key in set(attributes.keys())-set(('orderby', 'groupby', 'limitby', - 'required', 'cache', 'left', - 'distinct', 'having', 'join', - 'for_update', 'processor')): + tables = self.tables + for key in set(attributes.keys())-SELECT_ARGS: raise SyntaxError, 'invalid select attribute: %s' % key - - tablenames = self.tables(query) + args_get = attributes.get + tablenames = tables(query) for field in fields: if isinstance(field, basestring) \ and regex_table_field.match(field): tn,fn = field.split('.') field = self.db[tn][fn] - for tablename in self.tables(field): + for tablename in tables(field): if not tablename in tablenames: tablenames.append(tablename) @@ -1357,14 +1384,14 @@ class BaseAdapter(ConnectionPool): sql_w = '' sql_o = '' sql_s = '' - left = attributes.get('left', False) - inner_join = attributes.get('join', False) - distinct = attributes.get('distinct', False) - groupby = attributes.get('groupby', False) - orderby = attributes.get('orderby', False) - having = attributes.get('having', False) - limitby = attributes.get('limitby', False) - for_update = attributes.get('for_update', False) + left = args_get('left', False) + inner_join = args_get('join', False) + distinct = args_get('distinct', False) + groupby = args_get('groupby', False) + orderby = args_get('orderby', False) + having = args_get('having', False) + limitby = args_get('limitby', False) + for_update = args_get('for_update', False) if self.can_select_for_update is False and for_update is True: raise SyntaxError, 'invalid select attribute: for_update' if distinct is True: @@ -1380,7 +1407,7 @@ class BaseAdapter(ConnectionPool): ijoinon = [t for t in inner_join if isinstance(t, Expression)] itables_to_merge={} #issue 490 [itables_to_merge.update( - dict.fromkeys(self.tables(t))) for t in ijoinon] + dict.fromkeys(tables(t))) for t in ijoinon] ijoinont = [t.first._tablename for t in ijoinon] [itables_to_merge.pop(t) for t in ijoinont if t in itables_to_merge] #issue 490 @@ -1398,7 +1425,7 @@ class BaseAdapter(ConnectionPool): #patch join+left patch (solves problem with ordering in left joins) tables_to_merge={} [tables_to_merge.update( - dict.fromkeys(self.tables(t))) for t in joinon] + dict.fromkeys(tables(t))) for t in joinon] joinont = [t.first._tablename for t in joinon] [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge] important_tablenames = joint + joinont + tables_to_merge.keys() @@ -1463,28 +1490,36 @@ class BaseAdapter(ConnectionPool): return 'SELECT %s %s FROM %s%s%s;' % \ (sql_s, sql_f, sql_t, sql_w, sql_o) + def _select_aux(self,sql,fields,attributes): + args_get = attributes.get + self.execute(sql) + rows = self.cursor.fetchall() + if isinstance(rows,tuple): + rows = list(rows) + limitby = args_get('limitby', None) or (0,) + rows = self.rowslice(rows,limitby[0],None) + processor = args_get('processor',self.parse) + cacheable = args_get('cacheable',False) + return processor(rows,fields,self._colnames,cacheable=cacheable) + def select(self, query, fields, attributes): """ Always returns a Rows object, possibly empty. """ - def response(sql): - self.execute(sql) - return self.cursor.fetchall() sql = self._select(query, fields, attributes) - if attributes.get('cache', None): + if attributes.get('cache', None): + args = (sql,fields,attributes) (cache_model, time_expire) = attributes['cache'] del attributes['cache'] + attributes['cacheable'] = True key = self.uri + '/' + sql - key = (len(key)<=200) and key or hashlib.md5(key).hexdigest() - rows = cache_model(key, lambda: response(sql), time_expire) + if len(key)>200: key = hashlib.md5(key).hexdigest() + return cache_model( + key, + lambda self=self,args=args:self._select_aux(*args), + time_expire) else: - rows = response(sql) - if isinstance(rows,tuple): - rows = list(rows) - limitby = attributes.get('limitby', None) or (0,) - rows = self.rowslice(rows,limitby[0],None) - processor = attributes.get('processor',self.parse) - return processor(rows,fields,self._colnames) + return self._select_aux(sql,fields,attributes) def _count(self, query, distinct=None): tablenames = self.tables(query) @@ -1563,6 +1598,7 @@ class BaseAdapter(ConnectionPool): return self.log_execute(*a, **b) def represent(self, obj, fieldtype): + field_is_type = fieldtype.startswith if isinstance(obj, CALLABLETYPES): obj = obj() if isinstance(fieldtype, SQLCustomType): @@ -1572,12 +1608,12 @@ class BaseAdapter(ConnectionPool): return value if isinstance(obj, (Expression, Field)): return str(obj) - if fieldtype.startswith('list:'): + if field_is_type('list:'): if not obj: obj = [] elif not isinstance(obj, (list, tuple)): obj = [obj] - if fieldtype.startswith('list:string'): + if field_is_type('list:string'): obj = [str(item) for item in obj] else: obj = [int(item) for item in obj] @@ -1597,9 +1633,9 @@ class BaseAdapter(ConnectionPool): return "'F'" if fieldtype == 'id' or fieldtype == 'integer': return str(int(obj)) - if fieldtype.startswith('decimal'): + if field_is_type('decimal'): return str(obj) - elif fieldtype.startswith('reference'): # reference + elif field_is_type('reference'): # reference if fieldtype.find('.')>0: return repr(obj) elif isinstance(obj, (Row, Reference)): @@ -1778,41 +1814,48 @@ class BaseAdapter(ConnectionPool): 'list:string':self.parse_list_strings, } - def parse(self, rows, fields, colnames, blob_decode=True): + def parse(self, rows, fields, colnames, blob_decode=True, + cacheable = False): self.build_parsemap() db = self.db virtualtables = [] new_rows = [] + tmps = [] + for colname in colnames: + if not regex_table_field.match(colname): + tmps.append(None) + else: + (tablename, fieldname) = colname.split('.') + table = db[tablename] + field = table[fieldname] + ft = field.type + tmps.append((tablename,fieldname,table,field,ft)) for (i,row) in enumerate(rows): new_row = Row() - for j,colname in enumerate(colnames): - value = row[j] - if not regex_table_field.match(colnames[j]): - if not '_extra' in new_row: - new_row['_extra'] = Row() - new_row['_extra'][colnames[j]] = \ - self.parse_value(value, fields[j].type,blob_decode) - new_column_name = \ - regex_select_as_parser.search(colnames[j]) - if not new_column_name is None: - column_name = new_column_name.groups(0) - setattr(new_row,column_name[0],value) - else: - (tablename, fieldname) = colname.split('.') - table = db[tablename] - field = table[fieldname] - if not tablename in new_row: + for (j,colname) in enumerate(colnames): + value = row[j] + tmp = tmps[j] + if tmp: + (tablename,fieldname,table,field,ft) = tmp + if tablename in new_row: + colset = new_row[tablename] + else: colset = new_row[tablename] = Row() if tablename not in virtualtables: virtualtables.append(tablename) - else: - colset = new_row[tablename] - value = self.parse_value(value,field.type,blob_decode) - if field.filter_out: value = field.filter_out(value) + value = self.parse_value(value,ft,blob_decode) + if field.filter_out: + value = field.filter_out(value) colset[fieldname] = value + + # for backward compatibility + if ft=='id' and fieldname!='id' and \ + not 'id' in table.fields: + colset['id'] = value - if field.type == 'id': - # temporary hack to deal with GoogleDatastoreAdapter + if ft == 'id' and not cacheable: + # temporary hack to deal with + # GoogleDatastoreAdapter # references if isinstance(self, GoogleDatastoreAdapter): id = value.key().id_or_name() @@ -1820,27 +1863,34 @@ class BaseAdapter(ConnectionPool): colset.gae_item = value else: id = value - colset.update_record = ( - lambda _ = (colset, table, id), **a: - update_record(_, a)) - colset.delete_record = ( - lambda t = table, i = id: - t._db(t._id==i).delete()) - for (referee_table, referee_name) in table._referenced_by: - s = db[referee_table][referee_name] + colset.update_record = RecordUpdater(colset,table,id) + colset.delete_record = RecordDeleter(table,id) + for rfield in table._referenced_by: referee_link = db._referee_name and \ - db._referee_name % dict(table=referee_table,field=referee_name) + db._referee_name % dict( + table=rfield.tablename,field=rfield.name) if referee_link and not referee_link in colset: - colset[referee_link] = Set(db, s == id) + colset[referee_link] = Set(db,rfield == id) + else: + if not '_extra' in new_row: + new_row['_extra'] = Row() + new_row['_extra'][colname] = \ + self.parse_value(value, + fields[j].type,blob_decode) + new_column_name = \ + regex_select_as_parser.search(colname) + if not new_column_name is None: + column_name = new_column_name.groups(0) + setattr(new_row,column_name[0],value) new_rows.append(new_row) rowsobj = Rows(db, new_rows, colnames, rawrows=rows) for tablename in virtualtables: ### new style virtual fields table = db[tablename] - fields_virtual = [(f,v) for (f,v) in table.items() + fields_virtual = [(f,v) for (f,v) in table.iteritems() if isinstance(v,FieldVirtual)] - fields_lazy = [(f,v) for (f,v) in table.items() + fields_lazy = [(f,v) for (f,v) in table.iteritems() if isinstance(v,FieldLazy)] if fields_virtual or fields_lazy: for row in rowsobj.records: @@ -1854,7 +1904,7 @@ class BaseAdapter(ConnectionPool): for item in table.virtualfields: try: rowsobj = rowsobj.setvirtualfields(**{tablename:item}) - except KeyError: + except (KeyError, AttributeError): # to avoid breaking virtualfields when partial select pass return rowsobj @@ -1932,7 +1982,7 @@ class SQLiteAdapter(BaseAdapter): else: dbpath = uri.split('://')[1] if dbpath[0] != '/': - dbpath = os.path.join( + dbpath = pjoin( self.folder.decode(path_encoding).encode('utf8'), dbpath) if not 'check_same_thread' in driver_args: driver_args['check_same_thread'] = False @@ -1975,7 +2025,7 @@ class SpatiaLiteAdapter(SQLiteAdapter): import copy types = copy.copy(BaseAdapter.types) - types.update({'geometry': 'GEOMETRY'}) + types.update(geometry='GEOMETRY') def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8', credential_decoder=IDENTITY, driver_args={}, @@ -1997,7 +2047,7 @@ class SpatiaLiteAdapter(SQLiteAdapter): else: dbpath = uri.split('://')[1] if dbpath[0] != '/': - dbpath = os.path.join( + dbpath = pjoin( self.folder.decode(path_encoding).encode('utf8'), dbpath) if not 'check_same_thread' in driver_args: driver_args['check_same_thread'] = False @@ -2063,15 +2113,16 @@ class SpatiaLiteAdapter(SQLiteAdapter): self.expand(second, first.type)) def represent(self, obj, fieldtype): - if fieldtype.startswith('geo'): + field_is_type = fieldtype.startswith + if field_is_type('geo'): srid = 4326 # Spatialite default srid for geometry geotype, parms = fieldtype[:-1].split('(') parms = parms.split(',') if len(parms) >= 2: schema, srid = parms[:2] -# if fieldtype.startswith('geometry'): +# if field_is_type('geometry'): value = "ST_GeomFromText('%s',%s)" %(obj, srid) -# elif fieldtype.startswith('geography'): +# elif field_is_type('geography'): # value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj) # else: # raise SyntaxError, 'Invalid field type %s' %fieldtype @@ -2102,7 +2153,7 @@ class JDBCSQLiteAdapter(SQLiteAdapter): else: dbpath = uri.split('://')[1] if dbpath[0] != '/': - dbpath = os.path.join( + dbpath = pjoin( self.folder.decode(path_encoding).encode('utf8'), dbpath) def connect(dbpath=dbpath,driver_args=driver_args): return self.driver.connect( @@ -2212,12 +2263,12 @@ class MySQLAdapter(BaseAdapter): raise SyntaxError, 'Database name required' port = int(m.group('port') or '3306') charset = m.group('charset') or 'utf8' - driver_args.update(dict(db=db, - user=credential_decoder(user), - passwd=credential_decoder(password), - host=host, - port=port, - charset=charset)) + driver_args.update(db=db, + user=credential_decoder(user), + passwd=credential_decoder(password), + host=host, + port=port, + charset=charset) def connect(driver_args=driver_args): return self.driver.connect(**driver_args) self.pool_connection(connect) @@ -2471,15 +2522,16 @@ class PostgreSQLAdapter(BaseAdapter): return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type)) def represent(self, obj, fieldtype): - if fieldtype.startswith('geo'): + field_is_type = fieldtype.startswith + if field_is_type('geo'): srid = 4326 # postGIS default srid for geometry geotype, parms = fieldtype[:-1].split('(') parms = parms.split(',') if len(parms) >= 2: schema, srid = parms[:2] - if fieldtype.startswith('geometry'): + if field_is_type('geometry'): value = "ST_GeomFromText('%s',%s)" %(obj, srid) - elif fieldtype.startswith('geography'): + elif field_is_type('geography'): value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj) # else: # raise SyntaxError, 'Invalid field type %s' %fieldtype @@ -2523,12 +2575,13 @@ class NewPostgreSQLAdapter(PostgreSQLAdapter): return value def represent(self, obj, fieldtype): - if fieldtype.startswith('list:'): + field_is_type = fieldtype.startswith + if field_is_type('list:'): if not obj: obj = [] elif not isinstance(obj, (list, tuple)): obj = [obj] - if fieldtype.startswith('list:string'): + if field_is_type('list:string'): obj = [str(item) for item in obj] else: obj = [int(item) for item in obj] @@ -2871,7 +2924,7 @@ class MSSQLAdapter(BaseAdapter): argpattern = re.compile('(?P[^=]+)=(?P[^&]*)') for argmatch in argpattern.finditer(urlargs): argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue') - urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.items()]) + urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()]) cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \ % (host, port, db, user, password, urlargs) def connect(cnxn=cnxn,driver_args=driver_args): @@ -2924,7 +2977,8 @@ class MSSQLAdapter(BaseAdapter): return '%s.STWithin(%s)=1' %(self.expand(first), self.expand(second, first.type)) def represent(self, obj, fieldtype): - if fieldtype.startswith('geometry'): + field_is_type = fieldtype.startswith + if field_is_type('geometry'): srid = 0 # MS SQL default srid for geometry geotype, parms = fieldtype[:-1].split('(') if parms: @@ -2971,7 +3025,7 @@ class MSSQL2Adapter(MSSQLAdapter): def represent(self, obj, fieldtype): value = BaseAdapter.represent(self, obj, fieldtype) - if (fieldtype == 'string' or fieldtype == 'text') and value[:1]=="'": + if fieldtype in ('string','text') and value[:1]=="'": value = 'N'+value return value @@ -3060,9 +3114,9 @@ class SybaseAdapter(MSSQLAdapter): dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db) - driver_args.update(dict(user = credential_decoder(user), - password = credential_decoder(password), - locale = charset)) + driver_args.update(user = credential_decoder(user), + password = credential_decoder(password), + locale = charset) def connect(dsn=dsn,driver_args=driver_args): return self.driver.connect(dsn,**driver_args) @@ -3136,7 +3190,7 @@ class FireBirdAdapter(BaseAdapter): def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', credential_decoder=IDENTITY, driver_args={}, adapter_args={}): - if adapter_args.has_key('driver_name'): + if 'driver_name' in adapter_args: if adapter_args['driver_name'] == 'kinterbasdb': self.driver = kinterbasdb elif adapter_args['driver_name'] == 'firebirdsql': @@ -3171,11 +3225,11 @@ class FireBirdAdapter(BaseAdapter): if not db: raise SyntaxError, 'Database name required' charset = m.group('charset') or 'UTF8' - driver_args.update(dict(dsn='%s/%s:%s' % (host,port,db), - user = credential_decoder(user), - password = credential_decoder(password), - charset = charset)) - + driver_args.update(dsn='%s/%s:%s' % (host,port,db), + user = credential_decoder(user), + password = credential_decoder(password), + charset = charset) + def connect(driver_args=driver_args): return self.driver.connect(**driver_args) self.pool_connection(connect) @@ -3202,7 +3256,7 @@ class FireBirdEmbeddedAdapter(FireBirdAdapter): credential_decoder=IDENTITY, driver_args={}, adapter_args={}): - if adapter_args.has_key('driver_name'): + if 'driver_name' in adapter_args: if adapter_args['driver_name'] == 'kinterbasdb': self.driver = kinterbasdb elif adapter_args['driver_name'] == 'firebirdsql': @@ -3237,11 +3291,11 @@ class FireBirdEmbeddedAdapter(FireBirdAdapter): if not charset: charset = 'UTF8' host = '' - driver_args.update(dict(host=host, - database=pathdb, - user=credential_decoder(user), - password=credential_decoder(password), - charset=charset)) + driver_args.update(host=host, + database=pathdb, + user=credential_decoder(user), + password=credential_decoder(password), + charset=charset) #def connect(driver_args=driver_args): # return kinterbasdb.connect(**driver_args) @@ -3349,7 +3403,7 @@ class InformixAdapter(BaseAdapter): user = credential_decoder(user) password = credential_decoder(password) dsn = '%s@%s' % (db,host) - driver_args.update(dict(user=user,password=password,autocommit=True)) + driver_args.update(user=user,password=password,autocommit=True) def connect(dsn=dsn,driver_args=driver_args): return self.driver.connect(dsn,**driver_args) self.pool_connection(connect) @@ -3590,10 +3644,10 @@ class IngresAdapter(BaseAdapter): vnode = '(local)' servertype = 'ingres' trace = (0, None) # No tracing - driver_args.update(dict(database=database_name, - vnode=vnode, - servertype=servertype, - trace=trace)) + driver_args.update(database=database_name, + vnode=vnode, + servertype=servertype, + trace=trace) def connect(driver_args=driver_args): return self.driver.connect(**driver_args) self.pool_connection(connect) @@ -3813,7 +3867,7 @@ class DatabaseStoredFile: rows = self.db.executesql(query) if rows: self.data = rows[0][0] - elif os.path.exists(filename): + elif exists(filename): datafile = open(filename, 'r') try: self.data = datafile.read() @@ -3848,7 +3902,7 @@ class DatabaseStoredFile: @staticmethod def exists(db, filename): - if os.path.exists(filename): + if exists(filename): return True query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename if db.executesql(query): @@ -3885,7 +3939,7 @@ class GoogleSQLAdapter(UseDatabaseStoredFile,MySQLAdapter): self.uri = uri self.pool_size = pool_size self.db_codec = db_codec - self.folder = folder or os.path.join('$HOME',thread.folder.split( + self.folder = folder or pjoin('$HOME',thread.folder.split( os.sep+'applications'+os.sep,1)[1]) m = re.compile('^(?P.*)/(?P.*)$').match(self.uri[len('google:sql://'):]) @@ -3925,6 +3979,7 @@ class NoSQLAdapter(BaseAdapter): return obj def represent(self, obj, fieldtype): + field_is_type = fieldtype.startswith if isinstance(obj, CALLABLETYPES): obj = obj() if isinstance(fieldtype, SQLCustomType): @@ -3935,7 +3990,7 @@ class NoSQLAdapter(BaseAdapter): if isinstance(fieldtype, gae.Property): return obj is_string = isinstance(fieldtype,str) - is_list = is_string and fieldtype.startswith('list:') + is_list = is_string and field_is_type('list:') if is_list: if not obj: obj = [] @@ -3951,7 +4006,7 @@ class NoSQLAdapter(BaseAdapter): obj = long(obj) elif fieldtype == 'double': obj = float(obj) - elif is_string and fieldtype.startswith('reference'): + elif is_string and field_is_type('reference'): if isinstance(obj, (Row, Reference)): obj = obj['id'] obj = long(obj) @@ -3985,7 +4040,7 @@ class NoSQLAdapter(BaseAdapter): obj = datetime.datetime(y, m, d, h, mi, s) elif fieldtype == 'blob': pass - elif is_string and fieldtype.startswith('list:string'): + elif is_string and field_is_type('list:string'): return map(self.to_unicode,obj) elif is_list: return map(int,obj) @@ -4106,7 +4161,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): 'list:string': (lambda: gae.StringListProperty(default=None)), 'list:integer': (lambda: gae.ListProperty(int,default=None)), 'list:reference': (lambda: gae.ListProperty(int,default=None)), - }) + }) self.db = db self.uri = uri self.dbengine = 'google:datastore' @@ -4123,39 +4178,39 @@ class GoogleDatastoreAdapter(NoSQLAdapter): def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None): myfields = {} - for k in table.fields: + for field in table: if isinstance(polymodel,Table) and k in polymodel.fields(): continue - field = table[k] attr = {} - if isinstance(field.type, SQLCustomType): - ftype = self.types[field.type.native or field.type.type](**attr) - elif isinstance(field.type, gae.Property): - ftype = field.type - elif field.type.startswith('id'): + field_type = field.type + if isinstance(field_type, SQLCustomType): + ftype = self.types[field_type.native or field_type.type](**attr) + elif isinstance(field_type, gae.Property): + ftype = field_type + elif field_type.startswith('id'): continue - elif field.type.startswith('decimal'): - precision, scale = field.type[7:].strip('()').split(',') + elif field_type.startswith('decimal'): + precision, scale = field_type[7:].strip('()').split(',') precision = int(precision) scale = int(scale) ftype = GAEDecimalProperty(precision, scale, **attr) - elif field.type.startswith('reference'): + elif field_type.startswith('reference'): if field.notnull: attr = dict(required=True) - referenced = field.type[10:].strip() - ftype = self.types[field.type[:9]](referenced) - elif field.type.startswith('list:reference'): + referenced = field_type[10:].strip() + ftype = self.types[field_type[:9]](referenced) + elif field_type.startswith('list:reference'): if field.notnull: attr = dict(required=True) - referenced = field.type[15:].strip() - ftype = self.types[field.type[:14]](**attr) - elif field.type.startswith('list:'): - ftype = self.types[field.type](**attr) - elif not field.type in self.types\ - or not self.types[field.type]: - raise SyntaxError, 'Field: unknown field type: %s' % field.type + referenced = field_type[15:].strip() + ftype = self.types[field_type[:14]](**attr) + elif field_type.startswith('list:'): + ftype = self.types[field_type](**attr) + elif not field_type in self.types\ + or not self.types[field_type]: + raise SyntaxError, 'Field: unknown field type: %s' % field_type else: - ftype = self.types[field.type](**attr) + ftype = self.types[field_type](**attr) myfields[field.name] = ftype if not polymodel: table._tableobj = classobj(table._tablename, (gae.Model, ), myfields) @@ -4273,6 +4328,8 @@ class GoogleDatastoreAdapter(NoSQLAdapter): self.db(table._id > 0).delete() def select_raw(self,query,fields=None,attributes=None): + db = self.db + args_get = attributes.get fields = fields or [] attributes = attributes or {} new_fields = [] @@ -4295,14 +4352,14 @@ class GoogleDatastoreAdapter(NoSQLAdapter): query = self.common_filter(query,[tablename]) #tableobj is a GAE Model class (or subclass) - tableobj = self.db[tablename]._tableobj + tableobj = db[tablename]._tableobj filters = self.expand(query) projection = None - if len(self.db[tablename].fields) == len(fields): + if len(db[tablename].fields) == len(fields): #getting all fields, not a projection query projection = None - elif attributes.get('projection') == True: + elif args_get('projection') == True: projection = [] for f in fields: if f.type in ['text', 'blob']: @@ -4311,18 +4368,21 @@ class GoogleDatastoreAdapter(NoSQLAdapter): else: projection.append(f.name) - #projection's can't include 'id'. it will be added to the result later - query_projection = [p for p in projection if \ - p != self.db[tablename]._id.name] if projection \ - else None + # projection's can't include 'id'. + # it will be added to the result later + query_projection = [ + p for p in projection if \ + p != db[tablename]._id.name] if projection \ + else None cursor = None - if isinstance(attributes.get('reusecursor'), str): - cursor = attributes.get('reusecursor') - items = gae.Query(tableobj, projection=query_projection, cursor=cursor) + if isinstance(args_get('reusecursor'), str): + cursor = args_get('reusecursor') + items = gae.Query(tableobj, projection=query_projection, + cursor=cursor) for filter in filters: - if attributes.get('projection') == True and \ + if args_get('projection') == True and \ filter.name in query_projection and \ filter.op in ['=', '<=', '>=']: raise SyntaxError, \ @@ -4333,13 +4393,15 @@ class GoogleDatastoreAdapter(NoSQLAdapter): if filter.value==0: items = [] elif isinstance(filter.value, Key): - #key qeuries return a class instance, can't use projection - # extra values will be ignored in post-processing later + # key qeuries return a class instance, + # can't use projection + # extra values will be ignored in post-processing later item = tableobj.get(filter.value) items = (item and [item]) or [] else: - #key qeuries return a class instance, can't use projection - # extra values will be ignored in post-processing later + # key qeuries return a class instance, + # can't use projection + # extra values will be ignored in post-processing later item = tableobj.get_by_id(filter.value) items = (item and [item]) or [] elif isinstance(items,list): # i.e. there is a single record! @@ -4351,11 +4413,11 @@ class GoogleDatastoreAdapter(NoSQLAdapter): items = items.filter('%s %s' % (filter.name,filter.op), filter.value) if not isinstance(items,list): - if attributes.get('left', None): + if args_get('left', None): raise SyntaxError, 'Set: no left join in appengine' - if attributes.get('groupby', None): + if args_get('groupby', None): raise SyntaxError, 'Set: no groupby in appengine' - orderby = attributes.get('orderby', False) + orderby = args_get('orderby', False) if orderby: ### THIS REALLY NEEDS IMPROVEMENT !!! if isinstance(orderby, (list, tuple)): @@ -4366,16 +4428,16 @@ class GoogleDatastoreAdapter(NoSQLAdapter): for order in orders: order={'-id':'-__key__','id':'__key__'}.get(order,order) items = items.order(order) - if attributes.get('limitby', None): + if args_get('limitby', None): (lmin, lmax) = attributes['limitby'] (limit, offset) = (lmax - lmin, lmin) rows = items.fetch(limit,offset=offset) #cursor is only useful if there was a limit and we didn't return # all results - if attributes.get('reusecursor'): - self.db['_lastcursor'] = items.cursor() + if args_get('reusecursor'): + db['_lastcursor'] = items.cursor() items = rows - return (items, tablename, projection or self.db[tablename].fields) + return (items, tablename, projection or db[tablename].fields) def select(self,query,fields,attributes): """ @@ -4800,14 +4862,20 @@ class MongoDBAdapter(NoSQLAdapter): print "in expand and this is a query" # any query using 'id':= # set name as _id (as per pymongo/mongodb primary key) - # convert second arg to an objectid field (if its not already) - # if second arg is 0 convert to objectid - if isinstance(expression.first,Field) and expression.first.type == 'id': + # convert second arg to an objectid field + # (if its not already) + # if second arg is 0 convert to objectid + if isinstance(expression.first,Field) and \ + expression.first.type == 'id': expression.first.name = '_id' - if expression.second != 0 and not isinstance(expression.second,ObjectId): + if expression.second != 0 and \ + not isinstance(expression.second,ObjectId): if isinstance(expression.second,int): try: - #Because the reference field is by default an integer and therefore this must be an integer to be able to work with other databases + # Because the reference field is by default + # an integer and therefore this must be an + # integer to be able to work with other + # databases expression.second = ObjectId(("%X" % expression.second)) except: raise SyntaxError, 'The second argument must by an integer that can represent an objectid.' @@ -5448,7 +5516,7 @@ class IMAPAdapter(NoSQLAdapter): if port==993: over_ssl = True - driver_args.update(dict(host=host,port=port, password=password, user=user)) + driver_args.update(host=host,port=port, password=password, user=user) def connect(driver_args=driver_args): # it is assumed sucessful authentication alLways # TODO: support direct connection and login tests @@ -5481,6 +5549,7 @@ class IMAPAdapter(NoSQLAdapter): closing """ + pools = ConnectionPool.pools if not self.pool_size: self.connection = f() self.cursor = cursor and self.connection.cursor() @@ -5488,10 +5557,10 @@ class IMAPAdapter(NoSQLAdapter): uri = self.uri while True: sql_locker.acquire() - if not uri in ConnectionPool.pools: - ConnectionPool.pools[uri] = [] - if ConnectionPool.pools[uri]: - self.connection = ConnectionPool.pools[uri].pop() + if not uri in pools: + pools[uri] = [] + if pools[uri]: + self.connection = pools[uri].pop() sql_locker.release() self.cursor = cursor and self.connection.cursor() if self.cursor and self.check_active_connection: @@ -6174,6 +6243,7 @@ def sqlhtml_validators(field): makes sure the content of a field is in line with the declared fieldtype """ + db = field.db if not have_validators: return [] field_type, field_length = field.type, field.length @@ -6213,52 +6283,53 @@ def sqlhtml_validators(field): requires.append(validators.IS_TIME()) elif field_type == 'datetime': requires.append(validators.IS_DATETIME()) - elif field.db and field_type.startswith('reference') and \ + elif db and field_type.startswith('reference') and \ field_type.find('.') < 0 and \ - field_type[10:] in field.db.tables: - referenced = field.db[field_type[10:]] + field_type[10:] in db.tables: + referenced = db[field_type[10:]] def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id) field.represent = field.represent or repr_ref if hasattr(referenced, '_format') and referenced._format: - requires = validators.IS_IN_DB(field.db,referenced._id, + requires = validators.IS_IN_DB(db,referenced._id, referenced._format) if field.unique: - requires._and = validators.IS_NOT_IN_DB(field.db,field) + requires._and = validators.IS_NOT_IN_DB(db,field) if field.tablename == field_type[10:]: return validators.IS_EMPTY_OR(requires) return requires - elif field.db and field_type.startswith('list:reference') and \ + elif db and field_type.startswith('list:reference') and \ field_type.find('.') < 0 and \ - field_type[15:] in field.db.tables: - referenced = field.db[field_type[15:]] + field_type[15:] in db.tables: + referenced = db[field_type[15:]] def list_ref_repr(ids, row=None, r=referenced, f=ff): if not ids: return None refs = None - if isinstance(r._db._adapter, GoogleDatastoreAdapter): + db, id = r._db, r._id + if isinstance(db._adapter, GoogleDatastoreAdapter): for i in xrange(0, len(ids), 30): if not refs: - refs = r._db(r._id.belongs(ids[i:i+30])).select(r._id) + refs = db(id.belongs(ids[i:i+30])).select(id) else: - refs = refs & r._db(r._id.belongs(ids[i:i+30])).select(r._id) + refs = refs&db(id.belongs(ids[i:i+30])).select(id) else: - refs = r._db(r._id.belongs(ids)).select(r._id) - return (refs and ', '.join(str(f(r,ref.id)) for ref in refs) or '') + refs = db(id.belongs(ids)).select(id) + return (refs and ', '.join(str(f(r,x.id)) for x in refs) or '') field.represent = field.represent or list_ref_repr if hasattr(referenced, '_format') and referenced._format: - requires = validators.IS_IN_DB(field.db,referenced._id, + requires = validators.IS_IN_DB(db,referenced._id, referenced._format,multiple=True) else: - requires = validators.IS_IN_DB(field.db,referenced._id, + requires = validators.IS_IN_DB(db,referenced._id, multiple=True) if field.unique: - requires._and = validators.IS_NOT_IN_DB(field.db,field) + requires._and = validators.IS_NOT_IN_DB(db,field) return requires elif field_type.startswith('list:'): def repr_list(values,row=None): return', '.join(str(v) for v in (values or [])) field.represent = field.represent or repr_list if field.unique: - requires.insert(0,validators.IS_NOT_IN_DB(field.db,field)) + requires.insert(0,validators.IS_NOT_IN_DB(db,field)) sff = ['in', 'do', 'da', 'ti', 'de', 'bo'] if field.notnull and not field_type[:2] in sff: requires.insert(0, validators.IS_NOT_EMPTY()) @@ -6282,18 +6353,15 @@ def bar_decode_string(value): return [x.replace('||', '|') for x in string_unpack.split(value[1:-1]) if x.strip()] -class Row(dict): +class Row(object): """ a dictionary that lets you do d['a'] as well as d.a this is only used to store a Row """ - __slots__=() - - __setattr__ = dict.__setitem__ - __getattr__ = dict.__getitem__ - __delattr__ = dict.__delitem__ + def __init__(self,*args,**kwargs): + self.__dict__.update(*args,**kwargs) def __getitem__(self, key): key=str(key) @@ -6302,25 +6370,49 @@ class Row(dict): return self._extra[key] elif m: try: - return dict.__getitem__(self, m.group(1))[m.group(2)] - except (KeyError,TypeError): + return ogetattr(self, m.group(1))[m.group(2)] + except (KeyError,AttributeError,TypeError): key = m.group(2) - return dict.__getitem__(self, key) + return ogetattr(self, key) + + def __setitem__(self, key, value): + setattr(self, str(key), value) __call__ = __getitem__ - #def __call__(self,key): - # return self.__getitem__(key) + def get(self,key,default=None): + return self.__dict__.get(key,default) - def __setitem__(self, key, value): - dict.__setitem__(self, str(key), value) + def __contains__(self,key): + return key in self.__dict__ + + def __nonzero__(self): + return len(self.__dict__)>0 + + def update(self, *args, **kwargs): + self.__dict__.update(*args, **kwargs) + + def keys(self): + return self.__dict__.keys() + + def items(self): + return self.__dict__.items() + + def values(self): + return self.__dict__.values() + + def __iter__(self): + return self.__dict__.__iter__() + + def iteritems(self): + return self.__dict__.iteritems() def __str__(self): ### this could be made smarter - return '' % dict.__repr__(self) + return '' % self.__dict__ def __repr__(self): - return '' % dict.__repr__(self) + return '' % self.__dict__ def __int__(self): return dict.__getitem__(self,'id') @@ -6361,7 +6453,6 @@ class Row(dict): del d[k] return d - def Row_unpickler(data): return Row(cPickle.loads(data)) @@ -6492,7 +6583,7 @@ def smart_query(fields,text): return query -class DAL(dict): +class DAL(object): """ an instance of this class represents a database connection @@ -6503,7 +6594,6 @@ class DAL(dict): db.define_table('tablename', Field('fieldname1'), Field('fieldname2')) """ - __slots__ = () @staticmethod def set_folder(folder): @@ -6667,14 +6757,14 @@ class DAL(dict): return self._tables def import_table_definitions(self,path,migrate=False,fake_migrate=False): - pattern = os.path.join(path,self._uri_hash+'_*.table') + pattern = pjoin(path,self._uri_hash+'_*.table') for filename in glob.glob(pattern): tfile = self._adapter.file_open(filename, 'r') try: sql_fields = cPickle.load(tfile) name = filename[len(pattern)-7:-6] mf = [(value['sortable'],Field(key,type=value['type'])) \ - for key, value in sql_fields.items()] + for key, value in sql_fields.iteritems()] mf.sort(lambda a,b: cmp(a[0],b[0])) self.define_table(name,*[item[1] for item in mf], **dict(migrate=migrate,fake_migrate=fake_migrate)) @@ -6695,12 +6785,6 @@ class DAL(dict): if name.upper() in self.RSK[backend]: raise SyntaxError, 'invalid table/column name "%s" is a "%s" reserved SQL keyword' % (name, backend.upper()) - def __contains__(self, tablename): - if self.has_key(tablename): - return True - else: - return False - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True): """ EXAMPLE: @@ -6784,8 +6868,8 @@ def index(): patterns.append(tag) patterns.append(tag+'/:field') if depth>0: - for rtable,rfield in db[table]._referenced_by: - tag+='/%s[%s.%s]' % (rtable,rtable,rfield) + for f in db[table]._referenced_by: + tag+='/%s[%s.%s]' % (rtable,f.tablename,f.name) patterns.append(tag) patterns += auto_table(rtable,base=tag,depth=depth-1) return patterns @@ -6910,7 +6994,7 @@ def index(): ofields = vars.get('order',db[table]._id.name).split('|') try: orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields] - except KeyError: + except (KeyError, AttributeError): return Row({'status':400,'error':'invalid orderby','response':None}) fields = [field for field in db[table] if field.readable] count = dbset.count() @@ -6946,7 +7030,7 @@ def index(): elif self.check_reserved: self.check_reserved_keyword(tablename) else: - invalid_args = [key for key in args if not key in TABLE_ARGS] + invalid_args = set(args)-TABLE_ARGS if invalid_args: raise SyntaxError, 'invalid table "%s" attributes: %s' \ % (tablename,invalid_args) @@ -6964,21 +7048,23 @@ def index(): *fields, **args ): - if self._common_fields: - fields = list(fields) + list(self._common_fields) + args_get = args.get + common_fields = self._common_fields + if common_fields: + fields = list(fields) + list(common_fields) - table_class = args.get('table_class',Table) + table_class = args_get('table_class',Table) table = table_class(self, tablename, *fields, **args) table._actual = True self[tablename] = table table._create_references() # must follow above line to handle self references - migrate = self._migrate_enabled and args.get('migrate',self._migrate) + migrate = self._migrate_enabled and args_get('migrate',self._migrate) if migrate and not self._uri in (None,'None') \ or self._adapter.dbengine=='google:datastore': fake_migrate = self._fake_migrate_all or \ - args.get('fake_migrate',self._fake_migrate) - polymodel = args.get('polymodel',None) + args_get('fake_migrate',self._fake_migrate) + polymodel = args_get('polymodel',None) try: sql_locker.acquire() self._adapter.create_table(table,migrate=migrate, @@ -6988,10 +7074,16 @@ def index(): sql_locker.release() else: table._dbt = None - on_define = args.get('on_define',None) + on_define = args_get('on_define',None) if on_define: on_define(table) return table + def __contains__(self, tablename): + return tablename in self.tables + + def get(self,key,default): + return self.__dict__.get(key,default) + def __iter__(self): for tablename in self.tables: yield self[tablename] @@ -7000,19 +7092,22 @@ def index(): return self.__getattr__(str(key)) def __getattr__(self, key): - if not key is '_LAZY_TABLES' and key in self._LAZY_TABLES: + if ogetattr(self,'_lazy_tables') and \ + key in ogetattr(self,'_LAZY_TABLES'): tablename, fields, args = self._LAZY_TABLES.pop(key) return self.lazy_define_table(tablename,*fields,**args) - return dict.__getitem__(self, key) + return ogetattr(self, key) def __setitem__(self, key, value): - dict.__setitem__(self, str(key), value) + osetattr(self, str(key), value) def __setattr__(self, key, value): if key[:1]!='_' and key in self: raise SyntaxError, \ 'Object %s exists and cannot be redefined' % key - dict.__setitem__(self,key,value) + osetattr(self,key,value) + + __delitem__ = object.__delattr__ def __repr__(self): return '' % self._uri @@ -7034,9 +7129,10 @@ def index(): self._adapter.rollback() def close(self): - if self._adapter in thread.instances: - thread.instances.remove(self._adapter) - self._adapter.close() + adapter = self._adapter + if adapter in thread.instances: + thread.instances.remove(adapter) + adapter.close() def executesql(self, query, placeholders=None, as_dict=False, fields=None, colnames=None): @@ -7074,25 +7170,26 @@ def index(): via the "colnames" argument. Note, the fields and colnames must be in the same order as the fields in the results cursor returned from the DB. """ + adapter = self._adapter if placeholders: - self._adapter.execute(query, placeholders) + adapter.execute(query, placeholders) else: - self._adapter.execute(query) + adapter.execute(query) if as_dict: - if not hasattr(self._adapter.cursor,'description'): + if not hasattr(adapter.cursor,'description'): raise RuntimeError, "database does not support executesql(...,as_dict=True)" # Non-DAL legacy db query, converts cursor results to dict. # sequence of 7-item sequences. each sequence tells about a column. # first item is always the field name according to Python Database API specs - columns = self._adapter.cursor.description + columns = adapter.cursor.description # reduce the column info down to just the field names fields = [f[0] for f in columns] # will hold our finished resultset in a list - data = self._adapter.cursor.fetchall() + data = adapter.cursor.fetchall() # convert the list for each row into a dictionary so it's # easier to work with. row['field_name'] rather than row[0] return [dict(zip(fields,row)) for row in data] - data = self._adapter.cursor.fetchall() + data = adapter.cursor.fetchall() if fields: if not isinstance(fields, list): fields = [fields] @@ -7105,14 +7202,14 @@ def index(): if not colnames: colnames = ['%s.%s' % (f.tablename, f.name) for f in extracted_fields] - data = self._adapter.parse( + data = adapter.parse( data, fields=extracted_fields, colnames=colnames) return data - def _update_referenced_by(self, other): - for tablename in self.tables: - by = self[tablename]._referenced_by - by[:] = [item for item in by if not item[0] == other] + def _remove_references_to(self, thistable): + for table in self: + table._referenced_by = [field for field in table._referenced_by + if not field.table==thistable] def export_to_csv_file(self, ofile, *args, **kwargs): step = int(kwargs.get('max_fetch_rows,',500)) @@ -7210,7 +7307,7 @@ def Reference_pickler(data): copy_reg.pickle(Reference, Reference_pickler, Reference_unpickler) -class Table(dict): +class Table(object): """ an instance of this class represents a database table @@ -7222,7 +7319,6 @@ class Table(dict): db.users.insert(name='me') # print db.users._insert(...) to see SQL db.users.drop() """ - __slots__ = () def __init__( self, @@ -7304,7 +7400,7 @@ class Table(dict): self.virtualfields = [] fields = list(fields) - if db and self._db._adapter.uploads_in_blob==True: + if db and db._adapter.uploads_in_blob==True: for field in fields: if isinstance(field, Field) and field.type == 'upload'\ and field.uploadfield is True: @@ -7316,27 +7412,28 @@ class Table(dict): lower_fieldnames = set() reserved = dir(Table) + ['fields'] for field in fields: + field_name = field.name if db and db.check_reserved: - db.check_reserved_keyword(field.name) - elif field.name in reserved: - raise SyntaxError, "field name %s not allowed" % field.name + db.check_reserved_keyword(field_name) + elif field_name in reserved: + raise SyntaxError, "field name %s not allowed" % field_name - if field.name.lower() in lower_fieldnames: + if field_name.lower() in lower_fieldnames: raise SyntaxError, "duplicate field %s in table %s" \ - % (field.name, tablename) + % (field_name, tablename) else: - lower_fieldnames.add(field.name.lower()) + lower_fieldnames.add(field_name.lower()) - self.fields.append(field.name) - self[field.name] = field + self.fields.append(field_name) + self[field_name] = field if field.type == 'id': self['id'] = field field.tablename = field._tablename = tablename field.table = field._table = self - field.db = field._db = self._db - if self._db and not field.type in ('text','blob') and \ - self._db._adapter.maxcharlength < field.length: - field.length = self._db._adapter.maxcharlength + field.db = field._db = db + if db and not field.type in ('text','blob') and \ + db._adapter.maxcharlength < field.length: + field.length = db._adapter.maxcharlength if field.requires == DEFAULT: field.requires = sqlhtml_validators(field) self.ALL = SQLALL(self) @@ -7385,27 +7482,29 @@ class Table(dict): def _validate(self,**vars): errors = Row() - for key,value in vars.items(): + for key,value in vars.iteritems(): value,error = self[key].validate(value) if error: errors[key] = error return errors def _create_references(self): - pr = self._db._pending_references + db = self._db + pr = db._pending_references self._referenced_by = [] - for fieldname in self.fields: - field=self[fieldname] - if isinstance(field.type,str) and field.type[:10] == 'reference ': - ref = field.type[10:].strip() + for field in self: + fieldname = field.name + field_type = field.type + if isinstance(field_type,str) and field_type[:10] == 'reference ': + ref = field_type[10:].strip() if not ref.split(): raise SyntaxError, 'Table: reference to nothing: %s' %ref refs = ref.split('.') rtablename = refs[0] - if not rtablename in self._db: + if not rtablename in db: pr[rtablename] = pr.get(rtablename,[]) + [field] continue - rtable = self._db[rtablename] + rtable = db[rtablename] if len(refs)==2: rfieldname = refs[1] if not hasattr(rtable,'_primarykey'): @@ -7415,12 +7514,12 @@ class Table(dict): raise SyntaxError,\ "invalid field '%s' for referenced table '%s' in table '%s'" \ % (rfieldname, rtablename, self._tablename) - rtable._referenced_by.append((self._tablename, field.name)) + rtable._referenced_by.append(field) for referee in pr.get(self._tablename,[]): - self._referenced_by.append((referee._tablename,referee.name)) + self._referenced_by.append(referee) def _filter_fields(self, record, id=False): - return dict([(k, v) for (k, v) in record.items() if k + return dict([(k, v) for (k, v) in record.iteritems() if k in self.fields and (self[k].type!='id' or id)]) def _build_query(self,key): @@ -7451,7 +7550,7 @@ class Table(dict): elif str(key).isdigit() or 'google' in drivers and isinstance(key, Key): return self._db(self._id == key).select(limitby=(0,1)).first() elif key: - return dict.__getitem__(self, str(key)) + return ogetattr(self, str(key)) def __call__(self, key=DEFAULT, **kwargs): for_update = kwargs.get('_for_update',False) @@ -7466,11 +7565,11 @@ class Table(dict): record = self._db(self._id == key).select( limitby=(0,1),for_update=for_update).first() if record: - for k,v in kwargs.items(): + for k,v in kwargs.iteritems(): if record[k]!=v: return None return record elif kwargs: - query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.items()]) + query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()]) return self._db(query).select(limitby=(0,1),for_update=for_update).first() else: return None @@ -7500,10 +7599,15 @@ class Table(dict): if isinstance(key, dict): raise SyntaxError,\ 'value must be a dictionary: %s' % value - dict.__setitem__(self, str(key), value) + osetattr(self, str(key), value) __getattr__ = __getitem__ + def __setattr__(self, key, value): + if key[:1]!='_' and key in self: + raise SyntaxError, 'Object exists and cannot be redefined: %s' % key + osetattr(self,key,value) + def __delitem__(self, key): if isinstance(key, dict): query = self._build_query(key) @@ -7513,20 +7617,25 @@ class Table(dict): not self._db(self._id == key).delete(): raise SyntaxError, 'No such record: %s' % key - def __setattr__(self, key, value): - if key[:1]!='_' and key in self: - raise SyntaxError, 'Object exists and cannot be redefined: %s' % key - self[key] = value + def __contains__(self,key): + return hasattr(self,key) + + def items(self): + return self.__dict__.items() def __iter__(self): for fieldname in self.fields: yield self[fieldname] + def iteritems(self): + return self.__dict__.iteritems() + + def __repr__(self): return '' % (self._tablename,','.join(self.fields())) def __str__(self): - if self.get('_ot', None): + if hasattr(self,'_ot') and self._ot is not None: if 'Oracle' in str(type(self._db._adapter)): # <<< patch return '%s %s' % (self._ot, self._tablename) # <<< patch return '%s AS %s' % (self._ot, self._tablename) @@ -7540,6 +7649,7 @@ class Table(dict): def _listify(self,fields,update=False): new_fields = {} # format: new_fields[name] = (field,value) + # store all fields passed as input in new_fields for name in fields: if not name in self.fields: @@ -7552,24 +7662,36 @@ class Table(dict): if field.filter_in: value = field.filter_in(value) new_fields[name] = (field,value) - # check all fields that should be in the self table + + # check all fields that should be in the table but are not passed + to_compute = [] for ofield in self: name = ofield.name - # if field is supposed to be computed, compute it! - if ofield.compute: - try: - new_fields[name] = (ofield,ofield.compute(Row(fields))) - except KeyError: - pass - # if field is required, check its default value - elif not name in new_fields: - if not update and not ofield.default is None: + if not name in new_fields: + # if field is supposed to be computed, compute it! + if ofield.compute: # save those to compute for later + to_compute.append((name,ofield)) + # if field is required, check its default value + elif not update and not ofield.default is None: new_fields[name] = (ofield,ofield.default) + # if this is an update, user the update field instead elif update and not ofield.update is None: new_fields[name] = (ofield,ofield.update) - # error if field if required, record to be create and field missing - if not update and ofield.required and not name in new_fields: - raise SyntaxError, 'Table: missing required field: %s' % name + # if the field is still not there but it should, error + elif not update and ofield.required: + raise RuntimeError, \ + 'Table: missing required field: %s' % name + # now deal with fields that are supposed to be computed + if to_compute: + dummyrow = Row(new_fields) + for name,ofield in to_compute: + # try compute it + try: + new_fields[name] = (ofield,ofield.compute(dummyrow)) + except (KeyError, AttributeError): + # error sinlently unless field is required! + if ofield.required: + raise SyntaxError, 'unable to comput field: %s' % name return new_fields.values() def _attempt_upload(self, fields): @@ -7601,7 +7723,7 @@ class Table(dict): response = Row() response.errors = Row() new_fields = copy.copy(fields) - for key,value in fields.items(): + for key,value in fields.iteritems(): value,error = self[key].validate(value) if error: response.errors[key] = error @@ -7816,51 +7938,67 @@ class Expression(object): self.type = type def sum(self): - return Expression(self.db, self.db._adapter.AGGREGATE, self, 'SUM', self.type) + db = self.db + return Expression(db, db._adapter.AGGREGATE, self, 'SUM', self.type) def max(self): - return Expression(self.db, self.db._adapter.AGGREGATE, self, 'MAX', self.type) + db = self.db + return Expression(db, db._adapter.AGGREGATE, self, 'MAX', self.type) def min(self): - return Expression(self.db, self.db._adapter.AGGREGATE, self, 'MIN', self.type) + db = self.db + return Expression(db, db._adapter.AGGREGATE, self, 'MIN', self.type) def len(self): - return Expression(self.db, self.db._adapter.AGGREGATE, self, 'LENGTH', 'integer') + db = self.db + return Expression(db, db._adapter.AGGREGATE, self, 'LENGTH', 'integer') def avg(self): - return Expression(self.db, self.db._adapter.AGGREGATE, self, 'AVG', self.type) + db = self.db + return Expression(db, db._adapter.AGGREGATE, self, 'AVG', self.type) def lower(self): - return Expression(self.db, self.db._adapter.LOWER, self, None, self.type) + db = self.db + return Expression(db, db._adapter.LOWER, self, None, self.type) def upper(self): - return Expression(self.db, self.db._adapter.UPPER, self, None, self.type) + db = self.db + return Expression(db, db._adapter.UPPER, self, None, self.type) def year(self): - return Expression(self.db, self.db._adapter.EXTRACT, self, 'year', 'integer') + db = self.db + return Expression(db, db._adapter.EXTRACT, self, 'year', 'integer') def month(self): - return Expression(self.db, self.db._adapter.EXTRACT, self, 'month', 'integer') + db = self.db + return Expression(db, db._adapter.EXTRACT, self, 'month', 'integer') def day(self): - return Expression(self.db, self.db._adapter.EXTRACT, self, 'day', 'integer') + db = self.db + return Expression(db, db._adapter.EXTRACT, self, 'day', 'integer') def hour(self): - return Expression(self.db, self.db._adapter.EXTRACT, self, 'hour', 'integer') + db = self.db + return Expression(db, db._adapter.EXTRACT, self, 'hour', 'integer') def minutes(self): - return Expression(self.db, self.db._adapter.EXTRACT, self, 'minute', 'integer') + db = self.db + return Expression(db, db._adapter.EXTRACT, self, 'minute', 'integer') def coalesce(self,*others): - return Expression(self.db, self.db._adapter.COALESCE, self, others, self.type) + db = self.db + return Expression(db, db._adapter.COALESCE, self, others, self.type) def coalesce_zero(self): - return Expression(self.db, self.db._adapter.COALESCE_ZERO, self, None, self.type) + db = self.db + return Expression(db, db._adapter.COALESCE_ZERO, self, None, self.type) def seconds(self): - return Expression(self.db, self.db._adapter.EXTRACT, self, 'second', 'integer') + db = self.db + return Expression(db, db._adapter.EXTRACT, self, 'second', 'integer') def __getslice__(self, start, stop): + db = self.db if start < 0: pos0 = '(%s - %d)' % (self.len(), abs(start) - 1) else: @@ -7872,7 +8010,7 @@ class Expression(object): length = self.len() else: length = '(%s - %s)' % (stop + 1, pos0) - return Expression(self.db,self.db._adapter.SUBSTRING, + return Expression(db,db._adapter.SUBSTRING, self, (pos0, length), self.type) def __getitem__(self, i): @@ -7882,58 +8020,73 @@ class Expression(object): return self.db._adapter.expand(self,self.type) def __or__(self, other): # for use in sortby - return Expression(self.db,self.db._adapter.COMMA,self,other,self.type) + db = self.db + return Expression(db,db._adapter.COMMA,self,other,self.type) def __invert__(self): - if hasattr(self,'_op') and self.op == self.db._adapter.INVERT: + db = self.db + if hasattr(self,'_op') and self.op == db._adapter.INVERT: return self.first - return Expression(self.db,self.db._adapter.INVERT,self,type=self.type) + return Expression(db,db._adapter.INVERT,self,type=self.type) def __add__(self, other): - return Expression(self.db,self.db._adapter.ADD,self,other,self.type) + db = self.db + return Expression(db,db._adapter.ADD,self,other,self.type) def __sub__(self, other): + db = self.db if self.type in ('integer','bigint'): result_type = 'integer' elif self.type in ['date','time','datetime','double','float']: result_type = 'double' else: raise SyntaxError, "subtraction operation not supported for type" - return Expression(self.db,self.db._adapter.SUB,self,other, - result_type) + return Expression(db,db._adapter.SUB,self,other,result_type) + def __mul__(self, other): - return Expression(self.db,self.db._adapter.MUL,self,other,self.type) + db = self.db + return Expression(db,db._adapter.MUL,self,other,self.type) def __div__(self, other): - return Expression(self.db,self.db._adapter.DIV,self,other,self.type) + db = self.db + return Expression(db,db._adapter.DIV,self,other,self.type) def __mod__(self, other): - return Expression(self.db,self.db._adapter.MOD,self,other,self.type) + db = self.db + return Expression(db,db._adapter.MOD,self,other,self.type) def __eq__(self, value): - return Query(self.db, self.db._adapter.EQ, self, value) + db = self.db + return Query(db, db._adapter.EQ, self, value) def __ne__(self, value): - return Query(self.db, self.db._adapter.NE, self, value) + db = self.db + return Query(db, db._adapter.NE, self, value) def __lt__(self, value): - return Query(self.db, self.db._adapter.LT, self, value) + db = self.db + return Query(db, db._adapter.LT, self, value) def __le__(self, value): - return Query(self.db, self.db._adapter.LE, self, value) + db = self.db + return Query(db, db._adapter.LE, self, value) def __gt__(self, value): - return Query(self.db, self.db._adapter.GT, self, value) + db = self.db + return Query(db, db._adapter.GT, self, value) def __ge__(self, value): - return Query(self.db, self.db._adapter.GE, self, value) + db = self.db + return Query(db, db._adapter.GE, self, value) def like(self, value, case_sensitive=False): - op = case_sensitive and self.db._adapter.LIKE or self.db._adapter.ILIKE - return Query(self.db, op, self, value) + db = self.db + op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE + return Query(db, op, self, value) def regexp(self, value): - return Query(self.db, self.db._adapter.REGEXP, self, value) + db = self.db + return Query(db, db._adapter.REGEXP, self, value) def belongs(self, *value): """ @@ -7945,68 +8098,84 @@ class Expression(object): Does NOT accept: field.belongs(1) """ + db = self.db if len(value) == 1: value = value[0] if isinstance(value,Query): - value = self.db(value)._select(value.first._table._id) - return Query(self.db, self.db._adapter.BELONGS, self, value) + value = db(value)._select(value.first._table._id) + return Query(db, db._adapter.BELONGS, self, value) def startswith(self, value): + db = self.db if not self.type in ('string', 'text'): raise SyntaxError, "startswith used with incompatible field type" - return Query(self.db, self.db._adapter.STARTSWITH, self, value) + return Query(db, db._adapter.STARTSWITH, self, value) def endswith(self, value): + db = self.db if not self.type in ('string', 'text'): raise SyntaxError, "endswith used with incompatible field type" - return Query(self.db, self.db._adapter.ENDSWITH, self, value) + return Query(db, db._adapter.ENDSWITH, self, value) def contains(self, value, all=False): + db = self.db if isinstance(value,(list, tuple)): subqueries = [self.contains(str(v).strip()) for v in value if str(v).strip()] return reduce(all and AND or OR, subqueries) if not self.type in ('string', 'text') and not self.type.startswith('list:'): raise SyntaxError, "contains used with incompatible field type" - return Query(self.db, self.db._adapter.CONTAINS, self, value) + return Query(db, db._adapter.CONTAINS, self, value) def with_alias(self, alias): - return Expression(self.db, self.db._adapter.AS, self, alias, self.type) + db = self.db + return Expression(db, db._adapter.AS, self, alias, self.type) # GIS functions def st_asgeojson(self, precision=15, options=0, version=1): return Expression(self.db, self.db._adapter.ST_ASGEOJSON, self, - dict(precision=precision, options=options, version=version), 'dict') + dict(precision=precision, options=options, + version=version), 'dict') def st_astext(self): - return Expression(self.db, self.db._adapter.ST_ASTEXT, self) + db = self.db + return Expression(db, db._adapter.ST_ASTEXT, self) def st_contained(self, value): - return Query(self.db, self.db._adapter.ST_CONTAINS, value, self) + db = self.db + return Query(db, db._adapter.ST_CONTAINS, value, self) def st_contains(self, value): - return Query(self.db, self.db._adapter.ST_CONTAINS, self, value) + db = self.db + return Query(db, db._adapter.ST_CONTAINS, self, value) def st_distance(self, other): - return Expression(self.db,self.db._adapter.ST_DISTANCE,self,other,self.type) + db = self.db + return Expression(db,db._adapter.ST_DISTANCE,self,other,self.type) def st_equals(self, value): - return Query(self.db, self.db._adapter.ST_EQUALS, self, value) + db = self.db + return Query(db, db._adapter.ST_EQUALS, self, value) def st_intersects(self, value): - return Query(self.db, self.db._adapter.ST_INTERSECTS, self, value) + db = self.db + return Query(db, db._adapter.ST_INTERSECTS, self, value) def st_overlaps(self, value): - return Query(self.db, self.db._adapter.ST_OVERLAPS, self, value) + db = self.db + return Query(db, db._adapter.ST_OVERLAPS, self, value) def st_simplify(self, value): - return Expression(self.db, self.db._adapter.ST_SIMPLIFY, self, value) + db = self.db + return Expression(db, db._adapter.ST_SIMPLIFY, self, value) def st_touches(self, value): - return Query(self.db, self.db._adapter.ST_TOUCHES, self, value) + db = self.db + return Query(db, db._adapter.ST_TOUCHES, self, value) def st_within(self, value): - return Query(self.db, self.db._adapter.ST_WITHIN, self, value) + db = self.db + return Query(db, db._adapter.ST_WITHIN, self, value) # for use in both Query and sortby @@ -8222,29 +8391,30 @@ class Field(Expression): newfilename = '%s.%s.%s.%s' % \ (self._tablename, self.name, uuid_key, encoded_filename) newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension - if isinstance(self.uploadfield,Field): - blob_uploadfield_name = self.uploadfield.uploadfield - keys={self.uploadfield.name: newfilename, + self_uploadfield = self.uploadfield + if isinstance(self_uploadfield,Field): + blob_uploadfield_name = self_uploadfield.uploadfield + keys={self_uploadfield.name: newfilename, blob_uploadfield_name: file.read()} - self.uploadfield.table.insert(**keys) - elif self.uploadfield == True: + self_uploadfield.table.insert(**keys) + elif self_uploadfield == True: if path: pass elif self.uploadfolder: path = self.uploadfolder elif self.db._adapter.folder: - path = os.path.join(self.db._adapter.folder, '..', 'uploads') + path = pjoin(self.db._adapter.folder, '..', 'uploads') else: raise RuntimeError,\ "you must specify a Field(...,uploadfolder=...)" if self.uploadseparate: if self.uploadfs: raise RuntimeError, "not supported" - path = os.path.join(path,"%s.%s" %(self._tablename, self.name), + path = pjoin(path,"%s.%s" %(self._tablename, self.name), uuid_key[:2]) - if not os.path.exists(path): + if not exists(path): os.makedirs(path) - pathfilename = os.path.join(path, newfilename) + pathfilename = pjoin(path, newfilename) if self.uploadfs: dest_file = self.uploadfs.open(newfilename, 'wb') else: @@ -8257,10 +8427,11 @@ class Field(Expression): return newfilename def retrieve(self, name, path=None): + self_uploadfield = self.uploadfield if self.custom_retrieve: return self.custom_retrieve(name, path) import http - if self.authorize or isinstance(self.uploadfield, str): + if self.authorize or isinstance(self_uploadfield, str): row = self.db(self == name).select().first() if not row: raise http.HTTP(404) @@ -8271,22 +8442,23 @@ class Field(Expression): raise TypeError, 'Can\'t retrieve %s' % name file_properties = self.retrieve_file_properties(name,path) filename = file_properties['filename'] - if isinstance(self.uploadfield, str): # ## if file is in DB - stream = cStringIO.StringIO(row[self.uploadfield] or '') - elif isinstance(self.uploadfield,Field): - blob_uploadfield_name = self.uploadfield.uploadfield - query = self.uploadfield == name - data = self.uploadfield.table(query)[blob_uploadfield_name] + if isinstance(self_uploadfield, str): # ## if file is in DB + stream = cStringIO.StringIO(row[self_uploadfield] or '') + elif isinstance(self_uploadfield,Field): + blob_uploadfield_name = self_uploadfield.uploadfield + query = self_uploadfield == name + data = self_uploadfield.table(query)[blob_uploadfield_name] stream = cStringIO.StringIO(data) elif self.uploadfs: # ## if file is on pyfilesystem stream = self.uploadfs.open(name, 'rb') else: # ## if file is on regular filesystem - stream = open(os.path.join(file_properties['path'], name), 'rb') + stream = open(pjoin(file_properties['path'], name), 'rb') return (filename, stream) def retrieve_file_properties(self, name, path=None): + self_uploadfield = self.uploadfield if self.custom_retrieve_file_properties: return self.custom_retrieve_file_properties(name, path) try: @@ -8297,9 +8469,9 @@ class Field(Expression): filename = regex_cleanup_fn.sub('_', filename) except (TypeError, AttributeError): filename = name - if isinstance(self.uploadfield, str): # ## if file is in DB + if isinstance(self_uploadfield, str): # ## if file is in DB return dict(path=None,filename=filename) - elif isinstance(self.uploadfield,Field): + elif isinstance(self_uploadfield,Field): return dict(path=None,filename=filename) else: # ## if file is on filesystem @@ -8308,24 +8480,25 @@ class Field(Expression): elif self.uploadfolder: path = self.uploadfolder else: - path = os.path.join(self.db._adapter.folder, '..', 'uploads') + path = pjoin(self.db._adapter.folder, '..', 'uploads') if self.uploadseparate: t = m.group('table') f = m.group('field') u = m.group('uuidkey') - path = os.path.join(path,"%s.%s" % (t,f),u[:2]) + path = pjoin(path,"%s.%s" % (t,f),u[:2]) return dict(path=path,filename=filename) def formatter(self, value): - if value is None or not self.requires: + requires = self.requires + if value is None or not requires: return value - if not isinstance(self.requires, (list, tuple)): - requires = [self.requires] - elif isinstance(self.requires, tuple): - requires = list(self.requires) + if not isinstance(requires, (list, tuple)): + requires = [requires] + elif isinstance(requires, tuple): + requires = list(requires) else: - requires = copy.copy(self.requires) + requires = copy.copy(requires) requires.reverse() for item in requires: if hasattr(item, 'formatter'): @@ -8470,19 +8643,32 @@ class Set(object): return adapter._select(self.query,fields,attributes) def _delete(self): - tablename=self.db._adapter.get_table(self.query) - return self.db._adapter._delete(tablename,self.query) + db = self.db + tablename = db._adapter.get_table(self.query) + return db._adapter._delete(tablename,self.query) def _update(self, **update_fields): - tablename = self.db._adapter.get_table(self.query) - fields = self.db[tablename]._listify(update_fields,update=True) - return self.db._adapter._update(tablename,self.query,fields) + db = self.db + tablename = db._adapter.get_table(self.query) + fields = db[tablename]._listify(update_fields,update=True) + return db._adapter._update(tablename,self.query,fields) def isempty(self): return not self.select(limitby=(0,1)) - def count(self,distinct=None): - return self.db._adapter.count(self.query,distinct) + def count(self,distinct=None, cache=None): + db = self.db + if cache: + cache_model, time_expire = cache + sql = self._count(distinct=distinct) + key = db._uri + '/' + sql + if len(key)>200: key = hashlib.md5(key).hexdigest() + return cache_model( + key, + (lambda self=self,distinct=distinct: \ + db._adapter.count(self.query,distinct)), + time_expire) + return db._adapter.count(self.query,distinct) def select(self, *fields, **attributes): if self.query is None:# and fields[0]._table._common_filter != None: @@ -8495,21 +8681,25 @@ class Set(object): return Expression(self.db,self._select(*fields,**attributes)) def delete(self): - tablename=self.db._adapter.get_table(self.query) - table = self.db[tablename] + db = self.db + tablename = db._adapter.get_table(self.query) + table = db[tablename] if any(f(self) for f in table._before_delete): return 0 - ret = self.db._adapter.delete(tablename,self.query) + ret = db._adapter.delete(tablename,self.query) ret and [f(self) for f in table._after_delete] return ret def update(self, **update_fields): - tablename = self.db._adapter.get_table(self.query) - table = self.db[tablename] + db = self.db + tablename = db._adapter.get_table(self.query) + table = db[tablename] table._attempt_upload(update_fields) - if any(f(self,update_fields) for f in table._before_update): return 0 + if any(f(self,update_fields) for f in table._before_update): + return 0 fields = table._listify(update_fields,update=True) - if not fields: raise SyntaxError, "No fields to update" - ret = self.db._adapter.update(tablename,self.query,fields) + if not fields: + raise SyntaxError, "No fields to update" + ret = db._adapter.update(tablename,self.query,fields) ret and [f(self,update_fields) for f in table._after_update] return ret @@ -8529,7 +8719,7 @@ class Set(object): response = Row() response.errors = Row() new_fields = copy.copy(update_fields) - for key,value in update_fields.items(): + for key,value in update_fields.iteritems(): value,error = self.db[tablename][key].validate(value) if error: response.errors[key] = error @@ -8574,24 +8764,39 @@ class Set(object): else: uploadfolder = field.uploadfolder if not uploadfolder: - uploadfolder = os.path.join(self.db._adapter.folder, '..', 'uploads') + uploadfolder = pjoin( + self.db._adapter.folder, '..', 'uploads') if field.uploadseparate: items = oldname.split('.') - uploadfolder = os.path.join(uploadfolder, - "%s.%s" % (items[0], items[1]), - items[2][:2]) - oldpath = os.path.join(uploadfolder, oldname) - if os.path.exists(oldpath): + uploadfolder = pjoin( + uploadfolder, + "%s.%s" % (items[0], items[1]), + items[2][:2]) + oldpath = pjoin(uploadfolder, oldname) + if exists(oldpath): os.unlink(oldpath) return False -def update_record(pack, a=None): - (colset, table, id) = pack - b = a or dict(colset) - c = dict([(k,v) for (k,v) in b.items() if k in table.fields and table[k].type!='id']) - table._db(table._id==id,ignore_common_filters=True).update(**c) - for (k, v) in c.items(): - colset[k] = v +class RecordUpdater(object): + def __init__(self, colset, table, id): + self.colset, self.table, self.id = colset, table, id + + def __call__(self, **fields): + colset, table, id = self.colset, self.table, self.id + newfields = fields or dict(colset) + for fieldname in newfields.keys(): + if not fieldname in table.fields or table[fieldname].type=='id': + del newfields[fieldname] + table._db(table._id==id,ignore_common_filters=True).update(**newfields) + colset.update(newfields) + return colset + +class RecordDeleter(object): + def __init__(self, table, id): + self.table, self.id = table,id + def __call__(self): + return self.table._db(self.table._id==self.id).delete() + class VirtualCommand(object): def __init__(self,method,row): @@ -8651,7 +8856,7 @@ class Rows(object): if not keyed_virtualfields: return self for row in self.records: - for (tablename,virtualfields) in keyed_virtualfields.items(): + for (tablename,virtualfields) in keyed_virtualfields.iteritems(): attributes = dir(virtualfields) if not tablename in row: box = row[tablename] = Row() @@ -8894,10 +9099,11 @@ class Rows(object): """ serializes the table using sqlhtml.SQLTABLE (if present) """ + alphanumeric = re.compile('[a-zA-Z]\w*') if strict: ncols = len(self.colnames) def f(row,field,indent=' '): - if isinstance(row,dict): + if isinstance(row,Row): spc = indent+' \n' items = [f(row[x],x,indent+' ') for x in row] return '%s<%s>\n%s\n%s' % ( @@ -8907,7 +9113,12 @@ class Rows(object): indent, field) elif not callable(row): - return '%s<%s>%s' % (indent,field,row,field) + if alphanumeric.match(field): + return '%s<%s>%s' % \ + (indent,field,row,field) + else: + return '%s%s' % \ + (indent,field,row) else: return None return '<%s>\n%s\n' % ( @@ -9015,6 +9226,17 @@ def test_all(): >>> me = db(db.person.id==person_id).select()[0] # test select >>> me.name 'Massimo' + >>> db.person[2].name + 'Massimo' + >>> db.person(2).name + 'Massimo' + >>> db.person(name='Massimo').name + 'Massimo' + >>> db.person(db.person.name=='Massimo').name + 'Massimo' + >>> row = db.person[2] + >>> row.name == row['name'] == row['person.name'] == row('person.name') + True >>> db(db.person.name=='Massimo').update(name='massimo') # test update 1 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete diff --git a/gluon/main.py b/gluon/main.py index 85f57fca..efa30df7 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -425,14 +425,14 @@ def wsgibase(environ, responder): client = get_client(env) x_req_with = str(env.http_x_requested_with).lower() - request.update(dict( - client = client, - folder = abspath('applications',app) + os.sep, - ajax = x_req_with == 'xmlhttprequest', - cid = env.http_web2py_component_element, - is_local = env.remote_addr in local_hosts, - is_https = env.wsgi_url_scheme \ - in ['https', 'HTTPS'] or env.https=='on')) + request.update( + client = client, + folder = abspath('applications',app) + os.sep, + ajax = x_req_with == 'xmlhttprequest', + cid = env.http_web2py_component_element, + is_local = env.remote_addr in local_hosts, + is_https = env.wsgi_url_scheme \ + in ['https', 'HTTPS'] or env.https=='on') request.uuid = request.compute_uuid() # requires client # ################################################## @@ -561,15 +561,15 @@ def wsgibase(environ, responder): # ################################################## if request.cid: - rheaders + rheaders = response.headers if response.flash and \ not 'web2py-component-flash' in rheaders: rheaders['web2py-component-flash'] = \ urllib2.quote(xmlescape(response.flash)\ .replace('\n','')) if response.js and \ - not 'web2py-component-command' in readers: - readers['web2py-component-command'] = \ + not 'web2py-component-command' in rheaders: + rheaders['web2py-component-command'] = \ response.js.replace('\n','') rcookies = response.cookies if session._forget and \ diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 6f3b9220..20bfef25 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -292,7 +292,7 @@ class MultipleOptionsWidget(OptionsWidget): be shown """ - attributes.update(dict(_size=size, _multiple=True)) + attributes.update(_size=size, _multiple=True) return OptionsWidget.widget(field, value, **attributes) diff --git a/gluon/tools.py b/gluon/tools.py index e51e3d33..58ab9617 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -800,139 +800,139 @@ def addrow(form, a, b, c, style, _id, position=-1): class Auth(object): - default_settings = { - 'hideerror': False, - 'password_min_length': 4, - 'cas_maps': None, - 'reset_password_requires_verification': False, - 'registration_requires_verification': False, - 'registration_requires_approval': False, - 'login_after_registration': False, - 'login_after_password_change': True, - 'alternate_requires_registration': False, - 'create_user_groups': "user_%(id)s", - 'everybody_group_id': None, - 'login_captcha': None, - 'register_captcha': None, - 'retrieve_username_captcha': None, - 'retrieve_password_captcha': None, - 'captcha': None, - 'expiration': 3600, # one hour - 'long_expiration': 3600*30*24, # one month - 'remember_me_form': True, - 'allow_basic_login': False, - 'allow_basic_login_only': False, - 'on_failed_authentication': lambda x: redirect(x), - 'formstyle': 'table3cols', - 'label_separator': ': ', - 'password_field': 'password', - 'table_user_name': 'auth_user', - 'table_group_name': 'auth_group', - 'table_membership_name': 'auth_membership', - 'table_permission_name': 'auth_permission', - 'table_event_name': 'auth_event', - 'table_cas_name': 'auth_cas', - 'table_user': None, - 'table_group': None, - 'table_membership': None, - 'table_permission': None, - 'table_event': None, - 'table_cas': None, - 'showid': False, - 'login_email_validate': True, - 'login_userfield': None, - 'logout_onlogout': None, - 'register_fields': None, - 'register_verify_password': True, - 'profile_fields': None, - 'email_case_sensitive': True, - 'username_case_sensitive': True, - } + default_settings = dict( + hideerror = False, + password_min_length = 4, + cas_maps = None, + reset_password_requires_verification = False, + registration_requires_verification = False, + registration_requires_approval = False, + login_after_registration = False, + login_after_password_change = True, + alternate_requires_registration = False, + create_user_groups = "user_%(id)s", + everybody_group_id = None, + login_captcha = None, + register_captcha = None, + retrieve_username_captcha = None, + retrieve_password_captcha = None, + captcha = None, + expiration = 3600, # one hour + long_expiration = 3600*30*24, # one month + remember_me_form = True, + allow_basic_login = False, + allow_basic_login_only = False, + on_failed_authentication = lambda x: redirect(x), + formstyle = "table3cols", + label_separator = ": ", + password_field = 'password', + table_user_name = 'auth_user', + table_group_name = 'auth_group', + table_membership_name = 'auth_membership', + table_permission_name = 'auth_permission', + table_event_name = 'auth_event', + table_cas_name = 'auth_cas', + table_user = None, + table_group = None, + table_membership = None, + table_permission = None, + table_event = None, + table_cas = None, + showid = False, + login_email_validate = True, + login_userfield = None, + logout_onlogout = None, + register_fields = None, + register_verify_password = True, + profile_fields = None, + email_case_sensitive = True, + username_case_sensitive = True, + ) # ## these are messages that can be customized - default_messages = { - 'login_button': 'Login', - 'register_button': 'Register', - 'password_reset_button': 'Request reset password', - 'password_change_button': 'Change password', - 'profile_save_button': 'Save profile', - 'submit_button': 'Submit', - 'verify_password': 'Verify Password', - 'delete_label': 'Check to delete', - 'function_disabled': 'Function disabled', - 'access_denied': 'Insufficient privileges', - 'registration_verifying': 'Registration needs verification', - 'registration_pending': 'Registration is pending approval', - 'login_disabled': 'Login disabled by administrator', - 'logged_in': 'Logged in', - 'email_sent': 'Email sent', - 'unable_to_send_email': 'Unable to send email', - 'email_verified': 'Email verified', - 'logged_out': 'Logged out', - 'registration_successful': 'Registration successful', - 'invalid_email': 'Invalid email', - 'unable_send_email': 'Unable to send email', - 'invalid_login': 'Invalid login', - 'invalid_user': 'Invalid user', - 'invalid_password': 'Invalid password', - 'is_empty': "Cannot be empty", - 'mismatched_password': "Password fields don't match", - 'verify_email': 'Click on the link %(link)s to verify your email', - 'verify_email_subject': 'Email verification', - 'username_sent': 'Your username was emailed to you', - 'new_password_sent': 'A new password was emailed to you', - 'password_changed': 'Password changed', - 'retrieve_username': 'Your username is: %(username)s', - 'retrieve_username_subject': 'Username retrieve', - 'retrieve_password': 'Your password is: %(password)s', - 'retrieve_password_subject': 'Password retrieve', - 'reset_password': \ + default_messages = dict( + login_button = 'Login', + register_button = 'Register', + password_reset_button = 'Request reset password', + password_change_button = 'Change password', + profile_save_button = 'Save profile', + submit_button = 'Submit', + verify_password = 'Verify Password', + delete_label = 'Check to delete', + function_disabled = 'Function disabled', + access_denied = 'Insufficient privileges', + registration_verifying = 'Registration needs verification', + registration_pending = 'Registration is pending approval', + login_disabled = 'Login disabled by administrator', + logged_in = 'Logged in', + email_sent = 'Email sent', + unable_to_send_email = 'Unable to send email', + email_verified = 'Email verified', + logged_out = 'Logged out', + registration_successful = 'Registration successful', + invalid_email = 'Invalid email', + unable_send_email = 'Unable to send email', + invalid_login = 'Invalid login', + invalid_user = 'Invalid user', + invalid_password = 'Invalid password', + is_empty = "Cannot be empty", + mismatched_password = "Password fields don't match", + verify_email = 'Click on the link %(link)s to verify your email', + verify_email_subject = 'Email verification', + username_sent = 'Your username was emailed to you', + new_password_sent = 'A new password was emailed to you', + password_changed = 'Password changed', + retrieve_username = 'Your username is: %(username)s', + retrieve_username_subject = 'Username retrieve', + retrieve_password = 'Your password is: %(password)s', + retrieve_password_subject = 'Password retrieve', + reset_password = \ 'Click on the link %(link)s to reset your password', - 'reset_password_subject': 'Password reset', - 'invalid_reset_password': 'Invalid reset password', - 'profile_updated': 'Profile updated', - 'new_password': 'New password', - 'old_password': 'Old password', - 'group_description': 'Group uniquely assigned to user %(id)s', - 'register_log': 'User %(id)s Registered', - 'login_log': 'User %(id)s Logged-in', - 'login_failed_log': None, - 'logout_log': 'User %(id)s Logged-out', - 'profile_log': 'User %(id)s Profile updated', - 'verify_email_log': 'User %(id)s Verification email sent', - 'retrieve_username_log': 'User %(id)s Username retrieved', - 'retrieve_password_log': 'User %(id)s Password retrieved', - 'reset_password_log': 'User %(id)s Password reset', - 'change_password_log': 'User %(id)s Password changed', - 'add_group_log': 'Group %(group_id)s created', - 'del_group_log': 'Group %(group_id)s deleted', - 'add_membership_log': None, - 'del_membership_log': None, - 'has_membership_log': None, - 'add_permission_log': None, - 'del_permission_log': None, - 'has_permission_log': None, - 'impersonate_log': 'User %(id)s is impersonating %(other_id)s', - 'label_first_name': 'First name', - 'label_last_name': 'Last name', - 'label_username': 'Username', - 'label_email': 'E-mail', - 'label_password': 'Password', - 'label_registration_key': 'Registration key', - 'label_reset_password_key': 'Reset Password key', - 'label_registration_id': 'Registration identifier', - 'label_role': 'Role', - 'label_description': 'Description', - 'label_user_id': 'User ID', - 'label_group_id': 'Group ID', - 'label_name': 'Name', - 'label_table_name': 'Object or table name', - 'label_record_id': 'Record ID', - 'label_time_stamp': 'Timestamp', - 'label_client_ip': 'Client IP', - 'label_origin': 'Origin', - 'label_remember_me': "Remember me (for 30 days)", - 'verify_password_comment': 'please input your password again', - } + reset_password_subject = 'Password reset', + invalid_reset_password = 'Invalid reset password', + profile_updated = 'Profile updated', + new_password = 'New password', + old_password = 'Old password', + group_description = 'Group uniquely assigned to user %(id)s', + register_log = 'User %(id)s Registered', + login_log = 'User %(id)s Logged-in', + login_failed_log = None, + logout_log = 'User %(id)s Logged-out', + profile_log = 'User %(id)s Profile updated', + verify_email_log = 'User %(id)s Verification email sent', + retrieve_username_log = 'User %(id)s Username retrieved', + retrieve_password_log = 'User %(id)s Password retrieved', + reset_password_log = 'User %(id)s Password reset', + change_password_log = 'User %(id)s Password changed', + add_group_log = 'Group %(group_id)s created', + del_group_log = 'Group %(group_id)s deleted', + add_membership_log = None, + del_membership_log = None, + has_membership_log = None, + add_permission_log = None, + del_permission_log = None, + has_permission_log = None, + impersonate_log = 'User %(id)s is impersonating %(other_id)s', + label_first_name = 'First name', + label_last_name = 'Last name', + label_username = 'Username', + label_email = 'E-mail', + label_password = 'Password', + label_registration_key = 'Registration key', + label_reset_password_key = 'Reset Password key', + label_registration_id = 'Registration identifier', + label_role = 'Role', + label_description = 'Description', + label_user_id = 'User ID', + label_group_id = 'Group ID', + label_name = 'Name', + label_table_name = 'Object or table name', + label_record_id = 'Record ID', + label_time_stamp = 'Timestamp', + label_client_ip = 'Client IP', + label_origin = 'Origin', + label_remember_me = "Remember me (for 30 days)", + verify_password_comment = 'please input your password again', + ) """ Class for authentication, authorization, role based access control. @@ -1081,51 +1081,51 @@ class Auth(object): settings = self.settings = Settings() settings.update(Auth.default_settings) - settings.update({ - 'cas_domains': [request.env.http_host], - 'cas_provider': cas_provider, - 'cas_actions': {'login':'login', - 'validate':'validate', - 'servicevalidate':'serviceValidate', - 'proxyvalidate':'proxyValidate', - 'logout':'logout'}, - 'extra_fields': {}, - 'actions_disabled': [], - 'controller': controller, - 'function': function, - 'login_url': url_login, - 'logged_url': URL(controller, function, args='profile'), - 'download_url': URL(controller,'download'), - 'mailer': (mailer==True) and Mail() or mailer, - 'on_failed_authorization': \ - URL(controller,function, args='not_authorized'), - 'login_next': url_index, - 'login_onvalidation': [], - 'login_onaccept': [], - 'login_methods': [self], - 'login_form': self, - 'logout_next': url_index, - 'logout_onlogout': None, - 'register_next': url_index, - 'register_onvalidation': [], - 'register_onaccept': [], - 'verify_email_next': url_login, - 'verify_email_onaccept': [], - 'profile_next': url_index, - 'profile_onvalidation': [], - 'profile_onaccept': [], - 'retrieve_username_next': url_index, - 'retrieve_password_next': url_index, - 'request_reset_password_next': url_login, - 'reset_password_next': url_index, - 'change_password_next': url_index, - 'change_password_onvalidation': [], - 'change_password_onaccept': [], - 'retrieve_password_onvalidation': [], - 'reset_password_onvalidation': [], - 'reset_password_onaccept': [], - 'hmac_key': hmac_key, - }) + settings.update( + cas_domains = [request.env.http_host], + cas_provider = cas_provider, + cas_actions = dict(login ='login', + validate ='validate', + servicevalidate ='serviceValidate', + proxyvalidate ='proxyValidate', + logout ='logout'), + extra_fields = {}, + actions_disabled = [], + controller = controller, + function = function, + login_url = url_login, + logged_url = URL(controller, function, args='profile'), + download_url = URL(controller,'download'), + mailer = (mailer==True) and Mail() or mailer, + on_failed_authorization = \ + URL(controller,function, args='not_authorized'), + login_next = url_index, + login_onvalidation = [], + login_onaccept = [], + login_methods = [self], + login_form = self, + logout_next = url_index, + logout_onlogout = None, + register_next = url_index, + register_onvalidation = [], + register_onaccept = [], + verify_email_next = url_login, + verify_email_onaccept = [], + profile_next = url_index, + profile_onvalidation = [], + profile_onaccept = [], + retrieve_username_next = url_index, + retrieve_password_next = url_index, + request_reset_password_next = url_login, + reset_password_next = url_index, + change_password_next = url_index, + change_password_onvalidation = [], + change_password_onaccept = [], + retrieve_password_onvalidation = [], + reset_password_onvalidation = [], + reset_password_onaccept = [], + hmac_key = hmac_key, + ) settings.lock_keys = True # ## these are messages that can be customized @@ -2344,8 +2344,9 @@ class Auth(object): redirect(self.url(args=request.args)) password = self.random_password() passfield = self.settings.password_field - d = {passfield: str(table_user[passfield].validate(password)[0]), - 'registration_key': ''} + d = dict( + passfield = str(table_user[passfield].validate(password)[0]), + registration_key = '') user.update_record(**d) if self.settings.mailer and \ self.settings.mailer.send(to=form.vars.email, diff --git a/scripts/bench.py b/scripts/bench.py new file mode 100644 index 00000000..ffb98452 --- /dev/null +++ b/scripts/bench.py @@ -0,0 +1,15 @@ +import time +import sys +import urllib2 +import urllib2 + +n = int(sys.argv[1]) +url = sys.argv[2] +headers = {"Accept-Language" : "en" } +req = urllib2.Request(url, None, headers) + +t0 = time.time() +for k in xrange(n): + data = urllib2.urlopen(req).read() +print (time.time()-t0)/n +if n==1: print data diff --git a/scripts/fiximages.py b/scripts/fiximages.py new file mode 100644 index 00000000..d297939b --- /dev/null +++ b/scripts/fiximages.py @@ -0,0 +1,25 @@ +import os +import sys +paths = [sys.argv[1]] +paths1 = [] +paths2 = [] +while paths: + path = paths.pop() + for filename in os.listdir(path): + fullname = os.path.join(path,filename) + if os.path.isdir(fullname): + paths.append(fullname) + else: + extension = filename.split('.')[-1] + if extension.lower() in ('png','gif','jpg','jpeg','js','css'): + paths1.append((filename,fullname)) + if extension.lower() in ('css','js','py','html'): + paths2.append(fullname) +for filename,fullname in paths1: + for otherfullname in paths2: + if open(otherfullname).read().find(filename)>=0: + break + else: + print fullname + # os.system('hg rm '+fullname) + # os.system('rm '+fullname) diff --git a/scripts/rmorphans.py b/scripts/rmorphans.py new file mode 100644 index 00000000..d297939b --- /dev/null +++ b/scripts/rmorphans.py @@ -0,0 +1,25 @@ +import os +import sys +paths = [sys.argv[1]] +paths1 = [] +paths2 = [] +while paths: + path = paths.pop() + for filename in os.listdir(path): + fullname = os.path.join(path,filename) + if os.path.isdir(fullname): + paths.append(fullname) + else: + extension = filename.split('.')[-1] + if extension.lower() in ('png','gif','jpg','jpeg','js','css'): + paths1.append((filename,fullname)) + if extension.lower() in ('css','js','py','html'): + paths2.append(fullname) +for filename,fullname in paths1: + for otherfullname in paths2: + if open(otherfullname).read().find(filename)>=0: + break + else: + print fullname + # os.system('hg rm '+fullname) + # os.system('rm '+fullname)