From b05daddc5c61762832cf7defa1018c67bc750446 Mon Sep 17 00:00:00 2001 From: Oscar Fonts Date: Tue, 19 Nov 2013 19:17:11 +0100 Subject: [PATCH 01/52] Copy extract_pgsql_models as extract_oracle_models --- scripts/extract_oracle_models.py | 286 +++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 scripts/extract_oracle_models.py diff --git a/scripts/extract_oracle_models.py b/scripts/extract_oracle_models.py new file mode 100644 index 00000000..79a1699f --- /dev/null +++ b/scripts/extract_oracle_models.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Create web2py model (python code) to represent PostgreSQL tables. + +Features: + +* Uses ANSI Standard INFORMATION_SCHEMA (might work with other RDBMS) +* Detects legacy "keyed" tables (not having an "id" PK) +* Connects directly to running databases, no need to do a SQL dump +* Handles notnull, unique and referential constraints +* Detects most common datatypes and default values +* Support PostgreSQL columns comments (ie. for documentation) + +Requeriments: + +* Needs PostgreSQL pyscopg2 python connector (same as web2py) +* If used against other RDBMS, import and use proper connector (remove pg_ code) + + +Created by Mariano Reingart, based on a script to "generate schemas from dbs" +(mysql) by Alexandre Andrade + +""" + +_author__ = "Mariano Reingart " + +HELP = """ +USAGE: extract_pgsql_models db host port user passwd + +Call with PostgreSQL database connection parameters, +web2py model will be printed on standard output. + +EXAMPLE: python extract_pgsql_models.py mydb localhost 5432 reingart saraza +""" + +# Config options +DEBUG = False # print debug messages to STDERR +SCHEMA = 'public' # change if not using default PostgreSQL schema + +# Constant for Field keyword parameter order (and filter): +KWARGS = ('type', 'length', 'default', 'required', 'ondelete', + 'notnull', 'unique', 'label', 'comment') + + +import sys + + +def query(conn, sql, *args): + "Execute a SQL query and return rows as a list of dicts" + cur = conn.cursor() + ret = [] + try: + if DEBUG: + print >> sys.stderr, "QUERY: ", sql % args + cur.execute(sql, args) + for row in cur: + dic = {} + for i, value in enumerate(row): + field = cur.description[i][0] + dic[field] = value + if DEBUG: + print >> sys.stderr, "RET: ", dic + ret.append(dic) + return ret + finally: + cur.close() + + +def get_tables(conn, schema=SCHEMA): + "List table names in a given schema" + rows = query(conn, """SELECT table_name FROM information_schema.tables + WHERE table_schema = %s + ORDER BY table_name""", schema) + return [row['table_name'] for row in rows] + + +def get_fields(conn, table): + "Retrieve field list for a given table" + if DEBUG: + print >> sys.stderr, "Processing TABLE", table + rows = query(conn, """ + SELECT column_name, data_type, + is_nullable, + character_maximum_length, + numeric_precision, numeric_precision_radix, numeric_scale, + column_default + FROM information_schema.columns + WHERE table_name=%s + ORDER BY ordinal_position""", table) + return rows + + +def define_field(conn, table, field, pks): + "Determine field type, default value, references, etc." + f = {} + ref = references(conn, table, field['column_name']) + if ref: + f.update(ref) + elif field['column_default'] and \ + field['column_default'].startswith("nextval") and \ + field['column_name'] in pks: + # postgresql sequence (SERIAL) and primary key! + f['type'] = "'id'" + elif field['data_type'].startswith('character'): + f['type'] = "'string'" + if field['character_maximum_length']: + f['length'] = field['character_maximum_length'] + elif field['data_type'] in ('text', ): + f['type'] = "'text'" + elif field['data_type'] in ('boolean', 'bit'): + f['type'] = "'boolean'" + elif field['data_type'] in ('integer', 'smallint', 'bigint'): + f['type'] = "'integer'" + elif field['data_type'] in ('double precision', 'real'): + f['type'] = "'double'" + elif field['data_type'] in ('timestamp', 'timestamp without time zone'): + f['type'] = "'datetime'" + elif field['data_type'] in ('date', ): + f['type'] = "'date'" + elif field['data_type'] in ('time', 'time without time zone'): + f['type'] = "'time'" + elif field['data_type'] in ('numeric', 'currency'): + f['type'] = "'decimal'" + f['precision'] = field['numeric_precision'] + f['scale'] = field['numeric_scale'] or 0 + elif field['data_type'] in ('bytea', ): + f['type'] = "'blob'" + elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'): + f['type'] = "" # unsupported? + else: + raise RuntimeError("Data Type not supported: %s " % str(field)) + + try: + if field['column_default']: + if field['column_default'] == "now()": + d = "request.now" + elif field['column_default'] == "true": + d = "True" + elif field['column_default'] == "false": + d = "False" + else: + d = repr(eval(field['column_default'])) + f['default'] = str(d) + except (ValueError, SyntaxError): + pass + except Exception, e: + raise RuntimeError( + "Default unsupported '%s'" % field['column_default']) + + if not field['is_nullable']: + f['notnull'] = "True" + + comment = get_comment(conn, table, field) + if comment is not None: + f['comment'] = repr(comment) + return f + + +def is_unique(conn, table, field): + "Find unique columns (incomplete support)" + rows = query(conn, """ + SELECT information_schema.constraint_column_usage.column_name + FROM information_schema.table_constraints + NATURAL JOIN information_schema.constraint_column_usage + WHERE information_schema.table_constraints.table_name=%s + AND information_schema.constraint_column_usage.column_name=%s + AND information_schema.table_constraints.constraint_type='UNIQUE' + ;""", table, field['column_name']) + return rows and True or False + + +def get_comment(conn, table, field): + "Find the column comment (postgres specific)" + rows = query(conn, """ + SELECT d.description AS comment + FROM pg_class c + JOIN pg_description d ON c.oid=d.objoid + JOIN pg_attribute a ON c.oid = a.attrelid + WHERE c.relname=%s AND a.attname=%s + AND a.attnum = d.objsubid + ;""", table, field['column_name']) + return rows and rows[0]['comment'] or None + + +def primarykeys(conn, table): + "Find primary keys" + rows = query(conn, """ + SELECT information_schema.constraint_column_usage.column_name + FROM information_schema.table_constraints + NATURAL JOIN information_schema.constraint_column_usage + WHERE information_schema.table_constraints.table_name=%s + AND information_schema.table_constraints.constraint_type='PRIMARY KEY' + ;""", table) + return [row['column_name'] for row in rows] + + +def references(conn, table, field): + "Find a FK (fails if multiple)" + rows1 = query(conn, """ + SELECT table_name, column_name, constraint_name, + update_rule, delete_rule, ordinal_position + FROM information_schema.key_column_usage + NATURAL JOIN information_schema.referential_constraints + NATURAL JOIN information_schema.table_constraints + WHERE information_schema.key_column_usage.table_name=%s + AND information_schema.key_column_usage.column_name=%s + AND information_schema.table_constraints.constraint_type='FOREIGN KEY' + ;""", table, field) + if len(rows1) == 1: + rows2 = query(conn, """ + SELECT table_name, column_name, * + FROM information_schema.constraint_column_usage + WHERE constraint_name=%s + """, rows1[0]['constraint_name']) + row = None + if len(rows2) > 1: + row = rows2[int(rows1[0]['ordinal_position']) - 1] + keyed = True + if len(rows2) == 1: + row = rows2[0] + keyed = False + if row: + if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!! + ref = {'type': "'reference %s.%s'" % (row['table_name'], + row['column_name'])} + else: + ref = {'type': "'reference %s'" % (row['table_name'],)} + if rows1[0]['delete_rule'] != "NO ACTION": + ref['ondelete'] = repr(rows1[0]['delete_rule']) + return ref + elif rows2: + raise RuntimeError("Unsupported foreign key reference: %s" % + str(rows2)) + + elif rows1: + raise RuntimeError("Unsupported referential constraint: %s" % + str(rows1)) + + +def define_table(conn, table): + "Output single table definition" + fields = get_fields(conn, table) + pks = primarykeys(conn, table) + print "db.define_table('%s'," % (table, ) + for field in fields: + fname = field['column_name'] + fdef = define_field(conn, table, field, pks) + if fname not in pks and is_unique(conn, table, field): + fdef['unique'] = "True" + if fdef['type'] == "'id'" and fname in pks: + pks.pop(pks.index(fname)) + print " Field('%s', %s)," % (fname, + ', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS + if k in fdef and fdef[k]])) + if pks: + print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks]) + print " migrate=migrate)" + print + + +def define_db(conn, db, host, port, user, passwd): + "Output database definition (model)" + dal = 'db = DAL("postgres://%s:%s@%s:%s/%s", pool_size=10)' + print dal % (user, passwd, host, port, db) + print + print "migrate = False" + print + for table in get_tables(conn): + define_table(conn, table) + + +if __name__ == "__main__": + if len(sys.argv) < 6: + print HELP + else: + # Parse arguments from command line: + db, host, port, user, passwd = sys.argv[1:6] + + # Make the database connection (change driver if required) + import psycopg2 + cnn = psycopg2.connect(database=db, host=host, port=port, + user=user, password=passwd, + ) + # Start model code generation: + define_db(cnn, db, host, port, user, passwd) From 99a6dbeac273cc5fe848e1f838d3010410aa5d56 Mon Sep 17 00:00:00 2001 From: Oscar Fonts Date: Tue, 19 Nov 2013 19:27:34 +0100 Subject: [PATCH 02/52] extract_oracle_models.py script, for legacy Oracle 11g databases --- scripts/extract_oracle_models.py | 262 +++++++++++++++++-------------- 1 file changed, 144 insertions(+), 118 deletions(-) diff --git a/scripts/extract_oracle_models.py b/scripts/extract_oracle_models.py index 79a1699f..7758d625 100644 --- a/scripts/extract_oracle_models.py +++ b/scripts/extract_oracle_models.py @@ -1,42 +1,41 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""Create web2py model (python code) to represent PostgreSQL tables. +"""Create web2py model (python code) to represent Oracle 11g tables. Features: -* Uses ANSI Standard INFORMATION_SCHEMA (might work with other RDBMS) +* Uses Oracle's metadata tables * Detects legacy "keyed" tables (not having an "id" PK) * Connects directly to running databases, no need to do a SQL dump * Handles notnull, unique and referential constraints * Detects most common datatypes and default values -* Support PostgreSQL columns comments (ie. for documentation) +* Documents alternative datatypes as comments Requeriments: -* Needs PostgreSQL pyscopg2 python connector (same as web2py) -* If used against other RDBMS, import and use proper connector (remove pg_ code) +* Needs Oracle cx_Oracle python connector (same as web2py) -Created by Mariano Reingart, based on a script to "generate schemas from dbs" -(mysql) by Alexandre Andrade +Created by Oscar Fonts, based on extract_pgsql_models by Mariano Reingart, +based in turn on a script to "generate schemas from dbs" (mysql) +by Alexandre Andrade """ -_author__ = "Mariano Reingart " +_author__ = "Oscar Fonts " HELP = """ -USAGE: extract_pgsql_models db host port user passwd +USAGE: extract_oracle_models db host port user passwd -Call with PostgreSQL database connection parameters, +Call with Oracle database connection parameters, web2py model will be printed on standard output. -EXAMPLE: python extract_pgsql_models.py mydb localhost 5432 reingart saraza +EXAMPLE: python extract_oracle_models.py ORCL localhost 1521 user password """ # Config options DEBUG = False # print debug messages to STDERR -SCHEMA = 'public' # change if not using default PostgreSQL schema # Constant for Field keyword parameter order (and filter): KWARGS = ('type', 'length', 'default', 'required', 'ondelete', @@ -52,7 +51,7 @@ def query(conn, sql, *args): ret = [] try: if DEBUG: - print >> sys.stderr, "QUERY: ", sql % args + print >> sys.stderr, "QUERY: ", sql , args cur.execute(sql, args) for row in cur: dic = {} @@ -63,16 +62,18 @@ def query(conn, sql, *args): print >> sys.stderr, "RET: ", dic ret.append(dic) return ret + except cx_Oracle.DatabaseError, exc: + error, = exc.args + print >> sys.stderr, "Oracle-Error-Message:", error.message finally: cur.close() -def get_tables(conn, schema=SCHEMA): +def get_tables(conn): "List table names in a given schema" - rows = query(conn, """SELECT table_name FROM information_schema.tables - WHERE table_schema = %s - ORDER BY table_name""", schema) - return [row['table_name'] for row in rows] + rows = query(conn, """SELECT TABLE_NAME FROM USER_TABLES + ORDER BY TABLE_NAME""") + return [row['TABLE_NAME'] for row in rows] def get_fields(conn, table): @@ -80,154 +81,180 @@ def get_fields(conn, table): if DEBUG: print >> sys.stderr, "Processing TABLE", table rows = query(conn, """ - SELECT column_name, data_type, - is_nullable, - character_maximum_length, - numeric_precision, numeric_precision_radix, numeric_scale, - column_default - FROM information_schema.columns - WHERE table_name=%s - ORDER BY ordinal_position""", table) + SELECT COLUMN_NAME, DATA_TYPE, + NULLABLE AS IS_NULLABLE, + CHAR_LENGTH AS CHARACTER_MAXIMUM_LENGTH, + DATA_PRECISION AS NUMERIC_PRECISION, + DATA_SCALE AS NUMERIC_SCALE, + DATA_DEFAULT AS COLUMN_DEFAULT + FROM USER_TAB_COLUMNS + WHERE TABLE_NAME=:t + """, table) + return rows def define_field(conn, table, field, pks): "Determine field type, default value, references, etc." f = {} - ref = references(conn, table, field['column_name']) + ref = references(conn, table, field['COLUMN_NAME']) + # Foreign Keys if ref: f.update(ref) - elif field['column_default'] and \ - field['column_default'].startswith("nextval") and \ - field['column_name'] in pks: - # postgresql sequence (SERIAL) and primary key! + # PK & Numeric & autoincrement => id + elif field['COLUMN_NAME'] in pks and \ + field['DATA_TYPE'] in ('INT', 'NUMBER') and \ + is_autoincrement(conn, table, field): f['type'] = "'id'" - elif field['data_type'].startswith('character'): - f['type'] = "'string'" - if field['character_maximum_length']: - f['length'] = field['character_maximum_length'] - elif field['data_type'] in ('text', ): - f['type'] = "'text'" - elif field['data_type'] in ('boolean', 'bit'): - f['type'] = "'boolean'" - elif field['data_type'] in ('integer', 'smallint', 'bigint'): - f['type'] = "'integer'" - elif field['data_type'] in ('double precision', 'real'): + # Other data types + elif field['DATA_TYPE'] in ('BINARY_DOUBLE'): f['type'] = "'double'" - elif field['data_type'] in ('timestamp', 'timestamp without time zone'): - f['type'] = "'datetime'" - elif field['data_type'] in ('date', ): - f['type'] = "'date'" - elif field['data_type'] in ('time', 'time without time zone'): - f['type'] = "'time'" - elif field['data_type'] in ('numeric', 'currency'): - f['type'] = "'decimal'" - f['precision'] = field['numeric_precision'] - f['scale'] = field['numeric_scale'] or 0 - elif field['data_type'] in ('bytea', ): + elif field['DATA_TYPE'] in ('CHAR','NCHAR'): + f['type'] = "'string'" + f['comment'] = "'Alternative types: boolean, time'" + elif field['DATA_TYPE'] in ('BLOB', 'CLOB'): f['type'] = "'blob'" - elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'): - f['type'] = "" # unsupported? + f['comment'] = "'Alternative types: text, json, list:*'" + elif field['DATA_TYPE'] in ('DATE'): + f['type'] = "'datetime'" + f['comment'] = "'Alternative types: date'" + elif field['DATA_TYPE'] in ('FLOAT'): + f['type'] = "'float'" + elif field['DATA_TYPE'] in ('INT'): + f['type'] = "'integer'" + elif field['DATA_TYPE'] in ('NUMBER'): + f['type'] = "'bigint'" + elif field['DATA_TYPE'] in ('NUMERIC'): + f['type'] = "'decimal'" + f['precision'] = field['NUMERIC_PRECISION'] + f['scale'] = field['NUMERIC_SCALE'] or 0 + elif field['DATA_TYPE'] in ('VARCHAR2','NVARCHAR2'): + f['type'] = "'string'" + if field['CHARACTER_MAXIMUM_LENGTH']: + f['length'] = field['CHARACTER_MAXIMUM_LENGTH'] + f['comment'] = "'Other possible types: password, upload'" else: - raise RuntimeError("Data Type not supported: %s " % str(field)) + f['type'] = "'blob'" + f['comment'] = "'WARNING: Oracle Data Type %s was not mapped." % \ + str(field['DATA_TYPE']) + " Using 'blob' as fallback.'" try: - if field['column_default']: - if field['column_default'] == "now()": + if field['COLUMN_DEFAULT']: + if field['COLUMN_DEFAULT'] == "sysdate": d = "request.now" - elif field['column_default'] == "true": + elif field['COLUMN_DEFAULT'].upper() == "T": d = "True" - elif field['column_default'] == "false": + elif field['COLUMN_DEFAULT'].upper() == "F": d = "False" else: - d = repr(eval(field['column_default'])) + d = repr(eval(field['COLUMN_DEFAULT'])) f['default'] = str(d) except (ValueError, SyntaxError): pass except Exception, e: raise RuntimeError( - "Default unsupported '%s'" % field['column_default']) + "Default unsupported '%s'" % field['COLUMN_DEFAULT']) - if not field['is_nullable']: + if not field['IS_NULLABLE']: f['notnull'] = "True" - comment = get_comment(conn, table, field) - if comment is not None: - f['comment'] = repr(comment) return f def is_unique(conn, table, field): - "Find unique columns (incomplete support)" + "Find unique columns" rows = query(conn, """ - SELECT information_schema.constraint_column_usage.column_name - FROM information_schema.table_constraints - NATURAL JOIN information_schema.constraint_column_usage - WHERE information_schema.table_constraints.table_name=%s - AND information_schema.constraint_column_usage.column_name=%s - AND information_schema.table_constraints.constraint_type='UNIQUE' - ;""", table, field['column_name']) + SELECT COLS.COLUMN_NAME + FROM USER_CONSTRAINTS CONS, ALL_CONS_COLUMNS COLS + WHERE CONS.OWNER = COLS.OWNER + AND CONS.CONSTRAINT_NAME = COLS.CONSTRAINT_NAME + AND CONS.CONSTRAINT_TYPE = 'U' + AND COLS.TABLE_NAME = :t + AND COLS.COLUMN_NAME = :c + """, table, field['COLUMN_NAME']) return rows and True or False -def get_comment(conn, table, field): - "Find the column comment (postgres specific)" +# Returns True when a "BEFORE EACH ROW INSERT" trigger is found and: +# a) it mentions the "NEXTVAL" keyword (used by sequences) +# b) it operates on the given table and column +# +# On some (inelegant) database designs, SEQUENCE.NEXTVAL is called directly +# from each "insert" statement, instead of using triggers. Such cases cannot +# be detected by inspecting Oracle's metadata tables, as sequences are not +# logically bound to any specific table or field. +def is_autoincrement(conn, table, field): + "Find auto increment fields (best effort)" rows = query(conn, """ - SELECT d.description AS comment - FROM pg_class c - JOIN pg_description d ON c.oid=d.objoid - JOIN pg_attribute a ON c.oid = a.attrelid - WHERE c.relname=%s AND a.attname=%s - AND a.attnum = d.objsubid - ;""", table, field['column_name']) - return rows and rows[0]['comment'] or None + SELECT TRIGGER_NAME + FROM USER_TRIGGERS, + (SELECT NAME, LISTAGG(TEXT, ' ') WITHIN GROUP (ORDER BY LINE) TEXT + FROM USER_SOURCE + WHERE TYPE = 'TRIGGER' + GROUP BY NAME + ) TRIGGER_DEFINITION + WHERE TRIGGER_NAME = NAME + AND TRIGGERING_EVENT = 'INSERT' + AND TRIGGER_TYPE = 'BEFORE EACH ROW' + AND TABLE_NAME = :t + AND UPPER(TEXT) LIKE UPPER('%.NEXTVAL%') + AND UPPER(TEXT) LIKE UPPER('%:NEW.' || :c || '%') + """, table, field['COLUMN_NAME']) + return rows and True or False def primarykeys(conn, table): "Find primary keys" rows = query(conn, """ - SELECT information_schema.constraint_column_usage.column_name - FROM information_schema.table_constraints - NATURAL JOIN information_schema.constraint_column_usage - WHERE information_schema.table_constraints.table_name=%s - AND information_schema.table_constraints.constraint_type='PRIMARY KEY' - ;""", table) - return [row['column_name'] for row in rows] + SELECT COLS.COLUMN_NAME + FROM USER_CONSTRAINTS CONS, ALL_CONS_COLUMNS COLS + WHERE COLS.TABLE_NAME = :t + AND CONS.CONSTRAINT_TYPE = 'P' + AND CONS.OWNER = COLS.OWNER + AND CONS.CONSTRAINT_NAME = COLS.CONSTRAINT_NAME + """, table) + + return [row['COLUMN_NAME'] for row in rows] def references(conn, table, field): "Find a FK (fails if multiple)" rows1 = query(conn, """ - SELECT table_name, column_name, constraint_name, - update_rule, delete_rule, ordinal_position - FROM information_schema.key_column_usage - NATURAL JOIN information_schema.referential_constraints - NATURAL JOIN information_schema.table_constraints - WHERE information_schema.key_column_usage.table_name=%s - AND information_schema.key_column_usage.column_name=%s - AND information_schema.table_constraints.constraint_type='FOREIGN KEY' - ;""", table, field) + SELECT COLS.CONSTRAINT_NAME, + CONS.DELETE_RULE, + COLS.POSITION AS ORDINAL_POSITION + FROM USER_CONSTRAINTS CONS, ALL_CONS_COLUMNS COLS + WHERE COLS.TABLE_NAME = :t + AND COLS.COLUMN_NAME = :c + AND CONS.CONSTRAINT_TYPE = 'R' + AND CONS.OWNER = COLS.OWNER + AND CONS.CONSTRAINT_NAME = COLS.CONSTRAINT_NAME + """, table, field) + if len(rows1) == 1: rows2 = query(conn, """ - SELECT table_name, column_name, * - FROM information_schema.constraint_column_usage - WHERE constraint_name=%s - """, rows1[0]['constraint_name']) + SELECT COLS.TABLE_NAME, COLS.COLUMN_NAME + FROM USER_CONSTRAINTS CONS, ALL_CONS_COLUMNS COLS + WHERE CONS.CONSTRAINT_NAME = :k + AND CONS.R_CONSTRAINT_NAME = COLS.CONSTRAINT_NAME + ORDER BY COLS.POSITION ASC + """, rows1[0]['CONSTRAINT_NAME']) + row = None if len(rows2) > 1: - row = rows2[int(rows1[0]['ordinal_position']) - 1] + row = rows2[int(rows1[0]['ORDINAL_POSITION']) - 1] keyed = True if len(rows2) == 1: row = rows2[0] keyed = False if row: if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!! - ref = {'type': "'reference %s.%s'" % (row['table_name'], - row['column_name'])} + ref = {'type': "'reference %s.%s'" % (row['TABLE_NAME'], + row['COLUMN_NAME'])} else: - ref = {'type': "'reference %s'" % (row['table_name'],)} - if rows1[0]['delete_rule'] != "NO ACTION": - ref['ondelete'] = repr(rows1[0]['delete_rule']) + ref = {'type': "'reference %s'" % (row['TABLE_NAME'],)} + if rows1[0]['DELETE_RULE'] != "NO ACTION": + ref['ondelete'] = repr(rows1[0]['DELETE_RULE']) return ref elif rows2: raise RuntimeError("Unsupported foreign key reference: %s" % @@ -244,15 +271,15 @@ def define_table(conn, table): pks = primarykeys(conn, table) print "db.define_table('%s'," % (table, ) for field in fields: - fname = field['column_name'] + fname = field['COLUMN_NAME'] fdef = define_field(conn, table, field, pks) if fname not in pks and is_unique(conn, table, field): fdef['unique'] = "True" if fdef['type'] == "'id'" and fname in pks: pks.pop(pks.index(fname)) print " Field('%s', %s)," % (fname, - ', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS - if k in fdef and fdef[k]])) + ', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS + if k in fdef and fdef[k]])) if pks: print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks]) print " migrate=migrate)" @@ -261,7 +288,7 @@ def define_table(conn, table): def define_db(conn, db, host, port, user, passwd): "Output database definition (model)" - dal = 'db = DAL("postgres://%s:%s@%s:%s/%s", pool_size=10)' + dal = 'db = DAL("oracle://%s/%s@%s:%s/%s", pool_size=10)' print dal % (user, passwd, host, port, db) print print "migrate = False" @@ -278,9 +305,8 @@ if __name__ == "__main__": db, host, port, user, passwd = sys.argv[1:6] # Make the database connection (change driver if required) - import psycopg2 - cnn = psycopg2.connect(database=db, host=host, port=port, - user=user, password=passwd, - ) + import cx_Oracle + dsn = cx_Oracle.makedsn(host, port, db) + cnn = cx_Oracle.connect(user, passwd, dsn) # Start model code generation: define_db(cnn, db, host, port, user, passwd) From fa1c719c965c671fc4d6004af4061af2e7fc24fd Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 1 Dec 2013 10:00:05 -0600 Subject: [PATCH 03/52] removed conflict from languages, thanks Niphlod --- VERSION | 2 +- gluon/languages.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 518850f7..7e5465a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.11.28.07.51.37 +Version 2.8.2-stable+timestamp.2013.12.01.09.59.08 diff --git a/gluon/languages.py b/gluon/languages.py index 30d31e40..ecf0c9d6 100644 --- a/gluon/languages.py +++ b/gluon/languages.py @@ -319,11 +319,7 @@ def write_dict(filename, contents): if not settings.global_settings.web2py_runtime_gae: logging.warning('Unable to write to file %s' % filename) return -<<<<<<< HEAD - fp.write('# coding: utf-8\n{\n') -======= fp.write('# -*- coding: utf-8 -*-\n{\n') ->>>>>>> ad87d196e58d2ea0e46485e5b1e7cb710558ae95 for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())): fp.write('%s: %s,\n' % (repr(Utf8(key)), repr(Utf8(contents[key])))) fp.write('}\n') From f66d0e1200ca1f71000ce9a294e20e6f136530e4 Mon Sep 17 00:00:00 2001 From: samuel bonilla Date: Sun, 1 Dec 2013 11:37:29 -0500 Subject: [PATCH 04/52] Update buttons.py 'dict' object has no attribute 'is_mobile'" --- applications/admin/models/buttons.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/admin/models/buttons.py b/applications/admin/models/buttons.py index 41c8d7d2..67d9ec92 100644 --- a/applications/admin/models/buttons.py +++ b/applications/admin/models/buttons.py @@ -24,7 +24,7 @@ def button_enable(href, app): return A(label, _class='button btn', _id=id, callback=href, target=id) def sp_button(href, label): - if request.user_agent().is_mobile: + if request.user_agent().get('is_mobile'): ret = A_button(SPAN(label), _href=href) else: ret = A(SPAN(label), _class='button special btn btn-inverse', _href=href) @@ -37,4 +37,4 @@ def searchbox(elementid): return SPAN(LABEL(IMG(_id="search_start", _src=URL('static', 'images/search.png'), _alt=T('filter')), _class='icon', _for=elementid), ' ', INPUT(_id=elementid, _type='text', _size=12, _class="input-medium"), - _class="searchbox") \ No newline at end of file + _class="searchbox") From 593540e4674bcbba339ba51eaf1c4ff78ce2fa84 Mon Sep 17 00:00:00 2001 From: samuel bonilla Date: Sun, 1 Dec 2013 11:39:37 -0500 Subject: [PATCH 05/52] Update access.py 'dict' object has no attribute 'is_mobile'" --- applications/admin/models/access.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/admin/models/access.py b/applications/admin/models/access.py index fee8d464..502b90f8 100644 --- a/applications/admin/models/access.py +++ b/applications/admin/models/access.py @@ -144,7 +144,7 @@ if session.is_mobile == 'true': elif session.is_mobile == 'false': is_mobile = False else: - is_mobile = request.user_agent().is_mobile + is_mobile = request.user_agent().get('is_mobile') if DEMO_MODE: session.authorized = True From fcd9e0d5c684ae7de2ee6f3925630cc8b67a461d Mon Sep 17 00:00:00 2001 From: Anssi Hannula Date: Sun, 1 Dec 2013 23:36:25 +0200 Subject: [PATCH 06/52] Fix references with GAE in NDB mode For reference types the IntegerProperty class (no matter if NDB or not) constructor is passed the referenced table name as the first parameter. For non-NDB, the first parameter is 'verbose_name'. For NDB, the first parameter is 'name'. In NDB mode this causes the property to have a wrong identity and therefore references fail to work properly. The verbose_name set in non-NDB mode is relatively harmless in comparison, but seems to be wrong as well as it does not really make sense. Do not pass referenced table name to the IntegerProperty constructor in either case. --- gluon/dal.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gluon/dal.py b/gluon/dal.py index 8dc63f7f..58988d97 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -4839,12 +4839,10 @@ class GoogleDatastoreAdapter(NoSQLAdapter): elif field_type.startswith('reference'): if field.notnull: attr = dict(required=True) - referenced = field_type[10:].strip() - ftype = self.types[field_type[:9]](referenced, **attr) + ftype = self.types[field_type[:9]](**attr) elif field_type.startswith('list:reference'): if field.notnull: attr['required'] = True - referenced = field_type[15:].strip() ftype = self.types[field_type[:14]](**attr) elif field_type.startswith('list:'): ftype = self.types[field_type](**attr) From 55cbc1830a33af6d1eeefba677426f63362d36db Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 2 Dec 2013 02:07:17 -0600 Subject: [PATCH 07/52] fixed issue 1795:MongoDB: DAL instantiation dict definition error, thanks Alan --- VERSION | 2 +- gluon/dal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 7e5465a3..88d0464d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.01.09.59.08 +Version 2.8.2-stable+timestamp.2013.12.02.02.05.46 diff --git a/gluon/dal.py b/gluon/dal.py index 58988d97..7b173482 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -7808,11 +7808,11 @@ class DAL(object): def import_table_definitions(self, path, migrate=False, fake_migrate=False, tables=None): - pattern = pjoin(path,self._uri_hash+'_*.table') if tables: for table in tables: self.define_table(**table) else: + pattern = pjoin(path,self._uri_hash+'_*.table') for filename in glob.glob(pattern): tfile = self._adapter.file_open(filename, 'r') try: From 7703d959ef743a85cb48d497b5fc7ab96ec8c8e1 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 2 Dec 2013 10:28:13 -0600 Subject: [PATCH 08/52] fixed request.user_agent().get('is_mobile', False), thanks Jonathan --- VERSION | 2 +- applications/admin/models/access.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 88d0464d..cd5f07a1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.02.02.05.46 +Version 2.8.2-stable+timestamp.2013.12.02.10.26.58 diff --git a/applications/admin/models/access.py b/applications/admin/models/access.py index 502b90f8..823ec438 100644 --- a/applications/admin/models/access.py +++ b/applications/admin/models/access.py @@ -144,7 +144,7 @@ if session.is_mobile == 'true': elif session.is_mobile == 'false': is_mobile = False else: - is_mobile = request.user_agent().get('is_mobile') + is_mobile = request.user_agent().get('is_mobile',False) if DEMO_MODE: session.authorized = True From b7e7c8caf2274994ae91c9114d709fb05d426980 Mon Sep 17 00:00:00 2001 From: Anssi Hannula Date: Mon, 2 Dec 2013 23:15:06 +0200 Subject: [PATCH 09/52] Fix list types in GAE NDB after entity write As per http://code.google.com/p/appengine-ndb-experiment/issues/detail?id=208 ("List items from repeated StringProperty get mutated to _BaseValue on put") it seems to be expected behavior that lists in NDB models get mutated to _BaseValue lists after .put() is called. In web2py this corresponds to: ------------ db.define_table('listintegertest', Field('dummy', 'boolean'), Field('integerlist', 'list:integer'), Field('stringlist', 'list:string'), ) iidee = db.listintegertest.insert( integerlist=[1,2,3,4], stringlist=["1","2","3","4"] ) row = db.listintegertest(iidee) db.listintegertest[row.id] = dict(dummy=True) print "type is %s" % (type(row.integerlist[0])) print "type is %s" % (type(row.stringlist[0])) ------------ The output of the above is currently: type is type is while this is expected: type is type is The workaround is to copy the list from the NDB model instead of relying on it not being changed afterwards. list:reference is not affected since parse_list_references() already recreates the list in NoSQLAdapter case. --- gluon/dal.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gluon/dal.py b/gluon/dal.py index 58988d97..9a989d0c 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -5166,6 +5166,12 @@ class GoogleDatastoreAdapter(NoSQLAdapter): processor = attributes.get('processor',self.parse) return processor(rows,fields,colnames,False) + def parse_list_integers(self, value, field_type): + return value[:] if self.use_ndb else value + + def parse_list_strings(self, value, field_type): + return value[:] if self.use_ndb else value + def count(self,query,distinct=None,limit=None): if distinct: raise RuntimeError("COUNT DISTINCT not supported") From 645c3b10ddcee94629a53345e3c9c67a26d634fb Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 3 Dec 2013 00:41:51 -0600 Subject: [PATCH 10/52] fixed issue 1783 again, thanks Dean Kleissas --- VERSION | 2 +- gluon/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index cd5f07a1..3771e4c1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.02.10.26.58 +Version 2.8.2-stable+timestamp.2013.12.03.00.40.46 diff --git a/gluon/tools.py b/gluon/tools.py index 874ad220..afb258f8 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -2915,7 +2915,7 @@ class Auth(object): formname='reset_password', dbio=False, onvalidation=onvalidation, hideerror=self.settings.hideerror): - user = table_user(**{userfield:form.vars.email}) + user = table_user(**{userfield:form.vars.get(userlfield)}) if not user: session.flash = self.messages['invalid_%s' % userfield] redirect(self.url(args=request.args), From 14bcad629eb2b73266b11b4c9dbf63499c73d38b Mon Sep 17 00:00:00 2001 From: Michele Comitini Date: Fri, 8 Nov 2013 17:11:36 +0100 Subject: [PATCH 11/52] quoting of tablenames and field names fix for wrong sequence name quoting fix for wrong sequence name quoting 2 reverted test to quoted rname rname passed asis. freedom in table names and field names. Case sensitivity in field and table names removed test of allowed char sequence in table and field names, there is no limit anymore fixed some troubles with postgresql drop(). Better Row.__getitem__(): should not be much slower. Added a ignore_field_case parameter to allow Field('F')!=Field('f') on databases supporting case sesitivity. different mysql drivers handled in quoting test. improved handling of foreing keys fix with quoted names in SQLTABLE. fix for ORDER BY handle references on non "id" fields check reference expression against table list first --- gluon/dal.py | 426 ++++++++++++++++++++++++++-------------- gluon/sqlhtml.py | 6 +- gluon/tests/test_dal.py | 113 ++++++++++- 3 files changed, 389 insertions(+), 156 deletions(-) diff --git a/gluon/dal.py b/gluon/dal.py index d3c02239..6005b545 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -252,7 +252,8 @@ THREAD_LOCAL = threading.local() REGEX_TYPE = re.compile('^([\w\_\:]+)') REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*') REGEX_W = re.compile('^\w+$') -REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$') +REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.([^.]+)$') +REGEX_NO_GREEDY_ENTITY_NAME = r'(.+?)' REGEX_UPLOAD_PATTERN = re.compile('(?P[\w\-]+)\.(?P[\w\-]+)\.(?P[\w\-]+)(\.(?P\w+))?\.\w+$') REGEX_CLEANUP_FN = re.compile('[\'"\s;]+') REGEX_UNPACK = re.compile('(?1: - # then it has to be a table level FK - if rtablename not in TFK: - TFK[rtablename] = {} - TFK[rtablename][rfieldname] = field_name - else: - ftype = ftype + \ - types['reference FK'] % dict( - constraint_name = constraint_name, # should be quoted - foreign_key = '%s (%s)' % (rtablename, - rfieldname), - table_name = tablename, - field_name = field._rname or field.name, - on_delete_action=field.ondelete) + except Exception, e: + LOGGER.debug('Error: %s' %e) + raise KeyError('Cannot resolve reference %s in %s definition' % (referenced, table._tablename)) + + # must be PK reference or unique + if getattr(rtable, '_primarykey', None) and rfieldname in rtable._primarykey or \ + rfield.unique: + ftype = types[rfield.type[:9]] % \ + dict(length=rfield.length) + # multicolumn primary key reference? + if not rfield.unique and len(rtable._primarykey)>1: + # then it has to be a table level FK + if rtablename not in TFK: + TFK[rtablename] = {} + TFK[rtablename][rfieldname] = field_name else: - # make a guess here for circular references - if referenced in db: - id_fieldname = db[referenced]._id.name - elif referenced == tablename: - id_fieldname = table._id.name - else: #make a guess - id_fieldname = 'id' - #gotcha: the referenced table must be defined before - #the referencing one to be able to create the table - #Also if it's not recommended, we can still support - #references to tablenames without rname to make - #migrations and model relationship work also if tables - #are not defined in order - real_referenced = ( - (db[referenced]._rname or db[referenced]) - if referenced == tablename or referenced in db - else referenced) - - ftype = types[field_type[:9]] % dict( - index_name = field_name+'__idx', - field_name = field._rname or field.name, - constraint_name = constraint_name, - foreign_key = '%s (%s)' % (real_referenced, - id_fieldname), - on_delete_action=field.ondelete) + ftype = ftype + \ + types['reference FK'] % dict( + constraint_name = constraint_name, # should be quoted + foreign_key = rtable.sqlsafe + ' (' + rfield.sqlsafe_name + ')', + table_name = table.sqlsafe, + field_name = field.sqlsafe_name, + on_delete_action=field.ondelete) + else: + # make a guess here for circular references + if referenced in db: + id_fieldname = db[referenced]._id.sqlsafe_name + elif referenced == tablename: + id_fieldname = table._id.sqlsafe_name + else: #make a guess + id_fieldname = self.QUOTE_TEMPLATE % 'id' + #gotcha: the referenced table must be defined before + #the referencing one to be able to create the table + #Also if it's not recommended, we can still support + #references to tablenames without rname to make + #migrations and model relationship work also if tables + #are not defined in order + if referenced == tablename: + real_referenced = db[referenced].sqlsafe + else: + real_referenced = (referenced in db + and db[referenced].sqlsafe + or referenced) + rfield = db[referenced]._id + ftype = types[field_type[:9]] % dict( + index_name = self.QUOTE_TEMPLATE % (field_name+'__idx'), + field_name = field.sqlsafe_name, + constraint_name = self.QUOTE_TEMPLATE % constraint_name, + foreign_key = '%s (%s)' % (real_referenced, rfield.sqlsafe_name), + on_delete_action=field.ondelete) elif field_type.startswith('list:reference'): ftype = types[field_type[:14]] elif field_type.startswith('decimal'): @@ -995,39 +1025,37 @@ class BaseAdapter(ConnectionPool): # geometry fields are added after the table has been created, not now if not (self.dbengine == 'postgres' and \ field_type.startswith('geom')): - #fetch the rname if it's there - field_rname = "%s" % (field._rname or field_name) - fields.append('%s %s' % (field_rname, ftype)) + fields.append('%s %s' % (field.sqlsafe_name, ftype)) other = ';' # backend-specific extensions to fields if self.dbengine == 'mysql': if not hasattr(table, "_primarykey"): - fields.append('PRIMARY KEY(%s)' % (table._id.name or table._id._rname)) + fields.append('PRIMARY KEY (%s)' % (self.QUOTE_TEMPLATE % table._id.name)) other = ' ENGINE=InnoDB CHARACTER SET utf8;' fields = ',\n '.join(fields) for rtablename in TFK: rfields = TFK[rtablename] - pkeys = db[rtablename]._primarykey - fkeys = [ rfields[k] for k in pkeys ] + pkeys = [self.QUOTE_TEMPLATE % pk for pk in db[rtablename]._primarykey] + fkeys = [self.QUOTE_TEMPLATE % rfields[k].name for k in pkeys ] fields = fields + ',\n ' + \ types['reference TFK'] % dict( - table_name = tablename, + table_name = table.sqlsafe, field_name=', '.join(fkeys), - foreign_table = rtablename, + foreign_table = table.sqlsafe, foreign_key = ', '.join(pkeys), on_delete_action = field.ondelete) - #if there's a _rname, let's use that instead - table_rname = table._rname or tablename + + table_rname = table.sqlsafe if getattr(table,'_primarykey',None): query = "CREATE TABLE %s(\n %s,\n %s) %s" % \ - (table_rname, fields, - self.PRIMARY_KEY(', '.join(table._primarykey)),other) + (table.sqlsafe, fields, + self.PRIMARY_KEY(', '.join([self.QUOTE_TEMPLATE % pk for pk in table._primarykey])),other) else: query = "CREATE TABLE %s(\n %s\n)%s" % \ - (table_rname, fields, other) + (table.sqlsafe, fields, other) if self.uri.startswith('sqlite:///') \ or self.uri.startswith('spatialite:///'): @@ -1105,6 +1133,7 @@ class BaseAdapter(ConnectionPool): k,v=item if not isinstance(v,dict): v=dict(type='unknown',sql=v) + if self.ignore_field_case is not True: return k, v return k.lower(),v # make sure all field names are lower case to avoid # migrations because of case cahnge @@ -1132,7 +1161,7 @@ class BaseAdapter(ConnectionPool): query = [ sql_fields[key]['sql'] ] else: query = ['ALTER TABLE %s ADD %s %s;' % \ - (tablename, key, + (table.sqlsafe, key, sql_fields_aux[key]['sql'].replace(', ', new_add))] metadata_change = True elif self.dbengine in ('sqlite', 'spatialite'): @@ -1150,10 +1179,11 @@ class BaseAdapter(ConnectionPool): "'%(table)s', '%(field)s');" % dict(schema=schema, table=tablename, field=key,) ] elif self.dbengine in ('firebird',): - query = ['ALTER TABLE %s DROP %s;' % (tablename, key)] + query = ['ALTER TABLE %s DROP %s;' % + (self.QUOTE_TEMPLATE % tablename, self.QUOTE_TEMPLATE % key)] else: query = ['ALTER TABLE %s DROP COLUMN %s;' % - (tablename, key)] + (self.QUOTE_TEMPLATE % tablename, self.QUOTE_TEMPLATE % key)] metadata_change = True elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \ and not (key in table.fields and @@ -1169,12 +1199,15 @@ class BaseAdapter(ConnectionPool): else: drop_expr = 'ALTER TABLE %s DROP COLUMN %s;' key_tmp = key + '__tmp' - query = ['ALTER TABLE %s ADD %s %s;' % (t, key_tmp, tt), - 'UPDATE %s SET %s=%s;' % (t, key_tmp, key), - drop_expr % (t, key), - 'ALTER TABLE %s ADD %s %s;' % (t, key, tt), - 'UPDATE %s SET %s=%s;' % (t, key, key_tmp), - drop_expr % (t, key_tmp)] + query = ['ALTER TABLE %s ADD %s %s;' % (self.QUOTE_TEMPLATE % t, self.QUOTE_TEMPLATE % key_tmp, tt), + 'UPDATE %s SET %s=%s;' % + (self.QUOTE_TEMPLATE % t, self.QUOTE_TEMPLATE % key_tmp, self.QUOTE_TEMPLATE % key), + drop_expr % (self.QUOTE_TEMPLATE % t, self.QUOTE_TEMPLATE % key), + 'ALTER TABLE %s ADD %s %s;' % + (self.QUOTE_TEMPLATE % t, self.QUOTE_TEMPLATE % key, tt), + 'UPDATE %s SET %s=%s;' % + (self.QUOTE_TEMPLATE % t, self.QUOTE_TEMPLATE % key, self.QUOTE_TEMPLATE % key_tmp), + drop_expr % (self.QUOTE_TEMPLATE % t, self.QUOTE_TEMPLATE % key_tmp)] metadata_change = True elif sql_fields[key]['type'] != sql_fields_old[key]['type']: sql_fields_current[key] = sql_fields[key] @@ -1269,8 +1302,7 @@ class BaseAdapter(ConnectionPool): return 'PRIMARY KEY(%s)' % key def _drop(self, table, mode): - table_rname = table._rname or table - return ['DROP TABLE %s;' % table_rname] + return ['DROP TABLE %s;' % table.sqlsafe] def drop(self, table, mode=''): db = table._db @@ -1288,17 +1320,16 @@ class BaseAdapter(ConnectionPool): self.log('success!\n', table) def _insert(self, table, fields): - table_rname = table._rname or table + table_rname = table.sqlsafe if fields: - keys = ','.join(f._rname or f.name for f, v in fields) + keys = ','.join(f.sqlsafe_name for f, v in fields) values = ','.join(self.expand(v, f.type) for f, v in fields) return 'INSERT INTO %s(%s) VALUES (%s);' % (table_rname, keys, values) else: return self._insert_empty(table) def _insert_empty(self, table): - table_rname = table._rname or table - return 'INSERT INTO %s DEFAULT VALUES;' % table_rname + return 'INSERT INTO %s DEFAULT VALUES;' % (table.sqlsafe) def insert(self, table, fields): query = self._insert(table,fields) @@ -1451,13 +1482,13 @@ class BaseAdapter(ConnectionPool): self.expand(second, first.type)) def AS(self, first, second): - return '%s AS %s' % (self.expand(first), second) + return '%s AS %s' % (self.expand(first), second) def ON(self, first, second): - table_rname = first._ot and first or first._rname or first._tablename + table_rname = self.table_alias(first) if use_common_filters(second): second = self.common_filter(second,[first._tablename]) - return '%s ON %s' % (self.expand(table_rname), self.expand(second)) + return ('%s ON %s') % (self.expand(table_rname), self.expand(second)) def INVERT(self, first): return '%s DESC' % self.expand(first) @@ -1472,13 +1503,10 @@ class BaseAdapter(ConnectionPool): if isinstance(expression, Field): et = expression.table if not colnames: - table_rname = et._ot and et._tablename or et._rname or et._tablename + table_rname = et._ot and self.QUOTE_TEMPLATE % et._tablename or et._rname or self.QUOTE_TEMPLATE % et._tablename + out = '%s.%s' % (table_rname, expression._rname or (self.QUOTE_TEMPLATE % (expression.name))) else: - table_rname = et._tablename - if not colnames: - out = '%s.%s' % (table_rname, expression._rname or expression.name) - else: - out = '%s.%s' % (table_rname, expression.name) + out = '%s.%s' % (self.QUOTE_TEMPLATE % et._tablename, self.QUOTE_TEMPLATE % expression.name) if field_type == 'string' and not expression.type in ( 'string','text','json','password'): out = self.CAST(out, self.types['text']) @@ -1509,10 +1537,11 @@ class BaseAdapter(ConnectionPool): else: return str(expression) - def table_alias(self,name): - if not isinstance(name, Table): - name = self.db[name]._rname or self.db[name] - return str(name) + def table_alias(self, tbl): + if not isinstance(tbl, Table): + tbl = self.db[tbl] + return tbl.sqlsafe_alias + def alias(self, table, alias): """ @@ -1520,7 +1549,7 @@ class BaseAdapter(ConnectionPool): with alias name. """ other = copy.copy(table) - other['_ot'] = other._ot or other._rname or other._tablename + other['_ot'] = other._ot or other.sqlsafe other['ALL'] = SQLALL(other) other['_tablename'] = alias for fieldname in other.fields: @@ -1532,8 +1561,7 @@ class BaseAdapter(ConnectionPool): return other def _truncate(self, table, mode=''): - tablename = table._rname or table._tablename - return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')] + return ['TRUNCATE TABLE %s %s;' % (table.sqlsafe, mode or '')] def truncate(self, table, mode= ' '): # Prepare functions "write_to_logfile" and "close_logfile" @@ -1553,10 +1581,10 @@ class BaseAdapter(ConnectionPool): sql_w = ' WHERE ' + self.expand(query) else: sql_w = '' - sql_v = ','.join(['%s=%s' % (field._rname or field.name, + sql_v = ','.join(['%s=%s' % (field.sqlsafe_name, self.expand(value, field.type)) \ for (field, value) in fields]) - tablename = "%s" % (self.db[tablename]._rname or tablename) + tablename = self.db[tablename].sqlsafe return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w) def update(self, tablename, query, fields): @@ -1581,7 +1609,7 @@ class BaseAdapter(ConnectionPool): sql_w = ' WHERE ' + self.expand(query) else: sql_w = '' - tablename = '%s' % (self.db[tablename]._rname or tablename) + tablename = self.db[tablename].sqlsafe return 'DELETE FROM %s%s;' % (tablename, sql_w) def delete(self, tablename, query): @@ -1623,8 +1651,9 @@ class BaseAdapter(ConnectionPool): if isinstance(item,SQLALL): new_fields += item._table elif isinstance(item,str): - if REGEX_TABLE_DOT_FIELD.match(item): - tablename,fieldname = item.split('.') + m = self.REGEX_TABLE_DOT_FIELD.match(item) + if m: + tablename,fieldname = m.groups() append(db[tablename][fieldname]) else: append(Expression(db,lambda item=item:item)) @@ -1645,10 +1674,11 @@ class BaseAdapter(ConnectionPool): tablenames = tables(query) tablenames_for_common_filters = tablenames for field in fields: - if isinstance(field, basestring) \ - and REGEX_TABLE_DOT_FIELD.match(field): - tn,fn = field.split('.') - field = self.db[tn][fn] + if isinstance(field, basestring): + m = self.REGEX_TABLE_DOT_FIELD.match(field) + if m: + tn,fn = m.groups() + field = self.db[tn][fn] for tablename in tables(field): if not tablename in tablenames: tablenames.append(tablename) @@ -1732,7 +1762,7 @@ class BaseAdapter(ConnectionPool): tables_to_merge.keys()]) if joint: sql_t += ' %s %s' % (command, - ','.join([self.table_alias(t) for t in joint])) + ','.join([t for t in joint])) for t in joinon: sql_t += ' %s %s' % (command, t) elif inner_join and left: @@ -1747,7 +1777,7 @@ class BaseAdapter(ConnectionPool): sql_t += ' %s %s' % (icommand, t) if joint: sql_t += ' %s %s' % (command, - ','.join([self.table_alias(t) for t in joint])) + ','.join([t for t in joint])) for t in joinon: sql_t += ' %s %s' % (command, t) else: @@ -1767,9 +1797,9 @@ class BaseAdapter(ConnectionPool): sql_o += ' ORDER BY %s' % self.expand(orderby) if (limitby and not groupby and tablenames and orderby_on_limitby and not orderby): sql_o += ' ORDER BY %s' % ', '.join( - ['%s.%s'%(t,x) for t in tablenames for x in ( + [self.db[t][x].sqlsafe for t in tablenames for x in ( hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey - or [self.db[t]._id._rname or self.db[t]._id.name] + or ['_id'] ) ] ) @@ -1897,8 +1927,10 @@ class BaseAdapter(ConnectionPool): def create_sequence_and_triggers(self, query, table, **args): self.execute(query) + def log_execute(self, *a, **b): + if not self.connection: raise ValueError(a[0]) if not self.connection: return None command = a[0] if hasattr(self,'filter_sql_command'): @@ -1955,8 +1987,17 @@ class BaseAdapter(ConnectionPool): if field_is_type('decimal'): return str(obj) elif field_is_type('reference'): # reference - if fieldtype.find('.')>0: - return repr(obj) + # check for tablename first + referenced = fieldtype[9:].strip() + if referenced in self.db.tables: + return str(long(obj)) + p = referenced.partition('.') + if p[2] != '': + try: + ftype = self.db[p[0]][p[2]].type + return self.represent(obj, ftype) + except (ValueError, KeyError): + return repr(obj) elif isinstance(obj, (Row, Reference)): return str(obj['id']) return str(long(obj)) @@ -2162,14 +2203,15 @@ class BaseAdapter(ConnectionPool): new_rows = [] tmps = [] for colname in colnames: - if not REGEX_TABLE_DOT_FIELD.match(colname): + col_m = self.REGEX_TABLE_DOT_FIELD.match(colname) + if not col_m: tmps.append(None) else: - (tablename, _the_sep_, fieldname) = colname.partition('.') + tablename, fieldname = col_m.groups() table = db[tablename] field = table[fieldname] ft = field.type - tmps.append((tablename,fieldname,table,field,ft)) + tmps.append((tablename, fieldname, table, field, ft)) for (i,row) in enumerate(rows): new_row = Row() for (j,colname) in enumerate(colnames): @@ -2177,9 +2219,8 @@ class BaseAdapter(ConnectionPool): tmp = tmps[j] if tmp: (tablename,fieldname,table,field,ft) = tmp - if tablename in new_row: - colset = new_row[tablename] - else: + colset = new_row.get(tablename, None) + if colset is None: colset = new_row[tablename] = Row() if tablename not in virtualtables: virtualtables.append(tablename) @@ -2287,6 +2328,14 @@ class BaseAdapter(ConnectionPool): return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \ (self.expand(query),represent(t),represent(f))) + def sqlsafe_table(self, tablename, ot=None): + if ot is not None: + return ('%s AS ' + self.QUOTE_TEMPLATE) % (ot, tablename) + return self.QUOTE_TEMPLATE % tablename + + def sqlsafe_field(self, fieldname): + return self.QUOTE_TEMPLATE % fieldname + ################################################################################### # List of all the available adapters; they all extend BaseAdapter. ################################################################################### @@ -2564,6 +2613,7 @@ class MySQLAdapter(BaseAdapter): 'list:reference': 'LONGTEXT', 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL', 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', + 'reference FK': ', CONSTRAINT `FK_%(constraint_name)s` FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', } QUOTE_TEMPLATE = "`%s`" @@ -2590,12 +2640,12 @@ class MySQLAdapter(BaseAdapter): def _drop(self,table,mode): # breaks db integrity but without this mysql does not drop table - table_rname = table._rname or table + table_rname = table.sqlsafe return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table_rname, 'SET FOREIGN_KEY_CHECKS=1;'] def _insert_empty(self, table): - return 'INSERT INTO %s VALUES (DEFAULT);' % table + return 'INSERT INTO %s VALUES (DEFAULT);' % (table.sqlsafe) def distributed_transaction_begin(self,key): self.execute('XA START;') @@ -2668,6 +2718,8 @@ class MySQLAdapter(BaseAdapter): class PostgreSQLAdapter(BaseAdapter): drivers = ('psycopg2','pg8000') + QUOTE_TEMPLATE = '"%s"' + support_distributed_transaction = True types = { 'boolean': 'CHAR(1)', @@ -2694,12 +2746,11 @@ class PostgreSQLAdapter(BaseAdapter): 'geography': 'GEOGRAPHY', 'big-id': 'BIGSERIAL PRIMARY KEY', 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', - 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', - 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', + 'reference FK': ', CONSTRAINT "FK_%(constraint_name)s" FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s', + 'reference TFK': ' CONSTRAINT "FK_%(foreign_table)s_PK" FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', } - QUOTE_TEMPLATE = '"%s"' def varquote(self,name): return varquote_aux(name,'"%s"') @@ -2713,7 +2764,7 @@ class PostgreSQLAdapter(BaseAdapter): return "'%s'" % str(obj).replace("'","''") def sequence_name(self,table): - return '%s_id_seq' % table + return self.QUOTE_TEMPLATE % (table + '_id_seq') def RANDOM(self): return 'RANDOM()' @@ -2802,8 +2853,8 @@ class PostgreSQLAdapter(BaseAdapter): self.execute("SET standard_conforming_strings=on;") self.try_json() - def lastrowid(self,table): - self.execute("""select currval('"%s"')""" % table._sequence_name) + def lastrowid(self,table = None): + self.execute("select lastval()") return int(self.cursor.fetchone()[0]) def try_json(self): @@ -2942,6 +2993,11 @@ class PostgreSQLAdapter(BaseAdapter): return value return BaseAdapter.represent(self, obj, fieldtype) + def _drop(self, table, mode='restrict'): + if mode not in ['restrict', 'cascade', '']: + raise ValueError('Invalid mode: %s' % mode) + return ['DROP TABLE ' + table.sqlsafe + ' ' + str(mode) + ';'] + class NewPostgreSQLAdapter(PostgreSQLAdapter): drivers = ('psycopg2','pg8000') @@ -3074,8 +3130,6 @@ class OracleAdapter(BaseAdapter): 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', } - def sequence_name(self,tablename): - return '%s_sequence' % tablename def trigger_name(self,tablename): return '%s_trigger' % tablename @@ -3091,7 +3145,7 @@ class OracleAdapter(BaseAdapter): def _drop(self,table,mode): sequence_name = table._sequence_name - return ['DROP TABLE %s %s;' % (table, mode), 'DROP SEQUENCE %s;' % sequence_name] + return ['DROP TABLE %s %s;' % (table.sqlsafe, mode), 'DROP SEQUENCE %s;' % sequence_name] def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby): if limitby: @@ -3218,6 +3272,13 @@ class OracleAdapter(BaseAdapter): else: return self.cursor.fetchall() + def sqlsafe_table(self, tablename, ot=None): + if ot is not None: + return (self.QUOTE_TEMPLATE + ' ' \ + + self.QUOTE_TEMPLATE) % (ot, tablename) + return self.QUOTE_TEMPLATE % tablename + + class MSSQLAdapter(BaseAdapter): drivers = ('pyodbc',) T_SEP = 'T' @@ -3685,7 +3746,7 @@ class FireBirdAdapter(BaseAdapter): } def sequence_name(self,tablename): - return 'genid_%s' % tablename + return ('genid_' + self.QUOTE_TEMPLATE) % tablename def trigger_name(self,tablename): return 'trg_id_%s' % tablename @@ -3714,7 +3775,7 @@ class FireBirdAdapter(BaseAdapter): def _drop(self,table,mode): sequence_name = table._sequence_name - return ['DROP TABLE %s %s;' % (table, mode), 'DROP GENERATOR %s;' % sequence_name] + return ['DROP TABLE %s %s;' % (table.sqlsafe, mode), 'DROP GENERATOR %s;' % sequence_name] def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby): if limitby: @@ -4283,7 +4344,7 @@ class SAPDBAdapter(BaseAdapter): } def sequence_name(self,table): - return '%s_id_Seq' % table + return (self.QUOTE_TEMPLATE + '_id_Seq') % table def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby): if limitby: @@ -5446,8 +5507,8 @@ def cleanup(text): """ validates that the given text is clean: only contains [0-9a-zA-Z_] """ - if not REGEX_ALPHANUMERIC.match(text): - raise SyntaxError('invalid table or field name: %s' % text) + #if not REGEX_ALPHANUMERIC.match(text): + # raise SyntaxError('invalid table or field name: %s' % text) return text class MongoDBAdapter(NoSQLAdapter): @@ -7249,12 +7310,32 @@ class Row(object): __init__ = lambda self,*args,**kwargs: self.__dict__.update(*args,**kwargs) def __getitem__(self, k): + if isinstance(k, Table): + try: + return ogetattr(self, k._tablename) + except (KeyError,AttributeError,TypeError): + pass + elif isinstance(k, Field): + try: + return ogetattr(self, k.name) + except (KeyError,AttributeError,TypeError): + pass + try: + return ogetattr(ogetattr(self, k.tablename), k.name) + except (KeyError,AttributeError,TypeError): + pass + key=str(k) - _extra = self.__dict__.get('_extra', None) + _extra = ogetattr(self, '__dict__').get('_extra', None) if _extra is not None: v = _extra.get(key, DEFAULT) if v != DEFAULT: return v + try: + return ogetattr(self, key) + except (KeyError,AttributeError,TypeError): + pass + m = REGEX_TABLE_DOT_FIELD.match(key) if m: try: @@ -7656,7 +7737,7 @@ class DAL(object): adapter_args=None, attempts=5, auto_import=False, bigint_id=False, debug=False, lazy_tables=False, db_uid=None, do_connect=True, - after_connection=None, tables=None): + after_connection=None, tables=None, ignore_field_case=True): """ Creates a new Database Abstraction Layer instance. @@ -7741,6 +7822,7 @@ class DAL(object): self._decode_credentials = decode_credentials self._attempts = attempts self._do_connect = do_connect + self._ignore_field_case = ignore_field_case if not str(attempts).isdigit() or attempts < 0: attempts = 5 @@ -7772,6 +7854,7 @@ class DAL(object): # copy so multiple DAL() possible self._adapter.types = copy.copy(types) self._adapter.build_parsemap() + self._adapter.ignore_field_case = ignore_field_case if bigint_id: if 'big-id' in types and 'reference' in types: self._adapter.types['id'] = types['big-id'] @@ -8569,13 +8652,8 @@ class Table(object): self._tablename = tablename self._ot = None # args.get('rname') self._rname = args.get('rname') - if not self._rname: - self._sequence_name = args.get('sequence_name') or \ - db and db._adapter.sequence_name(tablename) - else: - tb = self._rname[1:-1] - self._sequence_name = args.get('sequence_name') or \ - db and db._adapter.sequence_name(tb) + self._sequence_name = args.get('sequence_name') or \ + db and db._adapter.sequence_name(self._rname or tablename) self._trigger_name = args.get('trigger_name') or \ db and db._adapter.trigger_name(tablename) self._common_filter = args.get('common_filter') @@ -8656,7 +8734,7 @@ class Table(object): fields.append(Field(fn,'blob',default='', writable=False,readable=False)) - lower_fieldnames = set() + fieldnames_set = set() reserved = dir(Table) + ['fields'] if (db and db.check_reserved): check_reserved = db.check_reserved_keyword @@ -8667,12 +8745,15 @@ class Table(object): for field in fields: field_name = field.name check_reserved(field_name) - fn_lower = field_name.lower() - if fn_lower in lower_fieldnames: + if db and db._ignore_field_case: + fname_item = field_name.lower() + else: + fname_item = field_name + if fname_item in fieldnames_set: raise SyntaxError("duplicate field %s in table %s" \ % (field_name, tablename)) else: - lower_fieldnames.add(fn_lower) + fieldnames_set.add(fname_item) self.fields.append(field_name) self[field_name] = field @@ -8914,8 +8995,23 @@ class Table(object): if 'Oracle' in str(type(self._db._adapter)): return '%s %s' % (ot, self._tablename) return '%s AS %s' % (ot, self._tablename) + return self._tablename + @property + def sqlsafe(self): + rname = self._rname + if rname: return rname + return self._db._adapter.sqlsafe_table(self._tablename) + + @property + def sqlsafe_alias(self): + rname = self._rname + ot = self._ot + if rname and not ot: return rname + return self._db._adapter.sqlsafe_table(self._tablename, self._ot) + + def _drop(self, mode = ''): return self._db._adapter._drop(self, mode) @@ -9742,7 +9838,12 @@ class Field(Expression): if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \ fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname): raise SyntaxError('Field: invalid field name: %s' % fieldname) - self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type + + if not isinstance(type, (Table,Field)): + self.type = type + else: + self.type = 'reference %s' % type + self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512) self.default = default if default!=DEFAULT else (update or None) self.required = required # is this field required @@ -10008,6 +10109,16 @@ class Field(Expression): except: return '.%s' % self.name + @property + def sqlsafe(self): + if self._table: + return self._table.sqlsafe + '.' + (self._rname or self._db._adapter.sqlsafe_field(self.name)) + return '.%s' % self.name + + @property + def sqlsafe_name(self): + return self._rname or self._db._adapter.sqlsafe_field(self.name) + class Query(object): @@ -10360,7 +10471,7 @@ class Set(object): fields = table._listify(update_fields,update=True) if not fields: raise SyntaxError("No fields to update") - ret = db._adapter.update("%s" % table,self.query,fields) + ret = db._adapter.update("%s" % table._tablename,self.query,fields) ret and [f(self,update_fields) for f in table._after_update] return ret @@ -10873,11 +10984,21 @@ class Rows(object): represent = kwargs.get('represent', False) writer = csv.writer(ofile, delimiter=delimiter, quotechar=quotechar, quoting=quoting) + def unquote_colnames(colnames): + unq_colnames = [] + for col in colnames: + m = self.db._adapter.REGEX_TABLE_DOT_FIELD.match(col) + if not m: + unq_colnames.append(col) + else: + unq_colnames.append('.'.join(m.groups())) + return unq_colnames + colnames = kwargs.get('colnames', self.colnames) write_colnames = kwargs.get('write_colnames',True) # a proper csv starting with the column names if write_colnames: - writer.writerow(colnames) + writer.writerow(unquote_colnames(colnames)) def none_exception(value): """ @@ -10900,10 +11021,11 @@ class Rows(object): for record in self: row = [] for col in colnames: - if not REGEX_TABLE_DOT_FIELD.match(col): + m = self.db._adapter.REGEX_TABLE_DOT_FIELD.match(col) + if not m: row.append(record._extra[col]) else: - (t, f) = col.split('.') + (t, f) = m.groups() field = self.db[t][f] if isinstance(record.get(t, None), (Row,dict)): value = record[t][f] diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index c970ab40..ef3b26f8 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -21,7 +21,7 @@ from gluon.html import FORM, INPUT, LABEL, OPTION, SELECT, COL, COLGROUP from gluon.html import TABLE, THEAD, TBODY, TR, TD, TH, STYLE from gluon.html import URL, truncate_string, FIELDSET from gluon.dal import DAL, Field, Table, Row, CALLABLETYPES, smart_query, \ - bar_encode, Reference, REGEX_TABLE_DOT_FIELD, Expression, SQLCustomType + bar_encode, Reference, Expression, SQLCustomType from gluon.storage import Storage from gluon.utils import md5_hash from gluon.validators import IS_EMPTY_OR, IS_NOT_EMPTY, IS_LIST_OF, IS_DATE, \ @@ -2893,7 +2893,7 @@ class SQLTABLE(TABLE): if not sqlrows: return if not columns: - columns = sqlrows.colnames + columns = ['.'.join(sqlrows.db._adapter.REGEX_TABLE_DOT_FIELD.match(c).groups()) for c in sqlrows.colnames] if headers == 'fieldname:capitalize': headers = {} for c in columns: @@ -3116,7 +3116,7 @@ class ExportClass(object): for record in self.rows: row = [] for col in self.rows.colnames: - if not REGEX_TABLE_DOT_FIELD.match(col): + if not self.rows.db._adapter.REGEX_TABLE_DOT_FIELD.match(col): row.append(record._extra[col]) else: (t, f) = col.split('.') diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index 1b4c5dcd..b77fdb13 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -79,6 +79,9 @@ def tearDownModule(): class TestFields(unittest.TestCase): def testFieldName(self): + return + + # Any table name is supported as long as underlying db does. The following code is ignored. # Check that Fields cannot start with underscores self.assertRaises(SyntaxError, Field, '_abc', 'string') @@ -1393,7 +1396,115 @@ class TestRNameFields(unittest.TestCase): self.assertEqual(len(db.person._referenced_by),0) db.person.drop() +class TestQuoting(unittest.TestCase): + # tests for complex table names + def testRun(self): + db = DAL(DEFAULT_URI, check_reserved=['all']) + t0 = db.define_table('A.table.with.dots and spaces', + Field('f', 'string')) + t1 = db.define_table('A.table', + Field('f.other', t0), + Field('words', 'text')) + + blather = 'blah blah and so' + t0[0] = {'f': 'content'} + t1[0] = {'f.other': int(t0[1]['id']), + 'words': blather} + + + r = db(t1['f.other']==t0.id).select() + self.assertEqual(r[0][db['A.table']].words, blather) + + db.define_table('t0', Field('f0')) + db.define_table('t1', Field('f1'), Field('t0', db['t0'])) + db.t0[0]=dict(f0=3) + db.t1[0]=dict(f1=3, t0=1) + + rows=db(db.t0.id==db.t1.t0).select() + self.assertEqual(rows[0].t1.t0, rows[0].t0.id) + + t0.drop('cascade') + t1.drop() + + db.t1.drop() + db.t0.drop() + + # tests for case sensitivity + def testCase(self): + db = DAL(DEFAULT_URI, check_reserved=['all'], ignore_field_case=False) + + + # test table case + t0 = db.define_table('B', + Field('f', 'string')) + try: + t1 = db.define_table('b', + Field('B', t0), + Field('words', 'text')) + except Exception, e: + # An error is expected when database does not support case + # sensitive entity names. + if DEFAULT_URI.startswith('sqlite:'): + self.assertTrue(isinstance(e, db._adapter.driver.OperationalError)) + return + raise e + + blather = 'blah blah and so' + t0[0] = {'f': 'content'} + t1[0] = {'B': int(t0[1]['id']), + 'words': blather} + + r = db(db.B.id==db.b.B).select() + + self.assertEqual(r[0].b.words, blather) + + t1.drop() + t0.drop() + + # test field case + try: + t0 = db.define_table('table is a test', + Field('a_a'), + Field('a_A')) + except Exception, e: + # some db does not support case sensitive field names mysql is one of them. + if DEFAULT_URI.startswith('mysql:'): + db.rollback() + return + raise e + + t0[0] = dict(a_a = 'a_a', a_A='a_A') + + self.assertEqual(t0[1].a_a, 'a_a') + self.assertEqual(t0[1].a_A, 'a_A') + + t0.drop() + + def testPKFK(self): + # test primary keys + + db = DAL(DEFAULT_URI, check_reserved=['all'], ignore_field_case=False) + + # test table without surrogate key. Length must is limited to + # 100 because of MySQL limitations: it cannot handle more than + # 767 bytes in unique keys. + + t0 = db.define_table('t0', Field('Code', length=100), primarykey=['Code']) + t22 = db.define_table('t22', Field('f'), Field('t0_Code', 'reference t0')) + t3 = db.define_table('t3', Field('f', length=100), Field('t0_Code', t0.Code), primarykey=['f']) + t4 = db.define_table('t4', Field('f', length=100), Field('t0', t0), primarykey=['f']) + + try: + t5 = db.define_table('t5', Field('f', length=100), Field('t0', 'reference no_table_wrong_reference'), primarykey=['f']) + except Exception, e: + self.assertTrue(isinstance(e, KeyError)) + + + t0.drop('cascade') + t22.drop() + t3.drop() + t4.drop() if __name__ == '__main__': unittest.main() - tearDownModule() \ No newline at end of file + tearDownModule() From 9d4cb09fdf8db1c245484c2833efe241ed6a69bc Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 3 Dec 2013 09:32:42 -0600 Subject: [PATCH 12/52] fixed typo, thanks Dean --- VERSION | 2 +- gluon/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 3771e4c1..c4e73ede 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.03.00.40.46 +Version 2.8.2-stable+timestamp.2013.12.03.09.31.36 diff --git a/gluon/tools.py b/gluon/tools.py index afb258f8..d4fa20f8 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -2915,7 +2915,7 @@ class Auth(object): formname='reset_password', dbio=False, onvalidation=onvalidation, hideerror=self.settings.hideerror): - user = table_user(**{userfield:form.vars.get(userlfield)}) + user = table_user(**{userfield:form.vars.get(userfield)}) if not user: session.flash = self.messages['invalid_%s' % userfield] redirect(self.url(args=request.args), From 5b7167436b6f5178712f212c9cd9198ce7a238fc Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 4 Dec 2013 10:43:50 -0600 Subject: [PATCH 13/52] optional label in checkbox widget, fixed issue 1797, thanks Richard --- VERSION | 2 +- gluon/sqlhtml.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c4e73ede..048f136d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.03.09.31.36 +Version 2.8.2-stable+timestamp.2013.12.04.10.42.46 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index ef3b26f8..0be5b768 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -400,6 +400,8 @@ class CheckboxesWidget(OptionsWidget): attr = cls._attributes(field, {}, **attributes) attr['_class'] = attr.get('_class', 'web2py_checkboxeswidget') + label = attr.get('label',True) + requires = field.requires if not isinstance(requires, (list, tuple)): requires = [requires] @@ -439,7 +441,8 @@ class CheckboxesWidget(OptionsWidget): requires=attr.get('requires', None), hideerror=True, _value=k, value=r_value), - LABEL(v, _for='%s%s' % (field.name, k)))) + LABEL(v, _for='%s%s' % (field.name, k)) + if label else '')) opts.append(child(tds)) if opts: From 3a6a8af9f629683efa980751a4943cb4526ff156 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 4 Dec 2013 19:35:43 -0600 Subject: [PATCH 14/52] fixed typo in spreadsheet, thanks Alan --- VERSION | 2 +- gluon/contrib/spreadsheet.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 048f136d..53c649a6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.04.10.42.46 +Version 2.8.2-stable+timestamp.2013.12.04.19.34.45 diff --git a/gluon/contrib/spreadsheet.py b/gluon/contrib/spreadsheet.py index 1ff05e73..7cca9c3d 100644 --- a/gluon/contrib/spreadsheet.py +++ b/gluon/contrib/spreadsheet.py @@ -858,8 +858,8 @@ class Sheet: sorted_headers = [TH(), ] + \ [TH(header[1]) for header in sorted(unsorted_headers)] table.insert(0, TR(*sorted_headers, - **{_class: "%s_fieldnames" % - attributes["_class"]})) + **{'_class': "%s_fieldnames" % + attributes["_class"]})) else: data = SCRIPT(""" // web2py Spreadsheets: no db data.""") From b61b0bf3ad1a80b1ab3ec7e65e206b24f0de01e6 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Thu, 5 Dec 2013 11:43:23 +0100 Subject: [PATCH 15/52] todo panel in admin editor --- applications/admin/controllers/default.py | 31 +++++++++++++++++++ .../admin/static/css/web2py-codemirror.css | 13 ++++++++ applications/admin/views/default/edit.html | 28 +++++++++++++++++ applications/admin/views/layout.html | 2 ++ 4 files changed, 74 insertions(+) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 221458de..aa95bb88 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -773,6 +773,37 @@ def edit(): else: return response.json(file_details) +def todolist(): + """ Returns all TODO of the requested app + """ + app = request.vars.app or '' + app_path = apath('%(app)s' % {'app':app}, r=request) + dirs=['models', 'controllers', 'modules', 'private' ] + def listfiles(app, dir, regexp='.*\.py$'): + files = sorted( listdir(apath('%(app)s/%(dir)s/' % {'app':app, 'dir':dir}, r=request), regexp)) + files = [x.replace(os.path.sep, '/') for x in files if not x.endswith('.bak')] + return files + + pattern = '#(todo)+\s+(.*)' + regex = re.compile(pattern, re.IGNORECASE) + + output = [] + for d in dirs: + for f in listfiles(app, d): + matches = [] + filename= apath(os.path.join(app, d, f), r=request) + + with open(filename, 'r') as f_s: + src = f_s.read() + for m in regex.finditer(src): + start = m.start() + lineno = src.count('\n', 0, start) + 1 + matches.append({'text':m.group(0), 'lineno':lineno}) + if len(matches) != 0: + output.append({'filename':f,'matches':matches, 'dir':d}) + + return {'todo':output, 'app': app} + def resolve(): """ diff --git a/applications/admin/static/css/web2py-codemirror.css b/applications/admin/static/css/web2py-codemirror.css index a11e8d88..f5f80211 100644 --- a/applications/admin/static/css/web2py-codemirror.css +++ b/applications/admin/static/css/web2py-codemirror.css @@ -1,3 +1,4 @@ +/* TODO rename this file as web2py-editor.css */ /* Fullscreen */ .CodeMirror-fullscreen { z-index: 1030; @@ -36,3 +37,15 @@ /*.nav-tabs>li { min-width: 100px; }*/ + +#windows_divs > div { + position: fixed; + height: 30%; + left: 0; + background: white; + right: 0; + bottom: 41px; + z-index: 1030; + overflow: inherit; + border-top: 1px solid #ddd; +} diff --git a/applications/admin/views/default/edit.html b/applications/admin/views/default/edit.html index 9187a4c4..15ab9317 100644 --- a/applications/admin/views/default/edit.html +++ b/applications/admin/views/default/edit.html @@ -227,7 +227,35 @@ $(document).on('click', 'a.font_button', function (e) {
+
+
+ {{=LOAD('default', 'todolist.load', vars={'app':app}, ajax=True, timeout=60000, times="infinity")}} +
+
+{{block footer}} +
+
+ +
+
+ + +{{end}} diff --git a/applications/admin/views/default/edit_js.html b/applications/admin/views/default/edit_js.html index 62e6fb54..51bd27ef 100644 --- a/applications/admin/views/default/edit_js.html +++ b/applications/admin/views/default/edit_js.html @@ -74,15 +74,15 @@ request.env['wsgi_url_scheme'], request.env['http_host'], URL(c='debug', f='toggle_breakpoint')))}}, sel); }); - function makeMarker() { - var marker = document.createElement("div"); - marker.style.color = "#822"; - marker.innerHTML = "●"; - marker.className = "breakpoint"; - return marker; - } + function makeMarker() { + var marker = document.createElement("div"); + marker.style.color = "#822"; + marker.innerHTML = "●"; + marker.className = "breakpoint"; + return marker; + } - {{if filetype in ('html', 'javascript', 'css'):}} + {{if filetype in ('html', 'javascript', 'css'):}} // must be here or break emmet/zencoding CodeMirror.defaults.extraKeys["Ctrl-S"] = function(instance) { @@ -94,26 +94,26 @@ CodeMirror.defaults.extraKeys["Shift-Esc"] = function(cm) { if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); } - {{pass}} - {{if filetype=='python':}} - // must be here or break emmet/zencoding for python - CodeMirror.defaults.extraKeys["Ctrl-S"] = - function(instance) { - doClickSave();}; - CodeMirror.defaults.extraKeys["Ctrl-Space"] = "autocomplete"; - CodeMirror.defaults.extraKeys["Tab"] = "indentMore"; - CodeMirror.defaults.extraKeys["Shift-Tab"] = "indentLess"; - CodeMirror.defaults.extraKeys["Ctrl-F11"] = function(cm) { - cm.setOption("fullScreen", !cm.getOption("fullScreen")); - }, - CodeMirror.defaults.extraKeys["Shift-Esc"] = function(cm) { - if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); - } - //for autocomplete - CodeMirror.commands.autocomplete = function(cm) { - CodeMirror.showHint(cm, CodeMirror.pythonHint); - } - {{pass}} + {{pass}} + {{if filetype=='python':}} + // must be here or break emmet/zencoding for python + CodeMirror.defaults.extraKeys["Ctrl-S"] = + function(instance) { + doClickSave();}; + CodeMirror.defaults.extraKeys["Ctrl-Space"] = "autocomplete"; + CodeMirror.defaults.extraKeys["Tab"] = "indentMore"; + CodeMirror.defaults.extraKeys["Shift-Tab"] = "indentLess"; + CodeMirror.defaults.extraKeys["Ctrl-F11"] = function(cm) { + cm.setOption("fullScreen", !cm.getOption("fullScreen")); + }, + CodeMirror.defaults.extraKeys["Shift-Esc"] = function(cm) { + if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); + } + //for autocomplete + CodeMirror.commands.autocomplete = function(cm) { + CodeMirror.showHint(cm, CodeMirror.pythonHint); + } + {{pass}} store_changes_function = function(instance, changeObj) { jQuery(instance).data('saved', false); instance.off("change", store_changes_function); @@ -128,6 +128,19 @@ request.env['wsgi_url_scheme'], request.env['http_host'], URL(c='debug', f='list_breakpoints')))}}, editor); + // TODO move it in a separated file + CodeMirror.defineExtension("centerOnCursor", function(limit) { + var coords = this.cursorCoords(null, "local"); + if (this.getScrollerElement().clientHeight === 0 && limit !== 10) { + if (limit === undefined) limit = 1; + else limit += 1; + editor = this; + setTimeout(function() {editor.centerOnCursor()}, 100); + return; + } + clientHeight = (this.getScrollerElement().clientHeight / 2) + this.scrollTo(null, (coords.top + coords.bottom)/2 - 10); + });
diff --git a/applications/admin/views/default/todolist.load b/applications/admin/views/default/todolist.load index 6d5bf494..00aceb92 100644 --- a/applications/admin/views/default/todolist.load +++ b/applications/admin/views/default/todolist.load @@ -7,7 +7,7 @@
  • {{=file['filename']}} ({{=len(file['matches'])}} TODO)
    • {{pass}} From 0ed01e5a21758cf29ebd5c47dd3aeeed6cd53fd9 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Fri, 6 Dec 2013 20:15:41 -0600 Subject: [PATCH 18/52] enable killing processed in windows, thanks Yair --- VERSION | 2 +- gluon/widget.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 53c649a6..f7d25eb0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.04.19.34.45 +Version 2.8.2-stable+timestamp.2013.12.06.20.14.05 diff --git a/gluon/widget.py b/gluon/widget.py index 2a8ddc98..371d1147 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -1224,7 +1224,10 @@ end tell if not options.nobanner: print 'please visit:' print '\t', url - print 'use "kill -SIGTERM %i" to shutdown the web2py server' % os.getpid() + if sys.platform.startswith('win'): + print 'use "taskkill /f /pid %i" to shutdown the web2py server' % os.getpid() + else: + print 'use "kill -SIGTERM %i" to shutdown the web2py server' % os.getpid() # enhance linecache.getline (used by debugger) to look at the source file # if the line was not found (under py2exe & when file was modified) From 91dc4f1d039dedab1bbfc0b3b76bdc8a054a9895 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 7 Dec 2013 10:16:21 -0600 Subject: [PATCH 19/52] fixed issue 1806:auth.wiki does not create permissions correctly, thanks Florian --- VERSION | 2 +- gluon/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index f7d25eb0..5f6efe32 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.06.20.14.05 +Version 2.8.2-stable+timestamp.2013.12.07.10.15.16 diff --git a/gluon/tools.py b/gluon/tools.py index d4fa20f8..f802d21f 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -5395,7 +5395,7 @@ class Wiki(object): if (auth.user and check_credentials(current.request, gae_login=False) and not 'wiki_editor' in auth.user_groups.values() and - self.settings.groups is None): + self.settings.groups == auth.user_groups.values()): group = db.auth_group(role='wiki_editor') gid = group.id if group else db.auth_group.insert( role='wiki_editor') From 35998633043f76859891d8c8bc81e73edd0789ad Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 7 Dec 2013 10:18:33 -0600 Subject: [PATCH 20/52] fixed 1801:error in DAL using MongoDB with 'list:reference
    ', thanks Alan --- VERSION | 2 +- gluon/dal.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 5f6efe32..8206c2d4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.07.10.15.16 +Version 2.8.2-stable+timestamp.2013.12.07.10.17.35 diff --git a/gluon/dal.py b/gluon/dal.py index 6005b545..bea06f8e 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -5647,9 +5647,6 @@ class MongoDBAdapter(NoSQLAdapter): value = obj else: value = NoSQLAdapter.represent(self, obj, fieldtype) - if isinstance(obj, (list, tuple)) and \ - (not fieldtype == "json" or fieldtype.startswith('list:')): - return value # reference types must be convert to ObjectID if fieldtype =='date': if value == None: From 1715b0c378f507209d0bdca25570f737d13196a0 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 7 Dec 2013 10:20:29 -0600 Subject: [PATCH 21/52] fixed 1807:Preserve quote template for NoSQLAdapter and subclasses, thanks Alan and Michele --- VERSION | 2 +- gluon/dal.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8206c2d4..af450676 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.07.10.17.35 +Version 2.8.2-stable+timestamp.2013.12.07.10.19.34 diff --git a/gluon/dal.py b/gluon/dal.py index bea06f8e..94a16620 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -4609,6 +4609,7 @@ class GoogleSQLAdapter(UseDatabaseStoredFile,MySQLAdapter): class NoSQLAdapter(BaseAdapter): can_select_for_update = False + QUOTE_TEMPLATE = '%s' @staticmethod def to_unicode(obj): From 86a19c25a8ab232e76329a957f120d40a050d972 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 7 Dec 2013 18:36:37 -0600 Subject: [PATCH 22/52] fixed issue 1808:HTML Injection on Terminal(shell) online --- VERSION | 2 +- applications/admin/controllers/shell.py | 3 +- applications/admin/views/shell/index.html | 69 ++++++++++++----------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/VERSION b/VERSION index af450676..4f14b643 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.07.10.19.34 +Version 2.8.2-stable+timestamp.2013.12.07.18.35.39 diff --git a/applications/admin/controllers/shell.py b/applications/admin/controllers/shell.py index 6d394861..3a835c7d 100644 --- a/applications/admin/controllers/shell.py +++ b/applications/admin/controllers/shell.py @@ -3,6 +3,7 @@ import cStringIO import gluon.contrib.shell import code import thread +import cgi from gluon.shell import env if DEMO_MODE or MULTI_USER_MODE: @@ -40,7 +41,7 @@ def callback(): k = len(session['commands:' + app]) - 1 #output = PRE(output) #return TABLE(TR('In[%i]:'%k,PRE(command)),TR('Out[%i]:'%k,output)) - return 'In [%i] : %s%s\n' % (k + 1, command, output) + return cgi.escape('In [%i] : %s%s\n' % (k + 1, command, output)) def reset(): diff --git a/applications/admin/views/shell/index.html b/applications/admin/views/shell/index.html index cb510f91..c90da324 100644 --- a/applications/admin/views/shell/index.html +++ b/applications/admin/views/shell/index.html @@ -2,40 +2,41 @@ {{block sectionclass}}shell{{end}}
    -
    -
    - -
    -
    - -
    -
    - - -
    - -
    -
    -
      -
    • Using the shell may lock the database to other users of this app.
      • -
    • Each db statement is automatically committed.
      • -
    • Creating new tables dynamically is not allowed.
      • -
    • Models are automatically imported in the shell.
      • -
    -
    +
    +
    + +
    +
    + +
    +
    + + +
    + +
    +
    +
      +
    • Using the shell may lock the database to other users of this app.
      • +
    • Each db statement is automatically committed.
      • +
    • Creating new tables dynamically is not allowed.
      • +
    • Models are automatically imported in the shell.
      • +
    +
    +
    @@ -113,4 +114,4 @@ jQuery(document).ready(function(){ }); }); - \ No newline at end of file + From 09277868d2335df3a887e9c91f0716e6a2c62e98 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 7 Dec 2013 20:47:08 -0600 Subject: [PATCH 23/52] accept bitcoin donations --- VERSION | 2 +- applications/examples/views/default/index.html | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 4f14b643..4ed1ee08 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.07.18.35.39 +Version 2.8.2-stable+timestamp.2013.12.07.20.46.12 diff --git a/applications/examples/views/default/index.html b/applications/examples/views/default/index.html index 2bfec2a6..28852551 100644 --- a/applications/examples/views/default/index.html +++ b/applications/examples/views/default/index.html @@ -43,7 +43,8 @@ random.shuffle(quotes)
    Download Now
    Try it now online
    - Sites Powered by web2py + Sites Powered by web2py

    + Donate Bitcoins
    From 483de0fc3021131eb92366694a377c0db2dc4f94 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sun, 8 Dec 2013 10:18:47 +0100 Subject: [PATCH 24/52] better style for todolist panel (admin editor) --- applications/admin/views/default/edit.html | 4 ++++ applications/admin/views/default/todolist.load | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/applications/admin/views/default/edit.html b/applications/admin/views/default/edit.html index 4e8d0427..31f7a040 100644 --- a/applications/admin/views/default/edit.html +++ b/applications/admin/views/default/edit.html @@ -251,6 +251,10 @@ $(document).on('click', 'a.font_button', function (e) { + diff --git a/applications/admin/views/default/edit_js.html b/applications/admin/views/default/edit_js.html index 51bd27ef..571839b8 100644 --- a/applications/admin/views/default/edit_js.html +++ b/applications/admin/views/default/edit_js.html @@ -114,6 +114,7 @@ CodeMirror.showHint(cm, CodeMirror.pythonHint); } {{pass}} + CodeMirror.defaults.extraKeys["Ctrl-/"] = "toggleComment"; store_changes_function = function(instance, changeObj) { jQuery(instance).data('saved', false); instance.off("change", store_changes_function); @@ -154,34 +155,26 @@
    - {{if filetype=='html':}} + {{if filetype=='html':}}

    {{=T('Key bindings for ZenCoding Plugin')}}

    -
      - {{=shortcut('Ctrl+S', T('Save via Ajax'))}} - {{=shortcut('Ctrl+F11', T('Toggle Fullscreen'))}} - {{=shortcut('Shift+Esc', T('Exit Fullscreen'))}} - {{=shortcut('Ctrl-F / Cmd-F', T('Start searching'))}} - {{=shortcut('Ctrl-G / Cmd-G', T('Find Next'))}} - {{=shortcut('Shift-Ctrl-G / Shift-Cmd-G', T('Find Previous'))}} - {{=shortcut('Shift-Ctrl-F / Cmd-Option-F', T('Replace'))}} - {{=shortcut('Shift-Ctrl-R / Shift-Cmd-Option-F', T('Replace All'))}} - {{=shortcut('Tab', T('Expand Abbreviation'))}} -
    - {{else:}} + {{else:}}

    {{=T("Key bindings")}}

    + {{pass}}
      {{=shortcut('Ctrl+S', T('Save via Ajax'))}} {{=shortcut('Ctrl+F11', T('Toggle Fullscreen'))}} {{=shortcut('Shift+Esc', T('Exit Fullscreen'))}} - {{if filetype=='python':}} - {{=shortcut('Ctrl-Space', T('Autocomplete Python Code'))}} - {{pass}} {{=shortcut('Ctrl-F / Cmd-F', T('Start searching'))}} {{=shortcut('Ctrl-G / Cmd-G', T('Find Next'))}} {{=shortcut('Shift-Ctrl-G / Shift-Cmd-G', T('Find Previous'))}} {{=shortcut('Shift-Ctrl-F / Cmd-Option-F', T('Replace'))}} {{=shortcut('Shift-Ctrl-R / Shift-Cmd-Option-F', T('Replace All'))}} -
    + {{=shortcut('Ctrl-/ ', T('Toggle comment'))}} + {{if filetype=='html':}} + {{=shortcut('Tab', T('Expand Abbreviation'))}} + {{elif filetype=='python':}} + {{=shortcut('Ctrl-Space', T('Autocomplete Python Code'))}} {{pass}} +
    From 53122bfa9f71a61a50bb4d376cf3644527ed00a7 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 9 Dec 2013 17:06:25 -0600 Subject: [PATCH 29/52] possible implementation of st_dwithin, thanks User --- VERSION | 2 +- gluon/dal.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 3798cc07..f7da9d28 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.08.09.32.29 +Version 2.8.2-stable+timestamp.2013.12.09.17.05.33 diff --git a/gluon/dal.py b/gluon/dal.py index b3dc64c3..df2c6425 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -2980,11 +2980,12 @@ class PostgreSQLAdapter(BaseAdapter): """ return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type)) - def ST_DWITHIN(self, first, second): + def ST_DWITHIN(self, first, (second, third)): """ http://postgis.org/docs/ST_Within.html """ - return 'ST_DWithin(%s,%s)' %(self.expand(first), self.expand(second, first.type)) + return 'ST_DWithin(%s,%s)' %(self.expand(first), self.expand(second, first.type), + self.expand(third, 'double')) def represent(self, obj, fieldtype): field_is_type = fieldtype.startswith @@ -9674,6 +9675,10 @@ class Expression(object): db = self.db return Query(db, db._adapter.ST_WITHIN, self, value) + def st_dwithin(self, value, distance): + db = self.db + return Query(db, db._adapter.ST_DWITHIN, self, (value, distance)) + # for use in both Query and sortby From cda7c666373df883859c95ca393857b645fce4f0 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 9 Dec 2013 17:55:53 -0600 Subject: [PATCH 30/52] fixed bug in last commit, thanks User --- VERSION | 2 +- gluon/dal.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index f7da9d28..9c204297 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.09.17.05.33 +Version 2.8.2-stable+timestamp.2013.12.09.17.54.55 diff --git a/gluon/dal.py b/gluon/dal.py index df2c6425..bc1f4ab0 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -2984,8 +2984,9 @@ class PostgreSQLAdapter(BaseAdapter): """ http://postgis.org/docs/ST_Within.html """ - return 'ST_DWithin(%s,%s)' %(self.expand(first), self.expand(second, first.type), - self.expand(third, 'double')) + return 'ST_DWithin(%s,%s,%s)' %(self.expand(first), + self.expand(second, first.type), + self.expand(third, 'double')) def represent(self, obj, fieldtype): field_is_type = fieldtype.startswith From bbbee21b0d5c402310f4a29a7b025a081282579b Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 9 Dec 2013 18:56:05 -0600 Subject: [PATCH 31/52] DAL(...,adapter_args=dict(engine='MyISAM')) --- VERSION | 2 +- gluon/dal.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 9c204297..810eb860 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.09.17.54.55 +Version 2.8.2-stable+timestamp.2013.12.09.18.55.10 diff --git a/gluon/dal.py b/gluon/dal.py index bc1f4ab0..1211e894 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -1032,7 +1032,8 @@ class BaseAdapter(ConnectionPool): if self.dbengine == 'mysql': if not hasattr(table, "_primarykey"): fields.append('PRIMARY KEY (%s)' % (self.QUOTE_TEMPLATE % table._id.name)) - other = ' ENGINE=InnoDB CHARACTER SET utf8;' + engine = self.adapter_args.get('engine','InnoDB') + other = ' ENGINE=%s CHARACTER SET utf8;' % engine fields = ',\n '.join(fields) for rtablename in TFK: From e6aaca1728cf53da26c0bccfae2940ac3c1d6530 Mon Sep 17 00:00:00 2001 From: robertop23 Date: Mon, 9 Dec 2013 21:51:55 -0430 Subject: [PATCH 32/52] Allow create files from editor page in "files toggle" Once file is created show success message and reload using ajax the files new file: applications/admin/views/default/files_menu.html modified: applications/admin/controllers/default.py modified: applications/admin/views/default/edit.html --- applications/admin/views/default/files_menu.html | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 applications/admin/views/default/files_menu.html diff --git a/applications/admin/views/default/files_menu.html b/applications/admin/views/default/files_menu.html new file mode 100644 index 00000000..85b55f22 --- /dev/null +++ b/applications/admin/views/default/files_menu.html @@ -0,0 +1,3 @@ +{{for files in result_files:}} + {{=files}} +{{pass}} \ No newline at end of file From 37df0a9338dc4f94e71274c1320051ebc1137f81 Mon Sep 17 00:00:00 2001 From: robertop23 Date: Mon, 9 Dec 2013 21:57:57 -0430 Subject: [PATCH 33/52] Allow create files from editor page in "files toggle" Once file is created show success message and reload using ajax the files modified: applications/admin/controllers/default.py modified: applications/admin/views/default/edit.html --- applications/admin/controllers/default.py | 42 ++++++++++- applications/admin/views/default/edit.html | 84 ++++++++++++++++------ 2 files changed, 103 insertions(+), 23 deletions(-) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index f048a3ae..bd22abed 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -1239,7 +1239,6 @@ def plugin(): languages=languages, crontab=crontab) - def create_file(): """ Create files handler """ if request.vars and not request.vars.token == session.token: @@ -1250,6 +1249,8 @@ def create_file(): app = get_app(request.vars.app) path = abspath(request.vars.location) else: + if request.vars.dir: + request.vars.location += request.vars.dir + '/' app = get_app(name=request.vars.location.split('/')[0]) path = apath(request.vars.location, r=request) filename = re.sub('[^\w./-]+', '_', request.vars.filename) @@ -1377,7 +1378,11 @@ def create_file(): safe_write(full_filename, text) log_progress(app, 'CREATE', filename) - session.flash = T('file "%(filename)s" created', + if request.vars.dir: + result = T('file "%(filename)s" created', + dict(filename=full_filename[len(path):])) + else: + session.flash = T('file "%(filename)s" created', dict(filename=full_filename[len(path):])) vars = {} if request.vars.id: @@ -1390,9 +1395,40 @@ def create_file(): if not isinstance(e, HTTP): session.flash = T('cannot create file') - redirect(request.vars.sender + anchor) + if request.vars.dir: + id_filename = '#' + request.vars.dir + '__' + filename.replace('.','__') + ' a' + return response.json({'result':result, 'id_filename':id_filename}) + else: + redirect(request.vars.sender + anchor) +def listfiles(app, dir, regexp='.*\.py$'): + files = sorted( + listdir(apath('%(app)s/%(dir)s/' % {'app':app, 'dir':dir}, r=request), regexp)) + files = [x.replace('\\', '/') for x in files if not x.endswith('.bak')] + return files + +def editfile(path,file,vars={}, app = None): + args=(path,file) if 'app' in vars else (app,path,file) + url = URL('edit', args=args, vars=vars) + return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;') + +def files_menu(): + app = 'welcome'#request.args[0] + dirs=[{'name':'models', 'reg':'.*\.py$'}, + {'name':'controllers', 'reg':'.*\.py$'}, + {'name':'views', 'reg':'[\w/\-]+(\.\w+)+$'}, + {'name':'modules', 'reg':'.*\.py$'}, + {'name':'static', 'reg': '[^\.#].*'}] + result_files = [] + for dir in dirs: + result_files.append(TAG[''](LI(dir['name'], _class="nav-header component", _onclick="collapse('" + dir['name'] + "_files');"), + LI(UL(*[LI(editfile(dir['name'], f, dict(id=dir['name'] + f.replace('.','__')), app), _style="overflow:hidden", _id=dir['name']+"__"+f.replace('.','__')) + for f in listfiles(app, dir['name'], regexp=dir['reg'])], + _class="nav nav-list small-font"), + _id=dir['name'] + '_files', _style="display: none;"))) + return dict(result_files = result_files) + def upload_file(): """ File uploading handler """ if request.vars and not request.vars.token == session.token: diff --git a/applications/admin/views/default/edit.html b/applications/admin/views/default/edit.html index 31f7a040..85d0e9d1 100644 --- a/applications/admin/views/default/edit.html +++ b/applications/admin/views/default/edit.html @@ -1,4 +1,32 @@ {{extend 'layout.html'}} +{{ +dirs=[{'name':'models', 'reg':'.*\.py$'}, + {'name':'controllers', 'reg':'.*\.py$'}, + {'name':'views', 'reg':'[\w/\-]+(\.\w+)+$'}, + {'name':'modules', 'reg':'.*\.py$'}, + {'name':'static', 'reg': '[^\.#].*'}] + +def file_create_form(location, anchor=None, helptext=""): + form=FORM( + LABEL(T("create file with filename:")), + SELECT(_name='dir', _style='width:100px;', + *[OPTION(dir['name'], _value=dir['name']) for dir in dirs]), + XML(' '),LABEL('/', _style='display:inline-block;'),XML(' '), + INPUT(_type="text",_name="filename",requires=IS_NOT_EMPTY(),_class=''), + TAG['SMALL'](helptext,_class="help-block"), + INPUT(_type='submit', name=T('filename'), _value=T('Create'), _style='display:block', _id='btn_file_create'), + INPUT(_type="hidden",_name="editor"), + INPUT(_type="hidden",_name="location",_value=location), + INPUT(_type="hidden",_name="sender",_value=URL('design',args=app)), + INPUT(_type="hidden",_name="token",_value=session.token), + #INPUT(_type="hidden",_name="id",_value=anchor), + _action=URL('create_file'), + _id='file_create_form', + _class="generatedbyw2p well well-small") + return form + +}} + {{ def shortcut(combo, description): @@ -203,27 +231,43 @@ $(document).on('click', 'a.font_button', function (e) {
    From 15dd46a91a2a80bd471a5cfae236d9643e8f868d Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 9 Dec 2013 21:28:26 -0600 Subject: [PATCH 34/52] reverted grouping of joined tables --- VERSION | 2 +- gluon/dal.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 810eb860..05a0c515 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.09.18.55.10 +Version 2.8.2-stable+timestamp.2013.12.09.21.27.36 diff --git a/gluon/dal.py b/gluon/dal.py index 1211e894..3108a781 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -2715,10 +2715,6 @@ class MySQLAdapter(BaseAdapter): self.execute('select last_insert_id();') return int(self.cursor.fetchone()[0]) - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby): - return BaseAdapter.select_limitby( - self, sql_s, sql_f, '(%s)' % sql_t, sql_w, sql_o, limitby) - class PostgreSQLAdapter(BaseAdapter): drivers = ('psycopg2','pg8000') From 8244472cb2f4f0a0aa4acce18f5e45ed76fd5477 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Tue, 10 Dec 2013 09:53:50 +0100 Subject: [PATCH 35/52] select the right app in the file list editor sidebar --- applications/admin/controllers/default.py | 2 +- applications/admin/views/default/edit.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index bd22abed..723b63c9 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -1414,7 +1414,7 @@ def editfile(path,file,vars={}, app = None): return A(file, _class='editor_filelink', _href=url, _style='word-wrap: nowrap;') def files_menu(): - app = 'welcome'#request.args[0] + app = request.vars.app or 'welcome' dirs=[{'name':'models', 'reg':'.*\.py$'}, {'name':'controllers', 'reg':'.*\.py$'}, {'name':'views', 'reg':'[\w/\-]+(\.\w+)+$'}, diff --git a/applications/admin/views/default/edit.html b/applications/admin/views/default/edit.html index b275a147..96c56def 100644 --- a/applications/admin/views/default/edit.html +++ b/applications/admin/views/default/edit.html @@ -267,7 +267,7 @@ $(document).on('click', 'a.font_button', function (e) {
    - {{=LOAD('default', 'files_menu', args=['welcome'])}} + {{=LOAD('default', 'files_menu', vars={'app':app})}}
    From ff94a22826cdf1e77a72c3c6b5acf434581c9247 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Tue, 10 Dec 2013 11:13:04 +0100 Subject: [PATCH 36/52] fix reload of file list in editor when a new file is created --- applications/admin/views/default/edit.html | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/applications/admin/views/default/edit.html b/applications/admin/views/default/edit.html index 96c56def..ba82c94c 100644 --- a/applications/admin/views/default/edit.html +++ b/applications/admin/views/default/edit.html @@ -241,20 +241,21 @@ $(document).on('click', 'a.font_button', function (e) {
    {{=file_create_form('%s/' % app, '')}}

    - {{=LOAD('default', 'files_menu', vars={'app':app})}} + {{=LOAD('default', 'files_menu', vars={'app':app}, ajax=True)}}
    @@ -318,6 +296,8 @@ $(document).ready(function() { var ow = filesMenu.outerWidth(); filesMenu.width(ow); $('#files').css('left', '-'+ow+'px'); + $.web2py.trap_form('url', 'form'); + $('#form form').addClass('no_trap'); // Let to reuse the same form }); From 42de43d052c2c64f52cdcde7db50d4e900ecb12f Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 14 Dec 2013 20:03:35 -0600 Subject: [PATCH 43/52] fixed 1814:session.connect: parameter migrate=False has no effect, thanks lucmurer --- VERSION | 2 +- gluon/globals.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 1b5a63e2..b276869c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.11.20.35.40 +Version 2.8.2-stable+timestamp.2013.12.14.20.02.42 diff --git a/gluon/globals.py b/gluon/globals.py index 58b2c0de..08e09f64 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -816,7 +816,10 @@ class Session(Storage): # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: request.tickets_db = db - table_migrate = (masterapp == request.application) + if masterapp == request.application: + table_migrate = migrate + else: + table_migrate = False tname = tablename + '_' + masterapp table = db.get(tname, None) Field = db.Field From 09c0069b43fe5a5246189ab5d14b108a96d51327 Mon Sep 17 00:00:00 2001 From: niphlod Date: Sun, 15 Dec 2013 15:06:42 +0100 Subject: [PATCH 44/52] fix mssql quoting issue, disabled some tests Tests now pass on mssql. I think we need to agree on what's going on with DAL, or we risk shipping a module with lots of changes that are breaking backward compatibility. I'll start a thread on web2py-developers --- gluon/dal.py | 4 ++-- gluon/tests/test_dal.py | 49 ++++++++++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/gluon/dal.py b/gluon/dal.py index 8c6420df..80ee34d7 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -3291,8 +3291,8 @@ class OracleAdapter(BaseAdapter): class MSSQLAdapter(BaseAdapter): drivers = ('pyodbc',) T_SEP = 'T' - - QUOTE_TEMPLATE = "[%s]" + + QUOTE_TEMPLATE = '"%s"' types = { 'boolean': 'BIT', diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index b77fdb13..0ca8b585 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -1396,24 +1396,26 @@ class TestRNameFields(unittest.TestCase): self.assertEqual(len(db.person._referenced_by),0) db.person.drop() + class TestQuoting(unittest.TestCase): # tests for complex table names def testRun(self): + return db = DAL(DEFAULT_URI, check_reserved=['all']) t0 = db.define_table('A.table.with.dots and spaces', Field('f', 'string')) t1 = db.define_table('A.table', - Field('f.other', t0), + Field('f_other', t0), Field('words', 'text')) blather = 'blah blah and so' t0[0] = {'f': 'content'} - t1[0] = {'f.other': int(t0[1]['id']), + t1[0] = {'f_other': int(t0[1]['id']), 'words': blather} - r = db(t1['f.other']==t0.id).select() + r = db(t1['f_other']==t0.id).select() self.assertEqual(r[0][db['A.table']].words, blather) db.define_table('t0', Field('f0')) @@ -1423,17 +1425,26 @@ class TestQuoting(unittest.TestCase): rows=db(db.t0.id==db.t1.t0).select() self.assertEqual(rows[0].t1.t0, rows[0].t0.id) - - t0.drop('cascade') - t1.drop() + if DEFAULT_URI.startswith('mssql'): + #there's no drop cascade in mssql + t1.drop() + t0.drop() + else: + t0.drop('cascade') + t1.drop() db.t1.drop() db.t0.drop() # tests for case sensitivity def testCase(self): + return db = DAL(DEFAULT_URI, check_reserved=['all'], ignore_field_case=False) - + if DEFAULT_URI.startswith('mssql'): + #multiple cascade gotcha + for key in ['reference','reference FK']: + db._adapter.types[key]=db._adapter.types[key].replace( + '%(on_delete_action)s','NO ACTION') # test table case t0 = db.define_table('B', @@ -1482,16 +1493,21 @@ class TestQuoting(unittest.TestCase): t0.drop() def testPKFK(self): + # test primary keys db = DAL(DEFAULT_URI, check_reserved=['all'], ignore_field_case=False) - + if DEFAULT_URI.startswith('mssql'): + #multiple cascade gotcha + for key in ['reference','reference FK']: + db._adapter.types[key]=db._adapter.types[key].replace( + '%(on_delete_action)s','NO ACTION') # test table without surrogate key. Length must is limited to # 100 because of MySQL limitations: it cannot handle more than # 767 bytes in unique keys. t0 = db.define_table('t0', Field('Code', length=100), primarykey=['Code']) - t22 = db.define_table('t22', Field('f'), Field('t0_Code', 'reference t0')) + t2 = db.define_table('t2', Field('f'), Field('t0_Code', 'reference t0')) t3 = db.define_table('t3', Field('f', length=100), Field('t0_Code', t0.Code), primarykey=['f']) t4 = db.define_table('t4', Field('f', length=100), Field('t0', t0), primarykey=['f']) @@ -1500,11 +1516,18 @@ class TestQuoting(unittest.TestCase): except Exception, e: self.assertTrue(isinstance(e, KeyError)) + if DEFAULT_URI.startswith('mssql'): + #there's no drop cascade in mssql + t3.drop() + t4.drop() + t2.drop() + t0.drop() + else: + t0.drop('cascade') + t2.drop() + t3.drop() + t4.drop() - t0.drop('cascade') - t22.drop() - t3.drop() - t4.drop() if __name__ == '__main__': unittest.main() tearDownModule() From 53a0718a7842514fd7224092039cd8d76c12fc13 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 15 Dec 2013 13:31:56 -0600 Subject: [PATCH 45/52] fixed problem with lack of connector in NoSQL --- VERSION | 2 +- gluon/dal.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b276869c..092f1d9d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.14.20.02.42 +Version 2.8.2-stable+timestamp.2013.12.15.13.31.12 diff --git a/gluon/dal.py b/gluon/dal.py index 80ee34d7..47be7dbc 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -703,6 +703,7 @@ class BaseAdapter(ConnectionPool): can_select_for_update = True dbpath = None folder = None + connector = lambda dbpath, adapter_args: None # __init__ should override this TRUE = 'T' FALSE = 'F' From 104ec5ba7302035111608274f665065996ff0a95 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 15 Dec 2013 22:23:57 -0600 Subject: [PATCH 46/52] fixed issue 1818, error in wiki/renderer, thanks Alan --- VERSION | 2 +- gluon/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 092f1d9d..98d07a7d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.15.13.31.12 +Version 2.8.2-stable+timestamp.2013.12.15.22.22.52 diff --git a/gluon/tools.py b/gluon/tools.py index f802d21f..b0d4d3a5 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -5523,7 +5523,7 @@ class Wiki(object): url = URL(args=('_edit', slug)) return dict(content=A('Create page "%s"' % slug, _href=url, _class="btn")) else: - html = page.html if not force_render else self.get_renderer(page) + html = page.html if not force_render else self.get_renderer()(page) content = XML(self.fix_hostname(html)) return dict(title=page.title, slug=page.slug, From 7646dc11fe33e0088a52df2881a51ad64d391301 Mon Sep 17 00:00:00 2001 From: Alan Etkin Date: Mon, 16 Dec 2013 17:24:51 -0300 Subject: [PATCH 47/52] Enable admin app for GAE (experimental) --- applications/admin/controllers/default.py | 23 ++++++++++++++++++----- examples/app.example.yaml | 4 ++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index b02e5eac..00082613 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -30,10 +30,18 @@ from gluon.languages import (read_possible_languages, read_dict, write_dict, read_plural_dict, write_plural_dict) -if DEMO_MODE and request.function in ['change_password', 'pack', 'pack_custom','pack_plugin', 'upgrade_web2py', 'uninstall', 'cleanup', 'compile_app', 'remove_compiled_app', 'delete', 'delete_plugin', 'create_file', 'upload_file', 'update_languages', 'reload_routes', 'git_push', 'git_pull']: +if DEMO_MODE and request.function in ['change_password', 'pack', +'pack_custom','pack_plugin', 'upgrade_web2py', 'uninstall', +'cleanup', 'compile_app', 'remove_compiled_app', 'delete', +'delete_plugin', 'create_file', 'upload_file', 'update_languages', +'reload_routes', 'git_push', 'git_pull', 'install_plugin']: session.flash = T('disabled in demo mode') redirect(URL('site')) +if is_gae and request.function in ('edit', 'edit_language', +'edit_plurals', 'update_languages', 'create_file', 'install_plugin'): + session.flash = T('disabled in GAE mode') + redirect(URL('site')) if not is_manager() and request.function in ['change_password', 'upgrade_web2py']: session.flash = T('disabled in multi user mode') @@ -63,10 +71,12 @@ def log_progress(app, mode='EDIT', filename=None, progress=0): def safe_open(a, b): - if DEMO_MODE and ('w' in b or 'a' in b): + if (DEMO_MODE or is_gae) and ('w' in b or 'a' in b): class tmp: def write(self, data): pass + def close(self): + pass return tmp() return open(a, b) @@ -1496,7 +1506,7 @@ def errors(): app = get_app() - method = request.args(1) or 'new' + method = request.args(1) or 'new' if not is_gae else "dbnew" db_ready = {} db_ready['status'] = get_ticket_storage(app) db_ready['errmessage'] = T( @@ -1588,7 +1598,7 @@ def errors(): decorated.sort(key=operator.itemgetter(0), reverse=True) - return dict(errors=[x[1] for x in decorated], app=app, method=method) + return dict(errors=[x[1] for x in decorated], app=app, method=method, db_ready=db_ready) elif method == 'dbold': tk_db, tk_table = get_ticket_storage(app) @@ -1601,7 +1611,7 @@ def errors(): times = dict( [(row.ticket_id, row.created_datetime) for row in tickets_]) - return dict(app=app, tickets=tickets, method=method, times=times) + return dict(app=app, tickets=tickets, method=method, times=times, db_ready=db_ready) else: for item in request.vars: @@ -1625,6 +1635,9 @@ def get_ticket_storage(app): if os.path.exists(ticket_file): db_string = open(ticket_file).read() db_string = db_string.strip().replace('\r', '').replace('\n', '') + elif is_gae: + # use Datastore as fallback if there is no ticket_file + db_string = "google:datastore" else: return False tickets_table = 'web2py_ticket' diff --git a/examples/app.example.yaml b/examples/app.example.yaml index b56a16e5..ff608af1 100644 --- a/examples/app.example.yaml +++ b/examples/app.example.yaml @@ -65,8 +65,8 @@ skip_files: | (.*\.py[co])| (.*/RCS/.*)| (\..*)| - (applications/(admin|examples)/.*)| - ((admin|examples|welcome)\.(w2p|tar))| + (applications/examples/.*)| + ((examples|welcome)\.(w2p|tar))| (applications/.*?/(cron|databases|errors|cache|sessions)/.*)| ((logs|scripts)/.*)| (anyserver\.py)| From c790b79393b52e622a8b5b13fad05d7232ae654b Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 16 Dec 2013 21:56:28 -0600 Subject: [PATCH 48/52] autolink emails --- VERSION | 2 +- gluon/contrib/autolinks.py | 2 ++ gluon/contrib/markmin/markmin2html.py | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 98d07a7d..ae14e219 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.15.22.22.52 +Version 2.8.2-stable+timestamp.2013.12.16.21.55.21 diff --git a/gluon/contrib/autolinks.py b/gluon/contrib/autolinks.py index af3d1247..23e9ed10 100644 --- a/gluon/contrib/autolinks.py +++ b/gluon/contrib/autolinks.py @@ -159,6 +159,8 @@ def extension(url): def expand_one(url, cdict): # try ombed but first check in cache + if '@' in url and not '://'in url: + return '%s' % (url, url) if cdict and url in cdict: r = cdict[url] else: diff --git a/gluon/contrib/markmin/markmin2html.py b/gluon/contrib/markmin/markmin2html.py index 2d3b2aa9..95f9dc56 100755 --- a/gluon/contrib/markmin/markmin2html.py +++ b/gluon/contrib/markmin/markmin2html.py @@ -551,7 +551,7 @@ regex_list=re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s*)?(.*)$') regex_bq_headline=re.compile('^(?:(\.+|\++|\-+)(\.)?\s+)?(-{3}-*)$') regex_tq=re.compile('^(-{3}-*)(?::(?P[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P

    [a-zA-Z][_a-zA-Z\-\d]*)\])?)?$') regex_proto = re.compile(r'(?/=])(?P

    \w+):(?P\w+://[\w\d\-+=?%&/:.]+)', re.M) -regex_auto = re.compile(r'(?/=])(?P\w+://[\w\d\-+_=?%&/:.,;#]+\w)',re.M) +regex_auto = re.compile(r'(?/=])(?P\w+://[\w\d\-+_=?%&/:.,;#]+\w|[\w\-.]+@[\w\-.]+)',re.M) regex_link=re.compile(r'('+LINK+r')|\[\[(?P.+?)\]\]',re.S) regex_link_level2=re.compile(r'^(?P\S.*?)?(?:\s+\[(?P.+?)\])?(?:\s+(?P\S+))?(?:\s+(?P

    popup))?\s*$',re.S) regex_media_level2=re.compile(r'^(?P\S.*?)?(?:\s+\[(?P.+?)\])?(?:\s+(?P\S+))?\s+(?P

    img|IMG|left|right|center|video|audio|blockleft|blockright)(?:\s+(?P\d+px))?\s*$',re.S) @@ -648,7 +648,9 @@ def autolinks_simple(url): image, video or audio tag """ u_url=url.lower() - if u_url.endswith(('.jpg','.jpeg','.gif','.png')): + if '@' in url and not '://' in url: + return '%s' % (url, url) + elif u_url.endswith(('.jpg','.jpeg','.gif','.png')): return '' % url elif u_url.endswith(('.mp4','.mpeg','.mov','.ogv')): return '' % url @@ -673,6 +675,9 @@ def protolinks_simple(proto, url): return 'QR Code'%url return proto+':'+url +def email_simple(email): + return '%s' % (email, email) + def render(text, extra={}, allowed={}, From a4416bd11f94bc05373c4463ac3f547932c0b53d Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 17 Dec 2013 16:40:44 -0600 Subject: [PATCH 49/52] fixed issue 1819:Datastore reconnect problem, thanks Alan --- VERSION | 2 +- gluon/dal.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index ae14e219..946fe36c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.16.21.55.21 +Version 2.8.2-stable+timestamp.2013.12.17.16.39.21 diff --git a/gluon/dal.py b/gluon/dal.py index 47be7dbc..3d7b33d2 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -703,7 +703,7 @@ class BaseAdapter(ConnectionPool): can_select_for_update = True dbpath = None folder = None - connector = lambda dbpath, adapter_args: None # __init__ should override this + connector = lambda *args, **kwargs: None # __init__ should override this TRUE = 'T' FALSE = 'F' @@ -4809,6 +4809,8 @@ class GoogleDatastoreAdapter(NoSQLAdapter): uploads_in_blob = True types = {} + # reconnect is not required for Datastore dbs + reconnect = lambda *args, **kwargs: None def file_exists(self, filename): pass def file_open(self, filename, mode='rb', lock=True): pass From 38de99fc9cefe0a4f7faaaba3fe710975fc261ad Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 17 Dec 2013 16:43:07 -0600 Subject: [PATCH 50/52] fixed 1820:uncaught pickling error with shell and app engine, thanks Alan --- VERSION | 2 +- gluon/contrib/shell.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 946fe36c..5e4667b0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.17.16.39.21 +Version 2.8.2-stable+timestamp.2013.12.17.16.41.58 diff --git a/gluon/contrib/shell.py b/gluon/contrib/shell.py index 4d206036..0a224100 100755 --- a/gluon/contrib/shell.py +++ b/gluon/contrib/shell.py @@ -258,7 +258,7 @@ def run(history, statement, env={}): if not name.startswith('__'): try: history.set_global(name, val) - except TypeError, ex: + except (TypeError, cPickle.PicklingError), ex: UNPICKLABLE_TYPES.append(type(val)) history.add_unpicklable(statement, new_globals.keys()) From 28b9d322a250d1b2c38aae039bd10ba0a7ba6fb5 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 17 Dec 2013 16:50:42 -0600 Subject: [PATCH 51/52] fixed 1821:RadioWidget and CheckboxesWidget wrong behavior, thanks Paolo Caruccio --- VERSION | 2 +- gluon/sqlhtml.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 5e4667b0..bee3b373 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.8.2-stable+timestamp.2013.12.17.16.41.58 +Version 2.8.2-stable+timestamp.2013.12.17.16.49.17 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 0be5b768..2a417c78 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -45,6 +45,9 @@ except ImportError: table_field = re.compile('[\w_]+\.[\w_]+') widget_class = re.compile('^\w*') +def add_class(a,b): + return a+' '+b if a else b + def represent(field, value, record): f = field.represent if not callable(f): @@ -334,7 +337,7 @@ class RadioWidget(OptionsWidget): attr = cls._attributes(field, {}, **attributes) - attr['_class'] = attr.get('_class', 'web2py_radiowidget') + attr['_class'] = add_class(attr.get('_class'), 'web2py_radiowidget') requires = field.requires if not isinstance(requires, (list, tuple)): @@ -398,7 +401,7 @@ class CheckboxesWidget(OptionsWidget): values = [str(value)] attr = cls._attributes(field, {}, **attributes) - attr['_class'] = attr.get('_class', 'web2py_checkboxeswidget') + attr['_class'] = add_class(attr.get('_class'), 'web2py_checkboxeswidget') label = attr.get('label',True) From 42c0b7b0fa8ebfcd5da2146d9ba7192c59b54ff1 Mon Sep 17 00:00:00 2001 From: Alan Etkin Date: Wed, 18 Dec 2013 15:44:45 -0300 Subject: [PATCH 52/52] admin app: enhanced ticket support for gae --- applications/admin/controllers/default.py | 70 ++++++++++++----------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 00082613..2e616459 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -1093,11 +1093,12 @@ def design(): #Get crontab cronfolder = apath('%s/cron' % app, r=request) - if not os.path.exists(cronfolder): - os.mkdir(cronfolder) crontab = apath('%s/cron/crontab' % app, r=request) - if not os.path.exists(crontab): - safe_write(crontab, '#crontab') + if not is_gae: + if not os.path.exists(cronfolder): + os.mkdir(cronfolder) + if not os.path.exists(crontab): + safe_write(crontab, '#crontab') plugins = [] @@ -1505,8 +1506,11 @@ def errors(): import hashlib app = get_app() - - method = request.args(1) or 'new' if not is_gae else "dbnew" + if is_gae: + method = 'dbold' if ('old' in + (request.args(1) or '')) else 'dbnew' + else: + method = request.args(1) or 'new' db_ready = {} db_ready['status'] = get_ticket_storage(app) db_ready['errmessage'] = T( @@ -1573,32 +1577,30 @@ def errors(): for fn in tk_db(tk_table.id > 0).select(): try: error = pickle.loads(fn.ticket_data) - except AttributeError: + hash = hashlib.md5(error['traceback']).hexdigest() + + if hash in delete_hashes: + tk_db(tk_table.id == fn.id).delete() + tk_db.commit() + else: + try: + hash2error[hash]['count'] += 1 + except KeyError: + error_lines = error['traceback'].split("\n") + last_line = error_lines[-2] + error_causer = os.path.split(error['layer'])[1] + hash2error[hash] = dict(count=1, + pickel=error, causer=error_causer, + last_line=last_line, hash=hash, + ticket=fn.ticket_id) + except AttributeError, e: tk_db(tk_table.id == fn.id).delete() tk_db.commit() - hash = hashlib.md5(error['traceback']).hexdigest() - - if hash in delete_hashes: - tk_db(tk_table.id == fn.id).delete() - tk_db.commit() - else: - try: - hash2error['hash']['count'] += 1 - except KeyError: - error_lines = error['traceback'].split("\n") - last_line = error_lines[-2] - error_causer = os.path.split(error['layer'])[1] - hash2error[hash] = dict(count=1, pickel=error, - causer=error_causer, - last_line=last_line, - hash=hash, ticket=fn.ticket_id) - decorated = [(x['count'], x) for x in hash2error.values()] - decorated.sort(key=operator.itemgetter(0), reverse=True) - - return dict(errors=[x[1] for x in decorated], app=app, method=method, db_ready=db_ready) + return dict(errors=[x[1] for x in decorated], app=app, + method=method, db_ready=db_ready) elif method == 'dbold': tk_db, tk_table = get_ticket_storage(app) @@ -1606,16 +1608,18 @@ def errors(): if item[:7] == 'delete_': tk_db(tk_table.ticket_id == item[7:]).delete() tk_db.commit() - tickets_ = tk_db(tk_table.id > 0).select(tk_table.ticket_id, tk_table.created_datetime, orderby=~tk_table.created_datetime) + tickets_ = tk_db(tk_table.id > 0).select(tk_table.ticket_id, + tk_table.created_datetime, + orderby=~tk_table.created_datetime) tickets = [row.ticket_id for row in tickets_] - times = dict( - [(row.ticket_id, row.created_datetime) for row in tickets_]) - - return dict(app=app, tickets=tickets, method=method, times=times, db_ready=db_ready) + times = dict([(row.ticket_id, row.created_datetime) for + row in tickets_]) + return dict(app=app, tickets=tickets, method=method, + times=times, db_ready=db_ready) else: for item in request.vars: - # delete_all} rows doesn't contain any ticket + # delete_all rows doesn't contain any ticket # Remove anything else as requested if item[:7] == 'delete_' and (not item == "delete_all}"): os.unlink(apath('%s/errors/%s' % (app, item[7:]), r=request))