dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pep8 in scripts/*.py

Browse Source
This commit is contained in:
mdipierro
2012-10-19 14:22:42 -05:00
parent d053900032
commit 295084976b
24 changed files with 781 additions and 720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 2.1.1 (2012-10-19 14:13:30) dev
Version 2.1.1 (2012-10-19 14:22:39) dev
scripts/autoroutes.py
+31 -21
View File
@@ -40,48 +40,58 @@ domain3.com /app3/defcon3
'''
if not config.strip():
try:
config_file = open('routes.conf','r')
config_file = open('routes.conf', 'r')
try:
config = config_file.read()
finally:
config_file.close()
except:
config=''
config = ''
def auto_in(apps):
routes = [
('/robots.txt','/welcome/static/robots.txt'),
('/favicon.ico','/welcome/static/favicon.ico'),
('/admin$anything','/admin$anything'),
]
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'): path = '/'+path
if path.endswith('/'): path = path[:-1]
('/robots.txt', '/welcome/static/robots.txt'),
('/favicon.ico', '/welcome/static/favicon.ico'),
('/admin$anything', '/admin$anything'),
]
for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'):
path = '/' + path
if path.endswith('/'):
path = path[:-1]
app = path.split('/')[1]
routes += [
('.*:https?://(.*\.)?%s:$method /' % domain,'%s' % path),
('.*:https?://(.*\.)?%s:$method /static/$anything' % domain,'/%s/static/$anything' % app),
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' % domain,'/%s/appadmin/$anything' % app),
('.*:https?://(.*\.)?%s:$method /$anything' % domain,'%s/$anything' % path),
]
('.*:https?://(.*\.)?%s:$method /' % domain, '%s' % path),
('.*:https?://(.*\.)?%s:$method /static/$anything' %
domain, '/%s/static/$anything' % app),
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' %
domain, '/%s/appadmin/$anything' % app),
('.*:https?://(.*\.)?%s:$method /$anything' %
domain, '%s/$anything' % path),
]
return routes
def auto_out(apps):
routes = []
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'): path = '/'+path
if path.endswith('/'): path = path[:-1]
for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'):
path = '/' + path
if path.endswith('/'):
path = path[:-1]
app = path.split('/')[1]
routes += [
('/%s/static/$anything' % app,'/static/$anything'),
('/%s/static/$anything' % app, '/static/$anything'),
('/%s/appadmin/$anything' % app, '/appadmin/$anything'),
('%s/$anything' % path, '/$anything'),
]
]
return routes
routes_in = auto_in(config)
routes_out = auto_out(config)
def __routes_doctest():
'''
Dummy function for doctesting autoroutes.py.
@@ -128,7 +138,8 @@ if __name__ == '__main__':
try:
import gluon.main
except ImportError:
import sys, os
import sys
import os
os.chdir(os.path.dirname(os.path.dirname(__file__)))
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import gluon.main
@@ -138,4 +149,3 @@ if __name__ == '__main__':
import doctest
doctest.testmod()
scripts/bench.py
+4 -3
View File
@@ -5,11 +5,12 @@ import urllib2
n = int(sys.argv[1])
url = sys.argv[2]
headers = {"Accept-Language" : "en" }
headers = {"Accept-Language": "en"}
req = urllib2.Request(url, None, headers)
t0 = time.time()
for k in xrange(n):
data = urllib2.urlopen(req).read()
print (time.time()-t0)/n
if n==1: print data
print (time.time() - t0) / n
if n == 1:
print data
scripts/cleancss.py
+12 -13
View File
@@ -8,26 +8,25 @@ filename = sys.argv[1]
datafile = open(filename, 'r')
try:
data = '\n'+datafile.read()
data = '\n' + datafile.read()
finally:
datafile.close()
SPACE = '\n ' if '-n' in sys.argv[1:] else ' '
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */',data)
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */', data)
data = re.compile('[ ]+').sub(' ', data)
data = re.compile('\s*{\s*').sub(' {'+SPACE, data)
data = re.compile('\s*;\s*').sub(';'+SPACE, data)
data = re.compile('\s*{\s*').sub(' {' + SPACE, data)
data = re.compile('\s*;\s*').sub(';' + SPACE, data)
data = re.compile(',\s*').sub(', ', data)
data = re.compile('\s*\*/\s*').sub('*/'+SPACE, data)
data = re.compile('\s*}\s*').sub(SPACE+'}\n', data)
data = re.compile('\s*\*/\s*').sub('*/' + SPACE, data)
data = re.compile('\s*}\s*').sub(SPACE + '}\n', data)
data = re.compile('\n\s*\n').sub('\n', data)
data = re.compile(';\s+/\*').sub('; /*',data)
data = re.compile('\*/\s+/\*').sub(' ',data)
data = re.compile(';\s+/\*').sub('; /*', data)
data = re.compile('\*/\s+/\*').sub(' ', data)
data = re.compile('[ ]+\n').sub('\n', data)
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/',re.DOTALL).sub(
'\n/*\g<a>*/\n',data)
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{',data)
data = data.replace('}','}\n')
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/', re.DOTALL).sub(
'\n/*\g<a>*/\n', data)
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{', data)
data = data.replace('}', '}\n')
print data
scripts/cleanhtml.py
+41 -38
View File
@@ -1,51 +1,55 @@
import sys
import re
def cleancss(text):
text=re.compile('\s+').sub(' ', text)
text=re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
text=re.compile('\s*;\s*').sub(';\n ', text)
text=re.compile('\s*\{\s*').sub(' {\n ', text)
text=re.compile('\s*\}\s*').sub('\n}\n\n', text)
text = re.compile('\s+').sub(' ', text)
text = re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
text = re.compile('\s*;\s*').sub(';\n ', text)
text = re.compile('\s*\{\s*').sub(' {\n ', text)
text = re.compile('\s*\}\s*').sub('\n}\n\n', text)
return text
def cleanhtml(text):
text=text.lower()
r=re.compile('\<script.+?/script\>', re.DOTALL)
scripts=r.findall(text)
text=r.sub('<script />', text)
r=re.compile('\<style.+?/style\>', re.DOTALL)
styles=r.findall(text)
text=r.sub('<style />', text)
text=re.compile(
text = text.lower()
r = re.compile('\<script.+?/script\>', re.DOTALL)
scripts = r.findall(text)
text = r.sub('<script />', text)
r = re.compile('\<style.+?/style\>', re.DOTALL)
styles = r.findall(text)
text = r.sub('<style />', text)
text = re.compile(
'<(?P<tag>(input|meta|link|hr|br|img|param))(?P<any>[^\>]*)\s*(?<!/)>')\
.sub('<\g<tag>\g<any> />', text)
text=text.replace('\n', ' ')
text=text.replace('>', '>\n')
text=text.replace('<', '\n<')
text=re.compile('\s*\n\s*').sub('\n', text)
lines=text.split('\n')
(indent, newlines)=(0, [])
text = text.replace('\n', ' ')
text = text.replace('>', '>\n')
text = text.replace('<', '\n<')
text = re.compile('\s*\n\s*').sub('\n', text)
lines = text.split('\n')
(indent, newlines) = (0, [])
for line in lines:
if line[:2]=='</': indent=indent-1
newlines.append(indent*' '+line)
if not line[:2]=='</' and line[-1:]=='>' and \
not line[-2:] in ['/>', '->']: indent=indent+1
text='\n'.join(newlines)
text=re.compile('\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>',text)
text=re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>',text)
text=re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>',text)
text=re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>',text)
text=re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>',text)
text=re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>',text)
text=re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>',text)
text=re.compile('\n\s*\n').sub('\n',text)
if line[:2] == '</': indent = indent - 1
newlines.append(indent * ' ' + line)
if not line[:2] == '</' and line[-1:] == '>' and \
not line[-2:] in ['/>', '->']: indent = indent + 1
text = '\n'.join(newlines)
text = re.compile(
'\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>', text)
text = re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>', text)
text = re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>', text)
text = re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>', text)
text = re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>', text)
text = re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>', text)
text = re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>', text)
text = re.compile('\n\s*\n').sub('\n', text)
for script in scripts:
text=text.replace('<script />', script, 1)
text = text.replace('<script />', script, 1)
for style in styles:
text=text.replace('<style />', cleancss(style), 1)
text = text.replace('<style />', cleancss(style), 1)
return text
def read_file(filename):
f = open(filename, 'r')
try:
@@ -53,9 +57,8 @@ def read_file(filename):
finally:
f.close()
file=sys.argv[1]
if file[-4:]=='.css':
file = sys.argv[1]
if file[-4:] == '.css':
print cleancss(read_file(file))
if file[-5:]=='.html':
if file[-5:] == '.html':
print cleanhtml(read_file(file))
scripts/cleanjs.py
+9 -9
View File
@@ -1,17 +1,17 @@
import re
def cleanjs(text):
text = re.sub('\s*}\s*','\n}\n',text)
text = re.sub('\s*{\s*',' {\n',text)
text = re.sub('\s*;\s*',';\n',text)
text = re.sub('\s*,\s*',', ',text)
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*',' \g<a> ',text)
text = re.sub('\s*}\s*', '\n}\n', text)
text = re.sub('\s*{\s*', ' {\n', text)
text = re.sub('\s*;\s*', ';\n', text)
text = re.sub('\s*,\s*', ', ', text)
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*', ' \g<a> ', text)
lines = text.split('\n')
text=''
indent=0
text = ''
indent = 0
for line in lines:
rline=line.strip()
rline = line.strip()
if rline:
pass
return text
scripts/contentparser.py
+6 -6
View File
@@ -22,7 +22,7 @@ Internet connection is required to perform the update.
OVERRIDE = [
('.pdb', 'chemical/x-pdb'),
('.xyz', 'chemical/x-pdb')
]
]
class MIMEParser(dict):
@@ -72,14 +72,15 @@ if __name__ == '__main__':
sys.stdout.write('Checking freedesktop.org database version:')
sys.stdout.flush()
try:
search = re.search('(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
search = re.search(
'(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
urllib.urlopen('http://www.freedesktop.org/wiki/Software/shared-mime-info').read())
url = search.group('url')
assert url != None
assert url is not None
nversion = search.group('version')
assert nversion != None
assert nversion is not None
ftype = search.group('type')
assert ftype != None
assert ftype is not None
sys.stdout.write('\t[OK] version %s\n' % nversion)
except:
sys.stdout.write('\t[ERROR] unknown version\n')
@@ -129,4 +130,3 @@ if __name__ == '__main__':
sys.stdout.write('\t\t\t[OK] done\n')
except Exception, e:
sys.stdout.write('\t\t\t[ERROR] %s\n' % e)
scripts/cpdb.py
+407 -397
View File
File diff suppressed because it is too large Load Diff
scripts/cpplugin.py
+20 -14
View File
@@ -1,26 +1,32 @@
import sys, glob, os, shutil
name=sys.argv[1]
app=sys.argv[2]
dest=sys.argv[3]
a=glob.glob('applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app,name=name))
b=glob.glob('applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app,name=name))
import sys
import glob
import os
import shutil
name = sys.argv[1]
app = sys.argv[2]
dest = sys.argv[3]
a = glob.glob(
'applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app, name=name))
b = glob.glob(
'applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app, name=name))
for f in a:
print 'cp %s ...' % f,
shutil.copyfile(f,os.path.join('applications',dest,*f.split('/')[2:]))
shutil.copyfile(f, os.path.join('applications', dest, *f.split('/')[2:]))
print 'done'
for f in b:
print 'cp %s ...' % f,
path = f.split('/')
for i in range(3,len(path)):
try: os.mkdir(os.path.join('applications',dest,*path[2:i]))
except: pass
path = os.path.join('applications',dest,*f.split('/')[2:])
for i in range(3, len(path)):
try:
os.mkdir(os.path.join('applications', dest, *path[2:i]))
except:
pass
path = os.path.join('applications', dest, *f.split('/')[2:])
if os.path.isdir(f):
if not os.path.exists(path):
shutil.copytree(f,path)
shutil.copytree(f, path)
else:
shutil.copyfile(f,path)
shutil.copyfile(f, path)
print 'done'
scripts/dbsessions2trash.py
+4 -5
View File
@@ -10,17 +10,16 @@ DB_URI = 'sqlite://sessions.sqlite'
EXPIRATION_MINUTES = 60
SLEEP_MINUTES = 5
while 1: # Infinite loop
now = time() # get current Unix timestamp
while 1: # Infinite loop
now = time() # get current Unix timestamp
for row in db().select(db.web2py_session_welcome.ALL):
t = row.modified_datetime
# Convert to a Unix timestamp
t = mktime(t.timetuple())+1e-6*t.microsecond
t = mktime(t.timetuple()) + 1e-6 * t.microsecond
if now - t > EXPIRATION_MINUTES * 60:
del db.web2py_session_welcome[row.id]
db.commit() # Write changes to database
db.commit() # Write changes to database
sleep(SLEEP_MINUTES * 60)
scripts/dict_diff.py
+17 -10
View File
@@ -6,11 +6,11 @@
@license: MIT
@since: 2011-06-17
Usage: dict_diff [OPTION]... dict1 dict2
Usage: dict_diff [OPTION]... dict1 dict2
Show the differences for two dictionaries.
-h, --help Display this help message.
dict1 and dict2 are two web2py dictionary files to compare. These are the files
located in the "languages" directory of a web2py app. The tools show the
differences between the two files.
@@ -22,34 +22,38 @@ import getopt
import os.path
import sys
def main(argv):
"""Parse the arguments and start the main process."""
try:
try:
opts, args = getopt.getopt(argv, "h", ["help"])
except getopt.GetoptError:
exit_with_parsing_error()
for opt, arg in opts:
arg = arg # To avoid a warning from Pydev
if opt in ("-h", "--help"):
usage()
usage()
sys.exit()
if len(args) == 2:
params = list(get_dicts(*args))
params.extend(get_dict_names(*args))
compare_dicts(*params)
else:
exit_with_parsing_error()
exit_with_parsing_error()
def exit_with_parsing_error():
def exit_with_parsing_error():
"""Report invalid arguments and usage."""
print("Invalid argument(s).")
usage()
sys.exit(2)
def usage():
"""Display the documentation"""
print(__doc__)
def get_dicts(dict_path1, dict_path2):
"""
Parse the dictionaries.
@@ -58,7 +62,8 @@ def get_dicts(dict_path1, dict_path2):
@return: The two dictionaries as a sequence.
"""
return eval(open(dict_path1).read()), eval(open(dict_path2).read())
return eval(open(dict_path1).read()), eval(open(dict_path2).read())
def get_dict_names(dict1_path, dict2_path):
"""
@@ -75,7 +80,8 @@ def get_dict_names(dict1_path, dict2_path):
dict1_name = "dict1"
dict2_name = "dict2"
return dict1_name, dict2_name
def compare_dicts(dict1, dict2, dict1_name, dict2_name):
"""
Compare the two dictionaries. Print out the result.
@@ -100,7 +106,8 @@ def compare_dicts(dict1, dict2, dict1_name, dict2_name):
has_value_differences = True
if not has_value_differences:
print " None"
def print_key_diff(key_diff, dict1_name, dict2_name):
"""
Prints the keys in the first dictionary and are in the second dictionary.
@@ -116,6 +123,6 @@ def print_key_diff(key_diff, dict1_name, dict2_name):
else:
print " None"
print
if __name__ == "__main__":
main(sys.argv[1:]) # Start the process (without the application name)
scripts/extract_mysql_models.py
+49 -43
View File
@@ -24,45 +24,48 @@ import subprocess
import re
import sys
data_type_map = dict(
varchar = 'string',
int = 'integer',
integer = 'integer',
tinyint = 'integer',
smallint = 'integer',
mediumint = 'integer',
bigint = 'integer',
float = 'double',
double = 'double',
char = 'string',
decimal = 'integer',
date = 'date',
varchar='string',
int='integer',
integer='integer',
tinyint='integer',
smallint='integer',
mediumint='integer',
bigint='integer',
float='double',
double='double',
char='string',
decimal='integer',
date='date',
#year = 'date',
time = 'time',
timestamp = 'datetime',
datetime = 'datetime',
binary = 'blob',
blob = 'blob',
tinyblob = 'blob',
mediumblob = 'blob',
longblob = 'blob',
text = 'text',
tinytext = 'text',
mediumtext = 'text',
longtext = 'text',
)
time='time',
timestamp='datetime',
datetime='datetime',
binary='blob',
blob='blob',
tinyblob='blob',
mediumblob='blob',
longblob='blob',
text='text',
tinytext='text',
mediumtext='text',
longtext='text',
)
def mysql(database_name, username, password):
p = subprocess.Popen(['mysql',
'--user=%s' % username,
'--password=%s'% password,
'--password=%s' % password,
'--execute=show tables;',
database_name],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sql_showtables, stderr = p.communicate()
tables = [re.sub('\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')"%(username, password, database_name)
tables = [re.sub(
'\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')" % (
username, password, database_name)
legacy_db_table_web2py_code = []
for table_name in tables:
#get the sql create statement
@@ -71,37 +74,40 @@ def mysql(database_name, username, password):
'--password=%s' % password,
'--skip-add-drop-table',
'--no-data', database_name,
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
sql_create_stmnt,stderr = p.communicate()
if 'CREATE' in sql_create_stmnt:#check if the table exists
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sql_create_stmnt, stderr = p.communicate()
if 'CREATE' in sql_create_stmnt: # check if the table exists
#remove garbage lines from sql statement
sql_lines = sql_create_stmnt.split('\n')
sql_lines = [x for x in sql_lines if not(x.startswith('--') or x.startswith('/*') or x =='')]
sql_lines = [x for x in sql_lines if not(
x.startswith('--') or x.startswith('/*') or x == '')]
#generate the web2py code from the create statement
web2py_table_code = ''
table_name = re.search('CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
table_name = re.search(
'CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
fields = []
for line in sql_lines[1:-1]:
if re.search('KEY', line) or re.search('PRIMARY', line) or re.search(' ID', line) or line.startswith(')'):
continue
hit = re.search('(\S+)\s+(\S+)(,| )( .*)?', line)
if hit!=None:
if hit is not None:
name, d_type = hit.group(1), hit.group(2)
d_type = re.sub(r'(\w+)\(.*',r'\1',d_type)
name = re.sub('`','',name)
web2py_table_code += "\n Field('%s','%s'),"%(name,data_type_map[d_type])
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)"%(table_name,web2py_table_code)
d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)
name = re.sub('`', '', name)
web2py_table_code += "\n Field('%s','%s')," % (
name, data_type_map[d_type])
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)" % (table_name, web2py_table_code)
legacy_db_table_web2py_code.append(web2py_table_code)
#----------------------------------------
#write the legacy db to file
legacy_db_web2py_code = connection_string+"\n\n"
legacy_db_web2py_code += "\n\n#--------\n".join(legacy_db_table_web2py_code)
legacy_db_web2py_code = connection_string + "\n\n"
legacy_db_web2py_code += "\n\n#--------\n".join(
legacy_db_table_web2py_code)
return legacy_db_web2py_code
regex = re.compile('(.*?):(.*?)@(.*)')
if len(sys.argv)<2 or not regex.match(sys.argv[1]):
if len(sys.argv) < 2 or not regex.match(sys.argv[1]):
print 'USAGE:\n\n extract_mysql_models.py username:password@data_basename\n\n'
else:
m = regex.match(sys.argv[1])
print mysql(m.group(3),m.group(1),m.group(2))
print mysql(m.group(3), m.group(1), m.group(2))
scripts/extract_pgsql_models.py
+29 -27
View File
@@ -46,19 +46,21 @@ KWARGS = ('type', 'length', 'default', 'required', 'ondelete',
import sys
def query(conn, sql,*args):
def query(conn, sql, *args):
"Execute a SQL query and return rows as a list of dicts"
cur = conn.cursor()
ret = []
try:
if DEBUG: print >> sys.stderr, "QUERY: ", sql % args
if DEBUG:
print >> sys.stderr, "QUERY: ", sql % args
cur.execute(sql, args)
for row in cur:
dic = {}
for i, value in enumerate(row):
field = cur.description[i][0]
dic[field] = value
if DEBUG: print >> sys.stderr, "RET: ", dic
if DEBUG:
print >> sys.stderr, "RET: ", dic
ret.append(dic)
return ret
finally:
@@ -75,7 +77,8 @@ def get_tables(conn, schema=SCHEMA):
def get_fields(conn, table):
"Retrieve field list for a given table"
if DEBUG: print >> sys.stderr, "Processing TABLE", table
if DEBUG:
print >> sys.stderr, "Processing TABLE", table
rows = query(conn, """
SELECT column_name, data_type,
is_nullable,
@@ -90,13 +93,13 @@ def get_fields(conn, table):
def define_field(conn, table, field, pks):
"Determine field type, default value, references, etc."
f={}
f = {}
ref = references(conn, table, field['column_name'])
if ref:
f.update(ref)
elif field['column_default'] and \
field['column_default'].startswith("nextval") and \
field['column_name'] in pks:
field['column_default'].startswith("nextval") and \
field['column_name'] in pks:
# postgresql sequence (SERIAL) and primary key!
f['type'] = "'id'"
elif field['data_type'].startswith('character'):
@@ -109,7 +112,7 @@ def define_field(conn, table, field, pks):
f['type'] = "'boolean'"
elif field['data_type'] in ('integer', 'smallint', 'bigint'):
f['type'] = "'integer'"
elif field['data_type'] in ('double precision', 'real' ):
elif field['data_type'] in ('double precision', 'real'):
f['type'] = "'double'"
elif field['data_type'] in ('timestamp', 'timestamp without time zone'):
f['type'] = "'datetime'"
@@ -124,17 +127,17 @@ def define_field(conn, table, field, pks):
elif field['data_type'] in ('bytea', ):
f['type'] = "'blob'"
elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'):
f['type'] = "" # unsupported?
f['type'] = "" # unsupported?
else:
raise RuntimeError("Data Type not supported: %s " % str(field))
try:
if field['column_default']:
if field['column_default']=="now()":
if field['column_default'] == "now()":
d = "request.now"
elif field['column_default']=="true":
elif field['column_default'] == "true":
d = "True"
elif field['column_default']=="false":
elif field['column_default'] == "false":
d = "False"
else:
d = repr(eval(field['column_default']))
@@ -142,7 +145,8 @@ def define_field(conn, table, field, pks):
except (ValueError, SyntaxError):
pass
except Exception, e:
raise RuntimeError("Default unsupported '%s'" % field['column_default'])
raise RuntimeError(
"Default unsupported '%s'" % field['column_default'])
if not field['is_nullable']:
f['notnull'] = "True"
@@ -203,40 +207,40 @@ def references(conn, table, field):
AND information_schema.key_column_usage.column_name=%s
AND information_schema.table_constraints.constraint_type='FOREIGN KEY'
;""", table, field)
if len(rows1)==1:
if len(rows1) == 1:
rows2 = query(conn, """
SELECT table_name, column_name, *
FROM information_schema.constraint_column_usage
WHERE constraint_name=%s
""", rows1[0]['constraint_name'])
row = None
if len(rows2)>1:
row = rows2[int(rows1[0]['ordinal_position'])-1]
if len(rows2) > 1:
row = rows2[int(rows1[0]['ordinal_position']) - 1]
keyed = True
if len(rows2)==1:
if len(rows2) == 1:
row = rows2[0]
keyed = False
if row:
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
ref = {'type': "'reference %s.%s'" % (row['table_name'],
row['column_name'])}
else:
ref = {'type': "'reference %s'" % (row['table_name'],)}
if rows1[0]['delete_rule']!="NO ACTION":
if rows1[0]['delete_rule'] != "NO ACTION":
ref['ondelete'] = repr(rows1[0]['delete_rule'])
return ref
elif rows2:
raise RuntimeError("Unsupported foreign key reference: %s" %
str(rows2))
str(rows2))
elif rows1:
raise RuntimeError("Unsupported referential constraint: %s" %
str(rows1))
str(rows1))
def define_table(conn, table):
"Output single table definition"
fields = get_fields(conn, table)
fields = get_fields(conn, table)
pks = primarykeys(conn, table)
print "db.define_table('%s'," % (table, )
for field in fields:
@@ -244,11 +248,11 @@ def define_table(conn, table):
fdef = define_field(conn, table, field, pks)
if fname not in pks and is_unique(conn, table, field):
fdef['unique'] = "True"
if fdef['type']=="'id'" and fname in pks:
if fdef['type'] == "'id'" and fname in pks:
pks.pop(pks.index(fname))
print " Field('%s', %s)," % (fname,
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
if k in fdef and fdef[k]]))
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
if k in fdef and fdef[k]]))
if pks:
print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks])
print " migrate=migrate)"
@@ -280,5 +284,3 @@ if __name__ == "__main__":
)
# Start model code generation:
define_db(cnn, db, host, port, user, passwd)
scripts/fixws.py
+7 -4
View File
@@ -1,4 +1,6 @@
import sys, glob
import sys
import glob
def read_fileb(filename, mode='rb'):
f = open(filename, mode)
@@ -7,6 +9,7 @@ def read_fileb(filename, mode='rb'):
finally:
f.close()
def write_fileb(filename, value, mode='wb'):
f = open(filename, mode)
try:
@@ -18,7 +21,7 @@ for filename in glob.glob(sys.argv[1]):
data1 = read_fileb(filename)
write_fileb(filename + '.bak2', data1)
data2lines = read_fileb(filename).strip().split('\n')
data2 = '\n'.join([line.rstrip().replace('\t',' '*2) for line in data2lines])+'\n'
data2 = '\n'.join([line.rstrip(
).replace('\t', ' ' * 2) for line in data2lines]) + '\n'
write_fileb(filename, data2)
print filename, len(data1)-len(data2)
print filename, len(data1) - len(data2)
scripts/layout_make.py
+28 -24
View File
@@ -5,10 +5,11 @@ import sys
import re
from BeautifulSoup import BeautifulSoup as BS
def head(styles):
title = '<title>{{=response.title or request.application}}</title>'
items = '\n'.join(["{{response.files.append(URL(request.application,'static','%s'))}}" % (style) for style in styles])
loc="""<style>
loc = """<style>
div.flash {
position: absolute;
float: right;
@@ -39,50 +40,54 @@ div.error {
border: 1px solid #666;
}
</style>"""
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title,items,loc)
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title, items, loc)
def content():
return """<div class="flash">{{=response.flash or ''}}</div>{{include}}"""
def process(folder):
indexfile = open(os.path.join(folder,'index.html'),'rb')
indexfile = open(os.path.join(folder, 'index.html'), 'rb')
try:
soup = BS(indexfile.read())
finally:
indexfile.close()
styles = [x['href'] for x in soup.findAll('link')]
soup.find('head').contents=BS(head(styles))
soup.find('head').contents = BS(head(styles))
try:
soup.find('h1').contents=BS('{{=response.title or request.application}}')
soup.find('h2').contents=BS("{{=response.subtitle or '=response.subtitle'}}")
soup.find(
'h1').contents = BS('{{=response.title or request.application}}')
soup.find('h2').contents = BS(
"{{=response.subtitle or '=response.subtitle'}}")
except:
pass
for match in (soup.find('div',id='menu'),
soup.find('div',{'class':'menu'}),
soup.find('div',id='nav'),
soup.find('div',{'class':'nav'})):
for match in (soup.find('div', id='menu'),
soup.find('div', {'class': 'menu'}),
soup.find('div', id='nav'),
soup.find('div', {'class': 'nav'})):
if match:
match.contents=BS('{{=MENU(response.menu)}}')
match.contents = BS('{{=MENU(response.menu)}}')
break
done=False
for match in (soup.find('div',id='content'),
soup.find('div',{'class':'content'}),
soup.find('div',id='main'),
soup.find('div',{'class':'main'})):
done = False
for match in (soup.find('div', id='content'),
soup.find('div', {'class': 'content'}),
soup.find('div', id='main'),
soup.find('div', {'class': 'main'})):
if match:
match.contents=BS(content())
done=True
match.contents = BS(content())
done = True
break
if done:
page = soup.prettify()
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*",re.MULTILINE)\
.sub("{{=response.flash or ''}}",page)
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*", re.MULTILINE)\
.sub("{{=response.flash or ''}}", page)
print page
else:
raise Exception, "Unable to convert"
raise Exception("Unable to convert")
if __name__=='__main__':
if len(sys.argv)<2:
if __name__ == '__main__':
if len(sys.argv) < 2:
print """USAGE:
1) start a new web2py application
2) Download a sample free layout from the web into the static/ folder of
@@ -96,4 +101,3 @@ if __name__=='__main__':
print 'Folder %s does not exist' % sys.argv[1]
else:
process(sys.argv[1])
scripts/make_min_web2py.py
+34 -25
View File
@@ -38,51 +38,60 @@ gluon/contrib/pyrtf/
gluon/contrib/pysimplesoap/
"""
import sys, os, shutil, glob
import sys
import os
import shutil
import glob
def main():
global REQUIRED, IGNORED
if len(sys.argv)<2:
if len(sys.argv) < 2:
print USAGE
# make target folder
target = sys.argv[1]
os.mkdir(target)
# change to os specificsep
REQUIRED = REQUIRED.replace('/',os.sep)
IGNORED = IGNORED.replace('/',os.sep)
REQUIRED = REQUIRED.replace('/', os.sep)
IGNORED = IGNORED.replace('/', os.sep)
# make a list of all files to include
files = [x.strip() for x in REQUIRED.split('\n') \
if x and not x[0]=='#']
ignore = [x.strip() for x in IGNORED.split('\n') \
if x and not x[0]=='#']
files = [x.strip() for x in REQUIRED.split('\n')
if x and not x[0] == '#']
ignore = [x.strip() for x in IGNORED.split('\n')
if x and not x[0] == '#']
def accept(filename):
for p in ignore:
if filename.startswith(p):
return False
return True
pattern = os.path.join('gluon','*.py')
pattern = os.path.join('gluon', '*.py')
while True:
newfiles = [x for x in glob.glob(pattern) if accept(x)]
if not newfiles: break
if not newfiles:
break
files += newfiles
pattern = os.path.join(pattern[:-3],'*.py')
pattern = os.path.join(pattern[:-3], '*.py')
# copy all files, make missing folder, build default.py
files.sort()
defaultpy = os.path.join('applications','welcome','controllers','default.py')
defaultpy = os.path.join(
'applications', 'welcome', 'controllers', 'default.py')
for f in files:
dirs = f.split(os.path.sep)
for i in range(1,len(dirs)):
try: os.mkdir(target+os.sep+os.path.join(*dirs[:i]))
except OSError: pass
if f==defaultpy:
open(os.path.join(target,f),'w').write('def index(): return "hello"\n')
for i in range(1, len(dirs)):
try:
os.mkdir(target + os.sep + os.path.join(*dirs[:i]))
except OSError:
pass
if f == defaultpy:
open(os.path.join(
target, f), 'w').write('def index(): return "hello"\n')
else:
shutil.copyfile(f,os.path.join(target,f))
if __name__=='__main__': main()
shutil.copyfile(f, os.path.join(target, f))
if __name__ == '__main__':
main()
scripts/rmorphans.py
+6 -6
View File
@@ -6,18 +6,18 @@ paths2 = []
while paths:
path = paths.pop()
for filename in os.listdir(path):
fullname = os.path.join(path,filename)
fullname = os.path.join(path, filename)
if os.path.isdir(fullname):
paths.append(fullname)
else:
extension = filename.split('.')[-1]
if extension.lower() in ('png','gif','jpg','jpeg','js','css'):
paths1.append((filename,fullname))
if extension.lower() in ('css','js','py','html'):
if extension.lower() in ('png', 'gif', 'jpg', 'jpeg', 'js', 'css'):
paths1.append((filename, fullname))
if extension.lower() in ('css', 'js', 'py', 'html'):
paths2.append(fullname)
for filename,fullname in paths1:
for filename, fullname in paths1:
for otherfullname in paths2:
if open(otherfullname).read().find(filename)>=0:
if open(otherfullname).read().find(filename) >= 0:
break
else:
print fullname
scripts/sessions2trash.py
+17 -17
View File
@@ -162,7 +162,7 @@ class SessionFile(object):
def last_visit_default(self):
return datetime.datetime.fromtimestamp(
os.stat(self.filename)[stat.ST_MTIME])
os.stat(self.filename)[stat.ST_MTIME])
def __str__(self):
return self.filename
@@ -175,7 +175,7 @@ def total_seconds(delta):
Args:
delta: datetime.timedelta instance.
"""
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) * \
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) *
10 ** 6) / 10 ** 6
@@ -186,25 +186,25 @@ def main():
parser = OptionParser(usage=usage)
parser.add_option('-f', '--force',
action='store_true', dest='force', default=False,
help=('Ignore session expiration. '
'Force expiry based on -x option or auth.settings.expiration.')
)
action='store_true', dest='force', default=False,
help=('Ignore session expiration. '
'Force expiry based on -x option or auth.settings.expiration.')
)
parser.add_option('-o', '--once',
action='store_true', dest='once', default=False,
help='Delete sessions, then exit.',
)
action='store_true', dest='once', default=False,
help='Delete sessions, then exit.',
)
parser.add_option('-s', '--sleep',
dest='sleep', default=SLEEP_MINUTES * 60, type="int",
help='Number of seconds to sleep between executions. Default 300.',
)
dest='sleep', default=SLEEP_MINUTES * 60, type="int",
help='Number of seconds to sleep between executions. Default 300.',
)
parser.add_option('-v', '--verbose',
default=0, action='count',
help="print verbose output, a second -v increases verbosity")
default=0, action='count',
help="print verbose output, a second -v increases verbosity")
parser.add_option('-x', '--expiration',
dest='expiration', default=None, type="int",
help='Expiration value for sessions without expiration (in seconds)',
)
dest='expiration', default=None, type="int",
help='Expiration value for sessions without expiration (in seconds)',
)
(options, unused_args) = parser.parse_args()
scripts/standalone_exe_cxfreeze.py
+30 -30
View File
@@ -30,35 +30,35 @@ if sys.platform == 'win32':
base_modules.remove('macpath')
buildOptions = dict(
compressed = True,
excludes = ["macpath","PyQt4"],
includes = base_modules,
include_files=[
'applications',
'ABOUT',
'LICENSE',
'VERSION',
'logging.example.conf',
'options_std.py',
'app.example.yaml',
'queue.example.yaml',
],
# append any extra module by extending the list below -
# "contributed_modules+["lxml"]"
packages = contributed_modules,
)
compressed=True,
excludes=["macpath", "PyQt4"],
includes=base_modules,
include_files=[
'applications',
'ABOUT',
'LICENSE',
'VERSION',
'logging.example.conf',
'options_std.py',
'app.example.yaml',
'queue.example.yaml',
],
# append any extra module by extending the list below -
# "contributed_modules+["lxml"]"
packages=contributed_modules,
)
setup(
name = "Web2py",
version=web2py_version,
author="Massimo DiPierro",
description="web2py web framework",
license = "LGPL v3",
options = dict(build_exe = buildOptions),
executables = [Executable("web2py.py",
base=base,
compress = True,
icon = "web2py.ico",
targetName="web2py.exe",
copyDependentFiles = True)],
)
name="Web2py",
version=web2py_version,
author="Massimo DiPierro",
description="web2py web framework",
license="LGPL v3",
options=dict(build_exe=buildOptions),
executables=[Executable("web2py.py",
base=base,
compress=True,
icon="web2py.ico",
targetName="web2py.exe",
copyDependentFiles=True)],
)
scripts/sync_languages.py
+10 -10
View File
@@ -14,26 +14,27 @@ sys.path.insert(0, '.')
file = sys.argv[1]
apps = sys.argv[2:]
def sync_language(d, data):
''' this function makes sure a translated string will be prefered over an untranslated
string when syncing languages between apps. when both are translated, it prefers the
''' this function makes sure a translated string will be prefered over an untranslated
string when syncing languages between apps. when both are translated, it prefers the
latter app, as did the original script
'''
for key in data:
# if this string is not in the allready translated data, add it
if key not in d:
d[key] = data[key]
# see if there is a translated string in the original list, but not in the new list
elif (
((d[key] != '') or (d[key] != key)) and
((data[key] == '') or (data[key] == key))
):
elif (
((d[key] != '') or (d[key] != key)) and
((data[key] == '') or (data[key] == key))
):
d[key] = d[key]
# any other case (wether there is or there isn't a translated string)
else:
d[key] = data[key]
return d
d = {}
@@ -45,7 +46,7 @@ for app in apps:
data = eval(langfile.read())
finally:
langfile.close()
d = sync_language(d, data)
path = 'applications/%s/' % apps[-1]
@@ -68,4 +69,3 @@ for app in oapps:
path2 = 'applications/%s/' % app
file2 = os.path.join(path2, 'languages', '%s.py' % file)
shutil.copyfile(file1, file2)
scripts/tickets2db.py
+4 -4
View File
@@ -15,7 +15,7 @@ SLEEP_MINUTES = 5
errors_path = os.path.join(request.folder, 'errors')
try:
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r','').replace('\n','').strip()
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r', '').replace('\n', '').strip()
except:
db_string = 'sqlite://storage.db'
@@ -23,7 +23,8 @@ db_path = os.path.join(request.folder, 'databases')
tk_db = DAL(db_string, folder=db_path, auto_import=True)
ts = TicketStorage(db=tk_db)
tk_table = ts._get_table(db=tk_db, tablename=ts.tablename, app=request.application)
tk_table = ts._get_table(
db=tk_db, tablename=ts.tablename, app=request.application)
hashes = {}
@@ -46,6 +47,5 @@ while 1:
)
tk_db.commit()
os.unlink(filename)
time.sleep(SLEEP_MINUTES * 60)
time.sleep(SLEEP_MINUTES * 60)
scripts/tickets2email.py
+2 -2
View File
@@ -42,8 +42,8 @@ while 1:
error = RestrictedError()
error.load(request, request.application, file)
mail.send(to=administrator_email, subject='new web2py ticket', message=error.traceback)
mail.send(to=administrator_email,
subject='new web2py ticket', message=error.traceback)
os.unlink(os.path.join(path, file))
time.sleep(SLEEP_MINUTES * 60)
scripts/update_web2py.py
+7 -7
View File
@@ -1,9 +1,9 @@
#!/usr/bin/env python
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
crontab -e
* 3 * * * root path/to/this/file
* 3 * * * root path/to/this/file
"""
USER = 'www-data'
@@ -14,7 +14,7 @@ import os
import urllib
import zipfile
if len(sys.argv)>1 and sys.argv[1] == 'nightly':
if len(sys.argv) > 1 and sys.argv[1] == 'nightly':
version = 'http://web2py.com/examples/static/nightly/web2py_src.zip'
else:
version = 'http://web2py.com/examples/static/web2py_src.zip'
@@ -23,11 +23,11 @@ realpath = os.path.realpath(__file__)
path = os.path.dirname(os.path.dirname(os.path.dirname(realpath)))
os.chdir(path)
try:
old_version = open('web2py/VERSION','r').read().strip()
old_version = open('web2py/VERSION', 'r').read().strip()
except IOError:
old_version = ''
open(TMPFILENAME,'wb').write(urllib.urlopen(version).read())
open(TMPFILENAME, 'wb').write(urllib.urlopen(version).read())
new_version = zipfile.ZipFile(TMPFILENAME).read('web2py/VERSION').strip()
if new_version>old_version:
os.system('sudo -u %s unzip -o %s' % (USER,TMPFILENAME))
if new_version > old_version:
os.system('sudo -u %s unzip -o %s' % (USER, TMPFILENAME))
os.system('apachectl restart | apache2ctl restart')
scripts/zip_static_files.py
+6 -4
View File
@@ -7,6 +7,7 @@
import os
import gzip
def zip_static(filelist=[]):
tsave = 0
for fi in filelist:
@@ -24,17 +25,18 @@ def zip_static(filelist=[]):
if zatime == atime and zmtime == mtime:
print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
continue
print 'gzipping %s to %s' % (os.path.basename(fi), os.path.basename(gfi))
print 'gzipping %s to %s' % (
os.path.basename(fi), os.path.basename(gfi))
f_in = open(fi, 'rb')
f_out = gzip.open(gfi, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.utime(gfi, (atime,mtime))
os.utime(gfi, (atime, mtime))
saved = fstats.st_size - os.stat(gfi).st_size
tsave+= saved
tsave += saved
print 'saved %s KB' % (int(tsave)/1000.0)
print 'saved %s KB' % (int(tsave) / 1000.0)
if __name__ == '__main__':
ALLOWED_EXTS = ['.css', '.js']