pep8 in scripts/*.py

This commit is contained in:
mdipierro committed 2012-10-19 14:22:42 -05:00
1 parent d053900032
commit 295084976b
24 files changed
+661 -600

No files matched your search

+1 -1
View File
@@ -1 +1 @@
Version 2.1.1 (2012-10-19 14:13:30) dev
Version 2.1.1 (2012-10-19 14:22:39) dev
+31 -21
View File
@@ -40,48 +40,58 @@ domain3.com /app3/defcon3
'''
if not config.strip():
try:
config_file = open('routes.conf','r')
config_file = open('routes.conf', 'r')
try:
config = config_file.read()
finally:
config_file.close()
except:
config=''
config = ''
def auto_in(apps):
routes = [
('/robots.txt','/welcome/static/robots.txt'),
('/favicon.ico','/welcome/static/favicon.ico'),
('/admin$anything','/admin$anything'),
]
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'): path = '/'+path
if path.endswith('/'): path = path[:-1]
('/robots.txt', '/welcome/static/robots.txt'),
('/favicon.ico', '/welcome/static/favicon.ico'),
('/admin$anything', '/admin$anything'),
]
for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'):
path = '/' + path
if path.endswith('/'):
path = path[:-1]
app = path.split('/')[1]
routes += [
('.*:https?://(.*\.)?%s:$method /' % domain,'%s' % path),
('.*:https?://(.*\.)?%s:$method /static/$anything' % domain,'/%s/static/$anything' % app),
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' % domain,'/%s/appadmin/$anything' % app),
('.*:https?://(.*\.)?%s:$method /$anything' % domain,'%s/$anything' % path),
]
('.*:https?://(.*\.)?%s:$method /' % domain, '%s' % path),
('.*:https?://(.*\.)?%s:$method /static/$anything' %
domain, '/%s/static/$anything' % app),
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' %
domain, '/%s/appadmin/$anything' % app),
('.*:https?://(.*\.)?%s:$method /$anything' %
domain, '%s/$anything' % path),
]
return routes
def auto_out(apps):
routes = []
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'): path = '/'+path
if path.endswith('/'): path = path[:-1]
for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'):
path = '/' + path
if path.endswith('/'):
path = path[:-1]
app = path.split('/')[1]
routes += [
('/%s/static/$anything' % app,'/static/$anything'),
('/%s/static/$anything' % app, '/static/$anything'),
('/%s/appadmin/$anything' % app, '/appadmin/$anything'),
('%s/$anything' % path, '/$anything'),
]
]
return routes
routes_in = auto_in(config)
routes_out = auto_out(config)
def __routes_doctest():
'''
Dummy function for doctesting autoroutes.py.
@@ -128,7 +138,8 @@ if __name__ == '__main__':
try:
import gluon.main
except ImportError:
import sys, os
import sys
import os
os.chdir(os.path.dirname(os.path.dirname(__file__)))
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import gluon.main
@@ -138,4 +149,3 @@ if __name__ == '__main__':
import doctest
doctest.testmod()
+4 -3
View File
@@ -5,11 +5,12 @@ import urllib2
n = int(sys.argv[1])
url = sys.argv[2]
headers = {"Accept-Language" : "en" }
headers = {"Accept-Language": "en"}
req = urllib2.Request(url, None, headers)
t0 = time.time()
for k in xrange(n):
data = urllib2.urlopen(req).read()
print (time.time()-t0)/n
if n==1: print data
print (time.time() - t0) / n
if n == 1:
print data
+12 -13
View File
@@ -8,26 +8,25 @@ filename = sys.argv[1]
datafile = open(filename, 'r')
try:
data = '\n'+datafile.read()
data = '\n' + datafile.read()
finally:
datafile.close()
SPACE = '\n ' if '-n' in sys.argv[1:] else ' '
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */',data)
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */', data)
data = re.compile('[ ]+').sub(' ', data)
data = re.compile('\s*{\s*').sub(' {'+SPACE, data)
data = re.compile('\s*;\s*').sub(';'+SPACE, data)
data = re.compile('\s*{\s*').sub(' {' + SPACE, data)
data = re.compile('\s*;\s*').sub(';' + SPACE, data)
data = re.compile(',\s*').sub(', ', data)
data = re.compile('\s*\*/\s*').sub('*/'+SPACE, data)
data = re.compile('\s*}\s*').sub(SPACE+'}\n', data)
data = re.compile('\s*\*/\s*').sub('*/' + SPACE, data)
data = re.compile('\s*}\s*').sub(SPACE + '}\n', data)
data = re.compile('\n\s*\n').sub('\n', data)
data = re.compile(';\s+/\*').sub('; /*',data)
data = re.compile('\*/\s+/\*').sub(' ',data)
data = re.compile(';\s+/\*').sub('; /*', data)
data = re.compile('\*/\s+/\*').sub(' ', data)
data = re.compile('[ ]+\n').sub('\n', data)
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/',re.DOTALL).sub(
'\n/*\g<a>*/\n',data)
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{',data)
data = data.replace('}','}\n')
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/', re.DOTALL).sub(
'\n/*\g<a>*/\n', data)
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{', data)
data = data.replace('}', '}\n')
print data
+41 -38
View File
@@ -1,51 +1,55 @@
import sys
import re
def cleancss(text):
text=re.compile('\s+').sub(' ', text)
text=re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
text=re.compile('\s*;\s*').sub(';\n ', text)
text=re.compile('\s*\{\s*').sub(' {\n ', text)
text=re.compile('\s*\}\s*').sub('\n}\n\n', text)
text = re.compile('\s+').sub(' ', text)
text = re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
text = re.compile('\s*;\s*').sub(';\n ', text)
text = re.compile('\s*\{\s*').sub(' {\n ', text)
text = re.compile('\s*\}\s*').sub('\n}\n\n', text)
return text
def cleanhtml(text):
text=text.lower()
r=re.compile('\<script.+?/script\>', re.DOTALL)
scripts=r.findall(text)
text=r.sub('<script />', text)
r=re.compile('\<style.+?/style\>', re.DOTALL)
styles=r.findall(text)
text=r.sub('<style />', text)
text=re.compile(
text = text.lower()
r = re.compile('\<script.+?/script\>', re.DOTALL)
scripts = r.findall(text)
text = r.sub('<script />', text)
r = re.compile('\<style.+?/style\>', re.DOTALL)
styles = r.findall(text)
text = r.sub('<style />', text)
text = re.compile(
'<(?P<tag>(input|meta|link|hr|br|img|param))(?P<any>[^\>]*)\s*(?<!/)>')\
.sub('<\g<tag>\g<any> />', text)
text=text.replace('\n', ' ')
text=text.replace('>', '>\n')
text=text.replace('<', '\n<')
text=re.compile('\s*\n\s*').sub('\n', text)
lines=text.split('\n')
(indent, newlines)=(0, [])
text = text.replace('\n', ' ')
text = text.replace('>', '>\n')
text = text.replace('<', '\n<')
text = re.compile('\s*\n\s*').sub('\n', text)
lines = text.split('\n')
(indent, newlines) = (0, [])
for line in lines:
if line[:2]=='</': indent=indent-1
newlines.append(indent*' '+line)
if not line[:2]=='</' and line[-1:]=='>' and \
not line[-2:] in ['/>', '->']: indent=indent+1
text='\n'.join(newlines)
text=re.compile('\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>',text)
text=re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>',text)
text=re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>',text)
text=re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>',text)
text=re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>',text)
text=re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>',text)
text=re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>',text)
text=re.compile('\n\s*\n').sub('\n',text)
if line[:2] == '</': indent = indent - 1
newlines.append(indent * ' ' + line)
if not line[:2] == '</' and line[-1:] == '>' and \
not line[-2:] in ['/>', '->']: indent = indent + 1
text = '\n'.join(newlines)
text = re.compile(
'\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>', text)
text = re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>', text)
text = re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>', text)
text = re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>', text)
text = re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>', text)
text = re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>', text)
text = re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>', text)
text = re.compile('\n\s*\n').sub('\n', text)
for script in scripts:
text=text.replace('<script />', script, 1)
text = text.replace('<script />', script, 1)
for style in styles:
text=text.replace('<style />', cleancss(style), 1)
text = text.replace('<style />', cleancss(style), 1)
return text
def read_file(filename):
f = open(filename, 'r')
try:
@@ -53,9 +57,8 @@ def read_file(filename):
finally:
f.close()
file=sys.argv[1]
if file[-4:]=='.css':
file = sys.argv[1]
if file[-4:] == '.css':
print cleancss(read_file(file))
if file[-5:]=='.html':
if file[-5:] == '.html':
print cleanhtml(read_file(file))
+9 -9
View File
@@ -1,17 +1,17 @@
import re
def cleanjs(text):
text = re.sub('\s*}\s*','\n}\n',text)
text = re.sub('\s*{\s*',' {\n',text)
text = re.sub('\s*;\s*',';\n',text)
text = re.sub('\s*,\s*',', ',text)
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*',' \g<a> ',text)
text = re.sub('\s*}\s*', '\n}\n', text)
text = re.sub('\s*{\s*', ' {\n', text)
text = re.sub('\s*;\s*', ';\n', text)
text = re.sub('\s*,\s*', ', ', text)
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*', ' \g<a> ', text)
lines = text.split('\n')
text=''
indent=0
text = ''
indent = 0
for line in lines:
rline=line.strip()
rline = line.strip()
if rline:
pass
return text
+6 -6
View File
@@ -22,7 +22,7 @@ Internet connection is required to perform the update.
OVERRIDE = [
('.pdb', 'chemical/x-pdb'),
('.xyz', 'chemical/x-pdb')
]
]
class MIMEParser(dict):
@@ -72,14 +72,15 @@ if __name__ == '__main__':
sys.stdout.write('Checking freedesktop.org database version:')
sys.stdout.flush()
try:
search = re.search('(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
search = re.search(
'(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
urllib.urlopen('http://www.freedesktop.org/wiki/Software/shared-mime-info').read())
url = search.group('url')
assert url != None
assert url is not None
nversion = search.group('version')
assert nversion != None
assert nversion is not None
ftype = search.group('type')
assert ftype != None
assert ftype is not None
sys.stdout.write('\t[OK] version %s\n' % nversion)
except:
sys.stdout.write('\t[ERROR] unknown version\n')
@@ -129,4 +130,3 @@ if __name__ == '__main__':
sys.stdout.write('\t\t\t[OK] done\n')
except Exception, e:
sys.stdout.write('\t\t\t[ERROR] %s\n' % e)
+311 -301
View File
@@ -1,9 +1,11 @@
import os,sys
import os
import sys
from collections import deque
import string
import argparse
import cStringIO,operator
import cStringIO
import operator
import cPickle as pickle
from collections import deque
import math
@@ -15,18 +17,19 @@ try:
except ImportError as err:
print('gluon path not found')
class refTable(object):
def __init__(self):
self.columns = None
self.rows = None
def getcolHeader(self,colHeader):
def getcolHeader(self, colHeader):
return "{0}".format(' | '.join([string.join(string.strip('**{0}**'.format(item)),
'') for item in colHeader]))
'') for item in colHeader]))
def wrapTable(self,rows, hasHeader=False, headerChar='-', delim=' | ', justify='left',
separateRows=False, prefix='', postfix='', wrapfunc=lambda x:x):
def wrapTable(
self, rows, hasHeader=False, headerChar='-', delim=' | ', justify='left',
separateRows=False, prefix='', postfix='', wrapfunc=lambda x: x):
def rowWrapper(row):
@@ -40,38 +43,36 @@ class refTable(object):
rowList gives like newRows but
formatted like [[w, x, y, z]]
---'''
rowList = [[substr or '' for substr in item] for item in map(None,*newRows)]
rowList = [[substr or '' for substr in item]
for item in map(None, *newRows)]
return rowList
logicalRows = [rowWrapper(row) for row in rows]
columns = map(None,*reduce(operator.add,logicalRows))
columns = map(None, *reduce(operator.add, logicalRows))
self.columns = columns
maxWidths = [max(\
[len(str\
(item)) for \
item in column]\
) for column \
in columns]
maxWidths = [max(
[len(str
(item)) for
item in column]
) for column
in columns]
rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) + \
len(delim)*(len(maxWidths)-1))
rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) +
len(delim) * (len(maxWidths) - 1))
justify = {'center'\
:str\
.center,
'right'\
:str\
.rjust,
'left'\
:str.\
ljust\
}[justify\
.lower(\
)]
justify = {'center': str
.center,
'right': str
.rjust,
'left': str.
ljust
}[justify
.lower(
)]
output=cStringIO.StringIO()
output = cStringIO.StringIO()
if separateRows:
print >> output, rowSeparator
@@ -79,191 +80,199 @@ class refTable(object):
for physicalRows in logicalRows:
for row in physicalRows:
print >> output,\
prefix + delim.join([\
justify(str(item),width) for (\
item,width) in zip(row,maxWidths)]\
) + postfix
prefix + delim.join([
justify(str(item), width) for (
item, width) in zip(row, maxWidths)]
) + postfix
if separateRows or hasHeader:
print >> output, rowSeparator; hasHeader=False
print >> output, rowSeparator
hasHeader = False
return output.getvalue()
def wrap_onspace(self,text,width):
return reduce(lambda line, word, width=width: '{0}{1}{2}'\
.format(line\
,' \n'[(len(\
line[line.rfind('\n'\
) + 1:]) + len(\
word.split('\n',1)[0]) >=\
width)],word),text.split(' '))
def wrap_onspace(self, text, width):
return reduce(lambda line, word, width=width: '{0}{1}{2}'
.format(line, ' \n'[(len(
line[line.rfind('\n'
) + 1:]) + len(
word.split('\n', 1)[0]) >=
width)], word), text.split(' '))
def wrap_onspace_strict(self,text,width):
wordRegex = re.compile(r'\S{'+str(width)+r',}')
return self.wrap_onspace(\
wordRegex.sub(\
lambda m: self.\
wrap_always(\
m.group(),width),text\
),width)
def wrap_onspace_strict(self, text, width):
wordRegex = re.compile(r'\S{' + str(width) + r',}')
return self.wrap_onspace(
wordRegex.sub(
lambda m: self.
wrap_always(
m.group(), width), text
), width)
def wrap_always(self, text, width):
return '\n'.join(
[text[width * i:width * (i + 1
)] for i in xrange(
int(math.ceil(1. * len(
text) / width)))])
def wrap_always(self,text,width):
return '\n'.join(\
[ text[width*i:width*(i+1\
)] for i in xrange(\
int(math.ceil(1.*len(\
text)/width))) ])
class tableHelper():
def __init__(self):
self.oTable = refTable()
def getAsRows(self,data):
return [row.strip().split(',') for row in data.splitlines()]
def getAsRows(self, data):
return [row.strip().split(',') for row in data.splitlines()]
def getTable_noWrap(self,data,header=None):
def getTable_noWrap(self, data, header=None):
rows = self.getAsRows(data)
if header is not None:hRows = [header]+rows
else:hRows = rows
if header is not None:
hRows = [header] + rows
else:
hRows = rows
table = self.oTable.wrapTable(hRows, hasHeader=True)
return table
def getTable_Wrap(self,data,wrapStyle,header=None,width=65):
def getTable_Wrap(self, data, wrapStyle, header=None, width=65):
wrapper = None
if len(wrapStyle) > 1:
rows = self.getAsRows(data)
if header is not None:hRows = [header]+rows
else:hRows = rows
if header is not None:
hRows = [header] + rows
else:
hRows = rows
for wrapper in (self.oTable.wrap_always,
self.oTable.wrap_onspace,
self.oTable.wrap_onspace_strict):
return self.oTable.wrapTable(hRows\
,hasHeader=True\
,separateRows=True\
,prefix='| '\
,postfix=' |'\
,wrapfunc\
=lambda x:\
wrapper(x,width))
self.oTable.wrap_onspace,
self.oTable.wrap_onspace_strict):
return self.oTable.wrapTable(hRows, hasHeader=True, separateRows=True, prefix='| ', postfix=' |', wrapfunc=lambda x:
wrapper(x, width))
else:
return self.getTable_noWrap(data,header)
return self.getTable_noWrap(data, header)
def getAsErrorTable(self,err):
return self.getTable_Wrap(err,None)
def getAsErrorTable(self, err):
return self.getTable_Wrap(err, None)
class console:
def __init__(self,prompt,banner=None):
self.prompt=prompt
self.banner=banner
self.commands={}
self.commandSort=[]
self.db=None
def __init__(self, prompt, banner=None):
self.prompt = prompt
self.banner = banner
self.commands = {}
self.commandSort = []
self.db = None
for i in dir(self):
if "cmd_"==i[:4]:
cmd=i.split("cmd_")[1].lower()
self.commands[cmd]=getattr(self,i)
try:self.commandSort.append((int(self\
.commands[cmd].__doc__.split(\
"|")[0]),cmd))
except:pass
if "cmd_" == i[:4]:
cmd = i.split("cmd_")[1].lower()
self.commands[cmd] = getattr(self, i)
try:
self.commandSort.append((int(self
.commands[cmd].__doc__.split(
"|")[0]), cmd))
except:
pass
self.commandSort.sort()
self.commandSort=[i[1] for i in self.commandSort]
self.commandSort = [i[1] for i in self.commandSort]
self.var_DEBUG=False
self.var_tableStyle=''
self.var_DEBUG = False
self.var_tableStyle = ''
self.configvars={}
self.configvars = {}
for i in dir(self):
if "var_"==i[:4]:
var=i.split("var_")[1]
self.configvars[var]=i
if "var_" == i[:4]:
var = i.split("var_")[1]
self.configvars[var] = i
def setBanner(self,banner):
self.banner=banner
def setBanner(self, banner):
self.banner = banner
def execCmd(self,db):
self.db=db
def execCmd(self, db):
self.db = db
print self.banner
while True:
try:
command=raw_input(self.prompt)
command = raw_input(self.prompt)
try:
self.execCommand(command)
except:
self.execute(command)
except KeyboardInterrupt:break
except EOFError:break
except Exception,a:self.printError (a)
except KeyboardInterrupt:
break
except EOFError:
break
except Exception, a:
self.printError(a)
print ("\r\n\r\nBye!...")
sys.exit(0)
def printError(self,err):
def printError(self, err):
sys.stderr.write("Error: {0}\r\n".format(str(err),))
if self.var_DEBUG:pass
if self.var_DEBUG:
pass
def execute(self,cmd):
def execute(self, cmd):
try:
if not '-table ' in cmd:
exec '{0}'.format(cmd)
else:
file=None
table=None
file = None
table = None
fields=[]
items=string.split(cmd,' ')
invalidParams=[]
table=self.getTable(items[1])
allowedParams=['fields','file']
fields = []
items = string.split(cmd, ' ')
invalidParams = []
table = self.getTable(items[1])
allowedParams = ['fields', 'file']
for i in items:
if '=' in i and not string.split(i,'=')[0] in allowedParams:
if '=' in i and not string.split(i, '=')[0] in allowedParams:
try:
invalidParams.append(i)
except Exception, err:
raise Exception, 'invalid parameter\n{0}'.format(i)
raise Exception('invalid parameter\n{0}'.format(i))
else:
if 'file=' in i:
file=os.path.abspath(string.strip(string.split(i,'=')[1]))
file = os.path.abspath(string.strip(string.split(
i, '=')[1]))
if 'fields=' in i:
for field in string.split(string.split(i,'=')[1],','):
for field in string.split(string.split(i, '=')[1], ','):
if field in self.db[table].fields:
fields.append(string.strip(field))
if len(invalidParams)>0:
print('the following parameter(s) is not valid\n{0}'.format(\
string.join(invalidParams,',')))
if len(invalidParams) > 0:
print('the following parameter(s) is not valid\n{0}'.format(
string.join(invalidParams, ',')))
else:
try:
self.cmd_table(table,file,fields)
self.cmd_table(table, file, fields)
except Exception, err:
print('could not generate table for table {0}\n{1}'\
.format(table,err))
print('could not generate table for table {0}\n{1}'
.format(table, err))
except Exception, err:
print('sorry, can not do that!\n{0}'.format(err))
def getTable(self,tbl):
def getTable(self, tbl):
for mTbl in db.tables:
if tbl in mTbl:
if mTbl.startswith(tbl):
return mTbl
def execCommand(self,cmd):
words=cmd.split(" ")
words=[i for i in words if i]
if not words:return
cmd,parameters=words[0].lower(),words[1:]
def execCommand(self, cmd):
words = cmd.split(" ")
words = [i for i in words if i]
if not words:
return
cmd, parameters = words[0].lower(), words[1:]
if not cmd in self.commands:
raise Exception("Command {0} not found. Try 'help'\r\n".format(cmd))
raise Exception(
"Command {0} not found. Try 'help'\r\n".format(cmd))
self.commands[cmd](*parameters)
'''---
DEFAULT COMMANDS (begins with cmd_)
---'''
def cmd_clear(self,numlines=100):
def cmd_clear(self, numlines=100):
"""-5|clear|clear the screen"""
if os.name == "posix":
'''---
@@ -279,9 +288,9 @@ class console:
'''---
Fallback for other operating systems.
---'''
print '\n'*numlines
print '\n' * numlines
def cmd_table(self,tbl,file=None,fields=[]):
def cmd_table(self, tbl, file=None, fields=[]):
"""-4|-table [TABLENAME] optional[file=None] [fields=None]|\
the default tableStyle is no_wrap - use the 'set x y' command to change the style\n\
style choices:
@@ -291,13 +300,13 @@ style choices:
\tno_wrap (value '')\n
\t the 2nd optional param is a path to a file where the table will be written
\t the 3rd optional param is a list of fields you want displayed\n"""
table=None
table = None
for mTbl in db.tables:
if tbl in mTbl:
if mTbl.startswith(tbl):
table=mTbl
table = mTbl
break
oTable=tableHelper()
oTable = tableHelper()
'''---
tablestyle:
wrap_always
@@ -305,98 +314,106 @@ style choices:
wrap_onspace_strict
or set set to "" for no wrapping
---'''
tableStyle=self.var_tableStyle
filedNotFound=[]
table_fields=None
if len(fields)==0:
table_fields=self.db[table].fields
tableStyle = self.var_tableStyle
filedNotFound = []
table_fields = None
if len(fields) == 0:
table_fields = self.db[table].fields
else:
table_fields=fields
table_fields = fields
for field in fields:
if not field in self.db[table].fields:
filedNotFound.append(field)
if len(filedNotFound)==0:
rows=self.db(self.db[table].id>0).select()
rows_data=[]
if len(filedNotFound) == 0:
rows = self.db(self.db[table].id > 0).select()
rows_data = []
for row in rows:
rowdata=[]
rowdata = []
for f in table_fields:
rowdata.append('{0}'.format(row[f]))
rows_data.append(string.join(rowdata,','))
data=string.join(rows_data,'\n')
dataTable=oTable.getTable_Wrap(data,tableStyle,table_fields)
print('TABLE {0}\n{1}'.format(table,dataTable))
if file!=None:
rows_data.append(string.join(rowdata, ','))
data = string.join(rows_data, '\n')
dataTable = oTable.getTable_Wrap(data, tableStyle, table_fields)
print('TABLE {0}\n{1}'.format(table, dataTable))
if file is not None:
try:
tail,head=os.path.split(file)
tail, head = os.path.split(file)
try:
os.makedirs(tail)
except:'do nothing, folders exist'
oFile=open(file,'w')
oFile.write('TABLE: {0}\n{1}'.format(table,dataTable))
except:
'do nothing, folders exist'
oFile = open(file, 'w')
oFile.write('TABLE: {0}\n{1}'.format(table, dataTable))
oFile.close()
print('{0} has been created and populated with all available data from table {1}\n'.format(file,table))
print('{0} has been created and populated with all available data from table {1}\n'.format(file, table))
except Exception, err:
print("EXCEPTION: could not create table {0}\n{1}".format(table,err))
print("EXCEPTION: could not create table {0}\n{1}".format(
table, err))
else:
print('the following fields are not valid [{0}]'.format(string.join(filedNotFound,',')))
print('the following fields are not valid [{0}]'.format(
string.join(filedNotFound, ',')))
def cmd_help(self,*args):
def cmd_help(self, *args):
'''-3|help|Show's help'''
alldata=[]
lengths=[]
alldata = []
lengths = []
for i in self.commandSort:alldata.append(\
self.commands[i].__doc__.split("|")[1:])
for i in self.commandSort:
alldata.append(
self.commands[i].__doc__.split("|")[1:])
for i in alldata:
if len(i) > len(lengths):
for j in range(len(i)\
-len(lengths)):
for j in range(len(i)
- len(lengths)):
lengths.append(0)
j=0
while j<len(i):
if len(i[j])>lengths[j]:
lengths[j]=len(i[j])
j+=1
j = 0
while j < len(i):
if len(i[j]) > lengths[j]:
lengths[j] = len(i[j])
j += 1
print ("-"*(lengths[0]+lengths[1]+4))
print ("-" * (lengths[0] + lengths[1] + 4))
for i in alldata:
print (("%-"+str(lengths[0])+"s - %-"+str(lengths[1])+"s") % (i[0],i[1]))
if len(i)>2:
for j in i[2:]:print (("%"+str(lengths[0]+9)+"s* %s") % (" ",j))
print (("%-" + str(lengths[0]) + "s - %-" + str(
lengths[1]) + "s") % (i[0], i[1]))
if len(i) > 2:
for j in i[2:]: print (("%" + str(lengths[
0] + 9) + "s* %s") % (" ", j))
print
def cmd_vars(self,*args):
def cmd_vars(self, *args):
'''-2|vars|Show variables'''
print ("variables\r\n"+"-"*79)
for i,j in self.configvars.items():
value=self.parfmt(repr(getattr(self,j)),52)
print ("| %20s | %52s |" % (i,value[0]))
for k in value[1:]:print ("| %20s | %52s |" % ("",k))
if len(value)>1:print("| %20s | %52s |" % ("",""))
print ("-"*79)
print ("variables\r\n" + "-" * 79)
for i, j in self.configvars.items():
value = self.parfmt(repr(getattr(self, j)), 52)
print ("| %20s | %52s |" % (i, value[0]))
for k in value[1:]: print ("| %20s | %52s |" % ("", k))
if len(value) > 1:
print("| %20s | %52s |" % ("", ""))
print ("-" * 79)
def parfmt(self,txt,width):
res=[]
pos=0
def parfmt(self, txt, width):
res = []
pos = 0
while True:
a=txt[pos:pos+width]
if not a:break
a = txt[pos:pos + width]
if not a:
break
res.append(a)
pos+=width
pos += width
return res
def cmd_set(self,*args):
def cmd_set(self, *args):
'''-1|set [variable_name] [value]|Set configuration variable value|Values are an expressions (100 | string.lower('ABC') | etc.'''
value=" ".join(args[1:])
value = " ".join(args[1:])
if args[0] not in self.configvars:
setattr(self,"var_{0}".format(args[0]),eval(value))
setattr(self,"var_{0}".format(args[0]),eval(value))
setattr(self, "var_{0}".format(args[0]), eval(value))
setattr(self, "var_{0}".format(args[0]), eval(value))
def cmd_clearscreen(self,numlines=50):
def cmd_clearscreen(self, numlines=50):
'''---Clear the console.
---'''
if os.name == "posix":
@@ -413,72 +430,78 @@ style choices:
'''---
Fallback for other operating systems.
---'''
print '\n'*numlines
print '\n' * numlines
class dalShell(console):
def __init__(self):
pass
def shell(self,db):
console.__init__(self,prompt=">>> ",banner='dal interactive shell')
def shell(self, db):
console.__init__(self, prompt=">>> ", banner='dal interactive shell')
self.execCmd(db)
class setCopyDB():
def __init__(self):
'''---
non source or target specific vars
---'''
self.strModel=None
self.dalPath=None
self.db=None
self.strModel = None
self.dalPath = None
self.db = None
'''---
source vars
---'''
self.sourceModel=None
self.sourceFolder=None
self.sourceConnectionString=None
self.sourcedbType=None
self.sourcedbName=None
self.sourceModel = None
self.sourceFolder = None
self.sourceConnectionString = None
self.sourcedbType = None
self.sourcedbName = None
'''---
target vars
---'''
self.targetdbType=None
self.targetdbName=None
self.targetModel=None
self.targetFolder=None
self.targetConnectionString=None
self.truncate=False
self.targetdbType = None
self.targetdbName = None
self.targetModel = None
self.targetFolder = None
self.targetConnectionString = None
self.truncate = False
def _getDal(self):
mDal=None
mDal = None
if self.dalPath is not None:
global DAL
sys.path.append(self.dalPath)
mDal=__import__('dal',globals={},locals={},fromlist=['DAL'],level=0)
DAL=mDal.DAL
mDal = __import__(
'dal', globals={}, locals={}, fromlist=['DAL'], level=0)
DAL = mDal.DAL
return mDal
def instDB(self,storageFolder,storageConnectionString,autoImport):
self.db=DAL(storageConnectionString,folder=os.path.abspath(storageFolder),auto_import=autoImport)
def instDB(self, storageFolder, storageConnectionString, autoImport):
self.db = DAL(storageConnectionString, folder=os.path.abspath(
storageFolder), auto_import=autoImport)
return self.db
def delete_DB_tables(self,storageFolder,storageType):
print 'delete_DB_tablesn\n\t{0}\n\t{1}'.format(storageFolder,storageType)
dataFiles=[storageType,"sql.log"]
def delete_DB_tables(self, storageFolder, storageType):
print 'delete_DB_tablesn\n\t{0}\n\t{1}'.format(
storageFolder, storageType)
dataFiles = [storageType, "sql.log"]
try:
for f in os.listdir(storageFolder):
if ".table" in f:
fTable="{0}/{1}".format(storageFolder,f)
fTable = "{0}/{1}".format(storageFolder, f)
os.remove(fTable)
print('deleted {0}'.format(fTable))
for dFile in dataFiles:
os.remove("{0}/{1}".format(storageFolder,dFile))
print('deleted {0}'.format("{0}/{1}".format(storageFolder,dFile)))
os.remove("{0}/{1}".format(storageFolder, dFile))
print('deleted {0}'.format(
"{0}/{1}".format(storageFolder, dFile)))
except Exception, errObj:
print(str(errObj))
def truncatetables(self,tables=[]):
if len(tables)!=0:
def truncatetables(self, tables=[]):
if len(tables) != 0:
try:
print 'table value: {0}'.format(tables)
for tbl in self.db.tables:
@@ -495,12 +518,14 @@ class setCopyDB():
print('EXCEPTION: {0}'.format(err))
def copyDB(self):
other_db=DAL("{0}://{1}".format(self.targetdbType,self.targetdbName),folder=self.targetFolder)
other_db = DAL("{0}://{1}".format(
self.targetdbType, self.targetdbName), folder=self.targetFolder)
print 'creating tables...'
for table in self.db:
other_db.define_table(table._tablename,*[field for field in table])
other_db.define_table(
table._tablename, *[field for field in table])
'''
should there be an option to truncAte target DB?
if yes, then change args to allow for choice
@@ -511,43 +536,38 @@ class setCopyDB():
'''
print 'exporting data...'
self.db.export_to_csv_file(open('tmp.sql','wb'))
self.db.export_to_csv_file(open('tmp.sql', 'wb'))
print 'importing data...'
other_db.import_from_csv_file(open('tmp.sql','rb'))
other_db.import_from_csv_file(open('tmp.sql', 'rb'))
other_db.commit()
print 'done!'
print 'Attention: do not run this program again or you end up with duplicate records'
def createfolderPath(self,folder):
def createfolderPath(self, folder):
try:
if folder!=None:os.makedirs(folder)
if folder is not None:
os.makedirs(folder)
except Exception, err:
pass
if __name__ == '__main__':
oCopy=setCopyDB()
db=None
targetDB=None
dbfolder=None
clean=False
model=None
truncate=False
oCopy = setCopyDB()
db = None
targetDB = None
dbfolder = None
clean = False
model = None
truncate = False
parser=argparse.ArgumentParser(description='\
parser = argparse.ArgumentParser(description='\
samplecmd line:\n\
-f ./blueLite/db_storage -i -y sqlite://storage.sqlite -Y sqlite://storage2.sqlite -d ./blueLite/pyUtils/sql/blueSQL -t True',
epilog = '')
reqGroup=parser.add_argument_group('Required arguments')
reqGroup.add_argument('-f','--sourceFolder'\
,required=True\
,help="path to the 'source' folder of the 'source' DB")
reqGroup.add_argument('-F','--targetFolder'\
,required=False\
,help="path to the 'target' folder of the 'target' DB")
reqGroup.add_argument('-y','--sourceConnectionString'\
,required=True\
,help="source db connection string ()\n\
epilog='')
reqGroup = parser.add_argument_group('Required arguments')
reqGroup.add_argument('-f', '--sourceFolder', required=True, help="path to the 'source' folder of the 'source' DB")
reqGroup.add_argument('-F', '--targetFolder', required=False, help="path to the 'target' folder of the 'target' DB")
reqGroup.add_argument('-y', '--sourceConnectionString', required=True, help="source db connection string ()\n\
------------------------------------------------\n\
\
sqlite://storage.db\n\
@@ -561,13 +581,10 @@ ingres://username:password@localhost/test\n\
informix://username:password@test\n\
\
------------------------------------------------")
reqGroup.add_argument('-Y','--targetConnectionString'\
,required=True\
,help="target db type (sqlite,mySql,etc.)")
autoImpGroup=parser.add_argument_group('optional args (auto_import)')
autoImpGroup.add_argument('-a','--autoimport'\
,required=False\
,help='set to True to bypass loading of the model')
reqGroup.add_argument('-Y', '--targetConnectionString', required=True,
help="target db type (sqlite,mySql,etc.)")
autoImpGroup = parser.add_argument_group('optional args (auto_import)')
autoImpGroup.add_argument('-a', '--autoimport', required=False, help='set to True to bypass loading of the model')
"""
@@ -584,66 +601,58 @@ informix://username:password@test\n\
"""
miscGroup = parser.add_argument_group('optional args/tasks')
miscGroup.add_argument('-i', '--interactive', required=False, action='store_true', help='run in interactive mode')
miscGroup.add_argument(
'-d', '--dal', required=False, help='path to dal.py')
miscGroup.add_argument('-t', '--truncate', choices=['True', 'False'], help='delete the records but *not* the table of the SOURCE DB')
miscGroup.add_argument('-b', '--tables', required=False, type=list, help='optional list (comma delimited) of SOURCE tables to truncate, defaults to all')
miscGroup.add_argument('-c', '--clean', required=False, help='delete the DB,tables and the log file, WARNING: this is unrecoverable')
miscGroup=parser.add_argument_group('optional args/tasks')
miscGroup.add_argument('-i','--interactive'\
,required=False\
,action='store_true'\
,help='run in interactive mode')
miscGroup.add_argument('-d','--dal'\
,required=False\
,help='path to dal.py')
miscGroup.add_argument('-t','--truncate'\
,choices=['True','False']\
,help='delete the records but *not* the table of the SOURCE DB')
miscGroup.add_argument('-b','--tables'\
,required=False\
,type=list\
,help='optional list (comma delimited) of SOURCE tables to truncate, defaults to all')
miscGroup.add_argument('-c','--clean'\
,required=False\
,help='delete the DB,tables and the log file, WARNING: this is unrecoverable')
args=parser.parse_args()
db=None
mDal=None
args = parser.parse_args()
db = None
mDal = None
try:
oCopy.sourceFolder=args.sourceFolder
oCopy.targetFolder=args.sourceFolder
sourceItems=string.split(args.sourceConnectionString,'://')
oCopy.sourcedbType=sourceItems[0]
oCopy.sourcedbName=sourceItems[1]
targetItems=string.split(args.targetConnectionString,'://')
oCopy.targetdbType=targetItems[0]
oCopy.targetdbName=targetItems[1]
oCopy.sourceFolder = args.sourceFolder
oCopy.targetFolder = args.sourceFolder
sourceItems = string.split(args.sourceConnectionString, '://')
oCopy.sourcedbType = sourceItems[0]
oCopy.sourcedbName = sourceItems[1]
targetItems = string.split(args.targetConnectionString, '://')
oCopy.targetdbType = targetItems[0]
oCopy.targetdbName = targetItems[1]
except Exception, err:
print('EXCEPTION: {0}'.format(err))
if args.dal:
try:
autoImport=True
if args.autoimport:autoImport=args.autoimport
autoImport = True
if args.autoimport:
autoImport = args.autoimport
#sif not DAL in globals:
#if not sys.path.__contains__():
oCopy.dalPath=args.dal
mDal=oCopy._getDal()
db=oCopy.instDB(args.sourceFolder,args.sourceConnectionString,autoImport)
oCopy.dalPath = args.dal
mDal = oCopy._getDal()
db = oCopy.instDB(args.sourceFolder, args.sourceConnectionString,
autoImport)
except Exception, err:
print('EXCEPTION: could not set DAL\n{0}'.format(err))
if args.truncate:
try:
if args.truncate:
if args.tables:tables=string.split(string.strip(args.tables),',')
else:oCopy.truncatetables([])
if args.tables:
tables = string.split(string.strip(args.tables), ',')
else:
oCopy.truncatetables([])
except Exception, err:
print('EXCEPTION: could not truncate tables\n{0}'.format(err))
try:
if args.clean:oCopy.delete_DB_tables(oCopy.targetFolder,oCopy.targetType)
if args.clean:
oCopy.delete_DB_tables(oCopy.targetFolder, oCopy.targetType)
except Exception, err:
print('EXCEPTION: could not clean db\n{0}'.format(err))
"""
*** goes with -m/-M options... removed for now
@@ -661,14 +670,15 @@ target model: {1}\n\
if args.sourceFolder:
try:
oCopy.sourceFolder=os.path.abspath(args.sourceFolder)
oCopy.sourceFolder = os.path.abspath(args.sourceFolder)
oCopy.createfolderPath(oCopy.sourceFolder)
except Exception, err:
print('EXCEPTION: could not create folder path\n{0}'.format(err))
else:oCopy.dbStorageFolder=os.path.abspath(os.getcwd())
else:
oCopy.dbStorageFolder = os.path.abspath(os.getcwd())
if args.targetFolder:
try:
oCopy.targetFolder=os.path.abspath(args.targetFolder)
oCopy.targetFolder = os.path.abspath(args.targetFolder)
oCopy.createfolderPath(oCopy.targetFolder)
except Exception, err:
print('EXCEPTION: could not create folder path\n{0}'.format(err))
@@ -678,5 +688,5 @@ target model: {1}\n\
except Exception, err:
print('EXCEPTION: could not make a copy of the database\n{0}'.format(err))
else:
s=dalShell()
s = dalShell()
s.shell(db)
+20 -14
View File
@@ -1,26 +1,32 @@
import sys, glob, os, shutil
name=sys.argv[1]
app=sys.argv[2]
dest=sys.argv[3]
a=glob.glob('applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app,name=name))
b=glob.glob('applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app,name=name))
import sys
import glob
import os
import shutil
name = sys.argv[1]
app = sys.argv[2]
dest = sys.argv[3]
a = glob.glob(
'applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app, name=name))
b = glob.glob(
'applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app, name=name))
for f in a:
print 'cp %s ...' % f,
shutil.copyfile(f,os.path.join('applications',dest,*f.split('/')[2:]))
shutil.copyfile(f, os.path.join('applications', dest, *f.split('/')[2:]))
print 'done'
for f in b:
print 'cp %s ...' % f,
path = f.split('/')
for i in range(3,len(path)):
try: os.mkdir(os.path.join('applications',dest,*path[2:i]))
except: pass
path = os.path.join('applications',dest,*f.split('/')[2:])
for i in range(3, len(path)):
try:
os.mkdir(os.path.join('applications', dest, *path[2:i]))
except:
pass
path = os.path.join('applications', dest, *f.split('/')[2:])
if os.path.isdir(f):
if not os.path.exists(path):
shutil.copytree(f,path)
shutil.copytree(f, path)
else:
shutil.copyfile(f,path)
shutil.copyfile(f, path)
print 'done'
+4 -5
View File
@@ -10,17 +10,16 @@ DB_URI = 'sqlite://sessions.sqlite'
EXPIRATION_MINUTES = 60
SLEEP_MINUTES = 5
while 1: # Infinite loop
now = time() # get current Unix timestamp
while 1: # Infinite loop
now = time() # get current Unix timestamp
for row in db().select(db.web2py_session_welcome.ALL):
t = row.modified_datetime
# Convert to a Unix timestamp
t = mktime(t.timetuple())+1e-6*t.microsecond
t = mktime(t.timetuple()) + 1e-6 * t.microsecond
if now - t > EXPIRATION_MINUTES * 60:
del db.web2py_session_welcome[row.id]
db.commit() # Write changes to database
db.commit() # Write changes to database
sleep(SLEEP_MINUTES * 60)
+7
View File
@@ -22,6 +22,7 @@ import getopt
import os.path
import sys
def main(argv):
"""Parse the arguments and start the main process."""
try:
@@ -40,16 +41,19 @@ def main(argv):
else:
exit_with_parsing_error()
def exit_with_parsing_error():
"""Report invalid arguments and usage."""
print("Invalid argument(s).")
usage()
sys.exit(2)
def usage():
"""Display the documentation"""
print(__doc__)
def get_dicts(dict_path1, dict_path2):
"""
Parse the dictionaries.
@@ -60,6 +64,7 @@ def get_dicts(dict_path1, dict_path2):
return eval(open(dict_path1).read()), eval(open(dict_path2).read())
def get_dict_names(dict1_path, dict2_path):
"""
Get the name of the dictionaries for the end user. Use the base name of the
@@ -76,6 +81,7 @@ def get_dict_names(dict1_path, dict2_path):
dict2_name = "dict2"
return dict1_name, dict2_name
def compare_dicts(dict1, dict2, dict1_name, dict2_name):
"""
Compare the two dictionaries. Print out the result.
@@ -101,6 +107,7 @@ def compare_dicts(dict1, dict2, dict1_name, dict2_name):
if not has_value_differences:
print " None"
def print_key_diff(key_diff, dict1_name, dict2_name):
"""
Prints the keys in the first dictionary and are in the second dictionary.
+49 -43
View File
@@ -24,45 +24,48 @@ import subprocess
import re
import sys
data_type_map = dict(
varchar = 'string',
int = 'integer',
integer = 'integer',
tinyint = 'integer',
smallint = 'integer',
mediumint = 'integer',
bigint = 'integer',
float = 'double',
double = 'double',
char = 'string',
decimal = 'integer',
date = 'date',
varchar='string',
int='integer',
integer='integer',
tinyint='integer',
smallint='integer',
mediumint='integer',
bigint='integer',
float='double',
double='double',
char='string',
decimal='integer',
date='date',
#year = 'date',
time = 'time',
timestamp = 'datetime',
datetime = 'datetime',
binary = 'blob',
blob = 'blob',
tinyblob = 'blob',
mediumblob = 'blob',
longblob = 'blob',
text = 'text',
tinytext = 'text',
mediumtext = 'text',
longtext = 'text',
)
time='time',
timestamp='datetime',
datetime='datetime',
binary='blob',
blob='blob',
tinyblob='blob',
mediumblob='blob',
longblob='blob',
text='text',
tinytext='text',
mediumtext='text',
longtext='text',
)
def mysql(database_name, username, password):
p = subprocess.Popen(['mysql',
'--user=%s' % username,
'--password=%s'% password,
'--password=%s' % password,
'--execute=show tables;',
database_name],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sql_showtables, stderr = p.communicate()
tables = [re.sub('\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')"%(username, password, database_name)
tables = [re.sub(
'\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')" % (
username, password, database_name)
legacy_db_table_web2py_code = []
for table_name in tables:
#get the sql create statement
@@ -71,37 +74,40 @@ def mysql(database_name, username, password):
'--password=%s' % password,
'--skip-add-drop-table',
'--no-data', database_name,
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
sql_create_stmnt,stderr = p.communicate()
if 'CREATE' in sql_create_stmnt:#check if the table exists
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sql_create_stmnt, stderr = p.communicate()
if 'CREATE' in sql_create_stmnt: # check if the table exists
#remove garbage lines from sql statement
sql_lines = sql_create_stmnt.split('\n')
sql_lines = [x for x in sql_lines if not(x.startswith('--') or x.startswith('/*') or x =='')]
sql_lines = [x for x in sql_lines if not(
x.startswith('--') or x.startswith('/*') or x == '')]
#generate the web2py code from the create statement
web2py_table_code = ''
table_name = re.search('CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
table_name = re.search(
'CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
fields = []
for line in sql_lines[1:-1]:
if re.search('KEY', line) or re.search('PRIMARY', line) or re.search(' ID', line) or line.startswith(')'):
continue
hit = re.search('(\S+)\s+(\S+)(,| )( .*)?', line)
if hit!=None:
if hit is not None:
name, d_type = hit.group(1), hit.group(2)
d_type = re.sub(r'(\w+)\(.*',r'\1',d_type)
name = re.sub('`','',name)
web2py_table_code += "\n Field('%s','%s'),"%(name,data_type_map[d_type])
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)"%(table_name,web2py_table_code)
d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)
name = re.sub('`', '', name)
web2py_table_code += "\n Field('%s','%s')," % (
name, data_type_map[d_type])
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)" % (table_name, web2py_table_code)
legacy_db_table_web2py_code.append(web2py_table_code)
#----------------------------------------
#write the legacy db to file
legacy_db_web2py_code = connection_string+"\n\n"
legacy_db_web2py_code += "\n\n#--------\n".join(legacy_db_table_web2py_code)
legacy_db_web2py_code = connection_string + "\n\n"
legacy_db_web2py_code += "\n\n#--------\n".join(
legacy_db_table_web2py_code)
return legacy_db_web2py_code
regex = re.compile('(.*?):(.*?)@(.*)')
if len(sys.argv)<2 or not regex.match(sys.argv[1]):
if len(sys.argv) < 2 or not regex.match(sys.argv[1]):
print 'USAGE:\n\n extract_mysql_models.py username:password@data_basename\n\n'
else:
m = regex.match(sys.argv[1])
print mysql(m.group(3),m.group(1),m.group(2))
print mysql(m.group(3), m.group(1), m.group(2))
+29 -27
View File
@@ -46,19 +46,21 @@ KWARGS = ('type', 'length', 'default', 'required', 'ondelete',
import sys
def query(conn, sql,*args):
def query(conn, sql, *args):
"Execute a SQL query and return rows as a list of dicts"
cur = conn.cursor()
ret = []
try:
if DEBUG: print >> sys.stderr, "QUERY: ", sql % args
if DEBUG:
print >> sys.stderr, "QUERY: ", sql % args
cur.execute(sql, args)
for row in cur:
dic = {}
for i, value in enumerate(row):
field = cur.description[i][0]
dic[field] = value
if DEBUG: print >> sys.stderr, "RET: ", dic
if DEBUG:
print >> sys.stderr, "RET: ", dic
ret.append(dic)
return ret
finally:
@@ -75,7 +77,8 @@ def get_tables(conn, schema=SCHEMA):
def get_fields(conn, table):
"Retrieve field list for a given table"
if DEBUG: print >> sys.stderr, "Processing TABLE", table
if DEBUG:
print >> sys.stderr, "Processing TABLE", table
rows = query(conn, """
SELECT column_name, data_type,
is_nullable,
@@ -90,13 +93,13 @@ def get_fields(conn, table):
def define_field(conn, table, field, pks):
"Determine field type, default value, references, etc."
f={}
f = {}
ref = references(conn, table, field['column_name'])
if ref:
f.update(ref)
elif field['column_default'] and \
field['column_default'].startswith("nextval") and \
field['column_name'] in pks:
field['column_default'].startswith("nextval") and \
field['column_name'] in pks:
# postgresql sequence (SERIAL) and primary key!
f['type'] = "'id'"
elif field['data_type'].startswith('character'):
@@ -109,7 +112,7 @@ def define_field(conn, table, field, pks):
f['type'] = "'boolean'"
elif field['data_type'] in ('integer', 'smallint', 'bigint'):
f['type'] = "'integer'"
elif field['data_type'] in ('double precision', 'real' ):
elif field['data_type'] in ('double precision', 'real'):
f['type'] = "'double'"
elif field['data_type'] in ('timestamp', 'timestamp without time zone'):
f['type'] = "'datetime'"
@@ -124,17 +127,17 @@ def define_field(conn, table, field, pks):
elif field['data_type'] in ('bytea', ):
f['type'] = "'blob'"
elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'):
f['type'] = "" # unsupported?
f['type'] = "" # unsupported?
else:
raise RuntimeError("Data Type not supported: %s " % str(field))
try:
if field['column_default']:
if field['column_default']=="now()":
if field['column_default'] == "now()":
d = "request.now"
elif field['column_default']=="true":
elif field['column_default'] == "true":
d = "True"
elif field['column_default']=="false":
elif field['column_default'] == "false":
d = "False"
else:
d = repr(eval(field['column_default']))
@@ -142,7 +145,8 @@ def define_field(conn, table, field, pks):
except (ValueError, SyntaxError):
pass
except Exception, e:
raise RuntimeError("Default unsupported '%s'" % field['column_default'])
raise RuntimeError(
"Default unsupported '%s'" % field['column_default'])
if not field['is_nullable']:
f['notnull'] = "True"
@@ -203,40 +207,40 @@ def references(conn, table, field):
AND information_schema.key_column_usage.column_name=%s
AND information_schema.table_constraints.constraint_type='FOREIGN KEY'
;""", table, field)
if len(rows1)==1:
if len(rows1) == 1:
rows2 = query(conn, """
SELECT table_name, column_name, *
FROM information_schema.constraint_column_usage
WHERE constraint_name=%s
""", rows1[0]['constraint_name'])
row = None
if len(rows2)>1:
row = rows2[int(rows1[0]['ordinal_position'])-1]
if len(rows2) > 1:
row = rows2[int(rows1[0]['ordinal_position']) - 1]
keyed = True
if len(rows2)==1:
if len(rows2) == 1:
row = rows2[0]
keyed = False
if row:
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
ref = {'type': "'reference %s.%s'" % (row['table_name'],
row['column_name'])}
else:
ref = {'type': "'reference %s'" % (row['table_name'],)}
if rows1[0]['delete_rule']!="NO ACTION":
if rows1[0]['delete_rule'] != "NO ACTION":
ref['ondelete'] = repr(rows1[0]['delete_rule'])
return ref
elif rows2:
raise RuntimeError("Unsupported foreign key reference: %s" %
str(rows2))
str(rows2))
elif rows1:
raise RuntimeError("Unsupported referential constraint: %s" %
str(rows1))
str(rows1))
def define_table(conn, table):
"Output single table definition"
fields = get_fields(conn, table)
fields = get_fields(conn, table)
pks = primarykeys(conn, table)
print "db.define_table('%s'," % (table, )
for field in fields:
@@ -244,11 +248,11 @@ def define_table(conn, table):
fdef = define_field(conn, table, field, pks)
if fname not in pks and is_unique(conn, table, field):
fdef['unique'] = "True"
if fdef['type']=="'id'" and fname in pks:
if fdef['type'] == "'id'" and fname in pks:
pks.pop(pks.index(fname))
print " Field('%s', %s)," % (fname,
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
if k in fdef and fdef[k]]))
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
if k in fdef and fdef[k]]))
if pks:
print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks])
print " migrate=migrate)"
@@ -280,5 +284,3 @@ if __name__ == "__main__":
)
# Start model code generation:
define_db(cnn, db, host, port, user, passwd)
+7 -4
View File
@@ -1,4 +1,6 @@
import sys, glob
import sys
import glob
def read_fileb(filename, mode='rb'):
f = open(filename, mode)
@@ -7,6 +9,7 @@ def read_fileb(filename, mode='rb'):
finally:
f.close()
def write_fileb(filename, value, mode='wb'):
f = open(filename, mode)
try:
@@ -18,7 +21,7 @@ for filename in glob.glob(sys.argv[1]):
data1 = read_fileb(filename)
write_fileb(filename + '.bak2', data1)
data2lines = read_fileb(filename).strip().split('\n')
data2 = '\n'.join([line.rstrip().replace('\t',' '*2) for line in data2lines])+'\n'
data2 = '\n'.join([line.rstrip(
).replace('\t', ' ' * 2) for line in data2lines]) + '\n'
write_fileb(filename, data2)
print filename, len(data1)-len(data2)
print filename, len(data1) - len(data2)
+28 -24
View File
@@ -5,10 +5,11 @@ import sys
import re
from BeautifulSoup import BeautifulSoup as BS
def head(styles):
title = '<title>{{=response.title or request.application}}</title>'
items = '\n'.join(["{{response.files.append(URL(request.application,'static','%s'))}}" % (style) for style in styles])
loc="""<style>
loc = """<style>
div.flash {
position: absolute;
float: right;
@@ -39,50 +40,54 @@ div.error {
border: 1px solid #666;
}
</style>"""
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title,items,loc)
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title, items, loc)
def content():
return """<div class="flash">{{=response.flash or ''}}</div>{{include}}"""
def process(folder):
indexfile = open(os.path.join(folder,'index.html'),'rb')
indexfile = open(os.path.join(folder, 'index.html'), 'rb')
try:
soup = BS(indexfile.read())
finally:
indexfile.close()
styles = [x['href'] for x in soup.findAll('link')]
soup.find('head').contents=BS(head(styles))
soup.find('head').contents = BS(head(styles))
try:
soup.find('h1').contents=BS('{{=response.title or request.application}}')
soup.find('h2').contents=BS("{{=response.subtitle or '=response.subtitle'}}")
soup.find(
'h1').contents = BS('{{=response.title or request.application}}')
soup.find('h2').contents = BS(
"{{=response.subtitle or '=response.subtitle'}}")
except:
pass
for match in (soup.find('div',id='menu'),
soup.find('div',{'class':'menu'}),
soup.find('div',id='nav'),
soup.find('div',{'class':'nav'})):
for match in (soup.find('div', id='menu'),
soup.find('div', {'class': 'menu'}),
soup.find('div', id='nav'),
soup.find('div', {'class': 'nav'})):
if match:
match.contents=BS('{{=MENU(response.menu)}}')
match.contents = BS('{{=MENU(response.menu)}}')
break
done=False
for match in (soup.find('div',id='content'),
soup.find('div',{'class':'content'}),
soup.find('div',id='main'),
soup.find('div',{'class':'main'})):
done = False
for match in (soup.find('div', id='content'),
soup.find('div', {'class': 'content'}),
soup.find('div', id='main'),
soup.find('div', {'class': 'main'})):
if match:
match.contents=BS(content())
done=True
match.contents = BS(content())
done = True
break
if done:
page = soup.prettify()
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*",re.MULTILINE)\
.sub("{{=response.flash or ''}}",page)
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*", re.MULTILINE)\
.sub("{{=response.flash or ''}}", page)
print page
else:
raise Exception, "Unable to convert"
raise Exception("Unable to convert")
if __name__=='__main__':
if len(sys.argv)<2:
if __name__ == '__main__':
if len(sys.argv) < 2:
print """USAGE:
1) start a new web2py application
2) Download a sample free layout from the web into the static/ folder of
@@ -96,4 +101,3 @@ if __name__=='__main__':
print 'Folder %s does not exist' % sys.argv[1]
else:
process(sys.argv[1])
+29 -20
View File
@@ -38,12 +38,16 @@ gluon/contrib/pyrtf/
gluon/contrib/pysimplesoap/
"""
import sys, os, shutil, glob
import sys
import os
import shutil
import glob
def main():
global REQUIRED, IGNORED
if len(sys.argv)<2:
if len(sys.argv) < 2:
print USAGE
# make target folder
@@ -51,38 +55,43 @@ def main():
os.mkdir(target)
# change to os specificsep
REQUIRED = REQUIRED.replace('/',os.sep)
IGNORED = IGNORED.replace('/',os.sep)
REQUIRED = REQUIRED.replace('/', os.sep)
IGNORED = IGNORED.replace('/', os.sep)
# make a list of all files to include
files = [x.strip() for x in REQUIRED.split('\n') \
if x and not x[0]=='#']
ignore = [x.strip() for x in IGNORED.split('\n') \
if x and not x[0]=='#']
files = [x.strip() for x in REQUIRED.split('\n')
if x and not x[0] == '#']
ignore = [x.strip() for x in IGNORED.split('\n')
if x and not x[0] == '#']
def accept(filename):
for p in ignore:
if filename.startswith(p):
return False
return True
pattern = os.path.join('gluon','*.py')
pattern = os.path.join('gluon', '*.py')
while True:
newfiles = [x for x in glob.glob(pattern) if accept(x)]
if not newfiles: break
if not newfiles:
break
files += newfiles
pattern = os.path.join(pattern[:-3],'*.py')
pattern = os.path.join(pattern[:-3], '*.py')
# copy all files, make missing folder, build default.py
files.sort()
defaultpy = os.path.join('applications','welcome','controllers','default.py')
defaultpy = os.path.join(
'applications', 'welcome', 'controllers', 'default.py')
for f in files:
dirs = f.split(os.path.sep)
for i in range(1,len(dirs)):
try: os.mkdir(target+os.sep+os.path.join(*dirs[:i]))
except OSError: pass
if f==defaultpy:
open(os.path.join(target,f),'w').write('def index(): return "hello"\n')
for i in range(1, len(dirs)):
try:
os.mkdir(target + os.sep + os.path.join(*dirs[:i]))
except OSError:
pass
if f == defaultpy:
open(os.path.join(
target, f), 'w').write('def index(): return "hello"\n')
else:
shutil.copyfile(f,os.path.join(target,f))
shutil.copyfile(f, os.path.join(target, f))
if __name__=='__main__': main()
if __name__ == '__main__':
main()
+6 -6
View File
@@ -6,18 +6,18 @@ paths2 = []
while paths:
path = paths.pop()
for filename in os.listdir(path):
fullname = os.path.join(path,filename)
fullname = os.path.join(path, filename)
if os.path.isdir(fullname):
paths.append(fullname)
else:
extension = filename.split('.')[-1]
if extension.lower() in ('png','gif','jpg','jpeg','js','css'):
paths1.append((filename,fullname))
if extension.lower() in ('css','js','py','html'):
if extension.lower() in ('png', 'gif', 'jpg', 'jpeg', 'js', 'css'):
paths1.append((filename, fullname))
if extension.lower() in ('css', 'js', 'py', 'html'):
paths2.append(fullname)
for filename,fullname in paths1:
for filename, fullname in paths1:
for otherfullname in paths2:
if open(otherfullname).read().find(filename)>=0:
if open(otherfullname).read().find(filename) >= 0:
break
else:
print fullname
+17 -17
View File
@@ -162,7 +162,7 @@ class SessionFile(object):
def last_visit_default(self):
return datetime.datetime.fromtimestamp(
os.stat(self.filename)[stat.ST_MTIME])
os.stat(self.filename)[stat.ST_MTIME])
def __str__(self):
return self.filename
@@ -175,7 +175,7 @@ def total_seconds(delta):
Args:
delta: datetime.timedelta instance.
"""
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) * \
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) *
10 ** 6) / 10 ** 6
@@ -186,25 +186,25 @@ def main():
parser = OptionParser(usage=usage)
parser.add_option('-f', '--force',
action='store_true', dest='force', default=False,
help=('Ignore session expiration. '
'Force expiry based on -x option or auth.settings.expiration.')
)
action='store_true', dest='force', default=False,
help=('Ignore session expiration. '
'Force expiry based on -x option or auth.settings.expiration.')
)
parser.add_option('-o', '--once',
action='store_true', dest='once', default=False,
help='Delete sessions, then exit.',
)
action='store_true', dest='once', default=False,
help='Delete sessions, then exit.',
)
parser.add_option('-s', '--sleep',
dest='sleep', default=SLEEP_MINUTES * 60, type="int",
help='Number of seconds to sleep between executions. Default 300.',
)
dest='sleep', default=SLEEP_MINUTES * 60, type="int",
help='Number of seconds to sleep between executions. Default 300.',
)
parser.add_option('-v', '--verbose',
default=0, action='count',
help="print verbose output, a second -v increases verbosity")
default=0, action='count',
help="print verbose output, a second -v increases verbosity")
parser.add_option('-x', '--expiration',
dest='expiration', default=None, type="int",
help='Expiration value for sessions without expiration (in seconds)',
)
dest='expiration', default=None, type="int",
help='Expiration value for sessions without expiration (in seconds)',
)
(options, unused_args) = parser.parse_args()
+30 -30
View File
@@ -30,35 +30,35 @@ if sys.platform == 'win32':
base_modules.remove('macpath')
buildOptions = dict(
compressed = True,
excludes = ["macpath","PyQt4"],
includes = base_modules,
include_files=[
'applications',
'ABOUT',
'LICENSE',
'VERSION',
'logging.example.conf',
'options_std.py',
'app.example.yaml',
'queue.example.yaml',
],
# append any extra module by extending the list below -
# "contributed_modules+["lxml"]"
packages = contributed_modules,
)
compressed=True,
excludes=["macpath", "PyQt4"],
includes=base_modules,
include_files=[
'applications',
'ABOUT',
'LICENSE',
'VERSION',
'logging.example.conf',
'options_std.py',
'app.example.yaml',
'queue.example.yaml',
],
# append any extra module by extending the list below -
# "contributed_modules+["lxml"]"
packages=contributed_modules,
)
setup(
name = "Web2py",
version=web2py_version,
author="Massimo DiPierro",
description="web2py web framework",
license = "LGPL v3",
options = dict(build_exe = buildOptions),
executables = [Executable("web2py.py",
base=base,
compress = True,
icon = "web2py.ico",
targetName="web2py.exe",
copyDependentFiles = True)],
)
name="Web2py",
version=web2py_version,
author="Massimo DiPierro",
description="web2py web framework",
license="LGPL v3",
options=dict(build_exe=buildOptions),
executables=[Executable("web2py.py",
base=base,
compress=True,
icon="web2py.ico",
targetName="web2py.exe",
copyDependentFiles=True)],
)
+4 -4
View File
@@ -14,6 +14,7 @@ sys.path.insert(0, '.')
file = sys.argv[1]
apps = sys.argv[2:]
def sync_language(d, data):
''' this function makes sure a translated string will be prefered over an untranslated
string when syncing languages between apps. when both are translated, it prefers the
@@ -26,9 +27,9 @@ def sync_language(d, data):
d[key] = data[key]
# see if there is a translated string in the original list, but not in the new list
elif (
((d[key] != '') or (d[key] != key)) and
((data[key] == '') or (data[key] == key))
):
((d[key] != '') or (d[key] != key)) and
((data[key] == '') or (data[key] == key))
):
d[key] = d[key]
# any other case (wether there is or there isn't a translated string)
else:
@@ -68,4 +69,3 @@ for app in oapps:
path2 = 'applications/%s/' % app
file2 = os.path.join(path2, 'languages', '%s.py' % file)
shutil.copyfile(file1, file2)
+3 -3
View File
@@ -15,7 +15,7 @@ SLEEP_MINUTES = 5
errors_path = os.path.join(request.folder, 'errors')
try:
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r','').replace('\n','').strip()
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r', '').replace('\n', '').strip()
except:
db_string = 'sqlite://storage.db'
@@ -23,7 +23,8 @@ db_path = os.path.join(request.folder, 'databases')
tk_db = DAL(db_string, folder=db_path, auto_import=True)
ts = TicketStorage(db=tk_db)
tk_table = ts._get_table(db=tk_db, tablename=ts.tablename, app=request.application)
tk_table = ts._get_table(
db=tk_db, tablename=ts.tablename, app=request.application)
hashes = {}
@@ -48,4 +49,3 @@ while 1:
os.unlink(filename)
time.sleep(SLEEP_MINUTES * 60)
+2 -2
View File
@@ -42,8 +42,8 @@ while 1:
error = RestrictedError()
error.load(request, request.application, file)
mail.send(to=administrator_email, subject='new web2py ticket', message=error.traceback)
mail.send(to=administrator_email,
subject='new web2py ticket', message=error.traceback)
os.unlink(os.path.join(path, file))
time.sleep(SLEEP_MINUTES * 60)
+5 -5
View File
@@ -14,7 +14,7 @@ import os
import urllib
import zipfile
if len(sys.argv)>1 and sys.argv[1] == 'nightly':
if len(sys.argv) > 1 and sys.argv[1] == 'nightly':
version = 'http://web2py.com/examples/static/nightly/web2py_src.zip'
else:
version = 'http://web2py.com/examples/static/web2py_src.zip'
@@ -23,11 +23,11 @@ realpath = os.path.realpath(__file__)
path = os.path.dirname(os.path.dirname(os.path.dirname(realpath)))
os.chdir(path)
try:
old_version = open('web2py/VERSION','r').read().strip()
old_version = open('web2py/VERSION', 'r').read().strip()
except IOError:
old_version = ''
open(TMPFILENAME,'wb').write(urllib.urlopen(version).read())
open(TMPFILENAME, 'wb').write(urllib.urlopen(version).read())
new_version = zipfile.ZipFile(TMPFILENAME).read('web2py/VERSION').strip()
if new_version>old_version:
os.system('sudo -u %s unzip -o %s' % (USER,TMPFILENAME))
if new_version > old_version:
os.system('sudo -u %s unzip -o %s' % (USER, TMPFILENAME))
os.system('apachectl restart | apache2ctl restart')
+6 -4
View File
@@ -7,6 +7,7 @@
import os
import gzip
def zip_static(filelist=[]):
tsave = 0
for fi in filelist:
@@ -24,17 +25,18 @@ def zip_static(filelist=[]):
if zatime == atime and zmtime == mtime:
print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
continue
print 'gzipping %s to %s' % (os.path.basename(fi), os.path.basename(gfi))
print 'gzipping %s to %s' % (
os.path.basename(fi), os.path.basename(gfi))
f_in = open(fi, 'rb')
f_out = gzip.open(gfi, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.utime(gfi, (atime,mtime))
os.utime(gfi, (atime, mtime))
saved = fstats.st_size - os.stat(gfi).st_size
tsave+= saved
tsave += saved
print 'saved %s KB' % (int(tsave)/1000.0)
print 'saved %s KB' % (int(tsave) / 1000.0)
if __name__ == '__main__':
ALLOWED_EXTS = ['.css', '.js']