dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pep8 in scripts/*.py

Browse Source
This commit is contained in:
mdipierro
2012-10-19 14:22:42 -05:00
parent 17f495e9c5
commit 6cda7a29fc
24 changed files with 781 additions and 720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 2.1.1 (2012-10-19 14:13:30) dev Version 2.1.1 (2012-10-19 14:22:39) dev
scripts/autoroutes.py
+31 -21
View File
@@ -40,48 +40,58 @@ domain3.com /app3/defcon3
''' '''
if not config.strip(): if not config.strip():
try: try:
config_file = open('routes.conf','r') config_file = open('routes.conf', 'r')
try: try:
config = config_file.read() config = config_file.read()
finally: finally:
config_file.close() config_file.close()
except: except:
config='' config = ''
def auto_in(apps): def auto_in(apps):
routes = [ routes = [
('/robots.txt','/welcome/static/robots.txt'), ('/robots.txt', '/welcome/static/robots.txt'),
('/favicon.ico','/welcome/static/favicon.ico'), ('/favicon.ico', '/welcome/static/favicon.ico'),
('/admin$anything','/admin$anything'), ('/admin$anything', '/admin$anything'),
] ]
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]: for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'): path = '/'+path if not path.startswith('/'):
if path.endswith('/'): path = path[:-1] path = '/' + path
if path.endswith('/'):
path = path[:-1]
app = path.split('/')[1] app = path.split('/')[1]
routes += [ routes += [
('.*:https?://(.*\.)?%s:$method /' % domain,'%s' % path), ('.*:https?://(.*\.)?%s:$method /' % domain, '%s' % path),
('.*:https?://(.*\.)?%s:$method /static/$anything' % domain,'/%s/static/$anything' % app), ('.*:https?://(.*\.)?%s:$method /static/$anything' %
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' % domain,'/%s/appadmin/$anything' % app), domain, '/%s/static/$anything' % app),
('.*:https?://(.*\.)?%s:$method /$anything' % domain,'%s/$anything' % path), ('.*:https?://(.*\.)?%s:$method /appadmin/$anything' %
] domain, '/%s/appadmin/$anything' % app),
('.*:https?://(.*\.)?%s:$method /$anything' %
domain, '%s/$anything' % path),
]
return routes return routes
def auto_out(apps): def auto_out(apps):
routes = [] routes = []
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]: for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
if not path.startswith('/'): path = '/'+path if not path.startswith('/'):
if path.endswith('/'): path = path[:-1] path = '/' + path
if path.endswith('/'):
path = path[:-1]
app = path.split('/')[1] app = path.split('/')[1]
routes += [ routes += [
('/%s/static/$anything' % app,'/static/$anything'), ('/%s/static/$anything' % app, '/static/$anything'),
('/%s/appadmin/$anything' % app, '/appadmin/$anything'), ('/%s/appadmin/$anything' % app, '/appadmin/$anything'),
('%s/$anything' % path, '/$anything'), ('%s/$anything' % path, '/$anything'),
] ]
return routes return routes
routes_in = auto_in(config) routes_in = auto_in(config)
routes_out = auto_out(config) routes_out = auto_out(config)
def __routes_doctest(): def __routes_doctest():
''' '''
Dummy function for doctesting autoroutes.py. Dummy function for doctesting autoroutes.py.
@@ -128,7 +138,8 @@ if __name__ == '__main__':
try: try:
import gluon.main import gluon.main
except ImportError: except ImportError:
import sys, os import sys
import os
os.chdir(os.path.dirname(os.path.dirname(__file__))) os.chdir(os.path.dirname(os.path.dirname(__file__)))
sys.path.append(os.path.dirname(os.path.dirname(__file__))) sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import gluon.main import gluon.main
@@ -138,4 +149,3 @@ if __name__ == '__main__':
import doctest import doctest
doctest.testmod() doctest.testmod()
scripts/bench.py
+4 -3
View File
@@ -5,11 +5,12 @@ import urllib2
n = int(sys.argv[1]) n = int(sys.argv[1])
url = sys.argv[2] url = sys.argv[2]
headers = {"Accept-Language" : "en" } headers = {"Accept-Language": "en"}
req = urllib2.Request(url, None, headers) req = urllib2.Request(url, None, headers)
t0 = time.time() t0 = time.time()
for k in xrange(n): for k in xrange(n):
data = urllib2.urlopen(req).read() data = urllib2.urlopen(req).read()
print (time.time()-t0)/n print (time.time() - t0) / n
if n==1: print data if n == 1:
print data
scripts/cleancss.py
+12 -13
View File
@@ -8,26 +8,25 @@ filename = sys.argv[1]
datafile = open(filename, 'r') datafile = open(filename, 'r')
try: try:
data = '\n'+datafile.read() data = '\n' + datafile.read()
finally: finally:
datafile.close() datafile.close()
SPACE = '\n ' if '-n' in sys.argv[1:] else ' ' SPACE = '\n ' if '-n' in sys.argv[1:] else ' '
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */',data) data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */', data)
data = re.compile('[ ]+').sub(' ', data) data = re.compile('[ ]+').sub(' ', data)
data = re.compile('\s*{\s*').sub(' {'+SPACE, data) data = re.compile('\s*{\s*').sub(' {' + SPACE, data)
data = re.compile('\s*;\s*').sub(';'+SPACE, data) data = re.compile('\s*;\s*').sub(';' + SPACE, data)
data = re.compile(',\s*').sub(', ', data) data = re.compile(',\s*').sub(', ', data)
data = re.compile('\s*\*/\s*').sub('*/'+SPACE, data) data = re.compile('\s*\*/\s*').sub('*/' + SPACE, data)
data = re.compile('\s*}\s*').sub(SPACE+'}\n', data) data = re.compile('\s*}\s*').sub(SPACE + '}\n', data)
data = re.compile('\n\s*\n').sub('\n', data) data = re.compile('\n\s*\n').sub('\n', data)
data = re.compile(';\s+/\*').sub('; /*',data) data = re.compile(';\s+/\*').sub('; /*', data)
data = re.compile('\*/\s+/\*').sub(' ',data) data = re.compile('\*/\s+/\*').sub(' ', data)
data = re.compile('[ ]+\n').sub('\n', data) data = re.compile('[ ]+\n').sub('\n', data)
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/',re.DOTALL).sub( data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/', re.DOTALL).sub(
'\n/*\g<a>*/\n',data) '\n/*\g<a>*/\n', data)
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{',data) data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{', data)
data = data.replace('}','}\n') data = data.replace('}', '}\n')
print data print data
scripts/cleanhtml.py
+41 -38
View File
@@ -1,51 +1,55 @@
import sys import sys
import re import re
def cleancss(text): def cleancss(text):
text=re.compile('\s+').sub(' ', text) text = re.compile('\s+').sub(' ', text)
text=re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text) text = re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
text=re.compile('\s*;\s*').sub(';\n ', text) text = re.compile('\s*;\s*').sub(';\n ', text)
text=re.compile('\s*\{\s*').sub(' {\n ', text) text = re.compile('\s*\{\s*').sub(' {\n ', text)
text=re.compile('\s*\}\s*').sub('\n}\n\n', text) text = re.compile('\s*\}\s*').sub('\n}\n\n', text)
return text return text
def cleanhtml(text): def cleanhtml(text):
text=text.lower() text = text.lower()
r=re.compile('\<script.+?/script\>', re.DOTALL) r = re.compile('\<script.+?/script\>', re.DOTALL)
scripts=r.findall(text) scripts = r.findall(text)
text=r.sub('<script />', text) text = r.sub('<script />', text)
r=re.compile('\<style.+?/style\>', re.DOTALL) r = re.compile('\<style.+?/style\>', re.DOTALL)
styles=r.findall(text) styles = r.findall(text)
text=r.sub('<style />', text) text = r.sub('<style />', text)
text=re.compile( text = re.compile(
'<(?P<tag>(input|meta|link|hr|br|img|param))(?P<any>[^\>]*)\s*(?<!/)>')\ '<(?P<tag>(input|meta|link|hr|br|img|param))(?P<any>[^\>]*)\s*(?<!/)>')\
.sub('<\g<tag>\g<any> />', text) .sub('<\g<tag>\g<any> />', text)
text=text.replace('\n', ' ') text = text.replace('\n', ' ')
text=text.replace('>', '>\n') text = text.replace('>', '>\n')
text=text.replace('<', '\n<') text = text.replace('<', '\n<')
text=re.compile('\s*\n\s*').sub('\n', text) text = re.compile('\s*\n\s*').sub('\n', text)
lines=text.split('\n') lines = text.split('\n')
(indent, newlines)=(0, []) (indent, newlines) = (0, [])
for line in lines: for line in lines:
if line[:2]=='</': indent=indent-1 if line[:2] == '</': indent = indent - 1
newlines.append(indent*' '+line) newlines.append(indent * ' ' + line)
if not line[:2]=='</' and line[-1:]=='>' and \ if not line[:2] == '</' and line[-1:] == '>' and \
not line[-2:] in ['/>', '->']: indent=indent+1 not line[-2:] in ['/>', '->']: indent = indent + 1
text='\n'.join(newlines) text = '\n'.join(newlines)
text=re.compile('\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>',text) text = re.compile(
text=re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>',text) '\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>', text)
text=re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>',text) text = re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>', text)
text=re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>',text) text = re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>', text)
text=re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>',text) text = re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>', text)
text=re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>',text) text = re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>', text)
text=re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>',text) text = re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>', text)
text=re.compile('\n\s*\n').sub('\n',text) text = re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>', text)
text = re.compile('\n\s*\n').sub('\n', text)
for script in scripts: for script in scripts:
text=text.replace('<script />', script, 1) text = text.replace('<script />', script, 1)
for style in styles: for style in styles:
text=text.replace('<style />', cleancss(style), 1) text = text.replace('<style />', cleancss(style), 1)
return text return text
def read_file(filename): def read_file(filename):
f = open(filename, 'r') f = open(filename, 'r')
try: try:
@@ -53,9 +57,8 @@ def read_file(filename):
finally: finally:
f.close() f.close()
file=sys.argv[1] file = sys.argv[1]
if file[-4:]=='.css': if file[-4:] == '.css':
print cleancss(read_file(file)) print cleancss(read_file(file))
if file[-5:]=='.html': if file[-5:] == '.html':
print cleanhtml(read_file(file)) print cleanhtml(read_file(file))
scripts/cleanjs.py
+9 -9
View File
@@ -1,17 +1,17 @@
import re import re
def cleanjs(text): def cleanjs(text):
text = re.sub('\s*}\s*','\n}\n',text) text = re.sub('\s*}\s*', '\n}\n', text)
text = re.sub('\s*{\s*',' {\n',text) text = re.sub('\s*{\s*', ' {\n', text)
text = re.sub('\s*;\s*',';\n',text) text = re.sub('\s*;\s*', ';\n', text)
text = re.sub('\s*,\s*',', ',text) text = re.sub('\s*,\s*', ', ', text)
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*',' \g<a> ',text) text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*', ' \g<a> ', text)
lines = text.split('\n') lines = text.split('\n')
text='' text = ''
indent=0 indent = 0
for line in lines: for line in lines:
rline=line.strip() rline = line.strip()
if rline: if rline:
pass pass
return text return text
scripts/contentparser.py
+6 -6
View File
@@ -22,7 +22,7 @@ Internet connection is required to perform the update.
OVERRIDE = [ OVERRIDE = [
('.pdb', 'chemical/x-pdb'), ('.pdb', 'chemical/x-pdb'),
('.xyz', 'chemical/x-pdb') ('.xyz', 'chemical/x-pdb')
] ]
class MIMEParser(dict): class MIMEParser(dict):
@@ -72,14 +72,15 @@ if __name__ == '__main__':
sys.stdout.write('Checking freedesktop.org database version:') sys.stdout.write('Checking freedesktop.org database version:')
sys.stdout.flush() sys.stdout.flush()
try: try:
search = re.search('(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))', search = re.search(
'(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
urllib.urlopen('http://www.freedesktop.org/wiki/Software/shared-mime-info').read()) urllib.urlopen('http://www.freedesktop.org/wiki/Software/shared-mime-info').read())
url = search.group('url') url = search.group('url')
assert url != None assert url is not None
nversion = search.group('version') nversion = search.group('version')
assert nversion != None assert nversion is not None
ftype = search.group('type') ftype = search.group('type')
assert ftype != None assert ftype is not None
sys.stdout.write('\t[OK] version %s\n' % nversion) sys.stdout.write('\t[OK] version %s\n' % nversion)
except: except:
sys.stdout.write('\t[ERROR] unknown version\n') sys.stdout.write('\t[ERROR] unknown version\n')
@@ -129,4 +130,3 @@ if __name__ == '__main__':
sys.stdout.write('\t\t\t[OK] done\n') sys.stdout.write('\t\t\t[OK] done\n')
except Exception, e: except Exception, e:
sys.stdout.write('\t\t\t[ERROR] %s\n' % e) sys.stdout.write('\t\t\t[ERROR] %s\n' % e)
scripts/cpdb.py
+311 -301
View File
@@ -1,9 +1,11 @@
import os,sys import os
import sys
from collections import deque from collections import deque
import string import string
import argparse import argparse
import cStringIO,operator import cStringIO
import operator
import cPickle as pickle import cPickle as pickle
from collections import deque from collections import deque
import math import math
@@ -15,18 +17,19 @@ try:
except ImportError as err: except ImportError as err:
print('gluon path not found') print('gluon path not found')
class refTable(object): class refTable(object):
def __init__(self): def __init__(self):
self.columns = None self.columns = None
self.rows = None self.rows = None
def getcolHeader(self,colHeader): def getcolHeader(self, colHeader):
return "{0}".format(' | '.join([string.join(string.strip('**{0}**'.format(item)), return "{0}".format(' | '.join([string.join(string.strip('**{0}**'.format(item)),
'') for item in colHeader])) '') for item in colHeader]))
def wrapTable(
def wrapTable(self,rows, hasHeader=False, headerChar='-', delim=' | ', justify='left', self, rows, hasHeader=False, headerChar='-', delim=' | ', justify='left',
separateRows=False, prefix='', postfix='', wrapfunc=lambda x:x): separateRows=False, prefix='', postfix='', wrapfunc=lambda x: x):
def rowWrapper(row): def rowWrapper(row):
@@ -40,38 +43,36 @@ class refTable(object):
rowList gives like newRows but rowList gives like newRows but
formatted like [[w, x, y, z]] formatted like [[w, x, y, z]]
---''' ---'''
rowList = [[substr or '' for substr in item] for item in map(None,*newRows)] rowList = [[substr or '' for substr in item]
for item in map(None, *newRows)]
return rowList return rowList
logicalRows = [rowWrapper(row) for row in rows] logicalRows = [rowWrapper(row) for row in rows]
columns = map(None,*reduce(operator.add,logicalRows)) columns = map(None, *reduce(operator.add, logicalRows))
self.columns = columns self.columns = columns
maxWidths = [max(\ maxWidths = [max(
[len(str\ [len(str
(item)) for \ (item)) for
item in column]\ item in column]
) for column \ ) for column
in columns] in columns]
rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) + \ rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) +
len(delim)*(len(maxWidths)-1)) len(delim) * (len(maxWidths) - 1))
justify = {'center'\ justify = {'center': str
:str\ .center,
.center, 'right': str
'right'\ .rjust,
:str\ 'left': str.
.rjust, ljust
'left'\ }[justify
:str.\ .lower(
ljust\ )]
}[justify\
.lower(\
)]
output=cStringIO.StringIO() output = cStringIO.StringIO()
if separateRows: if separateRows:
print >> output, rowSeparator print >> output, rowSeparator
@@ -79,191 +80,199 @@ class refTable(object):
for physicalRows in logicalRows: for physicalRows in logicalRows:
for row in physicalRows: for row in physicalRows:
print >> output,\ print >> output,\
prefix + delim.join([\ prefix + delim.join([
justify(str(item),width) for (\ justify(str(item), width) for (
item,width) in zip(row,maxWidths)]\ item, width) in zip(row, maxWidths)]
) + postfix ) + postfix
if separateRows or hasHeader: if separateRows or hasHeader:
print >> output, rowSeparator; hasHeader=False print >> output, rowSeparator
hasHeader = False
return output.getvalue() return output.getvalue()
def wrap_onspace(self,text,width): def wrap_onspace(self, text, width):
return reduce(lambda line, word, width=width: '{0}{1}{2}'\ return reduce(lambda line, word, width=width: '{0}{1}{2}'
.format(line\ .format(line, ' \n'[(len(
,' \n'[(len(\ line[line.rfind('\n'
line[line.rfind('\n'\ ) + 1:]) + len(
) + 1:]) + len(\ word.split('\n', 1)[0]) >=
word.split('\n',1)[0]) >=\ width)], word), text.split(' '))
width)],word),text.split(' '))
def wrap_onspace_strict(self,text,width): def wrap_onspace_strict(self, text, width):
wordRegex = re.compile(r'\S{'+str(width)+r',}') wordRegex = re.compile(r'\S{' + str(width) + r',}')
return self.wrap_onspace(\ return self.wrap_onspace(
wordRegex.sub(\ wordRegex.sub(
lambda m: self.\ lambda m: self.
wrap_always(\ wrap_always(
m.group(),width),text\ m.group(), width), text
),width) ), width)
def wrap_always(self, text, width):
return '\n'.join(
[text[width * i:width * (i + 1
)] for i in xrange(
int(math.ceil(1. * len(
text) / width)))])
def wrap_always(self,text,width):
return '\n'.join(\
[ text[width*i:width*(i+1\
)] for i in xrange(\
int(math.ceil(1.*len(\
text)/width))) ])
class tableHelper(): class tableHelper():
def __init__(self): def __init__(self):
self.oTable = refTable() self.oTable = refTable()
def getAsRows(self,data): def getAsRows(self, data):
return [row.strip().split(',') for row in data.splitlines()] return [row.strip().split(',') for row in data.splitlines()]
def getTable_noWrap(self,data,header=None): def getTable_noWrap(self, data, header=None):
rows = self.getAsRows(data) rows = self.getAsRows(data)
if header is not None:hRows = [header]+rows if header is not None:
else:hRows = rows hRows = [header] + rows
else:
hRows = rows
table = self.oTable.wrapTable(hRows, hasHeader=True) table = self.oTable.wrapTable(hRows, hasHeader=True)
return table return table
def getTable_Wrap(self,data,wrapStyle,header=None,width=65): def getTable_Wrap(self, data, wrapStyle, header=None, width=65):
wrapper = None wrapper = None
if len(wrapStyle) > 1: if len(wrapStyle) > 1:
rows = self.getAsRows(data) rows = self.getAsRows(data)
if header is not None:hRows = [header]+rows if header is not None:
else:hRows = rows hRows = [header] + rows
else:
hRows = rows
for wrapper in (self.oTable.wrap_always, for wrapper in (self.oTable.wrap_always,
self.oTable.wrap_onspace, self.oTable.wrap_onspace,
self.oTable.wrap_onspace_strict): self.oTable.wrap_onspace_strict):
return self.oTable.wrapTable(hRows\ return self.oTable.wrapTable(hRows, hasHeader=True, separateRows=True, prefix='| ', postfix=' |', wrapfunc=lambda x:
,hasHeader=True\ wrapper(x, width))
,separateRows=True\
,prefix='| '\
,postfix=' |'\
,wrapfunc\
=lambda x:\
wrapper(x,width))
else: else:
return self.getTable_noWrap(data,header) return self.getTable_noWrap(data, header)
def getAsErrorTable(self,err): def getAsErrorTable(self, err):
return self.getTable_Wrap(err,None) return self.getTable_Wrap(err, None)
class console: class console:
def __init__(self,prompt,banner=None): def __init__(self, prompt, banner=None):
self.prompt=prompt self.prompt = prompt
self.banner=banner self.banner = banner
self.commands={} self.commands = {}
self.commandSort=[] self.commandSort = []
self.db=None self.db = None
for i in dir(self): for i in dir(self):
if "cmd_"==i[:4]: if "cmd_" == i[:4]:
cmd=i.split("cmd_")[1].lower() cmd = i.split("cmd_")[1].lower()
self.commands[cmd]=getattr(self,i) self.commands[cmd] = getattr(self, i)
try:self.commandSort.append((int(self\ try:
.commands[cmd].__doc__.split(\ self.commandSort.append((int(self
"|")[0]),cmd)) .commands[cmd].__doc__.split(
except:pass "|")[0]), cmd))
except:
pass
self.commandSort.sort() self.commandSort.sort()
self.commandSort=[i[1] for i in self.commandSort] self.commandSort = [i[1] for i in self.commandSort]
self.var_DEBUG=False self.var_DEBUG = False
self.var_tableStyle='' self.var_tableStyle = ''
self.configvars={} self.configvars = {}
for i in dir(self): for i in dir(self):
if "var_"==i[:4]: if "var_" == i[:4]:
var=i.split("var_")[1] var = i.split("var_")[1]
self.configvars[var]=i self.configvars[var] = i
def setBanner(self,banner): def setBanner(self, banner):
self.banner=banner self.banner = banner
def execCmd(self,db): def execCmd(self, db):
self.db=db self.db = db
print self.banner print self.banner
while True: while True:
try: try:
command=raw_input(self.prompt) command = raw_input(self.prompt)
try: try:
self.execCommand(command) self.execCommand(command)
except: except:
self.execute(command) self.execute(command)
except KeyboardInterrupt:break except KeyboardInterrupt:
except EOFError:break break
except Exception,a:self.printError (a) except EOFError:
break
except Exception, a:
self.printError(a)
print ("\r\n\r\nBye!...") print ("\r\n\r\nBye!...")
sys.exit(0) sys.exit(0)
def printError(self,err): def printError(self, err):
sys.stderr.write("Error: {0}\r\n".format(str(err),)) sys.stderr.write("Error: {0}\r\n".format(str(err),))
if self.var_DEBUG:pass if self.var_DEBUG:
pass
def execute(self,cmd): def execute(self, cmd):
try: try:
if not '-table ' in cmd: if not '-table ' in cmd:
exec '{0}'.format(cmd) exec '{0}'.format(cmd)
else: else:
file=None file = None
table=None table = None
fields=[] fields = []
items=string.split(cmd,' ') items = string.split(cmd, ' ')
invalidParams=[] invalidParams = []
table=self.getTable(items[1]) table = self.getTable(items[1])
allowedParams=['fields','file'] allowedParams = ['fields', 'file']
for i in items: for i in items:
if '=' in i and not string.split(i,'=')[0] in allowedParams: if '=' in i and not string.split(i, '=')[0] in allowedParams:
try: try:
invalidParams.append(i) invalidParams.append(i)
except Exception, err: except Exception, err:
raise Exception, 'invalid parameter\n{0}'.format(i) raise Exception('invalid parameter\n{0}'.format(i))
else: else:
if 'file=' in i: if 'file=' in i:
file=os.path.abspath(string.strip(string.split(i,'=')[1])) file = os.path.abspath(string.strip(string.split(
i, '=')[1]))
if 'fields=' in i: if 'fields=' in i:
for field in string.split(string.split(i,'=')[1],','): for field in string.split(string.split(i, '=')[1], ','):
if field in self.db[table].fields: if field in self.db[table].fields:
fields.append(string.strip(field)) fields.append(string.strip(field))
if len(invalidParams)>0: if len(invalidParams) > 0:
print('the following parameter(s) is not valid\n{0}'.format(\ print('the following parameter(s) is not valid\n{0}'.format(
string.join(invalidParams,','))) string.join(invalidParams, ',')))
else: else:
try: try:
self.cmd_table(table,file,fields) self.cmd_table(table, file, fields)
except Exception, err: except Exception, err:
print('could not generate table for table {0}\n{1}'\ print('could not generate table for table {0}\n{1}'
.format(table,err)) .format(table, err))
except Exception, err: except Exception, err:
print('sorry, can not do that!\n{0}'.format(err)) print('sorry, can not do that!\n{0}'.format(err))
def getTable(self,tbl): def getTable(self, tbl):
for mTbl in db.tables: for mTbl in db.tables:
if tbl in mTbl: if tbl in mTbl:
if mTbl.startswith(tbl): if mTbl.startswith(tbl):
return mTbl return mTbl
def execCommand(self,cmd): def execCommand(self, cmd):
words=cmd.split(" ") words = cmd.split(" ")
words=[i for i in words if i] words = [i for i in words if i]
if not words:return if not words:
cmd,parameters=words[0].lower(),words[1:] return
cmd, parameters = words[0].lower(), words[1:]
if not cmd in self.commands: if not cmd in self.commands:
raise Exception("Command {0} not found. Try 'help'\r\n".format(cmd)) raise Exception(
"Command {0} not found. Try 'help'\r\n".format(cmd))
self.commands[cmd](*parameters) self.commands[cmd](*parameters)
'''--- '''---
DEFAULT COMMANDS (begins with cmd_) DEFAULT COMMANDS (begins with cmd_)
---''' ---'''
def cmd_clear(self,numlines=100): def cmd_clear(self, numlines=100):
"""-5|clear|clear the screen""" """-5|clear|clear the screen"""
if os.name == "posix": if os.name == "posix":
'''--- '''---
@@ -279,9 +288,9 @@ class console:
'''--- '''---
Fallback for other operating systems. Fallback for other operating systems.
---''' ---'''
print '\n'*numlines print '\n' * numlines
def cmd_table(self,tbl,file=None,fields=[]): def cmd_table(self, tbl, file=None, fields=[]):
"""-4|-table [TABLENAME] optional[file=None] [fields=None]|\ """-4|-table [TABLENAME] optional[file=None] [fields=None]|\
the default tableStyle is no_wrap - use the 'set x y' command to change the style\n\ the default tableStyle is no_wrap - use the 'set x y' command to change the style\n\
style choices: style choices:
@@ -291,13 +300,13 @@ style choices:
\tno_wrap (value '')\n \tno_wrap (value '')\n
\t the 2nd optional param is a path to a file where the table will be written \t the 2nd optional param is a path to a file where the table will be written
\t the 3rd optional param is a list of fields you want displayed\n""" \t the 3rd optional param is a list of fields you want displayed\n"""
table=None table = None
for mTbl in db.tables: for mTbl in db.tables:
if tbl in mTbl: if tbl in mTbl:
if mTbl.startswith(tbl): if mTbl.startswith(tbl):
table=mTbl table = mTbl
break break
oTable=tableHelper() oTable = tableHelper()
'''--- '''---
tablestyle: tablestyle:
wrap_always wrap_always
@@ -305,98 +314,106 @@ style choices:
wrap_onspace_strict wrap_onspace_strict
or set set to "" for no wrapping or set set to "" for no wrapping
---''' ---'''
tableStyle=self.var_tableStyle tableStyle = self.var_tableStyle
filedNotFound=[] filedNotFound = []
table_fields=None table_fields = None
if len(fields)==0: if len(fields) == 0:
table_fields=self.db[table].fields table_fields = self.db[table].fields
else: else:
table_fields=fields table_fields = fields
for field in fields: for field in fields:
if not field in self.db[table].fields: if not field in self.db[table].fields:
filedNotFound.append(field) filedNotFound.append(field)
if len(filedNotFound)==0: if len(filedNotFound) == 0:
rows=self.db(self.db[table].id>0).select() rows = self.db(self.db[table].id > 0).select()
rows_data=[] rows_data = []
for row in rows: for row in rows:
rowdata=[] rowdata = []
for f in table_fields: for f in table_fields:
rowdata.append('{0}'.format(row[f])) rowdata.append('{0}'.format(row[f]))
rows_data.append(string.join(rowdata,',')) rows_data.append(string.join(rowdata, ','))
data=string.join(rows_data,'\n') data = string.join(rows_data, '\n')
dataTable=oTable.getTable_Wrap(data,tableStyle,table_fields) dataTable = oTable.getTable_Wrap(data, tableStyle, table_fields)
print('TABLE {0}\n{1}'.format(table,dataTable)) print('TABLE {0}\n{1}'.format(table, dataTable))
if file!=None: if file is not None:
try: try:
tail,head=os.path.split(file) tail, head = os.path.split(file)
try: try:
os.makedirs(tail) os.makedirs(tail)
except:'do nothing, folders exist' except:
oFile=open(file,'w') 'do nothing, folders exist'
oFile.write('TABLE: {0}\n{1}'.format(table,dataTable)) oFile = open(file, 'w')
oFile.write('TABLE: {0}\n{1}'.format(table, dataTable))
oFile.close() oFile.close()
print('{0} has been created and populated with all available data from table {1}\n'.format(file,table)) print('{0} has been created and populated with all available data from table {1}\n'.format(file, table))
except Exception, err: except Exception, err:
print("EXCEPTION: could not create table {0}\n{1}".format(table,err)) print("EXCEPTION: could not create table {0}\n{1}".format(
table, err))
else: else:
print('the following fields are not valid [{0}]'.format(string.join(filedNotFound,','))) print('the following fields are not valid [{0}]'.format(
string.join(filedNotFound, ',')))
def cmd_help(self,*args): def cmd_help(self, *args):
'''-3|help|Show's help''' '''-3|help|Show's help'''
alldata=[] alldata = []
lengths=[] lengths = []
for i in self.commandSort:alldata.append(\ for i in self.commandSort:
self.commands[i].__doc__.split("|")[1:]) alldata.append(
self.commands[i].__doc__.split("|")[1:])
for i in alldata: for i in alldata:
if len(i) > len(lengths): if len(i) > len(lengths):
for j in range(len(i)\ for j in range(len(i)
-len(lengths)): - len(lengths)):
lengths.append(0) lengths.append(0)
j=0 j = 0
while j<len(i): while j < len(i):
if len(i[j])>lengths[j]: if len(i[j]) > lengths[j]:
lengths[j]=len(i[j]) lengths[j] = len(i[j])
j+=1 j += 1
print ("-"*(lengths[0]+lengths[1]+4)) print ("-" * (lengths[0] + lengths[1] + 4))
for i in alldata: for i in alldata:
print (("%-"+str(lengths[0])+"s - %-"+str(lengths[1])+"s") % (i[0],i[1])) print (("%-" + str(lengths[0]) + "s - %-" + str(
if len(i)>2: lengths[1]) + "s") % (i[0], i[1]))
for j in i[2:]:print (("%"+str(lengths[0]+9)+"s* %s") % (" ",j)) if len(i) > 2:
for j in i[2:]: print (("%" + str(lengths[
0] + 9) + "s* %s") % (" ", j))
print print
def cmd_vars(self,*args): def cmd_vars(self, *args):
'''-2|vars|Show variables''' '''-2|vars|Show variables'''
print ("variables\r\n"+"-"*79) print ("variables\r\n" + "-" * 79)
for i,j in self.configvars.items(): for i, j in self.configvars.items():
value=self.parfmt(repr(getattr(self,j)),52) value = self.parfmt(repr(getattr(self, j)), 52)
print ("| %20s | %52s |" % (i,value[0])) print ("| %20s | %52s |" % (i, value[0]))
for k in value[1:]:print ("| %20s | %52s |" % ("",k)) for k in value[1:]: print ("| %20s | %52s |" % ("", k))
if len(value)>1:print("| %20s | %52s |" % ("","")) if len(value) > 1:
print ("-"*79) print("| %20s | %52s |" % ("", ""))
print ("-" * 79)
def parfmt(self,txt,width): def parfmt(self, txt, width):
res=[] res = []
pos=0 pos = 0
while True: while True:
a=txt[pos:pos+width] a = txt[pos:pos + width]
if not a:break if not a:
break
res.append(a) res.append(a)
pos+=width pos += width
return res return res
def cmd_set(self,*args): def cmd_set(self, *args):
'''-1|set [variable_name] [value]|Set configuration variable value|Values are an expressions (100 | string.lower('ABC') | etc.''' '''-1|set [variable_name] [value]|Set configuration variable value|Values are an expressions (100 | string.lower('ABC') | etc.'''
value=" ".join(args[1:]) value = " ".join(args[1:])
if args[0] not in self.configvars: if args[0] not in self.configvars:
setattr(self,"var_{0}".format(args[0]),eval(value)) setattr(self, "var_{0}".format(args[0]), eval(value))
setattr(self,"var_{0}".format(args[0]),eval(value)) setattr(self, "var_{0}".format(args[0]), eval(value))
def cmd_clearscreen(self,numlines=50): def cmd_clearscreen(self, numlines=50):
'''---Clear the console. '''---Clear the console.
---''' ---'''
if os.name == "posix": if os.name == "posix":
@@ -413,72 +430,78 @@ style choices:
'''--- '''---
Fallback for other operating systems. Fallback for other operating systems.
---''' ---'''
print '\n'*numlines print '\n' * numlines
class dalShell(console): class dalShell(console):
def __init__(self): def __init__(self):
pass pass
def shell(self,db): def shell(self, db):
console.__init__(self,prompt=">>> ",banner='dal interactive shell') console.__init__(self, prompt=">>> ", banner='dal interactive shell')
self.execCmd(db) self.execCmd(db)
class setCopyDB(): class setCopyDB():
def __init__(self): def __init__(self):
'''--- '''---
non source or target specific vars non source or target specific vars
---''' ---'''
self.strModel=None self.strModel = None
self.dalPath=None self.dalPath = None
self.db=None self.db = None
'''--- '''---
source vars source vars
---''' ---'''
self.sourceModel=None self.sourceModel = None
self.sourceFolder=None self.sourceFolder = None
self.sourceConnectionString=None self.sourceConnectionString = None
self.sourcedbType=None self.sourcedbType = None
self.sourcedbName=None self.sourcedbName = None
'''--- '''---
target vars target vars
---''' ---'''
self.targetdbType=None self.targetdbType = None
self.targetdbName=None self.targetdbName = None
self.targetModel=None self.targetModel = None
self.targetFolder=None self.targetFolder = None
self.targetConnectionString=None self.targetConnectionString = None
self.truncate=False self.truncate = False
def _getDal(self): def _getDal(self):
mDal=None mDal = None
if self.dalPath is not None: if self.dalPath is not None:
global DAL global DAL
sys.path.append(self.dalPath) sys.path.append(self.dalPath)
mDal=__import__('dal',globals={},locals={},fromlist=['DAL'],level=0) mDal = __import__(
DAL=mDal.DAL 'dal', globals={}, locals={}, fromlist=['DAL'], level=0)
DAL = mDal.DAL
return mDal return mDal
def instDB(self,storageFolder,storageConnectionString,autoImport): def instDB(self, storageFolder, storageConnectionString, autoImport):
self.db=DAL(storageConnectionString,folder=os.path.abspath(storageFolder),auto_import=autoImport) self.db = DAL(storageConnectionString, folder=os.path.abspath(
storageFolder), auto_import=autoImport)
return self.db return self.db
def delete_DB_tables(self,storageFolder,storageType): def delete_DB_tables(self, storageFolder, storageType):
print 'delete_DB_tablesn\n\t{0}\n\t{1}'.format(storageFolder,storageType) print 'delete_DB_tablesn\n\t{0}\n\t{1}'.format(
dataFiles=[storageType,"sql.log"] storageFolder, storageType)
dataFiles = [storageType, "sql.log"]
try: try:
for f in os.listdir(storageFolder): for f in os.listdir(storageFolder):
if ".table" in f: if ".table" in f:
fTable="{0}/{1}".format(storageFolder,f) fTable = "{0}/{1}".format(storageFolder, f)
os.remove(fTable) os.remove(fTable)
print('deleted {0}'.format(fTable)) print('deleted {0}'.format(fTable))
for dFile in dataFiles: for dFile in dataFiles:
os.remove("{0}/{1}".format(storageFolder,dFile)) os.remove("{0}/{1}".format(storageFolder, dFile))
print('deleted {0}'.format("{0}/{1}".format(storageFolder,dFile))) print('deleted {0}'.format(
"{0}/{1}".format(storageFolder, dFile)))
except Exception, errObj: except Exception, errObj:
print(str(errObj)) print(str(errObj))
def truncatetables(self,tables=[]): def truncatetables(self, tables=[]):
if len(tables)!=0: if len(tables) != 0:
try: try:
print 'table value: {0}'.format(tables) print 'table value: {0}'.format(tables)
for tbl in self.db.tables: for tbl in self.db.tables:
@@ -495,12 +518,14 @@ class setCopyDB():
print('EXCEPTION: {0}'.format(err)) print('EXCEPTION: {0}'.format(err))
def copyDB(self): def copyDB(self):
other_db=DAL("{0}://{1}".format(self.targetdbType,self.targetdbName),folder=self.targetFolder) other_db = DAL("{0}://{1}".format(
self.targetdbType, self.targetdbName), folder=self.targetFolder)
print 'creating tables...' print 'creating tables...'
for table in self.db: for table in self.db:
other_db.define_table(table._tablename,*[field for field in table]) other_db.define_table(
table._tablename, *[field for field in table])
''' '''
should there be an option to truncAte target DB? should there be an option to truncAte target DB?
if yes, then change args to allow for choice if yes, then change args to allow for choice
@@ -511,43 +536,38 @@ class setCopyDB():
''' '''
print 'exporting data...' print 'exporting data...'
self.db.export_to_csv_file(open('tmp.sql','wb')) self.db.export_to_csv_file(open('tmp.sql', 'wb'))
print 'importing data...' print 'importing data...'
other_db.import_from_csv_file(open('tmp.sql','rb')) other_db.import_from_csv_file(open('tmp.sql', 'rb'))
other_db.commit() other_db.commit()
print 'done!' print 'done!'
print 'Attention: do not run this program again or you end up with duplicate records' print 'Attention: do not run this program again or you end up with duplicate records'
def createfolderPath(self,folder): def createfolderPath(self, folder):
try: try:
if folder!=None:os.makedirs(folder) if folder is not None:
os.makedirs(folder)
except Exception, err: except Exception, err:
pass pass
if __name__ == '__main__': if __name__ == '__main__':
oCopy=setCopyDB() oCopy = setCopyDB()
db=None db = None
targetDB=None targetDB = None
dbfolder=None dbfolder = None
clean=False clean = False
model=None model = None
truncate=False truncate = False
parser=argparse.ArgumentParser(description='\ parser = argparse.ArgumentParser(description='\
samplecmd line:\n\ samplecmd line:\n\
-f ./blueLite/db_storage -i -y sqlite://storage.sqlite -Y sqlite://storage2.sqlite -d ./blueLite/pyUtils/sql/blueSQL -t True', -f ./blueLite/db_storage -i -y sqlite://storage.sqlite -Y sqlite://storage2.sqlite -d ./blueLite/pyUtils/sql/blueSQL -t True',
epilog = '') epilog='')
reqGroup=parser.add_argument_group('Required arguments') reqGroup = parser.add_argument_group('Required arguments')
reqGroup.add_argument('-f','--sourceFolder'\ reqGroup.add_argument('-f', '--sourceFolder', required=True, help="path to the 'source' folder of the 'source' DB")
,required=True\ reqGroup.add_argument('-F', '--targetFolder', required=False, help="path to the 'target' folder of the 'target' DB")
,help="path to the 'source' folder of the 'source' DB") reqGroup.add_argument('-y', '--sourceConnectionString', required=True, help="source db connection string ()\n\
reqGroup.add_argument('-F','--targetFolder'\
,required=False\
,help="path to the 'target' folder of the 'target' DB")
reqGroup.add_argument('-y','--sourceConnectionString'\
,required=True\
,help="source db connection string ()\n\
------------------------------------------------\n\ ------------------------------------------------\n\
\ \
sqlite://storage.db\n\ sqlite://storage.db\n\
@@ -561,13 +581,10 @@ ingres://username:password@localhost/test\n\
informix://username:password@test\n\ informix://username:password@test\n\
\ \
------------------------------------------------") ------------------------------------------------")
reqGroup.add_argument('-Y','--targetConnectionString'\ reqGroup.add_argument('-Y', '--targetConnectionString', required=True,
,required=True\ help="target db type (sqlite,mySql,etc.)")
,help="target db type (sqlite,mySql,etc.)") autoImpGroup = parser.add_argument_group('optional args (auto_import)')
autoImpGroup=parser.add_argument_group('optional args (auto_import)') autoImpGroup.add_argument('-a', '--autoimport', required=False, help='set to True to bypass loading of the model')
autoImpGroup.add_argument('-a','--autoimport'\
,required=False\
,help='set to True to bypass loading of the model')
""" """
@@ -584,66 +601,58 @@ informix://username:password@test\n\
""" """
miscGroup = parser.add_argument_group('optional args/tasks')
miscGroup.add_argument('-i', '--interactive', required=False, action='store_true', help='run in interactive mode')
miscGroup.add_argument(
'-d', '--dal', required=False, help='path to dal.py')
miscGroup.add_argument('-t', '--truncate', choices=['True', 'False'], help='delete the records but *not* the table of the SOURCE DB')
miscGroup.add_argument('-b', '--tables', required=False, type=list, help='optional list (comma delimited) of SOURCE tables to truncate, defaults to all')
miscGroup.add_argument('-c', '--clean', required=False, help='delete the DB,tables and the log file, WARNING: this is unrecoverable')
miscGroup=parser.add_argument_group('optional args/tasks') args = parser.parse_args()
miscGroup.add_argument('-i','--interactive'\ db = None
,required=False\ mDal = None
,action='store_true'\
,help='run in interactive mode')
miscGroup.add_argument('-d','--dal'\
,required=False\
,help='path to dal.py')
miscGroup.add_argument('-t','--truncate'\
,choices=['True','False']\
,help='delete the records but *not* the table of the SOURCE DB')
miscGroup.add_argument('-b','--tables'\
,required=False\
,type=list\
,help='optional list (comma delimited) of SOURCE tables to truncate, defaults to all')
miscGroup.add_argument('-c','--clean'\
,required=False\
,help='delete the DB,tables and the log file, WARNING: this is unrecoverable')
args=parser.parse_args()
db=None
mDal=None
try: try:
oCopy.sourceFolder=args.sourceFolder oCopy.sourceFolder = args.sourceFolder
oCopy.targetFolder=args.sourceFolder oCopy.targetFolder = args.sourceFolder
sourceItems=string.split(args.sourceConnectionString,'://') sourceItems = string.split(args.sourceConnectionString, '://')
oCopy.sourcedbType=sourceItems[0] oCopy.sourcedbType = sourceItems[0]
oCopy.sourcedbName=sourceItems[1] oCopy.sourcedbName = sourceItems[1]
targetItems=string.split(args.targetConnectionString,'://') targetItems = string.split(args.targetConnectionString, '://')
oCopy.targetdbType=targetItems[0] oCopy.targetdbType = targetItems[0]
oCopy.targetdbName=targetItems[1] oCopy.targetdbName = targetItems[1]
except Exception, err: except Exception, err:
print('EXCEPTION: {0}'.format(err)) print('EXCEPTION: {0}'.format(err))
if args.dal: if args.dal:
try: try:
autoImport=True autoImport = True
if args.autoimport:autoImport=args.autoimport if args.autoimport:
autoImport = args.autoimport
#sif not DAL in globals: #sif not DAL in globals:
#if not sys.path.__contains__(): #if not sys.path.__contains__():
oCopy.dalPath=args.dal oCopy.dalPath = args.dal
mDal=oCopy._getDal() mDal = oCopy._getDal()
db=oCopy.instDB(args.sourceFolder,args.sourceConnectionString,autoImport) db = oCopy.instDB(args.sourceFolder, args.sourceConnectionString,
autoImport)
except Exception, err: except Exception, err:
print('EXCEPTION: could not set DAL\n{0}'.format(err)) print('EXCEPTION: could not set DAL\n{0}'.format(err))
if args.truncate: if args.truncate:
try: try:
if args.truncate: if args.truncate:
if args.tables:tables=string.split(string.strip(args.tables),',') if args.tables:
else:oCopy.truncatetables([]) tables = string.split(string.strip(args.tables), ',')
else:
oCopy.truncatetables([])
except Exception, err: except Exception, err:
print('EXCEPTION: could not truncate tables\n{0}'.format(err)) print('EXCEPTION: could not truncate tables\n{0}'.format(err))
try: try:
if args.clean:oCopy.delete_DB_tables(oCopy.targetFolder,oCopy.targetType) if args.clean:
oCopy.delete_DB_tables(oCopy.targetFolder, oCopy.targetType)
except Exception, err: except Exception, err:
print('EXCEPTION: could not clean db\n{0}'.format(err)) print('EXCEPTION: could not clean db\n{0}'.format(err))
""" """
*** goes with -m/-M options... removed for now *** goes with -m/-M options... removed for now
@@ -661,14 +670,15 @@ target model: {1}\n\
if args.sourceFolder: if args.sourceFolder:
try: try:
oCopy.sourceFolder=os.path.abspath(args.sourceFolder) oCopy.sourceFolder = os.path.abspath(args.sourceFolder)
oCopy.createfolderPath(oCopy.sourceFolder) oCopy.createfolderPath(oCopy.sourceFolder)
except Exception, err: except Exception, err:
print('EXCEPTION: could not create folder path\n{0}'.format(err)) print('EXCEPTION: could not create folder path\n{0}'.format(err))
else:oCopy.dbStorageFolder=os.path.abspath(os.getcwd()) else:
oCopy.dbStorageFolder = os.path.abspath(os.getcwd())
if args.targetFolder: if args.targetFolder:
try: try:
oCopy.targetFolder=os.path.abspath(args.targetFolder) oCopy.targetFolder = os.path.abspath(args.targetFolder)
oCopy.createfolderPath(oCopy.targetFolder) oCopy.createfolderPath(oCopy.targetFolder)
except Exception, err: except Exception, err:
print('EXCEPTION: could not create folder path\n{0}'.format(err)) print('EXCEPTION: could not create folder path\n{0}'.format(err))
@@ -678,5 +688,5 @@ target model: {1}\n\
except Exception, err: except Exception, err:
print('EXCEPTION: could not make a copy of the database\n{0}'.format(err)) print('EXCEPTION: could not make a copy of the database\n{0}'.format(err))
else: else:
s=dalShell() s = dalShell()
s.shell(db) s.shell(db)
scripts/cpplugin.py
+20 -14
View File
@@ -1,26 +1,32 @@
import sys, glob, os, shutil import sys
name=sys.argv[1] import glob
app=sys.argv[2] import os
dest=sys.argv[3] import shutil
a=glob.glob('applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app,name=name)) name = sys.argv[1]
b=glob.glob('applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app,name=name)) app = sys.argv[2]
dest = sys.argv[3]
a = glob.glob(
'applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app, name=name))
b = glob.glob(
'applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app, name=name))
for f in a: for f in a:
print 'cp %s ...' % f, print 'cp %s ...' % f,
shutil.copyfile(f,os.path.join('applications',dest,*f.split('/')[2:])) shutil.copyfile(f, os.path.join('applications', dest, *f.split('/')[2:]))
print 'done' print 'done'
for f in b: for f in b:
print 'cp %s ...' % f, print 'cp %s ...' % f,
path = f.split('/') path = f.split('/')
for i in range(3,len(path)): for i in range(3, len(path)):
try: os.mkdir(os.path.join('applications',dest,*path[2:i])) try:
except: pass os.mkdir(os.path.join('applications', dest, *path[2:i]))
path = os.path.join('applications',dest,*f.split('/')[2:]) except:
pass
path = os.path.join('applications', dest, *f.split('/')[2:])
if os.path.isdir(f): if os.path.isdir(f):
if not os.path.exists(path): if not os.path.exists(path):
shutil.copytree(f,path) shutil.copytree(f, path)
else: else:
shutil.copyfile(f,path) shutil.copyfile(f, path)
print 'done' print 'done'
scripts/dbsessions2trash.py
+4 -5
View File
@@ -10,17 +10,16 @@ DB_URI = 'sqlite://sessions.sqlite'
EXPIRATION_MINUTES = 60 EXPIRATION_MINUTES = 60
SLEEP_MINUTES = 5 SLEEP_MINUTES = 5
while 1: # Infinite loop while 1: # Infinite loop
now = time() # get current Unix timestamp now = time() # get current Unix timestamp
for row in db().select(db.web2py_session_welcome.ALL): for row in db().select(db.web2py_session_welcome.ALL):
t = row.modified_datetime t = row.modified_datetime
# Convert to a Unix timestamp # Convert to a Unix timestamp
t = mktime(t.timetuple())+1e-6*t.microsecond t = mktime(t.timetuple()) + 1e-6 * t.microsecond
if now - t > EXPIRATION_MINUTES * 60: if now - t > EXPIRATION_MINUTES * 60:
del db.web2py_session_welcome[row.id] del db.web2py_session_welcome[row.id]
db.commit() # Write changes to database db.commit() # Write changes to database
sleep(SLEEP_MINUTES * 60) sleep(SLEEP_MINUTES * 60)
scripts/dict_diff.py
+7
View File
@@ -22,6 +22,7 @@ import getopt
import os.path import os.path
import sys import sys
def main(argv): def main(argv):
"""Parse the arguments and start the main process.""" """Parse the arguments and start the main process."""
try: try:
@@ -40,16 +41,19 @@ def main(argv):
else: else:
exit_with_parsing_error() exit_with_parsing_error()
def exit_with_parsing_error(): def exit_with_parsing_error():
"""Report invalid arguments and usage.""" """Report invalid arguments and usage."""
print("Invalid argument(s).") print("Invalid argument(s).")
usage() usage()
sys.exit(2) sys.exit(2)
def usage(): def usage():
"""Display the documentation""" """Display the documentation"""
print(__doc__) print(__doc__)
def get_dicts(dict_path1, dict_path2): def get_dicts(dict_path1, dict_path2):
""" """
Parse the dictionaries. Parse the dictionaries.
@@ -60,6 +64,7 @@ def get_dicts(dict_path1, dict_path2):
return eval(open(dict_path1).read()), eval(open(dict_path2).read()) return eval(open(dict_path1).read()), eval(open(dict_path2).read())
def get_dict_names(dict1_path, dict2_path): def get_dict_names(dict1_path, dict2_path):
""" """
Get the name of the dictionaries for the end user. Use the base name of the Get the name of the dictionaries for the end user. Use the base name of the
@@ -76,6 +81,7 @@ def get_dict_names(dict1_path, dict2_path):
dict2_name = "dict2" dict2_name = "dict2"
return dict1_name, dict2_name return dict1_name, dict2_name
def compare_dicts(dict1, dict2, dict1_name, dict2_name): def compare_dicts(dict1, dict2, dict1_name, dict2_name):
""" """
Compare the two dictionaries. Print out the result. Compare the two dictionaries. Print out the result.
@@ -101,6 +107,7 @@ def compare_dicts(dict1, dict2, dict1_name, dict2_name):
if not has_value_differences: if not has_value_differences:
print " None" print " None"
def print_key_diff(key_diff, dict1_name, dict2_name): def print_key_diff(key_diff, dict1_name, dict2_name):
""" """
Prints the keys in the first dictionary and are in the second dictionary. Prints the keys in the first dictionary and are in the second dictionary.
scripts/extract_mysql_models.py
+49 -43
View File
@@ -24,45 +24,48 @@ import subprocess
import re import re
import sys import sys
data_type_map = dict( data_type_map = dict(
varchar = 'string', varchar='string',
int = 'integer', int='integer',
integer = 'integer', integer='integer',
tinyint = 'integer', tinyint='integer',
smallint = 'integer', smallint='integer',
mediumint = 'integer', mediumint='integer',
bigint = 'integer', bigint='integer',
float = 'double', float='double',
double = 'double', double='double',
char = 'string', char='string',
decimal = 'integer', decimal='integer',
date = 'date', date='date',
#year = 'date', #year = 'date',
time = 'time', time='time',
timestamp = 'datetime', timestamp='datetime',
datetime = 'datetime', datetime='datetime',
binary = 'blob', binary='blob',
blob = 'blob', blob='blob',
tinyblob = 'blob', tinyblob='blob',
mediumblob = 'blob', mediumblob='blob',
longblob = 'blob', longblob='blob',
text = 'text', text='text',
tinytext = 'text', tinytext='text',
mediumtext = 'text', mediumtext='text',
longtext = 'text', longtext='text',
) )
def mysql(database_name, username, password): def mysql(database_name, username, password):
p = subprocess.Popen(['mysql', p = subprocess.Popen(['mysql',
'--user=%s' % username, '--user=%s' % username,
'--password=%s'% password, '--password=%s' % password,
'--execute=show tables;', '--execute=show tables;',
database_name], database_name],
stdin=subprocess.PIPE, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
sql_showtables, stderr = p.communicate() sql_showtables, stderr = p.communicate()
tables = [re.sub('\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]] tables = [re.sub(
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')"%(username, password, database_name) '\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')" % (
username, password, database_name)
legacy_db_table_web2py_code = [] legacy_db_table_web2py_code = []
for table_name in tables: for table_name in tables:
#get the sql create statement #get the sql create statement
@@ -71,37 +74,40 @@ def mysql(database_name, username, password):
'--password=%s' % password, '--password=%s' % password,
'--skip-add-drop-table', '--skip-add-drop-table',
'--no-data', database_name, '--no-data', database_name,
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE) table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sql_create_stmnt,stderr = p.communicate() sql_create_stmnt, stderr = p.communicate()
if 'CREATE' in sql_create_stmnt:#check if the table exists if 'CREATE' in sql_create_stmnt: # check if the table exists
#remove garbage lines from sql statement #remove garbage lines from sql statement
sql_lines = sql_create_stmnt.split('\n') sql_lines = sql_create_stmnt.split('\n')
sql_lines = [x for x in sql_lines if not(x.startswith('--') or x.startswith('/*') or x =='')] sql_lines = [x for x in sql_lines if not(
x.startswith('--') or x.startswith('/*') or x == '')]
#generate the web2py code from the create statement #generate the web2py code from the create statement
web2py_table_code = '' web2py_table_code = ''
table_name = re.search('CREATE TABLE .(\S+). \(', sql_lines[0]).group(1) table_name = re.search(
'CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
fields = [] fields = []
for line in sql_lines[1:-1]: for line in sql_lines[1:-1]:
if re.search('KEY', line) or re.search('PRIMARY', line) or re.search(' ID', line) or line.startswith(')'): if re.search('KEY', line) or re.search('PRIMARY', line) or re.search(' ID', line) or line.startswith(')'):
continue continue
hit = re.search('(\S+)\s+(\S+)(,| )( .*)?', line) hit = re.search('(\S+)\s+(\S+)(,| )( .*)?', line)
if hit!=None: if hit is not None:
name, d_type = hit.group(1), hit.group(2) name, d_type = hit.group(1), hit.group(2)
d_type = re.sub(r'(\w+)\(.*',r'\1',d_type) d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)
name = re.sub('`','',name) name = re.sub('`', '', name)
web2py_table_code += "\n Field('%s','%s'),"%(name,data_type_map[d_type]) web2py_table_code += "\n Field('%s','%s')," % (
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)"%(table_name,web2py_table_code) name, data_type_map[d_type])
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)" % (table_name, web2py_table_code)
legacy_db_table_web2py_code.append(web2py_table_code) legacy_db_table_web2py_code.append(web2py_table_code)
#---------------------------------------- #----------------------------------------
#write the legacy db to file #write the legacy db to file
legacy_db_web2py_code = connection_string+"\n\n" legacy_db_web2py_code = connection_string + "\n\n"
legacy_db_web2py_code += "\n\n#--------\n".join(legacy_db_table_web2py_code) legacy_db_web2py_code += "\n\n#--------\n".join(
legacy_db_table_web2py_code)
return legacy_db_web2py_code return legacy_db_web2py_code
regex = re.compile('(.*?):(.*?)@(.*)') regex = re.compile('(.*?):(.*?)@(.*)')
if len(sys.argv)<2 or not regex.match(sys.argv[1]): if len(sys.argv) < 2 or not regex.match(sys.argv[1]):
print 'USAGE:\n\n extract_mysql_models.py username:password@data_basename\n\n' print 'USAGE:\n\n extract_mysql_models.py username:password@data_basename\n\n'
else: else:
m = regex.match(sys.argv[1]) m = regex.match(sys.argv[1])
print mysql(m.group(3),m.group(1),m.group(2)) print mysql(m.group(3), m.group(1), m.group(2))
scripts/extract_pgsql_models.py
+29 -27
View File
@@ -46,19 +46,21 @@ KWARGS = ('type', 'length', 'default', 'required', 'ondelete',
import sys import sys
def query(conn, sql,*args): def query(conn, sql, *args):
"Execute a SQL query and return rows as a list of dicts" "Execute a SQL query and return rows as a list of dicts"
cur = conn.cursor() cur = conn.cursor()
ret = [] ret = []
try: try:
if DEBUG: print >> sys.stderr, "QUERY: ", sql % args if DEBUG:
print >> sys.stderr, "QUERY: ", sql % args
cur.execute(sql, args) cur.execute(sql, args)
for row in cur: for row in cur:
dic = {} dic = {}
for i, value in enumerate(row): for i, value in enumerate(row):
field = cur.description[i][0] field = cur.description[i][0]
dic[field] = value dic[field] = value
if DEBUG: print >> sys.stderr, "RET: ", dic if DEBUG:
print >> sys.stderr, "RET: ", dic
ret.append(dic) ret.append(dic)
return ret return ret
finally: finally:
@@ -75,7 +77,8 @@ def get_tables(conn, schema=SCHEMA):
def get_fields(conn, table): def get_fields(conn, table):
"Retrieve field list for a given table" "Retrieve field list for a given table"
if DEBUG: print >> sys.stderr, "Processing TABLE", table if DEBUG:
print >> sys.stderr, "Processing TABLE", table
rows = query(conn, """ rows = query(conn, """
SELECT column_name, data_type, SELECT column_name, data_type,
is_nullable, is_nullable,
@@ -90,13 +93,13 @@ def get_fields(conn, table):
def define_field(conn, table, field, pks): def define_field(conn, table, field, pks):
"Determine field type, default value, references, etc." "Determine field type, default value, references, etc."
f={} f = {}
ref = references(conn, table, field['column_name']) ref = references(conn, table, field['column_name'])
if ref: if ref:
f.update(ref) f.update(ref)
elif field['column_default'] and \ elif field['column_default'] and \
field['column_default'].startswith("nextval") and \ field['column_default'].startswith("nextval") and \
field['column_name'] in pks: field['column_name'] in pks:
# postgresql sequence (SERIAL) and primary key! # postgresql sequence (SERIAL) and primary key!
f['type'] = "'id'" f['type'] = "'id'"
elif field['data_type'].startswith('character'): elif field['data_type'].startswith('character'):
@@ -109,7 +112,7 @@ def define_field(conn, table, field, pks):
f['type'] = "'boolean'" f['type'] = "'boolean'"
elif field['data_type'] in ('integer', 'smallint', 'bigint'): elif field['data_type'] in ('integer', 'smallint', 'bigint'):
f['type'] = "'integer'" f['type'] = "'integer'"
elif field['data_type'] in ('double precision', 'real' ): elif field['data_type'] in ('double precision', 'real'):
f['type'] = "'double'" f['type'] = "'double'"
elif field['data_type'] in ('timestamp', 'timestamp without time zone'): elif field['data_type'] in ('timestamp', 'timestamp without time zone'):
f['type'] = "'datetime'" f['type'] = "'datetime'"
@@ -124,17 +127,17 @@ def define_field(conn, table, field, pks):
elif field['data_type'] in ('bytea', ): elif field['data_type'] in ('bytea', ):
f['type'] = "'blob'" f['type'] = "'blob'"
elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'): elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'):
f['type'] = "" # unsupported? f['type'] = "" # unsupported?
else: else:
raise RuntimeError("Data Type not supported: %s " % str(field)) raise RuntimeError("Data Type not supported: %s " % str(field))
try: try:
if field['column_default']: if field['column_default']:
if field['column_default']=="now()": if field['column_default'] == "now()":
d = "request.now" d = "request.now"
elif field['column_default']=="true": elif field['column_default'] == "true":
d = "True" d = "True"
elif field['column_default']=="false": elif field['column_default'] == "false":
d = "False" d = "False"
else: else:
d = repr(eval(field['column_default'])) d = repr(eval(field['column_default']))
@@ -142,7 +145,8 @@ def define_field(conn, table, field, pks):
except (ValueError, SyntaxError): except (ValueError, SyntaxError):
pass pass
except Exception, e: except Exception, e:
raise RuntimeError("Default unsupported '%s'" % field['column_default']) raise RuntimeError(
"Default unsupported '%s'" % field['column_default'])
if not field['is_nullable']: if not field['is_nullable']:
f['notnull'] = "True" f['notnull'] = "True"
@@ -203,40 +207,40 @@ def references(conn, table, field):
AND information_schema.key_column_usage.column_name=%s AND information_schema.key_column_usage.column_name=%s
AND information_schema.table_constraints.constraint_type='FOREIGN KEY' AND information_schema.table_constraints.constraint_type='FOREIGN KEY'
;""", table, field) ;""", table, field)
if len(rows1)==1: if len(rows1) == 1:
rows2 = query(conn, """ rows2 = query(conn, """
SELECT table_name, column_name, * SELECT table_name, column_name, *
FROM information_schema.constraint_column_usage FROM information_schema.constraint_column_usage
WHERE constraint_name=%s WHERE constraint_name=%s
""", rows1[0]['constraint_name']) """, rows1[0]['constraint_name'])
row = None row = None
if len(rows2)>1: if len(rows2) > 1:
row = rows2[int(rows1[0]['ordinal_position'])-1] row = rows2[int(rows1[0]['ordinal_position']) - 1]
keyed = True keyed = True
if len(rows2)==1: if len(rows2) == 1:
row = rows2[0] row = rows2[0]
keyed = False keyed = False
if row: if row:
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!! if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
ref = {'type': "'reference %s.%s'" % (row['table_name'], ref = {'type': "'reference %s.%s'" % (row['table_name'],
row['column_name'])} row['column_name'])}
else: else:
ref = {'type': "'reference %s'" % (row['table_name'],)} ref = {'type': "'reference %s'" % (row['table_name'],)}
if rows1[0]['delete_rule']!="NO ACTION": if rows1[0]['delete_rule'] != "NO ACTION":
ref['ondelete'] = repr(rows1[0]['delete_rule']) ref['ondelete'] = repr(rows1[0]['delete_rule'])
return ref return ref
elif rows2: elif rows2:
raise RuntimeError("Unsupported foreign key reference: %s" % raise RuntimeError("Unsupported foreign key reference: %s" %
str(rows2)) str(rows2))
elif rows1: elif rows1:
raise RuntimeError("Unsupported referential constraint: %s" % raise RuntimeError("Unsupported referential constraint: %s" %
str(rows1)) str(rows1))
def define_table(conn, table): def define_table(conn, table):
"Output single table definition" "Output single table definition"
fields = get_fields(conn, table) fields = get_fields(conn, table)
pks = primarykeys(conn, table) pks = primarykeys(conn, table)
print "db.define_table('%s'," % (table, ) print "db.define_table('%s'," % (table, )
for field in fields: for field in fields:
@@ -244,11 +248,11 @@ def define_table(conn, table):
fdef = define_field(conn, table, field, pks) fdef = define_field(conn, table, field, pks)
if fname not in pks and is_unique(conn, table, field): if fname not in pks and is_unique(conn, table, field):
fdef['unique'] = "True" fdef['unique'] = "True"
if fdef['type']=="'id'" and fname in pks: if fdef['type'] == "'id'" and fname in pks:
pks.pop(pks.index(fname)) pks.pop(pks.index(fname))
print " Field('%s', %s)," % (fname, print " Field('%s', %s)," % (fname,
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS ', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
if k in fdef and fdef[k]])) if k in fdef and fdef[k]]))
if pks: if pks:
print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks]) print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks])
print " migrate=migrate)" print " migrate=migrate)"
@@ -280,5 +284,3 @@ if __name__ == "__main__":
) )
# Start model code generation: # Start model code generation:
define_db(cnn, db, host, port, user, passwd) define_db(cnn, db, host, port, user, passwd)
scripts/fixws.py
+7 -4
View File
@@ -1,4 +1,6 @@
import sys, glob import sys
import glob
def read_fileb(filename, mode='rb'): def read_fileb(filename, mode='rb'):
f = open(filename, mode) f = open(filename, mode)
@@ -7,6 +9,7 @@ def read_fileb(filename, mode='rb'):
finally: finally:
f.close() f.close()
def write_fileb(filename, value, mode='wb'): def write_fileb(filename, value, mode='wb'):
f = open(filename, mode) f = open(filename, mode)
try: try:
@@ -18,7 +21,7 @@ for filename in glob.glob(sys.argv[1]):
data1 = read_fileb(filename) data1 = read_fileb(filename)
write_fileb(filename + '.bak2', data1) write_fileb(filename + '.bak2', data1)
data2lines = read_fileb(filename).strip().split('\n') data2lines = read_fileb(filename).strip().split('\n')
data2 = '\n'.join([line.rstrip().replace('\t',' '*2) for line in data2lines])+'\n' data2 = '\n'.join([line.rstrip(
).replace('\t', ' ' * 2) for line in data2lines]) + '\n'
write_fileb(filename, data2) write_fileb(filename, data2)
print filename, len(data1)-len(data2) print filename, len(data1) - len(data2)
scripts/layout_make.py
+28 -24
View File
@@ -5,10 +5,11 @@ import sys
import re import re
from BeautifulSoup import BeautifulSoup as BS from BeautifulSoup import BeautifulSoup as BS
def head(styles): def head(styles):
title = '<title>{{=response.title or request.application}}</title>' title = '<title>{{=response.title or request.application}}</title>'
items = '\n'.join(["{{response.files.append(URL(request.application,'static','%s'))}}" % (style) for style in styles]) items = '\n'.join(["{{response.files.append(URL(request.application,'static','%s'))}}" % (style) for style in styles])
loc="""<style> loc = """<style>
div.flash { div.flash {
position: absolute; position: absolute;
float: right; float: right;
@@ -39,50 +40,54 @@ div.error {
border: 1px solid #666; border: 1px solid #666;
} }
</style>""" </style>"""
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title,items,loc) return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title, items, loc)
def content(): def content():
return """<div class="flash">{{=response.flash or ''}}</div>{{include}}""" return """<div class="flash">{{=response.flash or ''}}</div>{{include}}"""
def process(folder): def process(folder):
indexfile = open(os.path.join(folder,'index.html'),'rb') indexfile = open(os.path.join(folder, 'index.html'), 'rb')
try: try:
soup = BS(indexfile.read()) soup = BS(indexfile.read())
finally: finally:
indexfile.close() indexfile.close()
styles = [x['href'] for x in soup.findAll('link')] styles = [x['href'] for x in soup.findAll('link')]
soup.find('head').contents=BS(head(styles)) soup.find('head').contents = BS(head(styles))
try: try:
soup.find('h1').contents=BS('{{=response.title or request.application}}') soup.find(
soup.find('h2').contents=BS("{{=response.subtitle or '=response.subtitle'}}") 'h1').contents = BS('{{=response.title or request.application}}')
soup.find('h2').contents = BS(
"{{=response.subtitle or '=response.subtitle'}}")
except: except:
pass pass
for match in (soup.find('div',id='menu'), for match in (soup.find('div', id='menu'),
soup.find('div',{'class':'menu'}), soup.find('div', {'class': 'menu'}),
soup.find('div',id='nav'), soup.find('div', id='nav'),
soup.find('div',{'class':'nav'})): soup.find('div', {'class': 'nav'})):
if match: if match:
match.contents=BS('{{=MENU(response.menu)}}') match.contents = BS('{{=MENU(response.menu)}}')
break break
done=False done = False
for match in (soup.find('div',id='content'), for match in (soup.find('div', id='content'),
soup.find('div',{'class':'content'}), soup.find('div', {'class': 'content'}),
soup.find('div',id='main'), soup.find('div', id='main'),
soup.find('div',{'class':'main'})): soup.find('div', {'class': 'main'})):
if match: if match:
match.contents=BS(content()) match.contents = BS(content())
done=True done = True
break break
if done: if done:
page = soup.prettify() page = soup.prettify()
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*",re.MULTILINE)\ page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*", re.MULTILINE)\
.sub("{{=response.flash or ''}}",page) .sub("{{=response.flash or ''}}", page)
print page print page
else: else:
raise Exception, "Unable to convert" raise Exception("Unable to convert")
if __name__=='__main__': if __name__ == '__main__':
if len(sys.argv)<2: if len(sys.argv) < 2:
print """USAGE: print """USAGE:
1) start a new web2py application 1) start a new web2py application
2) Download a sample free layout from the web into the static/ folder of 2) Download a sample free layout from the web into the static/ folder of
@@ -96,4 +101,3 @@ if __name__=='__main__':
print 'Folder %s does not exist' % sys.argv[1] print 'Folder %s does not exist' % sys.argv[1]
else: else:
process(sys.argv[1]) process(sys.argv[1])
scripts/make_min_web2py.py
+29 -20
View File
@@ -38,12 +38,16 @@ gluon/contrib/pyrtf/
gluon/contrib/pysimplesoap/ gluon/contrib/pysimplesoap/
""" """
import sys, os, shutil, glob import sys
import os
import shutil
import glob
def main(): def main():
global REQUIRED, IGNORED global REQUIRED, IGNORED
if len(sys.argv)<2: if len(sys.argv) < 2:
print USAGE print USAGE
# make target folder # make target folder
@@ -51,38 +55,43 @@ def main():
os.mkdir(target) os.mkdir(target)
# change to os specificsep # change to os specificsep
REQUIRED = REQUIRED.replace('/',os.sep) REQUIRED = REQUIRED.replace('/', os.sep)
IGNORED = IGNORED.replace('/',os.sep) IGNORED = IGNORED.replace('/', os.sep)
# make a list of all files to include # make a list of all files to include
files = [x.strip() for x in REQUIRED.split('\n') \ files = [x.strip() for x in REQUIRED.split('\n')
if x and not x[0]=='#'] if x and not x[0] == '#']
ignore = [x.strip() for x in IGNORED.split('\n') \ ignore = [x.strip() for x in IGNORED.split('\n')
if x and not x[0]=='#'] if x and not x[0] == '#']
def accept(filename): def accept(filename):
for p in ignore: for p in ignore:
if filename.startswith(p): if filename.startswith(p):
return False return False
return True return True
pattern = os.path.join('gluon','*.py') pattern = os.path.join('gluon', '*.py')
while True: while True:
newfiles = [x for x in glob.glob(pattern) if accept(x)] newfiles = [x for x in glob.glob(pattern) if accept(x)]
if not newfiles: break if not newfiles:
break
files += newfiles files += newfiles
pattern = os.path.join(pattern[:-3],'*.py') pattern = os.path.join(pattern[:-3], '*.py')
# copy all files, make missing folder, build default.py # copy all files, make missing folder, build default.py
files.sort() files.sort()
defaultpy = os.path.join('applications','welcome','controllers','default.py') defaultpy = os.path.join(
'applications', 'welcome', 'controllers', 'default.py')
for f in files: for f in files:
dirs = f.split(os.path.sep) dirs = f.split(os.path.sep)
for i in range(1,len(dirs)): for i in range(1, len(dirs)):
try: os.mkdir(target+os.sep+os.path.join(*dirs[:i])) try:
except OSError: pass os.mkdir(target + os.sep + os.path.join(*dirs[:i]))
if f==defaultpy: except OSError:
open(os.path.join(target,f),'w').write('def index(): return "hello"\n') pass
if f == defaultpy:
open(os.path.join(
target, f), 'w').write('def index(): return "hello"\n')
else: else:
shutil.copyfile(f,os.path.join(target,f)) shutil.copyfile(f, os.path.join(target, f))
if __name__=='__main__': main() if __name__ == '__main__':
main()
scripts/rmorphans.py
+6 -6
View File
@@ -6,18 +6,18 @@ paths2 = []
while paths: while paths:
path = paths.pop() path = paths.pop()
for filename in os.listdir(path): for filename in os.listdir(path):
fullname = os.path.join(path,filename) fullname = os.path.join(path, filename)
if os.path.isdir(fullname): if os.path.isdir(fullname):
paths.append(fullname) paths.append(fullname)
else: else:
extension = filename.split('.')[-1] extension = filename.split('.')[-1]
if extension.lower() in ('png','gif','jpg','jpeg','js','css'): if extension.lower() in ('png', 'gif', 'jpg', 'jpeg', 'js', 'css'):
paths1.append((filename,fullname)) paths1.append((filename, fullname))
if extension.lower() in ('css','js','py','html'): if extension.lower() in ('css', 'js', 'py', 'html'):
paths2.append(fullname) paths2.append(fullname)
for filename,fullname in paths1: for filename, fullname in paths1:
for otherfullname in paths2: for otherfullname in paths2:
if open(otherfullname).read().find(filename)>=0: if open(otherfullname).read().find(filename) >= 0:
break break
else: else:
print fullname print fullname
scripts/sessions2trash.py
+17 -17
View File
@@ -162,7 +162,7 @@ class SessionFile(object):
def last_visit_default(self): def last_visit_default(self):
return datetime.datetime.fromtimestamp( return datetime.datetime.fromtimestamp(
os.stat(self.filename)[stat.ST_MTIME]) os.stat(self.filename)[stat.ST_MTIME])
def __str__(self): def __str__(self):
return self.filename return self.filename
@@ -175,7 +175,7 @@ def total_seconds(delta):
Args: Args:
delta: datetime.timedelta instance. delta: datetime.timedelta instance.
""" """
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) * \ return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) *
10 ** 6) / 10 ** 6 10 ** 6) / 10 ** 6
@@ -186,25 +186,25 @@ def main():
parser = OptionParser(usage=usage) parser = OptionParser(usage=usage)
parser.add_option('-f', '--force', parser.add_option('-f', '--force',
action='store_true', dest='force', default=False, action='store_true', dest='force', default=False,
help=('Ignore session expiration. ' help=('Ignore session expiration. '
'Force expiry based on -x option or auth.settings.expiration.') 'Force expiry based on -x option or auth.settings.expiration.')
) )
parser.add_option('-o', '--once', parser.add_option('-o', '--once',
action='store_true', dest='once', default=False, action='store_true', dest='once', default=False,
help='Delete sessions, then exit.', help='Delete sessions, then exit.',
) )
parser.add_option('-s', '--sleep', parser.add_option('-s', '--sleep',
dest='sleep', default=SLEEP_MINUTES * 60, type="int", dest='sleep', default=SLEEP_MINUTES * 60, type="int",
help='Number of seconds to sleep between executions. Default 300.', help='Number of seconds to sleep between executions. Default 300.',
) )
parser.add_option('-v', '--verbose', parser.add_option('-v', '--verbose',
default=0, action='count', default=0, action='count',
help="print verbose output, a second -v increases verbosity") help="print verbose output, a second -v increases verbosity")
parser.add_option('-x', '--expiration', parser.add_option('-x', '--expiration',
dest='expiration', default=None, type="int", dest='expiration', default=None, type="int",
help='Expiration value for sessions without expiration (in seconds)', help='Expiration value for sessions without expiration (in seconds)',
) )
(options, unused_args) = parser.parse_args() (options, unused_args) = parser.parse_args()
scripts/standalone_exe_cxfreeze.py
+30 -30
View File
@@ -30,35 +30,35 @@ if sys.platform == 'win32':
base_modules.remove('macpath') base_modules.remove('macpath')
buildOptions = dict( buildOptions = dict(
compressed = True, compressed=True,
excludes = ["macpath","PyQt4"], excludes=["macpath", "PyQt4"],
includes = base_modules, includes=base_modules,
include_files=[ include_files=[
'applications', 'applications',
'ABOUT', 'ABOUT',
'LICENSE', 'LICENSE',
'VERSION', 'VERSION',
'logging.example.conf', 'logging.example.conf',
'options_std.py', 'options_std.py',
'app.example.yaml', 'app.example.yaml',
'queue.example.yaml', 'queue.example.yaml',
], ],
# append any extra module by extending the list below - # append any extra module by extending the list below -
# "contributed_modules+["lxml"]" # "contributed_modules+["lxml"]"
packages = contributed_modules, packages=contributed_modules,
) )
setup( setup(
name = "Web2py", name="Web2py",
version=web2py_version, version=web2py_version,
author="Massimo DiPierro", author="Massimo DiPierro",
description="web2py web framework", description="web2py web framework",
license = "LGPL v3", license="LGPL v3",
options = dict(build_exe = buildOptions), options=dict(build_exe=buildOptions),
executables = [Executable("web2py.py", executables=[Executable("web2py.py",
base=base, base=base,
compress = True, compress=True,
icon = "web2py.ico", icon="web2py.ico",
targetName="web2py.exe", targetName="web2py.exe",
copyDependentFiles = True)], copyDependentFiles=True)],
) )
scripts/sync_languages.py
+4 -4
View File
@@ -14,6 +14,7 @@ sys.path.insert(0, '.')
file = sys.argv[1] file = sys.argv[1]
apps = sys.argv[2:] apps = sys.argv[2:]
def sync_language(d, data): def sync_language(d, data):
''' this function makes sure a translated string will be prefered over an untranslated ''' this function makes sure a translated string will be prefered over an untranslated
string when syncing languages between apps. when both are translated, it prefers the string when syncing languages between apps. when both are translated, it prefers the
@@ -26,9 +27,9 @@ def sync_language(d, data):
d[key] = data[key] d[key] = data[key]
# see if there is a translated string in the original list, but not in the new list # see if there is a translated string in the original list, but not in the new list
elif ( elif (
((d[key] != '') or (d[key] != key)) and ((d[key] != '') or (d[key] != key)) and
((data[key] == '') or (data[key] == key)) ((data[key] == '') or (data[key] == key))
): ):
d[key] = d[key] d[key] = d[key]
# any other case (wether there is or there isn't a translated string) # any other case (wether there is or there isn't a translated string)
else: else:
@@ -68,4 +69,3 @@ for app in oapps:
path2 = 'applications/%s/' % app path2 = 'applications/%s/' % app
file2 = os.path.join(path2, 'languages', '%s.py' % file) file2 = os.path.join(path2, 'languages', '%s.py' % file)
shutil.copyfile(file1, file2) shutil.copyfile(file1, file2)
scripts/tickets2db.py
+3 -3
View File
@@ -15,7 +15,7 @@ SLEEP_MINUTES = 5
errors_path = os.path.join(request.folder, 'errors') errors_path = os.path.join(request.folder, 'errors')
try: try:
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r','').replace('\n','').strip() db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r', '').replace('\n', '').strip()
except: except:
db_string = 'sqlite://storage.db' db_string = 'sqlite://storage.db'
@@ -23,7 +23,8 @@ db_path = os.path.join(request.folder, 'databases')
tk_db = DAL(db_string, folder=db_path, auto_import=True) tk_db = DAL(db_string, folder=db_path, auto_import=True)
ts = TicketStorage(db=tk_db) ts = TicketStorage(db=tk_db)
tk_table = ts._get_table(db=tk_db, tablename=ts.tablename, app=request.application) tk_table = ts._get_table(
db=tk_db, tablename=ts.tablename, app=request.application)
hashes = {} hashes = {}
@@ -48,4 +49,3 @@ while 1:
os.unlink(filename) os.unlink(filename)
time.sleep(SLEEP_MINUTES * 60) time.sleep(SLEEP_MINUTES * 60)
scripts/tickets2email.py
+2 -2
View File
@@ -42,8 +42,8 @@ while 1:
error = RestrictedError() error = RestrictedError()
error.load(request, request.application, file) error.load(request, request.application, file)
mail.send(to=administrator_email, subject='new web2py ticket', message=error.traceback) mail.send(to=administrator_email,
subject='new web2py ticket', message=error.traceback)
os.unlink(os.path.join(path, file)) os.unlink(os.path.join(path, file))
time.sleep(SLEEP_MINUTES * 60) time.sleep(SLEEP_MINUTES * 60)
scripts/update_web2py.py
+5 -5
View File
@@ -14,7 +14,7 @@ import os
import urllib import urllib
import zipfile import zipfile
if len(sys.argv)>1 and sys.argv[1] == 'nightly': if len(sys.argv) > 1 and sys.argv[1] == 'nightly':
version = 'http://web2py.com/examples/static/nightly/web2py_src.zip' version = 'http://web2py.com/examples/static/nightly/web2py_src.zip'
else: else:
version = 'http://web2py.com/examples/static/web2py_src.zip' version = 'http://web2py.com/examples/static/web2py_src.zip'
@@ -23,11 +23,11 @@ realpath = os.path.realpath(__file__)
path = os.path.dirname(os.path.dirname(os.path.dirname(realpath))) path = os.path.dirname(os.path.dirname(os.path.dirname(realpath)))
os.chdir(path) os.chdir(path)
try: try:
old_version = open('web2py/VERSION','r').read().strip() old_version = open('web2py/VERSION', 'r').read().strip()
except IOError: except IOError:
old_version = '' old_version = ''
open(TMPFILENAME,'wb').write(urllib.urlopen(version).read()) open(TMPFILENAME, 'wb').write(urllib.urlopen(version).read())
new_version = zipfile.ZipFile(TMPFILENAME).read('web2py/VERSION').strip() new_version = zipfile.ZipFile(TMPFILENAME).read('web2py/VERSION').strip()
if new_version>old_version: if new_version > old_version:
os.system('sudo -u %s unzip -o %s' % (USER,TMPFILENAME)) os.system('sudo -u %s unzip -o %s' % (USER, TMPFILENAME))
os.system('apachectl restart | apache2ctl restart') os.system('apachectl restart | apache2ctl restart')
scripts/zip_static_files.py
+6 -4
View File
@@ -7,6 +7,7 @@
import os import os
import gzip import gzip
def zip_static(filelist=[]): def zip_static(filelist=[]):
tsave = 0 tsave = 0
for fi in filelist: for fi in filelist:
@@ -24,17 +25,18 @@ def zip_static(filelist=[]):
if zatime == atime and zmtime == mtime: if zatime == atime and zmtime == mtime:
print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi) print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
continue continue
print 'gzipping %s to %s' % (os.path.basename(fi), os.path.basename(gfi)) print 'gzipping %s to %s' % (
os.path.basename(fi), os.path.basename(gfi))
f_in = open(fi, 'rb') f_in = open(fi, 'rb')
f_out = gzip.open(gfi, 'wb') f_out = gzip.open(gfi, 'wb')
f_out.writelines(f_in) f_out.writelines(f_in)
f_out.close() f_out.close()
f_in.close() f_in.close()
os.utime(gfi, (atime,mtime)) os.utime(gfi, (atime, mtime))
saved = fstats.st_size - os.stat(gfi).st_size saved = fstats.st_size - os.stat(gfi).st_size
tsave+= saved tsave += saved
print 'saved %s KB' % (int(tsave)/1000.0) print 'saved %s KB' % (int(tsave) / 1000.0)
if __name__ == '__main__': if __name__ == '__main__':
ALLOWED_EXTS = ['.css', '.js'] ALLOWED_EXTS = ['.css', '.js']