pep8 in scripts/*.py
This commit is contained in:
1 parent
d053900032
commit
295084976b
24 files changed
+661
-600
No files matched your search
@@ -1 +1 @@
|
||||
Version 2.1.1 (2012-10-19 14:13:30) dev
|
||||
Version 2.1.1 (2012-10-19 14:22:39) dev
|
||||
+31
-21
@@ -40,48 +40,58 @@ domain3.com /app3/defcon3
|
||||
'''
|
||||
if not config.strip():
|
||||
try:
|
||||
config_file = open('routes.conf','r')
|
||||
config_file = open('routes.conf', 'r')
|
||||
try:
|
||||
config = config_file.read()
|
||||
finally:
|
||||
config_file.close()
|
||||
except:
|
||||
config=''
|
||||
config = ''
|
||||
|
||||
|
||||
def auto_in(apps):
|
||||
routes = [
|
||||
('/robots.txt','/welcome/static/robots.txt'),
|
||||
('/favicon.ico','/welcome/static/favicon.ico'),
|
||||
('/admin$anything','/admin$anything'),
|
||||
]
|
||||
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
|
||||
if not path.startswith('/'): path = '/'+path
|
||||
if path.endswith('/'): path = path[:-1]
|
||||
('/robots.txt', '/welcome/static/robots.txt'),
|
||||
('/favicon.ico', '/welcome/static/favicon.ico'),
|
||||
('/admin$anything', '/admin$anything'),
|
||||
]
|
||||
for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
|
||||
if not path.startswith('/'):
|
||||
path = '/' + path
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
app = path.split('/')[1]
|
||||
routes += [
|
||||
('.*:https?://(.*\.)?%s:$method /' % domain,'%s' % path),
|
||||
('.*:https?://(.*\.)?%s:$method /static/$anything' % domain,'/%s/static/$anything' % app),
|
||||
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' % domain,'/%s/appadmin/$anything' % app),
|
||||
('.*:https?://(.*\.)?%s:$method /$anything' % domain,'%s/$anything' % path),
|
||||
]
|
||||
('.*:https?://(.*\.)?%s:$method /' % domain, '%s' % path),
|
||||
('.*:https?://(.*\.)?%s:$method /static/$anything' %
|
||||
domain, '/%s/static/$anything' % app),
|
||||
('.*:https?://(.*\.)?%s:$method /appadmin/$anything' %
|
||||
domain, '/%s/appadmin/$anything' % app),
|
||||
('.*:https?://(.*\.)?%s:$method /$anything' %
|
||||
domain, '%s/$anything' % path),
|
||||
]
|
||||
return routes
|
||||
|
||||
|
||||
def auto_out(apps):
|
||||
routes = []
|
||||
for domain,path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
|
||||
if not path.startswith('/'): path = '/'+path
|
||||
if path.endswith('/'): path = path[:-1]
|
||||
for domain, path in [x.strip().split() for x in apps.split('\n') if x.strip() and not x.strip().startswith('#')]:
|
||||
if not path.startswith('/'):
|
||||
path = '/' + path
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
app = path.split('/')[1]
|
||||
routes += [
|
||||
('/%s/static/$anything' % app,'/static/$anything'),
|
||||
('/%s/static/$anything' % app, '/static/$anything'),
|
||||
('/%s/appadmin/$anything' % app, '/appadmin/$anything'),
|
||||
('%s/$anything' % path, '/$anything'),
|
||||
]
|
||||
]
|
||||
return routes
|
||||
|
||||
routes_in = auto_in(config)
|
||||
routes_out = auto_out(config)
|
||||
|
||||
|
||||
def __routes_doctest():
|
||||
'''
|
||||
Dummy function for doctesting autoroutes.py.
|
||||
@@ -128,7 +138,8 @@ if __name__ == '__main__':
|
||||
try:
|
||||
import gluon.main
|
||||
except ImportError:
|
||||
import sys, os
|
||||
import sys
|
||||
import os
|
||||
os.chdir(os.path.dirname(os.path.dirname(__file__)))
|
||||
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
|
||||
import gluon.main
|
||||
@@ -138,4 +149,3 @@ if __name__ == '__main__':
|
||||
|
||||
import doctest
|
||||
doctest.testmod()
|
||||
|
||||
+4
-3
@@ -5,11 +5,12 @@ import urllib2
|
||||
|
||||
n = int(sys.argv[1])
|
||||
url = sys.argv[2]
|
||||
headers = {"Accept-Language" : "en" }
|
||||
headers = {"Accept-Language": "en"}
|
||||
req = urllib2.Request(url, None, headers)
|
||||
|
||||
t0 = time.time()
|
||||
for k in xrange(n):
|
||||
data = urllib2.urlopen(req).read()
|
||||
print (time.time()-t0)/n
|
||||
if n==1: print data
|
||||
print (time.time() - t0) / n
|
||||
if n == 1:
|
||||
print data
|
||||
+12
-13
@@ -8,26 +8,25 @@ filename = sys.argv[1]
|
||||
|
||||
datafile = open(filename, 'r')
|
||||
try:
|
||||
data = '\n'+datafile.read()
|
||||
data = '\n' + datafile.read()
|
||||
finally:
|
||||
datafile.close()
|
||||
SPACE = '\n ' if '-n' in sys.argv[1:] else ' '
|
||||
|
||||
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */',data)
|
||||
data = re.compile('(?<!\:)//(?P<a>.*)').sub('/* \g<a> */', data)
|
||||
data = re.compile('[ ]+').sub(' ', data)
|
||||
data = re.compile('\s*{\s*').sub(' {'+SPACE, data)
|
||||
data = re.compile('\s*;\s*').sub(';'+SPACE, data)
|
||||
data = re.compile('\s*{\s*').sub(' {' + SPACE, data)
|
||||
data = re.compile('\s*;\s*').sub(';' + SPACE, data)
|
||||
data = re.compile(',\s*').sub(', ', data)
|
||||
data = re.compile('\s*\*/\s*').sub('*/'+SPACE, data)
|
||||
data = re.compile('\s*}\s*').sub(SPACE+'}\n', data)
|
||||
data = re.compile('\s*\*/\s*').sub('*/' + SPACE, data)
|
||||
data = re.compile('\s*}\s*').sub(SPACE + '}\n', data)
|
||||
data = re.compile('\n\s*\n').sub('\n', data)
|
||||
data = re.compile(';\s+/\*').sub('; /*',data)
|
||||
data = re.compile('\*/\s+/\*').sub(' ',data)
|
||||
data = re.compile(';\s+/\*').sub('; /*', data)
|
||||
data = re.compile('\*/\s+/\*').sub(' ', data)
|
||||
data = re.compile('[ ]+\n').sub('\n', data)
|
||||
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/',re.DOTALL).sub(
|
||||
'\n/*\g<a>*/\n',data)
|
||||
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{',data)
|
||||
data = data.replace('}','}\n')
|
||||
data = re.compile('\n\s*/[\*]+(?P<a>.*?)[\*]+/', re.DOTALL).sub(
|
||||
'\n/*\g<a>*/\n', data)
|
||||
data = re.compile('[ \t]+(?P<a>\S.+?){').sub('\g<a>{', data)
|
||||
data = data.replace('}', '}\n')
|
||||
|
||||
print data
|
||||
|
||||
+41
-38
@@ -1,51 +1,55 @@
|
||||
import sys
|
||||
import re
|
||||
|
||||
|
||||
def cleancss(text):
|
||||
text=re.compile('\s+').sub(' ', text)
|
||||
text=re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
|
||||
text=re.compile('\s*;\s*').sub(';\n ', text)
|
||||
text=re.compile('\s*\{\s*').sub(' {\n ', text)
|
||||
text=re.compile('\s*\}\s*').sub('\n}\n\n', text)
|
||||
text = re.compile('\s+').sub(' ', text)
|
||||
text = re.compile('\s*(?P<a>,|:)\s*').sub('\g<a> ', text)
|
||||
text = re.compile('\s*;\s*').sub(';\n ', text)
|
||||
text = re.compile('\s*\{\s*').sub(' {\n ', text)
|
||||
text = re.compile('\s*\}\s*').sub('\n}\n\n', text)
|
||||
return text
|
||||
|
||||
|
||||
def cleanhtml(text):
|
||||
text=text.lower()
|
||||
r=re.compile('\<script.+?/script\>', re.DOTALL)
|
||||
scripts=r.findall(text)
|
||||
text=r.sub('<script />', text)
|
||||
r=re.compile('\<style.+?/style\>', re.DOTALL)
|
||||
styles=r.findall(text)
|
||||
text=r.sub('<style />', text)
|
||||
text=re.compile(
|
||||
text = text.lower()
|
||||
r = re.compile('\<script.+?/script\>', re.DOTALL)
|
||||
scripts = r.findall(text)
|
||||
text = r.sub('<script />', text)
|
||||
r = re.compile('\<style.+?/style\>', re.DOTALL)
|
||||
styles = r.findall(text)
|
||||
text = r.sub('<style />', text)
|
||||
text = re.compile(
|
||||
'<(?P<tag>(input|meta|link|hr|br|img|param))(?P<any>[^\>]*)\s*(?<!/)>')\
|
||||
.sub('<\g<tag>\g<any> />', text)
|
||||
text=text.replace('\n', ' ')
|
||||
text=text.replace('>', '>\n')
|
||||
text=text.replace('<', '\n<')
|
||||
text=re.compile('\s*\n\s*').sub('\n', text)
|
||||
lines=text.split('\n')
|
||||
(indent, newlines)=(0, [])
|
||||
text = text.replace('\n', ' ')
|
||||
text = text.replace('>', '>\n')
|
||||
text = text.replace('<', '\n<')
|
||||
text = re.compile('\s*\n\s*').sub('\n', text)
|
||||
lines = text.split('\n')
|
||||
(indent, newlines) = (0, [])
|
||||
for line in lines:
|
||||
if line[:2]=='</': indent=indent-1
|
||||
newlines.append(indent*' '+line)
|
||||
if not line[:2]=='</' and line[-1:]=='>' and \
|
||||
not line[-2:] in ['/>', '->']: indent=indent+1
|
||||
text='\n'.join(newlines)
|
||||
text=re.compile('\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>',text)
|
||||
text=re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>',text)
|
||||
text=re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>',text)
|
||||
text=re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>',text)
|
||||
text=re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>',text)
|
||||
text=re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>',text)
|
||||
text=re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>',text)
|
||||
text=re.compile('\n\s*\n').sub('\n',text)
|
||||
if line[:2] == '</': indent = indent - 1
|
||||
newlines.append(indent * ' ' + line)
|
||||
if not line[:2] == '</' and line[-1:] == '>' and \
|
||||
not line[-2:] in ['/>', '->']: indent = indent + 1
|
||||
text = '\n'.join(newlines)
|
||||
text = re.compile(
|
||||
'\<div(?P<a>( .+)?)\>\s+\</div\>').sub('<div\g<a>></div>', text)
|
||||
text = re.compile('\<a(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</a\>').sub('<a\g<a>>\g<b></a>', text)
|
||||
text = re.compile('\<b(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</b\>').sub('<b\g<a>>\g<b></b>', text)
|
||||
text = re.compile('\<i(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</i\>').sub('<i\g<a>>\g<b></i>', text)
|
||||
text = re.compile('\<span(?P<a>( .+)?)\>\s+(?P<b>[\w\s\(\)\/]+?)\s+\</span\>').sub('<span\g<a>>\g<b></span>', text)
|
||||
text = re.compile('\s+\<br(?P<a>.*?)\/\>').sub('<br\g<a>/>', text)
|
||||
text = re.compile('\>(?P<a>\s+)(?P<b>[\.\,\:\;])').sub('>\g<b>\g<a>', text)
|
||||
text = re.compile('\n\s*\n').sub('\n', text)
|
||||
for script in scripts:
|
||||
text=text.replace('<script />', script, 1)
|
||||
text = text.replace('<script />', script, 1)
|
||||
for style in styles:
|
||||
text=text.replace('<style />', cleancss(style), 1)
|
||||
text = text.replace('<style />', cleancss(style), 1)
|
||||
return text
|
||||
|
||||
|
||||
def read_file(filename):
|
||||
f = open(filename, 'r')
|
||||
try:
|
||||
@@ -53,9 +57,8 @@ def read_file(filename):
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
file=sys.argv[1]
|
||||
if file[-4:]=='.css':
|
||||
file = sys.argv[1]
|
||||
if file[-4:] == '.css':
|
||||
print cleancss(read_file(file))
|
||||
if file[-5:]=='.html':
|
||||
if file[-5:] == '.html':
|
||||
print cleanhtml(read_file(file))
|
||||
|
||||
+9
-9
@@ -1,17 +1,17 @@
|
||||
import re
|
||||
|
||||
|
||||
def cleanjs(text):
|
||||
text = re.sub('\s*}\s*','\n}\n',text)
|
||||
text = re.sub('\s*{\s*',' {\n',text)
|
||||
text = re.sub('\s*;\s*',';\n',text)
|
||||
text = re.sub('\s*,\s*',', ',text)
|
||||
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*',' \g<a> ',text)
|
||||
text = re.sub('\s*}\s*', '\n}\n', text)
|
||||
text = re.sub('\s*{\s*', ' {\n', text)
|
||||
text = re.sub('\s*;\s*', ';\n', text)
|
||||
text = re.sub('\s*,\s*', ', ', text)
|
||||
text = re.sub('\s*(?P<a>[\+\-\*/\=]+)\s*', ' \g<a> ', text)
|
||||
lines = text.split('\n')
|
||||
text=''
|
||||
indent=0
|
||||
text = ''
|
||||
indent = 0
|
||||
for line in lines:
|
||||
rline=line.strip()
|
||||
rline = line.strip()
|
||||
if rline:
|
||||
pass
|
||||
return text
|
||||
|
||||
@@ -22,7 +22,7 @@ Internet connection is required to perform the update.
|
||||
OVERRIDE = [
|
||||
('.pdb', 'chemical/x-pdb'),
|
||||
('.xyz', 'chemical/x-pdb')
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
class MIMEParser(dict):
|
||||
@@ -72,14 +72,15 @@ if __name__ == '__main__':
|
||||
sys.stdout.write('Checking freedesktop.org database version:')
|
||||
sys.stdout.flush()
|
||||
try:
|
||||
search = re.search('(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
|
||||
search = re.search(
|
||||
'(?P<url>http://freedesktop.org/.+?/shared-mime-info-(?P<version>.+?)\.tar\.(?P<type>[gb]z2?))',
|
||||
urllib.urlopen('http://www.freedesktop.org/wiki/Software/shared-mime-info').read())
|
||||
url = search.group('url')
|
||||
assert url != None
|
||||
assert url is not None
|
||||
nversion = search.group('version')
|
||||
assert nversion != None
|
||||
assert nversion is not None
|
||||
ftype = search.group('type')
|
||||
assert ftype != None
|
||||
assert ftype is not None
|
||||
sys.stdout.write('\t[OK] version %s\n' % nversion)
|
||||
except:
|
||||
sys.stdout.write('\t[ERROR] unknown version\n')
|
||||
@@ -129,4 +130,3 @@ if __name__ == '__main__':
|
||||
sys.stdout.write('\t\t\t[OK] done\n')
|
||||
except Exception, e:
|
||||
sys.stdout.write('\t\t\t[ERROR] %s\n' % e)
|
||||
|
||||
+311
-301
@@ -1,9 +1,11 @@
|
||||
|
||||
import os,sys
|
||||
import os
|
||||
import sys
|
||||
from collections import deque
|
||||
import string
|
||||
import argparse
|
||||
import cStringIO,operator
|
||||
import cStringIO
|
||||
import operator
|
||||
import cPickle as pickle
|
||||
from collections import deque
|
||||
import math
|
||||
@@ -15,18 +17,19 @@ try:
|
||||
except ImportError as err:
|
||||
print('gluon path not found')
|
||||
|
||||
|
||||
class refTable(object):
|
||||
def __init__(self):
|
||||
self.columns = None
|
||||
self.rows = None
|
||||
|
||||
def getcolHeader(self,colHeader):
|
||||
def getcolHeader(self, colHeader):
|
||||
return "{0}".format(' | '.join([string.join(string.strip('**{0}**'.format(item)),
|
||||
'') for item in colHeader]))
|
||||
'') for item in colHeader]))
|
||||
|
||||
|
||||
def wrapTable(self,rows, hasHeader=False, headerChar='-', delim=' | ', justify='left',
|
||||
separateRows=False, prefix='', postfix='', wrapfunc=lambda x:x):
|
||||
def wrapTable(
|
||||
self, rows, hasHeader=False, headerChar='-', delim=' | ', justify='left',
|
||||
separateRows=False, prefix='', postfix='', wrapfunc=lambda x: x):
|
||||
|
||||
def rowWrapper(row):
|
||||
|
||||
@@ -40,38 +43,36 @@ class refTable(object):
|
||||
rowList gives like newRows but
|
||||
formatted like [[w, x, y, z]]
|
||||
---'''
|
||||
rowList = [[substr or '' for substr in item] for item in map(None,*newRows)]
|
||||
rowList = [[substr or '' for substr in item]
|
||||
for item in map(None, *newRows)]
|
||||
return rowList
|
||||
|
||||
logicalRows = [rowWrapper(row) for row in rows]
|
||||
|
||||
columns = map(None,*reduce(operator.add,logicalRows))
|
||||
columns = map(None, *reduce(operator.add, logicalRows))
|
||||
self.columns = columns
|
||||
|
||||
maxWidths = [max(\
|
||||
[len(str\
|
||||
(item)) for \
|
||||
item in column]\
|
||||
) for column \
|
||||
in columns]
|
||||
maxWidths = [max(
|
||||
[len(str
|
||||
(item)) for
|
||||
item in column]
|
||||
) for column
|
||||
in columns]
|
||||
|
||||
rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) + \
|
||||
len(delim)*(len(maxWidths)-1))
|
||||
rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) +
|
||||
len(delim) * (len(maxWidths) - 1))
|
||||
|
||||
justify = {'center'\
|
||||
:str\
|
||||
.center,
|
||||
'right'\
|
||||
:str\
|
||||
.rjust,
|
||||
'left'\
|
||||
:str.\
|
||||
ljust\
|
||||
}[justify\
|
||||
.lower(\
|
||||
)]
|
||||
justify = {'center': str
|
||||
.center,
|
||||
'right': str
|
||||
.rjust,
|
||||
'left': str.
|
||||
ljust
|
||||
}[justify
|
||||
.lower(
|
||||
)]
|
||||
|
||||
output=cStringIO.StringIO()
|
||||
output = cStringIO.StringIO()
|
||||
|
||||
if separateRows:
|
||||
print >> output, rowSeparator
|
||||
@@ -79,191 +80,199 @@ class refTable(object):
|
||||
for physicalRows in logicalRows:
|
||||
for row in physicalRows:
|
||||
print >> output,\
|
||||
prefix + delim.join([\
|
||||
justify(str(item),width) for (\
|
||||
item,width) in zip(row,maxWidths)]\
|
||||
) + postfix
|
||||
prefix + delim.join([
|
||||
justify(str(item), width) for (
|
||||
item, width) in zip(row, maxWidths)]
|
||||
) + postfix
|
||||
|
||||
if separateRows or hasHeader:
|
||||
print >> output, rowSeparator; hasHeader=False
|
||||
print >> output, rowSeparator
|
||||
hasHeader = False
|
||||
return output.getvalue()
|
||||
|
||||
def wrap_onspace(self,text,width):
|
||||
return reduce(lambda line, word, width=width: '{0}{1}{2}'\
|
||||
.format(line\
|
||||
,' \n'[(len(\
|
||||
line[line.rfind('\n'\
|
||||
) + 1:]) + len(\
|
||||
word.split('\n',1)[0]) >=\
|
||||
width)],word),text.split(' '))
|
||||
def wrap_onspace(self, text, width):
|
||||
return reduce(lambda line, word, width=width: '{0}{1}{2}'
|
||||
.format(line, ' \n'[(len(
|
||||
line[line.rfind('\n'
|
||||
) + 1:]) + len(
|
||||
word.split('\n', 1)[0]) >=
|
||||
width)], word), text.split(' '))
|
||||
|
||||
def wrap_onspace_strict(self,text,width):
|
||||
wordRegex = re.compile(r'\S{'+str(width)+r',}')
|
||||
return self.wrap_onspace(\
|
||||
wordRegex.sub(\
|
||||
lambda m: self.\
|
||||
wrap_always(\
|
||||
m.group(),width),text\
|
||||
),width)
|
||||
def wrap_onspace_strict(self, text, width):
|
||||
wordRegex = re.compile(r'\S{' + str(width) + r',}')
|
||||
return self.wrap_onspace(
|
||||
wordRegex.sub(
|
||||
lambda m: self.
|
||||
wrap_always(
|
||||
m.group(), width), text
|
||||
), width)
|
||||
|
||||
def wrap_always(self, text, width):
|
||||
return '\n'.join(
|
||||
[text[width * i:width * (i + 1
|
||||
)] for i in xrange(
|
||||
int(math.ceil(1. * len(
|
||||
text) / width)))])
|
||||
|
||||
def wrap_always(self,text,width):
|
||||
return '\n'.join(\
|
||||
[ text[width*i:width*(i+1\
|
||||
)] for i in xrange(\
|
||||
int(math.ceil(1.*len(\
|
||||
text)/width))) ])
|
||||
|
||||
class tableHelper():
|
||||
def __init__(self):
|
||||
self.oTable = refTable()
|
||||
|
||||
def getAsRows(self,data):
|
||||
return [row.strip().split(',') for row in data.splitlines()]
|
||||
def getAsRows(self, data):
|
||||
return [row.strip().split(',') for row in data.splitlines()]
|
||||
|
||||
def getTable_noWrap(self,data,header=None):
|
||||
def getTable_noWrap(self, data, header=None):
|
||||
rows = self.getAsRows(data)
|
||||
if header is not None:hRows = [header]+rows
|
||||
else:hRows = rows
|
||||
if header is not None:
|
||||
hRows = [header] + rows
|
||||
else:
|
||||
hRows = rows
|
||||
table = self.oTable.wrapTable(hRows, hasHeader=True)
|
||||
return table
|
||||
|
||||
def getTable_Wrap(self,data,wrapStyle,header=None,width=65):
|
||||
def getTable_Wrap(self, data, wrapStyle, header=None, width=65):
|
||||
wrapper = None
|
||||
if len(wrapStyle) > 1:
|
||||
rows = self.getAsRows(data)
|
||||
if header is not None:hRows = [header]+rows
|
||||
else:hRows = rows
|
||||
if header is not None:
|
||||
hRows = [header] + rows
|
||||
else:
|
||||
hRows = rows
|
||||
|
||||
for wrapper in (self.oTable.wrap_always,
|
||||
self.oTable.wrap_onspace,
|
||||
self.oTable.wrap_onspace_strict):
|
||||
return self.oTable.wrapTable(hRows\
|
||||
,hasHeader=True\
|
||||
,separateRows=True\
|
||||
,prefix='| '\
|
||||
,postfix=' |'\
|
||||
,wrapfunc\
|
||||
=lambda x:\
|
||||
wrapper(x,width))
|
||||
self.oTable.wrap_onspace,
|
||||
self.oTable.wrap_onspace_strict):
|
||||
return self.oTable.wrapTable(hRows, hasHeader=True, separateRows=True, prefix='| ', postfix=' |', wrapfunc=lambda x:
|
||||
wrapper(x, width))
|
||||
else:
|
||||
return self.getTable_noWrap(data,header)
|
||||
return self.getTable_noWrap(data, header)
|
||||
|
||||
def getAsErrorTable(self,err):
|
||||
return self.getTable_Wrap(err,None)
|
||||
def getAsErrorTable(self, err):
|
||||
return self.getTable_Wrap(err, None)
|
||||
|
||||
|
||||
class console:
|
||||
def __init__(self,prompt,banner=None):
|
||||
self.prompt=prompt
|
||||
self.banner=banner
|
||||
self.commands={}
|
||||
self.commandSort=[]
|
||||
self.db=None
|
||||
def __init__(self, prompt, banner=None):
|
||||
self.prompt = prompt
|
||||
self.banner = banner
|
||||
self.commands = {}
|
||||
self.commandSort = []
|
||||
self.db = None
|
||||
|
||||
for i in dir(self):
|
||||
if "cmd_"==i[:4]:
|
||||
cmd=i.split("cmd_")[1].lower()
|
||||
self.commands[cmd]=getattr(self,i)
|
||||
try:self.commandSort.append((int(self\
|
||||
.commands[cmd].__doc__.split(\
|
||||
"|")[0]),cmd))
|
||||
except:pass
|
||||
if "cmd_" == i[:4]:
|
||||
cmd = i.split("cmd_")[1].lower()
|
||||
self.commands[cmd] = getattr(self, i)
|
||||
try:
|
||||
self.commandSort.append((int(self
|
||||
.commands[cmd].__doc__.split(
|
||||
"|")[0]), cmd))
|
||||
except:
|
||||
pass
|
||||
|
||||
self.commandSort.sort()
|
||||
self.commandSort=[i[1] for i in self.commandSort]
|
||||
self.commandSort = [i[1] for i in self.commandSort]
|
||||
|
||||
self.var_DEBUG=False
|
||||
self.var_tableStyle=''
|
||||
self.var_DEBUG = False
|
||||
self.var_tableStyle = ''
|
||||
|
||||
self.configvars={}
|
||||
self.configvars = {}
|
||||
for i in dir(self):
|
||||
if "var_"==i[:4]:
|
||||
var=i.split("var_")[1]
|
||||
self.configvars[var]=i
|
||||
if "var_" == i[:4]:
|
||||
var = i.split("var_")[1]
|
||||
self.configvars[var] = i
|
||||
|
||||
def setBanner(self,banner):
|
||||
self.banner=banner
|
||||
def setBanner(self, banner):
|
||||
self.banner = banner
|
||||
|
||||
def execCmd(self,db):
|
||||
self.db=db
|
||||
def execCmd(self, db):
|
||||
self.db = db
|
||||
print self.banner
|
||||
while True:
|
||||
try:
|
||||
command=raw_input(self.prompt)
|
||||
command = raw_input(self.prompt)
|
||||
try:
|
||||
self.execCommand(command)
|
||||
except:
|
||||
self.execute(command)
|
||||
except KeyboardInterrupt:break
|
||||
except EOFError:break
|
||||
except Exception,a:self.printError (a)
|
||||
except KeyboardInterrupt:
|
||||
break
|
||||
except EOFError:
|
||||
break
|
||||
except Exception, a:
|
||||
self.printError(a)
|
||||
print ("\r\n\r\nBye!...")
|
||||
sys.exit(0)
|
||||
|
||||
def printError(self,err):
|
||||
def printError(self, err):
|
||||
sys.stderr.write("Error: {0}\r\n".format(str(err),))
|
||||
if self.var_DEBUG:pass
|
||||
if self.var_DEBUG:
|
||||
pass
|
||||
|
||||
def execute(self,cmd):
|
||||
def execute(self, cmd):
|
||||
try:
|
||||
if not '-table ' in cmd:
|
||||
exec '{0}'.format(cmd)
|
||||
else:
|
||||
file=None
|
||||
table=None
|
||||
file = None
|
||||
table = None
|
||||
|
||||
fields=[]
|
||||
items=string.split(cmd,' ')
|
||||
invalidParams=[]
|
||||
table=self.getTable(items[1])
|
||||
allowedParams=['fields','file']
|
||||
fields = []
|
||||
items = string.split(cmd, ' ')
|
||||
invalidParams = []
|
||||
table = self.getTable(items[1])
|
||||
allowedParams = ['fields', 'file']
|
||||
for i in items:
|
||||
if '=' in i and not string.split(i,'=')[0] in allowedParams:
|
||||
if '=' in i and not string.split(i, '=')[0] in allowedParams:
|
||||
try:
|
||||
invalidParams.append(i)
|
||||
except Exception, err:
|
||||
raise Exception, 'invalid parameter\n{0}'.format(i)
|
||||
raise Exception('invalid parameter\n{0}'.format(i))
|
||||
else:
|
||||
if 'file=' in i:
|
||||
file=os.path.abspath(string.strip(string.split(i,'=')[1]))
|
||||
file = os.path.abspath(string.strip(string.split(
|
||||
i, '=')[1]))
|
||||
if 'fields=' in i:
|
||||
for field in string.split(string.split(i,'=')[1],','):
|
||||
for field in string.split(string.split(i, '=')[1], ','):
|
||||
if field in self.db[table].fields:
|
||||
fields.append(string.strip(field))
|
||||
|
||||
if len(invalidParams)>0:
|
||||
print('the following parameter(s) is not valid\n{0}'.format(\
|
||||
string.join(invalidParams,',')))
|
||||
if len(invalidParams) > 0:
|
||||
print('the following parameter(s) is not valid\n{0}'.format(
|
||||
string.join(invalidParams, ',')))
|
||||
else:
|
||||
try:
|
||||
self.cmd_table(table,file,fields)
|
||||
self.cmd_table(table, file, fields)
|
||||
except Exception, err:
|
||||
print('could not generate table for table {0}\n{1}'\
|
||||
.format(table,err))
|
||||
print('could not generate table for table {0}\n{1}'
|
||||
.format(table, err))
|
||||
except Exception, err:
|
||||
print('sorry, can not do that!\n{0}'.format(err))
|
||||
|
||||
def getTable(self,tbl):
|
||||
def getTable(self, tbl):
|
||||
for mTbl in db.tables:
|
||||
if tbl in mTbl:
|
||||
if mTbl.startswith(tbl):
|
||||
return mTbl
|
||||
|
||||
def execCommand(self,cmd):
|
||||
words=cmd.split(" ")
|
||||
words=[i for i in words if i]
|
||||
if not words:return
|
||||
cmd,parameters=words[0].lower(),words[1:]
|
||||
def execCommand(self, cmd):
|
||||
words = cmd.split(" ")
|
||||
words = [i for i in words if i]
|
||||
if not words:
|
||||
return
|
||||
cmd, parameters = words[0].lower(), words[1:]
|
||||
|
||||
if not cmd in self.commands:
|
||||
raise Exception("Command {0} not found. Try 'help'\r\n".format(cmd))
|
||||
raise Exception(
|
||||
"Command {0} not found. Try 'help'\r\n".format(cmd))
|
||||
|
||||
self.commands[cmd](*parameters)
|
||||
|
||||
'''---
|
||||
DEFAULT COMMANDS (begins with cmd_)
|
||||
---'''
|
||||
def cmd_clear(self,numlines=100):
|
||||
def cmd_clear(self, numlines=100):
|
||||
"""-5|clear|clear the screen"""
|
||||
if os.name == "posix":
|
||||
'''---
|
||||
@@ -279,9 +288,9 @@ class console:
|
||||
'''---
|
||||
Fallback for other operating systems.
|
||||
---'''
|
||||
print '\n'*numlines
|
||||
print '\n' * numlines
|
||||
|
||||
def cmd_table(self,tbl,file=None,fields=[]):
|
||||
def cmd_table(self, tbl, file=None, fields=[]):
|
||||
"""-4|-table [TABLENAME] optional[file=None] [fields=None]|\
|
||||
the default tableStyle is no_wrap - use the 'set x y' command to change the style\n\
|
||||
style choices:
|
||||
@@ -291,13 +300,13 @@ style choices:
|
||||
\tno_wrap (value '')\n
|
||||
\t the 2nd optional param is a path to a file where the table will be written
|
||||
\t the 3rd optional param is a list of fields you want displayed\n"""
|
||||
table=None
|
||||
table = None
|
||||
for mTbl in db.tables:
|
||||
if tbl in mTbl:
|
||||
if mTbl.startswith(tbl):
|
||||
table=mTbl
|
||||
table = mTbl
|
||||
break
|
||||
oTable=tableHelper()
|
||||
oTable = tableHelper()
|
||||
'''---
|
||||
tablestyle:
|
||||
wrap_always
|
||||
@@ -305,98 +314,106 @@ style choices:
|
||||
wrap_onspace_strict
|
||||
or set set to "" for no wrapping
|
||||
---'''
|
||||
tableStyle=self.var_tableStyle
|
||||
filedNotFound=[]
|
||||
table_fields=None
|
||||
if len(fields)==0:
|
||||
table_fields=self.db[table].fields
|
||||
tableStyle = self.var_tableStyle
|
||||
filedNotFound = []
|
||||
table_fields = None
|
||||
if len(fields) == 0:
|
||||
table_fields = self.db[table].fields
|
||||
else:
|
||||
table_fields=fields
|
||||
table_fields = fields
|
||||
|
||||
for field in fields:
|
||||
if not field in self.db[table].fields:
|
||||
filedNotFound.append(field)
|
||||
if len(filedNotFound)==0:
|
||||
rows=self.db(self.db[table].id>0).select()
|
||||
rows_data=[]
|
||||
if len(filedNotFound) == 0:
|
||||
rows = self.db(self.db[table].id > 0).select()
|
||||
rows_data = []
|
||||
for row in rows:
|
||||
rowdata=[]
|
||||
rowdata = []
|
||||
for f in table_fields:
|
||||
rowdata.append('{0}'.format(row[f]))
|
||||
rows_data.append(string.join(rowdata,','))
|
||||
data=string.join(rows_data,'\n')
|
||||
dataTable=oTable.getTable_Wrap(data,tableStyle,table_fields)
|
||||
print('TABLE {0}\n{1}'.format(table,dataTable))
|
||||
if file!=None:
|
||||
rows_data.append(string.join(rowdata, ','))
|
||||
data = string.join(rows_data, '\n')
|
||||
dataTable = oTable.getTable_Wrap(data, tableStyle, table_fields)
|
||||
print('TABLE {0}\n{1}'.format(table, dataTable))
|
||||
if file is not None:
|
||||
try:
|
||||
tail,head=os.path.split(file)
|
||||
tail, head = os.path.split(file)
|
||||
try:
|
||||
os.makedirs(tail)
|
||||
except:'do nothing, folders exist'
|
||||
oFile=open(file,'w')
|
||||
oFile.write('TABLE: {0}\n{1}'.format(table,dataTable))
|
||||
except:
|
||||
'do nothing, folders exist'
|
||||
oFile = open(file, 'w')
|
||||
oFile.write('TABLE: {0}\n{1}'.format(table, dataTable))
|
||||
oFile.close()
|
||||
print('{0} has been created and populated with all available data from table {1}\n'.format(file,table))
|
||||
print('{0} has been created and populated with all available data from table {1}\n'.format(file, table))
|
||||
except Exception, err:
|
||||
print("EXCEPTION: could not create table {0}\n{1}".format(table,err))
|
||||
print("EXCEPTION: could not create table {0}\n{1}".format(
|
||||
table, err))
|
||||
else:
|
||||
print('the following fields are not valid [{0}]'.format(string.join(filedNotFound,',')))
|
||||
print('the following fields are not valid [{0}]'.format(
|
||||
string.join(filedNotFound, ',')))
|
||||
|
||||
def cmd_help(self,*args):
|
||||
def cmd_help(self, *args):
|
||||
'''-3|help|Show's help'''
|
||||
alldata=[]
|
||||
lengths=[]
|
||||
alldata = []
|
||||
lengths = []
|
||||
|
||||
for i in self.commandSort:alldata.append(\
|
||||
self.commands[i].__doc__.split("|")[1:])
|
||||
for i in self.commandSort:
|
||||
alldata.append(
|
||||
self.commands[i].__doc__.split("|")[1:])
|
||||
|
||||
for i in alldata:
|
||||
if len(i) > len(lengths):
|
||||
for j in range(len(i)\
|
||||
-len(lengths)):
|
||||
for j in range(len(i)
|
||||
- len(lengths)):
|
||||
lengths.append(0)
|
||||
|
||||
j=0
|
||||
while j<len(i):
|
||||
if len(i[j])>lengths[j]:
|
||||
lengths[j]=len(i[j])
|
||||
j+=1
|
||||
j = 0
|
||||
while j < len(i):
|
||||
if len(i[j]) > lengths[j]:
|
||||
lengths[j] = len(i[j])
|
||||
j += 1
|
||||
|
||||
print ("-"*(lengths[0]+lengths[1]+4))
|
||||
print ("-" * (lengths[0] + lengths[1] + 4))
|
||||
for i in alldata:
|
||||
print (("%-"+str(lengths[0])+"s - %-"+str(lengths[1])+"s") % (i[0],i[1]))
|
||||
if len(i)>2:
|
||||
for j in i[2:]:print (("%"+str(lengths[0]+9)+"s* %s") % (" ",j))
|
||||
print (("%-" + str(lengths[0]) + "s - %-" + str(
|
||||
lengths[1]) + "s") % (i[0], i[1]))
|
||||
if len(i) > 2:
|
||||
for j in i[2:]: print (("%" + str(lengths[
|
||||
0] + 9) + "s* %s") % (" ", j))
|
||||
print
|
||||
|
||||
def cmd_vars(self,*args):
|
||||
def cmd_vars(self, *args):
|
||||
'''-2|vars|Show variables'''
|
||||
print ("variables\r\n"+"-"*79)
|
||||
for i,j in self.configvars.items():
|
||||
value=self.parfmt(repr(getattr(self,j)),52)
|
||||
print ("| %20s | %52s |" % (i,value[0]))
|
||||
for k in value[1:]:print ("| %20s | %52s |" % ("",k))
|
||||
if len(value)>1:print("| %20s | %52s |" % ("",""))
|
||||
print ("-"*79)
|
||||
print ("variables\r\n" + "-" * 79)
|
||||
for i, j in self.configvars.items():
|
||||
value = self.parfmt(repr(getattr(self, j)), 52)
|
||||
print ("| %20s | %52s |" % (i, value[0]))
|
||||
for k in value[1:]: print ("| %20s | %52s |" % ("", k))
|
||||
if len(value) > 1:
|
||||
print("| %20s | %52s |" % ("", ""))
|
||||
print ("-" * 79)
|
||||
|
||||
def parfmt(self,txt,width):
|
||||
res=[]
|
||||
pos=0
|
||||
def parfmt(self, txt, width):
|
||||
res = []
|
||||
pos = 0
|
||||
while True:
|
||||
a=txt[pos:pos+width]
|
||||
if not a:break
|
||||
a = txt[pos:pos + width]
|
||||
if not a:
|
||||
break
|
||||
res.append(a)
|
||||
pos+=width
|
||||
pos += width
|
||||
return res
|
||||
|
||||
def cmd_set(self,*args):
|
||||
def cmd_set(self, *args):
|
||||
'''-1|set [variable_name] [value]|Set configuration variable value|Values are an expressions (100 | string.lower('ABC') | etc.'''
|
||||
value=" ".join(args[1:])
|
||||
value = " ".join(args[1:])
|
||||
if args[0] not in self.configvars:
|
||||
setattr(self,"var_{0}".format(args[0]),eval(value))
|
||||
setattr(self,"var_{0}".format(args[0]),eval(value))
|
||||
setattr(self, "var_{0}".format(args[0]), eval(value))
|
||||
setattr(self, "var_{0}".format(args[0]), eval(value))
|
||||
|
||||
def cmd_clearscreen(self,numlines=50):
|
||||
def cmd_clearscreen(self, numlines=50):
|
||||
'''---Clear the console.
|
||||
---'''
|
||||
if os.name == "posix":
|
||||
@@ -413,72 +430,78 @@ style choices:
|
||||
'''---
|
||||
Fallback for other operating systems.
|
||||
---'''
|
||||
print '\n'*numlines
|
||||
print '\n' * numlines
|
||||
|
||||
|
||||
class dalShell(console):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def shell(self,db):
|
||||
console.__init__(self,prompt=">>> ",banner='dal interactive shell')
|
||||
def shell(self, db):
|
||||
console.__init__(self, prompt=">>> ", banner='dal interactive shell')
|
||||
self.execCmd(db)
|
||||
|
||||
|
||||
class setCopyDB():
|
||||
def __init__(self):
|
||||
'''---
|
||||
non source or target specific vars
|
||||
---'''
|
||||
self.strModel=None
|
||||
self.dalPath=None
|
||||
self.db=None
|
||||
self.strModel = None
|
||||
self.dalPath = None
|
||||
self.db = None
|
||||
'''---
|
||||
source vars
|
||||
---'''
|
||||
self.sourceModel=None
|
||||
self.sourceFolder=None
|
||||
self.sourceConnectionString=None
|
||||
self.sourcedbType=None
|
||||
self.sourcedbName=None
|
||||
self.sourceModel = None
|
||||
self.sourceFolder = None
|
||||
self.sourceConnectionString = None
|
||||
self.sourcedbType = None
|
||||
self.sourcedbName = None
|
||||
'''---
|
||||
target vars
|
||||
---'''
|
||||
self.targetdbType=None
|
||||
self.targetdbName=None
|
||||
self.targetModel=None
|
||||
self.targetFolder=None
|
||||
self.targetConnectionString=None
|
||||
self.truncate=False
|
||||
self.targetdbType = None
|
||||
self.targetdbName = None
|
||||
self.targetModel = None
|
||||
self.targetFolder = None
|
||||
self.targetConnectionString = None
|
||||
self.truncate = False
|
||||
|
||||
def _getDal(self):
|
||||
mDal=None
|
||||
mDal = None
|
||||
if self.dalPath is not None:
|
||||
global DAL
|
||||
sys.path.append(self.dalPath)
|
||||
mDal=__import__('dal',globals={},locals={},fromlist=['DAL'],level=0)
|
||||
DAL=mDal.DAL
|
||||
mDal = __import__(
|
||||
'dal', globals={}, locals={}, fromlist=['DAL'], level=0)
|
||||
DAL = mDal.DAL
|
||||
return mDal
|
||||
|
||||
def instDB(self,storageFolder,storageConnectionString,autoImport):
|
||||
self.db=DAL(storageConnectionString,folder=os.path.abspath(storageFolder),auto_import=autoImport)
|
||||
def instDB(self, storageFolder, storageConnectionString, autoImport):
|
||||
self.db = DAL(storageConnectionString, folder=os.path.abspath(
|
||||
storageFolder), auto_import=autoImport)
|
||||
return self.db
|
||||
|
||||
def delete_DB_tables(self,storageFolder,storageType):
|
||||
print 'delete_DB_tablesn\n\t{0}\n\t{1}'.format(storageFolder,storageType)
|
||||
dataFiles=[storageType,"sql.log"]
|
||||
def delete_DB_tables(self, storageFolder, storageType):
|
||||
print 'delete_DB_tablesn\n\t{0}\n\t{1}'.format(
|
||||
storageFolder, storageType)
|
||||
dataFiles = [storageType, "sql.log"]
|
||||
try:
|
||||
for f in os.listdir(storageFolder):
|
||||
if ".table" in f:
|
||||
fTable="{0}/{1}".format(storageFolder,f)
|
||||
fTable = "{0}/{1}".format(storageFolder, f)
|
||||
os.remove(fTable)
|
||||
print('deleted {0}'.format(fTable))
|
||||
for dFile in dataFiles:
|
||||
os.remove("{0}/{1}".format(storageFolder,dFile))
|
||||
print('deleted {0}'.format("{0}/{1}".format(storageFolder,dFile)))
|
||||
os.remove("{0}/{1}".format(storageFolder, dFile))
|
||||
print('deleted {0}'.format(
|
||||
"{0}/{1}".format(storageFolder, dFile)))
|
||||
except Exception, errObj:
|
||||
print(str(errObj))
|
||||
|
||||
def truncatetables(self,tables=[]):
|
||||
if len(tables)!=0:
|
||||
def truncatetables(self, tables=[]):
|
||||
if len(tables) != 0:
|
||||
try:
|
||||
print 'table value: {0}'.format(tables)
|
||||
for tbl in self.db.tables:
|
||||
@@ -495,12 +518,14 @@ class setCopyDB():
|
||||
print('EXCEPTION: {0}'.format(err))
|
||||
|
||||
def copyDB(self):
|
||||
other_db=DAL("{0}://{1}".format(self.targetdbType,self.targetdbName),folder=self.targetFolder)
|
||||
other_db = DAL("{0}://{1}".format(
|
||||
self.targetdbType, self.targetdbName), folder=self.targetFolder)
|
||||
|
||||
print 'creating tables...'
|
||||
|
||||
for table in self.db:
|
||||
other_db.define_table(table._tablename,*[field for field in table])
|
||||
other_db.define_table(
|
||||
table._tablename, *[field for field in table])
|
||||
'''
|
||||
should there be an option to truncAte target DB?
|
||||
if yes, then change args to allow for choice
|
||||
@@ -511,43 +536,38 @@ class setCopyDB():
|
||||
'''
|
||||
|
||||
print 'exporting data...'
|
||||
self.db.export_to_csv_file(open('tmp.sql','wb'))
|
||||
self.db.export_to_csv_file(open('tmp.sql', 'wb'))
|
||||
|
||||
print 'importing data...'
|
||||
other_db.import_from_csv_file(open('tmp.sql','rb'))
|
||||
other_db.import_from_csv_file(open('tmp.sql', 'rb'))
|
||||
other_db.commit()
|
||||
print 'done!'
|
||||
print 'Attention: do not run this program again or you end up with duplicate records'
|
||||
|
||||
def createfolderPath(self,folder):
|
||||
def createfolderPath(self, folder):
|
||||
try:
|
||||
if folder!=None:os.makedirs(folder)
|
||||
if folder is not None:
|
||||
os.makedirs(folder)
|
||||
except Exception, err:
|
||||
pass
|
||||
|
||||
if __name__ == '__main__':
|
||||
oCopy=setCopyDB()
|
||||
db=None
|
||||
targetDB=None
|
||||
dbfolder=None
|
||||
clean=False
|
||||
model=None
|
||||
truncate=False
|
||||
oCopy = setCopyDB()
|
||||
db = None
|
||||
targetDB = None
|
||||
dbfolder = None
|
||||
clean = False
|
||||
model = None
|
||||
truncate = False
|
||||
|
||||
parser=argparse.ArgumentParser(description='\
|
||||
parser = argparse.ArgumentParser(description='\
|
||||
samplecmd line:\n\
|
||||
-f ./blueLite/db_storage -i -y sqlite://storage.sqlite -Y sqlite://storage2.sqlite -d ./blueLite/pyUtils/sql/blueSQL -t True',
|
||||
epilog = '')
|
||||
reqGroup=parser.add_argument_group('Required arguments')
|
||||
reqGroup.add_argument('-f','--sourceFolder'\
|
||||
,required=True\
|
||||
,help="path to the 'source' folder of the 'source' DB")
|
||||
reqGroup.add_argument('-F','--targetFolder'\
|
||||
,required=False\
|
||||
,help="path to the 'target' folder of the 'target' DB")
|
||||
reqGroup.add_argument('-y','--sourceConnectionString'\
|
||||
,required=True\
|
||||
,help="source db connection string ()\n\
|
||||
epilog='')
|
||||
reqGroup = parser.add_argument_group('Required arguments')
|
||||
reqGroup.add_argument('-f', '--sourceFolder', required=True, help="path to the 'source' folder of the 'source' DB")
|
||||
reqGroup.add_argument('-F', '--targetFolder', required=False, help="path to the 'target' folder of the 'target' DB")
|
||||
reqGroup.add_argument('-y', '--sourceConnectionString', required=True, help="source db connection string ()\n\
|
||||
------------------------------------------------\n\
|
||||
\
|
||||
sqlite://storage.db\n\
|
||||
@@ -561,13 +581,10 @@ ingres://username:password@localhost/test\n\
|
||||
informix://username:password@test\n\
|
||||
\
|
||||
------------------------------------------------")
|
||||
reqGroup.add_argument('-Y','--targetConnectionString'\
|
||||
,required=True\
|
||||
,help="target db type (sqlite,mySql,etc.)")
|
||||
autoImpGroup=parser.add_argument_group('optional args (auto_import)')
|
||||
autoImpGroup.add_argument('-a','--autoimport'\
|
||||
,required=False\
|
||||
,help='set to True to bypass loading of the model')
|
||||
reqGroup.add_argument('-Y', '--targetConnectionString', required=True,
|
||||
help="target db type (sqlite,mySql,etc.)")
|
||||
autoImpGroup = parser.add_argument_group('optional args (auto_import)')
|
||||
autoImpGroup.add_argument('-a', '--autoimport', required=False, help='set to True to bypass loading of the model')
|
||||
|
||||
"""
|
||||
|
||||
@@ -584,66 +601,58 @@ informix://username:password@test\n\
|
||||
|
||||
"""
|
||||
|
||||
miscGroup = parser.add_argument_group('optional args/tasks')
|
||||
miscGroup.add_argument('-i', '--interactive', required=False, action='store_true', help='run in interactive mode')
|
||||
miscGroup.add_argument(
|
||||
'-d', '--dal', required=False, help='path to dal.py')
|
||||
miscGroup.add_argument('-t', '--truncate', choices=['True', 'False'], help='delete the records but *not* the table of the SOURCE DB')
|
||||
miscGroup.add_argument('-b', '--tables', required=False, type=list, help='optional list (comma delimited) of SOURCE tables to truncate, defaults to all')
|
||||
miscGroup.add_argument('-c', '--clean', required=False, help='delete the DB,tables and the log file, WARNING: this is unrecoverable')
|
||||
|
||||
miscGroup=parser.add_argument_group('optional args/tasks')
|
||||
miscGroup.add_argument('-i','--interactive'\
|
||||
,required=False\
|
||||
,action='store_true'\
|
||||
,help='run in interactive mode')
|
||||
miscGroup.add_argument('-d','--dal'\
|
||||
,required=False\
|
||||
,help='path to dal.py')
|
||||
miscGroup.add_argument('-t','--truncate'\
|
||||
,choices=['True','False']\
|
||||
,help='delete the records but *not* the table of the SOURCE DB')
|
||||
miscGroup.add_argument('-b','--tables'\
|
||||
,required=False\
|
||||
,type=list\
|
||||
,help='optional list (comma delimited) of SOURCE tables to truncate, defaults to all')
|
||||
miscGroup.add_argument('-c','--clean'\
|
||||
,required=False\
|
||||
,help='delete the DB,tables and the log file, WARNING: this is unrecoverable')
|
||||
|
||||
args=parser.parse_args()
|
||||
db=None
|
||||
mDal=None
|
||||
args = parser.parse_args()
|
||||
db = None
|
||||
mDal = None
|
||||
|
||||
try:
|
||||
oCopy.sourceFolder=args.sourceFolder
|
||||
oCopy.targetFolder=args.sourceFolder
|
||||
sourceItems=string.split(args.sourceConnectionString,'://')
|
||||
oCopy.sourcedbType=sourceItems[0]
|
||||
oCopy.sourcedbName=sourceItems[1]
|
||||
targetItems=string.split(args.targetConnectionString,'://')
|
||||
oCopy.targetdbType=targetItems[0]
|
||||
oCopy.targetdbName=targetItems[1]
|
||||
oCopy.sourceFolder = args.sourceFolder
|
||||
oCopy.targetFolder = args.sourceFolder
|
||||
sourceItems = string.split(args.sourceConnectionString, '://')
|
||||
oCopy.sourcedbType = sourceItems[0]
|
||||
oCopy.sourcedbName = sourceItems[1]
|
||||
targetItems = string.split(args.targetConnectionString, '://')
|
||||
oCopy.targetdbType = targetItems[0]
|
||||
oCopy.targetdbName = targetItems[1]
|
||||
except Exception, err:
|
||||
print('EXCEPTION: {0}'.format(err))
|
||||
|
||||
if args.dal:
|
||||
try:
|
||||
autoImport=True
|
||||
if args.autoimport:autoImport=args.autoimport
|
||||
autoImport = True
|
||||
if args.autoimport:
|
||||
autoImport = args.autoimport
|
||||
#sif not DAL in globals:
|
||||
#if not sys.path.__contains__():
|
||||
oCopy.dalPath=args.dal
|
||||
mDal=oCopy._getDal()
|
||||
db=oCopy.instDB(args.sourceFolder,args.sourceConnectionString,autoImport)
|
||||
oCopy.dalPath = args.dal
|
||||
mDal = oCopy._getDal()
|
||||
db = oCopy.instDB(args.sourceFolder, args.sourceConnectionString,
|
||||
autoImport)
|
||||
except Exception, err:
|
||||
print('EXCEPTION: could not set DAL\n{0}'.format(err))
|
||||
if args.truncate:
|
||||
try:
|
||||
if args.truncate:
|
||||
if args.tables:tables=string.split(string.strip(args.tables),',')
|
||||
else:oCopy.truncatetables([])
|
||||
if args.tables:
|
||||
tables = string.split(string.strip(args.tables), ',')
|
||||
else:
|
||||
oCopy.truncatetables([])
|
||||
except Exception, err:
|
||||
print('EXCEPTION: could not truncate tables\n{0}'.format(err))
|
||||
try:
|
||||
if args.clean:oCopy.delete_DB_tables(oCopy.targetFolder,oCopy.targetType)
|
||||
if args.clean:
|
||||
oCopy.delete_DB_tables(oCopy.targetFolder, oCopy.targetType)
|
||||
except Exception, err:
|
||||
print('EXCEPTION: could not clean db\n{0}'.format(err))
|
||||
|
||||
|
||||
"""
|
||||
*** goes with -m/-M options... removed for now
|
||||
|
||||
@@ -661,14 +670,15 @@ target model: {1}\n\
|
||||
|
||||
if args.sourceFolder:
|
||||
try:
|
||||
oCopy.sourceFolder=os.path.abspath(args.sourceFolder)
|
||||
oCopy.sourceFolder = os.path.abspath(args.sourceFolder)
|
||||
oCopy.createfolderPath(oCopy.sourceFolder)
|
||||
except Exception, err:
|
||||
print('EXCEPTION: could not create folder path\n{0}'.format(err))
|
||||
else:oCopy.dbStorageFolder=os.path.abspath(os.getcwd())
|
||||
else:
|
||||
oCopy.dbStorageFolder = os.path.abspath(os.getcwd())
|
||||
if args.targetFolder:
|
||||
try:
|
||||
oCopy.targetFolder=os.path.abspath(args.targetFolder)
|
||||
oCopy.targetFolder = os.path.abspath(args.targetFolder)
|
||||
oCopy.createfolderPath(oCopy.targetFolder)
|
||||
except Exception, err:
|
||||
print('EXCEPTION: could not create folder path\n{0}'.format(err))
|
||||
@@ -678,5 +688,5 @@ target model: {1}\n\
|
||||
except Exception, err:
|
||||
print('EXCEPTION: could not make a copy of the database\n{0}'.format(err))
|
||||
else:
|
||||
s=dalShell()
|
||||
s = dalShell()
|
||||
s.shell(db)
|
||||
+20
-14
@@ -1,26 +1,32 @@
|
||||
import sys, glob, os, shutil
|
||||
name=sys.argv[1]
|
||||
app=sys.argv[2]
|
||||
dest=sys.argv[3]
|
||||
a=glob.glob('applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app,name=name))
|
||||
b=glob.glob('applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app,name=name))
|
||||
import sys
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
name = sys.argv[1]
|
||||
app = sys.argv[2]
|
||||
dest = sys.argv[3]
|
||||
a = glob.glob(
|
||||
'applications/%(app)s/*/plugin_%(name)s.*' % dict(app=app, name=name))
|
||||
b = glob.glob(
|
||||
'applications/%(app)s/*/plugin_%(name)s/*' % dict(app=app, name=name))
|
||||
|
||||
for f in a:
|
||||
print 'cp %s ...' % f,
|
||||
shutil.copyfile(f,os.path.join('applications',dest,*f.split('/')[2:]))
|
||||
shutil.copyfile(f, os.path.join('applications', dest, *f.split('/')[2:]))
|
||||
print 'done'
|
||||
|
||||
for f in b:
|
||||
print 'cp %s ...' % f,
|
||||
path = f.split('/')
|
||||
for i in range(3,len(path)):
|
||||
try: os.mkdir(os.path.join('applications',dest,*path[2:i]))
|
||||
except: pass
|
||||
path = os.path.join('applications',dest,*f.split('/')[2:])
|
||||
for i in range(3, len(path)):
|
||||
try:
|
||||
os.mkdir(os.path.join('applications', dest, *path[2:i]))
|
||||
except:
|
||||
pass
|
||||
path = os.path.join('applications', dest, *f.split('/')[2:])
|
||||
if os.path.isdir(f):
|
||||
if not os.path.exists(path):
|
||||
shutil.copytree(f,path)
|
||||
shutil.copytree(f, path)
|
||||
else:
|
||||
shutil.copyfile(f,path)
|
||||
shutil.copyfile(f, path)
|
||||
print 'done'
|
||||
|
||||
@@ -10,17 +10,16 @@ DB_URI = 'sqlite://sessions.sqlite'
|
||||
EXPIRATION_MINUTES = 60
|
||||
SLEEP_MINUTES = 5
|
||||
|
||||
while 1: # Infinite loop
|
||||
now = time() # get current Unix timestamp
|
||||
while 1: # Infinite loop
|
||||
now = time() # get current Unix timestamp
|
||||
|
||||
for row in db().select(db.web2py_session_welcome.ALL):
|
||||
t = row.modified_datetime
|
||||
# Convert to a Unix timestamp
|
||||
t = mktime(t.timetuple())+1e-6*t.microsecond
|
||||
t = mktime(t.timetuple()) + 1e-6 * t.microsecond
|
||||
if now - t > EXPIRATION_MINUTES * 60:
|
||||
del db.web2py_session_welcome[row.id]
|
||||
|
||||
db.commit() # Write changes to database
|
||||
db.commit() # Write changes to database
|
||||
|
||||
sleep(SLEEP_MINUTES * 60)
|
||||
|
||||
@@ -22,6 +22,7 @@ import getopt
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
|
||||
def main(argv):
|
||||
"""Parse the arguments and start the main process."""
|
||||
try:
|
||||
@@ -40,16 +41,19 @@ def main(argv):
|
||||
else:
|
||||
exit_with_parsing_error()
|
||||
|
||||
|
||||
def exit_with_parsing_error():
|
||||
"""Report invalid arguments and usage."""
|
||||
print("Invalid argument(s).")
|
||||
usage()
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def usage():
|
||||
"""Display the documentation"""
|
||||
print(__doc__)
|
||||
|
||||
|
||||
def get_dicts(dict_path1, dict_path2):
|
||||
"""
|
||||
Parse the dictionaries.
|
||||
@@ -60,6 +64,7 @@ def get_dicts(dict_path1, dict_path2):
|
||||
|
||||
return eval(open(dict_path1).read()), eval(open(dict_path2).read())
|
||||
|
||||
|
||||
def get_dict_names(dict1_path, dict2_path):
|
||||
"""
|
||||
Get the name of the dictionaries for the end user. Use the base name of the
|
||||
@@ -76,6 +81,7 @@ def get_dict_names(dict1_path, dict2_path):
|
||||
dict2_name = "dict2"
|
||||
return dict1_name, dict2_name
|
||||
|
||||
|
||||
def compare_dicts(dict1, dict2, dict1_name, dict2_name):
|
||||
"""
|
||||
Compare the two dictionaries. Print out the result.
|
||||
@@ -101,6 +107,7 @@ def compare_dicts(dict1, dict2, dict1_name, dict2_name):
|
||||
if not has_value_differences:
|
||||
print " None"
|
||||
|
||||
|
||||
def print_key_diff(key_diff, dict1_name, dict2_name):
|
||||
"""
|
||||
Prints the keys in the first dictionary and are in the second dictionary.
|
||||
|
||||
@@ -24,45 +24,48 @@ import subprocess
|
||||
import re
|
||||
import sys
|
||||
data_type_map = dict(
|
||||
varchar = 'string',
|
||||
int = 'integer',
|
||||
integer = 'integer',
|
||||
tinyint = 'integer',
|
||||
smallint = 'integer',
|
||||
mediumint = 'integer',
|
||||
bigint = 'integer',
|
||||
float = 'double',
|
||||
double = 'double',
|
||||
char = 'string',
|
||||
decimal = 'integer',
|
||||
date = 'date',
|
||||
varchar='string',
|
||||
int='integer',
|
||||
integer='integer',
|
||||
tinyint='integer',
|
||||
smallint='integer',
|
||||
mediumint='integer',
|
||||
bigint='integer',
|
||||
float='double',
|
||||
double='double',
|
||||
char='string',
|
||||
decimal='integer',
|
||||
date='date',
|
||||
#year = 'date',
|
||||
time = 'time',
|
||||
timestamp = 'datetime',
|
||||
datetime = 'datetime',
|
||||
binary = 'blob',
|
||||
blob = 'blob',
|
||||
tinyblob = 'blob',
|
||||
mediumblob = 'blob',
|
||||
longblob = 'blob',
|
||||
text = 'text',
|
||||
tinytext = 'text',
|
||||
mediumtext = 'text',
|
||||
longtext = 'text',
|
||||
)
|
||||
time='time',
|
||||
timestamp='datetime',
|
||||
datetime='datetime',
|
||||
binary='blob',
|
||||
blob='blob',
|
||||
tinyblob='blob',
|
||||
mediumblob='blob',
|
||||
longblob='blob',
|
||||
text='text',
|
||||
tinytext='text',
|
||||
mediumtext='text',
|
||||
longtext='text',
|
||||
)
|
||||
|
||||
|
||||
def mysql(database_name, username, password):
|
||||
p = subprocess.Popen(['mysql',
|
||||
'--user=%s' % username,
|
||||
'--password=%s'% password,
|
||||
'--password=%s' % password,
|
||||
'--execute=show tables;',
|
||||
database_name],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
sql_showtables, stderr = p.communicate()
|
||||
tables = [re.sub('\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
|
||||
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')"%(username, password, database_name)
|
||||
tables = [re.sub(
|
||||
'\|\s+([^\|*])\s+.*', '\1', x) for x in sql_showtables.split()[1:]]
|
||||
connection_string = "legacy_db = DAL('mysql://%s:%s@localhost/%s')" % (
|
||||
username, password, database_name)
|
||||
legacy_db_table_web2py_code = []
|
||||
for table_name in tables:
|
||||
#get the sql create statement
|
||||
@@ -71,37 +74,40 @@ def mysql(database_name, username, password):
|
||||
'--password=%s' % password,
|
||||
'--skip-add-drop-table',
|
||||
'--no-data', database_name,
|
||||
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
||||
sql_create_stmnt,stderr = p.communicate()
|
||||
if 'CREATE' in sql_create_stmnt:#check if the table exists
|
||||
table_name], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
sql_create_stmnt, stderr = p.communicate()
|
||||
if 'CREATE' in sql_create_stmnt: # check if the table exists
|
||||
#remove garbage lines from sql statement
|
||||
sql_lines = sql_create_stmnt.split('\n')
|
||||
sql_lines = [x for x in sql_lines if not(x.startswith('--') or x.startswith('/*') or x =='')]
|
||||
sql_lines = [x for x in sql_lines if not(
|
||||
x.startswith('--') or x.startswith('/*') or x == '')]
|
||||
#generate the web2py code from the create statement
|
||||
web2py_table_code = ''
|
||||
table_name = re.search('CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
|
||||
table_name = re.search(
|
||||
'CREATE TABLE .(\S+). \(', sql_lines[0]).group(1)
|
||||
fields = []
|
||||
for line in sql_lines[1:-1]:
|
||||
if re.search('KEY', line) or re.search('PRIMARY', line) or re.search(' ID', line) or line.startswith(')'):
|
||||
continue
|
||||
hit = re.search('(\S+)\s+(\S+)(,| )( .*)?', line)
|
||||
if hit!=None:
|
||||
if hit is not None:
|
||||
name, d_type = hit.group(1), hit.group(2)
|
||||
d_type = re.sub(r'(\w+)\(.*',r'\1',d_type)
|
||||
name = re.sub('`','',name)
|
||||
web2py_table_code += "\n Field('%s','%s'),"%(name,data_type_map[d_type])
|
||||
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)"%(table_name,web2py_table_code)
|
||||
d_type = re.sub(r'(\w+)\(.*', r'\1', d_type)
|
||||
name = re.sub('`', '', name)
|
||||
web2py_table_code += "\n Field('%s','%s')," % (
|
||||
name, data_type_map[d_type])
|
||||
web2py_table_code = "legacy_db.define_table('%s',%s\n migrate=False)" % (table_name, web2py_table_code)
|
||||
legacy_db_table_web2py_code.append(web2py_table_code)
|
||||
#----------------------------------------
|
||||
#write the legacy db to file
|
||||
legacy_db_web2py_code = connection_string+"\n\n"
|
||||
legacy_db_web2py_code += "\n\n#--------\n".join(legacy_db_table_web2py_code)
|
||||
legacy_db_web2py_code = connection_string + "\n\n"
|
||||
legacy_db_web2py_code += "\n\n#--------\n".join(
|
||||
legacy_db_table_web2py_code)
|
||||
return legacy_db_web2py_code
|
||||
|
||||
regex = re.compile('(.*?):(.*?)@(.*)')
|
||||
if len(sys.argv)<2 or not regex.match(sys.argv[1]):
|
||||
if len(sys.argv) < 2 or not regex.match(sys.argv[1]):
|
||||
print 'USAGE:\n\n extract_mysql_models.py username:password@data_basename\n\n'
|
||||
else:
|
||||
m = regex.match(sys.argv[1])
|
||||
print mysql(m.group(3),m.group(1),m.group(2))
|
||||
|
||||
print mysql(m.group(3), m.group(1), m.group(2))
|
||||
@@ -46,19 +46,21 @@ KWARGS = ('type', 'length', 'default', 'required', 'ondelete',
|
||||
import sys
|
||||
|
||||
|
||||
def query(conn, sql,*args):
|
||||
def query(conn, sql, *args):
|
||||
"Execute a SQL query and return rows as a list of dicts"
|
||||
cur = conn.cursor()
|
||||
ret = []
|
||||
try:
|
||||
if DEBUG: print >> sys.stderr, "QUERY: ", sql % args
|
||||
if DEBUG:
|
||||
print >> sys.stderr, "QUERY: ", sql % args
|
||||
cur.execute(sql, args)
|
||||
for row in cur:
|
||||
dic = {}
|
||||
for i, value in enumerate(row):
|
||||
field = cur.description[i][0]
|
||||
dic[field] = value
|
||||
if DEBUG: print >> sys.stderr, "RET: ", dic
|
||||
if DEBUG:
|
||||
print >> sys.stderr, "RET: ", dic
|
||||
ret.append(dic)
|
||||
return ret
|
||||
finally:
|
||||
@@ -75,7 +77,8 @@ def get_tables(conn, schema=SCHEMA):
|
||||
|
||||
def get_fields(conn, table):
|
||||
"Retrieve field list for a given table"
|
||||
if DEBUG: print >> sys.stderr, "Processing TABLE", table
|
||||
if DEBUG:
|
||||
print >> sys.stderr, "Processing TABLE", table
|
||||
rows = query(conn, """
|
||||
SELECT column_name, data_type,
|
||||
is_nullable,
|
||||
@@ -90,13 +93,13 @@ def get_fields(conn, table):
|
||||
|
||||
def define_field(conn, table, field, pks):
|
||||
"Determine field type, default value, references, etc."
|
||||
f={}
|
||||
f = {}
|
||||
ref = references(conn, table, field['column_name'])
|
||||
if ref:
|
||||
f.update(ref)
|
||||
elif field['column_default'] and \
|
||||
field['column_default'].startswith("nextval") and \
|
||||
field['column_name'] in pks:
|
||||
field['column_default'].startswith("nextval") and \
|
||||
field['column_name'] in pks:
|
||||
# postgresql sequence (SERIAL) and primary key!
|
||||
f['type'] = "'id'"
|
||||
elif field['data_type'].startswith('character'):
|
||||
@@ -109,7 +112,7 @@ def define_field(conn, table, field, pks):
|
||||
f['type'] = "'boolean'"
|
||||
elif field['data_type'] in ('integer', 'smallint', 'bigint'):
|
||||
f['type'] = "'integer'"
|
||||
elif field['data_type'] in ('double precision', 'real' ):
|
||||
elif field['data_type'] in ('double precision', 'real'):
|
||||
f['type'] = "'double'"
|
||||
elif field['data_type'] in ('timestamp', 'timestamp without time zone'):
|
||||
f['type'] = "'datetime'"
|
||||
@@ -124,17 +127,17 @@ def define_field(conn, table, field, pks):
|
||||
elif field['data_type'] in ('bytea', ):
|
||||
f['type'] = "'blob'"
|
||||
elif field['data_type'] in ('point', 'lseg', 'polygon', 'unknown', 'USER-DEFINED'):
|
||||
f['type'] = "" # unsupported?
|
||||
f['type'] = "" # unsupported?
|
||||
else:
|
||||
raise RuntimeError("Data Type not supported: %s " % str(field))
|
||||
|
||||
try:
|
||||
if field['column_default']:
|
||||
if field['column_default']=="now()":
|
||||
if field['column_default'] == "now()":
|
||||
d = "request.now"
|
||||
elif field['column_default']=="true":
|
||||
elif field['column_default'] == "true":
|
||||
d = "True"
|
||||
elif field['column_default']=="false":
|
||||
elif field['column_default'] == "false":
|
||||
d = "False"
|
||||
else:
|
||||
d = repr(eval(field['column_default']))
|
||||
@@ -142,7 +145,8 @@ def define_field(conn, table, field, pks):
|
||||
except (ValueError, SyntaxError):
|
||||
pass
|
||||
except Exception, e:
|
||||
raise RuntimeError("Default unsupported '%s'" % field['column_default'])
|
||||
raise RuntimeError(
|
||||
"Default unsupported '%s'" % field['column_default'])
|
||||
|
||||
if not field['is_nullable']:
|
||||
f['notnull'] = "True"
|
||||
@@ -203,40 +207,40 @@ def references(conn, table, field):
|
||||
AND information_schema.key_column_usage.column_name=%s
|
||||
AND information_schema.table_constraints.constraint_type='FOREIGN KEY'
|
||||
;""", table, field)
|
||||
if len(rows1)==1:
|
||||
if len(rows1) == 1:
|
||||
rows2 = query(conn, """
|
||||
SELECT table_name, column_name, *
|
||||
FROM information_schema.constraint_column_usage
|
||||
WHERE constraint_name=%s
|
||||
""", rows1[0]['constraint_name'])
|
||||
row = None
|
||||
if len(rows2)>1:
|
||||
row = rows2[int(rows1[0]['ordinal_position'])-1]
|
||||
if len(rows2) > 1:
|
||||
row = rows2[int(rows1[0]['ordinal_position']) - 1]
|
||||
keyed = True
|
||||
if len(rows2)==1:
|
||||
if len(rows2) == 1:
|
||||
row = rows2[0]
|
||||
keyed = False
|
||||
if row:
|
||||
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
|
||||
if keyed: # THIS IS BAD, DON'T MIX "id" and primarykey!!!
|
||||
ref = {'type': "'reference %s.%s'" % (row['table_name'],
|
||||
row['column_name'])}
|
||||
else:
|
||||
ref = {'type': "'reference %s'" % (row['table_name'],)}
|
||||
if rows1[0]['delete_rule']!="NO ACTION":
|
||||
if rows1[0]['delete_rule'] != "NO ACTION":
|
||||
ref['ondelete'] = repr(rows1[0]['delete_rule'])
|
||||
return ref
|
||||
elif rows2:
|
||||
raise RuntimeError("Unsupported foreign key reference: %s" %
|
||||
str(rows2))
|
||||
str(rows2))
|
||||
|
||||
elif rows1:
|
||||
raise RuntimeError("Unsupported referential constraint: %s" %
|
||||
str(rows1))
|
||||
str(rows1))
|
||||
|
||||
|
||||
def define_table(conn, table):
|
||||
"Output single table definition"
|
||||
fields = get_fields(conn, table)
|
||||
fields = get_fields(conn, table)
|
||||
pks = primarykeys(conn, table)
|
||||
print "db.define_table('%s'," % (table, )
|
||||
for field in fields:
|
||||
@@ -244,11 +248,11 @@ def define_table(conn, table):
|
||||
fdef = define_field(conn, table, field, pks)
|
||||
if fname not in pks and is_unique(conn, table, field):
|
||||
fdef['unique'] = "True"
|
||||
if fdef['type']=="'id'" and fname in pks:
|
||||
if fdef['type'] == "'id'" and fname in pks:
|
||||
pks.pop(pks.index(fname))
|
||||
print " Field('%s', %s)," % (fname,
|
||||
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
|
||||
if k in fdef and fdef[k]]))
|
||||
', '.join(["%s=%s" % (k, fdef[k]) for k in KWARGS
|
||||
if k in fdef and fdef[k]]))
|
||||
if pks:
|
||||
print " primarykey=[%s]," % ", ".join(["'%s'" % pk for pk in pks])
|
||||
print " migrate=migrate)"
|
||||
@@ -280,5 +284,3 @@ if __name__ == "__main__":
|
||||
)
|
||||
# Start model code generation:
|
||||
define_db(cnn, db, host, port, user, passwd)
|
||||
|
||||
|
||||
+7
-4
@@ -1,4 +1,6 @@
|
||||
import sys, glob
|
||||
import sys
|
||||
import glob
|
||||
|
||||
|
||||
def read_fileb(filename, mode='rb'):
|
||||
f = open(filename, mode)
|
||||
@@ -7,6 +9,7 @@ def read_fileb(filename, mode='rb'):
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
|
||||
def write_fileb(filename, value, mode='wb'):
|
||||
f = open(filename, mode)
|
||||
try:
|
||||
@@ -18,7 +21,7 @@ for filename in glob.glob(sys.argv[1]):
|
||||
data1 = read_fileb(filename)
|
||||
write_fileb(filename + '.bak2', data1)
|
||||
data2lines = read_fileb(filename).strip().split('\n')
|
||||
data2 = '\n'.join([line.rstrip().replace('\t',' '*2) for line in data2lines])+'\n'
|
||||
data2 = '\n'.join([line.rstrip(
|
||||
).replace('\t', ' ' * 2) for line in data2lines]) + '\n'
|
||||
write_fileb(filename, data2)
|
||||
print filename, len(data1)-len(data2)
|
||||
|
||||
print filename, len(data1) - len(data2)
|
||||
+28
-24
@@ -5,10 +5,11 @@ import sys
|
||||
import re
|
||||
from BeautifulSoup import BeautifulSoup as BS
|
||||
|
||||
|
||||
def head(styles):
|
||||
title = '<title>{{=response.title or request.application}}</title>'
|
||||
items = '\n'.join(["{{response.files.append(URL(request.application,'static','%s'))}}" % (style) for style in styles])
|
||||
loc="""<style>
|
||||
loc = """<style>
|
||||
div.flash {
|
||||
position: absolute;
|
||||
float: right;
|
||||
@@ -39,50 +40,54 @@ div.error {
|
||||
border: 1px solid #666;
|
||||
}
|
||||
</style>"""
|
||||
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title,items,loc)
|
||||
return "\n%s\n%s\n{{include 'web2py_ajax.html'}}\n%s" % (title, items, loc)
|
||||
|
||||
|
||||
def content():
|
||||
return """<div class="flash">{{=response.flash or ''}}</div>{{include}}"""
|
||||
|
||||
|
||||
def process(folder):
|
||||
indexfile = open(os.path.join(folder,'index.html'),'rb')
|
||||
indexfile = open(os.path.join(folder, 'index.html'), 'rb')
|
||||
try:
|
||||
soup = BS(indexfile.read())
|
||||
finally:
|
||||
indexfile.close()
|
||||
styles = [x['href'] for x in soup.findAll('link')]
|
||||
soup.find('head').contents=BS(head(styles))
|
||||
soup.find('head').contents = BS(head(styles))
|
||||
try:
|
||||
soup.find('h1').contents=BS('{{=response.title or request.application}}')
|
||||
soup.find('h2').contents=BS("{{=response.subtitle or '=response.subtitle'}}")
|
||||
soup.find(
|
||||
'h1').contents = BS('{{=response.title or request.application}}')
|
||||
soup.find('h2').contents = BS(
|
||||
"{{=response.subtitle or '=response.subtitle'}}")
|
||||
except:
|
||||
pass
|
||||
for match in (soup.find('div',id='menu'),
|
||||
soup.find('div',{'class':'menu'}),
|
||||
soup.find('div',id='nav'),
|
||||
soup.find('div',{'class':'nav'})):
|
||||
for match in (soup.find('div', id='menu'),
|
||||
soup.find('div', {'class': 'menu'}),
|
||||
soup.find('div', id='nav'),
|
||||
soup.find('div', {'class': 'nav'})):
|
||||
if match:
|
||||
match.contents=BS('{{=MENU(response.menu)}}')
|
||||
match.contents = BS('{{=MENU(response.menu)}}')
|
||||
break
|
||||
done=False
|
||||
for match in (soup.find('div',id='content'),
|
||||
soup.find('div',{'class':'content'}),
|
||||
soup.find('div',id='main'),
|
||||
soup.find('div',{'class':'main'})):
|
||||
done = False
|
||||
for match in (soup.find('div', id='content'),
|
||||
soup.find('div', {'class': 'content'}),
|
||||
soup.find('div', id='main'),
|
||||
soup.find('div', {'class': 'main'})):
|
||||
if match:
|
||||
match.contents=BS(content())
|
||||
done=True
|
||||
match.contents = BS(content())
|
||||
done = True
|
||||
break
|
||||
if done:
|
||||
page = soup.prettify()
|
||||
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*",re.MULTILINE)\
|
||||
.sub("{{=response.flash or ''}}",page)
|
||||
page = re.compile("\s*\{\{=response\.flash or ''\}\}\s*", re.MULTILINE)\
|
||||
.sub("{{=response.flash or ''}}", page)
|
||||
print page
|
||||
else:
|
||||
raise Exception, "Unable to convert"
|
||||
raise Exception("Unable to convert")
|
||||
|
||||
if __name__=='__main__':
|
||||
if len(sys.argv)<2:
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print """USAGE:
|
||||
1) start a new web2py application
|
||||
2) Download a sample free layout from the web into the static/ folder of
|
||||
@@ -96,4 +101,3 @@ if __name__=='__main__':
|
||||
print 'Folder %s does not exist' % sys.argv[1]
|
||||
else:
|
||||
process(sys.argv[1])
|
||||
|
||||
+29
-20
@@ -38,12 +38,16 @@ gluon/contrib/pyrtf/
|
||||
gluon/contrib/pysimplesoap/
|
||||
"""
|
||||
|
||||
import sys, os, shutil, glob
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
import glob
|
||||
|
||||
|
||||
def main():
|
||||
global REQUIRED, IGNORED
|
||||
|
||||
if len(sys.argv)<2:
|
||||
if len(sys.argv) < 2:
|
||||
print USAGE
|
||||
|
||||
# make target folder
|
||||
@@ -51,38 +55,43 @@ def main():
|
||||
os.mkdir(target)
|
||||
|
||||
# change to os specificsep
|
||||
REQUIRED = REQUIRED.replace('/',os.sep)
|
||||
IGNORED = IGNORED.replace('/',os.sep)
|
||||
|
||||
REQUIRED = REQUIRED.replace('/', os.sep)
|
||||
IGNORED = IGNORED.replace('/', os.sep)
|
||||
|
||||
# make a list of all files to include
|
||||
files = [x.strip() for x in REQUIRED.split('\n') \
|
||||
if x and not x[0]=='#']
|
||||
ignore = [x.strip() for x in IGNORED.split('\n') \
|
||||
if x and not x[0]=='#']
|
||||
files = [x.strip() for x in REQUIRED.split('\n')
|
||||
if x and not x[0] == '#']
|
||||
ignore = [x.strip() for x in IGNORED.split('\n')
|
||||
if x and not x[0] == '#']
|
||||
|
||||
def accept(filename):
|
||||
for p in ignore:
|
||||
if filename.startswith(p):
|
||||
return False
|
||||
return True
|
||||
pattern = os.path.join('gluon','*.py')
|
||||
pattern = os.path.join('gluon', '*.py')
|
||||
while True:
|
||||
newfiles = [x for x in glob.glob(pattern) if accept(x)]
|
||||
if not newfiles: break
|
||||
if not newfiles:
|
||||
break
|
||||
files += newfiles
|
||||
pattern = os.path.join(pattern[:-3],'*.py')
|
||||
pattern = os.path.join(pattern[:-3], '*.py')
|
||||
# copy all files, make missing folder, build default.py
|
||||
files.sort()
|
||||
defaultpy = os.path.join('applications','welcome','controllers','default.py')
|
||||
defaultpy = os.path.join(
|
||||
'applications', 'welcome', 'controllers', 'default.py')
|
||||
for f in files:
|
||||
dirs = f.split(os.path.sep)
|
||||
for i in range(1,len(dirs)):
|
||||
try: os.mkdir(target+os.sep+os.path.join(*dirs[:i]))
|
||||
except OSError: pass
|
||||
if f==defaultpy:
|
||||
open(os.path.join(target,f),'w').write('def index(): return "hello"\n')
|
||||
for i in range(1, len(dirs)):
|
||||
try:
|
||||
os.mkdir(target + os.sep + os.path.join(*dirs[:i]))
|
||||
except OSError:
|
||||
pass
|
||||
if f == defaultpy:
|
||||
open(os.path.join(
|
||||
target, f), 'w').write('def index(): return "hello"\n')
|
||||
else:
|
||||
shutil.copyfile(f,os.path.join(target,f))
|
||||
shutil.copyfile(f, os.path.join(target, f))
|
||||
|
||||
if __name__=='__main__': main()
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -6,18 +6,18 @@ paths2 = []
|
||||
while paths:
|
||||
path = paths.pop()
|
||||
for filename in os.listdir(path):
|
||||
fullname = os.path.join(path,filename)
|
||||
fullname = os.path.join(path, filename)
|
||||
if os.path.isdir(fullname):
|
||||
paths.append(fullname)
|
||||
else:
|
||||
extension = filename.split('.')[-1]
|
||||
if extension.lower() in ('png','gif','jpg','jpeg','js','css'):
|
||||
paths1.append((filename,fullname))
|
||||
if extension.lower() in ('css','js','py','html'):
|
||||
if extension.lower() in ('png', 'gif', 'jpg', 'jpeg', 'js', 'css'):
|
||||
paths1.append((filename, fullname))
|
||||
if extension.lower() in ('css', 'js', 'py', 'html'):
|
||||
paths2.append(fullname)
|
||||
for filename,fullname in paths1:
|
||||
for filename, fullname in paths1:
|
||||
for otherfullname in paths2:
|
||||
if open(otherfullname).read().find(filename)>=0:
|
||||
if open(otherfullname).read().find(filename) >= 0:
|
||||
break
|
||||
else:
|
||||
print fullname
|
||||
|
||||
+17
-17
@@ -162,7 +162,7 @@ class SessionFile(object):
|
||||
|
||||
def last_visit_default(self):
|
||||
return datetime.datetime.fromtimestamp(
|
||||
os.stat(self.filename)[stat.ST_MTIME])
|
||||
os.stat(self.filename)[stat.ST_MTIME])
|
||||
|
||||
def __str__(self):
|
||||
return self.filename
|
||||
@@ -175,7 +175,7 @@ def total_seconds(delta):
|
||||
Args:
|
||||
delta: datetime.timedelta instance.
|
||||
"""
|
||||
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) * \
|
||||
return (delta.microseconds + (delta.seconds + (delta.days * 24 * 3600)) *
|
||||
10 ** 6) / 10 ** 6
|
||||
|
||||
|
||||
@@ -186,25 +186,25 @@ def main():
|
||||
parser = OptionParser(usage=usage)
|
||||
|
||||
parser.add_option('-f', '--force',
|
||||
action='store_true', dest='force', default=False,
|
||||
help=('Ignore session expiration. '
|
||||
'Force expiry based on -x option or auth.settings.expiration.')
|
||||
)
|
||||
action='store_true', dest='force', default=False,
|
||||
help=('Ignore session expiration. '
|
||||
'Force expiry based on -x option or auth.settings.expiration.')
|
||||
)
|
||||
parser.add_option('-o', '--once',
|
||||
action='store_true', dest='once', default=False,
|
||||
help='Delete sessions, then exit.',
|
||||
)
|
||||
action='store_true', dest='once', default=False,
|
||||
help='Delete sessions, then exit.',
|
||||
)
|
||||
parser.add_option('-s', '--sleep',
|
||||
dest='sleep', default=SLEEP_MINUTES * 60, type="int",
|
||||
help='Number of seconds to sleep between executions. Default 300.',
|
||||
)
|
||||
dest='sleep', default=SLEEP_MINUTES * 60, type="int",
|
||||
help='Number of seconds to sleep between executions. Default 300.',
|
||||
)
|
||||
parser.add_option('-v', '--verbose',
|
||||
default=0, action='count',
|
||||
help="print verbose output, a second -v increases verbosity")
|
||||
default=0, action='count',
|
||||
help="print verbose output, a second -v increases verbosity")
|
||||
parser.add_option('-x', '--expiration',
|
||||
dest='expiration', default=None, type="int",
|
||||
help='Expiration value for sessions without expiration (in seconds)',
|
||||
)
|
||||
dest='expiration', default=None, type="int",
|
||||
help='Expiration value for sessions without expiration (in seconds)',
|
||||
)
|
||||
|
||||
(options, unused_args) = parser.parse_args()
|
||||
|
||||
|
||||
@@ -30,35 +30,35 @@ if sys.platform == 'win32':
|
||||
|
||||
base_modules.remove('macpath')
|
||||
buildOptions = dict(
|
||||
compressed = True,
|
||||
excludes = ["macpath","PyQt4"],
|
||||
includes = base_modules,
|
||||
include_files=[
|
||||
'applications',
|
||||
'ABOUT',
|
||||
'LICENSE',
|
||||
'VERSION',
|
||||
'logging.example.conf',
|
||||
'options_std.py',
|
||||
'app.example.yaml',
|
||||
'queue.example.yaml',
|
||||
],
|
||||
# append any extra module by extending the list below -
|
||||
# "contributed_modules+["lxml"]"
|
||||
packages = contributed_modules,
|
||||
)
|
||||
compressed=True,
|
||||
excludes=["macpath", "PyQt4"],
|
||||
includes=base_modules,
|
||||
include_files=[
|
||||
'applications',
|
||||
'ABOUT',
|
||||
'LICENSE',
|
||||
'VERSION',
|
||||
'logging.example.conf',
|
||||
'options_std.py',
|
||||
'app.example.yaml',
|
||||
'queue.example.yaml',
|
||||
],
|
||||
# append any extra module by extending the list below -
|
||||
# "contributed_modules+["lxml"]"
|
||||
packages=contributed_modules,
|
||||
)
|
||||
|
||||
setup(
|
||||
name = "Web2py",
|
||||
version=web2py_version,
|
||||
author="Massimo DiPierro",
|
||||
description="web2py web framework",
|
||||
license = "LGPL v3",
|
||||
options = dict(build_exe = buildOptions),
|
||||
executables = [Executable("web2py.py",
|
||||
base=base,
|
||||
compress = True,
|
||||
icon = "web2py.ico",
|
||||
targetName="web2py.exe",
|
||||
copyDependentFiles = True)],
|
||||
)
|
||||
name="Web2py",
|
||||
version=web2py_version,
|
||||
author="Massimo DiPierro",
|
||||
description="web2py web framework",
|
||||
license="LGPL v3",
|
||||
options=dict(build_exe=buildOptions),
|
||||
executables=[Executable("web2py.py",
|
||||
base=base,
|
||||
compress=True,
|
||||
icon="web2py.ico",
|
||||
targetName="web2py.exe",
|
||||
copyDependentFiles=True)],
|
||||
)
|
||||
@@ -14,6 +14,7 @@ sys.path.insert(0, '.')
|
||||
file = sys.argv[1]
|
||||
apps = sys.argv[2:]
|
||||
|
||||
|
||||
def sync_language(d, data):
|
||||
''' this function makes sure a translated string will be prefered over an untranslated
|
||||
string when syncing languages between apps. when both are translated, it prefers the
|
||||
@@ -26,9 +27,9 @@ def sync_language(d, data):
|
||||
d[key] = data[key]
|
||||
# see if there is a translated string in the original list, but not in the new list
|
||||
elif (
|
||||
((d[key] != '') or (d[key] != key)) and
|
||||
((data[key] == '') or (data[key] == key))
|
||||
):
|
||||
((d[key] != '') or (d[key] != key)) and
|
||||
((data[key] == '') or (data[key] == key))
|
||||
):
|
||||
d[key] = d[key]
|
||||
# any other case (wether there is or there isn't a translated string)
|
||||
else:
|
||||
@@ -68,4 +69,3 @@ for app in oapps:
|
||||
path2 = 'applications/%s/' % app
|
||||
file2 = os.path.join(path2, 'languages', '%s.py' % file)
|
||||
shutil.copyfile(file1, file2)
|
||||
|
||||
@@ -15,7 +15,7 @@ SLEEP_MINUTES = 5
|
||||
|
||||
errors_path = os.path.join(request.folder, 'errors')
|
||||
try:
|
||||
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r','').replace('\n','').strip()
|
||||
db_string = open(os.path.join(request.folder, 'private', 'ticket_storage.txt')).read().replace('\r', '').replace('\n', '').strip()
|
||||
except:
|
||||
db_string = 'sqlite://storage.db'
|
||||
|
||||
@@ -23,7 +23,8 @@ db_path = os.path.join(request.folder, 'databases')
|
||||
|
||||
tk_db = DAL(db_string, folder=db_path, auto_import=True)
|
||||
ts = TicketStorage(db=tk_db)
|
||||
tk_table = ts._get_table(db=tk_db, tablename=ts.tablename, app=request.application)
|
||||
tk_table = ts._get_table(
|
||||
db=tk_db, tablename=ts.tablename, app=request.application)
|
||||
|
||||
|
||||
hashes = {}
|
||||
@@ -48,4 +49,3 @@ while 1:
|
||||
os.unlink(filename)
|
||||
|
||||
time.sleep(SLEEP_MINUTES * 60)
|
||||
|
||||
@@ -42,8 +42,8 @@ while 1:
|
||||
error = RestrictedError()
|
||||
error.load(request, request.application, file)
|
||||
|
||||
mail.send(to=administrator_email, subject='new web2py ticket', message=error.traceback)
|
||||
mail.send(to=administrator_email,
|
||||
subject='new web2py ticket', message=error.traceback)
|
||||
|
||||
os.unlink(os.path.join(path, file))
|
||||
time.sleep(SLEEP_MINUTES * 60)
|
||||
|
||||
@@ -14,7 +14,7 @@ import os
|
||||
import urllib
|
||||
import zipfile
|
||||
|
||||
if len(sys.argv)>1 and sys.argv[1] == 'nightly':
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'nightly':
|
||||
version = 'http://web2py.com/examples/static/nightly/web2py_src.zip'
|
||||
else:
|
||||
version = 'http://web2py.com/examples/static/web2py_src.zip'
|
||||
@@ -23,11 +23,11 @@ realpath = os.path.realpath(__file__)
|
||||
path = os.path.dirname(os.path.dirname(os.path.dirname(realpath)))
|
||||
os.chdir(path)
|
||||
try:
|
||||
old_version = open('web2py/VERSION','r').read().strip()
|
||||
old_version = open('web2py/VERSION', 'r').read().strip()
|
||||
except IOError:
|
||||
old_version = ''
|
||||
open(TMPFILENAME,'wb').write(urllib.urlopen(version).read())
|
||||
open(TMPFILENAME, 'wb').write(urllib.urlopen(version).read())
|
||||
new_version = zipfile.ZipFile(TMPFILENAME).read('web2py/VERSION').strip()
|
||||
if new_version>old_version:
|
||||
os.system('sudo -u %s unzip -o %s' % (USER,TMPFILENAME))
|
||||
if new_version > old_version:
|
||||
os.system('sudo -u %s unzip -o %s' % (USER, TMPFILENAME))
|
||||
os.system('apachectl restart | apache2ctl restart')
|
||||
@@ -7,6 +7,7 @@
|
||||
import os
|
||||
import gzip
|
||||
|
||||
|
||||
def zip_static(filelist=[]):
|
||||
tsave = 0
|
||||
for fi in filelist:
|
||||
@@ -24,17 +25,18 @@ def zip_static(filelist=[]):
|
||||
if zatime == atime and zmtime == mtime:
|
||||
print 'skipping %s, already gzipped to the latest version' % os.path.basename(fi)
|
||||
continue
|
||||
print 'gzipping %s to %s' % (os.path.basename(fi), os.path.basename(gfi))
|
||||
print 'gzipping %s to %s' % (
|
||||
os.path.basename(fi), os.path.basename(gfi))
|
||||
f_in = open(fi, 'rb')
|
||||
f_out = gzip.open(gfi, 'wb')
|
||||
f_out.writelines(f_in)
|
||||
f_out.close()
|
||||
f_in.close()
|
||||
os.utime(gfi, (atime,mtime))
|
||||
os.utime(gfi, (atime, mtime))
|
||||
saved = fstats.st_size - os.stat(gfi).st_size
|
||||
tsave+= saved
|
||||
tsave += saved
|
||||
|
||||
print 'saved %s KB' % (int(tsave)/1000.0)
|
||||
print 'saved %s KB' % (int(tsave) / 1000.0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
ALLOWED_EXTS = ['.css', '.js']
|
||||
|
||||
Reference in new issue
Block a user