populate fix and pep8
This commit is contained in:
mdipierro committed
1 parent
6cda7a29fc
commit
1a83487221
2 files changed
+140
-101
No files matched your search
@@ -1 +1 @@
|
||||
Version 2.1.1 (2012-10-19 14:22:39) dev
|
||||
Version 2.1.1 (2012-10-19 14:59:27) dev
|
||||
+139
-100
@@ -1,167 +1,206 @@
|
||||
import re, cPickle, random, datetime
|
||||
import re
|
||||
import cPickle
|
||||
import random
|
||||
import datetime
|
||||
|
||||
IUP = {'shoebill':{'a':1,'187':1},'trout-like':{'parr':1},'fig.':{'19':1},'mcintosh.':{'the':1,'illustration':1},'chiasmodon':{'niger':2},'yellow':{'and':4,'giant':1,'green':2,'red':1,'spots':1},'four':{'brightly':1,'hundred':1,'haunts':1,'feet':1,'solutions':2,'hours.':1,'things':1,'long':1,'.':2,'legs':1,'inches':4,'thousand':1,'million':1,'pairs':2,'hours':1,'satellites':1,'tufts':1,'great':1,'main':1,'of':1,'months':1,'days':1,'times':4,'volumes':1,'inches.':1,'frilled':2,'or':3},'snapping-blades':{'the':1},'aegir':{'on':4},'hanging':{'down':1,'on':2},'bird--evidences':{'of':1},'canes':{'venatici':1},'inheritance.':{'looking':1},'electricity':{'and':5,'belong':1,'is':4,'rotating':2,'held':1,'as':1,'sec':1,'in':1,'yet':1,'predominates':1,'from':1,'there':1,'.':9,'to':1,'which':4,';':4,'has':1,'we':1,'that':1,'here':1,'i.e':1,'like':1,'repel':1,'the':2,'or':1,'attracting':1},'similarity':{'of':3,'between':1},'sunlit':{'side':1},'superficially':{'indistinguishable':1,'like':1},'out-breeding':{'among':1,'when':1,'exogamy':1},'lord':{'popular':1,'kelvin':5,'avebury':1},'non-intelligent':{'experiments':1},'flicking':{'out':1},'meadows':{'and':2,'an':1},'sinking':{'down':2,'according':1,'in':1},'digit':{'of':1,'well':1},'co-operation':{'of':2},'oceans':{'lie':1,'of':1,'there':1,'are':1,'goes':1,'must':1},'pigment':{'and':1,'is':2,'.':1,'as':1,'which':1,';':1},'figs':{'.':1},'fingers.':{'but':1},'experimentally':{'on':1},'bringing':{'of':1,'about':1,'the':2},'elevations':{'and':1,'of':1},'meadow.':{'in':1},'internally':{'hot':1},'whitman':{'says':1,'took':1},'colour--the':{'result':2},'stars.':{'yet':1},'persisted':{'as':1,'until':1},'chameleons':{'the':1,'.':1},'intensification':{'of':1},'succession':{'giving':1,'of':5,'the':1,'has':1,'.':1},'tube--the':{'walls':1},'ultra-violet':{'and':1,'light':1,'waves--which':1,'or':1,'waves':1},'straight':{'ahead':1,'drop':1,'lines':1,'down':1,'through':1,'in':1,'path':2,'line':1,'legs':1,'along':1},'spiders':{'about':1,'were':1,'it':1,'.':1,'scorpions':1,'are':1,'have':1,'in':1},'specially':{'protected':1,'adapted':1},'symmetry.':{'illustration':1},'stiffens':{'.':1},'270':{'a':1,'silk':1,'reproduced':1},'271':{'electrical':1,'from':1},'second':{'and':5,'layer':1,'when':1,'year':1,'extinction':1,'yet':1,'any':1,'group':1,'would':3,'there':1,'question':2,'.':10,'to':1,'millennium':1,'which':1,';':1,'then':1,'we':1,'advantage':1,'that':1,'class':1,'screen.':1,'interglacial':1,'every':1,'longest':1,'not':1,'cousin':1,'copy':1,'opportunity':1,'by':1,'a':1,'great':1,'these':1,'of':1,'or':1,'preparation':1,'printing':1,'error':1,'position':1,'the':2,'chapter':1,'photograph':2},'attended':{'the':1,'by':2},'274':{'photo':1},'275':{'from':1},'278':{'hours':1,'photo':1},'279':{'rotating':1,'the':2},'inanimate':{'world':1},'dormancy':{'.':1},'errors':{'a':1,'in':1},'semicircular':{'canals':1},'phalangers':{'and':1,'flying':1},'thunder':{'and':1,'means':1},'nature.':{'the':1,'typical':1,'what':1,'illustration':1,'one':1},'contributed':{'every':1},'fingers':{'and':6,'to':1,';':1,'2':1,'knocking':1},'lowliness':{'.':1},'fossil':{'horses':1,'remains':2,'series':1,'forms':2,'scorpions':1,'of':2,'species':1},'increasing':{'control--a':1,'temperature':1,'facility':1,'intelligence':1,'complexity':2,'fullness':1,'the':1,'with':1,'day':1},'inducement':{'many':1},'surface--the':{'photosphere--is':1},'chins':{'and':1},'error':{'a':1,'we':1,'is':1,'here':1,'method':2,'to':1,';':1,'or':1,'must':1},'here':{'and':5,'says':1,'is':5,'within':1,'some':1,'it':1,'anticipating':1,'go':1,'seen':1,'again':4,'for':1,'also':1,'rather':1,'concerned':1,'since':1,'.':1,'to':3,'only':2,'cockchafers':1,'you':1,'was':2,'refer':1,'we':2,'let':1,'with':1,'nor':1,'a':1,'of':1,'spontaneously':1,'the':2,'stimulating':1,'are':1},'atoms':{'and':10,'being':1,'is':1,'an':1,'270':1,'whilst':1,'exist':1,'are':4,'have':1,'in':3,'seem':1,'seek':1,'containing':1,'indivisible':1,'from':1,'appear':1,'would':1,'with':1,'had':1,'.':5,'which':2,'got':1,';':2,'has':1,'was':1,'into':1,'do':2,'form':1,'were':4,'but':1,'atoms':2,'break':1,'most':1,'they':1,'put':1,'come':1,'themselves':1,'he':1,'for':1,'of':33,'enter':1,'the':2,'first':1,'at':1},'reported':{'to':1,'in':1,'that':1},'china':{'and':1},'evolutionist':{'s':1,'ideas':1,'suggestion':1},'fire.':{'chief':1,'saturn':1},'substance':{'in':1,'both':1,'from':2,'would':1,'is':1,'after':1,'glowing':1,'reaches':1,'.':3,'will':1,'to':1,'as':2,'but':1,'which':2,'were':1,'known':1,'has':1,'by':1},'contribute.':{'another':1},'k':{'.':1},'second--more':{'than':1},'reports':{'performances':1},'controversy':{'as':1},'forbes':{'observed':1,'tells':1,'history':1},'symmetrical':{'fishes.':1},'heidelbergensis':{'discovered':1},'appropriately':{'giants':1},'divide':{'a':1,'in':1},'owed':{'to':1},'explained':{'on':1,'because':1,'presently':1,'that':1,'later':1,'in':4,'by':1},'lengthen':{'the':1},'replace':{'the':4,'it':1},'brought':{'about':3,'to':3,'into':2,'up':1,'us':1,'their':1,'outside':1,'neLine truncated
|
||||
|
||||
class Learner:
|
||||
def __init__(self):
|
||||
self.db={}
|
||||
def learn(self,text):
|
||||
replacements1={'[^a-zA-Z0-9\.;:\-]':' ',
|
||||
'\s+':' ',', ':' , ', '\. ':' . ',
|
||||
': ':' : ','; ':' ; '}
|
||||
for key,value in replacements1.items(): text=re.sub(key,value,text)
|
||||
items=[item.lower() for item in text.split(' ')]
|
||||
for i in range(len(items)-1):
|
||||
item=items[i]
|
||||
nextitem=items[i+1]
|
||||
if not self.db.has_key(item): self.db[item]={}
|
||||
if not self.db[item].has_key(nextitem): self.db[item][nextitem]=1
|
||||
else: self.db[item][nextitem]+=1
|
||||
def save(self,filename):
|
||||
cPickle.dump(self.db,open(filename,'wb'))
|
||||
def load(self,filename):
|
||||
self.loadd(cPickle.load(open(filename,'rb')))
|
||||
def loadd(self,db):
|
||||
self.db=db
|
||||
def generate(self,length=10000,prefix=False):
|
||||
replacements2={' ,':',', ' \.':'.\n', ' :':':', ' ;':';', '\n\s+':'\n' }
|
||||
keys=self.db.keys()
|
||||
key=keys[random.randint(0,len(keys)-1)]
|
||||
words=key
|
||||
words=words.capitalize()
|
||||
regex=re.compile('[a-z]+')
|
||||
self.db = {}
|
||||
|
||||
def learn(self, text):
|
||||
replacements1 = {'[^a-zA-Z0-9\.;:\-]': ' ',
|
||||
'\s+': ' ', ', ': ' , ', '\. ': ' . ',
|
||||
': ': ' : ', '; ': ' ; '}
|
||||
for key, value in replacements1.items():
|
||||
text = re.sub(key, value, text)
|
||||
items = [item.lower() for item in text.split(' ')]
|
||||
for i in range(len(items) - 1):
|
||||
item = items[i]
|
||||
nextitem = items[i + 1]
|
||||
if item not in self.db:
|
||||
self.db[item] = {}
|
||||
if nextitem not in self.db[item]:
|
||||
self.db[item][nextitem] = 1
|
||||
else:
|
||||
self.db[item][nextitem] += 1
|
||||
|
||||
def save(self, filename):
|
||||
cPickle.dump(self.db, open(filename, 'wb'))
|
||||
|
||||
def load(self, filename):
|
||||
self.loadd(cPickle.load(open(filename, 'rb')))
|
||||
|
||||
def loadd(self, db):
|
||||
self.db = db
|
||||
|
||||
def generate(self, length=10000, prefix=False):
|
||||
replacements2 = {' ,': ',', ' \.': '.\n', ' :': ':', ' ;':
|
||||
';', '\n\s+': '\n'}
|
||||
keys = self.db.keys()
|
||||
key = keys[random.randint(0, len(keys) - 1)]
|
||||
words = key
|
||||
words = words.capitalize()
|
||||
regex = re.compile('[a-z]+')
|
||||
for i in range(length):
|
||||
okey=key
|
||||
if not key in self.db: break # should not happen
|
||||
db=self.db[key]
|
||||
s=sum(db.values())
|
||||
i=random.randint(0,s-1)
|
||||
for key,value in db.items():
|
||||
if i<value: break
|
||||
else: i-=value
|
||||
if okey=='.': key1=key.capitalize()
|
||||
else: key1=key
|
||||
okey = key
|
||||
if not key in self.db:
|
||||
break # should not happen
|
||||
db = self.db[key]
|
||||
s = sum(db.values())
|
||||
i = random.randint(0, s - 1)
|
||||
for key, value in db.items():
|
||||
if i < value:
|
||||
break
|
||||
else:
|
||||
i -= value
|
||||
if okey == '.':
|
||||
key1 = key.capitalize()
|
||||
else:
|
||||
key1 = key
|
||||
if prefix and regex.findall(key1) and \
|
||||
random.random()<0.01:
|
||||
key1='<a href="%s%s">%s</a>' % (prefix,key1,key1)
|
||||
words+=' '+key1
|
||||
text=words
|
||||
for key,value in replacements2.items():
|
||||
text=re.sub(key,value,text)
|
||||
return text+'.\n'
|
||||
random.random() < 0.01:
|
||||
key1 = '<a href="%s%s">%s</a>' % (prefix, key1, key1)
|
||||
words += ' ' + key1
|
||||
text = words
|
||||
for key, value in replacements2.items():
|
||||
text = re.sub(key, value, text)
|
||||
return text + '.\n'
|
||||
|
||||
|
||||
def da_du_ma(n=4):
|
||||
return ''.join([['da','du','ma','mo','ce','co',
|
||||
'pa','po','sa','so','ta','to']\
|
||||
[random.randint(0,11)] for i in range(n)])
|
||||
return ''.join([['da', 'du', 'ma', 'mo', 'ce', 'co',
|
||||
'pa', 'po', 'sa', 'so', 'ta', 'to']
|
||||
[random.randint(0, 11)] for i in range(n)])
|
||||
|
||||
|
||||
def populate(table, n, default=True, compute=False):
|
||||
ell=Learner()
|
||||
ell = Learner()
|
||||
#ell.learn(open('20417.txt','r').read())
|
||||
#ell.save('frequencies.pickle')
|
||||
#ell.load('frequencies.pickle')
|
||||
ell.loadd(IUP)
|
||||
ids={}
|
||||
ids = {}
|
||||
for i in range(n):
|
||||
record={}
|
||||
record = {}
|
||||
for fieldname in table.fields:
|
||||
field = table[fieldname]
|
||||
if not isinstance(field.type,(str,unicode)):
|
||||
if not isinstance(field.type, (str, unicode)):
|
||||
continue
|
||||
elif field.type == 'id':
|
||||
continue
|
||||
elif default and field.default !=None:
|
||||
record[fieldname]=field.default
|
||||
elif default and not field.default in (None, ''):
|
||||
record[fieldname] = field.default
|
||||
elif compute and field.compute:
|
||||
continue
|
||||
elif field.type == 'text':
|
||||
record[fieldname]=ell.generate(random.randint(10,100),prefix=None)
|
||||
record[fieldname] = ell.generate(
|
||||
random.randint(10, 100), prefix=None)
|
||||
elif field.type == 'boolean':
|
||||
record[fieldname]=random.random()>0.5
|
||||
record[fieldname] = random.random() > 0.5
|
||||
elif field.type == 'date':
|
||||
record[fieldname] = \
|
||||
datetime.date(2009,1,1) - \
|
||||
datetime.timedelta(days=random.randint(0,365))
|
||||
datetime.date(2009, 1, 1) - \
|
||||
datetime.timedelta(days=random.randint(0, 365))
|
||||
elif field.type == 'datetime':
|
||||
record[fieldname] = \
|
||||
datetime.datetime(2009,1,1) - \
|
||||
datetime.timedelta(days=random.randint(0,365))
|
||||
datetime.datetime(2009, 1, 1) - \
|
||||
datetime.timedelta(days=random.randint(0, 365))
|
||||
elif field.type == 'time':
|
||||
h = random.randint(0,23)
|
||||
m = 15*random.randint(0,3)
|
||||
record[fieldname] = datetime.time(h,m,0)
|
||||
h = random.randint(0, 23)
|
||||
m = 15 * random.randint(0, 3)
|
||||
record[fieldname] = datetime.time(h, m, 0)
|
||||
elif field.type == 'password':
|
||||
record[fieldname] = ''
|
||||
elif field.type == 'upload':
|
||||
record[fieldname] = None
|
||||
elif field.type=='integer' and hasattr(field.requires,'options'):
|
||||
options=field.requires.options(zero=False)
|
||||
if len(options)>0:
|
||||
record[fieldname] = options[random.randint(0,len(options)-1)][0]
|
||||
elif field.type == 'integer' and \
|
||||
hasattr(field.requires, 'options'):
|
||||
options = field.requires.options(zero=False)
|
||||
if len(options) > 0:
|
||||
record[fieldname] = options[
|
||||
random.randint(0, len(options) - 1)][0]
|
||||
else:
|
||||
record[fieldname] = None
|
||||
elif field.type=='list:integer' and hasattr(field.requires,'options'):
|
||||
options=field.requires.options(zero=False)
|
||||
elif field.type == 'list:integer' and hasattr(field.requires, 'options'):
|
||||
options = field.requires.options(zero=False)
|
||||
if len(options) > 0:
|
||||
record[fieldname] = [item[0] for item in random.sample(options, random.randint(0,len(options)-1)/2)]
|
||||
record[fieldname] = [item[0] for item in random.sample(
|
||||
options, random.randint(0, len(options) - 1) / 2)]
|
||||
elif field.type == 'integer':
|
||||
try:
|
||||
record[fieldname] = random.randint(field.requires.minimum,field.requires.maximum-1)
|
||||
record[fieldname] = random.randint(
|
||||
field.requires.minimum, field.requires.maximum - 1)
|
||||
except:
|
||||
record[fieldname] = random.randint(0,1000)
|
||||
elif field.type == 'double' or str(field.type).startswith('decimal'):
|
||||
if hasattr(field.requires,'minimum'):
|
||||
rand=random.random()
|
||||
record[fieldname] = random.randint(0, 1000)
|
||||
elif field.type == 'double' \
|
||||
or str(field.type).startswith('decimal'):
|
||||
if hasattr(field.requires, 'minimum'):
|
||||
rand = random.random()
|
||||
if str(field.type).startswith('decimal'):
|
||||
import decimal
|
||||
rand=decimal.Decimal(rand)
|
||||
record[fieldname] = field.requires.minimum+ \
|
||||
rand*(field.requires.maximum-field.requires.minimum)
|
||||
rand = decimal.Decimal(rand)
|
||||
record[fieldname] = field.requires.minimum + \
|
||||
rand * (field.requires.maximum -
|
||||
field.requires.minimum)
|
||||
else:
|
||||
record[fieldname] = random.random()*1000
|
||||
record[fieldname] = random.random() * 1000
|
||||
elif field.type[:10] == 'reference ':
|
||||
tablename = field.type[10:]
|
||||
if not tablename in ids:
|
||||
if table._db._dbname=='gql':
|
||||
ids[tablename] = [x.id for x in table._db(table._db[field.type[10:]].id>0).select()]
|
||||
if table._db._dbname == 'gql':
|
||||
ids[tablename] = [x.id for x in table._db(
|
||||
table._db[field.type[10:]].id > 0).select()]
|
||||
else:
|
||||
ids[tablename] = [x.id for x in table._db(table._db[field.type[10:]].id>0).select()]
|
||||
ids[tablename] = [x.id for x in table._db(
|
||||
table._db[field.type[10:]].id > 0).select()]
|
||||
n = len(ids[tablename])
|
||||
if n:
|
||||
record[fieldname] = ids[tablename][random.randint(0,n-1)]
|
||||
record[fieldname] = \
|
||||
ids[tablename][random.randint(0, n - 1)]
|
||||
else:
|
||||
record[fieldname] = 0
|
||||
elif field.type[:15] == 'list:reference ':
|
||||
tablename = field.type[15:]
|
||||
if not tablename in ids:
|
||||
if table._db._dbname=='gql':
|
||||
ids[tablename] = [x.id for x in table._db(table._db[field.type[15:]].id>0).select()]
|
||||
if table._db._dbname == 'gql':
|
||||
ids[tablename] = [x.id for x in table._db(
|
||||
table._db[field.type[15:]].id > 0).select()]
|
||||
else:
|
||||
ids[tablename] = [x.id for x in table._db(table._db[field.type[15:]].id>0).select()]
|
||||
ids[tablename] = [x.id for x in table._db(
|
||||
table._db[field.type[15:]].id > 0).select()]
|
||||
n = len(ids[tablename])
|
||||
if n:
|
||||
record[fieldname] = [item for item in random.sample(ids[tablename], random.randint(0,n-1)/2)]
|
||||
record[fieldname] = [item for item in random.sample(
|
||||
ids[tablename], random.randint(0, n - 1) / 2)]
|
||||
else:
|
||||
record[fieldname] = 0
|
||||
elif field.type=='list:string' and hasattr(field.requires,'options'):
|
||||
options=field.requires.options(zero=False)
|
||||
elif field.type == 'list:string' \
|
||||
and hasattr(field.requires, 'options'):
|
||||
options = field.requires.options(zero=False)
|
||||
if len(options) > 0:
|
||||
record[fieldname] = [item[0] for item in random.sample(options, random.randint(0,len(options)-1)/2)]
|
||||
elif field.type=='string' and hasattr(field.requires,'options'):
|
||||
options=field.requires.options(zero=False)
|
||||
record[fieldname] = options[random.randint(0,len(options)-1)][0]
|
||||
elif field.type=='string' and fieldname.find('url')>=0:
|
||||
record[fieldname] = 'http://%s.example.com' % da_du_ma(4)
|
||||
elif field.type=='string' and fieldname.find('email')>=0:
|
||||
record[fieldname] = [item[0] for item in random.sample(
|
||||
options, random.randint(0, len(options) - 1) / 2)]
|
||||
elif field.type == 'string' \
|
||||
and hasattr(field.requires, 'options'):
|
||||
options = field.requires.options(zero=False)
|
||||
record[fieldname] = \
|
||||
options[random.randint(0, len(options) - 1)][0]
|
||||
elif field.type == 'string' and fieldname.find('url') >= 0:
|
||||
record[fieldname] = 'http://%s.example.com' % \
|
||||
da_du_ma(4)
|
||||
elif field.type == 'string' and fieldname.find('email') >= 0:
|
||||
record[fieldname] = '%s@example.com' % da_du_ma(4)
|
||||
elif field.type=='string' and fieldname.find('name')>=0:
|
||||
elif field.type == 'string' and fieldname.find('name') >= 0:
|
||||
record[fieldname] = da_du_ma(4).capitalize()
|
||||
elif field.type=='string':
|
||||
record[fieldname] = ell.generate(10, prefix=False)[:field.length].replace('\n',' ')
|
||||
elif field.type == 'string':
|
||||
z = ell.generate(10, prefix=False)
|
||||
record[fieldname] = z[:field.length].replace('\n', ' ')
|
||||
table.insert(**record)
|
||||
table._db.commit()
|
||||
|
||||
if __name__ == '__main__':
|
||||
ell=Learner()
|
||||
ell = Learner()
|
||||
ell.loadd(eval(IUP))
|
||||
print ell.generate(1000,prefix=None)
|
||||
print ell.generate(1000, prefix=None)
|
||||
Reference in new issue
Block a user