many iteritems optimizations

This commit is contained in:
mdipierro
2012-08-28 09:35:36 -05:00
parent 18a2c2c68e
commit f9e35599c6
10 changed files with 147 additions and 108 deletions
+1 -1
View File
@@ -1 +1 @@
Version 2.00.1 (2012-08-28 08:28:31) rc4
Version 2.00.1 (2012-08-28 09:35:31) rc4
+23 -16
View File
@@ -870,9 +870,9 @@ class BaseAdapter(ConnectionPool):
v=dict(type='unkown',sql=v)
return k.lower(),v
### make sure all field names are lower case to avoid conflicts
sql_fields = dict(map(fix,sql_fields.items()))
sql_fields_old = dict(map(fix,sql_fields_old.items()))
sql_fields_aux = dict(map(fix,sql_fields_aux.items()))
sql_fields = dict(map(fix,sql_fields.iteritems()))
sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
keys = sql_fields.keys()
for key in sql_fields_old:
@@ -1859,9 +1859,9 @@ class BaseAdapter(ConnectionPool):
for tablename in virtualtables:
### new style virtual fields
table = db[tablename]
fields_virtual = [(f,v) for (f,v) in table.items()
fields_virtual = [(f,v) for (f,v) in table.iteritems()
if isinstance(v,FieldVirtual)]
fields_lazy = [(f,v) for (f,v) in table.items()
fields_lazy = [(f,v) for (f,v) in table.iteritems()
if isinstance(v,FieldLazy)]
if fields_virtual or fields_lazy:
for row in rowsobj.records:
@@ -2892,7 +2892,7 @@ class MSSQLAdapter(BaseAdapter):
argpattern = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
for argmatch in argpattern.finditer(urlargs):
argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.items()])
urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
% (host, port, db, user, password, urlargs)
def connect(cnxn=cnxn,driver_args=driver_args):
@@ -6351,6 +6351,9 @@ class Row(object):
def __iter__(self):
return self.__dict__.__iter__()
def iteritems(self):
return self.__dict__.iteritems()
def __str__(self):
### this could be made smarter
return '<Row %s>' % self.__dict__
@@ -6708,7 +6711,7 @@ class DAL(object):
sql_fields = cPickle.load(tfile)
name = filename[len(pattern)-7:-6]
mf = [(value['sortable'],Field(key,type=value['type'])) \
for key, value in sql_fields.items()]
for key, value in sql_fields.iteritems()]
mf.sort(lambda a,b: cmp(a[0],b[0]))
self.define_table(name,*[item[1] for item in mf],
**dict(migrate=migrate,fake_migrate=fake_migrate))
@@ -7420,7 +7423,7 @@ class Table(object):
def _validate(self,**vars):
errors = Row()
for key,value in vars.items():
for key,value in vars.iteritems():
value,error = self[key].validate(value)
if error:
errors[key] = error
@@ -7455,7 +7458,7 @@ class Table(object):
self._referenced_by.append((referee._tablename,referee.name))
def _filter_fields(self, record, id=False):
return dict([(k, v) for (k, v) in record.items() if k
return dict([(k, v) for (k, v) in record.iteritems() if k
in self.fields and (self[k].type!='id' or id)])
def _build_query(self,key):
@@ -7501,11 +7504,11 @@ class Table(object):
record = self._db(self._id == key).select(
limitby=(0,1),for_update=for_update).first()
if record:
for k,v in kwargs.items():
for k,v in kwargs.iteritems():
if record[k]!=v: return None
return record
elif kwargs:
query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.items()])
query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
return self._db(query).select(limitby=(0,1),for_update=for_update).first()
else:
return None
@@ -7563,6 +7566,10 @@ class Table(object):
for fieldname in self.fields:
yield self[fieldname]
def iteritems(self):
return self.__dict__.iteritems()
def __repr__(self):
return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
@@ -7642,7 +7649,7 @@ class Table(object):
response = Row()
response.errors = Row()
new_fields = copy.copy(fields)
for key,value in fields.items():
for key,value in fields.iteritems():
value,error = self[key].validate(value)
if error:
response.errors[key] = error
@@ -8572,7 +8579,7 @@ class Set(object):
response = Row()
response.errors = Row()
new_fields = copy.copy(update_fields)
for key,value in update_fields.items():
for key,value in update_fields.iteritems():
value,error = self.db[tablename][key].validate(value)
if error:
response.errors[key] = error
@@ -8631,9 +8638,9 @@ class Set(object):
def update_record(pack, a=None):
(colset, table, id) = pack
b = a or dict(colset)
c = dict([(k,v) for (k,v) in b.items() if k in table.fields and table[k].type!='id'])
c = dict([(k,v) for (k,v) in b.iteritems() if k in table.fields and table[k].type!='id'])
table._db(table._id==id,ignore_common_filters=True).update(**c)
for (k, v) in c.items():
for (k, v) in c.iteritems():
colset[k] = v
class VirtualCommand(object):
@@ -8694,7 +8701,7 @@ class Rows(object):
if not keyed_virtualfields:
return self
for row in self.records:
for (tablename,virtualfields) in keyed_virtualfields.items():
for (tablename,virtualfields) in keyed_virtualfields.iteritems():
attributes = dir(virtualfields)
if not tablename in row:
box = row[tablename] = Row()
+5 -4
View File
@@ -117,7 +117,8 @@ class Request(Storage):
user_agent_parser.detect(self.env.http_user_agent)
user_agent = Storage(user_agent)
for key,value in user_agent.items():
if isinstance(value,dict): user_agent[key] = Storage(value)
if isinstance(value,dict):
user_agent[key] = Storage(value)
return user_agent
def requires_https(self):
@@ -222,9 +223,9 @@ class Response(Storage):
return page
def include_meta(self):
s = '\n'
for key,value in (self.meta or {}).items():
s += '<meta name="%s" content="%s" />\n' % (key,xmlescape(value))
s = '\n'.join(
'<meta name="%s" content="%s" />\n' % (k,xmlescape(v))
for k,v in (self.meta or {}).iteritems())
self.write(s,escape=False)
def include_files(self):
+9 -9
View File
@@ -669,7 +669,7 @@ class DIV(XmlComponent):
dictionary like updating of the tag attributes
"""
for (key, value) in kargs.items():
for (key, value) in kargs.iteritems():
self[key] = value
return self
@@ -1043,7 +1043,7 @@ class DIV(XmlComponent):
tag = getattr(self,'tag').replace('/', '')
if args and tag not in args:
check = False
for (key, value) in kargs.items():
for (key, value) in kargs.iteritems():
if key not in ['first_only', 'replace', 'find_text']:
if isinstance(value, (str, int)):
if self[key] != str(value):
@@ -1122,7 +1122,7 @@ class DIV(XmlComponent):
tag = getattr(c,'tag').replace("/","")
if args and tag not in args:
check = False
for (key, value) in kargs.items():
for (key, value) in kargs.iteritems():
if c[key] != value:
check = False
if check:
@@ -1987,10 +1987,10 @@ class FORM(DIV):
def hidden_fields(self):
c = []
attr = self.attributes.get('hidden',{})
if 'hidden' in self.attributes:
for (key, value) in self.attributes.get('hidden',{}).items():
c.append(INPUT(_type='hidden', _name=key, _value=value))
c = [INPUT(_type='hidden', _name=key, _value=value)
for (key, value) in attr.iteritems()]
if hasattr(self, 'formkey') and self.formkey:
c.append(INPUT(_type='hidden', _name='_formkey',
_value=self.formkey))
@@ -2059,7 +2059,7 @@ class FORM(DIV):
onsuccess(self)
if next:
if self.vars:
for key,value in self.vars.items():
for key,value in self.vars.iteritems():
next = next.replace('[%s]' % key,
urllib.quote(str(value)))
if not next.startswith('/'):
@@ -2120,11 +2120,11 @@ class FORM(DIV):
inputs = [INPUT(_type='button',
_value=name,
_onclick=FORM.REDIRECT_JS % link) \
for name,link in buttons.items()]
for name,link in buttons.iteritems()]
inputs += [INPUT(_type='hidden',
_name=name,
_value=value)
for name,value in hidden.items()]
for name,value in hidden.iteritems()]
form = FORM(INPUT(_type='submit',_value=text),*inputs)
form.process()
return form
+3 -4
View File
@@ -92,14 +92,13 @@ class HTTP(BaseException):
body += '<!-- %s //-->' % ('x'*512) ### trick IE
self.headers['Content-Length'] = len(body)
headers = []
for (k, v) in self.headers.items():
for k, v in self.headers.iteritems():
if isinstance(v, list):
for item in v:
headers.append((k, str(item)))
headers += [(k, str(item)) for item in v]
else:
headers.append((k, str(v)))
responder(status, headers)
if hasattr(body, '__iter__') and not isinstance(self.body, str):
if hasattr(body, '__iter__') and not isinstance(self.body,str):
return body
return [str(body)]
+21 -12
View File
@@ -253,7 +253,8 @@ def middleware_aux(request, response, *middleware_apps):
for item in middleware_apps:
app=item(app)
def caller(app):
return app(request.wsgi.environ,request.wsgi.start_response)
return app(request.wsgi.environ,
request.wsgi.start_response)
return lambda caller=caller, app=app: caller(app)
return middleware
@@ -375,7 +376,8 @@ def wsgibase(environ, responder):
if not environ.get('PATH_INFO',None) and \
environ.get('REQUEST_URI',None):
# for fcgi, get path_info and query_string from request_uri
# for fcgi, get path_info and
# query_string from request_uri
items = environ['REQUEST_URI'].split('?')
environ['PATH_INFO'] = items[0]
if len(items) > 1:
@@ -383,13 +385,17 @@ def wsgibase(environ, responder):
else:
environ['QUERY_STRING'] = ''
if not environ.get('HTTP_HOST',None):
environ['HTTP_HOST'] = '%s:%s' % (environ.get('SERVER_NAME'),
environ.get('SERVER_PORT'))
environ['HTTP_HOST'] = '%s:%s' % (
environ.get('SERVER_NAME'),
environ.get('SERVER_PORT'))
(static_file, environ) = rewrite.url_in(request, environ)
(static_file, environ) = \
rewrite.url_in(request, environ)
if static_file:
if environ.get('QUERY_STRING', '')[:10] == 'attachment':
response.headers['Content-Disposition'] = 'attachment'
if environ.get('QUERY_STRING','').startswith(
'attachment'):
response.headers['Content-Disposition'] \
= 'attachment'
response.stream(static_file, request=request)
# ##################################################
@@ -401,14 +407,17 @@ def wsgibase(environ, responder):
local_hosts = [http_host,'::1','127.0.0.1','::ffff:127.0.0.1']
if not global_settings.web2py_runtime_gae:
local_hosts.append(socket.gethostname())
try: local_hosts.append(socket.gethostbyname(http_host))
except socket.gaierror: pass
try:
local_hosts.append(
socket.gethostbyname(http_host))
except socket.gaierror:
pass
request.client = get_client(request.env)
if not is_valid_ip_address(request.client):
raise HTTP(400,"Bad Request (request.client=%s)" % \
raise HTTP(400,"Bad Request (request.client=%s)" %\
request.client)
request.folder = abspath('applications',
request.application) + os.sep
request.folder = abspath(
'applications',request.application) + os.sep
x_req_with = str(request.env.http_x_requested_with).lower()
request.ajax = x_req_with == 'xmlhttprequest'
request.cid = request.env.http_web2py_component_element
+68 -49
View File
@@ -27,9 +27,17 @@ from http import HTTP
from fileutils import abspath, read_file
from settings import global_settings
logger = logging.getLogger('web2py.rewrite')
isdir = os.path.isdir
isfile = os.path.isfile
exists = os.path.exists
pjoin = os.path.join
thread = threading.local() # thread-local storage for routing parameters
logger = logging.getLogger('web2py.rewrite')
thread = threading.local() # thread-local storage for routing params
regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
regex_anything = re.compile(r'(?<!\\)\$anything')
regex_redirect = re.compile(r'(\d+)->(.*)')
def _router_default():
"return new copy of default base router"
@@ -75,7 +83,7 @@ def _params_default(app=None):
params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters
routers = None
def log_rewrite(string):
@@ -97,13 +105,18 @@ def log_rewrite(string):
else:
logger.debug(string)
ROUTER_KEYS = set(('default_application', 'applications', 'default_controller', 'controllers',
'default_function', 'functions', 'default_language', 'languages',
'domain', 'domains', 'root_static', 'path_prefix',
'exclusive_domain', 'map_hyphen', 'map_static',
'acfe_match', 'file_match', 'args_match'))
ROUTER_KEYS = set(
('default_application', 'applications',
'default_controller', 'controllers',
'default_function', 'functions',
'default_language', 'languages',
'domain', 'domains', 'root_static', 'path_prefix',
'exclusive_domain', 'map_hyphen', 'map_static',
'acfe_match', 'file_match', 'args_match'))
ROUTER_BASE_KEYS = set(('applications', 'default_application', 'domains', 'path_prefix'))
ROUTER_BASE_KEYS = set(
('applications', 'default_application',
'domains', 'path_prefix'))
# The external interface to rewrite consists of:
#
@@ -223,9 +236,7 @@ def try_redirect_on_error(http_object, request, ticket=None):
(redir,status,ticket,
urllib.quote_plus(request.env.request_uri),
request.url)
return HTTP(303,
'You are being redirected <a href="%s">here</a>' % url,
Location=url)
return HTTP(303,'You are being redirected <a href="%s">here</a>' % url,Location=url)
return http_object
@@ -258,7 +269,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
path = abspath(routes)
else:
path = abspath('applications', app, routes)
if not os.path.exists(path):
if not exists(path):
return
data = read_file(path).replace('\r\n','\n')
@@ -309,9 +320,11 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
# parse the app-specific routes.py if present
#
all_apps = []
for appname in [app for app in os.listdir(abspath('applications')) if not app.startswith('.')]:
if os.path.isdir(abspath('applications', appname)) and \
os.path.isdir(abspath('applications', appname, 'controllers')):
apppath = abspath('applications')
for appname in os.listdir(apppath):
if not appname.startswith('.') and \
isdir(abspath(apppath,appname)) and \
isdir(abspath(apppath,appname,'controllers')):
all_apps.append(appname)
if routers:
router = Storage(routers.BASE) # new copy
@@ -321,7 +334,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
raise SyntaxError, "BASE-only key '%s' in router '%s'" % (key, appname)
router.update(routers[appname])
routers[appname] = router
if os.path.exists(abspath('applications', appname, routes)):
if exists(abspath('applications', appname, routes)):
load(routes, appname)
if routers:
@@ -336,14 +349,9 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
log_rewrite('URL rewrite is on. configuration in %s' % path)
regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
regex_anything = re.compile(r'(?<!\\)\$anything')
regex_redirect = re.compile(r'(\d+)->(.*)')
def compile_regex(k, v):
"""
Preprocess and compile the regular expressions in routes_app/in/out
The resulting regex will match a pattern of the form:
[remote address]:[protocol]://[host]:[method] [path]
@@ -380,8 +388,9 @@ def compile_regex(k, v):
def load_routers(all_apps):
"load-time post-processing of routers"
for app in routers.keys():
# initialize apps with routers that aren't present, on behalf of unit tests
for app in routers:
# initialize apps with routers that aren't present,
# on behalf of unit tests
if app not in all_apps:
all_apps.append(app)
router = Storage(routers.BASE) # new copy
@@ -420,10 +429,10 @@ def load_routers(all_apps):
routers.BASE.domains[router.domain] = app
if isinstance(router.controllers, str) and router.controllers == 'DEFAULT':
router.controllers = set()
if os.path.isdir(abspath('applications', app)):
if isdir(abspath('applications', app)):
cpath = abspath('applications', app, 'controllers')
for cname in os.listdir(cpath):
if os.path.isfile(abspath(cpath, cname)) and cname.endswith('.py'):
if isfile(abspath(cpath, cname)) and cname.endswith('.py'):
router.controllers.add(cname[:-3])
if router.controllers:
router.controllers.add('static')
@@ -458,16 +467,20 @@ def load_routers(all_apps):
#
domains = dict()
if routers.BASE.domains:
for (domain, app) in [(d.strip(':'), a.strip('/')) for (d, a) in routers.BASE.domains.items()]:
port = None
for (d, a) in routers.BASE.domains.iteritems():
(domain, app) = (d.strip(':'), a.strip('/'))
if ':' in domain:
(domain, port) = domain.split(':')
ctlr = None
fcn = None
else:
port = None
if '/' in app:
(app, ctlr) = app.split('/', 1)
else:
ctlr = None
if ctlr and '/' in ctlr:
(ctlr, fcn) = ctlr.split('/')
else:
fcn = None
if app not in all_apps and app not in routers:
raise SyntaxError, "unknown app '%s' in domains" % app
domains[(domain, port)] = (app, ctlr, fcn)
@@ -514,7 +527,8 @@ def regex_filter_in(e):
query = e.get('QUERY_STRING', None)
e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
if thread.routes.routes_in:
path = regex_uri(e, thread.routes.routes_in, "routes_in", e['PATH_INFO'])
path = regex_uri(e, thread.routes.routes_in,
"routes_in", e['PATH_INFO'])
rmatch = regex_redirect.match(path)
if rmatch:
raise HTTP(int(rmatch.group(1)),location=rmatch.group(2))
@@ -581,6 +595,9 @@ regex_args = re.compile(r'''
/?$) # trailing slash
''', re.X)
def sluggify(key):
return key.lower().replace('.','_')
def regex_url_in(request, environ):
"rewrite and parse incoming URL"
@@ -594,9 +611,8 @@ def regex_url_in(request, environ):
if thread.routes.routes_in:
environ = regex_filter_in(environ)
for (key, value) in environ.items():
request.env[key.lower().replace('.', '_')] = value
request.env.update((sluggify(k),v) for k,v in environ.iteritems())
path = request.env.path_info.replace('\\', '/')
@@ -606,7 +622,7 @@ def regex_url_in(request, environ):
match = regex_static.match(regex_space.sub('_', path))
if match and match.group('x'):
static_file = os.path.join(request.env.applications_parent,
static_file = pjoin(request.env.applications_parent,
'applications', match.group('b'),
'static', match.group('x'))
return (static_file, environ)
@@ -908,7 +924,7 @@ class MapUrlIn(object):
'''
if len(self.args) == 1 and self.arg0 in self.router.root_static:
self.controller = self.request.controller = 'static'
root_static_file = os.path.join(self.request.env.applications_parent,
root_static_file = pjoin(self.request.env.applications_parent,
'applications', self.application,
self.controller, self.arg0)
log_rewrite("route: root static=%s" % root_static_file)
@@ -962,7 +978,8 @@ class MapUrlIn(object):
bad_static = bad_static or name in ('', '.', '..') or not self.router._file_match.match(name)
if bad_static:
log_rewrite('bad static path=%s' % file)
raise HTTP(400, thread.routes.error_message % 'invalid request',
raise HTTP(400,
thread.routes.error_message % 'invalid request',
web2py_error='invalid static file')
#
# support language-specific static subdirectories,
@@ -970,13 +987,13 @@ class MapUrlIn(object):
# if language-specific file doesn't exist, try same file in static
#
if self.language:
static_file = os.path.join(self.request.env.applications_parent,
'applications', self.application,
'static', self.language, file)
if not self.language or not os.path.isfile(static_file):
static_file = os.path.join(self.request.env.applications_parent,
'applications', self.application,
'static', file)
static_file = pjoin(self.request.env.applications_parent,
'applications', self.application,
'static', self.language, file)
if not self.language or not isfile(static_file):
static_file = pjoin(self.request.env.applications_parent,
'applications', self.application,
'static', file)
log_rewrite("route: static=%s" % static_file)
return static_file
@@ -1040,12 +1057,14 @@ class MapUrlIn(object):
uri += '.' + self.extension
if self.language:
uri = '/%s%s' % (self.language, uri)
uri = '/%s%s' % (app, uri)
uri += self.args and urllib.quote('/' + '/'.join([str(x) for x in self.args])) or ''
uri += (self.query and ('?' + self.query) or '')
uri = '/%s%s%s%s' % (
app,
uri,
urllib.quote('/'+'/'.join(str(x) for x in self.args)) if self.args else '',
('?' + self.query) if self.query else '')
self.env['REQUEST_URI'] = uri
for (key, value) in self.env.items():
self.request.env[key.lower().replace('.', '_')] = value
self.request.env.update(
(sluggify(k),v) for k,v in self.env.iteritems())
@property
def arg0(self):
+3 -2
View File
@@ -1407,7 +1407,7 @@ class Worker(Thread):
raise BadRequest
req = match.groupdict()
for k,v in req.items():
for k,v in req.iteritems():
if not v:
req[k] = ""
if k == 'path':
@@ -1694,7 +1694,8 @@ class FileSystemWorker(Worker):
try:
# Get our file path
headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()])
reader = self.read_headers(sock_file)
headers = dict((k.lower(),v) for k,v in reader.iteritems())
rpath = request.get('path', '').lstrip('/')
filepath = os.path.join(self.root, rpath)
filepath = os.path.abspath(filepath)
+9 -6
View File
@@ -18,7 +18,8 @@ if not hasattr(os, 'mkdir'):
if global_settings.db_sessions is not True:
global_settings.db_sessions = set()
global_settings.gluon_parent = os.environ.get('web2py_path', os.getcwd())
global_settings.gluon_parent = \
os.environ.get('web2py_path', os.getcwd())
global_settings.applications_parent = global_settings.gluon_parent
@@ -26,12 +27,14 @@ global_settings.app_folders = set()
global_settings.debugging = False
global_settings.is_pypy = hasattr(platform,'python_implementation') and \
platform.python_implementation() == 'PyPy'
global_settings.is_pypy = \
hasattr(platform,'python_implementation') and \
platform.python_implementation() == 'PyPy'
global_settings.is_jython = 'java' in sys.platform.lower() or \
hasattr(sys, 'JYTHON_JAR') or \
str(sys.copyright).find('Jython') > 0
global_settings.is_jython = \
'java' in sys.platform.lower() or \
hasattr(sys, 'JYTHON_JAR') or \
str(sys.copyright).find('Jython') > 0
+5 -5
View File
@@ -3138,8 +3138,7 @@ class Auth(object):
elif form.record and fieldname in form.record:
new_record[fieldname]=form.record[fieldname]
if fields:
for key,value in fields.items():
new_record[key] = value
new_record.update(fields)
id = archive_table.insert(**new_record)
return id
def wiki(self,slug=None,env=None,manage_permissions=False,force_prefix=''):
@@ -4132,7 +4131,7 @@ class Service(object):
prefix='pys',
documentation = documentation,
ns = True)
for method, (function, returns, args, doc) in procedures.items():
for method, (function, returns, args, doc) in procedures.iteritems():
dispatcher.register_function(method, function, returns, args, doc)
if request.env.request_method == 'POST':
# Process normal Soap Operation
@@ -4388,8 +4387,9 @@ class PluginManager(object):
self.__dict__.clear()
settings = self.__getattr__(plugin)
settings.installed = True
[settings.update({key:value}) for key,value in defaults.items() \
if not key in settings]
settings.update(
(k,v) for k,v in defaults.items() if not k in settings)
def __getattr__(self, key):
if not key in self.__dict__:
self.__dict__[key] = Storage()