dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved speed of regex_url_in

Browse Source
This commit is contained in:
Massimo
2012-10-12 11:33:02 -05:00
parent b8afce746b
commit a4fbf9b6b4
5 changed files with 100 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 2.1.0 (2012-10-11 22:03:15) dev Version 2.1.0 (2012-10-12 11:32:45) dev
applications/examples/controllers/simple_examples.py
-1
View File
@@ -38,7 +38,6 @@ def hello6():
def status(): def status():
""" page that shows internal status""" """ page that shows internal status"""
response.view = 'generic.html'
return dict(toolbar=response.toolbar()) return dict(toolbar=response.toolbar())
gluon/main.py
+1 -1
View File
@@ -87,7 +87,7 @@ from validators import CRYPT
from cache import CacheInRam from cache import CacheInRam
from html import URL, xmlescape from html import URL, xmlescape
from utils import is_valid_ip_address from utils import is_valid_ip_address
from rewrite import load, url_in, thread as rwthread, \ from rewrite import load, url_in, THREAD_LOCAL as rwthread, \
try_rewrite_on_error, fixup_missing_path_info try_rewrite_on_error, fixup_missing_path_info
import newcron import newcron
gluon/rewrite.py
+91 -136
View File
@@ -33,13 +33,29 @@ exists = os.path.exists
pjoin = os.path.join pjoin = os.path.join
logger = logging.getLogger('web2py.rewrite') logger = logging.getLogger('web2py.rewrite')
thread = threading.local() # thread-local storage for routing params THREAD_LOCAL = threading.local() # thread-local storage for routing params
regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*') regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
regex_anything = re.compile(r'(?<!\\)\$anything') regex_anything = re.compile(r'(?<!\\)\$anything')
regex_redirect = re.compile(r'(\d+)->(.*)') regex_redirect = re.compile(r'(\d+)->(.*)')
regex_full_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)') regex_full_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)')
regex_version = re.compile(r'^(_[\d]+\.[\d]+\.[\d]+)$') regex_version = re.compile(r'^(_[\d]+\.[\d]+\.[\d]+)$')
# pattern to replace spaces with underscore in URL
# also the html escaped variants '+' and '%20' are covered
regex_space = re.compile('(\+|\s|%20)+')
# pattern to find valid paths in url /application/controller/...
# this could be:
# for static pages:
# /<b:application>/static/<x:file>
# for dynamic pages:
# /<a:application>[/<c:controller>[/<f:function>[.<e:ext>][/<s:args>]]]
# application, controller, function and ext may only contain [a-zA-Z0-9_]
# file and args may also contain '-', '=', '.' and '/'
# apps in routes_apps_raw must parse raw_args into args
regex_url = re.compile('^/((?P<a>\w+)(/(?P<c>\w+)(/(?P<z>(?P<f>\w+)(\.(?P<e>[\w.]+))?(?P<s>[/\w@=-]*(\.[/\w@=-]+)*)))?)?)?$')
def _router_default(): def _router_default():
"return new copy of default base router" "return new copy of default base router"
@@ -85,7 +101,7 @@ def _params_default(app=None):
params_apps = dict() params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters THREAD_LOCAL.routes = params # default to base regex rewrite parameters
routers = None routers = None
def log_rewrite(string): def log_rewrite(string):
@@ -126,11 +142,11 @@ ROUTER_BASE_KEYS = set(
# url_in: parse and rewrite incoming URL # url_in: parse and rewrite incoming URL
# url_out: assemble and rewrite outgoing URL # url_out: assemble and rewrite outgoing URL
# #
# thread.routes.default_application # THREAD_LOCAL.routes.default_application
# thread.routes.error_message # THREAD_LOCAL.routes.error_message
# thread.routes.error_message_ticket # THREAD_LOCAL.routes.error_message_ticket
# thread.routes.try_redirect_on_error # THREAD_LOCAL.routes.try_redirect_on_error
# thread.routes.error_handler # THREAD_LOCAL.routes.error_handler
# #
# filter_url: helper for doctest & unittest # filter_url: helper for doctest & unittest
# filter_err: helper for doctest & unittest # filter_err: helper for doctest & unittest
@@ -177,6 +193,7 @@ def url_out(request, env, application, controller, function,
# fill in scheme and host if absolute URL is requested # fill in scheme and host if absolute URL is requested
# scheme can be a string, eg 'http', 'https', 'ws', 'wss' # scheme can be a string, eg 'http', 'https', 'ws', 'wss'
# #
if scheme or port is not None: if scheme or port is not None:
if host is None: # scheme or port implies host if host is None: # scheme or port implies host
host = True host = True
@@ -202,12 +219,12 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
called from main.wsgibase to rewrite the http response. called from main.wsgibase to rewrite the http response.
""" """
status = int(str(http_response.status).split()[0]) status = int(str(http_response.status).split()[0])
if status>=399 and thread.routes.routes_onerror: if status>=399 and THREAD_LOCAL.routes.routes_onerror:
keys=set(('%s/%s' % (request.application, status), keys=set(('%s/%s' % (request.application, status),
'%s/*' % (request.application), '%s/*' % (request.application),
'*/%s' % (status), '*/%s' % (status),
'*/*')) '*/*'))
for (key,uri) in thread.routes.routes_onerror: for (key,uri) in THREAD_LOCAL.routes.routes_onerror:
if key in keys: if key in keys:
if uri == '!': if uri == '!':
# do nothing! # do nothing!
@@ -245,12 +262,12 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
def try_redirect_on_error(http_object, request, ticket=None): def try_redirect_on_error(http_object, request, ticket=None):
"called from main.wsgibase to rewrite the http response" "called from main.wsgibase to rewrite the http response"
status = int(str(http_object.status).split()[0]) status = int(str(http_object.status).split()[0])
if status>399 and thread.routes.routes_onerror: if status>399 and THREAD_LOCAL.routes.routes_onerror:
keys=set(('%s/%s' % (request.application, status), keys=set(('%s/%s' % (request.application, status),
'%s/*' % (request.application), '%s/*' % (request.application),
'*/%s' % (status), '*/%s' % (status),
'*/*')) '*/*'))
for (key,redir) in thread.routes.routes_onerror: for (key,redir) in THREAD_LOCAL.routes.routes_onerror:
if key in keys: if key in keys:
if redir == '!': if redir == '!':
break break
@@ -283,7 +300,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
global params_apps global params_apps
params_apps = dict() params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters THREAD_LOCAL.routes = params # default to base regex rewrite parameters
routers = None routers = None
if isinstance(rdict, dict): if isinstance(rdict, dict):
@@ -330,7 +347,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
if app is None: if app is None:
params = p # install base rewrite parameters params = p # install base rewrite parameters
thread.routes = params # install default as current routes THREAD_LOCAL.routes = params # install default as current routes
# #
# create the BASE router if routers in use # create the BASE router if routers in use
# #
@@ -539,24 +556,25 @@ def regex_select(env=None, app=None, request=None):
select a set of regex rewrite params for the current request select a set of regex rewrite params for the current request
""" """
if app: if app:
thread.routes = params_apps.get(app, params) THREAD_LOCAL.routes = params_apps.get(app, params)
elif env and params.routes_app: elif env and params.routes_app:
if routers: if routers:
map_url_in(request, env, app=True) map_url_in(request, env, app=True)
else: else:
app = regex_uri(env, params.routes_app, "routes_app") app = regex_uri(env, params.routes_app, "routes_app")
thread.routes = params_apps.get(app, params) THREAD_LOCAL.routes = params_apps.get(app, params)
else: else:
thread.routes = params # default to base rewrite parameters THREAD_LOCAL.routes = params # default to base rewrite parameters
log_rewrite("select routing parameters: %s" % thread.routes.name) log_rewrite("select routing parameters: %s" % THREAD_LOCAL.routes.name)
return app # for doctest return app # for doctest
def regex_filter_in(e): def regex_filter_in(e):
"regex rewrite incoming URL" "regex rewrite incoming URL"
routes = THREAD_LOCAL.routes
query = e.get('QUERY_STRING', None) query = e.get('QUERY_STRING', None)
e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '') e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
if thread.routes.routes_in: if routes.routes_in:
path = regex_uri(e, thread.routes.routes_in, path = regex_uri(e, routes.routes_in,
"routes_in", e['PATH_INFO']) "routes_in", e['PATH_INFO'])
rmatch = regex_redirect.match(path) rmatch = regex_redirect.match(path)
if rmatch: if rmatch:
@@ -572,59 +590,7 @@ def regex_filter_in(e):
e['REQUEST_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '') e['REQUEST_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
return e return e
# pattern to replace spaces with underscore in URL
# also the html escaped variants '+' and '%20' are covered
regex_space = re.compile('(\+|\s|%20)+')
# pattern to find valid paths in url /application/controller/...
# this could be:
# for static pages:
# /<b:application>/static/<x:file>
# for dynamic pages:
# /<a:application>[/<c:controller>[/<f:function>[.<e:ext>][/<s:args>]]]
# application, controller, function and ext may only contain [a-zA-Z0-9_]
# file and args may also contain '-', '=', '.' and '/'
# apps in routes_apps_raw must parse raw_args into args
regex_static = re.compile(r'''
(^ # static pages
/(?P<b> \w+) # b=app
/static # /b/static
(/(?P<v>_[\d]+\.[\d]+\.[\d]+))? # version ?
/(?P<x> (\w[\-\=\./]?)* ) # x=file
$)
''', re.X)
regex_url = re.compile(r'''
(^( # (/a/c/f.e/s)
/(?P<a> [\w\s+]+ ) # /a=app
( # (/c.f.e/s)
/(?P<c> [\w\s+]+ ) # /a/c=controller
( # (/f.e/s)
/(?P<f> [\w\s+]+ ) # /a/c/f=function
( # (.e)
\.(?P<e> [\w\s+]+ ) # /a/c/f.e=extension
)?
( # (/s)
/(?P<r> # /a/c/f.e/r=raw_args
.*
)
)?
)?
)?
)?
/?$)
''', re.X)
regex_args = re.compile(r'''
(^
(?P<s>
( [\w@/-][=.]? )* # s=args
)?
/?$) # trailing slash
''', re.X)
def sluggify(key): def sluggify(key):
return key.lower().replace('.','_') return key.lower().replace('.','_')
@@ -638,73 +604,61 @@ def regex_url_in(request, environ):
# ################################################## # ##################################################
regex_select(env=environ, request=request) regex_select(env=environ, request=request)
routes = THREAD_LOCAL.routes
if thread.routes.routes_in: if routes.routes_in:
environ = regex_filter_in(environ) environ = regex_filter_in(environ)
request.env.update((sluggify(k),v) for k,v in environ.iteritems()) request.env.update((sluggify(k),v) for k,v in environ.iteritems())
path = request.env.path_info.replace('\\', '/')
# ################################################## # ##################################################
# serve if a static file # serve if a static file
# ################################################## # ##################################################
match = regex_static.match(regex_space.sub('_', path)) path = request.env.path_info.replace('\\', '/') or '/'
if match and match.group('x'): path = regex_space.sub('_',path)
version = match.group('v') if path.endswith('/') and len(path)>1: path = path[:-1]
static_file = pjoin(request.env.applications_parent,
'applications', match.group('b'),
'static', match.group('x'))
return (static_file, version, environ)
# ##################################################
# parse application, controller and function
# ##################################################
path = re.sub('%20', ' ', path)
match = regex_url.match(path) match = regex_url.match(path)
if not match or match.group('c') == 'static': if not match:
raise HTTP(400, raise HTTP(400,
thread.routes.error_message % 'invalid request', routes.error_message % 'invalid request',
web2py_error='invalid path') web2py_error='invalid path')
elif match.group('c')=='static':
request.application = \ application = match.group('a')
regex_space.sub('_', match.group('a') or thread.routes.default_application) version, filename = None, match.group('z')
request.controller = \ items = filename.split('/',1)
regex_space.sub('_', match.group('c') or thread.routes.default_controller) if regex_version.match(items[0]):
request.function = \ version,filename = items
regex_space.sub('_', match.group('f') or thread.routes.default_function) static_file = pjoin(request.env.applications_parent,
group_e = match.group('e') 'applications', application,
request.raw_extension = group_e and regex_space.sub('_', group_e) or None 'static', filename)
request.extension = request.raw_extension or 'html' return (static_file, version, environ)
request.raw_args = match.group('r') else:
request.args = List([]) # ##################################################
if request.application in thread.routes.routes_apps_raw: # parse application, controller and function
# application is responsible for parsing args # ##################################################
request.args = None request.application = match.group('a') or routes.default_application
elif request.raw_args: request.controller = match.group('c') or routes.default_controller
match = regex_args.match(request.raw_args.replace(' ', '_')) request.function = match.group('f') or routes.default_function
if match: request.raw_extension = match.group('e')
group_s = match.group('s') request.extension = request.raw_extension or 'html'
request.args = \ request.raw_args = match.group('s')
List((group_s and group_s.split('/')) or []) if request.application in routes.routes_apps_raw:
if request.args and request.args[-1] == '': # application is responsible for parsing args
request.args.pop() # adjust for trailing empty arg request.args = None
elif request.raw_args:
request.args = List(request.raw_args.split('/')[1:])
else: else:
raise HTTP(400, request.args = List([])
thread.routes.error_message % 'invalid request',
web2py_error='invalid path (args)')
return (None, None, environ) return (None, None, environ)
def regex_filter_out(url, e=None): def regex_filter_out(url, e=None):
"regex rewrite outgoing URL" "regex rewrite outgoing URL"
if not hasattr(thread, 'routes'): if not hasattr(THREAD_LOCAL, 'routes'):
regex_select() # ensure thread.routes is set (for application threads) regex_select() # ensure routes is set (for application threads)
routes = THREAD_LOCAL.routes
if routers: if routers:
return url # already filtered return url # already filtered
if thread.routes.routes_out: if routes.routes_out:
items = url.split('?', 1) items = url.split('?', 1)
if e: if e:
host = e.get('http_host', 'localhost').lower() host = e.get('http_host', 'localhost').lower()
@@ -717,7 +671,7 @@ def regex_filter_out(url, e=None):
e.get('request_method', 'get').lower(), items[0]) e.get('request_method', 'get').lower(), items[0])
else: else:
items[0] = ':http://localhost:get %s' % items[0] items[0] = ':http://localhost:get %s' % items[0]
for (regex, value, tmp) in thread.routes.routes_out: for (regex, value, tmp) in routes.routes_out:
if regex.match(items[0]): if regex.match(items[0]):
rewritten = '?'.join([regex.sub(value, items[0])] + items[1:]) rewritten = '?'.join([regex.sub(value, items[0])] + items[1:])
log_rewrite('routes_out: [%s] -> %s' % (url, rewritten)) log_rewrite('routes_out: [%s] -> %s' % (url, rewritten))
@@ -816,12 +770,13 @@ def filter_url(url, method='get', remote='0.0.0.0',
def filter_err(status, application='app', ticket='tkt'): def filter_err(status, application='app', ticket='tkt'):
"doctest/unittest interface to routes_onerror" "doctest/unittest interface to routes_onerror"
if status > 399 and thread.routes.routes_onerror: routes = THREAD_LOCAL.routes
if status > 399 and routes.routes_onerror:
keys = set(('%s/%s' % (application, status), keys = set(('%s/%s' % (application, status),
'%s/*' % (application), '%s/*' % (application),
'*/%s' % (status), '*/%s' % (status),
'*/*')) '*/*'))
for (key,redir) in thread.routes.routes_onerror: for (key,redir) in routes.routes_onerror:
if key in keys: if key in keys:
if redir == '!': if redir == '!':
break break
@@ -925,12 +880,12 @@ class MapUrlIn(object):
self.pop_arg_if(self.application == arg0) self.pop_arg_if(self.application == arg0)
if not base._acfe_match.match(self.application): if not base._acfe_match.match(self.application):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error="invalid application: '%s'" % self.application) web2py_error="invalid application: '%s'" % self.application)
if self.application not in routers and \ if self.application not in routers and \
(self.application != thread.routes.default_application or self.application == 'welcome'): (self.application != THREAD_LOCAL.routes.default_application or self.application == 'welcome'):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error="unknown application: '%s'" % self.application) web2py_error="unknown application: '%s'" % self.application)
# set the application router # set the application router
@@ -994,7 +949,7 @@ class MapUrlIn(object):
self.pop_arg_if(arg0 == self.controller) self.pop_arg_if(arg0 == self.controller)
log_rewrite("route: controller=%s" % self.controller) log_rewrite("route: controller=%s" % self.controller)
if not self.router._acfe_match.match(self.controller): if not self.router._acfe_match.match(self.controller):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid controller') web2py_error='invalid controller')
def map_static(self): def map_static(self):
@@ -1022,7 +977,7 @@ class MapUrlIn(object):
if bad_static: if bad_static:
log_rewrite('bad static path=%s' % file) log_rewrite('bad static path=%s' % file)
raise HTTP(400, raise HTTP(400,
thread.routes.error_message % 'invalid request', THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid static file') web2py_error='invalid static file')
# #
# support language-specific static subdirectories, # support language-specific static subdirectories,
@@ -1064,10 +1019,10 @@ class MapUrlIn(object):
log_rewrite("route: function.ext=%s.%s" % (self.function, self.extension)) log_rewrite("route: function.ext=%s.%s" % (self.function, self.extension))
if not self.router._acfe_match.match(self.function): if not self.router._acfe_match.match(self.function):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid function') web2py_error='invalid function')
if self.extension and not self.router._acfe_match.match(self.extension): if self.extension and not self.router._acfe_match.match(self.extension):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid extension') web2py_error='invalid extension')
def validate_args(self): def validate_args(self):
@@ -1076,7 +1031,7 @@ class MapUrlIn(object):
''' '''
for arg in self.args: for arg in self.args:
if not self.router._args_match.match(arg): if not self.router._args_match.match(arg):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid arg <%s>' % arg) web2py_error='invalid arg <%s>' % arg)
def sluggify(self): def sluggify(self):
@@ -1301,16 +1256,16 @@ def map_url_in(request, env, app=False):
# initialize router-url object # initialize router-url object
# #
thread.routes = params # default to base routes THREAD_LOCAL.routes = params # default to base routes
map = MapUrlIn(request=request, env=env) map = MapUrlIn(request=request, env=env)
map.sluggify() map.sluggify()
map.map_prefix() # strip prefix if present map.map_prefix() # strip prefix if present
map.map_app() # determine application map.map_app() # determine application
# configure thread.routes for error rewrite # configure THREAD_LOCAL.routes for error rewrite
# #
if params.routes_app: if params.routes_app:
thread.routes = params_apps.get(app, params) THREAD_LOCAL.routes = params_apps.get(app, params)
if app: if app:
return map.application return map.application
gluon/tests/test_routes.py
+7 -7
View File
@@ -163,17 +163,17 @@ default_application = 'defapp'
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/bad!ctl') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/bad!ctl')
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/bad!fcn') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/bad!fcn')
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/fcn.bad!ext') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/fcn.bad!ext')
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path \(args\)\]', filter_url, 'http://domain.com/appc/init/fcn/bad!arg') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/appc/init/fcn/bad!arg')
except AttributeError: except AttributeError:
pass pass
self.assertEqual(filter_url('http://domain.com/welcome/default/fcn_1'), "/welcome/default/fcn_1") self.assertEqual(filter_url('http://domain.com/welcome/default/fcn_1'), "/welcome/default/fcn_1")
self.assertRaises(HTTP, filter_url, 'http://domain.com/welcome/default/fcn-1') #self.assertRaises(HTTP, filter_url, 'http://domain.com/welcome/default/fcn-1')
try: #try:
# 2.7+ only # # 2.7+ only
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/welcome/default/fcn-1') # self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/welcome/default/fcn-1')
except AttributeError: #except AttributeError:
pass # pass
def test_routes_error(self): def test_routes_error(self):
''' '''