Merge github.com:web2py/web2py

This commit is contained in:
Michele Comitini
2012-10-12 21:17:03 +02:00
13 changed files with 234 additions and 438 deletions
+1 -1
View File
@@ -1 +1 @@
Version 2.1.0 (2012-10-10 12:20:21) dev Version 2.1.0 (2012-10-12 12:18:47) dev
@@ -38,7 +38,6 @@ def hello6():
def status(): def status():
""" page that shows internal status""" """ page that shows internal status"""
response.view = 'generic.html'
return dict(toolbar=response.toolbar()) return dict(toolbar=response.toolbar())
+13
View File
@@ -46,6 +46,19 @@ class CacheAbstract(object):
Main function is now to provide referenced api documentation. Main function is now to provide referenced api documentation.
Use CacheInRam or CacheOnDisk instead which are derived from this class. Use CacheInRam or CacheOnDisk instead which are derived from this class.
Attentions, Michele says:
There are signatures inside gdbm files that are used directly
by the python gdbm adapter that often are lagging behind in the
detection code in python part.
On every occasion that a gdbm store is probed by the python adapter,
the probe fails, because gdbm file version is newer.
Using gdbm directly from C would work, because there is backward
compatibility, but not from python!
The .shelve file is discarded and a new one created (with new
signature) and it works until it is probed again...
The possible consequences are memory leaks and broken sessions.
""" """
cache_stats_name = 'web2py_cache_statistics' cache_stats_name = 'web2py_cache_statistics'
+8 -8
View File
@@ -40,6 +40,7 @@ import imp
import logging import logging
logger = logging.getLogger("web2py") logger = logging.getLogger("web2py")
import rewrite import rewrite
from custom_import import custom_import_install
try: try:
import py_compile import py_compile
@@ -411,6 +412,7 @@ def build_environment(request, response, session, store_current=True):
local_import_aux(name,reload,app) local_import_aux(name,reload,app)
BaseAdapter.set_folder(pjoin(request.folder, 'databases')) BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
response._view_environment = copy.copy(environment) response._view_environment = copy.copy(environment)
custom_import_install()
return environment return environment
def save_pyc(filename): def save_pyc(filename):
@@ -530,7 +532,6 @@ def run_controller_in(controller, function, environment):
""" """
# if compiled should run compiled! # if compiled should run compiled!
folder = environment['request'].folder folder = environment['request'].folder
path = pjoin(folder, 'compiled') path = pjoin(folder, 'compiled')
badc = 'invalid controller (%s/%s)' % (controller, function) badc = 'invalid controller (%s/%s)' % (controller, function)
@@ -540,7 +541,7 @@ def run_controller_in(controller, function, environment):
% (controller, function)) % (controller, function))
if not os.path.exists(filename): if not os.path.exists(filename):
raise HTTP(404, raise HTTP(404,
rewrite.thread.routes.error_message % badf, rewrite.THREAD_LOCAL.routes.error_message % badf,
web2py_error=badf) web2py_error=badf)
restricted(read_pyc(filename), environment, layer=filename) restricted(read_pyc(filename), environment, layer=filename)
elif function == '_TEST': elif function == '_TEST':
@@ -555,7 +556,7 @@ def run_controller_in(controller, function, environment):
% controller) % controller)
if not os.path.exists(filename): if not os.path.exists(filename):
raise HTTP(404, raise HTTP(404,
rewrite.thread.routes.error_message % badc, rewrite.THREAD_LOCAL.routes.error_message % badc,
web2py_error=badc) web2py_error=badc)
environment['__symbols__'] = environment.keys() environment['__symbols__'] = environment.keys()
code = read_file(filename) code = read_file(filename)
@@ -566,13 +567,13 @@ def run_controller_in(controller, function, environment):
% controller) % controller)
if not os.path.exists(filename): if not os.path.exists(filename):
raise HTTP(404, raise HTTP(404,
rewrite.thread.routes.error_message % badc, rewrite.THREAD_LOCAL.routes.error_message % badc,
web2py_error=badc) web2py_error=badc)
code = read_file(filename) code = read_file(filename)
exposed = regex_expose.findall(code) exposed = regex_expose.findall(code)
if not function in exposed: if not function in exposed:
raise HTTP(404, raise HTTP(404,
rewrite.thread.routes.error_message % badf, rewrite.THREAD_LOCAL.routes.error_message % badf,
web2py_error=badf) web2py_error=badf)
code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function) code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
if is_gae: if is_gae:
@@ -596,7 +597,6 @@ def run_view_in(environment):
or `view/generic.extension` or `view/generic.extension`
It tries the pre-compiled views_controller_function.pyc before compiling it. It tries the pre-compiled views_controller_function.pyc before compiling it.
""" """
request = environment['request'] request = environment['request']
response = environment['response'] response = environment['response']
view = response.view view = response.view
@@ -632,7 +632,7 @@ def run_view_in(environment):
restricted(code, environment, layer=filename) restricted(code, environment, layer=filename)
return return
raise HTTP(404, raise HTTP(404,
rewrite.thread.routes.error_message % badv, rewrite.THREAD_LOCAL.routes.error_message % badv,
web2py_error=badv) web2py_error=badv)
else: else:
filename = pjoin(folder, 'views', view) filename = pjoin(folder, 'views', view)
@@ -641,7 +641,7 @@ def run_view_in(environment):
filename = pjoin(folder, 'views', view) filename = pjoin(folder, 'views', view)
if not os.path.exists(filename): if not os.path.exists(filename):
raise HTTP(404, raise HTTP(404,
rewrite.thread.routes.error_message % badv, rewrite.THREAD_LOCAL.routes.error_message % badv,
web2py_error=badv) web2py_error=badv)
layer = filename layer = filename
if is_gae: if is_gae:
+18 -11
View File
@@ -12,49 +12,56 @@ cache.ram=cache.disk=MemcacheClient(request)
import time import time
from google.appengine.api.memcache import Client from google.appengine.api.memcache import Client
class MemcacheClient(object):
class MemcacheClient(Client): client = Client()
def __init__(self, request): def __init__(self, request):
self.request = request self.request = request
Client.__init__(self)
def __call__( def __call__(
self, self,
key, key,
f, f,
time_expire=300, time_expire=300,
): ):
key = '%s/%s' % (self.request.application, key) key = '%s/%s' % (self.request.application, key)
dt = time_expire dt = time_expire
value = None value = None
obj = self.get(key) obj = self.client.get(key)
if obj and (dt == None or obj[0] > time.time() - dt): if obj and (dt == None or obj[0] > time.time() - dt):
value = obj[1] value = obj[1]
elif f is None: elif f is None:
if obj: if obj:
self.delete(key) self.client.delete(key)
else: else:
value = f() value = f()
self.set(key, (time.time(), value)) self.client.set(key, (time.time(), value))
return value return value
def increment(self, key, value=1): def increment(self, key, value=1):
key = '%s/%s' % (self.request.application, key) key = '%s/%s' % (self.request.application, key)
obj = self.get(key) obj = self.client.get(key)
if obj: if obj:
value = obj[1] + value value = obj[1] + value
self.set(key, (time.time(), value)) self.client.set(key, (time.time(), value))
return value return value
def clear(self, key = None): def clear(self, key = None):
if key: if key:
key = '%s/%s' % (self.request.application, key) key = '%s/%s' % (self.request.application, key)
self.delete(key) self.client.delete(key)
else: else:
self.flush_all() self.client.flush_all()
def delete(self,*a,**b):
return self.client.delete(*a,**b)
def get(self,*a,**b):
return self.client.delete(*a,**b)
def set(self,*a,**b):
return self.client.delete(*a,**b)
def flush_all(self,*a,**b):
return self.client.delete(*a,**b)
+77 -240
View File
@@ -6,153 +6,119 @@ import os
import re import re
import sys import sys
import threading import threading
import traceback
from gluon import current
# Install the new import function: NAIVE_IMPORTER = __builtin__.__import__
def custom_import_install(web2py_path): TRACK_CHANGES = False
global _web2py_importer INVALID_MODULES = set(sys.modules.keys()).union(('','gluon','applications','custom_import'))
global _web2py_path
if isinstance(__builtin__.__import__, _Web2pyImporter):
return #aready installed
_web2py_path = web2py_path
_web2py_importer = _Web2pyImporter(web2py_path)
__builtin__.__import__ = _web2py_importer
def is_tracking_changes(): # backward compatibility API
""" def custom_import_install():
@return: True: neo_importer is tracking changes made to Python source __builtin__.__import__ = custom_importer
files. False: neo_import does not reload Python modules.
"""
global _is_tracking_changes
return _is_tracking_changes
def track_changes(track=True): def track_changes(track=True):
assert track in (True,False), "must be True or False"
global TRACK_CHANGES
TRACK_CHANGES = track
def is_tracking_changes():
return TRACK_CHANGES
def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
""" """
Tell neo_importer to start/stop tracking changes made to Python modules. The web2py custom importer. Like the standard Python importer but it
@param track: True: Start tracking changes. False: Stop tracking changes. tries to transform import statements as something like
"import applications.app_name.modules.x".
If the import failed, fall back on naive_importer
""" """
global _is_tracking_changes globals = globals or {}
global _web2py_importer locals = locals or {}
global _web2py_date_tracker_importer fromlist = fromlist or []
assert track is True or track is False, "Boolean expected."
if track == _is_tracking_changes:
return
if track:
if not _web2py_date_tracker_importer:
_web2py_date_tracker_importer = \
_Web2pyDateTrackerImporter(_web2py_path)
__builtin__.__import__ = _web2py_date_tracker_importer
else:
__builtin__.__import__ = _web2py_importer
_is_tracking_changes = track
_STANDARD_PYTHON_IMPORTER = __builtin__.__import__ # Keep standard importer base_importer = TRACK_IMPORTER if TRACK_CHANGES else NAIVE_IMPORTER
_web2py_importer = None # The standard web2py importer
_web2py_date_tracker_importer = None # The web2py importer with date tracking
_web2py_path = None # Absolute path of the web2py directory
_is_tracking_changes = False # The tracking mode # if not relative and not from applications:
if hasattr(current,'request') \
class _BaseImporter(object): and level<=0 \
""" and not name.split('.')[0] in INVALID_MODULES \
The base importer. Dispatch the import the call to the standard Python and isinstance(globals, dict):
importer. try:
""" items = current.request.folder.split(os.path.sep)
if not items[-1]: items = items[:-1]
def begin(self): modules_prefix = '.'.join(items[-2:])+'.modules'
""" if not fromlist:
Many imports can be made for a single import statement. This method # import like "import x" or "import x.y"
help the management of this aspect. result = None
""" for itemname in name.split("."):
new_mod = base_importer(
def __init__(self): modules_prefix, globals,locals, [itemname], level)
self._STANDARD_PYTHON_IMPORTER = _STANDARD_PYTHON_IMPORTER try:
def __call__(self, name, globals=None, locals=None, result = result or new_mod.__dict__[itemname]
fromlist=None, level=-1): except KeyError, e:
""" raise ImportError, 'Cannot import module %s' % str(e)
The import method itself. modules_prefix += "." + itemname
""" return result
return self._STANDARD_PYTHON_IMPORTER(name, else:
globals, # import like "from x import a, b, ..."
locals, pname = modules_prefix + "." + name
fromlist, return base_importer(pname, globals, locals, fromlist, level)
level) except ImportError, e1:
pass # the module does not exist
def end(self): except Exception, e2:
""" raise e2 # there is an error in the module
Needed for clean up. return NAIVE_IMPORTER(name,globals,locals,fromlist,level)
"""
class _DateTrackerImporter(_BaseImporter): class TrackImporter(object):
""" """
An importer tracking the date of the module files and reloading them when An importer tracking the date of the module files and reloading them when
they have changed. they have changed.
""" """
_PACKAGE_PATH_SUFFIX = os.path.sep+"__init__.py" THREAD_LOCAL = threading.local()
PACKAGE_PATH_SUFFIX = os.path.sep+"__init__.py"
def __init__(self): def __init__(self):
super(_DateTrackerImporter, self).__init__()
self._import_dates = {} # Import dates of the files of the modules self._import_dates = {} # Import dates of the files of the modules
# Avoid reloading cause by file modifications of reload:
self._tl = threading.local()
self._tl._modules_loaded = None
def begin(self): def __call__(self,name,globals=None,locals=None,fromlist=None,level=-1):
self._tl._modules_loaded = set()
def __call__(self, name, globals=None, locals=None,
fromlist=None, level=-1):
""" """
The import method itself. The import method itself.
""" """
globals = globals or {} globals = globals or {}
locals = locals or {} locals = locals or {}
fromlist = fromlist or [] fromlist = fromlist or []
if not hasattr(self.THREAD_LOCAL,'_modules_loaded'):
call_begin_end = self._tl._modules_loaded is None self.THREAD_LOCAL._modules_loaded = set()
if call_begin_end:
self.begin()
try: try:
self._tl.globals = globals
self._tl.locals = locals
self._tl.level = level
# Check the date and reload if needed: # Check the date and reload if needed:
self._update_dates(name, fromlist) self._update_dates(name, globals, locals, fromlist, level)
# Try to load the module and update the dates if it works: # Try to load the module and update the dates if it works:
result = super(_DateTrackerImporter, self) \ result = NAIVE_IMPORTER(name, globals, locals, fromlist, level)
.__call__(name, globals, locals, fromlist, level)
# Module maybe loaded for the 1st time so we need to set the date # Module maybe loaded for the 1st time so we need to set the date
self._update_dates(name, fromlist) self._update_dates(name, globals, locals, fromlist, level)
return result return result
except Exception: except Exception, e:
raise # Don't hide something that went wrong raise # Don't hide something that went wrong
finally:
if call_begin_end:
self.end()
def _update_dates(self, name, fromlist): def _update_dates(self, name, globals, locals, fromlist, level):
""" """
Update all the dates associated to the statement import. A single Update all the dates associated to the statement import. A single
import statement may import many modules. import statement may import many modules.
""" """
self._reload_check(name) self._reload_check(name, globals, locals, level)
if fromlist: for fromlist_name in fromlist or []:
for fromlist_name in fromlist: pname = "%s.%s" % (name, fromlist_name)
self._reload_check("%s.%s" % (name, fromlist_name)) self._reload_check(pname, globals, locals, level)
def _reload_check(self, name): def _reload_check(self, name, globals, locals, level):
""" """
Update the date associated to the module and reload the module if Update the date associated to the module and reload the module if
the file has changed. the file has changed.
""" """
module = sys.modules.get(name) module = sys.modules.get(name)
file = self._get_module_file(module) file = self._get_module_file(module)
if file: if file:
@@ -170,7 +136,7 @@ class _DateTrackerImporter(_BaseImporter):
# Get path without file ext: # Get path without file ext:
file = os.path.splitext(file)[0] file = os.path.splitext(file)[0]
reload_mod = os.path.isdir(file) \ reload_mod = os.path.isdir(file) \
and os.path.isfile(file+self._PACKAGE_PATH_SUFFIX) and os.path.isfile(file+self.PACKAGE_PATH_SUFFIX)
mod_to_pack = reload_mod mod_to_pack = reload_mod
else: # Package turning into module? else: # Package turning into module?
file += ".py" file += ".py"
@@ -180,156 +146,27 @@ class _DateTrackerImporter(_BaseImporter):
if reload_mod or not date or new_date > date: if reload_mod or not date or new_date > date:
self._import_dates[file] = new_date self._import_dates[file] = new_date
if reload_mod or (date and new_date > date): if reload_mod or (date and new_date > date):
if module not in self._tl._modules_loaded: if module not in self.THREAD_LOCAL._modules_loaded:
if mod_to_pack: if mod_to_pack:
# Module turning into a package: # Module turning into a package:
mod_name = module.__name__ mod_name = module.__name__
del sys.modules[mod_name] # Delete the module del sys.modules[mod_name] # Delete the module
# Reload the module: # Reload the module:
super(_DateTrackerImporter, self).__call__ \ NAIVE_IMPORTER(mod_name, globals, locals, [], level)
(mod_name, self._tl.globals, self._tl.locals, [],
self._tl.level)
else: else:
reload(module) reload(module)
self._tl._modules_loaded.add(module) self.THREAD_LOCAL._modules_loaded.add(module)
def end(self): def _get_module_file(self, module):
self._tl._modules_loaded = None
@classmethod
def _get_module_file(cls, module):
""" """
Get the absolute path file associated to the module or None. Get the absolute path file associated to the module or None.
""" """
file = getattr(module, "__file__", None) file = getattr(module, "__file__", None)
if file: if file:
# Make path absolute if not: # Make path absolute if not:
#file = os.path.join(cls.web2py_path, file)
file = os.path.splitext(file)[0]+".py" # Change .pyc for .py file = os.path.splitext(file)[0]+".py" # Change .pyc for .py
if file.endswith(cls._PACKAGE_PATH_SUFFIX): if file.endswith(self.PACKAGE_PATH_SUFFIX):
file = os.path.dirname(file) # Track dir for packages file = os.path.dirname(file) # Track dir for packages
return file return file
class _Web2pyImporter(_BaseImporter): TRACK_IMPORTER = TrackImporter()
"""
The standard web2py importer. Like the standard Python importer but it
tries to transform import statements as something like
"import applications.app_name.modules.x". If the import failed, fall back
on _BaseImporter.
"""
_RE_ESCAPED_PATH_SEP = re.escape(os.path.sep) # os.path.sep escaped for re
def __init__(self, web2py_path):
"""
@param web2py_path: The absolute path of the web2py installation.
"""
global DEBUG
self.super_class = super(_Web2pyImporter, self)
self.super_class.__init__()
self.web2py_path = web2py_path
self.__web2py_path_os_path_sep = self.web2py_path+os.path.sep
self.__web2py_path_os_path_sep_len = len(self.__web2py_path_os_path_sep)
self.__RE_APP_DIR = re.compile(
self._RE_ESCAPED_PATH_SEP.join( \
( \
#"^" + re.escape(web2py_path), # Not working with Python 2.5
"^(" + "applications",
"[^",
"]+)",
"",
) ))
def _matchAppDir(self, file_path):
"""
Does the file in a directory inside the "applications" directory?
"""
if file_path.startswith(self.__web2py_path_os_path_sep):
file_path = file_path[self.__web2py_path_os_path_sep_len:]
return self.__RE_APP_DIR.match(file_path)
return False
def __call__(self, name, globals=None, locals=None,
fromlist=None, level=-1):
"""
The import method itself.
"""
globals = globals or {}
locals = locals or {}
fromlist = fromlist or []
self.begin()
#try:
# if not relative and not from applications:
if not name.startswith(".") and level <= 0 \
and not name.startswith("applications.") \
and isinstance(globals, dict):
# Get the name of the file do the import
caller_file_name = os.path.join(self.web2py_path, \
globals.get("__file__", ""))
# Is the path in an application directory?
match_app_dir = self._matchAppDir(caller_file_name)
if match_app_dir:
try:
# Get the prefix to add for the import
# (like applications.app_name.modules):
modules_prefix = \
".".join((match_app_dir.group(1). \
replace(os.path.sep, "."), "modules"))
if not fromlist:
# import like "import x" or "import x.y"
return self.__import__dot(modules_prefix, name,
globals, locals, fromlist, level)
else:
# import like "from x import a, b, ..."
return self.super_class \
.__call__(modules_prefix+"."+name,
globals, locals, fromlist, level)
except ImportError, e:
try:
return self.super_class.__call__(name, globals, locals,
fromlist, level)
except ImportError, e1:
raise e
return self.super_class.__call__(name, globals, locals,
fromlist, level)
def __import__dot(self, prefix, name, globals, locals, fromlist,
level):
"""
Here we will import x.y.z as many imports like:
from applications.app_name.modules import x
from applications.app_name.modules.x import y
from applications.app_name.modules.x.y import z.
x will be the module returned.
"""
result = None
for name in name.split("."):
new_mod = super(_Web2pyImporter, self).__call__(prefix, globals,
locals, [name], level)
try:
result = result or new_mod.__dict__[name]
except KeyError, e:
raise ImportError, 'Cannot import module %s' % str(e)
prefix += "." + name
return result
class _Web2pyDateTrackerImporter(_Web2pyImporter, _DateTrackerImporter):
"""
Like _Web2pyImporter but using a _DateTrackerImporter.
"""
+1 -6
View File
@@ -35,8 +35,6 @@ from settings import global_settings
from admin import add_path_first, create_missing_folders, create_missing_app_folders from admin import add_path_first, create_missing_folders, create_missing_app_folders
from globals import current from globals import current
from custom_import import custom_import_install
# Remarks: # Remarks:
# calling script has inserted path to script directory into sys.path # calling script has inserted path to script directory into sys.path
# applications_parent (path to applications/, site-packages/ etc) # applications_parent (path to applications/, site-packages/ etc)
@@ -55,8 +53,6 @@ from custom_import import custom_import_install
web2py_path = global_settings.applications_parent # backward compatibility web2py_path = global_settings.applications_parent # backward compatibility
custom_import_install(web2py_path)
create_missing_folders() create_missing_folders()
# set up logging for subsequent imports # set up logging for subsequent imports
@@ -91,7 +87,7 @@ from validators import CRYPT
from cache import CacheInRam from cache import CacheInRam
from html import URL, xmlescape from html import URL, xmlescape
from utils import is_valid_ip_address from utils import is_valid_ip_address
from rewrite import load, url_in, thread as rwthread, \ from rewrite import load, url_in, THREAD_LOCAL as rwthread, \
try_rewrite_on_error, fixup_missing_path_info try_rewrite_on_error, fixup_missing_path_info
import newcron import newcron
@@ -799,7 +795,6 @@ class HttpServer(object):
global_settings.applications_parent = path global_settings.applications_parent = path
os.chdir(path) os.chdir(path)
[add_path_first(p) for p in (path, abspath('site-packages'), "")] [add_path_first(p) for p in (path, abspath('site-packages'), "")]
custom_import_install(web2py_path)
if exists("logging.conf"): if exists("logging.conf"):
logging.config.fileConfig("logging.conf") logging.config.fileConfig("logging.conf")
+7 -3
View File
@@ -314,22 +314,26 @@ def crondance(applications_parent, ctype='soft', startup=False, apps=None):
(action,models,command) = (True,'-M',command[1:]) (action,models,command) = (True,'-M',command[1:])
else: else:
action=False action=False
if action and command.endswith('.py'): if action and command.endswith('.py'):
commands.extend(('-J', # cron job commands.extend(('-J', # cron job
models, # import models? models, # import models?
'-S', app, # app name '-S', app, # app name
'-a', '"<recycle>"', # password '-a', '"<recycle>"', # password
'-R', command)) # command '-R', command)) # command
shell = True
elif action: elif action:
commands.extend(('-J', # cron job commands.extend(('-J', # cron job
models, # import models? models, # import models?
'-S', app+'/'+command, # app name '-S', app+'/'+command, # app name
'-a', '"<recycle>"')) # password '-a', '"<recycle>"')) # password
shell = True
else: else:
commands = command commands = command
shell = False
# from python docs:
# You do not need shell=True to run a batch file or
# console-based executable.
shell = False
try: try:
cronlauncher(commands, shell=shell).start() cronlauncher(commands, shell=shell).start()
except Exception, e: except Exception, e:
+99 -155
View File
@@ -33,13 +33,29 @@ exists = os.path.exists
pjoin = os.path.join pjoin = os.path.join
logger = logging.getLogger('web2py.rewrite') logger = logging.getLogger('web2py.rewrite')
thread = threading.local() # thread-local storage for routing params THREAD_LOCAL = threading.local() # thread-local storage for routing params
regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*') regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
regex_anything = re.compile(r'(?<!\\)\$anything') regex_anything = re.compile(r'(?<!\\)\$anything')
regex_redirect = re.compile(r'(\d+)->(.*)') regex_redirect = re.compile(r'(\d+)->(.*)')
regex_full_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)') regex_full_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)')
regex_version = re.compile(r'^(_[\d]+\.[\d]+\.[\d]+)$') regex_version = re.compile(r'^(_[\d]+\.[\d]+\.[\d]+)$')
# pattern to replace spaces with underscore in URL
# also the html escaped variants '+' and '%20' are covered
regex_space = re.compile('(\+|\s|%20)+')
# pattern to find valid paths in url /application/controller/...
# this could be:
# for static pages:
# /<b:application>/static/<x:file>
# for dynamic pages:
# /<a:application>[/<c:controller>[/<f:function>[.<e:ext>][/<s:args>]]]
# application, controller, function and ext may only contain [a-zA-Z0-9_]
# file and args may also contain '-', '=', '.' and '/'
# apps in routes_apps_raw must parse raw_args into args
regex_url = re.compile('^/((?P<a>\w+)(/(?P<c>\w+)(/(?P<z>(?P<f>\w+)(\.(?P<e>[\w.]+))?(?P<s>[/\w@=-]*(\.[/\w@=-]+)*)))?)?)?$')
def _router_default(): def _router_default():
"return new copy of default base router" "return new copy of default base router"
@@ -85,7 +101,7 @@ def _params_default(app=None):
params_apps = dict() params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters THREAD_LOCAL.routes = params # default to base regex rewrite parameters
routers = None routers = None
def log_rewrite(string): def log_rewrite(string):
@@ -126,11 +142,11 @@ ROUTER_BASE_KEYS = set(
# url_in: parse and rewrite incoming URL # url_in: parse and rewrite incoming URL
# url_out: assemble and rewrite outgoing URL # url_out: assemble and rewrite outgoing URL
# #
# thread.routes.default_application # THREAD_LOCAL.routes.default_application
# thread.routes.error_message # THREAD_LOCAL.routes.error_message
# thread.routes.error_message_ticket # THREAD_LOCAL.routes.error_message_ticket
# thread.routes.try_redirect_on_error # THREAD_LOCAL.routes.try_redirect_on_error
# thread.routes.error_handler # THREAD_LOCAL.routes.error_handler
# #
# filter_url: helper for doctest & unittest # filter_url: helper for doctest & unittest
# filter_err: helper for doctest & unittest # filter_err: helper for doctest & unittest
@@ -163,38 +179,28 @@ def url_in(request, environ):
return map_url_in(request, environ) return map_url_in(request, environ)
return regex_url_in(request, environ) return regex_url_in(request, environ)
def url_out(request, env, application, controller, function, def url_out(request, environ, application, controller, function,
args, other, scheme, host, port): args, other, scheme, host, port):
"assemble and rewrite outgoing URL" "assemble and rewrite outgoing URL"
if routers: if routers:
acf = map_url_out(request, env, application, controller, acf = map_url_out(request, environ, application, controller,
function, args, other, scheme, host, port) function, args, other, scheme, host, port)
url = '%s%s' % (acf, other) url = '%s%s' % (acf, other)
else: else:
url = '/%s/%s/%s%s' % (application, controller, function, other) url = '/%s/%s/%s%s' % (application, controller, function, other)
url = regex_filter_out(url, env) url = regex_filter_out(url, environ)
# #
# fill in scheme and host if absolute URL is requested # fill in scheme and host if absolute URL is requested
# scheme can be a string, eg 'http', 'https', 'ws', 'wss' # scheme can be a string, eg 'http', 'https', 'ws', 'wss'
# #
if scheme or port is not None: if host is True or (host is None and (scheme or port!=None)):
if host is None: # scheme or port implies host host = request.env.http_host
host = True
if not scheme or scheme is True: if not scheme or scheme is True:
if request and request.env: scheme = request.env.get('wsgi_url_scheme', 'http').lower() \
scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
else:
scheme = 'http' # some reasonable default in case we need it
if host is not None:
if host is True:
host = request.env.http_host
if host: if host:
if port is None: host_port = host if not port else host.split(':',1)[0]+':%s'%port
port = '' url = '%s://%s%s' % (scheme, host_port, url)
else:
port = ':%s' % port
host = host.split(':')[0]
url = '%s://%s%s%s' % (scheme, host, port, url)
return url return url
def try_rewrite_on_error(http_response, request, environ, ticket=None): def try_rewrite_on_error(http_response, request, environ, ticket=None):
@@ -202,12 +208,12 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
called from main.wsgibase to rewrite the http response. called from main.wsgibase to rewrite the http response.
""" """
status = int(str(http_response.status).split()[0]) status = int(str(http_response.status).split()[0])
if status>=399 and thread.routes.routes_onerror: if status>=399 and THREAD_LOCAL.routes.routes_onerror:
keys=set(('%s/%s' % (request.application, status), keys=set(('%s/%s' % (request.application, status),
'%s/*' % (request.application), '%s/*' % (request.application),
'*/%s' % (status), '*/%s' % (status),
'*/*')) '*/*'))
for (key,uri) in thread.routes.routes_onerror: for (key,uri) in THREAD_LOCAL.routes.routes_onerror:
if key in keys: if key in keys:
if uri == '!': if uri == '!':
# do nothing! # do nothing!
@@ -245,12 +251,12 @@ def try_rewrite_on_error(http_response, request, environ, ticket=None):
def try_redirect_on_error(http_object, request, ticket=None): def try_redirect_on_error(http_object, request, ticket=None):
"called from main.wsgibase to rewrite the http response" "called from main.wsgibase to rewrite the http response"
status = int(str(http_object.status).split()[0]) status = int(str(http_object.status).split()[0])
if status>399 and thread.routes.routes_onerror: if status>399 and THREAD_LOCAL.routes.routes_onerror:
keys=set(('%s/%s' % (request.application, status), keys=set(('%s/%s' % (request.application, status),
'%s/*' % (request.application), '%s/*' % (request.application),
'*/%s' % (status), '*/%s' % (status),
'*/*')) '*/*'))
for (key,redir) in thread.routes.routes_onerror: for (key,redir) in THREAD_LOCAL.routes.routes_onerror:
if key in keys: if key in keys:
if redir == '!': if redir == '!':
break break
@@ -283,7 +289,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
global params_apps global params_apps
params_apps = dict() params_apps = dict()
params = _params_default(app=None) # regex rewrite parameters params = _params_default(app=None) # regex rewrite parameters
thread.routes = params # default to base regex rewrite parameters THREAD_LOCAL.routes = params # default to base regex rewrite parameters
routers = None routers = None
if isinstance(rdict, dict): if isinstance(rdict, dict):
@@ -330,7 +336,7 @@ def load(routes='routes.py', app=None, data=None, rdict=None):
if app is None: if app is None:
params = p # install base rewrite parameters params = p # install base rewrite parameters
thread.routes = params # install default as current routes THREAD_LOCAL.routes = params # install default as current routes
# #
# create the BASE router if routers in use # create the BASE router if routers in use
# #
@@ -539,24 +545,25 @@ def regex_select(env=None, app=None, request=None):
select a set of regex rewrite params for the current request select a set of regex rewrite params for the current request
""" """
if app: if app:
thread.routes = params_apps.get(app, params) THREAD_LOCAL.routes = params_apps.get(app, params)
elif env and params.routes_app: elif env and params.routes_app:
if routers: if routers:
map_url_in(request, env, app=True) map_url_in(request, env, app=True)
else: else:
app = regex_uri(env, params.routes_app, "routes_app") app = regex_uri(env, params.routes_app, "routes_app")
thread.routes = params_apps.get(app, params) THREAD_LOCAL.routes = params_apps.get(app, params)
else: else:
thread.routes = params # default to base rewrite parameters THREAD_LOCAL.routes = params # default to base rewrite parameters
log_rewrite("select routing parameters: %s" % thread.routes.name) log_rewrite("select routing parameters: %s" % THREAD_LOCAL.routes.name)
return app # for doctest return app # for doctest
def regex_filter_in(e): def regex_filter_in(e):
"regex rewrite incoming URL" "regex rewrite incoming URL"
routes = THREAD_LOCAL.routes
query = e.get('QUERY_STRING', None) query = e.get('QUERY_STRING', None)
e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '') e['WEB2PY_ORIGINAL_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
if thread.routes.routes_in: if routes.routes_in:
path = regex_uri(e, thread.routes.routes_in, path = regex_uri(e, routes.routes_in,
"routes_in", e['PATH_INFO']) "routes_in", e['PATH_INFO'])
rmatch = regex_redirect.match(path) rmatch = regex_redirect.match(path)
if rmatch: if rmatch:
@@ -572,59 +579,7 @@ def regex_filter_in(e):
e['REQUEST_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '') e['REQUEST_URI'] = e['PATH_INFO'] + (query and ('?' + query) or '')
return e return e
# pattern to replace spaces with underscore in URL
# also the html escaped variants '+' and '%20' are covered
regex_space = re.compile('(\+|\s|%20)+')
# pattern to find valid paths in url /application/controller/...
# this could be:
# for static pages:
# /<b:application>/static/<x:file>
# for dynamic pages:
# /<a:application>[/<c:controller>[/<f:function>[.<e:ext>][/<s:args>]]]
# application, controller, function and ext may only contain [a-zA-Z0-9_]
# file and args may also contain '-', '=', '.' and '/'
# apps in routes_apps_raw must parse raw_args into args
regex_static = re.compile(r'''
(^ # static pages
/(?P<b> \w+) # b=app
/static # /b/static
(/(?P<v>_[\d]+\.[\d]+\.[\d]+))? # version ?
/(?P<x> (\w[\-\=\./]?)* ) # x=file
$)
''', re.X)
regex_url = re.compile(r'''
(^( # (/a/c/f.e/s)
/(?P<a> [\w\s+]+ ) # /a=app
( # (/c.f.e/s)
/(?P<c> [\w\s+]+ ) # /a/c=controller
( # (/f.e/s)
/(?P<f> [\w\s+]+ ) # /a/c/f=function
( # (.e)
\.(?P<e> [\w\s+]+ ) # /a/c/f.e=extension
)?
( # (/s)
/(?P<r> # /a/c/f.e/r=raw_args
.*
)
)?
)?
)?
)?
/?$)
''', re.X)
regex_args = re.compile(r'''
(^
(?P<s>
( [\w@/-][=.]? )* # s=args
)?
/?$) # trailing slash
''', re.X)
def sluggify(key): def sluggify(key):
return key.lower().replace('.','_') return key.lower().replace('.','_')
@@ -638,73 +593,61 @@ def regex_url_in(request, environ):
# ################################################## # ##################################################
regex_select(env=environ, request=request) regex_select(env=environ, request=request)
routes = THREAD_LOCAL.routes
if thread.routes.routes_in: if routes.routes_in:
environ = regex_filter_in(environ) environ = regex_filter_in(environ)
request.env.update((sluggify(k),v) for k,v in environ.iteritems()) request.env.update((sluggify(k),v) for k,v in environ.iteritems())
path = request.env.path_info.replace('\\', '/')
# ################################################## # ##################################################
# serve if a static file # serve if a static file
# ################################################## # ##################################################
match = regex_static.match(regex_space.sub('_', path)) path = request.env.path_info.replace('\\', '/') or '/'
if match and match.group('x'): path = regex_space.sub('_',path)
version = match.group('v') if path.endswith('/') and len(path)>1: path = path[:-1]
static_file = pjoin(request.env.applications_parent,
'applications', match.group('b'),
'static', match.group('x'))
return (static_file, version, environ)
# ##################################################
# parse application, controller and function
# ##################################################
path = re.sub('%20', ' ', path)
match = regex_url.match(path) match = regex_url.match(path)
if not match or match.group('c') == 'static': if not match:
raise HTTP(400, raise HTTP(400,
thread.routes.error_message % 'invalid request', routes.error_message % 'invalid request',
web2py_error='invalid path') web2py_error='invalid path')
elif match.group('c')=='static':
request.application = \ application = match.group('a')
regex_space.sub('_', match.group('a') or thread.routes.default_application) version, filename = None, match.group('z')
request.controller = \ items = filename.split('/',1)
regex_space.sub('_', match.group('c') or thread.routes.default_controller) if regex_version.match(items[0]):
request.function = \ version,filename = items
regex_space.sub('_', match.group('f') or thread.routes.default_function) static_file = pjoin(request.env.applications_parent,
group_e = match.group('e') 'applications', application,
request.raw_extension = group_e and regex_space.sub('_', group_e) or None 'static', filename)
request.extension = request.raw_extension or 'html' return (static_file, version, environ)
request.raw_args = match.group('r') else:
request.args = List([]) # ##################################################
if request.application in thread.routes.routes_apps_raw: # parse application, controller and function
# application is responsible for parsing args # ##################################################
request.args = None request.application = match.group('a') or routes.default_application
elif request.raw_args: request.controller = match.group('c') or routes.default_controller
match = regex_args.match(request.raw_args.replace(' ', '_')) request.function = match.group('f') or routes.default_function
if match: request.raw_extension = match.group('e')
group_s = match.group('s') request.extension = request.raw_extension or 'html'
request.args = \ request.raw_args = match.group('s')
List((group_s and group_s.split('/')) or []) if request.application in routes.routes_apps_raw:
if request.args and request.args[-1] == '': # application is responsible for parsing args
request.args.pop() # adjust for trailing empty arg request.args = None
elif request.raw_args:
request.args = List(request.raw_args.split('/')[1:])
else: else:
raise HTTP(400, request.args = List([])
thread.routes.error_message % 'invalid request',
web2py_error='invalid path (args)')
return (None, None, environ) return (None, None, environ)
def regex_filter_out(url, e=None): def regex_filter_out(url, e=None):
"regex rewrite outgoing URL" "regex rewrite outgoing URL"
if not hasattr(thread, 'routes'): if not hasattr(THREAD_LOCAL, 'routes'):
regex_select() # ensure thread.routes is set (for application threads) regex_select() # ensure routes is set (for application threads)
routes = THREAD_LOCAL.routes
if routers: if routers:
return url # already filtered return url # already filtered
if thread.routes.routes_out: if routes.routes_out:
items = url.split('?', 1) items = url.split('?', 1)
if e: if e:
host = e.get('http_host', 'localhost').lower() host = e.get('http_host', 'localhost').lower()
@@ -717,7 +660,7 @@ def regex_filter_out(url, e=None):
e.get('request_method', 'get').lower(), items[0]) e.get('request_method', 'get').lower(), items[0])
else: else:
items[0] = ':http://localhost:get %s' % items[0] items[0] = ':http://localhost:get %s' % items[0]
for (regex, value, tmp) in thread.routes.routes_out: for (regex, value, tmp) in routes.routes_out:
if regex.match(items[0]): if regex.match(items[0]):
rewritten = '?'.join([regex.sub(value, items[0])] + items[1:]) rewritten = '?'.join([regex.sub(value, items[0])] + items[1:])
log_rewrite('routes_out: [%s] -> %s' % (url, rewritten)) log_rewrite('routes_out: [%s] -> %s' % (url, rewritten))
@@ -816,12 +759,13 @@ def filter_url(url, method='get', remote='0.0.0.0',
def filter_err(status, application='app', ticket='tkt'): def filter_err(status, application='app', ticket='tkt'):
"doctest/unittest interface to routes_onerror" "doctest/unittest interface to routes_onerror"
if status > 399 and thread.routes.routes_onerror: routes = THREAD_LOCAL.routes
if status > 399 and routes.routes_onerror:
keys = set(('%s/%s' % (application, status), keys = set(('%s/%s' % (application, status),
'%s/*' % (application), '%s/*' % (application),
'*/%s' % (status), '*/%s' % (status),
'*/*')) '*/*'))
for (key,redir) in thread.routes.routes_onerror: for (key,redir) in routes.routes_onerror:
if key in keys: if key in keys:
if redir == '!': if redir == '!':
break break
@@ -925,12 +869,12 @@ class MapUrlIn(object):
self.pop_arg_if(self.application == arg0) self.pop_arg_if(self.application == arg0)
if not base._acfe_match.match(self.application): if not base._acfe_match.match(self.application):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error="invalid application: '%s'" % self.application) web2py_error="invalid application: '%s'" % self.application)
if self.application not in routers and \ if self.application not in routers and \
(self.application != thread.routes.default_application or self.application == 'welcome'): (self.application != THREAD_LOCAL.routes.default_application or self.application == 'welcome'):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error="unknown application: '%s'" % self.application) web2py_error="unknown application: '%s'" % self.application)
# set the application router # set the application router
@@ -994,7 +938,7 @@ class MapUrlIn(object):
self.pop_arg_if(arg0 == self.controller) self.pop_arg_if(arg0 == self.controller)
log_rewrite("route: controller=%s" % self.controller) log_rewrite("route: controller=%s" % self.controller)
if not self.router._acfe_match.match(self.controller): if not self.router._acfe_match.match(self.controller):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid controller') web2py_error='invalid controller')
def map_static(self): def map_static(self):
@@ -1022,7 +966,7 @@ class MapUrlIn(object):
if bad_static: if bad_static:
log_rewrite('bad static path=%s' % file) log_rewrite('bad static path=%s' % file)
raise HTTP(400, raise HTTP(400,
thread.routes.error_message % 'invalid request', THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid static file') web2py_error='invalid static file')
# #
# support language-specific static subdirectories, # support language-specific static subdirectories,
@@ -1064,10 +1008,10 @@ class MapUrlIn(object):
log_rewrite("route: function.ext=%s.%s" % (self.function, self.extension)) log_rewrite("route: function.ext=%s.%s" % (self.function, self.extension))
if not self.router._acfe_match.match(self.function): if not self.router._acfe_match.match(self.function):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid function') web2py_error='invalid function')
if self.extension and not self.router._acfe_match.match(self.extension): if self.extension and not self.router._acfe_match.match(self.extension):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid extension') web2py_error='invalid extension')
def validate_args(self): def validate_args(self):
@@ -1076,7 +1020,7 @@ class MapUrlIn(object):
''' '''
for arg in self.args: for arg in self.args:
if not self.router._args_match.match(arg): if not self.router._args_match.match(arg):
raise HTTP(400, thread.routes.error_message % 'invalid request', raise HTTP(400, THREAD_LOCAL.routes.error_message % 'invalid request',
web2py_error='invalid arg <%s>' % arg) web2py_error='invalid arg <%s>' % arg)
def sluggify(self): def sluggify(self):
@@ -1301,16 +1245,16 @@ def map_url_in(request, env, app=False):
# initialize router-url object # initialize router-url object
# #
thread.routes = params # default to base routes THREAD_LOCAL.routes = params # default to base routes
map = MapUrlIn(request=request, env=env) map = MapUrlIn(request=request, env=env)
map.sluggify() map.sluggify()
map.map_prefix() # strip prefix if present map.map_prefix() # strip prefix if present
map.map_app() # determine application map.map_app() # determine application
# configure thread.routes for error rewrite # configure THREAD_LOCAL.routes for error rewrite
# #
if params.routes_app: if params.routes_app:
thread.routes = params_apps.get(app, params) THREAD_LOCAL.routes = params_apps.get(app, params)
if app: if app:
return map.application return map.application
+2 -3
View File
@@ -19,7 +19,7 @@ import optparse
import glob import glob
import traceback import traceback
import fileutils import fileutils
import settings from settings import global_settings
from utils import web2py_uuid from utils import web2py_uuid
from compileapp import build_environment, read_pyc, run_models_in from compileapp import build_environment, read_pyc, run_models_in
from restricted import RestrictedError from restricted import RestrictedError
@@ -28,7 +28,6 @@ from storage import Storage
from admin import w2p_unpack from admin import w2p_unpack
from dal import BaseAdapter from dal import BaseAdapter
logger = logging.getLogger("web2py") logger = logging.getLogger("web2py")
def exec_environment( def exec_environment(
@@ -112,7 +111,7 @@ def env(
request.env.path_info = '/%s/%s/%s' % (a, c, f) request.env.path_info = '/%s/%s/%s' % (a, c, f)
request.env.http_host = '127.0.0.1:8000' request.env.http_host = '127.0.0.1:8000'
request.env.remote_addr = '127.0.0.1' request.env.remote_addr = '127.0.0.1'
request.env.web2py_runtime_gae = settings.global_settings.web2py_runtime_gae request.env.web2py_runtime_gae = global_settings.web2py_runtime_gae
for k,v in extra_request.items(): for k,v in extra_request.items():
request[k] = v request[k] = v
+1 -1
View File
@@ -47,7 +47,7 @@ def stream_file_or_304_or_206(
error_message = None, error_message = None,
): ):
if error_message is None: if error_message is None:
error_message = rewrite.thread.routes.error_message % 'invalid request' error_message = rewrite.THREAD_LOCAL.routes.error_message % 'invalid request'
try: try:
fp = open(static_file) fp = open(static_file)
except IOError, e: except IOError, e:
+7 -7
View File
@@ -163,17 +163,17 @@ default_application = 'defapp'
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/bad!ctl') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/bad!ctl')
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/bad!fcn') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/bad!fcn')
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/fcn.bad!ext') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/init/ctlr/fcn.bad!ext')
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path \(args\)\]', filter_url, 'http://domain.com/appc/init/fcn/bad!arg') self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/appc/init/fcn/bad!arg')
except AttributeError: except AttributeError:
pass pass
self.assertEqual(filter_url('http://domain.com/welcome/default/fcn_1'), "/welcome/default/fcn_1") self.assertEqual(filter_url('http://domain.com/welcome/default/fcn_1'), "/welcome/default/fcn_1")
self.assertRaises(HTTP, filter_url, 'http://domain.com/welcome/default/fcn-1') #self.assertRaises(HTTP, filter_url, 'http://domain.com/welcome/default/fcn-1')
try: #try:
# 2.7+ only # # 2.7+ only
self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/welcome/default/fcn-1') # self.assertRaisesRegexp(HTTP, '400 BAD REQUEST \[invalid path\]', filter_url, 'http://domain.com/welcome/default/fcn-1')
except AttributeError: #except AttributeError:
pass # pass
def test_routes_error(self): def test_routes_error(self):
''' '''
-2
View File
@@ -94,8 +94,6 @@ class Web2pyService(Service):
os.chdir(dir) os.chdir(dir)
from gluon.settings import global_settings from gluon.settings import global_settings
global_settings.gluon_parent = dir global_settings.gluon_parent = dir
from gluon.custom_import import custom_import_install
custom_import_install(dir)
return True return True
except: except:
self.log("Can't change to web2py working path; server is stopped") self.log("Can't change to web2py working path; server is stopped")