From aadde5a791ad1ec8cb57ad309dfff295ac5cd63a Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 11 Oct 2012 22:03:20 -0500 Subject: [PATCH] much better custom_import, smaller, faster, and finally works with shell --- VERSION | 2 +- gluon/compileapp.py | 2 + gluon/custom_import.py | 309 ++++++++++------------------------------- gluon/main.py | 5 - gluon/shell.py | 1 - gluon/winservice.py | 2 - 6 files changed, 80 insertions(+), 241 deletions(-) diff --git a/VERSION b/VERSION index c9901797..5ab92aea 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.1.0 (2012-10-11 16:48:00) dev +Version 2.1.0 (2012-10-11 22:03:15) dev diff --git a/gluon/compileapp.py b/gluon/compileapp.py index f131b33b..3c4e3b74 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -40,6 +40,7 @@ import imp import logging logger = logging.getLogger("web2py") import rewrite +from custom_import import custom_import_install try: import py_compile @@ -411,6 +412,7 @@ def build_environment(request, response, session, store_current=True): local_import_aux(name,reload,app) BaseAdapter.set_folder(pjoin(request.folder, 'databases')) response._view_environment = copy.copy(environment) + custom_import_install() return environment def save_pyc(filename): diff --git a/gluon/custom_import.py b/gluon/custom_import.py index abddfce4..4ba79f26 100644 --- a/gluon/custom_import.py +++ b/gluon/custom_import.py @@ -6,149 +6,119 @@ import os import re import sys import threading +import traceback +from gluon import current -# Install the new import function: -def custom_import_install(web2py_path): - global _web2py_importer - global _web2py_path - if isinstance(__builtin__.__import__, _Web2pyImporter): - return #aready installed - _web2py_path = web2py_path - _web2py_importer = _Web2pyImporter(web2py_path) - __builtin__.__import__ = _web2py_importer +NAIVE_IMPORTER = __builtin__.__import__ +TRACK_CHANGES = False +INVALID_MODULES = set(sys.modules.keys()).union(('','gluon','applications','custom_import')) -def is_tracking_changes(): - """ - @return: True: neo_importer is tracking changes made to Python source - files. False: neo_import does not reload Python modules. - """ - global _is_tracking_changes - return _is_tracking_changes +# backward compatibility API +def custom_import_install(): + __builtin__.__import__ = custom_importer def track_changes(track=True): + assert track in (True,False), "must be True or False" + global TRACK_CHANGES + TRACK_CHANGES = track + +def is_tracking_changes(): + return TRACK_CHANGES + +def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1): """ - Tell neo_importer to start/stop tracking changes made to Python modules. - @param track: True: Start tracking changes. False: Stop tracking changes. + The web2py custom importer. Like the standard Python importer but it + tries to transform import statements as something like + "import applications.app_name.modules.x". + If the import failed, fall back on naive_importer """ - global _is_tracking_changes - global _web2py_importer - global _web2py_date_tracker_importer - assert track is True or track is False, "Boolean expected." - if track == _is_tracking_changes: - return - if track: - if not _web2py_date_tracker_importer: - _web2py_date_tracker_importer = \ - _Web2pyDateTrackerImporter(_web2py_path) - __builtin__.__import__ = _web2py_date_tracker_importer - else: - __builtin__.__import__ = _web2py_importer - _is_tracking_changes = track + globals = globals or {} + locals = locals or {} + fromlist = fromlist or [] -_STANDARD_PYTHON_IMPORTER = __builtin__.__import__ # Keep standard importer -_web2py_importer = None # The standard web2py importer -_web2py_date_tracker_importer = None # The web2py importer with date tracking -_web2py_path = None # Absolute path of the web2py directory + base_importer = TRACK_IMPORTER if TRACK_CHANGES else NAIVE_IMPORTER -_is_tracking_changes = False # The tracking mode - -class _BaseImporter(object): - """ - The base importer. Dispatch the import the call to the standard Python - importer. - """ - - def begin(self): - """ - Many imports can be made for a single import statement. This method - help the management of this aspect. - """ - - def __init__(self): - self._STANDARD_PYTHON_IMPORTER = _STANDARD_PYTHON_IMPORTER - def __call__(self, name, globals=None, locals=None, - fromlist=None, level=-1): - """ - The import method itself. - """ - return self._STANDARD_PYTHON_IMPORTER( - name,globals,locals,fromlist,level) - - def end(self): - """ - Needed for clean up. - """ + # if not relative and not from applications: + if hasattr(current,'request') \ + and level<=0 \ + and not name.split('.')[0] in INVALID_MODULES \ + and isinstance(globals, dict): + try: + items = current.request.folder.split(os.path.sep) + if not items[-1]: items = items[:-1] + modules_prefix = '.'.join(items[-2:])+'.modules' + if not fromlist: + # import like "import x" or "import x.y" + result = None + for itemname in name.split("."): + new_mod = base_importer( + modules_prefix, globals,locals, [itemname], level) + try: + result = result or new_mod.__dict__[itemname] + except KeyError, e: + raise ImportError, 'Cannot import module %s' % str(e) + modules_prefix += "." + itemname + return result + else: + # import like "from x import a, b, ..." + pname = modules_prefix + "." + name + return base_importer(pname, globals, locals, fromlist, level) + except ImportError, e1: + pass # the module does not exist + except Exception, e2: + raise e2 # there is an error in the module + return NAIVE_IMPORTER(name,globals,locals,fromlist,level) -class _DateTrackerImporter(_BaseImporter): +class TrackImporter(object): """ An importer tracking the date of the module files and reloading them when they have changed. """ - _PACKAGE_PATH_SUFFIX = os.path.sep+"__init__.py" + THREAD_LOCAL = threading.local() + PACKAGE_PATH_SUFFIX = os.path.sep+"__init__.py" def __init__(self): - super(_DateTrackerImporter, self).__init__() self._import_dates = {} # Import dates of the files of the modules - # Avoid reloading cause by file modifications of reload: - self._tl = threading.local() - self._tl._modules_loaded = None - def begin(self): - self._tl._modules_loaded = set() - - def __call__(self, name, globals=None, locals=None, - fromlist=None, level=-1): + def __call__(self,name,globals=None,locals=None,fromlist=None,level=-1): """ The import method itself. """ - globals = globals or {} locals = locals or {} fromlist = fromlist or [] - - call_begin_end = self._tl._modules_loaded is None - if call_begin_end: - self.begin() + if not hasattr(self.THREAD_LOCAL,'_modules_loaded'): + self.THREAD_LOCAL._modules_loaded = set() try: - self._tl.globals = globals - self._tl.locals = locals - self._tl.level = level - # Check the date and reload if needed: - self._update_dates(name, fromlist) - + self._update_dates(name, globals, locals, fromlist, level) # Try to load the module and update the dates if it works: - result = super(_DateTrackerImporter, self) \ - .__call__(name, globals, locals, fromlist, level) + result = NAIVE_IMPORTER(name, globals, locals, fromlist, level) # Module maybe loaded for the 1st time so we need to set the date - self._update_dates(name, fromlist) + self._update_dates(name, globals, locals, fromlist, level) return result - except Exception: + except Exception, e: raise # Don't hide something that went wrong - finally: - if call_begin_end: - self.end() - def _update_dates(self, name, fromlist): + def _update_dates(self, name, globals, locals, fromlist, level): """ Update all the dates associated to the statement import. A single import statement may import many modules. """ - self._reload_check(name) - if fromlist: - for fromlist_name in fromlist: - self._reload_check("%s.%s" % (name, fromlist_name)) + self._reload_check(name, globals, locals, level) + for fromlist_name in fromlist or []: + pname = "%s.%s" % (name, fromlist_name) + self._reload_check(pname, globals, locals, level) - def _reload_check(self, name): + def _reload_check(self, name, globals, locals, level): """ Update the date associated to the module and reload the module if the file has changed. """ - module = sys.modules.get(name) file = self._get_module_file(module) if file: @@ -166,7 +136,7 @@ class _DateTrackerImporter(_BaseImporter): # Get path without file ext: file = os.path.splitext(file)[0] reload_mod = os.path.isdir(file) \ - and os.path.isfile(file+self._PACKAGE_PATH_SUFFIX) + and os.path.isfile(file+self.PACKAGE_PATH_SUFFIX) mod_to_pack = reload_mod else: # Package turning into module? file += ".py" @@ -176,152 +146,27 @@ class _DateTrackerImporter(_BaseImporter): if reload_mod or not date or new_date > date: self._import_dates[file] = new_date if reload_mod or (date and new_date > date): - if module not in self._tl._modules_loaded: + if module not in self.THREAD_LOCAL._modules_loaded: if mod_to_pack: # Module turning into a package: mod_name = module.__name__ del sys.modules[mod_name] # Delete the module # Reload the module: - super(_DateTrackerImporter, self).__call__ \ - (mod_name, self._tl.globals, self._tl.locals, [], - self._tl.level) + NAIVE_IMPORTER(mod_name, globals, locals, [], level) else: reload(module) - self._tl._modules_loaded.add(module) + self.THREAD_LOCAL._modules_loaded.add(module) - def end(self): - self._tl._modules_loaded = None - - @classmethod - def _get_module_file(cls, module): + def _get_module_file(self, module): """ Get the absolute path file associated to the module or None. """ - file = getattr(module, "__file__", None) if file: # Make path absolute if not: - #file = os.path.join(cls.web2py_path, file) - file = os.path.splitext(file)[0]+".py" # Change .pyc for .py - if file.endswith(cls._PACKAGE_PATH_SUFFIX): + if file.endswith(self.PACKAGE_PATH_SUFFIX): file = os.path.dirname(file) # Track dir for packages return file -class _Web2pyImporter(_BaseImporter): - """ - The standard web2py importer. Like the standard Python importer but it - tries to transform import statements as something like - "import applications.app_name.modules.x". If the import failed, fall back - on _BaseImporter. - """ - - _RE_ESCAPED_PATH_SEP = re.escape(os.path.sep) # os.path.sep escaped for re - - def __init__(self, web2py_path): - """ - @param web2py_path: The absolute path of the web2py installation. - """ - - global DEBUG - self.super_class = super(_Web2pyImporter, self) - self.super_class.__init__() - self.web2py_path = web2py_path - self.__web2py_path_os_path_sep = self.web2py_path+os.path.sep - self.__web2py_path_os_path_sep_len = \ - len(self.__web2py_path_os_path_sep) - self.__RE_APP_DIR = re.compile( - self._RE_ESCAPED_PATH_SEP.join(( \ - #"^" + re.escape(web2py_path),# Not working with Python 2.5 - "^(" + "applications","[^","]+)",""))) - - def _matchAppDir(self, file_path): - """ - Does the file in a directory inside the "applications" directory? - """ - - if file_path.startswith(self.__web2py_path_os_path_sep): - file_path = file_path[self.__web2py_path_os_path_sep_len:] - return self.__RE_APP_DIR.match(file_path) - return False - - def __call__(self, name, globals=None, locals=None, - fromlist=None, level=-1): - """ - The import method itself. - """ - - globals = globals or {} - locals = locals or {} - fromlist = fromlist or [] - - self.begin() - #try: - # if not relative and not from applications: - if not name.startswith(".") and level <= 0 \ - and not name.startswith("applications.") \ - and isinstance(globals, dict): - # Get the name of the file do the import - caller_file_name = os.path.join( - self.web2py_path, globals.get("__file__", "")) - # Is the path in an application directory? - match_app_dir = self._matchAppDir(caller_file_name) - if match_app_dir: - try: - # Get the prefix to add for the import - # (like applications.app_name.modules): - modules_prefix = \ - ".".join((match_app_dir.group(1). \ - replace(os.path.sep, "."), "modules")) - if not fromlist: - # import like "import x" or "import x.y" - return self.__import__dot(modules_prefix, name, - globals, locals, fromlist, level) - else: - # import like "from x import a, b, ..." - return self.super_class \ - .__call__(modules_prefix+"."+name, - globals, locals, fromlist, level) - except ImportError, e: - try: - return self.super_class.__call__(name, globals, locals, - fromlist, level) - except ImportError, e1: - raise e - return self.super_class.__call__(name, globals, locals, - fromlist, level) - - def __import__dot(self, prefix, name, globals, locals, fromlist, - level): - """ - Here we will import x.y.z as many imports like: - from applications.app_name.modules import x - from applications.app_name.modules.x import y - from applications.app_name.modules.x.y import z. - x will be the module returned. - """ - - result = None - for name in name.split("."): - new_mod = super(_Web2pyImporter, self).__call__( - prefix, globals,locals, [name], level) - try: - result = result or new_mod.__dict__[name] - except KeyError, e: - raise ImportError, 'Cannot import module %s' % str(e) - prefix += "." + name - return result - -class _Web2pyDateTrackerImporter(_Web2pyImporter, _DateTrackerImporter): - """ - Like _Web2pyImporter but using a _DateTrackerImporter. - """ - - - - - - - - - +TRACK_IMPORTER = TrackImporter() diff --git a/gluon/main.py b/gluon/main.py index 9ce78eed..426e0272 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -35,8 +35,6 @@ from settings import global_settings from admin import add_path_first, create_missing_folders, create_missing_app_folders from globals import current -from custom_import import custom_import_install - # Remarks: # calling script has inserted path to script directory into sys.path # applications_parent (path to applications/, site-packages/ etc) @@ -55,8 +53,6 @@ from custom_import import custom_import_install web2py_path = global_settings.applications_parent # backward compatibility -custom_import_install(web2py_path) - create_missing_folders() # set up logging for subsequent imports @@ -799,7 +795,6 @@ class HttpServer(object): global_settings.applications_parent = path os.chdir(path) [add_path_first(p) for p in (path, abspath('site-packages'), "")] - custom_import_install(web2py_path) if exists("logging.conf"): logging.config.fileConfig("logging.conf") diff --git a/gluon/shell.py b/gluon/shell.py index d21e8dc8..4a336ba4 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -27,7 +27,6 @@ from globals import Request, Response, Session from storage import Storage from admin import w2p_unpack from dal import BaseAdapter -from custom_import import custom_import_install logger = logging.getLogger("web2py") diff --git a/gluon/winservice.py b/gluon/winservice.py index 27426c8b..3b518214 100644 --- a/gluon/winservice.py +++ b/gluon/winservice.py @@ -94,8 +94,6 @@ class Web2pyService(Service): os.chdir(dir) from gluon.settings import global_settings global_settings.gluon_parent = dir - from gluon.custom_import import custom_import_install - custom_import_install(dir) return True except: self.log("Can't change to web2py working path; server is stopped")