much better custom_import, smaller, faster, and finally works with shell

VERSION

@@ -1 +1 @@
-Version 2.1.0 (2012-10-11 16:48:00) dev
+Version 2.1.0 (2012-10-11 22:03:15) dev

gluon/compileapp.py

@@ -40,6 +40,7 @@ import imp
 import logging
 logger = logging.getLogger("web2py")
 import rewrite
+from custom_import import custom_import_install
 
 try:
     import py_compile
@@ -411,6 +412,7 @@ def build_environment(request, response, session, store_current=True):
        local_import_aux(name,reload,app)
     BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
     response._view_environment = copy.copy(environment)
+    custom_import_install()
     return environment
 
 def save_pyc(filename):

gluon/custom_import.py

@@ -6,149 +6,119 @@ import os
 import re
 import sys
 import threading
+import traceback
+from gluon import current
 
-# Install the new import function:
+NAIVE_IMPORTER = __builtin__.__import__
-def custom_import_install(web2py_path):
+TRACK_CHANGES = False
-    global _web2py_importer
+INVALID_MODULES = set(sys.modules.keys()).union(('','gluon','applications','custom_import'))
-    global _web2py_path
-    if isinstance(__builtin__.__import__, _Web2pyImporter):
-        return #aready installed
-    _web2py_path = web2py_path
-    _web2py_importer = _Web2pyImporter(web2py_path)
-    __builtin__.__import__ = _web2py_importer
 
-def is_tracking_changes():
+# backward compatibility API 
-    """
+def custom_import_install():
-    @return: True: neo_importer is tracking changes made to Python source
+    __builtin__.__import__ = custom_importer
-    files. False: neo_import does not reload Python modules.
-    """
-    global _is_tracking_changes
-    return _is_tracking_changes
 
 def track_changes(track=True):
-    """
+    assert track in (True,False), "must be True or False"
-    Tell neo_importer to start/stop tracking changes made to Python modules.
+    global TRACK_CHANGES
-    @param track: True: Start tracking changes. False: Stop tracking changes.
+    TRACK_CHANGES = track
-    """
 
-    global _is_tracking_changes
+def is_tracking_changes():    
-    global _web2py_importer
+    return TRACK_CHANGES
-    global _web2py_date_tracker_importer
-    assert track is True or track is False, "Boolean expected."
-    if track == _is_tracking_changes:
-        return
-    if track:
-        if not _web2py_date_tracker_importer:
-            _web2py_date_tracker_importer = \
-                _Web2pyDateTrackerImporter(_web2py_path)
-        __builtin__.__import__ = _web2py_date_tracker_importer
-    else:
-        __builtin__.__import__ = _web2py_importer
-    _is_tracking_changes = track
 
-_STANDARD_PYTHON_IMPORTER = __builtin__.__import__ # Keep standard importer
+def custom_importer(name, globals=None, locals=None, fromlist=None, level=-1):
-_web2py_importer = None # The standard web2py importer
-_web2py_date_tracker_importer = None # The web2py importer with date tracking
-_web2py_path = None # Absolute path of the web2py directory
-
-_is_tracking_changes = False # The tracking mode
-
-class _BaseImporter(object):
     """
-    The base importer. Dispatch the import the call to the standard Python
+    The web2py custom importer. Like the standard Python importer but it
-    importer.
+    tries to transform import statements as something like
-    """
+    "import applications.app_name.modules.x". 
-
+    If the import failed, fall back on naive_importer
-    def begin(self):
-        """
-        Many imports can be made for a single import statement. This method
-        help the management of this aspect.
-        """
-
-    def __init__(self):
-        self._STANDARD_PYTHON_IMPORTER = _STANDARD_PYTHON_IMPORTER
-    def __call__(self, name, globals=None, locals=None,
-                 fromlist=None, level=-1):
-        """
-        The import method itself.
-        """
-        return self._STANDARD_PYTHON_IMPORTER(
-            name,globals,locals,fromlist,level)
-            
-    def end(self):
-        """
-        Needed for clean up.
-        """
-
-
-class _DateTrackerImporter(_BaseImporter):
-    """
-    An importer tracking the date of the module files and reloading them when
-    they have changed.
-    """
-
-    _PACKAGE_PATH_SUFFIX = os.path.sep+"__init__.py"
-
-    def __init__(self):
-        super(_DateTrackerImporter, self).__init__()
-        self._import_dates = {} # Import dates of the files of the modules
-        # Avoid reloading cause by file modifications of reload:
-        self._tl = threading.local()
-        self._tl._modules_loaded = None
-
-    def begin(self):
-        self._tl._modules_loaded = set()
-
-    def __call__(self, name, globals=None, locals=None,
-                 fromlist=None, level=-1):
-        """
-        The import method itself.
     """
 
     globals = globals or {}
     locals = locals or {}
     fromlist = fromlist or []
 
-        call_begin_end = self._tl._modules_loaded is None
+    base_importer = TRACK_IMPORTER if TRACK_CHANGES else NAIVE_IMPORTER
-        if call_begin_end:
+
-            self.begin()
+    # if not relative and not from applications:
+    if hasattr(current,'request') \
+            and level<=0 \
+            and not name.split('.')[0] in INVALID_MODULES \
+            and isinstance(globals, dict):            
         try:
-            self._tl.globals = globals
+            items = current.request.folder.split(os.path.sep)
-            self._tl.locals = locals
+            if not items[-1]: items = items[:-1]
-            self._tl.level = level
+            modules_prefix = '.'.join(items[-2:])+'.modules'
-
+            if not fromlist:
-            # Check the date and reload if needed:
+                # import like "import x" or "import x.y"
-            self._update_dates(name, fromlist)
+                result = None
-
+                for itemname in name.split("."):
-            # Try to load the module and update the dates if it works:
+                    new_mod = base_importer(
-            result = super(_DateTrackerImporter, self) \
+                        modules_prefix, globals,locals, [itemname], level)
-              .__call__(name, globals, locals, fromlist, level)
+                    try:
-            # Module maybe loaded for the 1st time so we need to set the date
+                        result = result or new_mod.__dict__[itemname]
-            self._update_dates(name, fromlist)
+                    except KeyError, e:
+                        raise ImportError, 'Cannot import module %s' % str(e)
+                    modules_prefix += "." + itemname
                 return result
-        except Exception:
+            else:
-            raise # Don't hide something that went wrong
+                # import like "from x import a, b, ..."
-        finally:
+                pname = modules_prefix + "." + name
-            if call_begin_end:
+                return base_importer(pname, globals, locals, fromlist, level)
-                self.end()
+        except ImportError, e1:
+            pass # the module does not exist
+        except Exception, e2:
+            raise e2 # there is an error in the module
+    return NAIVE_IMPORTER(name,globals,locals,fromlist,level)
 
-    def _update_dates(self, name, fromlist):
+
+class TrackImporter(object):
+    """
+    An importer tracking the date of the module files and reloading them when
+    they have changed.
+    """
+
+    THREAD_LOCAL = threading.local()
+    PACKAGE_PATH_SUFFIX = os.path.sep+"__init__.py"
+
+    def __init__(self):
+        self._import_dates = {} # Import dates of the files of the modules
+
+    def __call__(self,name,globals=None,locals=None,fromlist=None,level=-1):
+        """
+        The import method itself.
+        """
+        globals = globals or {}
+        locals = locals or {}
+        fromlist = fromlist or []
+        if not hasattr(self.THREAD_LOCAL,'_modules_loaded'):
+            self.THREAD_LOCAL._modules_loaded = set()
+        try:
+            # Check the date and reload if needed:
+            self._update_dates(name, globals, locals, fromlist, level)
+            # Try to load the module and update the dates if it works:
+            result = NAIVE_IMPORTER(name, globals, locals, fromlist, level)
+            # Module maybe loaded for the 1st time so we need to set the date
+            self._update_dates(name, globals, locals, fromlist, level)
+            return result
+        except Exception, e:
+            raise # Don't hide something that went wrong
+
+    def _update_dates(self, name, globals, locals, fromlist, level):
         """
         Update all the dates associated to the statement import. A single
         import statement may import many modules.
         """
 
-        self._reload_check(name)
+        self._reload_check(name, globals, locals, level)
-        if fromlist:
+        for fromlist_name in fromlist or []:
-            for fromlist_name in fromlist:
+            pname = "%s.%s" % (name, fromlist_name)
-                self._reload_check("%s.%s" % (name, fromlist_name))
+            self._reload_check(pname, globals, locals, level)
 
-    def _reload_check(self, name):
+    def _reload_check(self, name, globals, locals, level):
         """
         Update the date associated to the module and reload the module if
         the file has changed.
         """
-
         module = sys.modules.get(name)
         file = self._get_module_file(module)
         if file:
@@ -166,7 +136,7 @@ class _DateTrackerImporter(_BaseImporter):
                     # Get path without file ext:
                     file = os.path.splitext(file)[0]
                     reload_mod = os.path.isdir(file) \
-                      and os.path.isfile(file+self._PACKAGE_PATH_SUFFIX)
+                      and os.path.isfile(file+self.PACKAGE_PATH_SUFFIX)
                     mod_to_pack = reload_mod
                 else: # Package turning into module?
                     file += ".py"
@@ -176,152 +146,27 @@ class _DateTrackerImporter(_BaseImporter):
             if reload_mod or not date or new_date > date:
                 self._import_dates[file] = new_date
             if reload_mod or (date and new_date > date):
-                if module not in self._tl._modules_loaded:
+                if module not in self.THREAD_LOCAL._modules_loaded:
                     if mod_to_pack:
                         # Module turning into a package:
                         mod_name = module.__name__
                         del sys.modules[mod_name] # Delete the module
                         # Reload the module:
-                        super(_DateTrackerImporter, self).__call__ \
+                        NAIVE_IMPORTER(mod_name, globals, locals, [], level)
-                          (mod_name, self._tl.globals, self._tl.locals, [],
-                           self._tl.level)
                     else:
                         reload(module)
-                        self._tl._modules_loaded.add(module)
+                        self.THREAD_LOCAL._modules_loaded.add(module)
 
-    def end(self):
+    def _get_module_file(self, module):
-        self._tl._modules_loaded = None
-
-    @classmethod
-    def _get_module_file(cls, module):
         """
         Get the absolute path file associated to the module or None.
         """
-
         file = getattr(module, "__file__", None)
         if file:
             # Make path absolute if not:
-            #file = os.path.join(cls.web2py_path, file)
-
             file = os.path.splitext(file)[0]+".py" # Change .pyc for .py
-            if file.endswith(cls._PACKAGE_PATH_SUFFIX):
+            if file.endswith(self.PACKAGE_PATH_SUFFIX):
                 file = os.path.dirname(file)  # Track dir for packages
         return file
 
-class _Web2pyImporter(_BaseImporter):
+TRACK_IMPORTER = TrackImporter()
-    """
-    The standard web2py importer. Like the standard Python importer but it
-    tries to transform import statements as something like
-    "import applications.app_name.modules.x". If the import failed, fall back
-    on _BaseImporter.
-    """
-
-    _RE_ESCAPED_PATH_SEP = re.escape(os.path.sep)  # os.path.sep escaped for re
-
-    def __init__(self, web2py_path):
-        """
-        @param web2py_path: The absolute path of the web2py installation.
-        """
-
-        global DEBUG
-        self.super_class = super(_Web2pyImporter, self)
-        self.super_class.__init__()
-        self.web2py_path =  web2py_path
-        self.__web2py_path_os_path_sep = self.web2py_path+os.path.sep
-        self.__web2py_path_os_path_sep_len = \
-            len(self.__web2py_path_os_path_sep)
-        self.__RE_APP_DIR = re.compile(
-            self._RE_ESCAPED_PATH_SEP.join(( \
-                    #"^" + re.escape(web2py_path),# Not working with Python 2.5
-                    "^(" + "applications","[^","]+)","")))
-
-    def _matchAppDir(self, file_path):
-        """
-        Does the file in a directory inside the "applications" directory?
-        """
-
-        if file_path.startswith(self.__web2py_path_os_path_sep):
-            file_path = file_path[self.__web2py_path_os_path_sep_len:]
-            return self.__RE_APP_DIR.match(file_path)
-        return False
-
-    def __call__(self, name, globals=None, locals=None,
-                 fromlist=None, level=-1):
-        """
-        The import method itself.
-        """
-
-        globals = globals or {}
-        locals = locals or {}
-        fromlist = fromlist or []
-
-        self.begin()
-        #try:
-        # if not relative and not from applications:
-        if not name.startswith(".") and level <= 0 \
-                    and not name.startswith("applications.") \
-                    and isinstance(globals, dict):
-            # Get the name of the file do the import
-            caller_file_name = os.path.join(
-                self.web2py_path, globals.get("__file__", ""))
-            # Is the path in an application directory?
-            match_app_dir = self._matchAppDir(caller_file_name)
-            if match_app_dir:
-                try:
-                    # Get the prefix to add for the import
-                    # (like applications.app_name.modules):
-                    modules_prefix = \
-                        ".".join((match_app_dir.group(1). \
-                        replace(os.path.sep, "."), "modules"))
-                    if not fromlist:
-                        # import like "import x" or "import x.y"
-                        return self.__import__dot(modules_prefix, name,
-                            globals, locals, fromlist, level)
-                    else:
-                        # import like "from x import a, b, ..."
-                        return self.super_class \
-                            .__call__(modules_prefix+"."+name,
-                                    globals, locals, fromlist, level)
-                except ImportError, e:
-                    try:
-                        return self.super_class.__call__(name, globals, locals,
-                                                         fromlist, level)
-                    except ImportError, e1:
-                        raise e
-        return self.super_class.__call__(name, globals, locals,
-                                         fromlist, level)
-
-    def __import__dot(self, prefix, name, globals, locals, fromlist,
-                      level):
-        """
-        Here we will import x.y.z as many imports like:
-        from applications.app_name.modules import x
-        from applications.app_name.modules.x import y
-        from applications.app_name.modules.x.y import z.
-        x will be the module returned.
-        """
-
-        result = None
-        for name in name.split("."):
-            new_mod = super(_Web2pyImporter, self).__call__(
-                prefix, globals,locals, [name], level)
-            try:
-                result = result or new_mod.__dict__[name]
-            except KeyError, e:
-                raise ImportError, 'Cannot import module %s' % str(e)
-            prefix += "." + name
-        return result
-
-class _Web2pyDateTrackerImporter(_Web2pyImporter, _DateTrackerImporter):
-    """
-    Like _Web2pyImporter but using a _DateTrackerImporter.
-    """
-
-
-
-
-
-
-
-
-

gluon/main.py

@@ -35,8 +35,6 @@ from settings import global_settings
 from admin import add_path_first, create_missing_folders, create_missing_app_folders
 from globals import current
 
-from custom_import import custom_import_install
-
 #  Remarks:
 #  calling script has inserted path to script directory into sys.path
 #  applications_parent (path to applications/, site-packages/ etc)
@@ -55,8 +53,6 @@ from custom_import import custom_import_install
 
 web2py_path = global_settings.applications_parent # backward compatibility
 
-custom_import_install(web2py_path)
-
 create_missing_folders()
 
 # set up logging for subsequent imports
@@ -799,7 +795,6 @@ class HttpServer(object):
             global_settings.applications_parent = path
             os.chdir(path)
             [add_path_first(p) for p in (path, abspath('site-packages'), "")]
-            custom_import_install(web2py_path)
             if exists("logging.conf"):
                 logging.config.fileConfig("logging.conf")
 

gluon/shell.py

@@ -27,7 +27,6 @@ from globals import Request, Response, Session
 from storage import Storage
 from admin import w2p_unpack
 from dal import BaseAdapter
-from custom_import import custom_import_install
 
 logger = logging.getLogger("web2py")
 

gluon/winservice.py

@@ -94,8 +94,6 @@ class Web2pyService(Service):
             os.chdir(dir)
             from gluon.settings import global_settings
             global_settings.gluon_parent = dir
-            from gluon.custom_import import custom_import_install
-            custom_import_install(dir)
             return True
         except:
             self.log("Can't change to web2py working path; server is stopped")