more regex improvements, thanks Paolo Pastori

applications/admin/controllers/default.py

@@ -166,9 +166,9 @@ def check_version():
     new_version, version = check_new_version(request.env.web2py_version,
                                              WEB2PY_VERSION_URL)
 
-    if new_version == -1:
+    if new_version in (-1, -2):
         return A(T('Unable to check for upgrades'), _href=WEB2PY_URL)
-    elif new_version != True:
+    elif not new_version:
         return A(T('web2py is up to date'), _href=WEB2PY_URL)
     elif platform.system().lower() in ('windows', 'win32', 'win64') and os.path.exists("web2py.exe"):
         return SPAN('You should upgrade to %s' % version.split('(')[0])
@@ -1082,9 +1082,6 @@ def about():
     return dict(app=app, about=MARKMIN(about), license=MARKMIN(license), progress=report_progress(app))
 
 
-regex_include = r"""(?P<all>\{\{\s*include\s+['"](?P<name>[^'"]*)['"]\s*\}\})"""
-regex_extend = r"""^\s*(?P<all>\{\{\s*extend\s+['"](?P<name>[^'"]+)['"]\s*\}\})"""
-
 def design():
     """ Application design handler """
     app = get_app()
@@ -1121,10 +1118,9 @@ def design():
     models = listdir(apath('%s/models/' % app, r=request), '.*\.py$')
     models = [x.replace('\\', '/') for x in models]
     defines = {}
-    regex_tables = r"""^[\w]+\.define_table\(\s*['"](?P<name>\w+)['"]"""
     for m in models:
         data = safe_read(apath('%s/models/%s' % (app, m), r=request))
-        defines[m] = re.findall(regex_tables, data, re.MULTILINE)
+        defines[m] = re.findall(REGEX_DEFINE_TABLE, data, re.MULTILINE)
         defines[m].sort()
 
     # Get all controllers
@@ -1148,12 +1144,12 @@ def design():
     include = {}
     for c in views:
         data = safe_read(apath('%s/views/%s' % (app, c), r=request))
-        items = re.findall(regex_extend, data, re.MULTILINE)
+        items = re.findall(REGEX_EXTEND, data, re.MULTILINE)
 
         if items:
             extend[c] = items[0][1]
 
-        items = re.findall(regex_include, data)
+        items = re.findall(REGEX_INCLUDE, data)
         include[c] = [i[1] for i in items]
 
     # Get all modules
@@ -1289,11 +1285,11 @@ def plugin():
     include = {}
     for c in views:
         data = safe_read(apath('%s/views/%s' % (app, c), r=request))
-        items = re.findall(regex_extend, data, re.MULTILINE)
+        items = re.findall(REGEX_EXTEND, data, re.MULTILINE)
         if items:
             extend[c] = items[0][1]
 
-        items = re.findall(regex_include, data)
+        items = re.findall(REGEX_INCLUDE, data)
         include[c] = [i[1] for i in items]
 
     # Get all modules

gluon/admin.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+# vim: set ts=4 sw=4 et ai:
 
 """
 | This file is part of the web2py Web Framework
@@ -10,24 +10,34 @@ Utility functions for the Admin application
 -------------------------------------------
 """
 
+from __future__ import print_function
+
 import os
-import sys
 import traceback
-import zipfile
 from shutil import rmtree, copyfileobj
+import zipfile
+import sys
+
 from gluon.fileutils import (w2p_pack, create_app, w2p_unpack,
     w2p_pack_plugin, w2p_unpack_plugin,
     up, fix_newlines, abspath, recursive_unlink,
-    read_file, write_file, parse_version)
+    read_file, write_file, parse_version, missing_app_folders)
 from gluon.restricted import RestrictedError
 from gluon.settings import global_settings
 from gluon.cache import CacheOnDisk
 from gluon._compat import urlopen, to_native
 
+# TODO: move into add_path_first
 if not global_settings.web2py_runtime_gae:
     import site
 
 
+REGEX_DEFINE_TABLE = r"""^\w+\.define_table\(\s*['"](?P<name>\w+)['"]"""
+REGEX_EXTEND = r"""^\s*(?P<all>\{\{\s*extend\s+['"](?P<name>[^'"]+)['"]\s*\}\})"""
+REGEX_INCLUDE = r"""(?P<all>\{\{\s*include\s+['"](?P<name>[^'"]+)['"]\s*\}\})"""
+
+
+# TODO: swap arguments, let first ('r' or whatever) be mandatory 
 def apath(path='', r=None):
     """Builds a path inside an application folder
 
@@ -38,8 +48,9 @@ def apath(path='', r=None):
     """
 
     opath = up(r.folder)
-    while path[:3] == '../':
+    while path.startswith('../'):
-        (opath, path) = (up(opath), path[3:])
+        opath = up(opath)
+        path = path[3:]
     return os.path.join(opath, path).replace('\\', '/')
 
 
@@ -81,7 +92,7 @@ def app_pack_compiled(app, request, raise_ex=False):
         filename = apath('../deposit/%s.w2p' % app, request)
         w2p_pack(filename, apath(app, request), compiled=True)
         return filename
-    except Exception as e:
+    except Exception:
         if raise_ex:
             raise
         return None
@@ -105,7 +116,7 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.':
+                if not f.startswith('.'):
                     os.unlink(os.path.join(path, f))
             except IOError:
                 r = False
@@ -115,7 +126,7 @@ def app_cleanup(app, request):
     if os.path.exists(path):
         for f in os.listdir(path):
             try:
-                if f[:1] != '.':
+                if not f.startswith('.'):
                     recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
@@ -126,7 +137,7 @@ def app_cleanup(app, request):
         CacheOnDisk(folder=path).clear()
         for f in os.listdir(path):
             try:
-                if f[:1] != '.':
+                if not f.startswith('.'):
                     recursive_unlink(os.path.join(path, f))
             except (OSError, IOError):
                 r = False
@@ -211,9 +222,9 @@ def app_install(app, fobj, request, filename, overwrite=None):
 
     """
     did_mkdir = False
-    if filename[-4:] == '.w2p':
+    if filename.endswith('.w2p'):
         extension = 'w2p'
-    elif filename[-7:] == '.tar.gz':
+    elif filename.endswith('.tar.gz'):
         extension = 'tar.gz'
     else:
         extension = 'tar'
@@ -300,7 +311,8 @@ def plugin_install(app, fobj, request, filename):
     upname = apath('../deposit/%s' % filename, request)
 
     try:
-        write_file(upname, fobj.read(), 'wb')
+        with open(upname, 'wb') as appfp:
+            copyfileobj(fobj, appfp, 4194304) # 4 MB buffer
         path = apath(app, request)
         w2p_unpack_plugin(upname, path)
         fix_newlines(path)
@@ -322,7 +334,9 @@ def check_new_version(myversion, version_url):
         tuple: state, version
 
         - state : `True` if upgrade available, `False` if current
-          version is up-to-date, -1 on error
+                  version is up-to-date, -1 on error,
+                  -2 when the system is likely to be offline (no
+                  internet link available)
         - version : the most up-to-version available
 
     """
@@ -330,10 +344,19 @@ def check_new_version(myversion, version_url):
         version = to_native(urlopen(version_url).read())
         pversion = parse_version(version)
         pmyversion = parse_version(myversion)
-    except IOError:
+    except IOError as e:
-        import traceback
+        from socket import gaierror
-        print(traceback.format_exc())
+        if isinstance(getattr(e, 'reason', None), gaierror) and \
-        return -1, myversion
+            e.reason.errno == -2:
+            # assuming the version_url is ok the socket.gaierror
+            # (gaierror stands for getaddrinfo() error) that
+            # originates the exception is probably due to a
+            # missing internet link (i.e. the system is offline)
+            print('system is offline, cannot retrieve latest web2py version')
+            return -2, myversion
+        else:
+            print(traceback.format_exc())
+            return -1, myversion
 
     if pversion[:3]+pversion[-6:] > pmyversion[:3]+pmyversion[-6:]:
         return True, version
@@ -360,7 +383,7 @@ def unzip(filename, dir, subfolder=''):
     for name in sorted(zf.namelist()):
         if not name.startswith(subfolder):
             continue
-        # print name[n:]
+        # print(name[n:])
         if name.endswith('/'):
             folder = os.path.join(dir, name[n:])
             if not os.path.exists(folder):
@@ -421,6 +444,7 @@ def upgrade(request, url='http://web2py.com'):
         return False, e
 
 
+# TODO: move to fileutils
 def add_path_first(path):
     sys.path = [path] + [p for p in sys.path if (
         not p == path and not p == (path + '/'))]
@@ -429,21 +453,24 @@ def add_path_first(path):
             site.addsitedir(path)
 
 
+# TODO: move to fileutils
 def try_mkdir(path):
     if not os.path.exists(path):
         try:
             if os.path.islink(path):
-                # path is a broken link, try to mkdir the target of the link instead of the link itself.
+                # path is a broken link, try to mkdir the target of the link
+                # instead of the link itself.
                 os.mkdir(os.path.realpath(path))
             else:
                 os.mkdir(path)
         except OSError as e:
-            if e.strerror == 'File exists':  # In case of race condition.
+            if e.errno == 17:  # "File exists" (race condition).
                 pass
             else:
-                raise e
+                raise
 
 
+# TODO: move to fileutils
 def create_missing_folders():
     if not global_settings.web2py_runtime_gae:
         for path in ('applications', 'deposit', 'site-packages', 'logs'):
@@ -453,16 +480,16 @@ def create_missing_folders():
     paths = (global_settings.gluon_parent, abspath(
         'site-packages', gluon=True), abspath('gluon', gluon=True), '')
     """
-    paths = (global_settings.gluon_parent, abspath(
+    for p in (global_settings.gluon_parent,
-        'site-packages', gluon=True), '')
+              abspath('site-packages', gluon=True),
-    [add_path_first(p) for p in paths]
+              ''):
+        add_path_first(p)
 
 
+# TODO: move to fileutils
 def create_missing_app_folders(request):
     if not global_settings.web2py_runtime_gae:
         if request.folder not in global_settings.app_folders:
-            for subfolder in ('models', 'views', 'controllers', 'databases',
+            for amf in missing_app_folders(request.folder):
-                              'modules', 'cron', 'errors', 'sessions',
+                try_mkdir(amf)
-                              'languages', 'static', 'private', 'uploads'):
-                try_mkdir(os.path.join(request.folder, subfolder))
             global_settings.app_folders.add(request.folder)

gluon/fileutils.py

@@ -18,6 +18,7 @@ import time
 import datetime
 import logging
 import shutil
+
 from gluon.http import HTTP
 from gzip import open as gzopen
 from gluon.recfile import generate
@@ -254,13 +255,11 @@ def w2p_pack(filename, path, compiled=False, filenames=None):
     os.unlink(tarname)
 
 
-def create_missing_folders(path):
+def missing_app_folders(path):
     for subfolder in ('models', 'views', 'controllers', 'databases',
                       'modules', 'cron', 'errors', 'sessions',
                       'languages', 'static', 'private', 'uploads'):
-        subpath = os.path.join(path, subfolder)
+        yield os.path.join(path, subfolder)
-        if not os.path.exists(subpath):
-            os.mkdir(subpath)
 
 
 def create_welcome_w2p():
@@ -269,7 +268,9 @@ def create_welcome_w2p():
         logger = logging.getLogger("web2py")
         try:
             app_path = 'applications/welcome'
-            create_missing_folders(app_path)
+            for amf in missing_app_folders(app_path):
+                if not os.path.exists(amf):
+                    os.mkdir(amf)
             w2p_pack('welcome.w2p', app_path)
             logger.info("New installation: created welcome.w2p file")
         except: