diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 928ba690..bf06d401 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -166,9 +166,9 @@ def check_version(): new_version, version = check_new_version(request.env.web2py_version, WEB2PY_VERSION_URL) - if new_version == -1: + if new_version in (-1, -2): return A(T('Unable to check for upgrades'), _href=WEB2PY_URL) - elif new_version != True: + elif not new_version: return A(T('web2py is up to date'), _href=WEB2PY_URL) elif platform.system().lower() in ('windows', 'win32', 'win64') and os.path.exists("web2py.exe"): return SPAN('You should upgrade to %s' % version.split('(')[0]) @@ -1082,9 +1082,6 @@ def about(): return dict(app=app, about=MARKMIN(about), license=MARKMIN(license), progress=report_progress(app)) -regex_include = r"""(?P\{\{\s*include\s+['"](?P[^'"]*)['"]\s*\}\})""" -regex_extend = r"""^\s*(?P\{\{\s*extend\s+['"](?P[^'"]+)['"]\s*\}\})""" - def design(): """ Application design handler """ app = get_app() @@ -1121,10 +1118,9 @@ def design(): models = listdir(apath('%s/models/' % app, r=request), '.*\.py$') models = [x.replace('\\', '/') for x in models] defines = {} - regex_tables = r"""^[\w]+\.define_table\(\s*['"](?P\w+)['"]""" for m in models: data = safe_read(apath('%s/models/%s' % (app, m), r=request)) - defines[m] = re.findall(regex_tables, data, re.MULTILINE) + defines[m] = re.findall(REGEX_DEFINE_TABLE, data, re.MULTILINE) defines[m].sort() # Get all controllers @@ -1148,12 +1144,12 @@ def design(): include = {} for c in views: data = safe_read(apath('%s/views/%s' % (app, c), r=request)) - items = re.findall(regex_extend, data, re.MULTILINE) + items = re.findall(REGEX_EXTEND, data, re.MULTILINE) if items: extend[c] = items[0][1] - items = re.findall(regex_include, data) + items = re.findall(REGEX_INCLUDE, data) include[c] = [i[1] for i in items] # Get all modules @@ -1289,11 +1285,11 @@ def plugin(): include = {} for c in views: data = safe_read(apath('%s/views/%s' % (app, c), r=request)) - items = re.findall(regex_extend, data, re.MULTILINE) + items = re.findall(REGEX_EXTEND, data, re.MULTILINE) if items: extend[c] = items[0][1] - items = re.findall(regex_include, data) + items = re.findall(REGEX_INCLUDE, data) include[c] = [i[1] for i in items] # Get all modules diff --git a/gluon/admin.py b/gluon/admin.py index 0c09bb69..79c8e7ff 100644 --- a/gluon/admin.py +++ b/gluon/admin.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- +# vim: set ts=4 sw=4 et ai: """ | This file is part of the web2py Web Framework @@ -10,24 +10,34 @@ Utility functions for the Admin application ------------------------------------------- """ +from __future__ import print_function + import os -import sys import traceback -import zipfile from shutil import rmtree, copyfileobj +import zipfile +import sys + from gluon.fileutils import (w2p_pack, create_app, w2p_unpack, w2p_pack_plugin, w2p_unpack_plugin, up, fix_newlines, abspath, recursive_unlink, - read_file, write_file, parse_version) + read_file, write_file, parse_version, missing_app_folders) from gluon.restricted import RestrictedError from gluon.settings import global_settings from gluon.cache import CacheOnDisk from gluon._compat import urlopen, to_native +# TODO: move into add_path_first if not global_settings.web2py_runtime_gae: import site +REGEX_DEFINE_TABLE = r"""^\w+\.define_table\(\s*['"](?P\w+)['"]""" +REGEX_EXTEND = r"""^\s*(?P\{\{\s*extend\s+['"](?P[^'"]+)['"]\s*\}\})""" +REGEX_INCLUDE = r"""(?P\{\{\s*include\s+['"](?P[^'"]+)['"]\s*\}\})""" + + +# TODO: swap arguments, let first ('r' or whatever) be mandatory def apath(path='', r=None): """Builds a path inside an application folder @@ -38,8 +48,9 @@ def apath(path='', r=None): """ opath = up(r.folder) - while path[:3] == '../': - (opath, path) = (up(opath), path[3:]) + while path.startswith('../'): + opath = up(opath) + path = path[3:] return os.path.join(opath, path).replace('\\', '/') @@ -81,7 +92,7 @@ def app_pack_compiled(app, request, raise_ex=False): filename = apath('../deposit/%s.w2p' % app, request) w2p_pack(filename, apath(app, request), compiled=True) return filename - except Exception as e: + except Exception: if raise_ex: raise return None @@ -105,7 +116,7 @@ def app_cleanup(app, request): if os.path.exists(path): for f in os.listdir(path): try: - if f[:1] != '.': + if not f.startswith('.'): os.unlink(os.path.join(path, f)) except IOError: r = False @@ -115,7 +126,7 @@ def app_cleanup(app, request): if os.path.exists(path): for f in os.listdir(path): try: - if f[:1] != '.': + if not f.startswith('.'): recursive_unlink(os.path.join(path, f)) except (OSError, IOError): r = False @@ -126,7 +137,7 @@ def app_cleanup(app, request): CacheOnDisk(folder=path).clear() for f in os.listdir(path): try: - if f[:1] != '.': + if not f.startswith('.'): recursive_unlink(os.path.join(path, f)) except (OSError, IOError): r = False @@ -211,9 +222,9 @@ def app_install(app, fobj, request, filename, overwrite=None): """ did_mkdir = False - if filename[-4:] == '.w2p': + if filename.endswith('.w2p'): extension = 'w2p' - elif filename[-7:] == '.tar.gz': + elif filename.endswith('.tar.gz'): extension = 'tar.gz' else: extension = 'tar' @@ -300,7 +311,8 @@ def plugin_install(app, fobj, request, filename): upname = apath('../deposit/%s' % filename, request) try: - write_file(upname, fobj.read(), 'wb') + with open(upname, 'wb') as appfp: + copyfileobj(fobj, appfp, 4194304) # 4 MB buffer path = apath(app, request) w2p_unpack_plugin(upname, path) fix_newlines(path) @@ -322,7 +334,9 @@ def check_new_version(myversion, version_url): tuple: state, version - state : `True` if upgrade available, `False` if current - version is up-to-date, -1 on error + version is up-to-date, -1 on error, + -2 when the system is likely to be offline (no + internet link available) - version : the most up-to-version available """ @@ -330,10 +344,19 @@ def check_new_version(myversion, version_url): version = to_native(urlopen(version_url).read()) pversion = parse_version(version) pmyversion = parse_version(myversion) - except IOError: - import traceback - print(traceback.format_exc()) - return -1, myversion + except IOError as e: + from socket import gaierror + if isinstance(getattr(e, 'reason', None), gaierror) and \ + e.reason.errno == -2: + # assuming the version_url is ok the socket.gaierror + # (gaierror stands for getaddrinfo() error) that + # originates the exception is probably due to a + # missing internet link (i.e. the system is offline) + print('system is offline, cannot retrieve latest web2py version') + return -2, myversion + else: + print(traceback.format_exc()) + return -1, myversion if pversion[:3]+pversion[-6:] > pmyversion[:3]+pmyversion[-6:]: return True, version @@ -360,7 +383,7 @@ def unzip(filename, dir, subfolder=''): for name in sorted(zf.namelist()): if not name.startswith(subfolder): continue - # print name[n:] + # print(name[n:]) if name.endswith('/'): folder = os.path.join(dir, name[n:]) if not os.path.exists(folder): @@ -421,6 +444,7 @@ def upgrade(request, url='http://web2py.com'): return False, e +# TODO: move to fileutils def add_path_first(path): sys.path = [path] + [p for p in sys.path if ( not p == path and not p == (path + '/'))] @@ -429,21 +453,24 @@ def add_path_first(path): site.addsitedir(path) +# TODO: move to fileutils def try_mkdir(path): if not os.path.exists(path): try: if os.path.islink(path): - # path is a broken link, try to mkdir the target of the link instead of the link itself. + # path is a broken link, try to mkdir the target of the link + # instead of the link itself. os.mkdir(os.path.realpath(path)) else: os.mkdir(path) except OSError as e: - if e.strerror == 'File exists': # In case of race condition. + if e.errno == 17: # "File exists" (race condition). pass else: - raise e + raise +# TODO: move to fileutils def create_missing_folders(): if not global_settings.web2py_runtime_gae: for path in ('applications', 'deposit', 'site-packages', 'logs'): @@ -453,16 +480,16 @@ def create_missing_folders(): paths = (global_settings.gluon_parent, abspath( 'site-packages', gluon=True), abspath('gluon', gluon=True), '') """ - paths = (global_settings.gluon_parent, abspath( - 'site-packages', gluon=True), '') - [add_path_first(p) for p in paths] + for p in (global_settings.gluon_parent, + abspath('site-packages', gluon=True), + ''): + add_path_first(p) +# TODO: move to fileutils def create_missing_app_folders(request): if not global_settings.web2py_runtime_gae: if request.folder not in global_settings.app_folders: - for subfolder in ('models', 'views', 'controllers', 'databases', - 'modules', 'cron', 'errors', 'sessions', - 'languages', 'static', 'private', 'uploads'): - try_mkdir(os.path.join(request.folder, subfolder)) + for amf in missing_app_folders(request.folder): + try_mkdir(amf) global_settings.app_folders.add(request.folder) diff --git a/gluon/fileutils.py b/gluon/fileutils.py index 82eefa4d..72004dab 100644 --- a/gluon/fileutils.py +++ b/gluon/fileutils.py @@ -18,6 +18,7 @@ import time import datetime import logging import shutil + from gluon.http import HTTP from gzip import open as gzopen from gluon.recfile import generate @@ -254,13 +255,11 @@ def w2p_pack(filename, path, compiled=False, filenames=None): os.unlink(tarname) -def create_missing_folders(path): +def missing_app_folders(path): for subfolder in ('models', 'views', 'controllers', 'databases', 'modules', 'cron', 'errors', 'sessions', 'languages', 'static', 'private', 'uploads'): - subpath = os.path.join(path, subfolder) - if not os.path.exists(subpath): - os.mkdir(subpath) + yield os.path.join(path, subfolder) def create_welcome_w2p(): @@ -269,7 +268,9 @@ def create_welcome_w2p(): logger = logging.getLogger("web2py") try: app_path = 'applications/welcome' - create_missing_folders(app_path) + for amf in missing_app_folders(app_path): + if not os.path.exists(amf): + os.mkdir(amf) w2p_pack('welcome.w2p', app_path) logger.info("New installation: created welcome.w2p file") except: