Merge branch 'master' into fix-missing-folders-in-script

gluon/admin.py

@@ -16,12 +16,11 @@ import os
 import traceback
 from shutil import rmtree, copyfileobj
 import zipfile
-import sys
 
 from gluon.fileutils import (w2p_pack, create_app, w2p_unpack,
-    w2p_pack_plugin, w2p_unpack_plugin,
-    up, fix_newlines, abspath, recursive_unlink,
-    read_file, write_file, parse_version, missing_app_folders)
+                             w2p_pack_plugin, w2p_unpack_plugin,
+                             up, fix_newlines, abspath, recursive_unlink,
+                             write_file, parse_version)
 from gluon.restricted import RestrictedError
 from gluon.settings import global_settings
 from gluon.cache import CacheOnDisk
@@ -29,7 +28,7 @@ from gluon._compat import urlopen, to_native
 
 # TODO: move into add_path_first
 if not global_settings.web2py_runtime_gae:
-    import site
+    pass
 
 
 REGEX_DEFINE_TABLE = r"""^\w+\.define_table\(\s*['"](?P<name>\w+)['"]"""
@@ -442,54 +441,3 @@ def upgrade(request, url='http://web2py.com'):
         return True, None
     except Exception as e:
         return False, e
-
-
-# TODO: move to fileutils
-def add_path_first(path):
-    sys.path = [path] + [p for p in sys.path if (
-        not p == path and not p == (path + '/'))]
-    if not global_settings.web2py_runtime_gae:
-        if not path in sys.path:
-            site.addsitedir(path)
-
-
-# TODO: move to fileutils
-def try_mkdir(path):
-    if not os.path.exists(path):
-        try:
-            if os.path.islink(path):
-                # path is a broken link, try to mkdir the target of the link
-                # instead of the link itself.
-                os.mkdir(os.path.realpath(path))
-            else:
-                os.mkdir(path)
-        except OSError as e:
-            if e.errno == 17:  # "File exists" (race condition).
-                pass
-            else:
-                raise
-
-
-# TODO: move to fileutils
-def create_missing_folders():
-    if not global_settings.web2py_runtime_gae:
-        for path in ('applications', 'deposit', 'site-packages', 'logs'):
-            try_mkdir(abspath(path, gluon=True))
-    """
-    OLD sys.path dance
-    paths = (global_settings.gluon_parent, abspath(
-        'site-packages', gluon=True), abspath('gluon', gluon=True), '')
-    """
-    for p in (global_settings.gluon_parent,
-              abspath('site-packages', gluon=True),
-              ''):
-        add_path_first(p)
-
-
-# TODO: move to fileutils
-def create_missing_app_folders(request):
-    if not global_settings.web2py_runtime_gae:
-        if request.folder not in global_settings.app_folders:
-            for amf in missing_app_folders(request.folder):
-                try_mkdir(amf)
-            global_settings.app_folders.add(request.folder)

gluon/compileapp.py

@@ -40,12 +40,10 @@ from gluon.validators import Validator
 from gluon.settings import global_settings
 from pydal.base import BaseAdapter
 from gluon.custom_import import custom_import_install
-from gluon.fileutils import mktree, listdir, read_file, write_file, abspath
+from gluon.fileutils import mktree, listdir, read_file, write_file, abspath, add_path_first
 from gluon.template import parse_template
 from gluon.cfs import getcfs
 from gluon.restricted import restricted, compile2
-from gluon.admin import add_path_first
-
 
 CACHED_REGEXES = {}
 CACHED_REGEXES_MAX_SIZE = 1000

gluon/console.py

@@ -259,12 +259,6 @@ web2py will attempt to run a GUI to ask for it when starting the web server
         '(default is %(default)s), see -S above. NOTE: when the APP_ENV '
         'argument of -S include a controller c automatic import of '
         'models is always enabled')
-    g.add_argument('--force_migrate',
-                   default=False,
-                   action='store_true',
-                   help=
-                   'force DAL to migrate all tables that should be migrated when enabled; '
-                   'monkeypatch in the DAL class to force _migrate_enabled=True')
     g.add_argument('-R', '--run',
                    type=existing_file,
                    metavar='PYTHON_FILE', help=

gluon/fileutils.py

@@ -11,6 +11,7 @@ File operations
 
 from gluon import storage
 import os
+import sys
 import re
 import tarfile
 import glob
@@ -47,6 +48,9 @@ __all__ = (
     'w2p_pack_plugin',
     'w2p_unpack_plugin',
     'fix_newlines',
+    'create_missing_folders',
+    'create_missing_app_folders',
+    'add_path_first',
 )
 
 
@@ -441,3 +445,50 @@ def abspath(*relpath, **kwargs):
     if kwargs.get('gluon', False):
         return os.path.join(global_settings.gluon_parent, path)
     return os.path.join(global_settings.applications_parent, path)
+
+
+def try_mkdir(path):
+    if not os.path.exists(path):
+        try:
+            if os.path.islink(path):
+                # path is a broken link, try to mkdir the target of the link
+                # instead of the link itself.
+                os.mkdir(os.path.realpath(path))
+            else:
+                os.mkdir(path)
+        except OSError as e:
+            if e.errno == 17:  # "File exists" (race condition).
+                pass
+            else:
+                raise
+
+
+def create_missing_folders():
+    if not global_settings.web2py_runtime_gae:
+        for path in ('applications', 'deposit', 'site-packages', 'logs'):
+            try_mkdir(abspath(path, gluon=True))
+    """
+    OLD sys.path dance
+    paths = (global_settings.gluon_parent, abspath(
+        'site-packages', gluon=True), abspath('gluon', gluon=True), '')
+    """
+    for p in (global_settings.gluon_parent,
+              abspath('site-packages', gluon=True),
+              ''):
+        add_path_first(p)
+
+
+def create_missing_app_folders(request):
+    if not global_settings.web2py_runtime_gae:
+        if request.folder not in global_settings.app_folders:
+            for amf in missing_app_folders(request.folder):
+                try_mkdir(amf)
+            global_settings.app_folders.add(request.folder)
+
+
+def add_path_first(path):
+    sys.path = [path] + [p for p in sys.path if (
+        not p == path and not p == (path + '/'))]
+    if not global_settings.web2py_runtime_gae:
+        if not path in sys.path:
+            site.addsitedir(path)

gluon/main.py

@@ -27,10 +27,10 @@ import string
 from gluon._compat import Cookie, urllib_quote
 # from thread import allocate_lock
 
-from gluon.fileutils import abspath, read_file, write_file
+from gluon.fileutils import abspath, read_file, write_file, create_missing_folders, create_missing_app_folders, \
+    add_path_first
 from gluon.settings import global_settings
 from gluon.utils import web2py_uuid, unlocalised_http_header_date
-from gluon.admin import add_path_first, create_missing_folders, create_missing_app_folders
 from gluon.globals import current
 
 #  Remarks:

gluon/shell.py

@@ -28,6 +28,7 @@ from gluon.restricted import RestrictedError
 from gluon.globals import Request, Response, Session
 from gluon.storage import Storage, List
 from gluon.admin import w2p_unpack
+from gluon.fileutils import create_missing_app_folders
 from pydal.base import BaseAdapter
 from gluon._compat import iteritems, ClassType, PY2
 
@@ -166,6 +167,9 @@ def env(
         path_info = '%s?%s' % (path_info, '&'.join(vars))
     request.env.path_info = path_info
 
+    # Ensure necessary folders are created
+    create_missing_app_folders(request)
+
     # Monkey patch so credentials checks pass.
 
     def check_credentials(request, other_application='admin'):
@@ -216,8 +220,7 @@ def run(
     bpython=False,
     python_code=None,
     cron_job=False,
-    scheduler_job=False,
-    force_migrate=False):
+    scheduler_job=False):
     """
     Start interactive shell or run Python script (startfile) in web2py
     controller environment. appname is formatted like:
@@ -247,19 +250,6 @@ def run(
             os.mkdir(adir)
             fileutils.create_app(adir)
 
-    if force_migrate:
-        import_models = True
-        from gluon.dal import DAL
-        orig_init = DAL.__init__
-
-        def custom_init(*args, **kwargs):
-            kwargs['migrate_enabled'] = True
-            kwargs['migrate'] = True
-            logger.info('Forcing migrate_enabled=True')
-            orig_init(*args, **kwargs)
-
-        DAL.__init__ = custom_init
-
     if c:
         import_models = True
     extra_request = {}
@@ -271,6 +261,7 @@ def run(
         # underscore necessary because request.vars is a property
         extra_request['_vars'] = vars
     _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request)
+
     if c:
         pyfile = os.path.join('applications', a, 'controllers', c + '.py')
         pycfile = os.path.join('applications', a, 'compiled',
@@ -312,15 +303,6 @@ def run(
             print(traceback.format_exc())
             if import_models:
                 BaseAdapter.close_all_instances('rollback')
-    elif force_migrate:
-        try:
-            execfile("scripts/migrator.py", _env)
-            if import_models:
-                BaseAdapter.close_all_instances('commit')
-        except:
-            print(traceback.format_exc())
-            if import_models:
-                BaseAdapter.close_all_instances('rollback')
     else:
         if not plain:
             if bpython:

gluon/widget.py

@@ -741,7 +741,7 @@ def start():
         sys.argv = [options.run or ''] + options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
             import_models=options.import_models, startfile=options.run,
-            cron_job=options.cron_job, force_migrate=options.force_migrate)
+            cron_job=options.cron_job)
         return
 
     if options.cron_run:

scripts/migrator.py

@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-'''
-To use, e.g. python .\web2py.py -S APPNAME --force_migrate
-'''
-
-import logging
-
-logger = logging.getLogger("web2py")
-
-
-def get_databases(request):
-    dbs = {}
-    global_env = globals()
-    for (key, value) in global_env.items():
-        try:
-            cond = isinstance(value, GQLDB)
-        except:
-            cond = isinstance(value, SQLDB)
-        if cond:
-            dbs[key] = value
-    return dbs
-
-
-logger.debug('Getting all databases')
-databases = get_databases(None)
-logger.debug('databases = %s', databases)
-for db_name in databases:
-    logger.debug('Migrating %s', db_name)
-    db = databases[db_name]
-    tables = db.tables
-    for table_name in tables:
-        # Force migration of lazy tables
-        logger.debug("Ensuring migration of table '%s'", table_name)
-        db(db[table_name]).isempty()
-    db.commit()