Merge branch 'master' into migrator-pr

gluon/admin.py

@@ -16,11 +16,12 @@ import os
 import traceback
 from shutil import rmtree, copyfileobj
 import zipfile
+import sys
 
 from gluon.fileutils import (w2p_pack, create_app, w2p_unpack,
-                             w2p_pack_plugin, w2p_unpack_plugin,
-                             up, fix_newlines, abspath, recursive_unlink,
-                             write_file, parse_version)
+    w2p_pack_plugin, w2p_unpack_plugin,
+    up, fix_newlines, abspath, recursive_unlink,
+    read_file, write_file, parse_version, missing_app_folders)
 from gluon.restricted import RestrictedError
 from gluon.settings import global_settings
 from gluon.cache import CacheOnDisk
@@ -28,7 +29,7 @@ from gluon._compat import urlopen, to_native
 
 # TODO: move into add_path_first
 if not global_settings.web2py_runtime_gae:
-    pass
+    import site
 
 
 REGEX_DEFINE_TABLE = r"""^\w+\.define_table\(\s*['"](?P<name>\w+)['"]"""
@@ -441,3 +442,54 @@ def upgrade(request, url='http://web2py.com'):
         return True, None
     except Exception as e:
         return False, e
+
+
+# TODO: move to fileutils
+def add_path_first(path):
+    sys.path = [path] + [p for p in sys.path if (
+        not p == path and not p == (path + '/'))]
+    if not global_settings.web2py_runtime_gae:
+        if not path in sys.path:
+            site.addsitedir(path)
+
+
+# TODO: move to fileutils
+def try_mkdir(path):
+    if not os.path.exists(path):
+        try:
+            if os.path.islink(path):
+                # path is a broken link, try to mkdir the target of the link
+                # instead of the link itself.
+                os.mkdir(os.path.realpath(path))
+            else:
+                os.mkdir(path)
+        except OSError as e:
+            if e.errno == 17:  # "File exists" (race condition).
+                pass
+            else:
+                raise
+
+
+# TODO: move to fileutils
+def create_missing_folders():
+    if not global_settings.web2py_runtime_gae:
+        for path in ('applications', 'deposit', 'site-packages', 'logs'):
+            try_mkdir(abspath(path, gluon=True))
+    """
+    OLD sys.path dance
+    paths = (global_settings.gluon_parent, abspath(
+        'site-packages', gluon=True), abspath('gluon', gluon=True), '')
+    """
+    for p in (global_settings.gluon_parent,
+              abspath('site-packages', gluon=True),
+              ''):
+        add_path_first(p)
+
+
+# TODO: move to fileutils
+def create_missing_app_folders(request):
+    if not global_settings.web2py_runtime_gae:
+        if request.folder not in global_settings.app_folders:
+            for amf in missing_app_folders(request.folder):
+                try_mkdir(amf)
+            global_settings.app_folders.add(request.folder)

gluon/compileapp.py

@@ -40,10 +40,12 @@ from gluon.validators import Validator
 from gluon.settings import global_settings
 from pydal.base import BaseAdapter
 from gluon.custom_import import custom_import_install
-from gluon.fileutils import mktree, listdir, read_file, write_file, abspath, add_path_first
+from gluon.fileutils import mktree, listdir, read_file, write_file, abspath
 from gluon.template import parse_template
 from gluon.cfs import getcfs
 from gluon.restricted import restricted, compile2
+from gluon.admin import add_path_first
+
 
 CACHED_REGEXES = {}
 CACHED_REGEXES_MAX_SIZE = 1000

gluon/console.py

@@ -259,6 +259,12 @@ web2py will attempt to run a GUI to ask for it when starting the web server
         '(default is %(default)s), see -S above. NOTE: when the APP_ENV '
         'argument of -S include a controller c automatic import of '
         'models is always enabled')
+    g.add_argument('--force_migrate',
+                   default=False,
+                   action='store_true',
+                   help=
+                   'force DAL to migrate all tables that should be migrated when enabled; '
+                   'monkeypatch in the DAL class to force _migrate_enabled=True')
     g.add_argument('-R', '--run',
                    type=existing_file,
                    metavar='PYTHON_FILE', help=

gluon/fileutils.py

@@ -11,7 +11,6 @@ File operations
 
 from gluon import storage
 import os
-import sys
 import re
 import tarfile
 import glob
@@ -48,9 +47,6 @@ __all__ = (
     'w2p_pack_plugin',
     'w2p_unpack_plugin',
     'fix_newlines',
-    'create_missing_folders',
-    'create_missing_app_folders',
-    'add_path_first',
 )
 
 
@@ -445,50 +441,3 @@ def abspath(*relpath, **kwargs):
     if kwargs.get('gluon', False):
         return os.path.join(global_settings.gluon_parent, path)
     return os.path.join(global_settings.applications_parent, path)
-
-
-def try_mkdir(path):
-    if not os.path.exists(path):
-        try:
-            if os.path.islink(path):
-                # path is a broken link, try to mkdir the target of the link
-                # instead of the link itself.
-                os.mkdir(os.path.realpath(path))
-            else:
-                os.mkdir(path)
-        except OSError as e:
-            if e.errno == 17:  # "File exists" (race condition).
-                pass
-            else:
-                raise
-
-
-def create_missing_folders():
-    if not global_settings.web2py_runtime_gae:
-        for path in ('applications', 'deposit', 'site-packages', 'logs'):
-            try_mkdir(abspath(path, gluon=True))
-    """
-    OLD sys.path dance
-    paths = (global_settings.gluon_parent, abspath(
-        'site-packages', gluon=True), abspath('gluon', gluon=True), '')
-    """
-    for p in (global_settings.gluon_parent,
-              abspath('site-packages', gluon=True),
-              ''):
-        add_path_first(p)
-
-
-def create_missing_app_folders(request):
-    if not global_settings.web2py_runtime_gae:
-        if request.folder not in global_settings.app_folders:
-            for amf in missing_app_folders(request.folder):
-                try_mkdir(amf)
-            global_settings.app_folders.add(request.folder)
-
-
-def add_path_first(path):
-    sys.path = [path] + [p for p in sys.path if (
-        not p == path and not p == (path + '/'))]
-    if not global_settings.web2py_runtime_gae:
-        if not path in sys.path:
-            site.addsitedir(path)

gluon/main.py

@@ -27,10 +27,10 @@ import string
 from gluon._compat import Cookie, urllib_quote
 # from thread import allocate_lock
 
-from gluon.fileutils import abspath, read_file, write_file, create_missing_folders, create_missing_app_folders, \
-    add_path_first
+from gluon.fileutils import abspath, read_file, write_file
 from gluon.settings import global_settings
 from gluon.utils import web2py_uuid, unlocalised_http_header_date
+from gluon.admin import add_path_first, create_missing_folders, create_missing_app_folders
 from gluon.globals import current
 
 #  Remarks:

gluon/shell.py

@@ -28,7 +28,6 @@ from gluon.restricted import RestrictedError
 from gluon.globals import Request, Response, Session
 from gluon.storage import Storage, List
 from gluon.admin import w2p_unpack
-from gluon.fileutils import create_missing_app_folders
 from pydal.base import BaseAdapter
 from gluon._compat import iteritems, ClassType, PY2
 
@@ -167,9 +166,6 @@ def env(
         path_info = '%s?%s' % (path_info, '&'.join(vars))
     request.env.path_info = path_info
 
-    # Ensure necessary folders are created
-    create_missing_app_folders(request)
-
     # Monkey patch so credentials checks pass.
 
     def check_credentials(request, other_application='admin'):
@@ -220,7 +216,8 @@ def run(
     bpython=False,
     python_code=None,
     cron_job=False,
-    scheduler_job=False):
+    scheduler_job=False,
+    force_migrate=False):
     """
     Start interactive shell or run Python script (startfile) in web2py
     controller environment. appname is formatted like:
@@ -250,6 +247,19 @@ def run(
             os.mkdir(adir)
             fileutils.create_app(adir)
 
+    if force_migrate:
+        import_models = True
+        from gluon.dal import DAL
+        orig_init = DAL.__init__
+
+        def custom_init(*args, **kwargs):
+            kwargs['migrate_enabled'] = True
+            kwargs['migrate'] = True
+            logger.info('Forcing migrate_enabled=True')
+            orig_init(*args, **kwargs)
+
+        DAL.__init__ = custom_init
+
     if c:
         import_models = True
     extra_request = {}
@@ -261,7 +271,6 @@ def run(
         # underscore necessary because request.vars is a property
         extra_request['_vars'] = vars
     _env = env(a, c=c, f=f, import_models=import_models, extra_request=extra_request)
-
     if c:
         pyfile = os.path.join('applications', a, 'controllers', c + '.py')
         pycfile = os.path.join('applications', a, 'compiled',
@@ -303,6 +312,15 @@ def run(
             print(traceback.format_exc())
             if import_models:
                 BaseAdapter.close_all_instances('rollback')
+    elif force_migrate:
+        try:
+            execfile("scripts/migrator.py", _env)
+            if import_models:
+                BaseAdapter.close_all_instances('commit')
+        except:
+            print(traceback.format_exc())
+            if import_models:
+                BaseAdapter.close_all_instances('rollback')
     else:
         if not plain:
             if bpython:

gluon/widget.py

@@ -741,7 +741,7 @@ def start():
         sys.argv = [options.run or ''] + options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
             import_models=options.import_models, startfile=options.run,
-            cron_job=options.cron_job)
+            cron_job=options.cron_job, force_migrate=options.force_migrate)
         return
 
     if options.cron_run:

scripts/migrator.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+'''
+To use, e.g. python .\web2py.py -S APPNAME --force_migrate
+'''
+
+import logging
+
+logger = logging.getLogger("web2py")
+
+
+def get_databases(request):
+    dbs = {}
+    global_env = globals()
+    for (key, value) in global_env.items():
+        try:
+            cond = isinstance(value, GQLDB)
+        except:
+            cond = isinstance(value, SQLDB)
+        if cond:
+            dbs[key] = value
+    return dbs
+
+
+logger.debug('Getting all databases')
+databases = get_databases(None)
+logger.debug('databases = %s', databases)
+for db_name in databases:
+    logger.debug('Migrating %s', db_name)
+    db = databases[db_name]
+    tables = db.tables
+    for table_name in tables:
+        # Force migration of lazy tables
+        logger.debug("Ensuring migration of table '%s'", table_name)
+        db(db[table_name]).isempty()
+    db.commit()