Merge remote-tracking branch 'remotes/oficial/master' into patch-2

gluon/authapi.py

@@ -803,7 +803,7 @@ class AuthAPI(object):
 
         # Finally verify the password
         passfield = settings.password_field
-        password = table_user[passfield].validate(kwargs.get(passfield, ''))[0]
+        password = table_user[passfield].validate(kwargs.get(passfield, ''), None)[0]
 
         if password == user[passfield]:
             self.login_user(user)

gluon/compileapp.py

@@ -565,7 +565,7 @@ def run_models_in(environment):
 
 TEST_CODE = r"""
 def _TEST():
-    import doctest, sys, cStringIO, types, cgi, gluon.fileutils
+    import doctest, sys, cStringIO, types, gluon.fileutils
     if not gluon.fileutils.check_credentials(request):
         raise HTTP(401, web2py_error='invalid credentials')
     stdout = sys.stdout

gluon/console.py

@@ -266,6 +266,12 @@ web2py will attempt to run a GUI to ask for it when starting the web server
         '(default is %(default)s), see -S above. NOTE: when the APP_ENV '
         'argument of -S include a controller c automatic import of '
         'models is always enabled')
+    g.add_argument('--fake_migrate',
+                   default=False,
+                   action='store_true',
+                   help=
+                   'force DAL to fake migrate all tables; '
+                   'monkeypatch in the DAL class to force _fake_migrate=True')
     g.add_argument('--force_migrate', '--force-migrate',
                    default=False,
                    action='store_true', help=

gluon/contrib/login_methods/freeipa_auth.py

@@ -0,0 +1,59 @@
+import sys
+import logging
+
+try:
+    import ldap
+
+    ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+except Exception as detail:
+    logging.error('missing ldap, try "pip install python-ldap"')
+    raise detail
+
+
+def freeipa_auth(server, basedn, group):
+    """
+    custom module for freeIPA auth in web2py
+    server: freeipa ip
+    base_dn: root of ldap tree containing user & groups
+    group: group authing user has to be a member of
+
+   """
+    logger = logging.getLogger("web2py.auth.freeipa_auth")
+
+    def freeipa_auth_aux(username, password):
+        if password == "" or username == "":
+            logger.warning("blank username / password not allowed")
+            return False
+
+        bind_user_base = "uid=" + username + ",cn=users," + basedn
+        ldap_filter = "memberof=cn=" + group + ",cn=groups," + basedn
+
+        session = ldap.initialize("ldaps://" + server + ":636")
+        try:
+            session.bind_s(bind_user_base, password)
+        except ldap.LDAPError:
+            import traceback
+
+            logger.warning("[%s] Error in ldap bind" % str(username))
+            logger.debug(traceback.format_exc())
+            return False
+
+        try:
+            result = session.search_s(
+                bind_user_base, ldap.SCOPE_SUBTREE, ldap_filter, ["member"]
+            )
+            session.unbind()
+        except ldap.LDAPError as detail:
+            logger.warning(
+                "ldap_auth: searc %s for %s resulted in %s: %s\n"
+                % (bind_user_base, ldap_filter, exc_type, exc_value)
+            )
+
+        try:
+            if result == list():
+                return False
+            return True
+        except:
+            return False
+
+    return freeipa_auth_aux

gluon/contrib/login_methods/saml2_auth.py

@@ -150,14 +150,14 @@ class Saml2Auth(object):
         self.config_file = config_file
         self.maps = maps
 
-    # URL for redirecting users to when they sign out
+        # URL for redirecting users to when they sign out
         self.saml_logout_url = logout_url
 
         # URL to let users change their password in the IDP system
         self.saml_change_password_url = change_password_url
     
-    # URL to specify an IDP if using federation metadata or an MDQ
-    self.entityid = entityid
+        # URL to specify an IDP if using federation metadata or an MDQ
+        self.entityid = entityid
 
     def login_url(self, next="/"):
         d = saml2_handler(current.session, current.request, entityid=self.entityid)

gluon/contrib/markmin/markmin2latex.py

@@ -237,6 +237,7 @@ WRAPPER = """
 \\usepackage{graphicx}
 \\usepackage{grffile}
 \\usepackage[utf8x]{inputenc}
+\\usepackage{textgreek}
 \\definecolor{lg}{rgb}{0.9,0.9,0.9}
 \\definecolor{dg}{rgb}{0.3,0.3,0.3}
 \\def\\ft{\\small\\tt}

gluon/contrib/markmin/markmin2pdf.py

@@ -80,7 +80,7 @@ def latex2pdf(latex, pdflatex='pdflatex', passes=3):
             outfile.close()
         re_errors = re.compile('^\!(.*)$', re.M)
         re_warnings = re.compile('^LaTeX Warning\:(.*)$', re.M)
-        flog = open(logname)
+        flog = open(logname,encoding="utf-8")
         try:
             loglines = flog.read()
         finally:

gluon/contrib/redis_cache.py

@@ -208,12 +208,12 @@ class RedisClient(object):
 
     def retry_call(self, key, f, time_expire, with_lock):
         self.RETRIES += 1
-        if self.RETRIES <= self.MAX_RETRIES:
-            logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))
-            time.sleep(2 * self.RETRIES)
+        if self.RETRIES <= self.MAX_RETRIES:            
             if self.fail_gracefully:
                 self.RETRIES = 0
                 return f()
+            logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))            
+            time.sleep(2 * self.RETRIES)
             return self.__call__(key, f, time_expire, with_lock)
         else:
             self.RETRIES = 0

gluon/contrib/redis_session.py

@@ -13,7 +13,7 @@ from gluon import current
 from gluon.storage import Storage
 from gluon.contrib.redis_utils import acquire_lock, release_lock
 from gluon.contrib.redis_utils import register_release_lock
-from gluon._compat import to_bytes
+from gluon._compat import to_native
 
 logger = logging.getLogger("web2py.session.redis")
 
@@ -182,11 +182,11 @@ class MockQuery(object):
             key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db.r_server, key + ':lock', self.value, 2)
-            rtn = self.db.r_server.hgetall(key)
+            rtn = {to_native(k.decode): v for k, v in self.db.r_server.hgetall(key).items()}
             if rtn:
                 if self.unique_key:
                     # make sure the id and unique_key are correct
-                    if rtn['unique_key'] == self.unique_key:
+                    if rtn['unique_key'] == to_native(self.unique_key):
                         rtn['update_record'] = self.update  # update record support
                     else:
                         rtn = None

gluon/contrib/simplejsonrpc.py

@@ -30,7 +30,7 @@ else:
     from io import StringIO
 import random
 import json
-
+from gluon._compat import basestring
 
 class JSONRPCError(RuntimeError):
     "Error object for remote procedure call fail"

gluon/globals.py

@@ -13,7 +13,7 @@ Contains the classes for the global used variables:
 
 """
 from gluon._compat import pickle, StringIO, copyreg, Cookie, urlparse, PY2, iteritems, to_unicode, to_native, \
-    to_bytes, unicodeT, long, hashlib_md5, urllib_quote
+    to_bytes, unicodeT, long, hashlib_md5, urllib_quote, to_native
 from gluon.storage import Storage, List
 from gluon.streamer import streamer, stream_file_or_304_or_206, DEFAULT_CHUNK_SIZE
 from gluon.contenttype import contenttype
@@ -1055,7 +1055,7 @@ class Session(Storage):
             if record_id.isdigit() and long(record_id) > 0:
                 new_unique_key = web2py_uuid()
                 row = table(record_id)
-                if row and row['unique_key'] == unique_key:
+                if row and to_native(row['unique_key']) == to_native(unique_key):
                     table._db(table.id == record_id).update(unique_key=new_unique_key)
                 else:
                     record_id = None
@@ -1231,9 +1231,9 @@ class Session(Storage):
 
         session_pickled = response.session_pickled or pickle.dumps(self, pickle.HIGHEST_PROTOCOL)
 
-        dd = dict(locked=False,
+        dd = dict(locked=0,
                   client_ip=response.session_client,
-                  modified_datetime=request.now,
+                  modified_datetime=request.now.isoformat(),
                   session_data=session_pickled,
                   unique_key=unique_key)
         if record_id:

gluon/languages.py

@@ -16,7 +16,6 @@ import re
 import sys
 import pkgutil
 import logging
-from cgi import escape
 from threading import RLock
 
 from pydal._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin

gluon/rewrite.py

@@ -204,7 +204,7 @@ def url_out(request, environ, application, controller, function,
     if host is True or (host is None and (scheme or port is not None)):
         host = request.env.http_host
     if not scheme or scheme is True:
-        scheme = 'https' if request and request.is_https else 'http'
+        scheme = request.env.get('wsgi_url_scheme', 'http').lower() if request else 'http'
     if host:
         host_port = host if not port else host.split(':', 1)[0] + ':%s' % port
         url = '%s://%s%s' % (scheme, host_port, url)

gluon/scheduler.py

@@ -530,7 +530,7 @@ class IS_CRONLINE(object):
     def __init__(self, error_message=None):
         self.error_message = error_message
 
-    def __call__(self, value):
+    def __call__(self, value, record_id=None):
         recur = CronParser(value, datetime.datetime.now())
         try:
             recur.next()
@@ -550,7 +550,7 @@ class TYPE(object):
         self.myclass = myclass
         self.parse = parse
 
-    def __call__(self, value):
+    def __call__(self, value, record_id=None):
         from gluon import current
         try:
             obj = loads(value)
@@ -596,12 +596,12 @@ class Scheduler(threading.Thread):
         utc_time(bool): do all datetime calculations assuming UTC as the
             timezone. Remember to pass `start_time` and `stop_time` to tasks
             accordingly
-
+        use_spawn(bool): use spawn for subprocess (only useable with python3)
     """
 
     def __init__(self, db, tasks=None, migrate=True,
                  worker_name=None, group_names=None, heartbeat=HEARTBEAT,
-                 max_empty_runs=0, discard_results=False, utc_time=False):
+                 max_empty_runs=0, discard_results=False, utc_time=False, use_spawn=False):
 
         threading.Thread.__init__(self)
         self.setDaemon(True)
@@ -639,6 +639,7 @@ class Scheduler(threading.Thread):
         current._scheduler = self
 
         self.define_tables(db, migrate=migrate)
+        self.use_spawn = use_spawn
 
     def execute(self, task):
         """Start the background process.
@@ -649,10 +650,19 @@ class Scheduler(threading.Thread):
         Returns:
             a `TaskReport` object
         """
-        outq = multiprocessing.Queue()
-        retq = multiprocessing.Queue(maxsize=1)
-        self.process = p = \
-            multiprocessing.Process(target=executor, args=(retq, task, outq))
+        outq = None
+        retq = None
+        if (self.use_spawn and not PY2):
+            ctx = multiprocessing.get_context('spawn')
+            outq = ctx.Queue()
+            retq = ctx.Queue(maxsize=1)
+            sel.process = p = ctx.Process(target=executor, args=(retq, task, outq))
+        else:
+            outq = multiprocessing.Queue()
+            retq = multiprocessing.Queue(maxsize=1)
+            self.process = p = \
+                multiprocessing.Process(target=executor, args=(retq, task, outq))
+
         self.process_queues = (retq, outq)
 
         logger.debug('   task starting')

gluon/serializers.py

@@ -82,7 +82,7 @@ def custom_json(o):
     elif isinstance(o, integer_types):
         return int(o)
     elif isinstance(o, decimal.Decimal):
-        return str(o)
+        return float(o)
     elif isinstance(o, (bytes, bytearray)):
         return str(o)
     elif isinstance(o, lazyT):

gluon/shell.py

@@ -217,7 +217,8 @@ def run(
     python_code=None,
     cron_job=False,
     scheduler_job=False,
-    force_migrate=False):
+    force_migrate=False,
+    fake_migrate=False):
     """
     Start interactive shell or run Python script (startfile) in web2py
     controller environment. appname is formatted like:
@@ -248,6 +249,7 @@ def run(
             fileutils.create_app(adir)
 
     if force_migrate:
+        c = 'appadmin' # Load all models (hack already used for appadmin controller)
         import_models = True
         from gluon.dal import DAL
         orig_init = DAL.__init__
@@ -255,6 +257,7 @@ def run(
         def custom_init(*args, **kwargs):
             kwargs['migrate_enabled'] = True
             kwargs['migrate'] = True
+            kwargs['fake_migrate'] = fake_migrate
             logger.info('Forcing migrate_enabled=True')
             orig_init(*args, **kwargs)
 

gluon/tests/test_serializers.py

@@ -44,7 +44,7 @@ class TestSerializers(unittest.TestCase):
         # self.assertEqual(json(1), json(1))
         # decimal stringified
         obj = {'a': decimal.Decimal('4.312312312312')}
-        self.assertEqual(json(obj), u'{"a": "4.312312312312"}')
+        self.assertEqual(json(obj), u'{"a": 4.312312312312}')
         # lazyT translated
         T = TranslatorFactory('', 'en')
         lazy_translation = T('abc')

gluon/tools.py

@@ -2301,7 +2301,7 @@ class Auth(AuthAPI):
         # in this case they will have to reset their password to login
         if fields.get(settings.passfield):
             fields[settings.passfield] = \
-                settings.table_user[settings.passfield].validate(fields[settings.passfield])[0]
+                settings.table_user[settings.passfield].validate(fields[settings.passfield], None)[0]
         if not fields.get(settings.userfield):
             raise ValueError('register_bare: userfield not provided or invalid')
         user = self.get_or_create_user(fields, login=False, get=False,
@@ -2638,9 +2638,7 @@ class Auth(AuthAPI):
                         # invalid login
                         session.flash = specific_error if self.settings.login_specify_error else self.messages.invalid_login
                         callback(onfail, None)
-                        if 'password' in request.post_vars:
-                            del request.post_vars['password']
-                        redirect(self.url(args=request.args, vars=request.vars),client_side=settings.client_side)
+                        redirect(self.url(args=request.args, vars=request.get_vars),client_side=settings.client_side)
 
             else:  # use a central authentication server
                 cas = settings.login_form
@@ -3173,12 +3171,12 @@ class Auth(AuthAPI):
                         formname='retrieve_password', dbio=False,
                         onvalidation=onvalidation, hideerror=self.settings.hideerror):
             user = table_user(email=form.vars.email)
-            key = user.registration_key
             if not user:
                 current.session.flash = \
                     self.messages.invalid_email
                 redirect(self.url(args=request.args))
-            elif key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
+            key = user.registration_key
+            if key in ('pending', 'disabled', 'blocked') or (key or '').startswith('pending'):
                 current.session.flash = \
                     self.messages.registration_pending
                 redirect(self.url(args=request.args))

gluon/widget.py

@@ -741,7 +741,8 @@ def start():
         sys.argv = [options.run or ''] + options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
             import_models=options.import_models, startfile=options.run,
-            cron_job=options.cron_job, force_migrate=options.force_migrate)
+            cron_job=options.cron_job, force_migrate=options.force_migrate,
+            fake_migrate=options.fake_migrate)
         return
 
     # set size of cron thread pools