fixed markmin test, change beahvior

CHANGELOG

@@ -30,7 +30,7 @@
   improved Teradata support
   improved mongodb support
   overall refactoring
-  experimental support for Google Cloud SQL v2 (TODO)
+  experimental support for Google Cloud SQL v2
 
 ## 2.15.x
 - web2py does not support python 2.6 anymore

Makefile

@@ -32,7 +32,7 @@ update:
 	echo "remember that pymysql was tweaked"
 src:
 	### Use semantic versioning
-	echo 'Version 2.14.6-stable+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
+	echo 'Version 2.16.0-beta2+timestamp.'`date +%Y.%m.%d.%H.%M.%S` > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
@@ -54,7 +54,7 @@ src:
 	### build web2py_src.zip
 	echo '' > NEWINSTALL
 	mv web2py_src.zip web2py_src_old.zip | echo 'no old'
-	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
+	cd ..; zip -r web2py/web2py_src.zip web2py/web2py.py web2py/anyserver.py web2py/fabfile.py web2py/gluon/* web2py/extras/* web2py/handlers/* web2py/examples/* web2py/README.markdown  web2py/LICENSE web2py/CHANGELOG web2py/NEWINSTALL web2py/VERSION web2py/MANIFEST.in web2py/scripts/*.sh web2py/scripts/*.py web2py/applications/admin web2py/applications/examples/ web2py/applications/welcome web2py/applications/__init__.py web2py/site-packages/__init__.py web2py/gluon/tests/*.sh web2py/gluon/tests/*.py
 
 mdp:
 	make src

VERSION

@@ -1 +1 @@
-Version 2.14.6-stable+timestamp.2016.05.09.19.18.48
+Version 2.16.0-beta2+timestamp.2017.07.10.03.14.16

applications/admin/controllers/appadmin.py

@@ -690,6 +690,6 @@ def d3_graph_model():
         nodes.append(dict(name=tablename, type="table", fields = fields))
 
     # d3 v4 allows individual modules to be specified.  The complete d3 library is included below.
-    response.files.append(URL('static','js/d3.min.js'))
+    response.files.append(URL('admin','static','js/d3.min.js'))
-    response.files.append(URL('static','js/d3_graph.js'))
+    response.files.append(URL('admin','static','js/d3_graph.js'))
     return dict(databases=databases, nodes=nodes, links=links)

applications/welcome/routes.example.py

@@ -20,8 +20,8 @@
 # YOU CAN COPY THIS FILE TO ANY APPLICATION'S ROOT DIRECTORY WITHOUT CHANGES!
 # ----------------------------------------------------------------------------------------------------------------------
 
-from fileutils import abspath
+from gluon.fileutils import abspath
-from languages import read_possible_languages
+from gluon.languages import read_possible_languages
 
 possible_languages = read_possible_languages(abspath('applications', app))
 # ----------------------------------------------------------------------------------------------------------------------

gluon/contrib/markmin/markmin2html.py

@@ -5,12 +5,19 @@
 # license MIT/BSD/GPL
 from __future__ import print_function
 import re
+import sys
 import urllib
-from gluon._compat import maketrans, urllib_quote, unicodeT, to_bytes, to_native, xrange
-from gluon.utils import local_html_escape as escape
-from ast import parse as ast_parse
 import ast
 
+PY2 = sys.version_info[0] == 2
+
+if PY2:
+    from urllib import quote as urllib_quote
+    from string import maketrans
+else:
+    from urllib.parse import quote as urllib_quote
+    maketrans = str.maketrans
+
 
 """
 TODO: next version should use MathJax
@@ -564,6 +571,29 @@ ttab_in = maketrans("'`:*~\\[]{}@$+-.#\n", '\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14
 ttab_out = maketrans('\x0b\x0c\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x05', "'`:*~\\[]{}@$+-.#\n")
 regex_quote = re.compile('(?P<name>\w+?)\s*\=\s*')
 
+def local_html_escape(data, quote=False):
+    """
+    Works with bytes.
+    Replace special characters "&", "<" and ">" to HTML-safe sequences.
+    If the optional flag quote is true (the default), the quotation mark
+    characters, both double quote (") and single quote (') characters are also
+    translated.
+    """
+    if PY2:
+        import cgi
+        data = cgi.escape(data, quote)
+        return data.replace("'", "&#x27;") if quote else data
+    else:
+        import html
+        if isinstance(data, str):
+            return html.escape(data, quote=quote)
+        data = data.replace(b"&", b"&amp;")  # Must be done first!                                                                                           
+        data = data.replace(b"<", b"&lt;")
+        data = data.replace(b">", b"&gt;")
+        if quote:
+            data = data.replace(b'"', b"&quot;")
+            data = data.replace(b'\'', b"&#x27;")
+        return data
 
 def make_dict(b):
     return '{%s}' % regex_quote.sub("'\g<name>':", b)
@@ -579,7 +609,7 @@ def safe_eval(node_or_string, env):
     _safe_names = {'None': None, 'True': True, 'False': False}
     _safe_names.update(env)
     if isinstance(node_or_string, basestring):
-        node_or_string = ast_parse(node_or_string, mode='eval')
+        node_or_string = ast.parse(node_or_string, mode='eval')
     if isinstance(node_or_string, ast.Expression):
         node_or_string = node_or_string.body
 
@@ -599,14 +629,14 @@ def safe_eval(node_or_string, env):
             if node.id in _safe_names:
                 return _safe_names[node.id]
         elif isinstance(node, ast.BinOp) and \
-                isinstance(node.op, (Add, Sub)) and \
+                isinstance(node.op, (ast.Add, ast.Sub)) and \
-                isinstance(node.right, Num) and \
+                isinstance(node.right, ast.Num) and \
                 isinstance(node.right.n, complex) and \
-                isinstance(node.left, Num) and \
+                isinstance(node.left, ast.Num) and \
                 isinstance(node.left.n, (int, long, float)):
             left = node.left.n
             right = node.right.n
-            if isinstance(node.op, Add):
+            if isinstance(node.op, ast.Add):
                 return left + right
             else:
                 return left - right
@@ -765,7 +795,7 @@ def render(text,
     '<table><tbody><tr class="first"><td>a</td><td>b</td></tr><tr class="even"><td>c</td><td>d</td></tr></tbody></table>'
 
     >>> render("----\\nhello world\\n----\\n")
-    '<blockquote>hello world</blockquote>'
+    '<blockquote><p>hello world</p></blockquote>'
 
     >>> render('[[myanchor]]')
     '<p><span class="anchor" id="markmin_myanchor"></span></p>'
@@ -946,7 +976,8 @@ def render(text,
     if protolinks == "default":
         protolinks = protolinks_simple
     pp = '\n' if pretty_print else ''
-    text = to_native(text)
+    text = text if text is None or isinstance(text, str) else text.decode('utf8', 'strict')
+
     if not (isinstance(text, str)):
         text = str(text or '')
     text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)
@@ -994,7 +1025,7 @@ def render(text,
         return LINK
 
     text = regex_link.sub(mark_link, text)
-    text = escape(text)
+    text = local_html_escape(text)
 
     if protolinks:
         text = regex_proto.sub(lambda m: protolinks(*m.group('p', 'k')), text)
@@ -1035,7 +1066,7 @@ def render(text,
             if pend and mtag == '.':  # paragraph in a list:
                 out.append(etags.pop())
                 ltags.pop()
-            for i in xrange(lent - lev):
+            for i in range(lent - lev):
                 out.append('<' + tag + '>' + pp)
                 etags.append('</' + tag + '>' + pp)
                 lev += 1
@@ -1044,7 +1075,7 @@ def render(text,
         elif lent == lev:
             if tlev[-1] != tag:
                 # type of list is changed (ul<=>ol):
-                for i in xrange(ltags.count(lent)):
+                for i in range(ltags.count(lent)):
                     ltags.pop()
                     out.append(etags.pop())
                 tlev[-1] = tag
@@ -1209,7 +1240,7 @@ def render(text,
                 s = '<blockquote%s%s>%s</blockquote>%s' \
                     % (t_cls,
                        t_id,
-                       '\n'.join(strings[bq_begin:lineno]), pp)
+                       render('\n'.join(strings[bq_begin:lineno])), pp)
                 mtag = 'q'
         else:
             s = '<hr />'
@@ -1322,10 +1353,10 @@ def render(text,
         t, a, k, p, w = m.group('t', 'a', 'k', 'p', 'w')
         if not k:
             return m.group(0)
-        k = escape(k)
+        k = local_html_escape(k)
         t = t or ''
         style = 'width:%s' % w if w else ''
-        title = ' title="%s"' % escape(a).replace(META, DISABLED_META) if a else ''
+        title = ' title="%s"' % local_html_escape(a).replace(META, DISABLED_META) if a else ''
         p_begin = p_end = ''
         if p == 'center':
             p_begin = '<p style="text-align:center">'
@@ -1349,7 +1380,7 @@ def render(text,
                        autolinks, protolinks, class_prefix, id_prefix, pretty_print)
             return '<%(p)s controls="controls"%(title)s%(style)s><source src="%(k)s" />%(t)s</%(p)s>' \
                    % dict(p=p, title=title, style=style, k=k, t=t)
-        alt = ' alt="%s"' % escape(t).replace(META, DISABLED_META) if t else ''
+        alt = ' alt="%s"' % local_html_escape(t).replace(META, DISABLED_META) if t else ''
         return '%(begin)s<img src="%(k)s"%(alt)s%(title)s%(style)s />%(end)s' \
                % dict(begin=p_begin, k=k, alt=alt, title=title, style=style, end=p_end)
 
@@ -1358,12 +1389,12 @@ def render(text,
         if not k and not t:
             return m.group(0)
         t = t or ''
-        a = escape(a) if a else ''
+        a = local_html_escape(a) if a else ''
         if k:
             if '#' in k and ':' not in k.split('#')[0]:
                 # wikipage, not external url
                 k = k.replace('#', '#' + id_prefix)
-            k = escape(k)
+            k = local_html_escape(k)
             title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
             target = ' target="_blank"' if p == 'popup' else ''
             t = render(t, {}, {}, 'br', URL, environment, latex, None,
@@ -1373,7 +1404,7 @@ def render(text,
         if t == 'NEWLINE' and not a:
             return '<br />' + pp
         return '<span class="anchor" id="%s">%s</span>' % (
-            escape(id_prefix + t),
+            local_html_escape(id_prefix + t),
             render(a, {}, {}, 'br', URL,
                    environment, latex, autolinks,
                    protolinks, class_prefix,
@@ -1399,7 +1430,7 @@ def render(text,
     def expand_meta(m):
         code, b, p, s = segments.pop(0)
         if code is None or m.group() == DISABLED_META:
-            return escape(s)
+            return local_html_escape(s)
         if b in extra:
             if code[:1] == '\n':
                 code = code[1:]
@@ -1411,7 +1442,7 @@ def render(text,
                 return str(extra[b](code))
         elif b == 'cite':
             return '[' + ','.join('<a href="#%s" class="%s">%s</a>' %
-                                  (id_prefix + d, b, d) for d in escape(code).split(',')) + ']'
+                                  (id_prefix + d, b, d) for d in local_html_escape(code).split(',')) + ']'
         elif b == 'latex':
             return LATEX % urllib_quote(code)
         elif b in html_colors:
@@ -1426,12 +1457,12 @@ def render(text,
                    % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
                                      autolinks, protolinks, class_prefix, id_prefix, pretty_print))
         cls = ' class="%s%s"' % (class_prefix, b) if b and b != 'id' else ''
-        id = ' id="%s%s"' % (id_prefix, escape(p)) if p else ''
+        id = ' id="%s%s"' % (id_prefix, local_html_escape(p)) if p else ''
         beg = (code[:1] == '\n')
         end = [None, -1][code[-1:] == '\n']
         if beg and end:
-            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
+            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, local_html_escape(code[1:-1]), pp)
-        return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
+        return '<code%s%s>%s</code>' % (cls, id, local_html_escape(code[beg:end]))
 
     text = regex_expand_meta.sub(expand_meta, text)