From 3292f760ca7255c878ed84470898c5893d53b23f Mon Sep 17 00:00:00 2001 From: geomapdev Date: Thu, 20 Apr 2017 10:21:16 -0700 Subject: [PATCH 001/525] handle reference fields for keyed tables --- gluon/dal.py | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/gluon/dal.py b/gluon/dal.py index 95476a9f..e60cf82d 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -51,11 +51,29 @@ def _default_validators(db, field): if hasattr(referenced, '_format') and referenced._format: requires = validators.IS_IN_DB(db, referenced._id, referenced._format) - if field.unique: - requires._and = validators.IS_NOT_IN_DB(db, field) - if field.tablename == field_type[10:]: - return validators.IS_EMPTY_OR(requires) - return requires + else: + requires = validators.IS_IN_DB(db, referenced._id) + if field.unique: + requires._and = validators.IS_NOT_IN_DB(db, field) + if field.tablename == field_type[10:]: + return validators.IS_EMPTY_OR(requires) + return requires + elif db and field_type.startswith('reference') and \ + field_type.find('.') > 0 and \ + field_type[10:].split('.')[0] in db.tables: + table_name=field_type[10:].split('.')[0] + field_name=field_type[10:].split('.')[1] + referenced = db[table_name] + if hasattr(referenced, '_format') and referenced._format: + requires = validators.IS_IN_DB(db, referenced[field_name], + referenced._format) + else: + requires = validators.IS_IN_DB(db, referenced[field_name]) + if field.unique: + requires._and = validators.IS_NOT_IN_DB(db, field) + if field.tablename == table_name: + return validators.IS_EMPTY_OR(requires) + return requires elif db and field_type.startswith('list:reference') and \ field_type.find('.') < 0 and \ field_type[15:] in db.tables: @@ -71,6 +89,23 @@ def _default_validators(db, field): if not field.notnull: requires = validators.IS_EMPTY_OR(requires) return requires + elif db and field_type.startswith('list:reference') and \ + field_type.find('.') > 0 and \ + field_type[15:].split('.')[0] in db.tables: + table_name=field_type[15:].split('.')[0] + field_name=field_type[15:].split('.')[1] + referenced = db[table_name] + if hasattr(referenced, '_format') and referenced._format: + requires = validators.IS_IN_DB(db, referenced[field_name], + referenced._format, multiple=True) + else: + requires = validators.IS_IN_DB(db, referenced[field_name], + multiple=True) + if field.unique: + requires._and = validators.IS_NOT_IN_DB(db, field) + if not field.notnull: + requires = validators.IS_EMPTY_OR(requires) + return requires # does not get here for reference and list:reference if field.unique: requires.insert(0, validators.IS_NOT_IN_DB(db, field)) From 69e6e79e232db922b0ae4514a6820ee5d2dcf9bb Mon Sep 17 00:00:00 2001 From: geomapdev Date: Mon, 1 May 2017 08:39:31 -0700 Subject: [PATCH 002/525] Update dal.py updated to handle references without format --- gluon/dal.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gluon/dal.py b/gluon/dal.py index 95476a9f..27195f1a 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -51,11 +51,13 @@ def _default_validators(db, field): if hasattr(referenced, '_format') and referenced._format: requires = validators.IS_IN_DB(db, referenced._id, referenced._format) - if field.unique: - requires._and = validators.IS_NOT_IN_DB(db, field) - if field.tablename == field_type[10:]: - return validators.IS_EMPTY_OR(requires) - return requires + else: + requires = validators.IS_IN_DB(db, referenced._id) + if field.unique: + requires._and = validators.IS_NOT_IN_DB(db, field) + if not field.notnull: + return validators.IS_EMPTY_OR(requires) + return requires elif db and field_type.startswith('list:reference') and \ field_type.find('.') < 0 and \ field_type[15:] in db.tables: From cf1ea982176087850222e8a85a42d9d1f18b68ef Mon Sep 17 00:00:00 2001 From: ilvalle Date: Fri, 5 May 2017 21:12:19 +0200 Subject: [PATCH 003/525] fix TAG helper on PY3, updated web2pyHTMLParser --- gluon/decoder.py | 4 +-- gluon/html.py | 36 ++++++++---------------- gluon/sanitizer.py | 4 +-- gluon/tests/test_html.py | 61 +++++++++++++++++++++++++++++++--------- 4 files changed, 63 insertions(+), 42 deletions(-) diff --git a/gluon/decoder.py b/gluon/decoder.py index d04aaf9a..4fc068dd 100644 --- a/gluon/decoder.py +++ b/gluon/decoder.py @@ -9,7 +9,7 @@ Based on http://code.activestate.com/recipes/52257/ Licensed under the PSF License """ - +from gluon._compat import to_unicode import codecs # None represents a potentially variable byte. "##" in the XML spec... @@ -77,4 +77,4 @@ def autoDetectXMLEncoding(buffer): def decoder(buffer): encoding = autoDetectXMLEncoding(buffer) - return buffer.decode(encoding).encode('utf8') + return to_unicode(buffer, charset=encoding) diff --git a/gluon/html.py b/gluon/html.py index ceef7bd2..5ae5f4cd 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -20,7 +20,7 @@ import urllib import base64 from gluon import sanitizer, decoder import itertools -from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type +from gluon._compat import reduce, pickle, copyreg, HTMLParser, name2codepoint, iteritems, unichr, unicodeT, urllib_quote, to_bytes, to_native, to_unicode, basestring, urlencode, implements_bool, text_type, long from gluon.utils import local_html_escape import marshal @@ -998,9 +998,9 @@ class DIV(XmlComponent): if isinstance(c, XmlComponent): s = c.flatten(render) elif render: - s = render(str(c)) + s = render(to_native(c)) else: - s = str(c) + s = to_native(c) text += s if render: text = render(text, self.tag, self.attributes) @@ -1281,7 +1281,6 @@ class __TAG__(XmlComponent): def __getattr__(self, name): if name[-1:] == '_': name = name[:-1] + '/' - name=to_bytes(name) return lambda *a, **b: __tag_div__(name, *a, **b) def __call__(self, html): @@ -2376,17 +2375,17 @@ class FORM(DIV): def as_json(self, sanitize=True): d = self.as_dict(flat=True, sanitize=sanitize) - from serializers import json + from gluon.serializers import json return json(d) def as_yaml(self, sanitize=True): d = self.as_dict(flat=True, sanitize=sanitize) - from serializers import yaml + from gluon.serializers import yaml return yaml(d) def as_xml(self, sanitize=True): d = self.as_dict(flat=True, sanitize=sanitize) - from serializers import xml + from gluon.serializers import xml return xml(d) @@ -2655,36 +2654,24 @@ class web2pyHTMLParser(HTMLParser): """ obj = web2pyHTMLParser(text) parses and html/xml text into web2py helpers. obj.tree contains the root of the tree, and tree can be manipulated - - >>> str(web2pyHTMLParser('hello
wor<ldxxxyyy
zzz' - >>> str(web2pyHTMLParser('
ab
c').tree) - '
ab
c' - >>> tree = web2pyHTMLParser('hello
world
').tree - >>> tree.element(_a='b')['_c']=5 - >>> str(tree) - 'hello
world
' """ + def __init__(self, text, closed=('input', 'link')): HTMLParser.__init__(self) self.tree = self.parent = TAG['']() self.closed = closed - self.tags = [x for x in __all__ if isinstance(eval(x), DIV)] self.last = None self.feed(text) def handle_starttag(self, tagname, attrs): - if tagname.upper() in self.tags: - tag = eval(tagname.upper()) - else: - if tagname in self.closed: - tagname += '/' - tag = TAG[tagname]() + if tagname in self.closed: + tagname += '/' + tag = TAG[tagname]() for key, value in attrs: tag['_' + key] = value tag.parent = self.parent self.parent.append(tag) - if not tag.tag.endswith(b'/'): + if not tag.tag.endswith('/'): self.parent = tag else: self.last = tag.tag[:-1] @@ -2707,7 +2694,6 @@ class web2pyHTMLParser(HTMLParser): self.parent.append(entitydefs[name]) def handle_endtag(self, tagname): - tagname = to_bytes(tagname) # this deals with unbalanced tags if tagname == self.last: return diff --git a/gluon/sanitizer.py b/gluon/sanitizer.py index dbf84eca..5cd2ea4a 100644 --- a/gluon/sanitizer.py +++ b/gluon/sanitizer.py @@ -11,7 +11,7 @@ Cross-site scripting (XSS) defense """ from gluon._compat import HTMLParser, urlparse, entitydefs, basestring -from cgi import escape +from gluon.utils import local_html_escape from formatter import AbstractFormatter from xml.sax.saxutils import quoteattr @@ -21,7 +21,7 @@ __all__ = ['sanitize'] def xssescape(text): """Gets rid of < and > and & and, for good measure, :""" - return escape(text, quote=True).replace(':', ':') + return local_html_escape(text, quote=True).replace(':', ':') class XssCleaner(HTMLParser): diff --git a/gluon/tests/test_html.py b/gluon/tests/test_html.py index e091d1ce..6ee181d5 100644 --- a/gluon/tests/test_html.py +++ b/gluon/tests/test_html.py @@ -11,11 +11,13 @@ import unittest from gluon.html import A, ASSIGNJS, B, BEAUTIFY, P, BODY, BR, BUTTON, CAT, CENTER, CODE, COL, COLGROUP, DIV, SPAN, URL, verifyURL from gluon.html import truncate_string, EM, FIELDSET, FORM, H1, H2, H3, H4, H5, H6, HEAD, HR, HTML, I, IFRAME, IMG, INPUT, EMBED from gluon.html import LABEL, LEGEND, LI, LINK, MARKMIN, MENU, META, OBJECT, OL, OPTGROUP, OPTION, PRE, SCRIPT, SELECT, STRONG -from gluon.html import STYLE, TABLE, TR, TD, TAG, TBODY, THEAD, TEXTAREA, TFOOT, TH, TITLE, TT, UL, XHTML, XML +from gluon.html import STYLE, TABLE, TR, TD, TAG, TBODY, THEAD, TEXTAREA, TFOOT, TH, TITLE, TT, UL, XHTML, XML, web2pyHTMLParser from gluon.storage import Storage from gluon.html import XML_pickle, XML_unpickle from gluon.html import TAG_pickler, TAG_unpickler from gluon._compat import xrange, PY2, to_native +from gluon.decoder import decoder +import re class TestBareHelpers(unittest.TestCase): @@ -155,7 +157,7 @@ class TestBareHelpers(unittest.TestCase): self.assertEqual(rtn, True) # TODO: def test_XmlComponent(self): - @unittest.skipIf(not PY2, "Skipping Python 3.x tests for XML.__repr__") + def test_XML(self): # sanitization process self.assertEqual(XML('

HelloWorld

').xml(), @@ -179,19 +181,18 @@ class TestBareHelpers(unittest.TestCase): # you can compare them ##self.assertEqual(XML('a') == XML('a'), True) # beware that the comparison is made on the XML repr - self.assertEqual(XML('

HelloWorld

', sanitize=True), - XML('

HelloWorld

')) + + self.assertEqual(XML('

HelloWorld

', sanitize=True).__repr__(), + XML('

HelloWorld

').__repr__()) # bug check for the sanitizer for closing no-close tags - self.assertEqual(XML('

Test


Test


', sanitize=True), - XML('

Test


Test


')) + self.assertEqual(XML('

Test


Test


', sanitize=True).xml(), + XML('

Test


Test


').xml()) # basic flatten test self.assertEqual(XML('

Test

').flatten(), '

Test

') self.assertEqual(XML('

Test

').flatten(render=lambda text, tag, attr: text), '

Test

') - @unittest.skipIf(not PY2, "Skipping Python 3.x tests for XML_unpickle.__repr__") def test_XML_pickle_unpickle(self): - # weird test - self.assertEqual(XML_unpickle(XML_pickle('data to be pickle')[1][0]), 'data to be pickle') + self.assertEqual(str(XML_unpickle(XML_pickle('data to be pickle')[1][0])), 'data to be pickle') def test_DIV(self): # Empty DIV() @@ -255,6 +256,11 @@ class TestBareHelpers(unittest.TestCase): self.assertEqual(DIV('

Test

', _class="class_test").get('_class'), 'class_test') self.assertEqual(DIV(b'a').xml(), b'
a
') + def test_decoder(self): + tag_html = '
hello

world

' + a = decoder(tag_html) + self.assertEqual(a, tag_html) + def test_CAT(self): # Empty CAT() self.assertEqual(CAT().xml(), b'') @@ -636,8 +642,8 @@ class TestBareHelpers(unittest.TestCase): # These 2 crash AppVeyor and Travis with: "ImportError: No YAML serializer available" # self.assertEqual(FORM('<>', _a='1', _b='2').as_yaml(), # "accepted: null\nattributes: {_a: '1', _action: '#', _b: '2', _enctype: multipart/form-data, _method: post}\ncomponents: [<>]\nerrors: {}\nlatest: {}\nparent: null\nvars: {}\n") - # self.assertEqual(FORM('<>', _a='1', _b='2').as_xml(), - # 'None<_enctype>multipart/form-data<_action>#<_b>2<_a>1<_method>post&lt;&gt;None') + # TODO check tags content + self.assertEqual(len(FORM('<>', _a='1', _b='2').as_xml()), 334) def test_BEAUTIFY(self): #self.assertEqual(BEAUTIFY(['a', 'b', {'hello': 'world'}]).xml(), @@ -670,13 +676,42 @@ class TestBareHelpers(unittest.TestCase): # TODO: def test_embed64(self): - # TODO: def test_web2pyHTMLParser(self): + def test_web2pyHTMLParser(self): + #tag should not be a byte + self.assertEqual(web2pyHTMLParser("
").tree.components[0].tag, 'div') + a = str(web2pyHTMLParser('
ab
c').tree) + self.assertEqual(a, "
ab
c") + + tree = web2pyHTMLParser('hello
world
').tree + tree.element(_a='b')['_c']=5 + self.assertEqual(str(tree), 'hello
world
') + + a = str(web2pyHTMLParser('
', closed=['img']).tree) + self.assertEqual(a, '
') + + #greater-than sign ( > ) --> decimal > --> hexadecimal > + #Less-than sign ( < ) --> decimal < --> hexadecimal < + # test decimal + a = str(web2pyHTMLParser('
< >
').tree) + self.assertEqual(a, '
< >
') + # test hexadecimal + a = str(web2pyHTMLParser('
< >
').tree) + self.assertEqual(a, '
< >
') + + def test_markdown(self): + def markdown(text, tag=None, attributes={}): + r = {None: re.sub('\s+',' ',text), \ + 'h1':'#'+text+'\\n\\n', \ + 'p':text+'\\n'}.get(tag,text) + return r + a=TAG('

Header

this is a test

') + ret = a.flatten(markdown) + self.assertEqual(ret, '#Header\\n\\nthis is a test\\n') # TODO: def test_markdown_serializer(self): # TODO: def test_markmin_serializer(self): - @unittest.skipIf(not PY2, "Skipping Python 3.x tests for MARKMIN") def test_MARKMIN(self): # This test pass with python 2.7 but expected to fail under 2.6 # with self.assertRaises(TypeError) as cm: From 49bf14e79a601a2fca5783857a217992ec326e58 Mon Sep 17 00:00:00 2001 From: Scimonster Date: Sun, 7 May 2017 14:32:52 +0300 Subject: [PATCH 004/525] Fix #1624 -- Unicode in XML sanitizing causes error Add unit test for it --- gluon/html.py | 4 ++-- gluon/tests/test_html.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gluon/html.py b/gluon/html.py index ceef7bd2..bc6f5fd1 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -596,10 +596,10 @@ class XML(XmlComponent): for A, IMG and BlockQuote). The key is the tag; the value is a list of allowed attributes. """ - if sanitize: - text = sanitizer.sanitize(text, permitted_tags, allowed_attributes) if isinstance(text, unicodeT): text = to_native(text.encode('utf8', 'xmlcharrefreplace')) + if sanitize: + text = sanitizer.sanitize(text, permitted_tags, allowed_attributes) elif isinstance(text, bytes): text = to_native(text) elif not isinstance(text, str): diff --git a/gluon/tests/test_html.py b/gluon/tests/test_html.py index e091d1ce..ad611ed3 100644 --- a/gluon/tests/test_html.py +++ b/gluon/tests/test_html.py @@ -168,6 +168,8 @@ class TestBareHelpers(unittest.TestCase): # seams that __repr__ is no longer enough ##self.assertEqual(XML('1.3'), '1.3') self.assertEqual(XML(u'
è
').xml(), b'
\xc3\xa8
') + # make sure unicode works with sanitize + self.assertEqual(XML(u'
è
', sanitize=True).xml(), b'
\xc3\xa8
') # you can calc len on the class, that equals the xml() and the str() ##self.assertEqual(len(XML('1.3')), len('1.3')) self.assertEqual(len(XML('1.3').xml()), len('1.3')) From 3505e372d860075fd9b7c506f5da146804924740 Mon Sep 17 00:00:00 2001 From: Andrew Willimott Date: Mon, 8 May 2017 06:40:50 +1200 Subject: [PATCH 005/525] Remove graphviz graph code from appadmin and design files --- applications/admin/controllers/appadmin.py | 60 ------------------- applications/admin/views/appadmin.html | 49 +++++---------- applications/admin/views/default/design.html | 2 - applications/examples/controllers/appadmin.py | 60 ------------------- applications/examples/views/appadmin.html | 49 +++++---------- applications/welcome/controllers/appadmin.py | 60 ------------------- applications/welcome/views/appadmin.html | 49 +++++---------- 7 files changed, 42 insertions(+), 287 deletions(-) diff --git a/applications/admin/controllers/appadmin.py b/applications/admin/controllers/appadmin.py index e72b3dab..bd4d2966 100644 --- a/applications/admin/controllers/appadmin.py +++ b/applications/admin/controllers/appadmin.py @@ -12,12 +12,6 @@ import gluon.contenttype import gluon.fileutils from gluon._compat import iteritems -# d3_graph_model added but leaving pygraphviz code as is for initial tests. -try: - import pygraphviz as pgv -except ImportError: - pgv = None - is_gae = request.env.web2py_runtime_gae or False # ## critical --- make a copy of the environment @@ -566,60 +560,6 @@ def table_template(table): _cellborder=0, _cellspacing=0) ).xml() - -# d3_graph_model added but leaving pygraphviz code as is for initial tests. -# The Graph Model button in admin app views/default/design.html has been redirected -# to the d3_graph_model function. -def bg_graph_model(): - graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') - - subgraphs = dict() - for tablename in db.tables: - if hasattr(db[tablename],'_meta_graphmodel'): - meta_graphmodel = db[tablename]._meta_graphmodel - else: - meta_graphmodel = dict(group=request.application, color='#ECECEC') - - group = meta_graphmodel['group'].replace(' ', '') - if group not in subgraphs: - subgraphs[group] = dict(meta=meta_graphmodel, tables=[]) - subgraphs[group]['tables'].append(tablename) - - graph.add_node(tablename, name=tablename, shape='plaintext', - label=table_template(tablename)) - - for n, key in enumerate(subgraphs.iterkeys()): - graph.subgraph(nbunch=subgraphs[key]['tables'], - name='cluster%d' % n, - style='filled', - color=subgraphs[key]['meta']['color'], - label=subgraphs[key]['meta']['group']) - - for tablename in db.tables: - for field in db[tablename]: - f_type = field.type - if isinstance(f_type,str) and ( - f_type.startswith('reference') or - f_type.startswith('list:reference')): - referenced_table = f_type.split()[1].split('.')[0] - n1 = graph.get_node(tablename) - n2 = graph.get_node(referenced_table) - graph.add_edge(n1, n2, color="#4C4C4C", label='') - - graph.layout() - if not request.args: - response.headers['Content-Type'] = 'image/png' - return graph.draw(format='png', prog='dot') - else: - response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0) - if request.args(0) == 'dot': - return graph.string() - else: - return graph.draw(format=request.args(0), prog='dot') - -def graph_model(): - return dict(databases=databases, pgv=pgv) - def manage(): tables = manager_action['tables'] if isinstance(tables[0], str): diff --git a/applications/admin/views/appadmin.html b/applications/admin/views/appadmin.html index c92e4058..5330b163 100644 --- a/applications/admin/views/appadmin.html +++ b/applications/admin/views/appadmin.html @@ -233,43 +233,22 @@
{{pass}} -{{if request.function=='graph_model':}} -

{{=T("Graph Model")}}

- {{if not pgv:}} - {{=T('pygraphviz library not found')}} - {{elif not databases:}} - {{=T("No databases in this application")}} - {{else:}} -
- - {{=T('Save model as...')}} - - - -
-
- {{=IMG(_src=URL('appadmin', 'bg_graph_model'))}} - {{pass}} -{{pass}} - {{if request.function=='d3_graph_model':}}

{{=T("Graph Model")}}

-
- - -{{pass}} + {{if not databases:}} + {{=T("No databases in this application")}} + {{else:}} +
+ + + {{pass}} +{{pass}} {{if request.function == 'manage':}}

{{=heading}}

diff --git a/applications/admin/views/default/design.html b/applications/admin/views/default/design.html index ec3ca099..5b81f83a 100644 --- a/applications/admin/views/default/design.html +++ b/applications/admin/views/default/design.html @@ -107,8 +107,6 @@ def deletefile(arglist, vars={}): {{pass}} {{if os.access(os.path.join(request.folder,'..',app,'static','js','d3_graph.js'),os.R_OK):}} {{=button(URL(a=app, c='appadmin',f='d3_graph_model'), T('graph model'))}} - {{else:}} - {{=button(URL(a=app, c='appadmin',f='graph_model'), T('graph model'))}} {{pass}}