From 356643d50914eaf694a4326d85bf9973f9276a96 Mon Sep 17 00:00:00 2001 From: Massimo Date: Thu, 24 Jan 2013 10:51:44 -0600 Subject: [PATCH 01/39] check whether pgsql supports json fields, thanks Alan --- VERSION | 2 +- gluon/dal.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fb34ff20..312f3df5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.23.08.48.16 +Version 2.4.1-alpha.2+timestamp.2013.01.24.10.51.01 diff --git a/gluon/dal.py b/gluon/dal.py index 4d4daa48..69c0af9e 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -2569,11 +2569,25 @@ class PostgreSQLAdapter(BaseAdapter): def after_connection(self): self.connection.set_client_encoding('UTF8') self.execute("SET standard_conforming_strings=on;") + self.try_json() def lastrowid(self,table): self.execute("select currval('%s')" % table._sequence_name) return int(self.cursor.fetchone()[0]) + def try_json(self): + # check JSON data type support + # (to be added to after_connection) + if self.driver_name == "pg8000": + supports_json = self.connection.server_version >= "9.2.0" + elif (self.driver_name == "psycopg2") and \ + (self.driver.__version__ >= "2.0.12"): + supports_json = self.connection.server_version >= 90200 + elif self.driver_name == "zxJDBC": + supports_json = self.connection.dbversion >= "9.2.0" + else: supports_json = None + if supports_json: self.types["json"] = "JSON" + else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)") def LIKE(self,first,second): args = (self.expand(first), self.expand(second,'string')) @@ -2797,6 +2811,7 @@ class JDBCPostgreSQLAdapter(PostgreSQLAdapter): self.connection.set_client_encoding('UTF8') self.execute('BEGIN;') self.execute("SET CLIENT_ENCODING TO 'UNICODE';") + self.try_json() class OracleAdapter(BaseAdapter): From d4e68e6c1849be235e94633aea5219412874a67f Mon Sep 17 00:00:00 2001 From: Massimo Date: Thu, 24 Jan 2013 15:52:21 -0600 Subject: [PATCH 02/39] more modular setup-web2py-nginx-uwsgi-ubuntu, thanks Paolo and Niphlod --- VERSION | 2 +- scripts/setup-web2py-nginx-uwsgi-ubuntu.sh | 90 +++++++++++++++++----- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/VERSION b/VERSION index 312f3df5..da23749e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.24.10.51.01 +Version 2.4.1-alpha.2+timestamp.2013.01.24.15.51.43 diff --git a/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh b/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh index e70b6605..2f48a349 100644 --- a/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh +++ b/scripts/setup-web2py-nginx-uwsgi-ubuntu.sh @@ -19,19 +19,50 @@ apt-get -y install build-essential python-dev libxml2-dev python-pip unzip pip install --upgrade pip PIPPATH=`which pip` $PIPPATH install --upgrade uwsgi +# Create common nginx sections +mkdir /etc/nginx/conf.d/web2py +echo ' +gzip_static on; +gzip_http_version 1.1; +gzip_proxied expired no-cache no-store private auth; +gzip_disable "MSIE [1-6]\."; +gzip_vary on; +' > /etc/nginx/conf.d/web2py/gzip_static.conf +echo ' +gzip on; +gzip_disable "msie6"; +gzip_vary on; +gzip_proxied any; +gzip_comp_level 6; +gzip_buffers 16 8k; +gzip_http_version 1.1; +gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; +' > /etc/nginx/conf.d/web2py/gzip.conf # Create configuration file /etc/nginx/sites-available/web2py echo 'server { listen 80; server_name $hostname; - #to enable correct use of response.static_version + ###to enable correct use of response.static_version #location ~* /(\w+)/static(?:/_[\d]+\.[\d]+\.[\d]+)?/(.*)$ { # alias /home/www-data/web2py/applications/$1/static/$2; # expires max; #} + ### + + ###if you use something like myapp = dict(languages=['en', 'it', 'jp'], default_language='en') in your routes.py + #location ~* /(\w+)/(en|it|jp)/static/(.*)$ { + # alias /home/www-data/web2py/applications/$1/; + # try_files static/$2/$3 static/$3 =404; + #} + ### location ~* /(\w+)/static/ { root /home/www-data/web2py/applications/; #remove next comment on production #expires max; + ### if you want to use pre-gzipped static files (recommended) + ### check scripts/zip_static_files.py and remove the comments + # include /etc/nginx/conf.d/web2py/gzip_static.conf; + ### } location / { #uwsgi_pass 127.0.0.1:9001; @@ -39,6 +70,14 @@ echo 'server { include uwsgi_params; uwsgi_param UWSGI_SCHEME $scheme; uwsgi_param SERVER_SOFTWARE nginx/$nginx_version; + + ###remove the comments to turn on if you want gzip compression of your pages + # include /etc/nginx/conf.d/web2py/gzip.conf; + ### end gzip section + + ### remove the comments if you use uploads (max 10 MB) + #client_max_body_size 10m; + ### } } server { @@ -58,7 +97,15 @@ server { include uwsgi_params; uwsgi_param UWSGI_SCHEME $scheme; uwsgi_param SERVER_SOFTWARE nginx/$nginx_version; + ###remove the comments to turn on if you want gzip compression of your pages + # include /etc/nginx/conf.d/web2py/gzip.conf; + ### end gzip section + ### remove the comments if you want to enable uploads (max 10 MB) + #client_max_body_size 10m; + ### } + ## if you serve static files through https, copy here the section + ## from the previous server instance to manage static files }' >/etc/nginx/sites-available/web2py @@ -78,25 +125,26 @@ sudo mkdir /etc/uwsgi sudo mkdir /var/log/uwsgi # Create configuration file /etc/uwsgi/web2py.xml -echo ' - /tmp/web2py.socket - /home/www-data/web2py/ - /=wsgihandler:application - - 4 - 60 - 8 - 1 - /tmp/stats.socket - 2000 - 512 - 256 - 192 - www-data - www-data - 0 0 -1 -1 -1 python /home/www-data/web2py/web2py.py -Q -S welcome -M -R scripts/sessions2trash.py -A -o - -' >/etc/uwsgi/web2py.xml +echo '[uwsgi] + +socket = /tmp/web2py.socket +pythonpath = /home/www-data/web2py/ +mount = /=wsgihandler:application +processes = 4 +master = true +harakiri = 60 +reload-mercy = 8 +cpu-affinity = 1 +stats = /tmp/stats.socket +max-requests = 2000 +limit-as = 512 +reload-on-as = 256 +reload-on-rss = 192 +uid = www-data +gid = www-data +cron = 0 0 -1 -1 -1 python /home/www-data/web2py/web2py.py -Q -S welcome -M -R scripts/sessions2trash.py -A -o +no-orphans = true +' >/etc/uwsgi/web2py.ini #Create a configuration file for uwsgi in emperor-mode #for Upstart in /etc/init/uwsgi-emperor.conf @@ -135,4 +183,4 @@ start uwsgi-emperor ## and stop it with # stop uwsgi-emperor ## to reload web2py only (without restarting uwsgi) -# touch /etc/uwsgi/web2py.xml \ No newline at end of file +# touch /etc/uwsgi/web2py.xml From ca330a99e2e2bbf9a1be7ea6ffb24fa109efb05d Mon Sep 17 00:00:00 2001 From: Massimo Date: Fri, 25 Jan 2013 15:29:04 -0600 Subject: [PATCH 03/39] issue 1302, support for expressions in CONTAINS, thanks Paolo --- VERSION | 2 +- gluon/dal.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index da23749e..629d9d45 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.24.15.51.43 +Version 2.4.1-alpha.2+timestamp.2013.01.25.15.28.24 diff --git a/gluon/dal.py b/gluon/dal.py index 69c0af9e..055f4eaa 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -1234,11 +1234,22 @@ class BaseAdapter(ConnectionPool): self.expand('%'+second, 'string')) def CONTAINS(self, first, second): - if first.type in ('string', 'text', 'json'): - key = '%'+str(second).replace('%','%%')+'%' - elif first.type.startswith('list:'): - key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%' - return '(%s LIKE %s)' % (self.expand(first),self.expand(key,'string')) + field = self.expand(first) + if isinstance(second,Expression): + expr = self.expand(second,'string') + if first.type.startswith('list:'): + expr = 'CONCAT("|", %s, "|")' % expr + elif not first.type in ('string', 'text', 'json'): + raise RuntimeError("Expression Not Supported") + return 'INSTR(%s,%s)' % (field, expr) + else: + if first.type in ('string', 'text', 'json'): + key = '%'+str(second).replace('%','%%')+'%' + elif first.type.startswith('list:'): + key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%' + else: + raise RuntimeError("Expression Not Supported") + return '(%s LIKE %s)' % (field,self.expand(key,'string')) def EQ(self, first, second=None): if second is None: From 147b2d889d8d60426012732f2a133ff648479b57 Mon Sep 17 00:00:00 2001 From: Massimo Date: Fri, 25 Jan 2013 15:36:23 -0600 Subject: [PATCH 04/39] db.as_json/as_xml thanks Alan (This is tentative and experimental) --- VERSION | 2 +- gluon/dal.py | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 629d9d45..0d50d7f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.25.15.28.24 +Version 2.4.1-alpha.2+timestamp.2013.01.25.15.35.52 diff --git a/gluon/dal.py b/gluon/dal.py index 055f4eaa..b48c0bb9 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -7551,6 +7551,32 @@ def index(): if on_define: on_define(table) return table + def as_dict(self, flat=False, sanitize=True): + dbname = codec = uid = uri = None + if not sanitize: + uri, dbname, codec, uid = (str(self), self._dbname, + self._db_codec, self._db_uid) + db_as_dict = dict(items={}, tables=[], uri=uri, dbname=dbname, + codec=codec, uid=uid) + for table in self: + tablename = str(table) + db_as_dict["tables"].append(tablename) + db_as_dict["items"][tablename] = table.as_dict(flat=flat, + sanitize=sanitize) + return db_as_dict + + def as_xml(self, sanitize=True): + if not have_serializers: + raise ImportError("No xml serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return serializers.xml(d) + + def as_json(self, sanitize=True): + if not have_serializers: + raise ImportError("No json serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return serializers.json(d) + def __contains__(self, tablename): try: return tablename in self.tables @@ -8437,6 +8463,28 @@ class Table(object): if id_map and cid is not None: id_map_self[int(line[cid])] = new_id + def as_dict(self, flat=False, sanitize=True): + tablename = str(self) + table_as_dict = dict(name=tablename, items={}, fields=[]) + for field in self: + if (field.readable or field.writable) or (not sanitize): + table_as_dict["fields"].append(field.name) + table_as_dict["items"][field.name] = \ + field.as_dict(flat=flat, sanitize=sanitize) + return table_as_dict + + def as_xml(self, sanitize=True): + if not have_serializers: + raise ImportError("No xml serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return serializers.xml(d) + + def as_json(self, sanitize=True): + if not have_serializers: + raise ImportError("No json serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return serializers.json(d) + def with_alias(self, alias): return self._db._adapter.alias(self,alias) @@ -9082,6 +9130,69 @@ class Field(Expression): def count(self, distinct=None): return Expression(self.db, self.db._adapter.COUNT, self, distinct, 'integer') + def as_dict(self, flat=False, sanitize=True): + attrs = ("readable", "writable", "label", "default", "name", + "type", "represent", "compute") + SERIALIZABLE = (int, long, basestring, dict, list, float, + tuple, bool, None.__class__) + def flatten(obj): + if flat: + if isinstance(obj, flatten.__class__): + return str(type(obj)) + elif isinstance(obj, type): + try: + obj = str(obj).split("'")[1] + except IndexError: + obj = str(obj) + elif not isinstance(obj, SERIALIZABLE): + obj = str(obj) + return obj + + def filter_requires(t, r): + if sanitize and any([keyword in str(t).upper() for + keyword in ("CRYPT", "IS_STRONG")]): + return None + for k, v in r.items(): + if k == "other": + if not isinstance(v, dict): + other = v.__dict__ + else: other = v + r[k] = {flatten(type(v)): + filter_requires(type(v), other)} + elif flat and (not isinstance(v, SERIALIZABLE)): + r[k] = str(v) + return r + + if isinstance(self.requires, (tuple, list, set)): + requires = dict([(flatten(type(r)), + filter_requires(type(r), r.__dict__)) for + r in self.requires]) + else: + requires = {flatten(type(self.requires)): + filter_requires(type(self.requires), + self.requires.__dict__)} + + d = dict(colname="%s.%s" % (self.tablename, self.name), + requires=requires) + d.update([(attr, flatten(getattr(self, attr))) for attr in attrs]) + return d + + def as_xml(self, sanitize=True): + if have_serializers: + xml = serializers.xml + else: + raise ImportError("No xml serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return xml(d) + + def as_json(self, sanitize=True): + if have_serializers: + json = serializers.json + else: + raise ImportError("No json serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return json(d) + def __nonzero__(self): return True From ec7c8f0248516328cf81679a0aa067278d740565 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 27 Jan 2013 09:46:22 -0600 Subject: [PATCH 05/39] added bottom right save button in edit_languages, issue 1300 --- VERSION | 2 +- applications/admin/views/default/edit_language.html | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 0d50d7f5..4d5d468d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.25.15.35.52 +Version 2.4.1-alpha.2+timestamp.2013.01.27.09.45.36 diff --git a/applications/admin/views/default/edit_language.html b/applications/admin/views/default/edit_language.html index e1f9f4de..ebfa7ed7 100644 --- a/applications/admin/views/default/edit_language.html +++ b/applications/admin/views/default/edit_language.html @@ -22,4 +22,5 @@ function hideShowTranslated(){
{{=form}}
- \ No newline at end of file + + From 8fe3934c6e37caa09506567055224a0fc5944feb Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 27 Jan 2013 09:59:19 -0600 Subject: [PATCH 06/39] fixed codemirror syntax highlighting, thanks Mariano --- VERSION | 2 +- applications/admin/controllers/default.py | 3 ++- applications/admin/static/js/ajax_editor.js | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 4d5d468d..b167470f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.27.09.45.36 +Version 2.4.1-alpha.2+timestamp.2013.01.27.09.58.34 diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index caeb5d83..019b25d0 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -645,6 +645,7 @@ def edit(): code = request.vars.data.rstrip().replace('\r\n', '\n') + '\n' compile(code, path, "exec", _ast.PyCF_ONLY_AST) except Exception, e: + # offset calculation is only used for textarea (start/stop) start = sum([len(line) + 1 for l, line in enumerate(request.vars.data.split("\n")) if l < e.lineno - 1]) @@ -654,7 +655,7 @@ def edit(): else: offset = 0 highlight = {'start': start, 'end': start + - offset + 1, 'lineno': e.lineno} + offset + 1, 'lineno': e.lineno, 'offset': offset} try: ex_name = e.__class__.__name__ except: diff --git a/applications/admin/static/js/ajax_editor.js b/applications/admin/static/js/ajax_editor.js index 44d5de7d..d2d733ee 100644 --- a/applications/admin/static/js/ajax_editor.js +++ b/applications/admin/static/js/ajax_editor.js @@ -43,8 +43,9 @@ function doHighlight(highlight) { if (window.ace_editor) { window.ace_editor.gotoLine(highlight.lineno); } else if (window.mirror) { - window.mirror.setSelection({line:highlight.lineno,ch:0}, - {line:highlight.end,ch:0}); + // Put the cursor at the offending line: + window.mirror.setCursor({line:highlight.lineno-1, + ch:highlight.offset+1}); } else if (window.eamy) { // not implemented } else if (window.textarea) { From 3bb8d5e235ec8615e4a3368c7a9b8b6d83303f70 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 27 Jan 2013 10:25:02 -0600 Subject: [PATCH 07/39] configurable appadmin graphs, thanks Jose --- VERSION | 2 +- applications/admin/controllers/appadmin.py | 45 +++++++++++++++---- applications/admin/views/appadmin.html | 16 ++++++- applications/examples/controllers/appadmin.py | 45 +++++++++++++++---- applications/examples/views/appadmin.html | 16 ++++++- applications/welcome/controllers/appadmin.py | 45 +++++++++++++++---- applications/welcome/views/appadmin.html | 16 ++++++- 7 files changed, 157 insertions(+), 28 deletions(-) diff --git a/VERSION b/VERSION index b167470f..98d8ae98 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.27.09.58.34 +Version 2.4.1-alpha.2+timestamp.2013.01.27.10.24.17 diff --git a/applications/admin/controllers/appadmin.py b/applications/admin/controllers/appadmin.py index 99a0186d..a2b2c6ab 100644 --- a/applications/admin/controllers/appadmin.py +++ b/applications/admin/controllers/appadmin.py @@ -469,10 +469,10 @@ def ccache(): def table_template(table): from gluon.html import TR, TD, TABLE, TAG - + def FONT(*args, **kwargs): return TAG.font(*args, **kwargs) - + def types(field): f_type = field.type if not isinstance(f_type,str): @@ -480,7 +480,7 @@ def table_template(table): elif f_type == 'string': return field.length elif f_type == 'id': - return B('pk') + return B('pk') elif f_type.startswith('reference') or \ f_type.startswith('list:reference'): return B('fk') @@ -495,7 +495,7 @@ def table_template(table): face = "Helvetica" face_bold = "Helvetica Bold" border = 0 - + rows.append(TR(TD(FONT(table, _face=face_bold, _color=bgcolor), _colspan=3, _cellpadding=cellpadding, _align="center", _bgcolor=color))) @@ -515,10 +515,31 @@ def table_template(table): def bg_graph_model(): - graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') + graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') + + subgraphs = dict() for tablename in db.tables: - graph.add_node(tablename, name=tablename, shape='plaintext', + if hasattr(db[tablename],'_meta_graphmodel'): + meta_graphmodel = db[tablename]._meta_graphmodel + else: + meta_graphmodel = dict(group='Undefined', color='#ECECEC') + + group = meta_graphmodel['group'].replace(' ', '') + if not subgraphs.has_key(group): + subgraphs[group] = dict(meta=meta_graphmodel, tables=[]) + subgraphs[group]['tables'].append(tablename) + else: + subgraphs[group]['tables'].append(tablename) + + graph.add_node(tablename, name=tablename, shape='plaintext', label=table_template(tablename)) + + for n, key in enumerate(subgraphs.iterkeys()): + graph.subgraph(nbunch=subgraphs[key]['tables'], + name='cluster%d' % n, + style='filled', + color=subgraphs[key]['meta']['color'], + label=subgraphs[key]['meta']['group']) for tablename in db.tables: for field in db[tablename]: @@ -532,7 +553,15 @@ def bg_graph_model(): graph.add_edge(n1, n2, color="#4C4C4C", label='') graph.layout() - return graph.draw(format='png', prog='dot') + #return graph.draw(format='png', prog='dot') + if not request.args: + return graph.draw(format='png', prog='dot') + else: + response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0) + if request.args(0) == 'dot': + return graph.string() + else: + return graph.draw(format=request.args(0), prog='dot') -def graph_model(): +def graph_model(): return dict(databases=databases, pgv=pgv) diff --git a/applications/admin/views/appadmin.html b/applications/admin/views/appadmin.html index a9f6dab8..95c60ecf 100644 --- a/applications/admin/views/appadmin.html +++ b/applications/admin/views/appadmin.html @@ -229,7 +229,21 @@ {{=T('pygraphviz library not found')}} {{elif not databases:}} {{=T("No databases in this application")}} - {{else:}} + {{else:}} +
+ + {{=T('Save model as...')}} + + + +
+
{{=IMG(_src=URL('appadmin', 'bg_graph_model'))}} {{pass}} {{pass}} diff --git a/applications/examples/controllers/appadmin.py b/applications/examples/controllers/appadmin.py index 99a0186d..a2b2c6ab 100644 --- a/applications/examples/controllers/appadmin.py +++ b/applications/examples/controllers/appadmin.py @@ -469,10 +469,10 @@ def ccache(): def table_template(table): from gluon.html import TR, TD, TABLE, TAG - + def FONT(*args, **kwargs): return TAG.font(*args, **kwargs) - + def types(field): f_type = field.type if not isinstance(f_type,str): @@ -480,7 +480,7 @@ def table_template(table): elif f_type == 'string': return field.length elif f_type == 'id': - return B('pk') + return B('pk') elif f_type.startswith('reference') or \ f_type.startswith('list:reference'): return B('fk') @@ -495,7 +495,7 @@ def table_template(table): face = "Helvetica" face_bold = "Helvetica Bold" border = 0 - + rows.append(TR(TD(FONT(table, _face=face_bold, _color=bgcolor), _colspan=3, _cellpadding=cellpadding, _align="center", _bgcolor=color))) @@ -515,10 +515,31 @@ def table_template(table): def bg_graph_model(): - graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') + graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') + + subgraphs = dict() for tablename in db.tables: - graph.add_node(tablename, name=tablename, shape='plaintext', + if hasattr(db[tablename],'_meta_graphmodel'): + meta_graphmodel = db[tablename]._meta_graphmodel + else: + meta_graphmodel = dict(group='Undefined', color='#ECECEC') + + group = meta_graphmodel['group'].replace(' ', '') + if not subgraphs.has_key(group): + subgraphs[group] = dict(meta=meta_graphmodel, tables=[]) + subgraphs[group]['tables'].append(tablename) + else: + subgraphs[group]['tables'].append(tablename) + + graph.add_node(tablename, name=tablename, shape='plaintext', label=table_template(tablename)) + + for n, key in enumerate(subgraphs.iterkeys()): + graph.subgraph(nbunch=subgraphs[key]['tables'], + name='cluster%d' % n, + style='filled', + color=subgraphs[key]['meta']['color'], + label=subgraphs[key]['meta']['group']) for tablename in db.tables: for field in db[tablename]: @@ -532,7 +553,15 @@ def bg_graph_model(): graph.add_edge(n1, n2, color="#4C4C4C", label='') graph.layout() - return graph.draw(format='png', prog='dot') + #return graph.draw(format='png', prog='dot') + if not request.args: + return graph.draw(format='png', prog='dot') + else: + response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0) + if request.args(0) == 'dot': + return graph.string() + else: + return graph.draw(format=request.args(0), prog='dot') -def graph_model(): +def graph_model(): return dict(databases=databases, pgv=pgv) diff --git a/applications/examples/views/appadmin.html b/applications/examples/views/appadmin.html index a9f6dab8..95c60ecf 100644 --- a/applications/examples/views/appadmin.html +++ b/applications/examples/views/appadmin.html @@ -229,7 +229,21 @@ {{=T('pygraphviz library not found')}} {{elif not databases:}} {{=T("No databases in this application")}} - {{else:}} + {{else:}} +
+ + {{=T('Save model as...')}} + + + +
+
{{=IMG(_src=URL('appadmin', 'bg_graph_model'))}} {{pass}} {{pass}} diff --git a/applications/welcome/controllers/appadmin.py b/applications/welcome/controllers/appadmin.py index 99a0186d..a2b2c6ab 100644 --- a/applications/welcome/controllers/appadmin.py +++ b/applications/welcome/controllers/appadmin.py @@ -469,10 +469,10 @@ def ccache(): def table_template(table): from gluon.html import TR, TD, TABLE, TAG - + def FONT(*args, **kwargs): return TAG.font(*args, **kwargs) - + def types(field): f_type = field.type if not isinstance(f_type,str): @@ -480,7 +480,7 @@ def table_template(table): elif f_type == 'string': return field.length elif f_type == 'id': - return B('pk') + return B('pk') elif f_type.startswith('reference') or \ f_type.startswith('list:reference'): return B('fk') @@ -495,7 +495,7 @@ def table_template(table): face = "Helvetica" face_bold = "Helvetica Bold" border = 0 - + rows.append(TR(TD(FONT(table, _face=face_bold, _color=bgcolor), _colspan=3, _cellpadding=cellpadding, _align="center", _bgcolor=color))) @@ -515,10 +515,31 @@ def table_template(table): def bg_graph_model(): - graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') + graph = pgv.AGraph(layout='dot', directed=True, strict=False, rankdir='LR') + + subgraphs = dict() for tablename in db.tables: - graph.add_node(tablename, name=tablename, shape='plaintext', + if hasattr(db[tablename],'_meta_graphmodel'): + meta_graphmodel = db[tablename]._meta_graphmodel + else: + meta_graphmodel = dict(group='Undefined', color='#ECECEC') + + group = meta_graphmodel['group'].replace(' ', '') + if not subgraphs.has_key(group): + subgraphs[group] = dict(meta=meta_graphmodel, tables=[]) + subgraphs[group]['tables'].append(tablename) + else: + subgraphs[group]['tables'].append(tablename) + + graph.add_node(tablename, name=tablename, shape='plaintext', label=table_template(tablename)) + + for n, key in enumerate(subgraphs.iterkeys()): + graph.subgraph(nbunch=subgraphs[key]['tables'], + name='cluster%d' % n, + style='filled', + color=subgraphs[key]['meta']['color'], + label=subgraphs[key]['meta']['group']) for tablename in db.tables: for field in db[tablename]: @@ -532,7 +553,15 @@ def bg_graph_model(): graph.add_edge(n1, n2, color="#4C4C4C", label='') graph.layout() - return graph.draw(format='png', prog='dot') + #return graph.draw(format='png', prog='dot') + if not request.args: + return graph.draw(format='png', prog='dot') + else: + response.headers['Content-Disposition']='attachment;filename=graph.%s'%request.args(0) + if request.args(0) == 'dot': + return graph.string() + else: + return graph.draw(format=request.args(0), prog='dot') -def graph_model(): +def graph_model(): return dict(databases=databases, pgv=pgv) diff --git a/applications/welcome/views/appadmin.html b/applications/welcome/views/appadmin.html index a9f6dab8..95c60ecf 100644 --- a/applications/welcome/views/appadmin.html +++ b/applications/welcome/views/appadmin.html @@ -229,7 +229,21 @@ {{=T('pygraphviz library not found')}} {{elif not databases:}} {{=T("No databases in this application")}} - {{else:}} + {{else:}} +
+ + {{=T('Save model as...')}} + + + +
+
{{=IMG(_src=URL('appadmin', 'bg_graph_model'))}} {{pass}} {{pass}} From d238b5e86d41aa9a7418f4c67d331c2d377352ed Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 28 Jan 2013 20:48:02 -0600 Subject: [PATCH 08/39] Row.__delitem__ --- VERSION | 2 +- gluon/dal.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 98d8ae98..c9613f13 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.27.10.24.17 +Version 2.4.1-alpha.2+timestamp.2013.01.28.20.47.12 diff --git a/gluon/dal.py b/gluon/dal.py index b48c0bb9..622c581c 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -6670,6 +6670,8 @@ class Row(object): def __setitem__(self, key, value): setattr(self, str(key), value) + __delitem__ = delattr + __copy__ = lambda self: Row(self) __call__ = __getitem__ From 698286dbe09fa036bf19ea8b1679a14f2f996cc9 Mon Sep 17 00:00:00 2001 From: Massimo Date: Tue, 29 Jan 2013 16:25:10 -0600 Subject: [PATCH 09/39] Query/Set enhancements for client/service apps, thanks Alan --- VERSION | 2 +- gluon/dal.py | 147 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index c9613f13..66670966 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.28.20.47.12 +Version 2.4.1-alpha.2+timestamp.2013.01.29.16.24.13 diff --git a/gluon/dal.py b/gluon/dal.py index 622c581c..e20b7cb3 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -7630,6 +7630,9 @@ def index(): query = self._adapter.id_query(query) elif isinstance(query,Field): query = query!=None + elif isinstance(query, dict): + icf = query.get("ignore_common_filters") + if icf: ignore_common_filters = icf return Set(self, query, ignore_common_filters=ignore_common_filters) def commit(self): @@ -9135,8 +9138,8 @@ class Field(Expression): def as_dict(self, flat=False, sanitize=True): attrs = ("readable", "writable", "label", "default", "name", "type", "represent", "compute") - SERIALIZABLE = (int, long, basestring, dict, list, float, - tuple, bool, None.__class__) + SERIALIZABLE_TYPES = (int, long, basestring, dict, list, + float, tuple, bool, type(None)) def flatten(obj): if flat: if isinstance(obj, flatten.__class__): @@ -9146,7 +9149,7 @@ class Field(Expression): obj = str(obj).split("'")[1] except IndexError: obj = str(obj) - elif not isinstance(obj, SERIALIZABLE): + elif not isinstance(obj, SERIALIZABLE_TYPES): obj = str(obj) return obj @@ -9161,7 +9164,7 @@ class Field(Expression): else: other = v r[k] = {flatten(type(v)): filter_requires(type(v), other)} - elif flat and (not isinstance(v, SERIALIZABLE)): + elif flat and (not isinstance(v, SERIALIZABLE_TYPES)): r[k] = str(v) return r @@ -9253,7 +9256,61 @@ class Query(object): def case(self,t=1,f=0): return self.db._adapter.CASE(self,t,f) + def as_dict(self, flat=False, sanitize=True): + """Experimental stuff + This allows to return a plain dictionary with the basic + query representation. Can be used with json/xml services + for client-side db I/O + + Example: + >>> q = db.auth_user.id != 0 + >>> q.as_dict(flat=True) + {"op": "NE", "first":{"tablename": "auth_user", + "fieldname": "id"}, + "second":0} + """ + SERIALIZABLE_TYPES = (tuple, dict, list, int, long, float, + basestring, type(None), bool) + def loop(d): + newd = dict() + for k, v in d.items(): + if k in ("first", "second"): + if isinstance(v, self.__class__): + newd[k] = loop(v.__dict__) + elif isinstance(v, Field): + newd[k] = {"tablename": v._tablename, + "fieldname": v.name} + elif isinstance(v, Expression): + newd[k] = loop(v.__dict__) + elif isinstance(v, SERIALIZABLE_TYPES): + newd[k] = v + else: pass + elif k == "op": + newd[k] = v.__name__ + else: pass + return newd + + if flat: + d = loop(self.__dict__) + else: d = self.__dict__ + return d + + def as_xml(self, sanitize=True): + if have_serializers: + xml = serializers.xml + else: + raise ImportError("No xml serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return xml(d) + + def as_json(self, sanitize=True): + if have_serializers: + json = serializers.json + else: + raise ImportError("No json serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return json(d) def xorify(orderby): if not orderby: @@ -9287,6 +9344,12 @@ class Set(object): def __init__(self, db, query, ignore_common_filters = None): self.db = db self._db = db # for backward compatibility + self.dquery = None + + # if query is a dict, parse it + if isinstance(query, dict): + query = self.parse(query) + if not ignore_common_filters is None and \ use_common_filters(query) == ignore_common_filters: query = copy.copy(query) @@ -9334,6 +9397,82 @@ class Set(object): fields = db[tablename]._listify(update_fields,update=True) return db._adapter._update(tablename,self.query,fields) + def as_dict(self, flat=False, sanitize=True): + if flat: + uid = dbname = uri = None + codec = self.db._db_codec + if not sanitize: + uri, dbname, uid = (self.db._dbname, str(self.db), + self.db._db_uid) + d = {"query": self.query.as_dict(flat=flat)} + d["db"] = {"uid": uid, "codec": codec, + "name": dbname, "uri": uri} + return d + else: return self.__dict__ + + def as_xml(self, sanitize=True): + if have_serializers: + xml = serializers.xml + else: + raise ImportError("No xml serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return xml(d) + + def as_json(self, sanitize=True): + if have_serializers: + json = serializers.json + else: + raise ImportError("No json serializers available") + d = self.as_dict(flat=True, sanitize=sanitize) + return json(d) + + def parse(self, dquery): + "Experimental: Turn a dictionary into a Query object" + self.dquery = dquery + return self.build(self.dquery) + + def build(self, d): + "Experimental: see .parse()" + op, first, second = (d["op"], d["first"], + d.get("second", None)) + left = right = built = None + + if op in ("AND", "OR"): + if not (type(first), type(second)) == (dict, dict): + raise SyntaxError("Invalid AND/OR query") + if op == "AND": + built = self.build(first) & self.build(second) + else: built = self.build(first) | self.build(second) + + elif op == "NOT": + if first is None: + raise SyntaxError("Invalid NOT query") + built = ~self.build(first) + else: + # normal operation (GT, EQ, LT, ...) + for k, v in {"left": first, "right": second}.items(): + if isinstance(v, dict) and v.get("op"): + v = self.build(v) + if isinstance(v, dict) and ("tablename" in v): + v = self.db[v["tablename"]][v["fieldname"]] + if k == "left": + left = v + else: + right = v + + if op == "EQ": built = left == right + elif op == "NE": built = left != right + elif op == "GT": built = left > right + elif op == "GE": built = left >= right + elif op == "LT": built = left < right + elif op == "LE": built = left <= right + elif op == "CONTAINS": built = left.contains(right) + elif op == "BELONGS": built = left.belongs(right) + elif op == "INVERT": built = ~left + else: raise SyntaxError("Operator not supported") + + return built + def isempty(self): return not self.select(limitby=(0,1)) From 45969c7628e2f618fda3effd4ca95ce75facf61e Mon Sep 17 00:00:00 2001 From: Massimo Date: Tue, 29 Jan 2013 16:27:46 -0600 Subject: [PATCH 10/39] added note to scripts/setup-web2py-nginx-uwsgi-on-centos.sh, thanks Alan --- VERSION | 2 +- scripts/setup-web2py-nginx-uwsgi-on-centos.sh | 52 ++++++++++++++----- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/VERSION b/VERSION index 66670966..35c6106a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.29.16.24.13 +Version 2.4.1-alpha.2+timestamp.2013.01.29.16.27.18 diff --git a/scripts/setup-web2py-nginx-uwsgi-on-centos.sh b/scripts/setup-web2py-nginx-uwsgi-on-centos.sh index 0892026d..09f1b81b 100644 --- a/scripts/setup-web2py-nginx-uwsgi-on-centos.sh +++ b/scripts/setup-web2py-nginx-uwsgi-on-centos.sh @@ -1,19 +1,43 @@ #!/bin/bash -# Script for installing Web2py with Nginx and Uwsgi on Centos 5 -# Created By Hutchinson -# Modified by spametki -# License: BSD - -# It was originally posted in this web2py-users group thread: -# https://groups.google.com/forum/?fromgroups#!topic/web2py/O4c4Jfr18tM - -# There are lots of subtleties of ownership, and one has to take care -# when installing python 2.6 not to stop the systems python2.4 from working. - -# NOTE: The only thing that should need changing for -# each installation is the $BASEARCH (base architecture) of the machine. -# This is determined by doing uname -i. This is needed for the nginx installation. +# ------------------------------------------------------------------- +# Description : Installation and basic configuration of web2py, +# uWSGI, andNGINX. +# in CentOS 5.x GNU/Linux +# Usage : Copy the script in /home/username and run it as root, +# you may need to allow execution (chmod +x) +# +# WARNING: This script was modified to install compiled +# versions of Python and other packages that may be +# available at your Centos package repository. +# This change was made in order to get the latest +# stable libraries available for avoiding compatibility +# issues. +# +# It was originally posted in this web2py-users group +# thread: https://groups.google.com/forum/?fromgroups# +# !topic/web2py/O4c4Jfr18tM +# +# There are lots of subtleties of ownership, and one +# has to take care when installing python 2.6 not to +# stop the systems python2.4 from working. +# +# NOTE: The only thing that should need changing for +# each installation is the $BASEARCH (base +# architecture) +# of the machine. This is determined by doing uname -i. +# This is needed for the nginx installation. +# +# File : setup-web2py-nginx-uwsgi-on-centos.sh +# Author : Hutchinson +# Modified by : Alan Etkin +# Email : spametki@gmail.com +# Copyright : web2py +# Date : 2013-01-28 +# Disclaimers : This script is provided "as is", without warranty of +# any kind. +# Licence : BSD +# ------------------------------------------------------------------- # Retrieve base architecture BASEARCH=$(uname -i) From 8740fea7c1c10fb0c2648348eda5dbeab2e63236 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 30 Jan 2013 09:29:49 -0600 Subject: [PATCH 11/39] allow multiple left joins in grid --- VERSION | 2 +- gluon/sqlhtml.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 35c6106a..db093562 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.29.16.27.18 +Version 2.4.1-alpha.2+timestamp.2013.01.30.09.29.07 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index bf9e8ea0..90ccf722 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -1867,7 +1867,10 @@ class SQLFORM(FORM): dbset = db(query) tablenames = db._adapter.tables(dbset.query) if left is not None: - tablenames += db._adapter.tables(left) + if not isinstance(left, (list, tuple)): + left = [left] + for join in left: + tablenames += db._adapter.tables(join) tables = [db[tablename] for tablename in tablenames] if not fields: fields = reduce(lambda a, b: a + b, From 14bd35e348154fd66d8ed63b4d44e248045d52f9 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 30 Jan 2013 09:41:41 -0600 Subject: [PATCH 12/39] fixed issue 1306 rendering of width in markmin, thanks dickschrauwen --- VERSION | 2 +- gluon/contrib/markmin/markmin2html.py | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/VERSION b/VERSION index db093562..863d2fec 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2013.01.30.09.29.07 +Version 2.4.1-alpha.2+timestamp.2013.01.30.09.40.42 diff --git a/gluon/contrib/markmin/markmin2html.py b/gluon/contrib/markmin/markmin2html.py index 0f56b603..cbdcec47 100755 --- a/gluon/contrib/markmin/markmin2html.py +++ b/gluon/contrib/markmin/markmin2html.py @@ -609,7 +609,7 @@ def protolinks_simple(proto, url): #elif proto == 'embed': # NOTE: embed is a synonym to iframe now # return '%s>'%(url,class_prefix,url) elif proto == 'qr': - return 'qr code'%url + return 'qr code'%url return proto+':'+url def render(text, @@ -701,7 +701,7 @@ def render(text, '

this is an image

' >>> render('[[this is an image http://example.com left 200px]]') - '

this is an image

' + '

this is an image

' >>> render("[[Your browser doesn't support