diff --git a/Makefile b/Makefile index ccb5f5c7..f38e1bc3 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,6 @@ commit: push: hg push git push - git push mdipierro tag: git tag -l '$(S)' hg tag -l '$(S)' diff --git a/VERSION b/VERSION index c163047c..73372cfc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.4.1-alpha.2+timestamp.2012.12.28.01.21.20 +Version 2.4.1-alpha.2+timestamp.2013.01.09.16.56.34 diff --git a/anyserver.py b/anyserver.py index 473cceaa..ebf556c8 100644 --- a/anyserver.py +++ b/anyserver.py @@ -163,6 +163,22 @@ class Servers: "tcp://127.0.0.1:9996") mongrel2_handler(app, conn, debug=False) + @staticmethod + def motor(app, address, **options): + #https://github.com/rpedroso/motor + import motor + app = motor.WSGIContainer(app) + http_server = motor.HTTPServer(app) + http_server.listen(address=address[0], port=address[1]) + #http_server.start(2) + motor.IOLoop.instance().start() + + @staticmethod + def pulsar(app, address, **options): + from pulsar.apps import wsgi + sys.argv = ['anyserver.py'] + s = wsgi.WSGIServer(callable=app, bind="%s:%d" % address) + s.start() def run(servername, ip, port, softcron=True, logging=False, profiler=None): if logging: diff --git a/applications/admin/controllers/debug.py b/applications/admin/controllers/debug.py index f02b2866..97ef49f8 100644 --- a/applications/admin/controllers/debug.py +++ b/applications/admin/controllers/debug.py @@ -212,3 +212,26 @@ def toggle_breakpoint(): except Exception, e: session.flash = str(e) return response.json({'ok': ok, 'lineno': lineno}) + +def list_breakpoints(): + "Return a list of linenumbers for current breakpoints" + + breakpoints = [] + ok = None + try: + filename = os.path.join(request.env['applications_parent'], + 'applications', request.vars.filename) + # normalize path name: replace slashes, references, etc... + filename = os.path.normpath(os.path.normcase(filename)) + for bp in qdb_debugger.do_list_breakpoint(): + no, bp_filename, bp_lineno, temporary, enabled, hits, cond = bp + # normalize path name: replace slashes, references, etc... + bp_filename = os.path.normpath(os.path.normcase(bp_filename)) + if filename == bp_filename: + breakpoints.append(bp_lineno) + ok = True + except Exception, e: + session.flash = str(e) + ok = False + return response.json({'ok': ok, 'breakpoints': breakpoints}) + diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 4004402a..ba67a9f5 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -1493,7 +1493,9 @@ def errors(): else: for item in request.vars: - if item[:7] == 'delete_': + # delete_all} rows doesn't contain any ticket + # Remove anything else as requested + if item[:7] == 'delete_' and (not item == "delete_all}"): os.unlink(apath('%s/errors/%s' % (app, item[7:]), r=request)) func = lambda p: os.stat(apath('%s/errors/%s' % (app, p), r=request)).st_mtime diff --git a/applications/admin/static/js/ajax_editor.js b/applications/admin/static/js/ajax_editor.js index b24b2fd0..44d5de7d 100644 --- a/applications/admin/static/js/ajax_editor.js +++ b/applications/admin/static/js/ajax_editor.js @@ -148,8 +148,12 @@ function getSelectionRange() { return sel; } -function doToggleBreakpoint(filename, url) { - var sel = getSelectionRange(); +function doToggleBreakpoint(filename, url, sel) { + if (sel==null) { + // use cursor position to determine the breakpoint line + // (gutter already tell us the selected line) + sel = getSelectionRange(); + } var dataForPost = prepareMultiPartPOST(new Array( prepareDataForSave('filename', filename), prepareDataForSave('sel_start', sel["start"]), @@ -199,6 +203,43 @@ function doToggleBreakpoint(filename, url) { return false; } +// on load, update all breakpoints markers: +function doListBreakpoints(filename, url) { + var dataForPost = prepareMultiPartPOST(new Array( + prepareDataForSave('filename', filename) + )); + jQuery.ajax({ + type: "POST", + contentType: 'multipart/form-data;boundary="'+dataForPost[1]+'"', + url: url, + dataType: "json", + data: dataForPost[0], + timeout: 5000, + beforeSend: function(xhr) { + xhr.setRequestHeader('web2py-component-location', + document.location); + xhr.setRequestHeader('web2py-component-element', + 'doListBreakpoints');}, + success: function(json,text,xhr){ + try { + if (json.error) { + window.location.href=json.redirect; + } else { + if (window.mirror) { + for (i in json.breakpoints) { + lineno = json.breakpoints[i]; + // mark the breakpoint if ok=True + editor.setMarker(lineno-1, + "● %N%"); + } + } + } + } catch(e) { on_error(); } + }, + error: function(json) { on_error(); } + }); + return false; +} function keepalive(url) { jQuery.ajax({ diff --git a/applications/admin/static/js/web2py.js b/applications/admin/static/js/web2py.js index 2c58fbf0..e3076795 100644 --- a/applications/admin/static/js/web2py.js +++ b/applications/admin/static/js/web2py.js @@ -44,12 +44,25 @@ function web2py_event_handlers() { doc.on('keyup', 'input.double, input.decimal', function(){this.value=this.value.reverse().replace(/[^0-9\-\.,]|[\-](?=.)|[\.,](?=[0-9]*[\.,])/g,'').reverse();}); var confirm_message = (typeof w2p_ajax_confirm_message != 'undefined') ? w2p_ajax_confirm_message : "Are you sure you want to delete this object?"; doc.on('click', "input[type='checkbox'].delete", function(){if(this.checked) if(!confirm(confirm_message)) this.checked=false;}); + doc.ajaxSuccess(function(e, xhr) { var redirect=xhr.getResponseHeader('web2py-redirect-location'); - if (redirect != null) { + var command=xhr.getResponseHeader('web2py-component-command'); + var flash=xhr.getResponseHeader('web2py-component-flash'); + if (redirect !== null) { window.location = redirect; }; + if(command !== null){ + eval(decodeURIComponent(command)); + } + if(flash) { + jQuery('.flash') + .html(decodeURIComponent(flash)) + .append('×') + .slideDown(); + } }); + doc.ajaxError(function(e, xhr, settings, exception) { doc.off('click', '.flash') switch(xhr.status){ @@ -98,8 +111,6 @@ function web2py_ajax_page(method, action, data, target) { 'complete':function(xhr,text){ var html=xhr.responseText; var content=xhr.getResponseHeader('web2py-component-content'); - var command=xhr.getResponseHeader('web2py-component-command'); - var flash=xhr.getResponseHeader('web2py-component-flash'); var t = jQuery('#'+target); if(content=='prepend') t.prepend(html); else if(content=='append') t.append(html); @@ -107,14 +118,6 @@ function web2py_ajax_page(method, action, data, target) { web2py_trap_form(action,target); web2py_trap_link(target); web2py_ajax_init('#'+target); - if(command) - eval(decodeURIComponent(command)); - if(flash) { - jQuery('.flash') - .html(decodeURIComponent(flash)) - .append('×') - .slideDown(); - } } }); } diff --git a/applications/admin/views/default/edit.html b/applications/admin/views/default/edit.html index 19c416e4..cb1b6035 100644 --- a/applications/admin/views/default/edit.html +++ b/applications/admin/views/default/edit.html @@ -25,6 +25,13 @@ + {{elif TEXT_EDITOR == 'ace':}} @@ -103,7 +110,7 @@ jQuery(document).ready(function(){ {{if filetype=='python':}} {{=A(SPAN(T('toggle breakpoint')), _value="breakpoint", _name="breakpoint", - _onclick="return doToggleBreakpoint('%s','%s://%s%s');" % (filename, + _onclick="return doToggleBreakpoint('%s','%s://%s%s',null);" % (filename, request.env['wsgi_url_scheme'], request.env['http_host'], URL(c='debug', f='toggle_breakpoint')), _class="button special btn btn-inverse")}} @@ -202,7 +209,13 @@ jQuery(document).ready(function(){ autofocus: true, onCursorActivity: function() { editor.setLineClass(hlLine, null, null); - hlLine = editor.setLineClass(editor.getCursor().line, null, "activeline");} + hlLine = editor.setLineClass(editor.getCursor().line, null, "activeline");}, + onGutterClick: function(cm, n) { + sel = {start: n, end: n, data: ''}; + doToggleBreakpoint({{=XML("'%s','%s://%s%s',sel" % (filename, + request.env['wsgi_url_scheme'], request.env['http_host'], + URL(c='debug', f='toggle_breakpoint')))}}); + } }; var editor = CodeMirror.fromTextArea( document.getElementById("body"),cm_opts); @@ -293,4 +306,4 @@ window.onload = function() { {{pass}} - \ No newline at end of file + diff --git a/applications/examples/static/js/web2py.js b/applications/examples/static/js/web2py.js index 2c58fbf0..e3076795 100644 --- a/applications/examples/static/js/web2py.js +++ b/applications/examples/static/js/web2py.js @@ -44,12 +44,25 @@ function web2py_event_handlers() { doc.on('keyup', 'input.double, input.decimal', function(){this.value=this.value.reverse().replace(/[^0-9\-\.,]|[\-](?=.)|[\.,](?=[0-9]*[\.,])/g,'').reverse();}); var confirm_message = (typeof w2p_ajax_confirm_message != 'undefined') ? w2p_ajax_confirm_message : "Are you sure you want to delete this object?"; doc.on('click', "input[type='checkbox'].delete", function(){if(this.checked) if(!confirm(confirm_message)) this.checked=false;}); + doc.ajaxSuccess(function(e, xhr) { var redirect=xhr.getResponseHeader('web2py-redirect-location'); - if (redirect != null) { + var command=xhr.getResponseHeader('web2py-component-command'); + var flash=xhr.getResponseHeader('web2py-component-flash'); + if (redirect !== null) { window.location = redirect; }; + if(command !== null){ + eval(decodeURIComponent(command)); + } + if(flash) { + jQuery('.flash') + .html(decodeURIComponent(flash)) + .append('×') + .slideDown(); + } }); + doc.ajaxError(function(e, xhr, settings, exception) { doc.off('click', '.flash') switch(xhr.status){ @@ -98,8 +111,6 @@ function web2py_ajax_page(method, action, data, target) { 'complete':function(xhr,text){ var html=xhr.responseText; var content=xhr.getResponseHeader('web2py-component-content'); - var command=xhr.getResponseHeader('web2py-component-command'); - var flash=xhr.getResponseHeader('web2py-component-flash'); var t = jQuery('#'+target); if(content=='prepend') t.prepend(html); else if(content=='append') t.append(html); @@ -107,14 +118,6 @@ function web2py_ajax_page(method, action, data, target) { web2py_trap_form(action,target); web2py_trap_link(target); web2py_ajax_init('#'+target); - if(command) - eval(decodeURIComponent(command)); - if(flash) { - jQuery('.flash') - .html(decodeURIComponent(flash)) - .append('×') - .slideDown(); - } } }); } diff --git a/applications/welcome/static/js/web2py.js b/applications/welcome/static/js/web2py.js index 2c58fbf0..e3076795 100644 --- a/applications/welcome/static/js/web2py.js +++ b/applications/welcome/static/js/web2py.js @@ -44,12 +44,25 @@ function web2py_event_handlers() { doc.on('keyup', 'input.double, input.decimal', function(){this.value=this.value.reverse().replace(/[^0-9\-\.,]|[\-](?=.)|[\.,](?=[0-9]*[\.,])/g,'').reverse();}); var confirm_message = (typeof w2p_ajax_confirm_message != 'undefined') ? w2p_ajax_confirm_message : "Are you sure you want to delete this object?"; doc.on('click', "input[type='checkbox'].delete", function(){if(this.checked) if(!confirm(confirm_message)) this.checked=false;}); + doc.ajaxSuccess(function(e, xhr) { var redirect=xhr.getResponseHeader('web2py-redirect-location'); - if (redirect != null) { + var command=xhr.getResponseHeader('web2py-component-command'); + var flash=xhr.getResponseHeader('web2py-component-flash'); + if (redirect !== null) { window.location = redirect; }; + if(command !== null){ + eval(decodeURIComponent(command)); + } + if(flash) { + jQuery('.flash') + .html(decodeURIComponent(flash)) + .append('×') + .slideDown(); + } }); + doc.ajaxError(function(e, xhr, settings, exception) { doc.off('click', '.flash') switch(xhr.status){ @@ -98,8 +111,6 @@ function web2py_ajax_page(method, action, data, target) { 'complete':function(xhr,text){ var html=xhr.responseText; var content=xhr.getResponseHeader('web2py-component-content'); - var command=xhr.getResponseHeader('web2py-component-command'); - var flash=xhr.getResponseHeader('web2py-component-flash'); var t = jQuery('#'+target); if(content=='prepend') t.prepend(html); else if(content=='append') t.append(html); @@ -107,14 +118,6 @@ function web2py_ajax_page(method, action, data, target) { web2py_trap_form(action,target); web2py_trap_link(target); web2py_ajax_init('#'+target); - if(command) - eval(decodeURIComponent(command)); - if(flash) { - jQuery('.flash') - .html(decodeURIComponent(flash)) - .append('×') - .slideDown(); - } } }); } diff --git a/applications/welcome/views/default/user.html b/applications/welcome/views/default/user.html index 475be5b3..b4bd0a15 100644 --- a/applications/welcome/views/default/user.html +++ b/applications/welcome/views/default/user.html @@ -4,7 +4,7 @@ {{ if request.args(0)=='login': if not 'register' in auth.settings.actions_disabled: - form.add_button(T('Register'),URL(args='register'),_class='btn') + form.add_button(T('Register'),URL(args='register', vars={'_next': request.vars._next} if request.vars._next else None),_class='btn') pass if not 'request_reset_password' in auth.settings.actions_disabled: form.add_button(T('Lost Password'),URL(args='request_reset_password'),_class='btn') diff --git a/gluon/__init__.py b/gluon/__init__.py index 3e9801f7..da8d5587 100644 --- a/gluon/__init__.py +++ b/gluon/__init__.py @@ -10,7 +10,7 @@ Web2Py framework modules ======================== """ -__all__ = ['A', 'B', 'BEAUTIFY', 'BODY', 'BR', 'CAT', 'CENTER', 'CLEANUP', 'CODE', 'CRYPT', 'DAL', 'DIV', 'EM', 'EMBED', 'FIELDSET', 'FORM', 'Field', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'HTTP', 'I', 'IFRAME', 'IMG', 'INPUT', 'IS_ALPHANUMERIC', 'IS_DATE', 'IS_DATETIME', 'IS_DATETIME_IN_RANGE', 'IS_DATE_IN_RANGE', 'IS_DECIMAL_IN_RANGE', 'IS_EMAIL', 'IS_EMPTY_OR', 'IS_EQUAL_TO', 'IS_EXPR', 'IS_FLOAT_IN_RANGE', 'IS_IMAGE', 'IS_INT_IN_RANGE', 'IS_IN_DB', 'IS_IN_SET', 'IS_IPV4', 'IS_LENGTH', 'IS_LIST_OF', 'IS_LOWER', 'IS_MATCH', 'IS_NOT_EMPTY', 'IS_NOT_IN_DB', 'IS_NULL_OR', 'IS_SLUG', 'IS_STRONG', 'IS_TIME', 'IS_UPLOAD_FILENAME', 'IS_UPPER', 'IS_URL', 'LABEL', 'LEGEND', 'LI', 'LINK', 'LOAD', 'MARKMIN', 'MENU', 'META', 'OBJECT', 'OL', 'ON', 'OPTGROUP', 'OPTION', 'P', 'PRE', 'SCRIPT', 'SELECT', 'SPAN', 'SQLFORM', 'SQLTABLE', 'STRONG', 'STYLE', 'TABLE', 'TAG', 'TBODY', 'TD', 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'UL', 'URL', 'XHTML', 'XML', 'redirect', 'current', 'embed64'] +__all__ = ['A', 'B', 'BEAUTIFY', 'BODY', 'BR', 'CAT', 'CENTER', 'CLEANUP', 'CODE', 'CRYPT', 'DAL', 'DIV', 'EM', 'EMBED', 'FIELDSET', 'FORM', 'Field', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'HTTP', 'I', 'IFRAME', 'IMG', 'INPUT', 'IS_ALPHANUMERIC', 'IS_DATE', 'IS_DATETIME', 'IS_DATETIME_IN_RANGE', 'IS_DATE_IN_RANGE', 'IS_DECIMAL_IN_RANGE', 'IS_EMAIL', 'IS_EMPTY_OR', 'IS_EQUAL_TO', 'IS_EXPR', 'IS_FLOAT_IN_RANGE', 'IS_IMAGE', 'IS_JSON', 'IS_INT_IN_RANGE', 'IS_IN_DB', 'IS_IN_SET', 'IS_IPV4', 'IS_LENGTH', 'IS_LIST_OF', 'IS_LOWER', 'IS_MATCH', 'IS_NOT_EMPTY', 'IS_NOT_IN_DB', 'IS_NULL_OR', 'IS_SLUG', 'IS_STRONG', 'IS_TIME', 'IS_UPLOAD_FILENAME', 'IS_UPPER', 'IS_URL', 'LABEL', 'LEGEND', 'LI', 'LINK', 'LOAD', 'MARKMIN', 'MENU', 'META', 'OBJECT', 'OL', 'ON', 'OPTGROUP', 'OPTION', 'P', 'PRE', 'SCRIPT', 'SELECT', 'SPAN', 'SQLFORM', 'SQLTABLE', 'STRONG', 'STYLE', 'TABLE', 'TAG', 'TBODY', 'TD', 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'UL', 'URL', 'XHTML', 'XML', 'redirect', 'current', 'embed64'] from globals import current from html import * diff --git a/gluon/contrib/login_methods/oauth20_account.py b/gluon/contrib/login_methods/oauth20_account.py index 0ed3adb2..e5c51c76 100644 --- a/gluon/contrib/login_methods/oauth20_account.py +++ b/gluon/contrib/login_methods/oauth20_account.py @@ -121,13 +121,13 @@ server for requests. It can be used for the optional"scope" parameters for Face """ # Create an OpenerDirector with support # for Basic HTTP Authentication... - - auth_handler = urllib2.HTTPBasicAuthHandler() - auth_handler.add_password(None, - uri, - self.client_id, - self.client_secret) - opener = urllib2.build_opener(auth_handler) + password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(realm=None, + uri=uri, + user=self.client_id, + passwd=self.client_secret) + handler = urllib2.HTTPBasicAuthHandler(password_mgr) + opener = urllib2.build_opener(handler) return opener def accessToken(self): diff --git a/gluon/contrib/login_methods/openid_auth.py b/gluon/contrib/login_methods/openid_auth.py index 4e977030..0c763e81 100644 --- a/gluon/contrib/login_methods/openid_auth.py +++ b/gluon/contrib/login_methods/openid_auth.py @@ -129,18 +129,21 @@ class OpenIDAuth(object): def _define_alt_login_table(self): """ Define the OpenID login table. - Note: type is what I used for our project. We're going to support 'fackbook' and - 'plurk' alternate login methods. Otherwise it's always 'openid' and you + Note: oidtype is what I used for our project. + We're going to support 'fackbook' and + 'plurk' alternate login methods. + Otherwise it's always 'openid' and you may not need it. This should be easy to changed. (Just remove the field of "type" and remove the - "and db.alt_logins.type == type_" in _find_matched_openid function) + "and db.alt_logins.oidtype == type_" + in _find_matched_openid function) """ db = self.db table = db.define_table( self.table_alt_logins_name, Field('username', length=512, default=''), - Field('type', length=128, default='openid', readable=False), - Field('user', self.table_user, readable=False), + Field('oidtype', length=128, default='openid', readable=False), + Field('oiduser', self.table_user, readable=False), ) table.username.requires = IS_NOT_IN_DB(db, table.username) self.table_alt_logins = table @@ -213,7 +216,7 @@ class OpenIDAuth(object): # Get existed OpenID user user = db( - self.table_user.id == alt_login.user).select().first() + self.table_user.id == alt_login.oiduser).select().first() if user: if current.session.w2popenid: del(current.session.w2popenid) @@ -230,7 +233,7 @@ class OpenIDAuth(object): Get the matched OpenID for given """ query = ( - (db.alt_logins.username == oid) & (db.alt_logins.type == type_)) + (db.alt_logins.username == oid) & (db.alt_logins.oidtype == type_)) alt_login = db(query).select().first() # Get the OpenID record return alt_login @@ -239,7 +242,7 @@ class OpenIDAuth(object): Associate the user logged in with given OpenID """ # print "[DB] %s authenticated" % oid - self.db.alt_logins.insert(username=oid, user=user.id) + self.db.alt_logins.insert(username=oid, oiduser=user.id) def _form_with_notification(self): """ @@ -400,7 +403,7 @@ width: 400px; if 'delete_openid' in request.vars: self.remove_openid(request.vars.delete_openid) - query = self.db.alt_logins.user == self.auth.user.id + query = self.db.alt_logins.oiduser == self.auth.user.id alt_logins = self.db(query).select() l = [] for alt_login in alt_logins: @@ -529,7 +532,7 @@ class Web2pyStore(OpenIDStore): self.database.define_table(self.table_oid_nonces_name, Field('server_url', 'string', length=2047, required=True), - Field('timestamp', + Field('itimestamp', 'integer', required=True), Field('salt', 'string', length=40, required=True) @@ -591,12 +594,12 @@ class Web2pyStore(OpenIDStore): db = self.database if abs(timestamp - time.time()) > nonce.SKEW: return False - query = (db.oid_nonces.server_url == server_url) & (db.oid_nonces.timestamp == timestamp) & (db.oid_nonces.salt == salt) + query = (db.oid_nonces.server_url == server_url) & (db.oid_nonces.itimestamp == timestamp) & (db.oid_nonces.salt == salt) if db(query).count() > 0: return False else: db.oid_nonces.insert(server_url=server_url, - timestamp=timestamp, + itimestamp=timestamp, salt=salt) return True @@ -628,7 +631,7 @@ class Web2pyStore(OpenIDStore): """ db = self.database - query = (db.oid_nonces.timestamp < time.time() - nonce.SKEW) + query = (db.oid_nonces.itimestamp < time.time() - nonce.SKEW) return db(query).delete() def cleanupAssociations(self): diff --git a/gluon/dal.py b/gluon/dal.py index b4df67e9..2e7a1569 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -621,6 +621,7 @@ class BaseAdapter(ConnectionPool): 'boolean': 'CHAR(1)', 'string': 'CHAR(%(length)s)', 'text': 'TEXT', + 'json': 'TEXT', 'password': 'CHAR(%(length)s)', 'blob': 'BLOB', 'upload': 'CHAR(%(length)s)', @@ -1160,8 +1161,8 @@ class BaseAdapter(ConnectionPool): logfile.write('success!\n') def _insert(self, table, fields): - keys = ','.join(f.name for f,v in fields) - values = ','.join(self.expand(v,f.type) for f,v in fields) + keys = ','.join(f.name for f, v in fields) + values = ','.join(self.expand(v, f.type) for f, v in fields) return 'INSERT INTO %s(%s) VALUES (%s);' % (table, keys, values) def insert(self, table, fields): @@ -1225,7 +1226,7 @@ class BaseAdapter(ConnectionPool): self.expand('%'+second, 'string')) def CONTAINS(self, first, second): - if first.type in ('string', 'text'): + if first.type in ('string', 'text', 'json'): key = '%'+str(second).replace('%','%%')+'%' elif first.type.startswith('list:'): key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%' @@ -1291,6 +1292,8 @@ class BaseAdapter(ConnectionPool): return '%s AS %s' % (self.expand(first), second) def ON(self, first, second): + if use_common_filters(second): + second = self.common_filter(second,[first._tablename]) return '%s ON %s' % (self.expand(first), self.expand(second)) def INVERT(self, first): @@ -1466,9 +1469,6 @@ class BaseAdapter(ConnectionPool): if not tablename in tablenames: tablenames.append(tablename) - if use_common_filters(query): - query = self.common_filter(query,tablenames) - if len(tablenames) < 1: raise SyntaxError('Set: no tables selected') self._colnames = map(self.expand, fields) @@ -1477,10 +1477,6 @@ class BaseAdapter(ConnectionPool): field = field.st_astext() return self.expand(field) sql_f = ', '.join(map(geoexpand, fields)) - if query: - sql_w = ' WHERE ' + self.expand(query) - else: - sql_w = '' sql_o = '' sql_s = '' left = args_get('left', False) @@ -1530,6 +1526,13 @@ class BaseAdapter(ConnectionPool): important_tablenames = joint + joinont + tables_to_merge.keys() excluded = [t for t in tablenames if not t in important_tablenames ] + else: + excluded = tablenames + + if use_common_filters(query): + query = self.common_filter(query,excluded) + sql_w = ' WHERE ' + self.expand(query) if query else '' + def alias(t): return str(self.db[t]) if inner_join and not left: @@ -1717,7 +1720,7 @@ class BaseAdapter(ConnectionPool): obj = obj() if isinstance(fieldtype, SQLCustomType): value = fieldtype.encoder(obj) - if fieldtype.type in ('string','text'): + if fieldtype.type in ('string','text', 'json'): return self.adapt(value) return value if isinstance(obj, (Expression, Field)): @@ -1731,11 +1734,12 @@ class BaseAdapter(ConnectionPool): obj = map(str,obj) else: obj = map(int,obj) - if isinstance(obj, (list, tuple)): + # we don't want to bar_encode json objects + if isinstance(obj, (list, tuple)) and (not fieldtype == "json"): obj = bar_encode(obj) if obj is None: return 'NULL' - if obj == '' and not fieldtype[:2] in ['st', 'te', 'pa', 'up']: + if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']: return 'NULL' r = self.represent_exceptions(obj, fieldtype) if not r is None: @@ -1778,6 +1782,16 @@ class BaseAdapter(ConnectionPool): obj = obj.isoformat()[:10] else: obj = str(obj) + elif (fieldtype == 'json'): + if not isinstance(obj, basestring): + if have_serializers: + obj = serializers.json(obj) + else: + try: + import json as simplejson + except ImportError: + import gluon.contrib.simplejson as simplejson + obj = simplejson.dumps(items) if not isinstance(obj,bytes): obj = bytes(obj) try: @@ -1909,6 +1923,20 @@ class BaseAdapter(ConnectionPool): def parse_double(self, value, field_type): return float(value) + def parse_json(self, value, field_type): + if isinstance(value, basestring): + if isinstance(value, unicode): + value = value.encode('utf-8') + if have_serializers: + value = serializers.loads_json(value) + else: + try: + import json as simplejson + except ImportError: + import gluon.contrib.simplejson as simplejson + value = simplejson.loads(value) + return value + def build_parsemap(self): self.parsemap = { 'id':self.parse_id, @@ -1923,6 +1951,7 @@ class BaseAdapter(ConnectionPool): 'datetime':self.parse_datetime, 'blob':self.parse_blob, 'decimal':self.parse_decimal, + 'json':self.parse_json, 'list:integer':self.parse_list_integers, 'list:reference':self.parse_list_references, 'list:string':self.parse_list_strings, @@ -2305,6 +2334,7 @@ class MySQLAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'LONGTEXT', + 'json': 'LONGTEXT', 'password': 'VARCHAR(%(length)s)', 'blob': 'LONGBLOB', 'upload': 'VARCHAR(%(length)s)', @@ -2417,6 +2447,7 @@ class PostgreSQLAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'TEXT', + 'json': 'TEXT', 'password': 'VARCHAR(%(length)s)', 'blob': 'BYTEA', 'upload': 'VARCHAR(%(length)s)', @@ -2458,7 +2489,7 @@ class PostgreSQLAdapter(BaseAdapter): def ADD(self, first, second): t = first.type - if t in ('text','string','password','upload','blob'): + if t in ('text','string','password', 'json', 'upload','blob'): return '(%s || %s)' % (self.expand(first), self.expand(second, t)) else: return '(%s + %s)' % (self.expand(first), self.expand(second, t)) @@ -2559,7 +2590,7 @@ class PostgreSQLAdapter(BaseAdapter): self.expand('%'+second,'string')) def CONTAINS(self,first,second): - if first.type in ('string','text'): + if first.type in ('string','text', 'json'): key = '%'+str(second).replace('%','%%')+'%' elif first.type.startswith('list:'): key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%' @@ -2580,12 +2611,17 @@ class PostgreSQLAdapter(BaseAdapter): """ return 'ST_AsText(%s)' %(self.expand(first)) -# def ST_CONTAINED(self, first, second): -# """ -# non-standard function based on ST_Contains with parameters reversed -# http://postgis.org/docs/ST_Contains.html -# """ -# return 'ST_Contains(%s,%s)' % (self.expand(second, first.type), self.expand(first)) + def ST_X(self, first): + """ + http://postgis.org/docs/ST_X.html + """ + return 'ST_X(%s)' %(self.expand(first)) + + def ST_Y(self, first): + """ + http://postgis.org/docs/ST_Y.html + """ + return 'ST_Y(%s)' %(self.expand(first)) def ST_CONTAINS(self, first, second): """ @@ -2659,6 +2695,7 @@ class NewPostgreSQLAdapter(PostgreSQLAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'TEXT', + 'json': 'TEXT', 'password': 'VARCHAR(%(length)s)', 'blob': 'BYTEA', 'upload': 'VARCHAR(%(length)s)', @@ -2758,6 +2795,7 @@ class OracleAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR2(%(length)s)', 'text': 'CLOB', + 'json': 'CLOB', 'password': 'VARCHAR2(%(length)s)', 'blob': 'CLOB', 'upload': 'VARCHAR2(%(length)s)', @@ -2928,6 +2966,7 @@ class MSSQLAdapter(BaseAdapter): 'boolean': 'BIT', 'string': 'VARCHAR(%(length)s)', 'text': 'TEXT', + 'json': 'TEXT', 'password': 'VARCHAR(%(length)s)', 'blob': 'IMAGE', 'upload': 'VARCHAR(%(length)s)', @@ -3141,6 +3180,7 @@ class MSSQL2Adapter(MSSQLAdapter): 'boolean': 'CHAR(1)', 'string': 'NVARCHAR(%(length)s)', 'text': 'NTEXT', + 'json': 'NTEXT', 'password': 'NVARCHAR(%(length)s)', 'blob': 'IMAGE', 'upload': 'NVARCHAR(%(length)s)', @@ -3165,7 +3205,7 @@ class MSSQL2Adapter(MSSQLAdapter): def represent(self, obj, fieldtype): value = BaseAdapter.represent(self, obj, fieldtype) - if fieldtype in ('string','text') and value[:1]=="'": + if fieldtype in ('string','text', 'json') and value[:1]=="'": value = 'N'+value return value @@ -3179,6 +3219,7 @@ class SybaseAdapter(MSSQLAdapter): 'boolean': 'BIT', 'string': 'CHAR VARYING(%(length)s)', 'text': 'TEXT', + 'json': 'TEXT', 'password': 'CHAR VARYING(%(length)s)', 'blob': 'IMAGE', 'upload': 'CHAR VARYING(%(length)s)', @@ -3273,6 +3314,7 @@ class FireBirdAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'BLOB SUB_TYPE 1', + 'json': 'BLOB SUB_TYPE 1', 'password': 'VARCHAR(%(length)s)', 'blob': 'BLOB SUB_TYPE 0', 'upload': 'VARCHAR(%(length)s)', @@ -3441,6 +3483,7 @@ class InformixAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'BLOB SUB_TYPE 1', + 'json': 'BLOB SUB_TYPE 1', 'password': 'VARCHAR(%(length)s)', 'blob': 'BLOB SUB_TYPE 0', 'upload': 'VARCHAR(%(length)s)', @@ -3568,6 +3611,7 @@ class DB2Adapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'CLOB', + 'json': 'CLOB', 'password': 'VARCHAR(%(length)s)', 'blob': 'BLOB', 'upload': 'VARCHAR(%(length)s)', @@ -3653,6 +3697,7 @@ class TeradataAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'CLOB', + 'json': 'CLOB', 'password': 'VARCHAR(%(length)s)', 'blob': 'BLOB', 'upload': 'VARCHAR(%(length)s)', @@ -3719,6 +3764,7 @@ class IngresAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'CLOB', + 'json': 'CLOB', 'password': 'VARCHAR(%(length)s)', ## Not sure what this contains utf8 or nvarchar. Or even bytes? 'blob': 'BLOB', 'upload': 'VARCHAR(%(length)s)', ## FIXME utf8 or nvarchar... or blob? what is this type? @@ -3821,6 +3867,7 @@ class IngresUnicodeAdapter(IngresAdapter): 'boolean': 'CHAR(1)', 'string': 'NVARCHAR(%(length)s)', 'text': 'NCLOB', + 'json': 'NCLOB', 'password': 'NVARCHAR(%(length)s)', ## Not sure what this contains utf8 or nvarchar. Or even bytes? 'blob': 'BLOB', 'upload': 'VARCHAR(%(length)s)', ## FIXME utf8 or nvarchar... or blob? what is this type? @@ -3851,6 +3898,7 @@ class SAPDBAdapter(BaseAdapter): 'boolean': 'CHAR(1)', 'string': 'VARCHAR(%(length)s)', 'text': 'LONG', + 'json': 'LONG', 'password': 'VARCHAR(%(length)s)', 'blob': 'LONG', 'upload': 'VARCHAR(%(length)s)', @@ -4152,7 +4200,7 @@ class NoSQLAdapter(BaseAdapter): if not isinstance(obj, (list, tuple)): obj = [obj] if obj == '' and not \ - (is_string and fieldtype[:2] in ['st','te','pa','up']): + (is_string and fieldtype[:2] in ['st','te', 'pa','up']): return None if not obj is None: if isinstance(obj, list) and not is_list: @@ -4195,6 +4243,17 @@ class NoSQLAdapter(BaseAdapter): obj = datetime.datetime(y, m, d, h, mi, s) elif fieldtype == 'blob': pass + elif fieldtype == 'json': + if isinstance(obj, basestring): + obj = self.to_unicode(obj) + if have_serializers: + obj = serializers.loads_json(obj) + else: + try: + import json as simplejson + except ImportError: + import gluon.contrib.simplejson as simplejson + obj = simplejson.loads(obj) elif is_string and field_is_type('list:string'): return map(self.to_unicode,obj) elif is_list: @@ -4302,6 +4361,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): 'boolean': gae.BooleanProperty, 'string': (lambda: gae.StringProperty(multiline=True)), 'text': gae.TextProperty, + 'json': gae.TextProperty, 'password': gae.StringProperty, 'blob': gae.BlobProperty, 'upload': gae.StringProperty, @@ -4381,7 +4441,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): def expand(self,expression,field_type=None): if isinstance(expression,Field): - if expression.type in ('text','blob'): + if expression.type in ('text', 'blob', 'json'): raise SyntaxError('AppEngine does not index by: %s' % expression.type) return expression.name elif isinstance(expression, (Expression, Query)): @@ -4520,7 +4580,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter): elif args_get('projection') == True: projection = [] for f in fields: - if f.type in ['text', 'blob']: + if f.type in ['text', 'blob', 'json']: raise SyntaxError( "text and blob field types not allowed in projection queries") else: @@ -4701,6 +4761,7 @@ class CouchDBAdapter(NoSQLAdapter): 'boolean': bool, 'string': str, 'text': str, + 'json': str, 'password': str, 'blob': str, 'upload': str, @@ -4900,6 +4961,7 @@ class MongoDBAdapter(NoSQLAdapter): 'boolean': bool, 'string': str, 'text': str, + 'json': str, 'password': str, 'blob': str, 'upload': str, @@ -4917,170 +4979,205 @@ class MongoDBAdapter(NoSQLAdapter): 'list:reference': list, } + error_messages = {"javascript_needed": "This must yet be replaced" + + " with javascript in order to work."} + def __init__(self,db,uri='mongodb://127.0.0.1:5984/db', - pool_size=0,folder=None,db_codec ='UTF-8', + pool_size=0, folder=None, db_codec ='UTF-8', credential_decoder=IDENTITY, driver_args={}, adapter_args={}, do_connect=True): + self.db = db self.uri = uri if do_connect: self.find_driver(adapter_args) + import random + from bson.objectid import ObjectId + from bson.son import SON + import pymongo.uri_parser + + m = pymongo.uri_parser.parse_uri(uri) + + self.SON = SON + self.ObjectId = ObjectId + self.random = random - m=None - try: - #Since version 2 - import pymongo.uri_parser - m = pymongo.uri_parser.parse_uri(uri) - except ImportError: - try: - #before version 2 of pymongo - import pymongo.connection - m = pymongo.connection._parse_uri(uri) - except ImportError: - raise ImportError("Uriparser for mongodb is not available") - except: - raise SyntaxError("This type of uri is not supported by the mongodb uri parser") self.dbengine = 'mongodb' self.folder = folder db['_lastsql'] = '' self.db_codec = 'UTF-8' self.pool_size = pool_size - #this is the minimum amount of replicates that it should wait for on insert/update + #this is the minimum amount of replicates that it should wait + # for on insert/update self.minimumreplication = adapter_args.get('minimumreplication',0) - #by default alle insert and selects are performand asynchronous, but now the default is - #synchronous, except when overruled by either this default or function parameter + # by default all inserts and selects are performand asynchronous, + # but now the default is + # synchronous, except when overruled by either this default or + # function parameter self.safe = adapter_args.get('safe',True) - if isinstance(m,tuple): m = {"database" : m[1]} if m.get('database')==None: raise SyntaxError("Database is required!") def connector(uri=self.uri,m=m): try: - return self.driver.Connection(uri)[m.get('database')] + # Connection() is deprecated + if hasattr(self.driver, "MongoClient"): + Connection = self.driver.MongoClient + else: + Connection = self.driver.Connection + return Connection(uri)[m.get('database')] except self.driver.errors.ConnectionFailure: inst = sys.exc_info()[1] - raise SyntaxError("The connection to " + uri + " could not be made") - except Exception: - inst = sys.exc_info()[1] - if inst == "cannot specify database without a username and password": - raise SyntaxError("You are probebly running version 1.1 of pymongo which contains a bug which requires authentication. Update your pymongo.") - else: - raise SyntaxError("This is not an official Mongodb uri (http://www.mongodb.org/display/DOCS/Connections) Error : %s" % inst) + raise SyntaxError("The connection to " + + uri + " could not be made") + self.reconnect(connector,cursor=False) + def object_id(self, arg=None): + """ Convert input to a valid Mongodb ObjectId instance + + self.object_id("") -> ObjectId (not unique) instance """ + if not arg: + arg = 0 + if isinstance(arg, basestring): + # we assume an integer as default input + rawhex = len(arg.replace("0x", "").replace("L", "")) == 24 + if arg.isdigit() and (not rawhex): + arg = int(arg) + elif arg == "": + arg = int("0x%sL" % \ + "".join([self.random.choice("0123456789abcdef") \ + for x in range(24)]), 0) + elif arg.isalnum(): + if not arg.startswith("0x"): + arg = "0x%s" % arg + try: + arg = int(arg, 0) + except ValueError, e: + raise ValueError( + "invalid objectid argument string: %s" % e) + else: + raise ValueError("Invalid objectid argument string. " + + "Requires an integer or base 16 value") + elif isinstance(arg, self.ObjectId): + return arg + if not isinstance(arg, (int, long)): + raise TypeError("object_id argument must be of type " + + "ObjectId or an objectid representable integer") + if arg == 0: + hexvalue = "".zfill(24) + else: + hexvalue = hex(arg)[2:].replace("L", "") + return self.ObjectId(hexvalue) + def represent(self, obj, fieldtype): value = NoSQLAdapter.represent(self, obj, fieldtype) if fieldtype =='date': if value == None: return value - t = datetime.time(0, 0, 0)#this piece of data can be stripped of based on the fieldtype - return datetime.datetime.combine(value, t) #mongodb doesn't has a date object and so it must datetime, string or integer + # this piece of data can be stripped off based on the fieldtype + t = datetime.time(0, 0, 0) + # mongodb doesn't has a date object and so it must datetime, + # string or integer + return datetime.datetime.combine(value, t) elif fieldtype == 'time': if value == None: return value - d = datetime.date(2000, 1, 1) #this piece of data can be stripped of based on the fieldtype - return datetime.datetime.combine(d, value) #mongodb doesn't has a time object and so it must datetime, string or integer - elif fieldtype == 'list:string' or fieldtype == 'list:integer' or fieldtype == 'list:reference': - return value #raise SyntaxError("Not Supported") + # this piece of data can be stripped of based on the fieldtype + d = datetime.date(2000, 1, 1) + # mongodb doesn't has a time object and so it must datetime, + # string or integer + return datetime.datetime.combine(d, value) + elif fieldtype == 'list:string' or \ + fieldtype == 'list:integer' or \ + fieldtype == 'list:reference': + return value return value - #Safe determines whether a asynchronious request is done or a synchronious action is done - #For safety, we use by default synchronious requests - def insert(self,table,fields,safe=None): + # Safe determines whether a asynchronious request is done or a + # synchronious action is done + # For safety, we use by default synchronious requests + def insert(self, table, fields, safe=None): if safe==None: - safe=self.safe + safe = self.safe ctable = self.connection[table._tablename] - values = dict((k.name,self.represent(v,table[k.name].type)) for k,v in fields) - ctable.insert(values,safe=safe) + values = dict() + for k, v in fields: + if not k.name in ["id", "safe"]: + fieldname = k.name + fieldtype = table[k.name].type + if ("reference" in fieldtype) or (fieldtype=="id"): + values[fieldname] = self.object_id(v) + else: + values[fieldname] = self.represent(v, fieldtype) + ctable.insert(values, safe=safe) return int(str(values['_id']), 16) - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None, isCapped=False): + def create_table(self, table, migrate=True, fake_migrate=False, + polymodel=None, isCapped=False): if isCapped: raise RuntimeError("Not implemented") - else: - pass - def count(self,query,distinct=None,snapshot=True): + def count(self, query, distinct=None, snapshot=True): if distinct: raise RuntimeError("COUNT DISTINCT not supported") if not isinstance(query,Query): raise SyntaxError("Not Supported") tablename = self.get_table(query) - return int(self.select(query,[self.db[tablename]._id],{},count=True,snapshot=snapshot)['count']) - #Maybe it would be faster if we just implemented the pymongo .count() function which is probably quicker? - # therefor call __select() connection[table].find(query).count() Since this will probably reduce the return set? + return int(self.select(query,[self.db[tablename]._id], {}, + count=True,snapshot=snapshot)['count']) + # Maybe it would be faster if we just implemented the pymongo + # .count() function which is probably quicker? + # therefor call __select() connection[table].find(query).count() + # Since this will probably reduce the return set? def expand(self, expression, field_type=None): - try: - from pymongo.objectid import ObjectId - except ImportError: - from bson.objectid import ObjectId - #if isinstance(expression,Field): - # if expression.type=='id': - # return {_id}" if isinstance(expression, Query): # any query using 'id':= - # set name as _id (as per pymongo/mongodb primary key) + # set name as _id (as per pymongo/mongodb primary key) # convert second arg to an objectid field # (if its not already) # if second arg is 0 convert to objectid if isinstance(expression.first,Field) and \ - expression.first.type == 'id': - expression.first.name = '_id' - if expression.second != 0 and \ - not isinstance(expression.second,ObjectId): - if isinstance(expression.second,int): - try: - # Because the reference field is by default - # an integer and therefore this must be an - # integer to be able to work with other - # databases - expression.second = ObjectId(("%X" % expression.second)) - except: - raise SyntaxError('The second argument must by an integer that can represent an objectid.') - else: - try: - #But a direct id is also possible - expression.second = ObjectId(expression.second) - except: - raise SyntaxError('second argument must be of type ObjectId or an objectid representable integer') - elif expression.second == 0: - expression.second = ObjectId('000000000000000000000000') - return expression.op(expression.first, expression.second) + ((expression.first.type == 'id') or \ + ("reference" in expression.first.type)): + if expression.first.type == 'id': + expression.first.name = '_id' + # cast to Mongo ObjectId + expression.second = self.object_id(expression.second) + result = expression.op(expression.first, expression.second) if isinstance(expression, Field): if expression.type=='id': - return "_id" + result = "_id" else: - return expression.name - #return expression + result = expression.name + elif isinstance(expression, (Expression, Query)): if not expression.second is None: - return expression.op(expression.first, expression.second) + result = expression.op(expression.first, expression.second) elif not expression.first is None: - return expression.op(expression.first) + result = expression.op(expression.first) elif not isinstance(expression.op, str): - return expression.op() + result = expression.op() else: - return expression.op + result = expression.op elif field_type: - return str(self.represent(expression,field_type)) + result = str(self.represent(expression,field_type)) elif isinstance(expression,(list,tuple)): - return ','.join(self.represent(item,field_type) for item in expression) + result = ','.join(self.represent(item,field_type) for + item in expression) else: - return expression - - def _select(self,query,fields,attributes): - try: - from bson.son import SON - except ImportError: - from pymongo.son import SON + result = expression + return result + def _select(self, query, fields, attributes): if 'for_update' in attributes: logging.warn('mongodb does not support for_update') - for key in set(attributes.keys())-set(('limitby','orderby','for_update')): + for key in set(attributes.keys())-set(('limitby', + 'orderby','for_update')): if attributes[key]!=None: - raise SyntaxError('invalid select attribute: %s' % key) + logging.warn('select attribute not implemented: %s' % key) new_fields=[] mongosort_list = [] @@ -5088,28 +5185,27 @@ class MongoDBAdapter(NoSQLAdapter): # try an orderby attribute orderby = attributes.get('orderby', False) limitby = attributes.get('limitby', False) - #distinct = attributes.get('distinct', False) + # distinct = attributes.get('distinct', False) if orderby: - #print "in if orderby %s" % orderby if isinstance(orderby, (list, tuple)): orderby = xorify(orderby) # !!!! need to add 'random' for f in self.expand(orderby).split(','): if f.startswith('-'): - mongosort_list.append((f[1:],-1)) + mongosort_list.append((f[1:], -1)) else: - mongosort_list.append((f,1)) + mongosort_list.append((f, 1)) if limitby: limitby_skip, limitby_limit = limitby else: limitby_skip = limitby_limit = 0 - mongofields_dict = SON() + mongofields_dict = self.SON() mongoqry_dict = {} for item in fields: - if isinstance(item,SQLALL): + if isinstance(item, SQLALL): new_fields += item._table else: new_fields.append(item) @@ -5119,60 +5215,73 @@ class MongoDBAdapter(NoSQLAdapter): elif len(fields) != 0: tablename = fields[0].tablename else: - raise SyntaxError("The table name could not be found in the query nor from the select statement.") + raise SyntaxError("The table name could not be found in " + + "the query nor from the select statement.") + mongoqry_dict = self.expand(query) fields = fields or self.db[tablename] for field in fields: mongofields_dict[field.name] = 1 - return tablename, mongoqry_dict, mongofields_dict, \ - mongosort_list, limitby_limit, limitby_skip - # need to define all the 'sql' methods gt,lt etc.... + return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \ + limitby_limit, limitby_skip - def select(self,query,fields,attributes,count=False,snapshot=False): - try: - from pymongo.objectid import ObjectId - except ImportError: - from bson.objectid import ObjectId - tablename, mongoqry_dict, mongofields_dict, \ - mongosort_list, limitby_limit, limitby_skip = \ - self._select(query,fields,attributes) + + def select(self, query, fields, attributes, count=False, + snapshot=False): + # TODO: support joins + tablename, mongoqry_dict, mongofields_dict, mongosort_list, \ + limitby_limit, limitby_skip = self._select(query, fields, attributes) ctable = self.connection[tablename] + if count: return {'count' : ctable.find( mongoqry_dict, mongofields_dict, skip=limitby_skip, limit=limitby_limit, sort=mongosort_list, snapshot=snapshot).count()} else: - mongo_list_dicts = ctable.find( - mongoqry_dict, mongofields_dict, - skip=limitby_skip, limit=limitby_limit, - sort=mongosort_list, snapshot=snapshot) # pymongo cursor object - # DEBUG: print "mongo_list_dicts=%s" % mongo_list_dicts + # pymongo cursor object + mongo_list_dicts = ctable.find(mongoqry_dict, + mongofields_dict, skip=limitby_skip, + limit=limitby_limit, sort=mongosort_list, + snapshot=snapshot) rows = [] - ### populate row in proper order - colnames = [str(field) for field in fields] - for k,record in enumerate(mongo_list_dicts): + # populate row in proper order + # Here we replace ._id with .id to follow the standard naming + colnames = [] + newnames = [] + for field in fields: + colname = str(field) + colnames.append(colname) + tablename, fieldname = colname.split(".") + if fieldname == "_id": + # Mongodb reserved uuid key + field.name = "id" + newnames.append(".".join((tablename, field.name))) + + for record in mongo_list_dicts: row=[] - for fullcolname in colnames: - colname = fullcolname.split('.')[1] - column = '_id' if colname=='id' else colname - if column in record: - if column == '_id' and isinstance( - record[column],ObjectId): - value = int(str(record[column]),16) - elif column != '_id': - value = record[column] + for colname in colnames: + tablename, fieldname = colname.split(".") + # switch to Mongo _id uuids for retrieving + # record id's + if fieldname == "id": fieldname = "_id" + if fieldname in record: + if isinstance(record[fieldname], + self.ObjectId): + value = int(str(record[fieldname]), 16) else: - value = None + value = record[fieldname] else: value = None row.append(value) rows.append(row) - processor = attributes.get('processor',self.parse) - return processor(rows,fields,colnames,False) + processor = attributes.get('processor', self.parse) + result = processor(rows, fields, newnames, False) + return result - def INVERT(self,first): + + def INVERT(self, first): #print "in invert first=%s" % first return '-%s' % self.expand(first) @@ -5181,62 +5290,69 @@ class MongoDBAdapter(NoSQLAdapter): ctable.drop() - def truncate(self,table,mode,safe=None): - if safe==None: + def truncate(self, table, mode, safe=None): + if safe == None: safe=self.safe ctable = self.connection[table._tablename] ctable.remove(None, safe=True) - #the update function should return a string - def oupdate(self,tablename,query,fields): - if not isinstance(query,Query): + def oupdate(self, tablename, query, fields): + if not isinstance(query, Query): raise SyntaxError("Not Supported") filter = None if query: filter = self.expand(query) - f_v = [] + modify = {'$set': dict((k.name, self.represent(v, k.type)) for + k, v in fields)} + return modify, filter - - modify = { '$set' : dict(((k.name,self.represent(v,k.type)) for k,v in fields)) } - return modify,filter - - #TODO implement update - #TODO implement set operator - #TODO implement find and modify - #todo implement complex update - def update(self,tablename,query,fields,safe=None): - if safe==None: - safe=self.safe - #return amount of adjusted rows or zero, but no exceptions related not finding the result - if not isinstance(query,Query): + def update(self, tablename, query, fields, safe=None): + if safe == None: + safe = self.safe + # return amount of adjusted rows or zero, but no exceptions + # @ related not finding the result + if not isinstance(query, Query): raise RuntimeError("Not implemented") - amount = self.count(query,False) - modify,filter = self.oupdate(tablename,query,fields) + amount = self.count(query, False) + modify, filter = self.oupdate(tablename, query, fields) try: + result = self.connection[tablename].update(filter, + modify, multi=True, safe=safe) if safe: - return self.connection[tablename].update(filter,modify,multi=True,safe=safe).n + try: + # if result count is available fetch it + return result["n"] + except (KeyError, AttributeError, TypeError): + return amount else: - amount =self.count(query) - self.connection[tablename].update(filter,modify,multi=True,safe=safe) return amount - except: - #TODO Reverse update query to verifiy that the query succeded - return 0 - """ - An special update operator that enables the update of specific field - return a dict - """ - - + except Exception, e: + # TODO Reverse update query to verifiy that the query succeded + raise RuntimeError("uncaught exception when updating rows: %s" % e) #this function returns a dict with the where clause and update fields def _update(self,tablename,query,fields): - return str(self.oupdate(tablename,query,fields)) + return str(self.oupdate(tablename, query, fields)) + + def delete(self, tablename, query, safe=None): + if safe is None: + safe = self.safe + amount = 0 + amount = self.count(query, False) + if not isinstance(query, Query): + raise RuntimeError("query type %s is not supported" % \ + type(query)) + filter = self.expand(query) + self._delete(tablename, filter, safe=safe) + return amount + + def _delete(self, tablename, filter, safe=None): + return self.connection[tablename].remove(filter, safe=safe) def bulk_insert(self, table, items): return [self.insert(table,item) for item in items] - #TODO This will probably not work:( + # TODO This will probably not work:( def NOT(self, first): result = {} result["$not"] = self.expand(first) @@ -5249,7 +5365,7 @@ class MongoDBAdapter(NoSQLAdapter): return f def OR(self,first,second): - # pymongo expects: .find( {'$or' : [{'name':'1'}, {'name':'2'}] } ) + # pymongo expects: .find({'$or': [{'name':'1'}, {'name':'2'}]}) result = {} f = self.expand(first) s = self.expand(second) @@ -5266,9 +5382,6 @@ class MongoDBAdapter(NoSQLAdapter): def EQ(self,first,second): result = {} - #if second is None: - #return '(%s == null)' % self.expand(first) - #return '(%s == %s)' % (self.expand(first),self.expand(second,first.type)) result[self.expand(first)] = self.expand(second) return result @@ -5304,84 +5417,101 @@ class MongoDBAdapter(NoSQLAdapter): return result def ADD(self, first, second): - raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") - return '%s + %s' % (self.expand(first), self.expand(second, first.type)) + raise NotImplementedError(self.error_messages["javascript_needed"]) + return '%s + %s' % (self.expand(first), + self.expand(second, first.type)) def SUB(self, first, second): - raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") - return '(%s - %s)' % (self.expand(first), self.expand(second, first.type)) + raise NotImplementedError(self.error_messages["javascript_needed"]) + return '(%s - %s)' % (self.expand(first), + self.expand(second, first.type)) def MUL(self, first, second): - raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") - return '(%s * %s)' % (self.expand(first), self.expand(second, first.type)) + raise NotImplementedError(self.error_messages["javascript_needed"]) + return '(%s * %s)' % (self.expand(first), + self.expand(second, first.type)) def DIV(self, first, second): - raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") - return '(%s / %s)' % (self.expand(first), self.expand(second, first.type)) + raise NotImplementedError(self.error_messages["javascript_needed"]) + return '(%s / %s)' % (self.expand(first), + self.expand(second, first.type)) def MOD(self, first, second): - raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") - return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type)) + raise NotImplementedError(self.error_messages["javascript_needed"]) + return '(%s %% %s)' % (self.expand(first), + self.expand(second, first.type)) def AS(self, first, second): - raise NotImplementedError("This must yet be replaced with javascript in order to accomplish this. Sorry") + raise NotImplementedError(self.error_messages["javascript_needed"]) return '%s AS %s' % (self.expand(first), second) - #We could implement an option that simulates a full featured SQL database. But I think the option should be set explicit or implemented as another library. + # We could implement an option that simulates a full featured SQL + # database. But I think the option should be set explicit or + # implemented as another library. def ON(self, first, second): - raise NotImplementedError("This is not possible in NoSQL, but can be simulated with a wrapper.") + raise NotImplementedError("This is not possible in NoSQL" + + " but can be simulated with a wrapper.") return '%s ON %s' % (self.expand(first), self.expand(second)) - # # BLOW ARE TWO IMPLEMENTATIONS OF THE SAME FUNCITONS # WHICH ONE IS BEST? - # def COMMA(self, first, second): return '%s, %s' % (self.expand(first), self.expand(second)) def LIKE(self, first, second): #escaping regex operators? - return {self.expand(first) : ('%s' % self.expand(second, 'string').replace('%','/'))} + return {self.expand(first): ('%s' % \ + self.expand(second, 'string').replace('%','/'))} def STARTSWITH(self, first, second): #escaping regex operators? - return {self.expand(first) : ('/^%s/' % self.expand(second, 'string'))} + return {self.expand(first): ('/^%s/' % \ + self.expand(second, 'string'))} def ENDSWITH(self, first, second): #escaping regex operators? - return {self.expand(first) : ('/%s^/' % self.expand(second, 'string'))} + return {self.expand(first): ('/%s^/' % \ + self.expand(second, 'string'))} def CONTAINS(self, first, second): - #There is a technical difference, but mongodb doesn't support that, but the result will be the same - return {self.expand(first) : ('/%s/' % self.expand(second, 'string'))} + #There is a technical difference, but mongodb doesn't support + # that, but the result will be the same + return {self.expand(first) : ('/%s/' % \ + self.expand(second, 'string'))} def LIKE(self, first, second): import re - return {self.expand(first) : {'$regex' : re.escape(self.expand(second, 'string')).replace('%','.*')}} + return {self.expand(first): {'$regex': \ + re.escape(self.expand(second, + 'string')).replace('%','.*')}} #TODO verify full compatibilty with official SQL Like operator def STARTSWITH(self, first, second): #TODO Solve almost the same problem as with endswith import re - return {self.expand(first) : {'$regex' : '^' + re.escape(self.expand(second, 'string'))}} + return {self.expand(first): {'$regex' : '^' + + re.escape(self.expand(second, + 'string'))}} #TODO verify full compatibilty with official SQL Like operator def ENDSWITH(self, first, second): #escaping regex operators? - #TODO if searched for a name like zsa_corbitt and the function is endswith('a') then this is also returned. Aldo it end with a t + #TODO if searched for a name like zsa_corbitt and the function + # is endswith('a') then this is also returned. + # Aldo it end with a t import re - return {self.expand(first) : {'$regex' : re.escape(self.expand(second, 'string')) + '$'}} + return {self.expand(first): {'$regex': \ + re.escape(self.expand(second, 'string')) + '$'}} #TODO verify full compatibilty with official oracle contains operator def CONTAINS(self, first, second): - #There is a technical difference, but mongodb doesn't support that, but the result will be the same + #There is a technical difference, but mongodb doesn't support + # that, but the result will be the same #TODO contains operators need to be transformed to Regex - return {self.expand(first) : {' $regex' : ".*" + re.escape(self.expand(second, 'string')) + ".*"}} + return {self.expand(first) : {' $regex': \ + ".*" + re.escape(self.expand(second, 'string')) + ".*"}} - # - # END REDUNDANCY - # class IMAPAdapter(NoSQLAdapter): drivers = ('imaplib',) @@ -5814,17 +5944,21 @@ class IMAPAdapter(NoSQLAdapter): def create_table(self, *args, **kwargs): # not implemented - LOGGER.debug("Create table feature is not implemented for %s" % type(self)) + # but required by DAL + pass - def _select(self,query,fields,attributes): - """ Search and Fetch records and return web2py - rows + def _select(self, query, fields, attributes): + if use_common_filters(query): + query = self.common_filter(query, [self.get_query_mailbox(query),]) + return str(query) + + def select(self,query,fields,attributes): + """ Search and Fetch records and return web2py rows """ - + # move this statement elsewhere (upper-level) if use_common_filters(query): query = self.common_filter(query, [self.get_query_mailbox(query),]) - # move this statement elsewhere (upper-level) import email import email.header decode_header = email.header.decode_header @@ -5832,68 +5966,62 @@ class IMAPAdapter(NoSQLAdapter): # convert results to a dictionary tablename = None fetch_results = list() - if isinstance(query, (Expression, Query)): + if isinstance(query, Query): tablename = self.get_table(query) mailbox = self.connection.mailbox_names.get(tablename, None) - if isinstance(query, Expression): - pass - elif isinstance(query, Query): - if mailbox is not None: - # select with readonly - selected = self.connection.select(mailbox, True) - self.mailbox_size = int(selected[1][0]) - search_query = "(%s)" % str(query).strip() - search_result = self.connection.uid("search", None, search_query) - # Normal IMAP response OK is assumed (change this) - if search_result[0] == "OK": - # For "light" remote server responses just get the first - # ten records (change for non-experimental implementation) - # However, light responses are not guaranteed with this - # approach, just fewer messages. - # TODO: change limitby single to 2-tuple argument - limitby = attributes.get('limitby', None) - messages_set = search_result[1][0].split() - # descending order - messages_set.reverse() - if limitby is not None: - # TODO: asc/desc attributes - messages_set = messages_set[int(limitby[0]):int(limitby[1])] - # Partial fetches are not used since the email - # library does not seem to support it (it converts - # partial messages to mangled message instances) - imap_fields = "(RFC822)" - if len(messages_set) > 0: - # create fetch results object list - # fetch each remote message and store it in memmory - # (change to multi-fetch command syntax for faster - # transactions) - for uid in messages_set: - # fetch the RFC822 message body - typ, data = self.connection.uid("fetch", uid, imap_fields) - if typ == "OK": - fr = {"message": int(data[0][0].split()[0]), - "uid": int(uid), - "email": email.message_from_string(data[0][1]), - "raw_message": data[0][1] - } - fr["multipart"] = fr["email"].is_multipart() - # fetch flags for the message - ftyp, fdata = self.connection.uid("fetch", uid, "(FLAGS)") - if ftyp == "OK": - fr["flags"] = self.driver.ParseFlags(fdata[0]) - fetch_results.append(fr) - else: - # error retrieving the flags for this message - pass + if mailbox is not None: + # select with readonly + selected = self.connection.select(mailbox, True) + self.mailbox_size = int(selected[1][0]) + search_query = "(%s)" % str(query).strip() + search_result = self.connection.uid("search", None, search_query) + # Normal IMAP response OK is assumed (change this) + if search_result[0] == "OK": + # For "light" remote server responses just get the first + # ten records (change for non-experimental implementation) + # However, light responses are not guaranteed with this + # approach, just fewer messages. + # TODO: change limitby single to 2-tuple argument + limitby = attributes.get('limitby', None) + messages_set = search_result[1][0].split() + # descending order + messages_set.reverse() + if limitby is not None: + # TODO: asc/desc attributes + messages_set = messages_set[int(limitby[0]):int(limitby[1])] + # Partial fetches are not used since the email + # library does not seem to support it (it converts + # partial messages to mangled message instances) + imap_fields = "(RFC822)" + if len(messages_set) > 0: + # create fetch results object list + # fetch each remote message and store it in memmory + # (change to multi-fetch command syntax for faster + # transactions) + for uid in messages_set: + # fetch the RFC822 message body + typ, data = self.connection.uid("fetch", uid, imap_fields) + if typ == "OK": + fr = {"message": int(data[0][0].split()[0]), + "uid": int(uid), + "email": email.message_from_string(data[0][1]), + "raw_message": data[0][1]} + fr["multipart"] = fr["email"].is_multipart() + # fetch flags for the message + ftyp, fdata = self.connection.uid("fetch", uid, "(FLAGS)") + if ftyp == "OK": + fr["flags"] = self.driver.ParseFlags(fdata[0]) + fetch_results.append(fr) else: - # error retrieving the message body + # error retrieving the flags for this message pass - - elif isinstance(query, basestring): - # not implemented - pass + else: + # error retrieving the message body + pass + elif isinstance(query, (Expression, basestring)): + raise NotImplementedError() else: - pass + raise TypeError("Unexpected query type") imapqry_dict = {} imapfields_dict = {} @@ -5981,6 +6109,7 @@ class IMAPAdapter(NoSQLAdapter): item_dict["%s.answered" % tablename] = "\\Answered" in flags if "%s.mime" % tablename in fieldnames: item_dict["%s.mime" % tablename] = message.get_content_type() + # Here goes the whole RFC822 body as an email instance # for controller side custom processing # The message is stored as a raw string @@ -5988,6 +6117,7 @@ class IMAPAdapter(NoSQLAdapter): # returns a Message object for enhanced object processing if "%s.email" % tablename in fieldnames: item_dict["%s.email" % tablename] = self.encode_text(raw_message, charset) + # Size measure as suggested in a Velocity Reviews post # by Tim Williams: "how to get size of email attachment" # Note: len() and server RFC822.SIZE reports doesn't match @@ -6004,11 +6134,9 @@ class IMAPAdapter(NoSQLAdapter): if "%s.size" % tablename in fieldnames: if part is not None: size += len(str(part)) - item_dict["%s.content" % tablename] = bar_encode(content) item_dict["%s.attachments" % tablename] = bar_encode(attachments) item_dict["%s.size" % tablename] = size - imapqry_list.append(item_dict) # extra object mapping for the sake of rows object @@ -6019,23 +6147,18 @@ class IMAPAdapter(NoSQLAdapter): imapqry_array_item.append(item_dict[fieldname]) imapqry_array.append(imapqry_array_item) - return tablename, imapqry_array, fieldnames - - def select(self,query,fields,attributes): - tablename, imapqry_array , fieldnames = self._select(query,fields,attributes) # parse result and return a rows object colnames = fieldnames processor = attributes.get('processor',self.parse) return processor(imapqry_array, fields, colnames) - def update(self, tablename, query, fields): + def _update(self, tablename, query, fields, commit=False): + # TODO: the adapter should implement an .expand method + commands = list() if use_common_filters(query): query = self.common_filter(query, [tablename,]) - mark = [] unmark = [] - rowcount = 0 - query = str(query) if query: for item in fields: field = item[0] @@ -6048,26 +6171,33 @@ class IMAPAdapter(NoSQLAdapter): mark.append(flag) else: unmark.append(flag) - result, data = self.connection.select( self.connection.mailbox_names[tablename]) string_query = "(%s)" % query result, data = self.connection.search(None, string_query) store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()] - # change marked flags + # build commands for marked flags for number in store_list: result = None if len(mark) > 0: - result, data = self.connection.store( - number, "+FLAGS", "(%s)" % " ".join(mark)) + commands.append((number, "+FLAGS", "(%s)" % " ".join(mark))) if len(unmark) > 0: - result, data = self.connection.store( - number, "-FLAGS", "(%s)" % " ".join(unmark)) - if result == "OK": - rowcount += 1 + commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark))) + return commands + + def update(self, tablename, query, fields): + rowcount = 0 + commands = self._update(tablename, query, fields) + for command in commands: + result, data = self.connection.store(*command) + if result == "OK": + rowcount += 1 return rowcount + def _count(self, query, distinct=None): + raise NotImplementedError() + def count(self,query,distinct=None): counter = 0 tablename = self.get_query_mailbox(query) @@ -6343,12 +6473,10 @@ def sqlhtml_validators(field): return r._format(row) else: return id - if field_type == 'string': - requires.append(validators.IS_LENGTH(field_length)) - elif field_type == 'text': - requires.append(validators.IS_LENGTH(field_length)) - elif field_type == 'password': + if field_type in (('string', 'text', 'password')): requires.append(validators.IS_LENGTH(field_length)) + elif field_type == 'json': + requires.append(validators.IS_EMPTY_OR(validators.IS_JSON())) elif field_type == 'double' or field_type == 'float': requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100)) elif field_type in ('integer','bigint'): @@ -6455,6 +6583,8 @@ class Row(object): def __setitem__(self, key, value): setattr(self, str(key), value) + __copy__ = lambda self: Row(self) + __call__ = __getitem__ def get(self,key,default=None): @@ -6628,7 +6758,7 @@ def smart_query(fields,text): value = constants[item[1:]] else: value = item - if field.type in ('text','string'): + if field.type in ('text', 'string', 'json'): if op == '=': op = 'like' if op == '=': new_query = field==value elif op == '<': new_query = field :db_codec: string encoding of the database (default: 'UTF-8') :check_reserved: list of adapters to check tablenames and column names - against sql reserved keywords. (Default None) + against sql/nosql reserved keywords. (Default None) * 'common' List of sql keywords that are common to all database types such as "SELECT, INSERT". (recommended) @@ -6831,6 +6961,7 @@ class DAL(object): self._migrated = [] self._LAZY_TABLES = {} self._lazy_tables = lazy_tables + self._tables = SQLCallableList() if not str(attempts).isdigit() or attempts < 0: attempts = 5 if uri: @@ -6883,7 +7014,6 @@ class DAL(object): migrate = fake_migrate = False adapter = self._adapter self._uri_hash = hashlib_md5(adapter.uri).hexdigest() - self._tables = SQLCallableList() self.check_reserved = check_reserved if self.check_reserved: from reserved_sql_keywords import ADAPTERS as RSK @@ -6932,7 +7062,7 @@ class DAL(object): for backend in self.check_reserved: if name.upper() in self.RSK[backend]: raise SyntaxError( - 'invalid table/column name "%s" is a "%s" reserved SQL keyword' % (name, backend.upper())) + 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper())) def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True): """ @@ -7023,7 +7153,7 @@ def index(): patterns += auto_table(table,base=tag,depth=depth-1) return patterns - if patterns==DEFAULT: + if patterns == 'auto': patterns=[] for table in db.tables: if not table.startswith('auth_'): @@ -7045,9 +7175,15 @@ def index(): return Row({'status':200,'pattern':'list', 'error':None,'response':patterns}) for pattern in patterns: + if isinstance(pattern,tuple): + pattern, basequery = pattern + else: + basequery = None otable=table=None if not isinstance(queries,dict): dbset=db(queries) + if basequery is not None: + dbset = dbset(basequery) i=0 tags = pattern[1:].split('/') if len(tags)!=len(args): @@ -7094,6 +7230,8 @@ def index(): raise RuntimeError("invalid pattern: %s" % pattern) if not otable and isinstance(queries,dict): dbset = db(queries[table]) + if basequery is not None: + dbset = dbset(basequery) dbset=dbset(query) else: raise RuntimeError("missing relation in pattern: %s" % pattern) @@ -7449,8 +7587,8 @@ class SQLALL(object): def __str__(self): return ', '.join([str(field) for field in self._table]) - -class Reference(int): +# class Reference(int): +class Reference(long): def __allocate(self): if not self._record: @@ -7628,7 +7766,7 @@ class Table(object): field.tablename = field._tablename = tablename field.table = field._table = self field.db = field._db = db - if db and not field.type in ('text','blob') and \ + if db and not field.type in ('text', 'blob', 'json') and \ db._adapter.maxcharlength < field.length: field.length = db._adapter.maxcharlength self.ALL = SQLALL(self) @@ -7914,13 +8052,24 @@ class Table(object): raise RuntimeError("Unable to handle upload") fields[field.name] = new_name + def _defaults(self, fields): + "If there are no fields/values specified, return table defaults" + if not fields: + fields = {} + for field in self: + if field.type != "id": + fields[field.name] = field.default + return fields + def _insert(self, **fields): - return self._db._adapter._insert(self,self._listify(fields)) + fields = self._default(fields) + return self._db._adapter._insert(self, self._listify(fields)) def insert(self, **fields): + fields = self._defaults(fields) self._attempt_upload(fields) if any(f(fields) for f in self._before_insert): return 0 - ret = self._db._adapter.insert(self,self._listify(fields)) + ret = self._db._adapter.insert(self, self._listify(fields)) if ret and self._after_insert: fields = Row(fields) [f(fields,ret) for f in self._after_insert] @@ -8326,13 +8475,13 @@ class Expression(object): def startswith(self, value): db = self.db - if not self.type in ('string', 'text'): + if not self.type in ('string', 'text', 'json'): raise SyntaxError("startswith used with incompatible field type") return Query(db, db._adapter.STARTSWITH, self, value) def endswith(self, value): db = self.db - if not self.type in ('string', 'text'): + if not self.type in ('string', 'text', 'json'): raise SyntaxError("endswith used with incompatible field type") return Query(db, db._adapter.ENDSWITH, self, value) @@ -8344,7 +8493,7 @@ class Expression(object): return self.contains('') else: return reduce(all and AND or OR,subqueries) - if not self.type in ('string', 'text') and not self.type.startswith('list:'): + if not self.type in ('string', 'text', 'json') and not self.type.startswith('list:'): raise SyntaxError("contains used with incompatible field type") return Query(db, db._adapter.CONTAINS, self, value) @@ -8352,8 +8501,8 @@ class Expression(object): db = self.db return Expression(db, db._adapter.AS, self, alias, self.type) - # GIS functions - + # GIS expressions + def st_asgeojson(self, precision=15, options=0, version=1): return Expression(self.db, self.db._adapter.ST_ASGEOJSON, self, dict(precision=precision, options=options, @@ -8363,18 +8512,28 @@ class Expression(object): db = self.db return Expression(db, db._adapter.ST_ASTEXT, self, type='string') - def st_contained(self, value): + def st_x(self): db = self.db - return Query(db, db._adapter.ST_CONTAINS, value, self) + return Expression(db, db._adapter.ST_X, self, type='string') + + def st_y(self): + db = self.db + return Expression(db, db._adapter.ST_Y, self, type='string') + + def st_distance(self, other): + db = self.db + return Expression(db,db._adapter.ST_DISTANCE,self,other, 'double') + + def st_simplify(self, value): + db = self.db + return Expression(db, db._adapter.ST_SIMPLIFY, self, value, self.type) + + # GIS queries def st_contains(self, value): db = self.db return Query(db, db._adapter.ST_CONTAINS, self, value) - def st_distance(self, other): - db = self.db - return Expression(db,db._adapter.ST_DISTANCE,self,other,self.type) - def st_equals(self, value): db = self.db return Query(db, db._adapter.ST_EQUALS, self, value) @@ -8387,10 +8546,6 @@ class Expression(object): db = self.db return Query(db, db._adapter.ST_OVERLAPS, self, value) - def st_simplify(self, value): - db = self.db - return Expression(db, db._adapter.ST_SIMPLIFY, self, value) - def st_touches(self, value): db = self.db return Query(db, db._adapter.ST_TOUCHES, self, value) @@ -9441,7 +9596,9 @@ class Rows(object): items = [[inner_loop(record, col) for col in self.colnames] for record in self] if have_serializers: - return serializers.json(items,default=default or serializers.custom_json) + return serializers.json(items, + default=default or + serializers.custom_json) else: try: import json as simplejson @@ -9467,6 +9624,7 @@ def test_all(): Field('blobf', 'blob'),\ Field('integerf', 'integer', unique=True),\ Field('doublef', 'double', unique=True,notnull=True),\ + Field('jsonf', 'json'),\ Field('datef', 'date', default=datetime.date.today()),\ Field('timef', 'time'),\ Field('datetimef', 'datetime'),\ @@ -9476,6 +9634,7 @@ def test_all(): >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\ uploadf=None, integerf=5, doublef=3.14,\ + jsonf={"j": True},\ datef=datetime.date(2001, 1, 1),\ timef=datetime.time(12, 30, 15),\ datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15)) @@ -9675,7 +9834,7 @@ DAL.Table = Table # was necessary in gluon/globals.py session.connect # Geodal utils ################################################################################ -def geoPoint(*line): +def geoPoint(x,y): return "POINT (%f %f)" % (x,y) def geoLine(*line): diff --git a/gluon/html.py b/gluon/html.py index 09783e78..265f5500 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -991,7 +991,7 @@ class DIV(XmlComponent): >>> a=FORM( INPUT(_type='text'), SELECT(range(1)), TEXTAREA() ) >>> for c in a.elements('input, select, textarea'): c['_disabled'] = 'disabled' >>> a.xml() - '
' + '
' Elements that are matched can also be replaced or removed by specifying a "replace" argument (note, a list of the original matching elements @@ -1478,9 +1478,9 @@ class A(DIV): (self['component'], self['target'] or '', d) self['_href'] = self['_href'] or '#null' elif self['callback']: - returnfalse = "var e = arguments[0] || window.event; e.cancelBubble=true; if (e.stopPropagation) e.stopPropagation();" + returnfalse = "var e = arguments[0] || window.event; e.cancelBubble=true; if (e.stopPropagation) {e.stopPropagation(); e.stopImmediatePropagation(); e.preventDefault();}" if d: - self['_onclick'] = "if(confirm(w2p_ajax_confirm_message||'Are you sure you want o delete this object?')){ajax('%s',[],'%s');%s};%s" % \ + self['_onclick'] = "if(confirm(w2p_ajax_confirm_message||'Are you sure you want to delete this object?')){ajax('%s',[],'%s');%s};%s" % \ (self['callback'], self['target'] or '', d, returnfalse) else: self['_onclick'] = "ajax('%s',[],'%s');%sreturn false" % \ @@ -1932,7 +1932,7 @@ class FORM(DIV): >>> from validators import IS_NOT_EMPTY >>> form=FORM(INPUT(_name=\"test\", requires=IS_NOT_EMPTY())) >>> form.xml() - '
' + '
' a FORM is container for INPUT, TEXTAREA, SELECT and other helpers @@ -2033,7 +2033,7 @@ class FORM(DIV): def _postprocessing(self): if not '_action' in self.attributes: - self['_action'] = '' + self['_action'] = '#' if not '_method' in self.attributes: self['_method'] = 'post' if not '_enctype' in self.attributes: @@ -2415,25 +2415,25 @@ def test():
abc
def
123
>>> form=FORM(INPUT(_type='text', _name='myvar', requires=IS_EXPR('int(value)<10'))) >>> print form.xml() -
+
>>> print form.accepts({'myvar':'34'}, formname=None) False >>> print form.xml() -
invalid expression
+
invalid expression
>>> print form.accepts({'myvar':'4'}, formname=None, keepvalues=True) True >>> print form.xml() -
+
>>> form=FORM(SELECT('cat', 'dog', _name='myvar')) >>> print form.accepts({'myvar':'dog'}, formname=None, keepvalues=True) True >>> print form.xml() -
+
>>> form=FORM(INPUT(_type='text', _name='myvar', requires=IS_MATCH('^\w+$', 'only alphanumeric!'))) >>> print form.accepts({'myvar':'as df'}, formname=None) False >>> print form.xml() -
only alphanumeric!
+
only alphanumeric!
>>> session={} >>> form=FORM(INPUT(value=\"Hello World\", _name=\"var\", requires=IS_MATCH('^\w+$'))) >>> if form.accepts({}, session,formname=None): print 'passed' diff --git a/gluon/reserved_sql_keywords.py b/gluon/reserved_sql_keywords.py index e5b0fd14..9ff314ae 100644 --- a/gluon/reserved_sql_keywords.py +++ b/gluon/reserved_sql_keywords.py @@ -1686,6 +1686,9 @@ SQLITE = set(( 'WHERE', )) + +MONGODB_NONRESERVED = set(('SAFE',)) + # remove from here when you add a list. JDBCSQLITE = SQLITE DB2 = INFORMIX = INGRES = JDBCPOSTGRESQL = COMMON @@ -1708,6 +1711,7 @@ ADAPTERS = { 'jdbc:sqlite': JDBCSQLITE, 'jdbc:postgres': JDBCPOSTGRESQL, 'common': COMMON, + 'mongodb_nonreserved': MONGODB_NONRESERVED } ADAPTERS['all'] = reduce(lambda a, b: a.union(b), ( diff --git a/gluon/serializers.py b/gluon/serializers.py index 3cdb91fc..e29061a1 100644 --- a/gluon/serializers.py +++ b/gluon/serializers.py @@ -20,6 +20,10 @@ except ImportError: import contrib.simplejson as json_parser # fallback to pure-Python module +def loads_json(o): + # deserialize a json string + return json_parser.loads(o) + def custom_json(o): if hasattr(o, 'custom_json') and callable(o.custom_json): return o.custom_json() diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index a5827e96..973185c4 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -30,6 +30,7 @@ from utils import md5_hash from validators import IS_EMPTY_OR, IS_NOT_EMPTY, IS_LIST_OF, IS_DATE, \ IS_DATETIME, IS_INT_IN_RANGE, IS_FLOAT_IN_RANGE, IS_STRONG +import serializers import datetime import urllib import re @@ -39,6 +40,9 @@ import inspect import settings is_gae = settings.global_settings.web2py_runtime_gae + + + table_field = re.compile('[\w_]+\.[\w_]+') widget_class = re.compile('^\w*') @@ -164,11 +168,9 @@ class TimeWidget(StringWidget): class DateWidget(StringWidget): _class = 'date' - class DatetimeWidget(StringWidget): _class = 'datetime' - class TextWidget(FormWidget): _class = 'text' @@ -184,6 +186,22 @@ class TextWidget(FormWidget): attr = cls._attributes(field, default, **attributes) return TEXTAREA(**attr) +class JSONWidget(FormWidget): + _class = 'json' + + @classmethod + def widget(cls, field, value, **attributes): + """ + generates a TEXTAREA for JSON notation. + + see also: :meth:`FormWidget.widget` + """ + if not isinstance(value, basestring): + if value is not None: + value = serializers.json(value) + default = dict(value=value) + attr = cls._attributes(field, default, **attributes) + return TEXTAREA(**attr) class BooleanWidget(FormWidget): _class = 'boolean' @@ -235,7 +253,6 @@ class OptionsWidget(FormWidget): raise SyntaxError( 'widget cannot determine options of %s' % field) opts = [OPTION(v, _value=k) for (k, v) in options] - return SELECT(*opts, **attr) @@ -523,20 +540,25 @@ class UploadWidget(FormWidget): requires = attr["requires"] if requires == [] or isinstance(requires, IS_EMPTY_OR): - inp = DIV(inp, '[', - A(current.T( - UploadWidget.GENERIC_DESCRIPTION), _href=url), - '|', - INPUT(_type='checkbox', - _name=field.name + cls.ID_DELETE_SUFFIX, - _id=field.name + cls.ID_DELETE_SUFFIX), - LABEL(current.T(cls.DELETE_FILE), - _for=field.name + cls.ID_DELETE_SUFFIX), - ']', br, image) + inp = DIV(inp, + SPAN('[', + A(current.T( + UploadWidget.GENERIC_DESCRIPTION), _href=url), + '|', + INPUT(_type='checkbox', + _name=field.name + cls.ID_DELETE_SUFFIX, + _id=field.name + cls.ID_DELETE_SUFFIX), + LABEL(current.T(cls.DELETE_FILE), + _for=field.name + cls.ID_DELETE_SUFFIX, + _style='display:inline'), + ']', _style='white-space:nowrap'), + br, image) else: - inp = DIV(inp, '[', - A(cls.GENERIC_DESCRIPTION, _href=url), - ']', br, image) + inp = DIV(inp, + SPAN('[', + A(cls.GENERIC_DESCRIPTION, _href=url), + ']', _style='white-space:nowrap'), + br, image) return inp @classmethod @@ -716,7 +738,7 @@ def formstyle_table2cols(form, fields): def formstyle_divs(form, fields): ''' divs only ''' - table = TAG['']() + table = FIELDSET() for id, label, controls, help in fields: _help = DIV(help, _class='w2p_fc') _controls = DIV(controls, _class='w2p_fw') @@ -748,7 +770,7 @@ def formstyle_ul(form, fields): def formstyle_bootstrap(form, fields): ''' bootstrap format form layout ''' - form['_class'] = 'form-horizontal' + form.add_class('form-horizontal') parent = FIELDSET() for id, label, controls, help in fields: # wrappers @@ -782,12 +804,12 @@ def formstyle_bootstrap(form, fields): if _submit: # submit button has unwrapped label and controls, different class - parent.append(DIV(label, controls, _class='form-actions')) + parent.append(DIV(label, controls, _class='form-actions', _id=id)) # unflag submit (possible side effect) _submit = False else: # unwrapped label - parent.append(DIV(label, _controls, _class='control-group')) + parent.append(DIV(label, _controls, _class='control-group', _id=id)) return parent @@ -843,6 +865,7 @@ class SQLFORM(FORM): widgets = Storage(dict( string=StringWidget, text=TextWidget, + json=JSONWidget, password=PasswordWidget, integer=IntegerWidget, double=DoubleWidget, @@ -1132,7 +1155,7 @@ class SQLFORM(FORM): # # when deletable, add delete? checkbox - self.custom.deletable = '' + self.custom.delete = self.custom.deletable = '' if record and deletable: #add secondary css class for cascade delete warning css = 'delete' @@ -1154,7 +1177,8 @@ class SQLFORM(FORM): _id=self.FIELDKEY_DELETE_RECORD + SQLFORM.ID_LABEL_SUFFIX), widget, col3.get(self.FIELDKEY_DELETE_RECORD, ''))) - self.custom.deletable = widget + self.custom.delete = self.custom.deletable = widget + # when writable, add submit button self.custom.submit = '' @@ -1807,28 +1831,21 @@ class SQLFORM(FORM): buttonurl=url(args=[]), callback=None, delete=None, trap=True): if showbuttontext: - if callback: - return A(SPAN(_class=ui.get(buttonclass)), - SPAN(T(buttontext), _title=buttontext, - _class=ui.get('buttontext')), - callback=callback, delete=delete, - _class=trap_class(ui.get('button'), trap)) - else: - return A(SPAN(_class=ui.get(buttonclass)), - SPAN(T(buttontext), _title=buttontext, - _class=ui.get('buttontext')), - _href=buttonurl, - _class=trap_class(ui.get('button'), trap)) + return A(SPAN(_class=ui.get(buttonclass)), + SPAN(T(buttontext), _title=buttontext, + _class=ui.get('buttontext')), + _href=buttonurl, + callback=callback, + delete=delete, + _class=trap_class(ui.get('button'), trap)) else: - if callback: - return A(SPAN(_class=ui.get(buttonclass)), - callback=callback, delete=delete, - _title=buttontext, - _class=trap_class(ui.get('buttontext'), trap)) - else: - return A(SPAN(_class=ui.get(buttonclass)), - _href=buttonurl, _title=buttontext, - _class=trap_class(ui.get('buttontext'), trap)) + return A(SPAN(_class=ui.get(buttonclass)), + _href=buttonurl, + callback=callback, + delete=delete, + _title=buttontext, + _class=trap_class(ui.get('buttontext'), trap)) + dbset = db(query) tablenames = db._adapter.tables(dbset.query) if left is not None: @@ -1948,8 +1965,8 @@ class SQLFORM(FORM): table = db[request.args[-2]] if ondelete: ondelete(table, request.args[-1]) - ret = db(table[table._id.name] == request.args[-1]).delete() - return ret + db(table[table._id.name] == request.args[-1]).delete() + redirect(referrer) exportManager = dict( csv_with_hidden_cols=(ExporterCSV, 'CSV (hidden cols)'), @@ -2267,7 +2284,7 @@ class SQLFORM(FORM): elif not isinstance(value, DIV): value = field.formatter(value) trcols.append(TD(value)) - row_buttons = TD(_class='row_buttons') + row_buttons = TD(_class='row_buttons',_nowrap=True) if links and links_in_grid: toadd = [] for link in links: @@ -2293,6 +2310,7 @@ class SQLFORM(FORM): if deletable and (not callable(deletable) or deletable(row)): row_buttons.append(gridbutton( 'buttondelete', 'Delete', + url(args=['delete', tablename, id]), callback=url(args=['delete', tablename, id]), delete='tr')) if buttons_placement in ['right', 'both']: diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index d92ac0e6..a703801d 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -31,6 +31,7 @@ ALLOWED_DATATYPES = [ 'datetime', 'upload', 'password', + 'json', ] @@ -62,14 +63,14 @@ class TestFields(unittest.TestCase): def testFieldTypes(self): - # Check that string, text, and password default length is 512 + # Check that string, and password default length is 512 for typ in ['string', 'password']: self.assert_(Field('abc', typ).length == 512, "Default length for type '%s' is not 512 or 255" % typ) # Check that upload default length is 512 self.assert_(Field('abc', 'upload').length == 512, - "Default length for type 'upload' is not 128") + "Default length for type 'upload' is not 512") # Check that Tables passed in the type creates a reference self.assert_(Field('abc', Table(None, 'temp')).type @@ -113,6 +114,10 @@ class TestFields(unittest.TestCase): self.assertEqual(db.t.insert(a=True), 1) self.assertEqual(db().select(db.t.a)[0].a, True) db.t.drop() + db.define_table('t', Field('a', 'json', default={})) + self.assertEqual(db.t.insert(a={}), 1) + self.assertEqual(db().select(db.t.a)[0].a, {}) + db.t.drop() db.define_table('t', Field('a', 'date', default=datetime.date.today())) t0 = datetime.date.today() diff --git a/gluon/tests/test_html.py b/gluon/tests/test_html.py index 336eff9b..1273f2bd 100644 --- a/gluon/tests/test_html.py +++ b/gluon/tests/test_html.py @@ -74,7 +74,7 @@ class TestBareHelpers(unittest.TestCase): def testFORM(self): self.assertEqual(FORM('<>', _a='1', _b='2').xml(), - '
<>
') + '
<>
') def testH1(self): self.assertEqual(H1('<>', _a='1', _b='2').xml(), diff --git a/gluon/tools.py b/gluon/tools.py index 4644cc61..0244fee9 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -272,7 +272,7 @@ class Mail(object): cc=None, bcc=None, reply_to=None, - sender='%(sender)s', + sender=None, encoding='utf-8', raw=False, headers={} @@ -360,9 +360,11 @@ class Mail(object): text = encode_header(text) return text + sender = sender or self.settings.sender + if not isinstance(self.settings.server, str): raise Exception('Server address not specified') - if not isinstance(self.settings.sender, str): + if not isinstance(sender, str): raise Exception('Sender address not specified') if not raw: @@ -458,11 +460,11 @@ class Mail(object): c.set_armor(1) c.signers_clear() # search for signing key for From: - for sigkey in c.op_keylist_all(self.settings.sender, 1): + for sigkey in c.op_keylist_all(sender, 1): if sigkey.can_sign: c.signers_add(sigkey) if not c.signers_enum(0): - self.error = 'No key for signing [%s]' % self.settings.sender + self.error = 'No key for signing [%s]' % sender return False c.set_passphrase_cb(lambda x, y, z: sign_passphrase) try: @@ -619,7 +621,6 @@ class Mail(object): # no cryptography process as usual payload = payload_in - sender = sender % dict(sender=self.settings.sender) payload['From'] = encoded_or_raw(sender.decode(encoding)) origTo = to[:] if to: @@ -655,16 +656,16 @@ class Mail(object): attachments = attachments and [(a.my_filename, a.my_payload) for a in attachments if not raw] if attachments: result = mail.send_mail( - sender=self.settings.sender, to=origTo, + sender=sender, to=origTo, subject=subject, body=text, html=html, attachments=attachments, **xcc) elif html and (not raw): result = mail.send_mail( - sender=self.settings.sender, to=origTo, + sender=sender, to=origTo, subject=subject, body=text, html=html, **xcc) else: result = mail.send_mail( - sender=self.settings.sender, to=origTo, + sender=sender, to=origTo, subject=subject, body=text, **xcc) else: smtp_args = self.settings.server.split(':') @@ -679,7 +680,7 @@ class Mail(object): if self.settings.login: server.login(*self.settings.login.split(':', 1)) result = server.sendmail( - self.settings.sender, to, payload.as_string()) + sender, to, payload.as_string()) server.quit() except Exception, e: logger.warn('Mail.send failure:%s' % e) @@ -881,6 +882,7 @@ class Auth(object): profile_fields=None, email_case_sensitive=True, username_case_sensitive=True, + update_fields = ['email'], ) # ## these are messages that can be customized default_messages = dict( @@ -2047,7 +2049,8 @@ class Auth(object): if not self in self.settings.login_methods: # do not store password in db form.vars[passfield] = None - user = self.get_or_create_user(form.vars) + user = self.get_or_create_user( + form.vars, self.settings.update_fields) break if not user: # alternates have failed, maybe because service inaccessible @@ -2067,7 +2070,8 @@ class Auth(object): if not self in self.settings.login_methods: # do not store password in db form.vars[passfield] = None - user = self.get_or_create_user(form.vars) + user = self.get_or_create_user( + form.vars, self.settings.update_fields) break if not user: self.log_event(self.messages.login_failed_log, @@ -2087,7 +2091,8 @@ class Auth(object): if cas_user: cas_user[passfield] = None user = self.get_or_create_user( - table_user._filter_fields(cas_user)) + table_user._filter_fields(cas_user), + self.settings.update_fields) elif hasattr(cas, 'login_form'): return cas.login_form() else: @@ -3309,14 +3314,14 @@ class Auth(object): restrict_search=False, resolve=True, extra=None, - menugroups=None): + menu_groups=None): if not hasattr(self, '_wiki'): self._wiki = Wiki(self, render=render, manage_permissions=manage_permissions, force_prefix=force_prefix, restrict_search=restrict_search, env=env, extra=extra or {}, - menugroups=menugroups) + menu_groups=menu_groups) else: self._wiki.env.update(env or {}) # if resolve is set to True, process request as wiki call @@ -4694,15 +4699,19 @@ class Expose(object): class Wiki(object): everybody = 'everybody' rows_page = 25 - - def markmin_render(self, page): - html = MARKMIN(page.body, extra=self.extra, + def markmin_base(self,body): + return MARKMIN(body, extra=self.extra, url=True, environment=self.env, autolinks=lambda link: expand_one(link, {})).xml() - html += DIV(_class='w2p_wiki_tags', - *[A(t.strip(), _href=URL(args='_search', vars=dict(q=t))) - for t in page.tags or [] if t.strip()]).xml() - return html + + def render_tags(self, tags): + return DIV( + _class='w2p_wiki_tags', + *[A(t.strip(), _href=URL(args='_search', vars=dict(q=t))) + for t in tags or [] if t.strip()]) + + def markmin_render(self, page): + return self.markmin_base(page.body) + self.render_tags(page.tags).xml() def html_render(self, page): html = page.body @@ -4712,6 +4721,7 @@ class Wiki(object): html = replace_autolinks(html, lambda link: expand_one(link, {})) # @{component:name} -> html = replace_components(html, self.env) + html = html + self.render_tags(page.tags).xml() return html @staticmethod @@ -4726,7 +4736,7 @@ class Wiki(object): def __init__(self, auth, env=None, render='markmin', manage_permissions=False, force_prefix='', - restrict_search=False, extra=None, menugroups=None): + restrict_search=False, extra=None, menu_groups=None): self.env = env or {} self.env['component'] = Wiki.component if render == 'markmin': @@ -4735,7 +4745,7 @@ class Wiki(object): render = self.html_render self.render = render self.auth = auth - self.menugroups = menugroups + self.menu_groups = menu_groups if self.auth.user: self.force_prefix = force_prefix % self.auth.user else: @@ -4853,11 +4863,11 @@ class Wiki(object): return True def can_see_menu(self): - if self.menugroups is None: + if self.menu_groups is None: return True if self.auth.user: groups = self.auth.user_groups.values() - if any(t in self.menugroups for t in groups): + if any(t in self.menu_groups for t in groups): return True return False diff --git a/gluon/validators.py b/gluon/validators.py index 6ec44656..276ed395 100644 --- a/gluon/validators.py +++ b/gluon/validators.py @@ -21,6 +21,16 @@ import unicodedata from cStringIO import StringIO from utils import simple_hash, web2py_uuid, DIGEST_ALG_BY_SIZE +JSONErrors = (NameError, TypeError, ValueError, AttributeError, + KeyError) +try: + import json as simplejson +except ImportError: + from gluon.contrib import simplejson + from gluon.contrib.simplejson.decoder import JSONDecodeError + JSONErrors += (JSONDecodeError,) + + __all__ = [ 'CLEANUP', 'CRYPT', @@ -53,6 +63,7 @@ __all__ = [ 'IS_UPLOAD_FILENAME', 'IS_UPPER', 'IS_URL', + 'IS_JSON', ] try: @@ -300,6 +311,30 @@ class IS_LENGTH(Validator): return (value, translate(self.error_message) % dict(min=self.minsize, max=self.maxsize)) +class IS_JSON(Validator): + """ + example:: + INPUT(_type='text', _name='name', + requires=IS_JSON(error_message="This is not a valid json input") + + >>> IS_JSON()('{"a": 100}') + ('{"a": 100}', None) + + >>> IS_JSON()('spam1234') + ('spam1234', 'invalid json') + """ + + def __init__(self, error_message='invalid json'): + self.error_message = error_message + + def __call__(self, value): + try: + simplejson.loads(value) + return (value, None) + except JSONErrors: + pass + return (value, translate(self.error_message)) + class IS_IN_SET(Validator): """ diff --git a/scripts/setup-ubuntu-12-04-redmine-unicorn-web2py-uwsgi-nginx.sh b/scripts/setup-ubuntu-12-04-redmine-unicorn-web2py-uwsgi-nginx.sh new file mode 100644 index 00000000..84a0b0f2 --- /dev/null +++ b/scripts/setup-ubuntu-12-04-redmine-unicorn-web2py-uwsgi-nginx.sh @@ -0,0 +1,464 @@ +#!/bin/bash + +# ------------------------------------------------------------------------------ +# Description : Installation and basic configuration of web2py, uWSGI, Redmine, +# Unicorn, Nginx and PostgreSQL. +# Usage : Copy the script in /home/username and run it as root, you may +# need to allow exectuion (chmod +x). Ex.: +# sudo ./setup-ubuntu-12-04-redmine-unicorn-web2py-uwsgi-nginx.sh +# File : setup-ubuntu-12-04-redmine-unicorn-web2py-uwsgi-nginx.sh +# Author : Richard V?zina +# Email : ml.richard.vezina@gmail.com +# Copyright : Richard V?zina +# Date : ven 28 d?c 2012 13:27:11 EST +# Disclaimers : This script is provided "as is", without warranty of any kind. +# Licence : CC BY-NC 2.5 CA +# ------------------------------------------------------------------------------ + +echo 'setup-ubuntu-12-04-redmine-unicorn-web2py-uwsgi-nginx.sh' +echo 'Requires Ubuntu = 12.04 (May works with 12.10 not tested) and installs Redmine + Unicorn + Web2py + uWSGI + Nginx + PostgreSQL' +# Check if user has root privileges +if [[ $EUID -ne 0 ]]; then + echo "You must run the script as root or using sudo" + exit 1 +fi + +# ------------------------------------------------------------------------------ +# We concentrate here user prompts!! +# Get Redmine Postgres Database Password +echo -e "Redmine Postgres Database Password: \c " +read REDMINEPASSWORD +# Get Web2py Admin Password +echo -e "Web2py Admin Password: \c " +read PW + +cd ~ +openssl genrsa 1024 > self_signed.key +chmod 400 self_signed.key +openssl req -new -x509 -nodes -sha1 -days 1780 -key self_signed.key > self_signed.cert +openssl x509 -noout -fingerprint -text < self_signed.cert > self_signed.info +# ------------------------------------------------------------------------------ + +apt-get update +apt-get -y upgrade +apt-get autoremove +apt-get autoclean +apt-get -y install postgresql +apt-get -y install nginx-full +apt-get -y install build-essential python-dev libxml2-dev python-pip unzip +apt-get -y install ruby1.9.3 # Ref.: http://askubuntu.com/questions/137485/rails-3-not-using-rvm +apt-get -y install libpq-dev # Required for gem1.9.3 install pg Ref.: http://stackoverflow.com/questions/6040583/unable-to-install-pg-gem-on-ubuntu-cant-find-the-libpq-fe-h-header + +gem1.9.3 install rails --no-rdoc --no-ri # For testing (faster) --no-rdoc --no-ri +gem1.9.3 install unicorn --no-rdoc --no-ri # For testing (faster) --no-rdoc --no-ri +gem1.9.3 install pg --no-rdoc --no-ri # For testing (faster) --no-rdoc --no-ri +cd /opt +wget http://rubyforge.org/frs/download.php/76627/redmine-2.2.0.tar.gz +wget http://rubyforge.org/frs/download.php/76628/redmine-2.2.0.tar.gz.md5 +md5sum --check redmine-2.2.0.tar.gz.md5 > redmine_md5_checked_successfully +if [ -f redmine_md5_checked_successfully ] +then + tar xvfz redmine-2.2.0.tar.gz + rm redmine_md5_checked_successfully +else + echo "Redmine md5 check sum failed..." + exit 1 +fi +cd redmine-2.2.0 +bundle install --without development test rmagick sqlite mysql +mkdir /var/www +ln -s /opt/redmine-2.2.0/public /var/www/redmine +chown -R www-data.www-data /var/www +chown -R www-data.www-data /opt/redmine-2.2.0/public +# To avoid prompt during execution of the script use psql instead of createuser +#echo "Enter a postgres redmine user password twice:" +#createuser -P -S -D -R -l -e redmine +# createuser switch: -P --pwprompt -S --no-superuser -D --no-createdb -R --no-createrole -l --login -e --echo +sudo -u postgres psql -c "CREATE ROLE redmine LOGIN; ALTER ROLE redmine WITH ENCRYPTED PASSWORD '$REDMINEPASSWORD';" +# createdb wouldn't work without having root password +#createdb -U postgres -w -E UTF8 -O redmine -e redmine +# createdb switch: -U username --username=username -w --no-password -E Encoding -O owner --owner=owner -e --echo +sudo -u postgres psql -c "CREATE DATABASE redmine WITH ENCODING='UTF8' OWNER=redmine;" +cd /opt/redmine-2.2.0/config +# Here we change related to an issue with new rails version as far as I understand +# Ref1.: http://www.redmine.org/projects/redmine/wiki/HowTo_Install_Redmine_in_a_sub-URI # Preferred solution used +# Ref2.: http://www.redmine.org/issues/12102 # JS and CSS was not working until I add this line 'RedmineApp::Application.routes.default_scope = { :path => "/redmine", :shallow_path => "/redmine" }' before 'RedmineApp::Application.initialize!' +cp environment.rb environment.rb_original # Backup default environment.rb +sed '/RedmineApp::Application.initialize!/c \RedmineApp::Application.routes.default_scope = { :path => "/redmine", :shallow_path => "/redmine" }\nRedmineApp::Application.initialize!\nRedmine::Utils::relative_url_root = "/redmine"' environment.rb_original > environment.rb +# Now we configure Redmine database access +#nano database.yml +# paste : +echo 'production: + adapter: postgresql + database: redmine + host: localhost + username: redmine + password: "'$REDMINEPASSWORD'" + encoding: utf8' > database.yml +rake generate_secret_token +RAILS_ENV=production rake db:migrate +RAILS_ENV=production rake redmine:load_default_data +mkdir /opt/redmine-2.2.0/tmp/pids +#mkdir /opt/redmine-2.2.0/log # if not there +cd /opt/redmine-2.2.0/config +# Create Unicorn specific Redmine config in /opt/redmine-2.2.0/config/unicorn.rb +echo '#unicorn.rb Starts here +worker_processes 1 +working_directory "/opt/redmine-2.2.0" # needs to be the correct directory for redmine + +# This loads the application in the master process before forking +# worker processes +# Read more about it here: +# http://unicorn.bogomips.org/Unicorn/Configurator.html +preload_app true +timeout 45 + +# This is where we specify the socket. +# We will point the upstream Nginx module to this socket later on +listen "/tmp/unicorn_rails.socket", :backlog => 64 #directory structure needs to be created. +pid "/opt/redmine-2.2.0/tmp/pids/unicorn_rails.pid" # make sure this points to a valid directory. Make sure it is named the same as the real process name in order to allow init.d script start-stop-daemon command to kill unicorn process properly + +# Set the path of the log files inside the log folder of the testapp +stderr_path "/opt/redmine-2.2.0/log/unicorn_rails.stderr.log" +stdout_path "/opt/redmine-2.2.0/log/unicorn_rails.stdout.log" + +before_fork do |server, worker| +# This option works in together with preload_app true setting +# What is does is prevent the master process from holding +# the database connection +defined?(ActiveRecord::Base) and +ActiveRecord::Base.connection.disconnect! +end + +after_fork do |server, worker| +# Here we are establishing the connection after forking worker +# processes +defined?(ActiveRecord::Base) and +ActiveRecord::Base.establish_connection +# change below if your redmine instance is running differently +worker.user('\''www-data'\'', '\''www-data'\'') if Process.euid == 0 +end +#unicorn.rb Ends here' > unicorn.rb +chown www-data:www-data unicorn.rb +chown -R www-data:www-data /opt/redmine-2.2.0/tmp +mkdir /etc/unicorn +# Set some config for Unicorn in /etc/unicorn/redmine +echo 'RAILS_ROOT=/opt/redmine-2 +RAILS_ENV=production' > /etc/unicorn/redmine +# Create a Unicorn Redmine start script in /etc/init.d/redmine +echo '#! /bin/sh +### BEGIN INIT INFO +# Provides: redmine +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: redmine initscript +# Description: This script startup unicorn server and redmine and should +# be placed in /etc/init.d. +### END INIT INFO + +# ------------------------------------------------------------------------------ +# Author: Richard V?zina +# Base on Ubuntu 12.04 : /etc/init.d/skeleton +# ven 21 d?c 2012 11:08:31 EST +# ------------------------------------------------------------------------------ + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +APP=/opt/redmine-2.2.0/ +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Unicorn and Redmine" +NAME=unicorn_rails +DAEMON=/usr/local/bin/$NAME +DAEMON_ARGS=" -E production -c $APP/config/unicorn.rb -D" +PIDFILE=/opt/redmine-2.2.0/tmp/pids/$NAME.pid +SCRIPTNAME=/etc/init.d/redmine + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don'\''t delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave '\''force-reload'\'' as an alias for '\''restart'\''. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # '\''force-reload'\'' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +:' > /etc/init.d/redmine +chmod +x /etc/init.d/redmine +# Backup default Nginx site and replace it +cp /etc/nginx/sites-available/default /etc/nginx/sites-available/default_original +rm /etc/nginx/sites-available/default +# Create configuration file /etc/nginx/sites-available/default +echo 'upstream unicorn_server { + # This is the socket we configured in unicorn.rb + server unix:/tmp/unicorn_rails.socket + fail_timeout=0; +} +server { + listen 80; + #return 301 https://192.168.1.126$request_uri; # http://$hostname$request_uri; # idem #http://wiki.nginx.org/Pitfalls#Taxing_Rewrites + charset utf-8; + server_name localhost; # $hostname; + root /var/www; + access_log /var/log/nginx/yoursite.access.log; + error_log /var/log/nginx/yoursite.error.log; + #to enable correct use of response.static_version + #location ~* /(\w+)/static(?:/_[\d]+\.[\d]+\.[\d]+)?/(.*)$ { + # alias /home/www-data/web2py/applications/$1/static/$2; + # expires max; + #} + location ~* /(\w+)/static/ { + root /home/www-data/web2py/applications/; + #remove next comment on production + #expires max; + } + location ~^\/(?!redmine(.*)) { + #uwsgi_pass 127.0.0.1:9001; + uwsgi_pass unix:///tmp/web2py.socket; + include uwsgi_params; + uwsgi_param UWSGI_SCHEME $scheme; + uwsgi_param SERVER_SOFTWARE nginx/$nginx_version; + } + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + + if (!-f $request_filename) { + proxy_pass http://unicorn_server; + break; + } + } +} +server { + listen 443 default_server ssl; + charset utf-8; + server_name localhost; # $hostname; + root /var/www; + ssl_certificate /etc/nginx/ssl/self_signed.cert; + ssl_certificate_key /etc/nginx/ssl/self_signed.key; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + ssl_ciphers ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA; + ssl_protocols SSLv3 TLSv1; + keepalive_timeout 70; + location ~^\/(?!redmine(.*)) { + #uwsgi_pass 127.0.0.1:9001; + uwsgi_pass unix:///tmp/web2py.socket; + include uwsgi_params; + uwsgi_param UWSGI_SCHEME $scheme; + uwsgi_param SERVER_SOFTWARE nginx/$nginx_version; + } + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + + if (!-f $request_filename) { + proxy_pass http://unicorn_server; + break; + } + } + +}' >/etc/nginx/sites-available/default + +#ln -s /etc/nginx/sites-available/web2py /etc/nginx/sites-enabled/web2py +#rm /etc/nginx/sites-enabled/default + +# We copy ssl files we previously created +if [ -f /etc/nginx/ssl ] +then + cp ~/self_signed.* /etc/nginx/ssl/ + rm ~/self_signed.* +else + mkdir /etc/nginx/ssl + cp ~/self_signed.* /etc/nginx/ssl/ + rm ~/self_signed.* +fi + +pip install --upgrade pip +PIPPATH=`which pip` +$PIPPATH install --upgrade uwsgi + +# Prepare folders for uwsgi +sudo mkdir /etc/uwsgi +sudo mkdir /var/log/uwsgi + +# Create configuration file /etc/uwsgi/web2py.xml +echo ' + /tmp/web2py.socket + /home/www-data/web2py/ + /=wsgihandler:application + + 4 + 60 + 8 + 1 + /tmp/stats.socket + 2000 + 512 + 256 + 192 + www-data + www-data + 0 0 -1 -1 -1 python /home/www-data/web2py/web2py.py -Q -S welcome -M -R scripts/sessions2trash.py -A -o + +' > /etc/uwsgi/web2py.xml + +#Create a configuration file for uwsgi in emperor-mode +#for Upstart in /etc/init/uwsgi-emperor.conf +echo '# Emperor uWSGI script + +description "uWSGI Emperor" +start on runlevel [2345] +stop on runlevel [06] +## +#remove the comments in the next section to enable static file compression for the welcome app +#in that case, turn on gzip_static on; on /etc/nginx/nginx.conf +## +#pre-start script +# python /home/www-data/web2py/web2py.py -S welcome -R scripts/zip_static_files.py +# chown -R www-data:www-data /home/www-data/web2py/* +#end script +respawn +exec uwsgi --master --die-on-term --emperor /etc/uwsgi --logto /var/log/uwsgi/uwsgi.log +' > /etc/init/uwsgi-emperor.conf +# Install Web2py +mkdir /home/www-data +cd /home/www-data +wget http://web2py.com/examples/static/web2py_src.zip +unzip web2py_src.zip +rm web2py_src.zip +# Download latest version of sessions2trash.py +wget http://web2py.googlecode.com/hg/scripts/sessions2trash.py -O /home/www-data/web2py/scripts/sessions2trash.py +chown -R www-data:www-data web2py +cd /home/www-data/web2py +sudo -u www-data python -c "from gluon.main import save_password; save_password('$PW',443)" +/etc/init.d/redmine start +start uwsgi-emperor +/etc/init.d/nginx restart +ufw allow 80 # Or check your firewall configuration