From 61d81c01c497d6ac29ac60d8edb9b4ab88e73be1 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 15 Aug 2013 09:09:19 -0500 Subject: [PATCH 01/24] refactoring of belongs, thanks Jonathan --- VERSION | 2 +- gluon/dal.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/VERSION b/VERSION index be967c51..cf73e38f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.09.16.07.32 +Version 2.6.0-development+timestamp.2013.08.15.09.08.28 diff --git a/gluon/dal.py b/gluon/dal.py index 9ad7bb17..6e7f4431 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -1263,17 +1263,12 @@ class BaseAdapter(ConnectionPool): return '(%s OR %s)' % (self.expand(first), self.expand(second)) def BELONGS(self, first, second): - if isinstance(second, str): - return '(%s IN (%s))' % (self.expand(first), second[:-1]) - if not second: - return '(1=0)' - if isinstance(second, (list,tuple,frozenset)): - second = set(second) # remove duplicates, make mutable - if isinstance(second, set) and None in second: - second.remove(None) - return self.OR(self.EQ(first, None), self.BELONGS(first, second)) - items = ','.join(self.expand(item, first.type) for item in second) - return '(%s IN (%s))' % (self.expand(first), items) + if isinstance(second, str): + return '(%s IN (%s))' % (self.expand(first), second[:-1]) + if not second: + return '(1=0)' + items = ','.join(self.expand(item, first.type) for item in second) + return '(%s IN (%s))' % (self.expand(first), items) def REGEXP(self, first, second): "regular expression operator" @@ -9070,7 +9065,7 @@ class Expression(object): db = self.db return Query(db, db._adapter.REGEXP, self, value) - def belongs(self, *value): + def belongs(self, *value, **kwattr): """ Accepts the following inputs: field.belongs(1,2) @@ -9085,6 +9080,11 @@ class Expression(object): value = value[0] if isinstance(value,Query): value = db(value)._select(value.first._table._id) + elif not isinstance(value, basestring): + value = set(value) + if kwattr.get('null') and None in value: + value.remove(None) + return (self == None) | Query(db, db._adapter.BELONGS, self, value) return Query(db, db._adapter.BELONGS, self, value) def startswith(self, value): From 2cdcdaf9320146a6904b9d57da46dc8053a47eff Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 15 Aug 2013 09:21:58 -0500 Subject: [PATCH 02/24] possibly fixed Issue 1630:memdb id error -- tries casting None to long., thanks Luca --- VERSION | 2 +- gluon/contrib/memdb.py | 5 ++--- gluon/globals.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index cf73e38f..e95fe15e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.15.09.08.28 +Version 2.6.0-development+timestamp.2013.08.15.09.21.06 diff --git a/gluon/contrib/memdb.py b/gluon/contrib/memdb.py index 6396cfe2..f619bdeb 100644 --- a/gluon/contrib/memdb.py +++ b/gluon/contrib/memdb.py @@ -502,9 +502,8 @@ class Query(object): 'Query: right side of filter must be a value or entity') if isinstance(left, Field) and left.name == 'id': if op == '=': - self.get_one = \ - QueryException(tablename=left._tablename, - id=long(right)) + self.get_one = QueryException( + tablename=left._tablename, id=long(right or 0)) return else: raise SyntaxError('only equality by id is supported') diff --git a/gluon/globals.py b/gluon/globals.py index 9798cb1a..1f856f90 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -741,7 +741,7 @@ class Session(Storage): if record_id == '0': raise Exception('record_id == 0') # Select from database - row = db(table.id == record_id).select() + row = record_id and db(table.id == record_id).select() row = row and row[0] or None # Make sure the session data exists in the database if not row or row.unique_key != unique_key: From c4f7c4984629588bc5c17b5abc20d76f9729726d Mon Sep 17 00:00:00 2001 From: mdipierro Date: Thu, 15 Aug 2013 10:06:21 -0500 Subject: [PATCH 03/24] fixed issue 1631:elements(..., replace=None) deletes the first element only --- VERSION | 2 +- gluon/html.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/VERSION b/VERSION index e95fe15e..43a6dc81 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.15.09.21.06 +Version 2.6.0-development+timestamp.2013.08.15.10.05.27 diff --git a/gluon/html.py b/gluon/html.py index 61a5bee8..da092a33 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -1075,10 +1075,8 @@ class DIV(XmlComponent): matches = [] # check if the component has an attribute with the same # value as provided - check = True tag = getattr(self, 'tag').replace('/', '') - if args and tag not in args: - check = False + check = not (args and tag not in args) for (key, value) in kargs.iteritems(): if key not in ['first_only', 'replace', 'find_text']: if isinstance(value, (str, int)): @@ -1109,24 +1107,28 @@ class DIV(XmlComponent): def replace_component(i): if replace is None: del self[i] - elif callable(replace): - self[i] = replace(self[i]) + return i else: - self[i] = replace + self[i] = replace(self[i]) if callable(replace) else replace + return i+1 # loop the components if find_text or find_components: - for i, c in enumerate(self.components): + i = 0 + while i Date: Thu, 15 Aug 2013 17:15:59 -0500 Subject: [PATCH 04/24] http://www.451unavailable.org/ --- Introduction | 0 VERSION | 2 +- gluon/http.py | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 Introduction diff --git a/Introduction b/Introduction deleted file mode 100644 index e69de29b..00000000 diff --git a/VERSION b/VERSION index 43a6dc81..fa42ba84 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.15.10.05.27 +Version 2.6.0-development+timestamp.2013.08.15.17.14.24 diff --git a/gluon/http.py b/gluon/http.py index 6f1b2e1e..c10c6a63 100644 --- a/gluon/http.py +++ b/gluon/http.py @@ -44,6 +44,7 @@ defined_status = { 416: 'REQUESTED RANGE NOT SATISFIABLE', 417: 'EXPECTATION FAILED', 422: 'UNPROCESSABLE ENTITY', + 451: 'UNAVAILABLE FOR LEGAL REASONS', # http://www.451unavailable.org/ 500: 'INTERNAL SERVER ERROR', 501: 'NOT IMPLEMENTED', 502: 'BAD GATEWAY', From e4d8c4ea179c600ee670ba91271dbd4015f02299 Mon Sep 17 00:00:00 2001 From: Mariano Reingart Date: Fri, 16 Aug 2013 02:27:47 -0300 Subject: [PATCH 05/24] updated pyfpdf (html enhancements) --- gluon/contrib/fpdf/html.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gluon/contrib/fpdf/html.py b/gluon/contrib/fpdf/html.py index a1269bc9..0863d5ed 100644 --- a/gluon/contrib/fpdf/html.py +++ b/gluon/contrib/fpdf/html.py @@ -33,7 +33,6 @@ class HTML2FPDF(HTMLParser): self.href = '' self.align = '' self.page_links = {} - self.font_list = ("times","courier", "helvetica") self.font = None self.font_stack = [] self.pdf = pdf @@ -55,6 +54,7 @@ class HTML2FPDF(HTMLParser): self.thead = None self.tfoot = None self.theader_out = self.tfooter_out = False + self.hsize = dict(h1=2, h2=1.5, h3=1.17, h4=1, h5=0.83, h6=0.67) def width2mm(self, length): if length[-1]=='%': @@ -88,7 +88,7 @@ class HTML2FPDF(HTMLParser): else: self.set_style('B',True) border = border or 'B' - align = 'C' + align = self.td.get('align', 'C')[0].upper() bgcolor = hex2dec(self.td.get('bgcolor', self.tr.get('bgcolor', ''))) # parsing table header/footer (drawn later): if self.thead is not None: @@ -179,8 +179,8 @@ class HTML2FPDF(HTMLParser): self.pdf.ln(5) if attrs: if attrs: self.align = attrs.get('align') - if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'): - k = (2, 1.5, 1.17, 1, 0.83, 0.67)[int(tag[1])] + if tag in self.hsize: + k = self.hsize[tag] self.pdf.ln(5*k) self.pdf.set_text_color(150,0,0) self.pdf.set_font_size(12 * k) @@ -219,7 +219,7 @@ class HTML2FPDF(HTMLParser): self.color = hex2dec(attrs['color']) self.set_text_color(*color) self.color = color - if 'face' in attrs and attrs['face'].lower() in self.font_list: + if 'face' in attrs: face = attrs.get('face').lower() self.pdf.set_font(face) self.font_face = face From f391c9cf22fd67f31b35e455cfd1d8abacbfbaa8 Mon Sep 17 00:00:00 2001 From: Mariano Reingart Date: Fri, 16 Aug 2013 02:27:53 -0300 Subject: [PATCH 06/24] updated pyfpdf (template Row.copy, fixes issue 1255) --- gluon/contrib/fpdf/template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluon/contrib/fpdf/template.py b/gluon/contrib/fpdf/template.py index 2d38c9f9..916e8355 100644 --- a/gluon/contrib/fpdf/template.py +++ b/gluon/contrib/fpdf/template.py @@ -110,8 +110,8 @@ class Template: pdf.set_auto_page_break(False,margin=0) for element in sorted(self.elements,key=lambda x: x['priority']): - #print "dib",element['type'], element['name'], element['x1'], element['y1'], element['x2'], element['y2'] - element = element.copy() + # make a copy of the element: + element = dict(element) element['text'] = self.texts[pg].get(element['name'].lower(), element['text']) if 'rotate' in element: pdf.rotate(element['rotate'], element['x1'], element['y1']) From ad29ebe86643c60c1c5071d6edecbcbb17861fe0 Mon Sep 17 00:00:00 2001 From: niphlod Date: Sat, 17 Aug 2013 14:02:46 +0200 Subject: [PATCH 07/24] profiler now save dumps in a dir --- fcgihandler.py | 2 +- gluon/main.py | 54 +++++++++++++++++++++++++++---------------------- gluon/widget.py | 29 +++++++++++++------------- scgihandler.py | 2 +- wsgihandler.py | 2 +- 5 files changed, 48 insertions(+), 41 deletions(-) diff --git a/fcgihandler.py b/fcgihandler.py index a3d4a5df..7d5719fb 100755 --- a/fcgihandler.py +++ b/fcgihandler.py @@ -42,7 +42,7 @@ import gluon.contrib.gateways.fcgi as fcgi if LOGGING: application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase, logfilename='httpserver.log', - profilerfilename=None) + profiler_dir=None) else: application = gluon.main.wsgibase diff --git a/gluon/main.py b/gluon/main.py index 4dc3695d..62af77c7 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -26,6 +26,7 @@ import signal import socket import random import urllib2 +import string try: @@ -40,6 +41,7 @@ from thread import allocate_lock from fileutils import abspath, write_file from settings import global_settings +from utils import web2py_uuid from admin import add_path_first, create_missing_folders, create_missing_app_folders from globals import current @@ -379,7 +381,7 @@ def wsgibase(environ, responder): # ################################################## # access the requested application # ################################################## - + disabled = pjoin(request.folder, 'DISABLED') if not exists(request.folder): if app == rwthread.routes.default_application \ @@ -396,7 +398,7 @@ def wsgibase(environ, responder): % 'invalid request', web2py_error='invalid application') elif request.is_local and exists(disabled): - data = dict([item.strip() for item in line.split(':',1)] + data = dict([item.strip() for item in line.split(':',1)] for line in open(disabled) if line.strip()) if data.get('disabled','True').lower() != 'false': if 'redirect' in data: @@ -605,7 +607,7 @@ def save_password(password, port): def appfactory(wsgiapp=wsgibase, logfilename='httpserver.log', - profilerfilename='profiler.log'): + profiler_dir=None): """ generates a wsgi application that does logging and profiling and calls wsgibase @@ -616,9 +618,22 @@ def appfactory(wsgiapp=wsgibase, [, profilerfilename='profiler.log']]]) """ - if profilerfilename and exists(profilerfilename): - os.unlink(profilerfilename) - locker = allocate_lock() + + if profiler_dir: + profiler_dir = abspath(profiler_dir) + logger.warn('profiler is on. will use dir %s', profiler_dir) + if not os.path.isdir(profiler_dir): + try: + os.makedirs(profiler_dir) + except: + raise BaseException, "Can't create dir %s" % profiler_dir + filepath = pjoin(profiler_dir, 'wtest') + try: + filehandle = open( filepath, 'w' ) + filehandle.close() + os.unlink(filepath) + except IOError: + raise BaseException, "Unable to write to dir %s" % profiler_dir def app_with_logging(environ, responder): """ @@ -636,25 +651,17 @@ def appfactory(wsgiapp=wsgibase, time_in = time.time() ret = [0] - if not profilerfilename: + if not profiler_dir: ret[0] = wsgiapp(environ, responder2) else: import cProfile - import pstats - logger.warn('profiler is on. this makes web2py slower and serial') + prof = cProfile.Profile() + prof.enable() + ret[0] = wsgiapp(environ, responder2) + prof.disable() + destfile = pjoin(profiler_dir, "req_%s.prof" % web2py_uuid()) + prof.dump_stats(destfile) - locker.acquire() - cProfile.runctx('ret[0] = wsgiapp(environ, responder2)', - globals(), locals(), profilerfilename + '.tmp') - stat = pstats.Stats(profilerfilename + '.tmp') - stat.stream = cStringIO.StringIO() - stat.strip_dirs().sort_stats("time").print_stats(80) - profile_out = stat.stream.getvalue() - profile_file = open(profilerfilename, 'a') - profile_file.write('%s\n%s\n%s\n%s\n\n' % - ('=' * 60, environ['PATH_INFO'], '=' * 60, profile_out)) - profile_file.close() - locker.release() try: line = '%s, %s, %s, %s, %s, %s, %f\n' % ( environ['REMOTE_ADDR'], @@ -677,7 +684,6 @@ def appfactory(wsgiapp=wsgibase, return app_with_logging - class HttpServer(object): """ the web2py web server (Rocket) @@ -690,7 +696,7 @@ class HttpServer(object): password='', pid_filename='httpserver.pid', log_filename='httpserver.log', - profiler_filename=None, + profiler_dir=None, ssl_certificate=None, ssl_private_key=None, ssl_ca_certificate=None, @@ -755,7 +761,7 @@ class HttpServer(object): logger.info('SSL is ON') app_info = {'wsgi_app': appfactory(wsgibase, log_filename, - profiler_filename)} + profiler_dir)} self.server = rocket.Rocket(interfaces or tuple(sock_list), method='wsgi', diff --git a/gluon/widget.py b/gluon/widget.py index 2ea47e77..1ba8224a 100644 --- a/gluon/widget.py +++ b/gluon/widget.py @@ -66,9 +66,9 @@ def run_system_tests(options): coverage_config = os.environ.get( "COVERAGE_PROCESS_START", os.path.join('gluon', 'tests', 'coverage.ini')) - - call_args = ['coverage', 'run', '--rcfile=%s' % - coverage_config, + + call_args = ['coverage', 'run', '--rcfile=%s' % + coverage_config, '-m', 'unittest', '-v', 'gluon.tests'] except: sys.stderr.write('Coverage was not installed, skipping\n') @@ -159,7 +159,7 @@ def presentation(root): # Prevent garbage collection of img pnl.image = img - def add_label(text='Change Me', font_size=12, + def add_label(text='Change Me', font_size=12, foreground='#195866', height=1): return Tkinter.Label( master=canvas, @@ -339,7 +339,7 @@ class web2pyDialog(object): if start: #the widget takes care of starting the scheduler if self.options.scheduler and self.options.with_scheduler: - apps = [app.strip() for app + apps = [app.strip() for app in self.options.scheduler.split(',') if app in available_apps] for app in apps: @@ -431,7 +431,7 @@ class web2pyDialog(object): url = self.url + arq self.pagesmenu.add_command( label=url, command=lambda u=url: start_browser(u)) - + def quit(self, justHide=False): """ Finish the program execution """ if justHide: @@ -484,7 +484,7 @@ class web2pyDialog(object): return self.error('invalid port number') # Check for non default value for ssl inputs - if (len(self.options.ssl_certificate) > 0 or + if (len(self.options.ssl_certificate) > 0 or len(self.options.ssl_private_key) > 0): proto = 'https' else: @@ -503,7 +503,7 @@ class web2pyDialog(object): password, pid_filename=options.pid_filename, log_filename=options.log_filename, - profiler_filename=options.profiler_filename, + profiler_dir=options.profiler_dir, ssl_certificate=options.ssl_certificate, ssl_private_key=options.ssl_private_key, ssl_ca_certificate=options.ssl_ca_certificate, @@ -865,9 +865,9 @@ def console(): parser.add_option('-F', '--profiler', - dest='profiler_filename', + dest='profiler_dir', default=None, - help='profiler filename') + help='profiler dir') parser.add_option('-t', '--taskbar', @@ -914,7 +914,7 @@ def console(): dest='run_system_tests', default=False, help=msg) - + msg = ('adds coverage reporting (needs --run_system_tests), ' 'python 2.7 and the coverage module installed. ' 'You can alter the default path setting the environmental ' @@ -925,7 +925,7 @@ def console(): dest='with_coverage', default=False, help=msg) - + if '-A' in sys.argv: k = sys.argv.index('-A') elif '--args' in sys.argv: @@ -1117,7 +1117,8 @@ def start(cron=True): if not options.args is None: sys.argv[:] = options.args run(options.shell, plain=options.plain, bpython=options.bpython, - import_models=options.import_models, startfile=options.run, cronjob=options.cronjob) + import_models=options.import_models, startfile=options.run, + cronjob=options.cronjob) return # ## if -C start cron run (extcron) and exit @@ -1267,7 +1268,7 @@ end tell password=options.password, pid_filename=options.pid_filename, log_filename=options.log_filename, - profiler_filename=options.profiler_filename, + profiler_dir=options.profiler_dir, ssl_certificate=options.ssl_certificate, ssl_private_key=options.ssl_private_key, ssl_ca_certificate=options.ssl_ca_certificate, diff --git a/scgihandler.py b/scgihandler.py index c7f51cfd..2c3fb03c 100755 --- a/scgihandler.py +++ b/scgihandler.py @@ -61,7 +61,7 @@ wsgiapp = WSGIFilterMiddleware(gluon.main.wsgibase, GzipWSGIFilter) if LOGGING: application = gluon.main.appfactory(wsgiapp=wsgiapp, logfilename='httpserver.log', - profilerfilename=None) + profiler_dir=None) else: application = wsgiapp diff --git a/wsgihandler.py b/wsgihandler.py index e98c45e4..eadf2ab1 100644 --- a/wsgihandler.py +++ b/wsgihandler.py @@ -35,7 +35,7 @@ import gluon.main if LOGGING: application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase, logfilename='httpserver.log', - profilerfilename=None) + profiler_dir=None) else: application = gluon.main.wsgibase From 28cfd0a0f23505f698f62efeebecb297c6e76f88 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 17 Aug 2013 11:31:36 -0500 Subject: [PATCH 08/24] fixed issue 1634:retrieve password regression, thanks christophe.varoqui --- VERSION | 2 +- gluon/tools.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index fa42ba84..09c7ce06 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.15.17.14.24 +Version 2.6.0-development+timestamp.2013.08.17.11.30.09 diff --git a/gluon/tools.py b/gluon/tools.py index e33f250b..aaa7414b 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -2642,9 +2642,10 @@ class Auth(object): redirect(self.url(args=request.args)) password = self.random_password() passfield = self.settings.password_field - d = dict( - passfield=str(table_user[passfield].validate(password)[0]), - registration_key='') + d = { + passfield: str(table_user[passfield].validate(password)[0]), + 'registration_key': '' + } user.update_record(**d) if self.settings.mailer and \ self.settings.mailer.send(to=form.vars.email, From aa644ceeb0e197e5944ae9020439c6de78b8e98b Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 18 Aug 2013 02:09:14 -0500 Subject: [PATCH 09/24] isProgrammingError, thanks Alan --- VERSION | 2 +- gluon/dal.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 09c7ce06..7d55c734 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.17.11.30.09 +Version 2.6.0-development+timestamp.2013.08.18.02.08.22 diff --git a/gluon/dal.py b/gluon/dal.py index 6e7f4431..d061953d 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -684,6 +684,11 @@ class BaseAdapter(ConnectionPool): return None return isinstance(exception, self.driver.OperationalError) + def isProgrammingError(self,exception): + if not hasattr(self.driver, "ProgrammingError"): + return None + return isinstance(exception, self.driver.ProgrammingError) + def id_query(self, table): return table._id != None @@ -4328,7 +4333,8 @@ class DatabaseStoredFile: if db.executesql(query): return True except Exception, e: - if not db._adapter.isOperationalError(e): + if not (db._adapter.isOperationalError(e) or + db._adapter.isProgrammingError(e)): raise # no web2py_filesystem found? tb = traceback.format_exc() From 8121e2b99c720c5c8ba4fb6e88d8fcc4bc4f69a0 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sun, 18 Aug 2013 02:50:26 -0500 Subject: [PATCH 10/24] fixed regex in app.example.yaml, thanks Luca --- VERSION | 2 +- app.example.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 7d55c734..c21cf34d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.18.02.08.22 +Version 2.6.0-development+timestamp.2013.08.18.02.49.16 diff --git a/app.example.yaml b/app.example.yaml index eead4703..62cd405b 100644 --- a/app.example.yaml +++ b/app.example.yaml @@ -25,13 +25,13 @@ handlers: # the parametric router's language logic. # You cannot use them together. -- url: /(.+?)/[^_]*\/?static/_\d.\d.\d\/?(.+) +- url: /(.+?)/static/_\d.\d.\d\/(.+) static_files: applications/\1/static/\2 upload: applications/(.+?)/static/(.+) secure: optional expiration: "365d" -- url: /(.+?)/[^_]*\/?static/?(.+) +- url: /(.+?)/static/(.+) static_files: applications/\1/static/\2 upload: applications/(.+?)/static/(.+) secure: optional From c49ae3df046b504716e2b650c7e7f443eb9c9d47 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 19 Aug 2013 08:04:33 -0500 Subject: [PATCH 11/24] fixed again issue 1630:memdb id error -- tries casting None to long. --- VERSION | 2 +- gluon/globals.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c21cf34d..bf89d0bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.18.02.49.16 +Version 2.6.0-development+timestamp.2013.08.19.08.03.43 diff --git a/gluon/globals.py b/gluon/globals.py index 1f856f90..d5d1999b 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -738,9 +738,10 @@ class Session(Storage): # Get session data out of the database (record_id, unique_key) = response.session_id.split(':') - if record_id == '0': + if not record_id.isdigit() or long(record_id)<1: raise Exception('record_id == 0') # Select from database + record_id = long(record_id) row = record_id and db(table.id == record_id).select() row = row and row[0] or None # Make sure the session data exists in the database From 90931657e454491266231d027ebe414c701ab35c Mon Sep 17 00:00:00 2001 From: mdipierro Date: Mon, 19 Aug 2013 08:44:57 -0500 Subject: [PATCH 12/24] fixed issue 1633:IMAP support for storing messages, thanks Alan and Prachi --- VERSION | 2 +- gluon/dal.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bf89d0bf..b23fa922 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.19.08.03.43 +Version 2.6.0-development+timestamp.2013.08.19.08.43.59 diff --git a/gluon/dal.py b/gluon/dal.py index d061953d..36241fd9 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -6489,6 +6489,71 @@ class IMAPAdapter(NoSQLAdapter): processor = attributes.get('processor',self.parse) return processor(imapqry_array, fields, colnames) + def _insert(self, table, fields): + def add_payload(message, obj): + payload = Message() + charset = obj.get("encoding", "utf-8") + payload.set_type(obj.get("mime", None)) + payload.set_charset(charset) + if "text" in obj: + payload.set_payload(obj["text"]) + elif "payload" in obj: + payload.set_payload(obj["payload"]) + if "filename" in obj and obj["filename"]: + payload.add_header("Content-Disposition", + "attachment", filename=obj["filename"]) + message.attach(payload) + + mailbox = table.mailbox + d = dict(((k.name, v) for k, v in fields)) + date_time = (d.get("created", datetime.datetime.now())).timetuple() + if len(d) > 0: + message = d.get("email", None) + attachments = d.get("attachments", []) + content = d.get("content", []) + flags = " ".join(["\\%s" % flag.capitalize() for flag in + ("answered", "deleted", "draft", "flagged", + "recent", "seen") if d.get(flag, False)]) + if not message: + from email.message import Message + mime = d.get("mime", None) + charset = d.get("encoding", None) + message = Message() + message["from"] = d.get("sender", "") + message["subject"] = d.get("subject", "") + if mime: + message.set_type(mime) + if charset: + message.set_charset(charset) + for item in ("to", "cc", "bcc"): + value = d.get(item, "") + if isinstance(value, basestring): + message[item] = value + else: + message[item] = ";".join([i for i in + value]) + if not message.is_multipart(): + if isinstance(content, basestring): + message.set_payload(content) + elif len(content) > 0: + message.set_payload(content[0]["text"]) + else: + [add_payload(message, c) for c in content] + [add_payload(message, a) for a in attachments] + message = message.as_string() + return (mailbox, flags, date_time, message) + else: + raise NotImplementedError("IMAP empty insert is not implemented") + + def insert(self, table, fields): + values = self._insert(table, fields) + result, data = self.connection.append(*values) + if result == "OK": + uid = int(re.findall("\d+", str(data))[-1]) + return self.db(table.uid==uid).select(table.id).first().id + else: + raise Exception("IMAP message append failed: %s" % data) + def _update(self, tablename, query, fields, commit=False): # TODO: the adapter should implement an .expand method commands = list() From 9a86f4f0f326c8133882d9309cacf598afc955c1 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 20 Aug 2013 01:34:30 -0500 Subject: [PATCH 13/24] removed print from sqlhtml.py --- VERSION | 2 +- gluon/sqlhtml.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/VERSION b/VERSION index b23fa922..395edf99 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.19.08.43.59 +Version 2.6.0-development+timestamp.2013.08.20.01.33.36 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index faccbabb..7c87ae6e 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -2209,7 +2209,6 @@ class SQLFORM(FORM): elif key == order[1:]: marker = sorter_icons[1] else: - print 'a', key, ordermatch if key == ordermatch: key, marker = '~' + order, sorter_icons[0] elif key == ordermatch[1:]: From 56dcaff2369680dd7cacf483cad904aed2f5c9b1 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 20 Aug 2013 03:22:54 -0500 Subject: [PATCH 14/24] fixed a problem with sortable grid, new icons --- VERSION | 2 +- gluon/sqlhtml.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 395edf99..906b14c1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.20.01.33.36 +Version 2.6.0-development+timestamp.2013.08.20.03.21.53 diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index 7c87ae6e..b439628e 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -1750,7 +1750,7 @@ class SQLFORM(FORM): oncreate=None, onupdate=None, ondelete=None, - sorter_icons=(XML('↑'), XML('↓')), + sorter_icons=(XML('▲'), XML('▼')), ui = 'web2py', showbuttontext=True, _class="web2py_grid", @@ -2210,7 +2210,7 @@ class SQLFORM(FORM): marker = sorter_icons[1] else: if key == ordermatch: - key, marker = '~' + order, sorter_icons[0] + key, marker = '~' + ordermatch, sorter_icons[0] elif key == ordermatch[1:]: marker = sorter_icons[1] header = A(header, marker, _href=url(vars=dict( From 05f3e01ece2d5663111cfcabff5f3743bb528a44 Mon Sep 17 00:00:00 2001 From: gi0baro Date: Tue, 20 Aug 2013 16:53:08 +0200 Subject: [PATCH 15/24] Fix in redis_session.py select: prevent errors when value is integer/long --- gluon/contrib/redis_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py index ce8fb167..55cd7f9a 100644 --- a/gluon/contrib/redis_session.py +++ b/gluon/contrib/redis_session.py @@ -168,7 +168,7 @@ class MockQuery(object): def select(self): if self.op == 'eq' and self.field == 'id' and self.value: #means that someone wants to retrieve the key self.value - key = self.keyprefix + ':' + self.value + key = self.keyprefix + ':' + str(self.value) if self.with_lock: acquire_lock(self.db, key + ':lock', self.value) rtn = self.db.hgetall(key) From dee4e6a980713f0a3a5bbd350e4b367c4c410327 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 20 Aug 2013 10:58:36 -0500 Subject: [PATCH 16/24] new session logic --- VERSION | 2 +- gluon/globals.py | 411 +++++++++++++++++++++++------------------------ gluon/tools.py | 11 +- 3 files changed, 200 insertions(+), 224 deletions(-) diff --git a/VERSION b/VERSION index 906b14c1..a248a49e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.20.03.21.53 +Version 2.6.0-development+timestamp.2013.08.20.10.57.14 diff --git a/gluon/globals.py b/gluon/globals.py index d5d1999b..11fede2b 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -651,116 +651,34 @@ class Response(Storage): class Session(Storage): - """ defines the session object and the default values of its members (None) + + response.session_storage_type : 'file', 'db', or 'cookie' + response.session_cookie_compression_level : + response.session_cookie_expires : cookie expiration + response.session_cookie_key : for encrypted sessions in cookies + response.session_id : a number or None if no session + response.session_id_name : + response.session_locked : + response.session_masterapp : + response.session_new : a new session obj is being created + + if session in cookie: + + response.session_data_name : name of the cookie for session data + + if session in db: + + response.session_db_record_id : + response.session_db_table : + response.session_db_unique_key : + + if session in file: + + response.session_file : + response.session_filename : """ - def renew( - self, - request=None, - response=None, - db=None, - tablename='web2py_session', - masterapp=None, - clear_session=False - ): - if request is None: - request = current.request - if response is None: - response = current.response - - #check if session is separate - separate = None - if response.session and response.session_id[2:3] == "/": - separate = lambda session_name: session_name[-2:] - - self._unlock(response) - - if not masterapp: - masterapp = request.application - - # Load session data from cookie - cookies = request.cookies - - # check if there is a session_id in cookies - if response.session_id_name in cookies: - response.session_id = \ - cookies[response.session_id_name].value - else: - response.session_id = None - - # if the session goes in file - if response.session_storage_type == 'file': - if global_settings.db_sessions is True \ - or masterapp in global_settings.db_sessions: - return - - client = request.client and request.client.replace(':', '.') - - uuid = web2py_uuid() - response.session_id = '%s-%s' % (client, uuid) - if separate: - prefix = separate(response.session_id) - response.session_id = '%s/%s' % \ - (prefix, response.session_id) - response.session_filename = \ - os.path.join(up(request.folder), masterapp, - 'sessions', response.session_id) - response.session_new = True - - # else the session goes in db - elif response.session_storage_type == 'db': - # verify that session_id exists - if not response.session_id: - return - - # verify if tablename was set or used in connect - if response.session_table_name and tablename == 'web2py_session': - tablename = response.session_table_name - - if global_settings.db_sessions is not True: - global_settings.db_sessions.add(masterapp) - - if response.session_file: - self._close(response) - if settings.global_settings.web2py_runtime_gae: - # in principle this could work without GAE - request.tickets_db = db - - tname = tablename + '_' + masterapp - - if not db: - raise Exception('No database parameter passed: "db=database"') - - table = db.get(tname, None) - if table is None: - raise Exception('No session to renew') - - # Get session data out of the database - (record_id, unique_key) = response.session_id.split(':') - if not record_id.isdigit() or long(record_id)<1: - raise Exception('record_id == 0') - # Select from database - record_id = long(record_id) - row = record_id and db(table.id == record_id).select() - row = row and row[0] or None - # Make sure the session data exists in the database - if not row or row.unique_key != unique_key: - raise Exception('No record') - - unique_key = web2py_uuid() - db(table.id == record_id).update(unique_key=unique_key) - response.session_id = '%s:%s' % (record_id, unique_key) - response.session_db_table = table - response.session_db_record_id = record_id - response.session_db_unique_key = unique_key - - rcookies = response.cookies - if response.session_id_name: - rcookies[response.session_id_name] = response.session_id - rcookies[response.session_id_name]['path'] = '/' - if clear_session: - self.clear() def connect( self, @@ -781,109 +699,102 @@ class Session(Storage): and it is used to determine a session prefix. separate can be True and it is set to session_name[-2:] """ - if request is None: - request = current.request - if response is None: - response = current.response - if separate is True: - separate = lambda session_name: session_name[-2:] + request = request or current.request + response = response or current.response + masterapp = masterapp or request.application + cookies = request.cookies + self._unlock(response) - if not masterapp: - masterapp = request.application + + response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() response.session_cookie_expires = cookie_expires - - # Load session data from cookie - cookies = request.cookies + response.session_client = str(request.client).replace(':', '.') + response.session_cookie_key = cookie_key + response.session_cookie_compression_level = compression_level # check if there is a session_id in cookies - if response.session_id_name in cookies: - response.session_id = \ - cookies[response.session_id_name].value - else: + try: + response.session_id = cookies[response.session_id_name].value + except KeyError: response.session_id = None - # check if there is session data in cookies - if response.session_data_name in cookies: - session_cookie_data = cookies[response.session_data_name].value - else: - session_cookie_data = None - # if we are supposed to use cookie based session data if cookie_key: response.session_storage_type = 'cookie' - response.session_cookie_key = cookie_key - response.session_cookie_compression_level = compression_level + elif db: + response.session_storage_type = 'db' + else: + response.session_storage_type = 'file' + # why do we do this? + # because connect may be called twice, by web2py and in models. + # the first time there is no db yet so it should do nothing + if (global_settings.db_sessions is True or + masterapp in global_settings.db_sessions): + return + + if response.session_storage_type == 'cookie': + # check if there is session data in cookies + if response.session_data_name in cookies: + session_cookie_data = cookies[response.session_data_name].value + else: + session_cookie_data = None if session_cookie_data: data = secure_loads(session_cookie_data, cookie_key, compression_level=compression_level) if data: self.update(data) + response.session_id = True + # else if we are supposed to use file based sessions - elif not db: - response.session_storage_type = 'file' - if global_settings.db_sessions is True \ - or masterapp in global_settings.db_sessions: - return + elif response.session_storage_type == 'file': response.session_new = False - client = request.client and request.client.replace(':', '.') + response.session_file = None + # check if the session_id points to a valid sesion filename if response.session_id: - if regex_session_id.match(response.session_id): + if not regex_session_id.match(response.session_id): + response.session_id = None + else: response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) - else: - response.session_id = None - # do not try load the data from file is these was data in cookie - if response.session_id and not session_cookie_data: - # os.path.exists(response.session_filename): - try: - response.session_file = \ - open(response.session_filename, 'rb+') try: + response.session_file = \ + open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True self.update(cPickle.load(response.session_file)) response.session_file.seek(0) - oc = response.session_filename.split('/')[-1]\ - .split('-')[0] - if check_client and client != oc: + oc = response.session_filename.split('/')[-1].split('-')[0] + if check_client and response.session_client != oc: raise Exception("cookie attack") except: response.session_id = None - finally: - pass - #This causes admin login to break. Must find out why. - #self._close(response) - except: - response.session_file = None if not response.session_id: uuid = web2py_uuid() - response.session_id = '%s-%s' % (client, uuid) + response.session_id = '%s-%s' % (response.session_client, uuid) + separate = separate and (lambda session_name: session_name[-2:]) if separate: prefix = separate(response.session_id) - response.session_id = '%s/%s' % \ - (prefix, response.session_id) + response.session_id = '%s/%s' % (prefix, response.session_id) response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) response.session_new = True + # else the session goes in db - else: - response.session_storage_type = 'db' + elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) + # if had a session on file alreday, close it (yes, can happen) if response.session_file: self._close(response) + # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: - # in principle this could work without GAE request.tickets_db = db - if masterapp == request.application: - table_migrate = migrate - else: - table_migrate = False + table_migrate = (masterapp == request.application) tname = tablename + '_' + masterapp table = db.get(tname, None) Field = db.Field @@ -900,46 +811,111 @@ class Session(Storage): migrate=table_migrate, ) table = db[tname] # to allow for lazy table - try: - - # Get session data out of the database - (record_id, unique_key) = response.session_id.split(':') - if record_id == '0': - raise Exception('record_id == 0') - # Select from database - if not session_cookie_data: - rows = db(table.id == record_id).select() - # Make sure the session data exists in the database - if len(rows) == 0 or rows[0].unique_key != unique_key: - raise Exception('No record') - # rows[0].update_record(locked=True) - # Unpickle the data - session_data = cPickle.loads(rows[0].session_data) - self.update(session_data) - except Exception: - record_id = None - unique_key = web2py_uuid() - session_data = {} - response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_table = table - response.session_db_record_id = record_id - response.session_db_unique_key = unique_key - # keep tablename parameter for use in session renew - response.session_table_name = tablename + if response.session_id: + # Get session data out of the database + try: + (record_id, unique_key) = response.session_id.split(':') + record_id = long(record_id) + except (TypeError,ValueError): + record_id = None + + # Select from database + if record_id: + row = table(record_id) #,unique_key=unique_key) + # Make sure the session data exists in the database + if row: + # rows[0].update_record(locked=True) + # Unpickle the data + session_data = cPickle.loads(row.session_data) + self.update(session_data) + else: + record_id = None + if record_id: + response.session_id = '%s:%s' % (record_id, unique_key) + response.session_db_unique_key = unique_key + response.session_db_record_id = record_id + else: + response.session_id = None + response.session_new = True + + if self.flash: + (response.flash, self.flash) = (self.flash, None) + + def renew(self, clear_session=False): + + if clear_session: + self.clear() + + request = current.request + response = current.response + session = response.session + masterapp = response.session_masterapp + cookies = request.cookies + + if response.session_storage_type == 'cookie': + return + + # if the session goes in file + if response.session_storage_type == 'file': + self._close(response) + uuid = web2py_uuid() + response.session_id = '%s-%s' % (response.session_client, uuid) + separate = (lambda s: s[-2:]) if session and response.session_id[2:3]=="/" else None + if separate: + prefix = separate(response.session_id) + response.session_id = '%s/%s' % \ + (prefix, response.session_id) + response.session_filename = \ + os.path.join(up(request.folder), masterapp, + 'sessions', response.session_id) + response.session_new = True + + # else the session goes in db + elif response.session_storage_type == 'db': + table = response.session_db_table + + # verify that session_id exists + if response.session_file: + self._close(response) + if response.session_new: + return + # Get session data out of the database + (record_id, unique_key) = response.session_id.split(':') + + if record_id.isdigit() and long(record_id)>1: + new_unique_key = web2py_uuid() + rows = db(table.id==record_id)(table.unique_key==unique_key)\ + .update(unique_key=new_unique_key) + else: + rows = None + if rows: + response.session_id = '%s:%s' % (record_id, unique_key) + response.session_db_record_id = record_id + response.session_db_unique_key = new_unique_key + else: + response.session_new = True + + def save_session_id_cookie(self): + request = current.request + response = current.response + session = response.session + masterapp = response.session_masterapp + cookies = request.cookies rcookies = response.cookies - rcookies[response.session_id_name] = response.session_id - rcookies[response.session_id_name]['path'] = '/' - if cookie_expires: - rcookies[response.session_id_name][ - 'expires'] = cookie_expires.strftime(FMT) + # if not cookie_key, but session_data_name in cookies # expire session_data_name from cookies - if session_cookie_data: + if response.session_data_name in cookies: rcookies[response.session_data_name] = 'expired' rcookies[response.session_data_name]['path'] = '/' rcookies[response.session_data_name]['expires'] = PAST - if self.flash: - (response.flash, self.flash) = (self.flash, None) + if response.session_id: + rcookies[response.session_id_name] = response.session_id + rcookies[response.session_id_name]['path'] = '/' + if response.session_cookie_expires: + rcookies[response.session_id_name]['expires'] = \ + response.session_cookie_expires.strftime(FMT) def clear(self): previous_session_hash = self.pop('_session_hash', None) @@ -971,10 +947,13 @@ class Session(Storage): self._forget = True def _try_store_in_cookie(self, request, response): - if response.session_storage_type != 'cookie': + if self._forget or self._unchanged(): return False name = response.session_data_name - value = secure_dumps(dict(self), response.session_cookie_key, compression_level=response.session_cookie_compression_level) + compression_level = response.session_cookie_compression_level + value = secure_dumps(dict(self), + response.session_cookie_key, + compression_level=compression_level) expires = response.session_cookie_expires rcookies = response.cookies rcookies.pop(name, None) @@ -1001,37 +980,40 @@ class Session(Storage): # don't save if file-based sessions, # no session id, or session being forgotten # or no changes to session - if response.session_storage_type != 'db' or not response.session_id \ - or self._forget or self._unchanged(): + + if not response.session_db_table or self._forget or self._unchanged(): return False table = response.session_db_table record_id = response.session_db_record_id - unique_key = response.session_db_unique_key + if response.session_new: + unique_key = web2py_uuid() + else: + unique_key = response.session_db_unique_key dd = dict(locked=False, - client_ip=request.client.replace(':', '.'), + client_ip=response.session_client, modified_datetime=request.now, session_data=cPickle.dumps(dict(self)), unique_key=unique_key) if record_id: - table._db(table.id == record_id).update(**dd) - else: + table(record_id).update_record(**dd) + if not record_id: record_id = table.insert(**dd) + response.session_id = '%s:%s' % (record_id, unique_key) + response.session_db_unique_key = unique_key + response.session_db_record_id = record_id - cookies, session_id_name = response.cookies, response.session_id_name - cookies[session_id_name] = '%s:%s' % (record_id, unique_key) - cookies[session_id_name]['path'] = '/' + self.save_session_id_cookie() return True def _try_store_in_cookie_or_file(self, request, response): - return \ - self._try_store_in_cookie(request, response) or \ - self._try_store_in_file(request, response) + if response.session_storage_type == 'file': + return self._try_store_in_file(request, response) + if response.session_storage_type == 'cookie': + return self._try_store_in_cookie(request, response) def _try_store_in_file(self, request, response): - if response.session_storage_type != 'file': - return False try: if not response.session_id or self._forget or self._unchanged(): return False @@ -1043,12 +1025,13 @@ class Session(Storage): response.session_file = open(response.session_filename, 'wb') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True - if response.session_file: cPickle.dump(dict(self), response.session_file) response.session_file.truncate() finally: self._close(response) + + self.save_session_id_cookie() return True def _unlock(self, response): diff --git a/gluon/tools.py b/gluon/tools.py index aaa7414b..7c47b1c7 100644 --- a/gluon/tools.py +++ b/gluon/tools.py @@ -1868,11 +1868,7 @@ class Auth(object): for key, value in user.items(): if callable(value) or key=='password': delattr(user,key) - sessdb = current.response.session_db_table and current.response.session_db_table._db or None - current.session.renew( - clear_session=not self.settings.keep_session_onlogin, - db=sessdb - ) + current.session.renew(clear_session=not self.settings.keep_session_onlogin) current.session.auth = Storage( user = user, last_visit=current.request.now, @@ -2304,10 +2300,7 @@ class Auth(object): current.session.auth = None current.session.flash = self.messages.logged_out - sessdb = current.response.session_db_table and current.response.session_db_table._db or None - current.session.renew( - clear_session=not self.settings.keep_session_onlogout, - db=sessdb) + current.session.renew(clear_session=not self.settings.keep_session_onlogout) if not next is None: redirect(next) From 6448518ce094589a617bf1d80ef879a1b037b238 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 20 Aug 2013 11:15:51 -0500 Subject: [PATCH 17/24] allow to change session in db to session in file --- VERSION | 2 +- gluon/globals.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a248a49e..c7d859a7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.20.10.57.14 +Version 2.6.0-development+timestamp.2013.08.20.11.14.41 diff --git a/gluon/globals.py b/gluon/globals.py index 11fede2b..13083c48 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -982,6 +982,10 @@ class Session(Storage): # or no changes to session if not response.session_db_table or self._forget or self._unchanged(): + if (not response.session_db_table and + global_settings.db_sessions is not True and + response.session_masterapp in global_settings.db_sessions): + global_settings.db_sessions.remove(response.session_masterapp) return False table = response.session_db_table From 8fe821ed0ca51cd0dbf21b495977b7f4640fd983 Mon Sep 17 00:00:00 2001 From: niphlod Date: Tue, 20 Aug 2013 20:59:14 +0200 Subject: [PATCH 18/24] standardized keyprefix --- gluon/contrib/redis_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py index 55cd7f9a..2d50d35c 100644 --- a/gluon/contrib/redis_session.py +++ b/gluon/contrib/redis_session.py @@ -198,7 +198,7 @@ class MockQuery(object): def update(self, **kwargs): #means that the session has been found and needs an update if self.op == 'eq' and self.field == 'id' and self.value: - key = "%s:%s" % (self.keyprefix, self.value) + key = self.keyprefix + ':' + str(self.value) with self.db.pipeline() as pipe: pipe.hmset(key, kwargs) if self.session_expiry: From ab58af3c87c7d307e35c0b9c65511fc518ce6868 Mon Sep 17 00:00:00 2001 From: niphlod Date: Tue, 20 Aug 2013 21:23:18 +0200 Subject: [PATCH 19/24] figure out correctly the content-type for request.restful. Thanks @mcabo --- gluon/globals.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/gluon/globals.py b/gluon/globals.py index 13083c48..884aac6b 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -310,10 +310,9 @@ class Request(Storage): self.is_restful = True method = _self.env.request_method if len(_self.args) and '.' in _self.args[-1]: - _self.args[- - 1], _self.extension = _self.args[-1].rsplit('.', 1) + _self.args[-1], _, self.extension = self.args[-1].rpartition('.') current.response.headers['Content-Type'] = \ - contenttype(_self.extension.lower()) + contenttype('.' + _self.extension.lower()) if not method in ['GET', 'POST', 'DELETE', 'PUT']: raise HTTP(400, "invalid method") rest_action = _action().get(method, None) @@ -656,7 +655,7 @@ class Session(Storage): response.session_storage_type : 'file', 'db', or 'cookie' response.session_cookie_compression_level : - response.session_cookie_expires : cookie expiration + response.session_cookie_expires : cookie expiration response.session_cookie_key : for encrypted sessions in cookies response.session_id : a number or None if no session response.session_id_name : @@ -666,7 +665,7 @@ class Session(Storage): if session in cookie: - response.session_data_name : name of the cookie for session data + response.session_data_name : name of the cookie for session data if session in db: @@ -705,7 +704,7 @@ class Session(Storage): cookies = request.cookies self._unlock(response) - + response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() @@ -733,7 +732,7 @@ class Session(Storage): if (global_settings.db_sessions is True or masterapp in global_settings.db_sessions): return - + if response.session_storage_type == 'cookie': # check if there is session data in cookies if response.session_data_name in cookies: @@ -761,7 +760,7 @@ class Session(Storage): 'sessions', response.session_id) try: response.session_file = \ - open(response.session_filename, 'rb+') + open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True @@ -821,7 +820,7 @@ class Session(Storage): record_id = None # Select from database - if record_id: + if record_id: row = table(record_id) #,unique_key=unique_key) # Make sure the session data exists in the database if row: @@ -838,7 +837,7 @@ class Session(Storage): else: response.session_id = None response.session_new = True - + if self.flash: (response.flash, self.flash) = (self.flash, None) @@ -853,7 +852,7 @@ class Session(Storage): masterapp = response.session_masterapp cookies = request.cookies - if response.session_storage_type == 'cookie': + if response.session_storage_type == 'cookie': return # if the session goes in file @@ -882,14 +881,14 @@ class Session(Storage): return # Get session data out of the database (record_id, unique_key) = response.session_id.split(':') - + if record_id.isdigit() and long(record_id)>1: new_unique_key = web2py_uuid() rows = db(table.id==record_id)(table.unique_key==unique_key)\ .update(unique_key=new_unique_key) else: rows = None - if rows: + if rows: response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_record_id = record_id response.session_db_unique_key = new_unique_key @@ -982,7 +981,7 @@ class Session(Storage): # or no changes to session if not response.session_db_table or self._forget or self._unchanged(): - if (not response.session_db_table and + if (not response.session_db_table and global_settings.db_sessions is not True and response.session_masterapp in global_settings.db_sessions): global_settings.db_sessions.remove(response.session_masterapp) From 3421447d7fc9b7100343bfab2f5b855884f6c149 Mon Sep 17 00:00:00 2001 From: niphlod Date: Tue, 20 Aug 2013 22:22:29 +0200 Subject: [PATCH 20/24] fix for issue 1642 --- applications/admin/controllers/default.py | 4 ++-- applications/admin/views/default/site.html | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 2a6ce41f..be6ca8fb 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -296,8 +296,8 @@ def site(): apps = [f for f in apps if f in FILTER_APPS] apps = sorted(apps, lambda a, b: cmp(a.upper(), b.upper())) - - return dict(app=None, apps=apps, myversion=myversion, + myplatform = platform.python_version() + return dict(app=None, apps=apps, myversion=myversion, myplatform=myplatform, form_create=form_create, form_update=form_update) diff --git a/applications/admin/views/default/site.html b/applications/admin/views/default/site.html index c618d923..f1051c16 100644 --- a/applications/admin/views/default/site.html +++ b/applications/admin/views/default/site.html @@ -78,7 +78,8 @@

{{=T("Version")}}

{{=myversion}}
- ({{=T("Running on %s", request.env.server_software)}}) + {{running_on = T("Running on %s", request.env.server_software or 'Unknown')}} + ({{="%s, Python %s" % (running_on, myplatform)}})

{{if session.check_version:}} From fd857b15f6cd7eefc7b12bd7c17a9814e0ae7e2a Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 21 Aug 2013 02:34:14 -0500 Subject: [PATCH 21/24] fixed issue 1638:imap: content as list field type and other small changes, thanks Alan --- VERSION | 2 +- gluon/dal.py | 47 ++++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/VERSION b/VERSION index c7d859a7..566d3fde 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.20.11.14.41 +Version 2.6.0-development+timestamp.2013.08.21.02.33.07 diff --git a/gluon/dal.py b/gluon/dal.py index 36241fd9..c3c16e75 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -5836,7 +5836,7 @@ class IMAPAdapter(NoSQLAdapter): uid string answered boolean Flag created date - content list:string A list of text or html parts + content list:string A list of dict text or html parts to string cc string bcc string @@ -5978,8 +5978,9 @@ class IMAPAdapter(NoSQLAdapter): """ MESSAGE is an identifier for sequence number""" - self.flags = ['\\Deleted', '\\Draft', '\\Flagged', - '\\Recent', '\\Seen', '\\Answered'] + self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft', + 'flagged': '\\Flagged', 'recent': '\\Recent', + 'seen': '\\Seen', 'answered': '\\Answered'} self.search_fields = { 'id': 'MESSAGE', 'created': 'DATE', 'uid': 'UID', 'sender': 'FROM', @@ -6202,7 +6203,7 @@ class IMAPAdapter(NoSQLAdapter): return tablename def is_flag(self, flag): - if self.search_fields.get(flag, None) in self.flags: + if self.search_fields.get(flag, None) in self.flags.values(): return True else: return False @@ -6234,7 +6235,7 @@ class IMAPAdapter(NoSQLAdapter): Field("uid", "string", writable=False), Field("answered", "boolean"), Field("created", "datetime", writable=False), - Field("content", "list:string", writable=False), + Field("content", list, writable=False), Field("to", "string", writable=False), Field("cc", "string", writable=False), Field("bcc", "string", writable=False), @@ -6451,23 +6452,23 @@ class IMAPAdapter(NoSQLAdapter): maintype = part.get_content_maintype() if ("%s.attachments" % tablename in colnames) or \ ("%s.content" % tablename in colnames): - if "%s.attachments" % tablename in colnames: - if not ("text" in maintype): - payload = part.get_payload(decode=True) - if payload: - attachment = { - "payload": payload, - "filename": part.get_filename(), - "encoding": part.get_content_charset(), - "mime": part.get_content_type(), - "disposition": part["Content-Disposition"]} - attachments.append(attachment) - if "%s.content" % tablename in colnames: - payload = part.get_payload(decode=True) - part_charset = self.get_charset(part) - if "text" in maintype: - if payload: - content.append(self.encode_text(payload, part_charset)) + payload = part.get_payload(decode=True) + if payload: + filename = part.get_filename() + values = {"mime": part.get_content_type()} + if ((filename or not "text" in maintype) and + ("%s.attachments" % tablename in colnames)): + values.update({"payload": payload, + "filename": filename, + "encoding": part.get_content_charset(), + "disposition": part["Content-Disposition"]}) + attachments.append(values) + elif (("text" in maintype) and + ("%s.content" % tablename in colnames)): + values.update({"text": self.encode_text(payload, + self.get_charset(part))}) + content.append(values) + if "%s.size" % tablename in colnames: if part is not None: size += len(str(part)) @@ -6787,7 +6788,7 @@ class IMAPAdapter(NoSQLAdapter): elif name == "DATE": result = "ON %s" % self.convert_date(second) - elif name in self.flags: + elif name in self.flags.values(): if second: result = "%s" % (name.upper()[1:]) else: From f9940825a1e3cfd2add9db6b58fabdd863160c9b Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 21 Aug 2013 02:37:31 -0500 Subject: [PATCH 22/24] fixed issue 1639:DAL, define_table and Field unpacked serialized attributes, thanks Alan --- VERSION | 2 +- gluon/dal.py | 254 ++++++++++++++-------------------------- gluon/tests/test_dal.py | 62 +++++----- 3 files changed, 119 insertions(+), 199 deletions(-) diff --git a/VERSION b/VERSION index 566d3fde..687b7af8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.21.02.33.07 +Version 2.6.0-development+timestamp.2013.08.21.02.36.16 diff --git a/gluon/dal.py b/gluon/dal.py index c3c16e75..d1edcd62 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -7247,7 +7247,7 @@ class DAL(object): or - db = DAL({"uri": ..., "items": ...}) # experimental + db = DAL(**{"uri": ..., "tables": [...]...}) # experimental db.define_table('tablename', Field('fieldname1'), Field('fieldname2')) @@ -7362,8 +7362,9 @@ class DAL(object): migrate_enabled=True, fake_migrate_all=False, decode_credentials=False, driver_args=None, adapter_args=None, attempts=5, auto_import=False, - bigint_id=False,debug=False,lazy_tables=False, - db_uid=None, do_connect=True, after_connection=None): + bigint_id=False, debug=False, lazy_tables=False, + db_uid=None, do_connect=True, + after_connection=None, tables=None): """ Creates a new Database Abstraction Layer instance. @@ -7375,7 +7376,7 @@ class DAL(object): experimental: you can specify a dictionary as uri parameter i.e. with db = DAL({"uri": "sqlite://storage.sqlite", - "items": {...}, ...}) + "tables": {...}, ...}) for an example of dict input you can check the output of the scaffolding db model with @@ -7419,18 +7420,6 @@ class DAL(object): :lazy_tables (defaults to False): delay table definition until table access :after_connection (defaults to None): a callable that will be execute after the connection """ - - items = None - if isinstance(uri, dict): - if "items" in uri: - items = uri.pop("items") - try: - newuri = uri.pop("uri") - except KeyError: - newuri = DEFAULT_URI - locals().update(uri) - uri = newuri - if uri == '' and db_uid is not None: return if not decode_credentials: credential_decoder = lambda cred: cred @@ -7523,31 +7512,20 @@ class DAL(object): self._fake_migrate = fake_migrate self._migrate_enabled = migrate_enabled self._fake_migrate_all = fake_migrate_all - if auto_import or items: + if auto_import or tables: self.import_table_definitions(adapter.folder, - items=items) + tables=tables) @property def tables(self): return self._tables def import_table_definitions(self, path, migrate=False, - fake_migrate=False, items=None): + fake_migrate=False, tables=None): pattern = pjoin(path,self._uri_hash+'_*.table') - if items: - for tablename, table in items.iteritems(): - # TODO: read all field/table options - fields = [] - # remove unsupported/illegal Table arguments - [table.pop(name) for name in ("name", "fields") if - name in table] - if "items" in table: - for fieldname, field in table.pop("items").iteritems(): - # remove unsupported/illegal Field arguments - [field.pop(key) for key in ("requires", "name", - "compute", "colname") if key in field] - fields.append(Field(str(fieldname), **field)) - self.define_table(str(tablename), *fields, **table) + if tables: + for table in tables: + self.define_table(**table) else: for filename in glob.glob(pattern): tfile = self._adapter.file_open(filename, 'r') @@ -7845,8 +7823,14 @@ def index(): ): if not fields and 'fields' in args: fields = args.get('fields',()) - if not isinstance(tablename,str): - raise SyntaxError("missing table name") + if not isinstance(tablename, str): + if isinstance(tablename, unicode): + try: + tablename = str(tablename) + except UnicodeEncodeError: + raise SyntaxError("invalid unicode table name") + else: + raise SyntaxError("missing table name") elif hasattr(self,tablename) or tablename in self.tables: if not args.get('redefine',False): raise SyntaxError('table already defined: %s' % tablename) @@ -7910,48 +7894,40 @@ def index(): if on_define: on_define(table) return table - def as_dict(self, flat=False, sanitize=True, field_options=True): - dbname = db_uid = uri = None + def as_dict(self, flat=False, sanitize=True): + db_uid = uri = None if not sanitize: - uri, dbname, db_uid = (self._uri, self._dbname, self._db_uid) - db_as_dict = dict(items={}, tables=[], uri=uri, dbname=dbname, - db_uid=db_uid, - **dict([(k, getattr(self, "_" + k)) for - k in 'pool_size','folder','db_codec', + uri, db_uid = (self._uri, self._db_uid) + db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid, + **dict([(k, getattr(self, "_" + k, None)) + for k in 'pool_size','folder','db_codec', 'check_reserved','migrate','fake_migrate', 'migrate_enabled','fake_migrate_all', 'decode_credentials','driver_args', 'adapter_args', 'attempts', 'bigint_id','debug','lazy_tables', 'do_connect'])) - for table in self: - tablename = str(table) - db_as_dict["tables"].append(tablename) - db_as_dict["items"][tablename] = table.as_dict(flat=flat, - sanitize=sanitize, - field_options=field_options) + db_as_dict["tables"].append(table.as_dict(flat=flat, + sanitize=sanitize)) return db_as_dict - def as_xml(self, sanitize=True, field_options=True): + def as_xml(self, sanitize=True): if not have_serializers: raise ImportError("No xml serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.xml(d) - def as_json(self, sanitize=True, field_options=True): + def as_json(self, sanitize=True): if not have_serializers: raise ImportError("No json serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.json(d) - def as_yaml(self, sanitize=True, field_options=True): + def as_yaml(self, sanitize=True): if not have_serializers: raise ImportError("No YAML serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.yaml(d) def __contains__(self, tablename): @@ -8319,7 +8295,9 @@ class Table(object): if len(self._primarykey)==1: self._id = [f for f in fields if isinstance(f,Field) \ and f.name==self._primarykey[0]][0] - elif not [f for f in fields if isinstance(f,Field) and f.type=='id']: + elif not [f for f in fields if (isinstance(f,Field) and + f.type=='id') or (isinstance(f, dict) and + f.get("type", None)=="id")]: field = Field('id', 'id') newfields.append(field) fieldnames.add('id') @@ -8337,8 +8315,7 @@ class Table(object): if field.db is not None: field = copy.copy(field) include_new(field) - elif isinstance(field, dict) and 'fieldname' and \ - not field['fieldname'] in fieldnames: + elif isinstance(field, dict) and not field['fieldname'] in fieldnames: include_new(Field(**field)) elif isinstance(field, Table): table = field @@ -8893,9 +8870,8 @@ class Table(object): if id_map and cid is not None: id_map_self[long(line[cid])] = new_id - def as_dict(self, flat=False, sanitize=True, field_options=True): - tablename = str(self) - table_as_dict = dict(name=tablename, items={}, fields=[], + def as_dict(self, flat=False, sanitize=True): + table_as_dict = dict(tablename=str(self), fields=[], sequence_name=self._sequence_name, trigger_name=self._trigger_name, common_filter=self._common_filter, format=self._format, @@ -8903,31 +8879,26 @@ class Table(object): for field in self: if (field.readable or field.writable) or (not sanitize): - table_as_dict["fields"].append(field.name) - table_as_dict["items"][field.name] = \ - field.as_dict(flat=flat, sanitize=sanitize, - options=field_options) + table_as_dict["fields"].append(field.as_dict( + flat=flat, sanitize=sanitize)) return table_as_dict - def as_xml(self, sanitize=True, field_options=True): + def as_xml(self, sanitize=True): if not have_serializers: raise ImportError("No xml serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.xml(d) - def as_json(self, sanitize=True, field_options=True): + def as_json(self, sanitize=True): if not have_serializers: raise ImportError("No json serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.json(d) - def as_yaml(self, sanitize=True, field_options=True): + def as_yaml(self, sanitize=True): if not have_serializers: raise ImportError("No YAML serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.yaml(d) def with_alias(self, alias): @@ -9408,8 +9379,13 @@ class Field(Expression): self.op = None self.first = None self.second = None + if isinstance(fieldname, unicode): + try: + fieldname = str(fieldname) + except UnicodeEncodeError: + raise SyntaxError('Field: invalid unicode field name') self.name = fieldname = cleanup(fieldname) - if not isinstance(fieldname,str) or hasattr(Table,fieldname) or \ + if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \ fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname): raise SyntaxError('Field: invalid field name: %s' % fieldname) self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type @@ -9609,113 +9585,61 @@ class Field(Expression): def count(self, distinct=None): return Expression(self.db, self.db._adapter.COUNT, self, distinct, 'integer') - def as_dict(self, flat=False, sanitize=True, options=True): - - attrs = ('type', 'length', 'default', 'required', - 'ondelete', 'notnull', 'unique', 'uploadfield', - 'widget', 'label', 'comment', 'writable', 'readable', - 'update', 'authorize', 'autodelete', 'represent', - 'uploadfolder', 'uploadseparate', 'uploadfs', - 'compute', 'custom_store', 'custom_retrieve', - 'custom_retrieve_file_properties', 'custom_delete', - 'filter_in', 'filter_out', 'custom_qualifier', - 'map_none', 'name') - - SERIALIZABLE_TYPES = (int, long, basestring, dict, list, - float, tuple, bool, type(None)) + def as_dict(self, flat=False, sanitize=True): + attrs = ("name", 'authorize', 'represent', 'ondelete', + 'custom_store', 'autodelete', 'custom_retrieve', + 'filter_out', 'uploadseparate', 'widget', 'uploadfs', + 'update', 'custom_delete', 'uploadfield', 'uploadfolder', + 'custom_qualifier', 'unique', 'writable', 'compute', + 'map_none', 'default', 'type', 'required', 'readable', + 'requires', 'comment', 'label', 'length', 'notnull', + 'custom_retrieve_file_properties', 'filter_in') + serializable = (int, long, basestring, float, tuple, + bool, type(None)) def flatten(obj): - if flat: - if isinstance(obj, flatten.__class__): - return str(type(obj)) - elif isinstance(obj, type): - try: - return str(obj).split("'")[1] - except IndexError: - return str(obj) - elif not isinstance(obj, SERIALIZABLE_TYPES): - return str(obj) - elif isinstance(obj, dict): - newobj = dict() - for k, v in obj.items(): - newobj[k] = flatten(v) - return newobj - elif isinstance(obj, (list, tuple, set)): - return [flatten(v) for v in obj] - else: - return obj - elif isinstance(obj, (dict, set)): - return obj.copy() - else: return obj - - def filter_requires(t, r, options=True): - if sanitize and any([keyword in str(t).upper() for - keyword in ("CRYPT", "IS_STRONG")]): + if isinstance(obj, dict): + return dict((flatten(k), flatten(v)) for k, v in + obj.items()) + elif isinstance(obj, (tuple, list, set)): + return [flatten(v) for v in obj] + elif isinstance(obj, serializable): + return obj + elif isinstance(obj, (datetime.datetime, + datetime.date, datetime.time)): + return str(obj) + else: return None - if not isinstance(r, dict): - if options and hasattr(r, "options"): - if callable(r.options): - r.options() - newr = r.__dict__.copy() - else: - newr = r.copy() - - # remove options if not required - if not options and newr.has_key("labels"): - [newr.update({key:None}) for key in - ("labels", "theset") if (key in newr)] - - for k, v in newr.items(): - if k == "other": - if isinstance(v, dict): - otype, other = v.popitem() - else: - otype = flatten(type(v)) - other = v - newr[k] = {otype: filter_requires(otype, other, - options=options)} - else: - newr[k] = flatten(v) - return newr - - if isinstance(self.requires, (tuple, list, set)): - requires = dict([(flatten(type(r)), - filter_requires(type(r), r, - options=options)) for - r in self.requires]) - else: - requires = {flatten(type(self.requires)): - filter_requires(type(self.requires), - self.requires, options=options)} - - d = dict(colname="%s.%s" % (self.tablename, self.name), - requires=requires) - d.update([(attr, flatten(getattr(self, attr))) for attr in attrs]) + d = dict() + if not (sanitize and not (self.readable or self.writable)): + for attr in attrs: + if flat: + d.update({attr: flatten(getattr(self, attr))}) + else: + d.update({attr: getattr(self, attr)}) + d["fieldname"] = d.pop("name") return d - def as_xml(self, sanitize=True, options=True): + def as_xml(self, sanitize=True): if have_serializers: xml = serializers.xml else: raise ImportError("No xml serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - options=options) + d = self.as_dict(flat=True, sanitize=sanitize) return xml(d) - def as_json(self, sanitize=True, options=True): + def as_json(self, sanitize=True): if have_serializers: json = serializers.json else: raise ImportError("No json serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - options=options) + d = self.as_dict(flat=True, sanitize=sanitize) return json(d) - def as_yaml(self, sanitize=True, options=True): + def as_yaml(self, sanitize=True): if have_serializers: - d = self.as_dict(flat=True, sanitize=sanitize, - options=options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.yaml(d) else: raise ImportError("No YAML serializers available") diff --git a/gluon/tests/test_dal.py b/gluon/tests/test_dal.py index 8a83282a..cbcb69a8 100644 --- a/gluon/tests/test_dal.py +++ b/gluon/tests/test_dal.py @@ -701,13 +701,12 @@ class TestDALDictImportExport(unittest.TestCase): assert isinstance(dbdict, dict) uri = dbdict["uri"] assert isinstance(uri, basestring) and uri - assert len(dbdict["items"]) == 2 - assert len(dbdict["items"]["person"]["items"]) == 3 - assert dbdict["items"]["person"]["items"]["name"]["type"] == db.person.name.type - assert dbdict["items"]["person"]["items"]["name"]["default"] == db.person.name.default - assert dbdict + assert len(dbdict["tables"]) == 2 + assert len(dbdict["tables"][0]["fields"]) == 3 + assert dbdict["tables"][0]["fields"][1]["type"] == db.person.name.type + assert dbdict["tables"][0]["fields"][1]["default"] == db.person.name.default - db2 = DAL(dbdict, check_reserved=['all']) + db2 = DAL(**dbdict) assert len(db.tables) == len(db2.tables) assert hasattr(db2, "pet") and isinstance(db2.pet, Table) assert hasattr(db2.pet, "friend") and isinstance(db2.pet.friend, Field) @@ -725,7 +724,7 @@ class TestDALDictImportExport(unittest.TestCase): unicode_keys = True if sys.version < "2.6.5": unicode_keys = False - db3 = DAL(serializers.loads_json(dbjson, + db3 = DAL(**serializers.loads_json(dbjson, unicode_keys=unicode_keys)) assert hasattr(db3, "person") and hasattr(db3.person, "uuid") and\ db3.person.uuid.type == db.person.uuid.type @@ -736,18 +735,19 @@ class TestDALDictImportExport(unittest.TestCase): mpfc = "Monty Python's Flying Circus" dbdict4 = {"uri": DEFAULT_URI, - "items":{"staff":{"items": {"name": - {"default":"Michael"}, - "food": - {"default":"Spam"}, - "tvshow": - {"type": "reference tvshow"} - }}, - "tvshow":{"items": {"name": - {"default":mpfc}, - "rating": - {"type":"double"}}}}} - db4 = DAL(dbdict4, check_reserved=['all']) + "tables":[{"tablename": "staff", + "fields": [{"fieldname": "name", + "default":"Michael"}, + {"fieldname": "food", + "default":"Spam"}, + {"fieldname": "tvshow", + "type": "reference tvshow"}]}, + {"tablename": "tvshow", + "fields": [{"fieldname": "name", + "default":mpfc}, + {"fieldname": "rating", + "type":"double"}]}]} + db4 = DAL(**dbdict4) assert "staff" in db4.tables assert "name" in db4.staff assert db4.tvshow.rating.type == "double" @@ -761,20 +761,19 @@ class TestDALDictImportExport(unittest.TestCase): db4.commit() dbdict5 = {"uri": DEFAULT_URI} - db5 = DAL(dbdict5, check_reserved=['all']) + db5 = DAL(**dbdict5) assert db5.tables in ([], None) assert not (str(db5) in ("", None)) dbdict6 = {"uri": DEFAULT_URI, - "items":{"staff":{}, - "tvshow":{"items": {"name": {}, - "rating": - {"type":"double"} - } - } - } - } - db6 = DAL(dbdict6, check_reserved=['all']) + "tables":[{"tablename": "staff"}, + {"tablename": "tvshow", + "fields": [{"fieldname": "name"}, + {"fieldname": "rating", "type":"double"} + ] + }] + } + db6 = DAL(**dbdict6) assert len(db6["staff"].fields) == 1 assert "name" in db6["tvshow"].fields @@ -782,15 +781,12 @@ class TestDALDictImportExport(unittest.TestCase): assert db6.staff.insert() is not None assert db6(db6.staff).select().first().id == 1 - db6.staff.drop() db6.tvshow.drop() db6.commit() - - - if __name__ == '__main__': unittest.main() tearDownModule() + From 4f1d6e6f314e7da5edd61362e8ed1b1b70a11201 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 21 Aug 2013 02:54:23 -0500 Subject: [PATCH 23/24] fixed possible memdb issue with table(id), thanks Luca --- VERSION | 2 +- gluon/contrib/memdb.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 687b7af8..ada9fdbe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.21.02.36.16 +Version 2.6.0-development+timestamp.2013.08.21.02.52.57 diff --git a/gluon/contrib/memdb.py b/gluon/contrib/memdb.py index f619bdeb..c17fa322 100644 --- a/gluon/contrib/memdb.py +++ b/gluon/contrib/memdb.py @@ -299,6 +299,11 @@ class Table(DALStorage): def __str__(self): return self._tablename + def __call__(self, id): + return self.get(id) + + def __getitem__(self,id): + return self.get(id) class Expression(object): From d1525cf6ecf300d2650c8cad6b410af56da11f1e Mon Sep 17 00:00:00 2001 From: mdipierro Date: Wed, 21 Aug 2013 04:29:39 -0500 Subject: [PATCH 24/24] adapter.connect = None after adapter.close_connection --- VERSION | 2 +- gluon/dal.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index ada9fdbe..59dc1a2c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.21.02.52.57 +Version 2.6.0-development+timestamp.2013.08.21.04.28.27 diff --git a/gluon/dal.py b/gluon/dal.py index d1edcd62..f21f19c5 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -1778,13 +1778,18 @@ class BaseAdapter(ConnectionPool): return list(tables) def commit(self): - if self.connection: return self.connection.commit() + if self.connection: + return self.connection.commit() def rollback(self): - if self.connection: return self.connection.rollback() + if self.connection: + return self.connection.rollback() def close_connection(self): - if self.connection: return self.connection.close() + if self.connection: + r = self.connection.close() + self.connection = None + return r def distributed_transaction_begin(self, key): return