diff --git a/Introduction b/Introduction deleted file mode 100644 index e69de29b..00000000 diff --git a/VERSION b/VERSION index be967c51..59dc1a2c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.09.16.07.32 +Version 2.6.0-development+timestamp.2013.08.21.04.28.27 diff --git a/app.example.yaml b/app.example.yaml index eead4703..62cd405b 100644 --- a/app.example.yaml +++ b/app.example.yaml @@ -25,13 +25,13 @@ handlers: # the parametric router's language logic. # You cannot use them together. -- url: /(.+?)/[^_]*\/?static/_\d.\d.\d\/?(.+) +- url: /(.+?)/static/_\d.\d.\d\/(.+) static_files: applications/\1/static/\2 upload: applications/(.+?)/static/(.+) secure: optional expiration: "365d" -- url: /(.+?)/[^_]*\/?static/?(.+) +- url: /(.+?)/static/(.+) static_files: applications/\1/static/\2 upload: applications/(.+?)/static/(.+) secure: optional diff --git a/applications/admin/controllers/default.py b/applications/admin/controllers/default.py index 2a6ce41f..be6ca8fb 100644 --- a/applications/admin/controllers/default.py +++ b/applications/admin/controllers/default.py @@ -296,8 +296,8 @@ def site(): apps = [f for f in apps if f in FILTER_APPS] apps = sorted(apps, lambda a, b: cmp(a.upper(), b.upper())) - - return dict(app=None, apps=apps, myversion=myversion, + myplatform = platform.python_version() + return dict(app=None, apps=apps, myversion=myversion, myplatform=myplatform, form_create=form_create, form_update=form_update) diff --git a/applications/admin/views/default/site.html b/applications/admin/views/default/site.html index c618d923..f1051c16 100644 --- a/applications/admin/views/default/site.html +++ b/applications/admin/views/default/site.html @@ -78,7 +78,8 @@

{{=T("Version")}}

{{=myversion}}
- ({{=T("Running on %s", request.env.server_software)}}) + {{running_on = T("Running on %s", request.env.server_software or 'Unknown')}} + ({{="%s, Python %s" % (running_on, myplatform)}})

{{if session.check_version:}} diff --git a/fcgihandler.py b/fcgihandler.py index a3d4a5df..7d5719fb 100755 --- a/fcgihandler.py +++ b/fcgihandler.py @@ -42,7 +42,7 @@ import gluon.contrib.gateways.fcgi as fcgi if LOGGING: application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase, logfilename='httpserver.log', - profilerfilename=None) + profiler_dir=None) else: application = gluon.main.wsgibase diff --git a/gluon/contrib/fpdf/html.py b/gluon/contrib/fpdf/html.py index a1269bc9..0863d5ed 100644 --- a/gluon/contrib/fpdf/html.py +++ b/gluon/contrib/fpdf/html.py @@ -33,7 +33,6 @@ class HTML2FPDF(HTMLParser): self.href = '' self.align = '' self.page_links = {} - self.font_list = ("times","courier", "helvetica") self.font = None self.font_stack = [] self.pdf = pdf @@ -55,6 +54,7 @@ class HTML2FPDF(HTMLParser): self.thead = None self.tfoot = None self.theader_out = self.tfooter_out = False + self.hsize = dict(h1=2, h2=1.5, h3=1.17, h4=1, h5=0.83, h6=0.67) def width2mm(self, length): if length[-1]=='%': @@ -88,7 +88,7 @@ class HTML2FPDF(HTMLParser): else: self.set_style('B',True) border = border or 'B' - align = 'C' + align = self.td.get('align', 'C')[0].upper() bgcolor = hex2dec(self.td.get('bgcolor', self.tr.get('bgcolor', ''))) # parsing table header/footer (drawn later): if self.thead is not None: @@ -179,8 +179,8 @@ class HTML2FPDF(HTMLParser): self.pdf.ln(5) if attrs: if attrs: self.align = attrs.get('align') - if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'): - k = (2, 1.5, 1.17, 1, 0.83, 0.67)[int(tag[1])] + if tag in self.hsize: + k = self.hsize[tag] self.pdf.ln(5*k) self.pdf.set_text_color(150,0,0) self.pdf.set_font_size(12 * k) @@ -219,7 +219,7 @@ class HTML2FPDF(HTMLParser): self.color = hex2dec(attrs['color']) self.set_text_color(*color) self.color = color - if 'face' in attrs and attrs['face'].lower() in self.font_list: + if 'face' in attrs: face = attrs.get('face').lower() self.pdf.set_font(face) self.font_face = face diff --git a/gluon/contrib/fpdf/template.py b/gluon/contrib/fpdf/template.py index 2d38c9f9..916e8355 100644 --- a/gluon/contrib/fpdf/template.py +++ b/gluon/contrib/fpdf/template.py @@ -110,8 +110,8 @@ class Template: pdf.set_auto_page_break(False,margin=0) for element in sorted(self.elements,key=lambda x: x['priority']): - #print "dib",element['type'], element['name'], element['x1'], element['y1'], element['x2'], element['y2'] - element = element.copy() + # make a copy of the element: + element = dict(element) element['text'] = self.texts[pg].get(element['name'].lower(), element['text']) if 'rotate' in element: pdf.rotate(element['rotate'], element['x1'], element['y1']) diff --git a/gluon/contrib/memdb.py b/gluon/contrib/memdb.py index 6396cfe2..c17fa322 100644 --- a/gluon/contrib/memdb.py +++ b/gluon/contrib/memdb.py @@ -299,6 +299,11 @@ class Table(DALStorage): def __str__(self): return self._tablename + def __call__(self, id): + return self.get(id) + + def __getitem__(self,id): + return self.get(id) class Expression(object): @@ -502,9 +507,8 @@ class Query(object): 'Query: right side of filter must be a value or entity') if isinstance(left, Field) and left.name == 'id': if op == '=': - self.get_one = \ - QueryException(tablename=left._tablename, - id=long(right)) + self.get_one = QueryException( + tablename=left._tablename, id=long(right or 0)) return else: raise SyntaxError('only equality by id is supported') diff --git a/gluon/contrib/redis_session.py b/gluon/contrib/redis_session.py index ce8fb167..2d50d35c 100644 --- a/gluon/contrib/redis_session.py +++ b/gluon/contrib/redis_session.py @@ -168,7 +168,7 @@ class MockQuery(object): def select(self): if self.op == 'eq' and self.field == 'id' and self.value: #means that someone wants to retrieve the key self.value - key = self.keyprefix + ':' + self.value + key = self.keyprefix + ':' + str(self.value) if self.with_lock: acquire_lock(self.db, key + ':lock', self.value) rtn = self.db.hgetall(key) @@ -198,7 +198,7 @@ class MockQuery(object): def update(self, **kwargs): #means that the session has been found and needs an update if self.op == 'eq' and self.field == 'id' and self.value: - key = "%s:%s" % (self.keyprefix, self.value) + key = self.keyprefix + ':' + str(self.value) with self.db.pipeline() as pipe: pipe.hmset(key, kwargs) if self.session_expiry: diff --git a/gluon/dal.py b/gluon/dal.py index 9ad7bb17..f21f19c5 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -684,6 +684,11 @@ class BaseAdapter(ConnectionPool): return None return isinstance(exception, self.driver.OperationalError) + def isProgrammingError(self,exception): + if not hasattr(self.driver, "ProgrammingError"): + return None + return isinstance(exception, self.driver.ProgrammingError) + def id_query(self, table): return table._id != None @@ -1263,17 +1268,12 @@ class BaseAdapter(ConnectionPool): return '(%s OR %s)' % (self.expand(first), self.expand(second)) def BELONGS(self, first, second): - if isinstance(second, str): - return '(%s IN (%s))' % (self.expand(first), second[:-1]) - if not second: - return '(1=0)' - if isinstance(second, (list,tuple,frozenset)): - second = set(second) # remove duplicates, make mutable - if isinstance(second, set) and None in second: - second.remove(None) - return self.OR(self.EQ(first, None), self.BELONGS(first, second)) - items = ','.join(self.expand(item, first.type) for item in second) - return '(%s IN (%s))' % (self.expand(first), items) + if isinstance(second, str): + return '(%s IN (%s))' % (self.expand(first), second[:-1]) + if not second: + return '(1=0)' + items = ','.join(self.expand(item, first.type) for item in second) + return '(%s IN (%s))' % (self.expand(first), items) def REGEXP(self, first, second): "regular expression operator" @@ -1778,13 +1778,18 @@ class BaseAdapter(ConnectionPool): return list(tables) def commit(self): - if self.connection: return self.connection.commit() + if self.connection: + return self.connection.commit() def rollback(self): - if self.connection: return self.connection.rollback() + if self.connection: + return self.connection.rollback() def close_connection(self): - if self.connection: return self.connection.close() + if self.connection: + r = self.connection.close() + self.connection = None + return r def distributed_transaction_begin(self, key): return @@ -4333,7 +4338,8 @@ class DatabaseStoredFile: if db.executesql(query): return True except Exception, e: - if not db._adapter.isOperationalError(e): + if not (db._adapter.isOperationalError(e) or + db._adapter.isProgrammingError(e)): raise # no web2py_filesystem found? tb = traceback.format_exc() @@ -5835,7 +5841,7 @@ class IMAPAdapter(NoSQLAdapter): uid string answered boolean Flag created date - content list:string A list of text or html parts + content list:string A list of dict text or html parts to string cc string bcc string @@ -5977,8 +5983,9 @@ class IMAPAdapter(NoSQLAdapter): """ MESSAGE is an identifier for sequence number""" - self.flags = ['\\Deleted', '\\Draft', '\\Flagged', - '\\Recent', '\\Seen', '\\Answered'] + self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft', + 'flagged': '\\Flagged', 'recent': '\\Recent', + 'seen': '\\Seen', 'answered': '\\Answered'} self.search_fields = { 'id': 'MESSAGE', 'created': 'DATE', 'uid': 'UID', 'sender': 'FROM', @@ -6201,7 +6208,7 @@ class IMAPAdapter(NoSQLAdapter): return tablename def is_flag(self, flag): - if self.search_fields.get(flag, None) in self.flags: + if self.search_fields.get(flag, None) in self.flags.values(): return True else: return False @@ -6233,7 +6240,7 @@ class IMAPAdapter(NoSQLAdapter): Field("uid", "string", writable=False), Field("answered", "boolean"), Field("created", "datetime", writable=False), - Field("content", "list:string", writable=False), + Field("content", list, writable=False), Field("to", "string", writable=False), Field("cc", "string", writable=False), Field("bcc", "string", writable=False), @@ -6450,23 +6457,23 @@ class IMAPAdapter(NoSQLAdapter): maintype = part.get_content_maintype() if ("%s.attachments" % tablename in colnames) or \ ("%s.content" % tablename in colnames): - if "%s.attachments" % tablename in colnames: - if not ("text" in maintype): - payload = part.get_payload(decode=True) - if payload: - attachment = { - "payload": payload, - "filename": part.get_filename(), - "encoding": part.get_content_charset(), - "mime": part.get_content_type(), - "disposition": part["Content-Disposition"]} - attachments.append(attachment) - if "%s.content" % tablename in colnames: - payload = part.get_payload(decode=True) - part_charset = self.get_charset(part) - if "text" in maintype: - if payload: - content.append(self.encode_text(payload, part_charset)) + payload = part.get_payload(decode=True) + if payload: + filename = part.get_filename() + values = {"mime": part.get_content_type()} + if ((filename or not "text" in maintype) and + ("%s.attachments" % tablename in colnames)): + values.update({"payload": payload, + "filename": filename, + "encoding": part.get_content_charset(), + "disposition": part["Content-Disposition"]}) + attachments.append(values) + elif (("text" in maintype) and + ("%s.content" % tablename in colnames)): + values.update({"text": self.encode_text(payload, + self.get_charset(part))}) + content.append(values) + if "%s.size" % tablename in colnames: if part is not None: size += len(str(part)) @@ -6488,6 +6495,71 @@ class IMAPAdapter(NoSQLAdapter): processor = attributes.get('processor',self.parse) return processor(imapqry_array, fields, colnames) + def _insert(self, table, fields): + def add_payload(message, obj): + payload = Message() + charset = obj.get("encoding", "utf-8") + payload.set_type(obj.get("mime", None)) + payload.set_charset(charset) + if "text" in obj: + payload.set_payload(obj["text"]) + elif "payload" in obj: + payload.set_payload(obj["payload"]) + if "filename" in obj and obj["filename"]: + payload.add_header("Content-Disposition", + "attachment", filename=obj["filename"]) + message.attach(payload) + + mailbox = table.mailbox + d = dict(((k.name, v) for k, v in fields)) + date_time = (d.get("created", datetime.datetime.now())).timetuple() + if len(d) > 0: + message = d.get("email", None) + attachments = d.get("attachments", []) + content = d.get("content", []) + flags = " ".join(["\\%s" % flag.capitalize() for flag in + ("answered", "deleted", "draft", "flagged", + "recent", "seen") if d.get(flag, False)]) + if not message: + from email.message import Message + mime = d.get("mime", None) + charset = d.get("encoding", None) + message = Message() + message["from"] = d.get("sender", "") + message["subject"] = d.get("subject", "") + if mime: + message.set_type(mime) + if charset: + message.set_charset(charset) + for item in ("to", "cc", "bcc"): + value = d.get(item, "") + if isinstance(value, basestring): + message[item] = value + else: + message[item] = ";".join([i for i in + value]) + if not message.is_multipart(): + if isinstance(content, basestring): + message.set_payload(content) + elif len(content) > 0: + message.set_payload(content[0]["text"]) + else: + [add_payload(message, c) for c in content] + [add_payload(message, a) for a in attachments] + message = message.as_string() + return (mailbox, flags, date_time, message) + else: + raise NotImplementedError("IMAP empty insert is not implemented") + + def insert(self, table, fields): + values = self._insert(table, fields) + result, data = self.connection.append(*values) + if result == "OK": + uid = int(re.findall("\d+", str(data))[-1]) + return self.db(table.uid==uid).select(table.id).first().id + else: + raise Exception("IMAP message append failed: %s" % data) + def _update(self, tablename, query, fields, commit=False): # TODO: the adapter should implement an .expand method commands = list() @@ -6721,7 +6793,7 @@ class IMAPAdapter(NoSQLAdapter): elif name == "DATE": result = "ON %s" % self.convert_date(second) - elif name in self.flags: + elif name in self.flags.values(): if second: result = "%s" % (name.upper()[1:]) else: @@ -7180,7 +7252,7 @@ class DAL(object): or - db = DAL({"uri": ..., "items": ...}) # experimental + db = DAL(**{"uri": ..., "tables": [...]...}) # experimental db.define_table('tablename', Field('fieldname1'), Field('fieldname2')) @@ -7295,8 +7367,9 @@ class DAL(object): migrate_enabled=True, fake_migrate_all=False, decode_credentials=False, driver_args=None, adapter_args=None, attempts=5, auto_import=False, - bigint_id=False,debug=False,lazy_tables=False, - db_uid=None, do_connect=True, after_connection=None): + bigint_id=False, debug=False, lazy_tables=False, + db_uid=None, do_connect=True, + after_connection=None, tables=None): """ Creates a new Database Abstraction Layer instance. @@ -7308,7 +7381,7 @@ class DAL(object): experimental: you can specify a dictionary as uri parameter i.e. with db = DAL({"uri": "sqlite://storage.sqlite", - "items": {...}, ...}) + "tables": {...}, ...}) for an example of dict input you can check the output of the scaffolding db model with @@ -7352,18 +7425,6 @@ class DAL(object): :lazy_tables (defaults to False): delay table definition until table access :after_connection (defaults to None): a callable that will be execute after the connection """ - - items = None - if isinstance(uri, dict): - if "items" in uri: - items = uri.pop("items") - try: - newuri = uri.pop("uri") - except KeyError: - newuri = DEFAULT_URI - locals().update(uri) - uri = newuri - if uri == '' and db_uid is not None: return if not decode_credentials: credential_decoder = lambda cred: cred @@ -7456,31 +7517,20 @@ class DAL(object): self._fake_migrate = fake_migrate self._migrate_enabled = migrate_enabled self._fake_migrate_all = fake_migrate_all - if auto_import or items: + if auto_import or tables: self.import_table_definitions(adapter.folder, - items=items) + tables=tables) @property def tables(self): return self._tables def import_table_definitions(self, path, migrate=False, - fake_migrate=False, items=None): + fake_migrate=False, tables=None): pattern = pjoin(path,self._uri_hash+'_*.table') - if items: - for tablename, table in items.iteritems(): - # TODO: read all field/table options - fields = [] - # remove unsupported/illegal Table arguments - [table.pop(name) for name in ("name", "fields") if - name in table] - if "items" in table: - for fieldname, field in table.pop("items").iteritems(): - # remove unsupported/illegal Field arguments - [field.pop(key) for key in ("requires", "name", - "compute", "colname") if key in field] - fields.append(Field(str(fieldname), **field)) - self.define_table(str(tablename), *fields, **table) + if tables: + for table in tables: + self.define_table(**table) else: for filename in glob.glob(pattern): tfile = self._adapter.file_open(filename, 'r') @@ -7778,8 +7828,14 @@ def index(): ): if not fields and 'fields' in args: fields = args.get('fields',()) - if not isinstance(tablename,str): - raise SyntaxError("missing table name") + if not isinstance(tablename, str): + if isinstance(tablename, unicode): + try: + tablename = str(tablename) + except UnicodeEncodeError: + raise SyntaxError("invalid unicode table name") + else: + raise SyntaxError("missing table name") elif hasattr(self,tablename) or tablename in self.tables: if not args.get('redefine',False): raise SyntaxError('table already defined: %s' % tablename) @@ -7843,48 +7899,40 @@ def index(): if on_define: on_define(table) return table - def as_dict(self, flat=False, sanitize=True, field_options=True): - dbname = db_uid = uri = None + def as_dict(self, flat=False, sanitize=True): + db_uid = uri = None if not sanitize: - uri, dbname, db_uid = (self._uri, self._dbname, self._db_uid) - db_as_dict = dict(items={}, tables=[], uri=uri, dbname=dbname, - db_uid=db_uid, - **dict([(k, getattr(self, "_" + k)) for - k in 'pool_size','folder','db_codec', + uri, db_uid = (self._uri, self._db_uid) + db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid, + **dict([(k, getattr(self, "_" + k, None)) + for k in 'pool_size','folder','db_codec', 'check_reserved','migrate','fake_migrate', 'migrate_enabled','fake_migrate_all', 'decode_credentials','driver_args', 'adapter_args', 'attempts', 'bigint_id','debug','lazy_tables', 'do_connect'])) - for table in self: - tablename = str(table) - db_as_dict["tables"].append(tablename) - db_as_dict["items"][tablename] = table.as_dict(flat=flat, - sanitize=sanitize, - field_options=field_options) + db_as_dict["tables"].append(table.as_dict(flat=flat, + sanitize=sanitize)) return db_as_dict - def as_xml(self, sanitize=True, field_options=True): + def as_xml(self, sanitize=True): if not have_serializers: raise ImportError("No xml serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.xml(d) - def as_json(self, sanitize=True, field_options=True): + def as_json(self, sanitize=True): if not have_serializers: raise ImportError("No json serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.json(d) - def as_yaml(self, sanitize=True, field_options=True): + def as_yaml(self, sanitize=True): if not have_serializers: raise ImportError("No YAML serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.yaml(d) def __contains__(self, tablename): @@ -8252,7 +8300,9 @@ class Table(object): if len(self._primarykey)==1: self._id = [f for f in fields if isinstance(f,Field) \ and f.name==self._primarykey[0]][0] - elif not [f for f in fields if isinstance(f,Field) and f.type=='id']: + elif not [f for f in fields if (isinstance(f,Field) and + f.type=='id') or (isinstance(f, dict) and + f.get("type", None)=="id")]: field = Field('id', 'id') newfields.append(field) fieldnames.add('id') @@ -8270,8 +8320,7 @@ class Table(object): if field.db is not None: field = copy.copy(field) include_new(field) - elif isinstance(field, dict) and 'fieldname' and \ - not field['fieldname'] in fieldnames: + elif isinstance(field, dict) and not field['fieldname'] in fieldnames: include_new(Field(**field)) elif isinstance(field, Table): table = field @@ -8826,9 +8875,8 @@ class Table(object): if id_map and cid is not None: id_map_self[long(line[cid])] = new_id - def as_dict(self, flat=False, sanitize=True, field_options=True): - tablename = str(self) - table_as_dict = dict(name=tablename, items={}, fields=[], + def as_dict(self, flat=False, sanitize=True): + table_as_dict = dict(tablename=str(self), fields=[], sequence_name=self._sequence_name, trigger_name=self._trigger_name, common_filter=self._common_filter, format=self._format, @@ -8836,31 +8884,26 @@ class Table(object): for field in self: if (field.readable or field.writable) or (not sanitize): - table_as_dict["fields"].append(field.name) - table_as_dict["items"][field.name] = \ - field.as_dict(flat=flat, sanitize=sanitize, - options=field_options) + table_as_dict["fields"].append(field.as_dict( + flat=flat, sanitize=sanitize)) return table_as_dict - def as_xml(self, sanitize=True, field_options=True): + def as_xml(self, sanitize=True): if not have_serializers: raise ImportError("No xml serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.xml(d) - def as_json(self, sanitize=True, field_options=True): + def as_json(self, sanitize=True): if not have_serializers: raise ImportError("No json serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.json(d) - def as_yaml(self, sanitize=True, field_options=True): + def as_yaml(self, sanitize=True): if not have_serializers: raise ImportError("No YAML serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - field_options=field_options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.yaml(d) def with_alias(self, alias): @@ -9070,7 +9113,7 @@ class Expression(object): db = self.db return Query(db, db._adapter.REGEXP, self, value) - def belongs(self, *value): + def belongs(self, *value, **kwattr): """ Accepts the following inputs: field.belongs(1,2) @@ -9085,6 +9128,11 @@ class Expression(object): value = value[0] if isinstance(value,Query): value = db(value)._select(value.first._table._id) + elif not isinstance(value, basestring): + value = set(value) + if kwattr.get('null') and None in value: + value.remove(None) + return (self == None) | Query(db, db._adapter.BELONGS, self, value) return Query(db, db._adapter.BELONGS, self, value) def startswith(self, value): @@ -9336,8 +9384,13 @@ class Field(Expression): self.op = None self.first = None self.second = None + if isinstance(fieldname, unicode): + try: + fieldname = str(fieldname) + except UnicodeEncodeError: + raise SyntaxError('Field: invalid unicode field name') self.name = fieldname = cleanup(fieldname) - if not isinstance(fieldname,str) or hasattr(Table,fieldname) or \ + if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \ fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname): raise SyntaxError('Field: invalid field name: %s' % fieldname) self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type @@ -9537,113 +9590,61 @@ class Field(Expression): def count(self, distinct=None): return Expression(self.db, self.db._adapter.COUNT, self, distinct, 'integer') - def as_dict(self, flat=False, sanitize=True, options=True): - - attrs = ('type', 'length', 'default', 'required', - 'ondelete', 'notnull', 'unique', 'uploadfield', - 'widget', 'label', 'comment', 'writable', 'readable', - 'update', 'authorize', 'autodelete', 'represent', - 'uploadfolder', 'uploadseparate', 'uploadfs', - 'compute', 'custom_store', 'custom_retrieve', - 'custom_retrieve_file_properties', 'custom_delete', - 'filter_in', 'filter_out', 'custom_qualifier', - 'map_none', 'name') - - SERIALIZABLE_TYPES = (int, long, basestring, dict, list, - float, tuple, bool, type(None)) + def as_dict(self, flat=False, sanitize=True): + attrs = ("name", 'authorize', 'represent', 'ondelete', + 'custom_store', 'autodelete', 'custom_retrieve', + 'filter_out', 'uploadseparate', 'widget', 'uploadfs', + 'update', 'custom_delete', 'uploadfield', 'uploadfolder', + 'custom_qualifier', 'unique', 'writable', 'compute', + 'map_none', 'default', 'type', 'required', 'readable', + 'requires', 'comment', 'label', 'length', 'notnull', + 'custom_retrieve_file_properties', 'filter_in') + serializable = (int, long, basestring, float, tuple, + bool, type(None)) def flatten(obj): - if flat: - if isinstance(obj, flatten.__class__): - return str(type(obj)) - elif isinstance(obj, type): - try: - return str(obj).split("'")[1] - except IndexError: - return str(obj) - elif not isinstance(obj, SERIALIZABLE_TYPES): - return str(obj) - elif isinstance(obj, dict): - newobj = dict() - for k, v in obj.items(): - newobj[k] = flatten(v) - return newobj - elif isinstance(obj, (list, tuple, set)): - return [flatten(v) for v in obj] - else: - return obj - elif isinstance(obj, (dict, set)): - return obj.copy() - else: return obj - - def filter_requires(t, r, options=True): - if sanitize and any([keyword in str(t).upper() for - keyword in ("CRYPT", "IS_STRONG")]): + if isinstance(obj, dict): + return dict((flatten(k), flatten(v)) for k, v in + obj.items()) + elif isinstance(obj, (tuple, list, set)): + return [flatten(v) for v in obj] + elif isinstance(obj, serializable): + return obj + elif isinstance(obj, (datetime.datetime, + datetime.date, datetime.time)): + return str(obj) + else: return None - if not isinstance(r, dict): - if options and hasattr(r, "options"): - if callable(r.options): - r.options() - newr = r.__dict__.copy() - else: - newr = r.copy() - - # remove options if not required - if not options and newr.has_key("labels"): - [newr.update({key:None}) for key in - ("labels", "theset") if (key in newr)] - - for k, v in newr.items(): - if k == "other": - if isinstance(v, dict): - otype, other = v.popitem() - else: - otype = flatten(type(v)) - other = v - newr[k] = {otype: filter_requires(otype, other, - options=options)} - else: - newr[k] = flatten(v) - return newr - - if isinstance(self.requires, (tuple, list, set)): - requires = dict([(flatten(type(r)), - filter_requires(type(r), r, - options=options)) for - r in self.requires]) - else: - requires = {flatten(type(self.requires)): - filter_requires(type(self.requires), - self.requires, options=options)} - - d = dict(colname="%s.%s" % (self.tablename, self.name), - requires=requires) - d.update([(attr, flatten(getattr(self, attr))) for attr in attrs]) + d = dict() + if not (sanitize and not (self.readable or self.writable)): + for attr in attrs: + if flat: + d.update({attr: flatten(getattr(self, attr))}) + else: + d.update({attr: getattr(self, attr)}) + d["fieldname"] = d.pop("name") return d - def as_xml(self, sanitize=True, options=True): + def as_xml(self, sanitize=True): if have_serializers: xml = serializers.xml else: raise ImportError("No xml serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - options=options) + d = self.as_dict(flat=True, sanitize=sanitize) return xml(d) - def as_json(self, sanitize=True, options=True): + def as_json(self, sanitize=True): if have_serializers: json = serializers.json else: raise ImportError("No json serializers available") - d = self.as_dict(flat=True, sanitize=sanitize, - options=options) + d = self.as_dict(flat=True, sanitize=sanitize) return json(d) - def as_yaml(self, sanitize=True, options=True): + def as_yaml(self, sanitize=True): if have_serializers: - d = self.as_dict(flat=True, sanitize=sanitize, - options=options) + d = self.as_dict(flat=True, sanitize=sanitize) return serializers.yaml(d) else: raise ImportError("No YAML serializers available") diff --git a/gluon/globals.py b/gluon/globals.py index 9798cb1a..884aac6b 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -310,10 +310,9 @@ class Request(Storage): self.is_restful = True method = _self.env.request_method if len(_self.args) and '.' in _self.args[-1]: - _self.args[- - 1], _self.extension = _self.args[-1].rsplit('.', 1) + _self.args[-1], _, self.extension = self.args[-1].rpartition('.') current.response.headers['Content-Type'] = \ - contenttype(_self.extension.lower()) + contenttype('.' + _self.extension.lower()) if not method in ['GET', 'POST', 'DELETE', 'PUT']: raise HTTP(400, "invalid method") rest_action = _action().get(method, None) @@ -651,115 +650,34 @@ class Response(Storage): class Session(Storage): - """ defines the session object and the default values of its members (None) + + response.session_storage_type : 'file', 'db', or 'cookie' + response.session_cookie_compression_level : + response.session_cookie_expires : cookie expiration + response.session_cookie_key : for encrypted sessions in cookies + response.session_id : a number or None if no session + response.session_id_name : + response.session_locked : + response.session_masterapp : + response.session_new : a new session obj is being created + + if session in cookie: + + response.session_data_name : name of the cookie for session data + + if session in db: + + response.session_db_record_id : + response.session_db_table : + response.session_db_unique_key : + + if session in file: + + response.session_file : + response.session_filename : """ - def renew( - self, - request=None, - response=None, - db=None, - tablename='web2py_session', - masterapp=None, - clear_session=False - ): - if request is None: - request = current.request - if response is None: - response = current.response - - #check if session is separate - separate = None - if response.session and response.session_id[2:3] == "/": - separate = lambda session_name: session_name[-2:] - - self._unlock(response) - - if not masterapp: - masterapp = request.application - - # Load session data from cookie - cookies = request.cookies - - # check if there is a session_id in cookies - if response.session_id_name in cookies: - response.session_id = \ - cookies[response.session_id_name].value - else: - response.session_id = None - - # if the session goes in file - if response.session_storage_type == 'file': - if global_settings.db_sessions is True \ - or masterapp in global_settings.db_sessions: - return - - client = request.client and request.client.replace(':', '.') - - uuid = web2py_uuid() - response.session_id = '%s-%s' % (client, uuid) - if separate: - prefix = separate(response.session_id) - response.session_id = '%s/%s' % \ - (prefix, response.session_id) - response.session_filename = \ - os.path.join(up(request.folder), masterapp, - 'sessions', response.session_id) - response.session_new = True - - # else the session goes in db - elif response.session_storage_type == 'db': - # verify that session_id exists - if not response.session_id: - return - - # verify if tablename was set or used in connect - if response.session_table_name and tablename == 'web2py_session': - tablename = response.session_table_name - - if global_settings.db_sessions is not True: - global_settings.db_sessions.add(masterapp) - - if response.session_file: - self._close(response) - if settings.global_settings.web2py_runtime_gae: - # in principle this could work without GAE - request.tickets_db = db - - tname = tablename + '_' + masterapp - - if not db: - raise Exception('No database parameter passed: "db=database"') - - table = db.get(tname, None) - if table is None: - raise Exception('No session to renew') - - # Get session data out of the database - (record_id, unique_key) = response.session_id.split(':') - if record_id == '0': - raise Exception('record_id == 0') - # Select from database - row = db(table.id == record_id).select() - row = row and row[0] or None - # Make sure the session data exists in the database - if not row or row.unique_key != unique_key: - raise Exception('No record') - - unique_key = web2py_uuid() - db(table.id == record_id).update(unique_key=unique_key) - response.session_id = '%s:%s' % (record_id, unique_key) - response.session_db_table = table - response.session_db_record_id = record_id - response.session_db_unique_key = unique_key - - rcookies = response.cookies - if response.session_id_name: - rcookies[response.session_id_name] = response.session_id - rcookies[response.session_id_name]['path'] = '/' - if clear_session: - self.clear() def connect( self, @@ -780,109 +698,102 @@ class Session(Storage): and it is used to determine a session prefix. separate can be True and it is set to session_name[-2:] """ - if request is None: - request = current.request - if response is None: - response = current.response - if separate is True: - separate = lambda session_name: session_name[-2:] + request = request or current.request + response = response or current.response + masterapp = masterapp or request.application + cookies = request.cookies + self._unlock(response) - if not masterapp: - masterapp = request.application + + response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() response.session_cookie_expires = cookie_expires - - # Load session data from cookie - cookies = request.cookies + response.session_client = str(request.client).replace(':', '.') + response.session_cookie_key = cookie_key + response.session_cookie_compression_level = compression_level # check if there is a session_id in cookies - if response.session_id_name in cookies: - response.session_id = \ - cookies[response.session_id_name].value - else: + try: + response.session_id = cookies[response.session_id_name].value + except KeyError: response.session_id = None - # check if there is session data in cookies - if response.session_data_name in cookies: - session_cookie_data = cookies[response.session_data_name].value - else: - session_cookie_data = None - # if we are supposed to use cookie based session data if cookie_key: response.session_storage_type = 'cookie' - response.session_cookie_key = cookie_key - response.session_cookie_compression_level = compression_level + elif db: + response.session_storage_type = 'db' + else: + response.session_storage_type = 'file' + # why do we do this? + # because connect may be called twice, by web2py and in models. + # the first time there is no db yet so it should do nothing + if (global_settings.db_sessions is True or + masterapp in global_settings.db_sessions): + return + + if response.session_storage_type == 'cookie': + # check if there is session data in cookies + if response.session_data_name in cookies: + session_cookie_data = cookies[response.session_data_name].value + else: + session_cookie_data = None if session_cookie_data: data = secure_loads(session_cookie_data, cookie_key, compression_level=compression_level) if data: self.update(data) + response.session_id = True + # else if we are supposed to use file based sessions - elif not db: - response.session_storage_type = 'file' - if global_settings.db_sessions is True \ - or masterapp in global_settings.db_sessions: - return + elif response.session_storage_type == 'file': response.session_new = False - client = request.client and request.client.replace(':', '.') + response.session_file = None + # check if the session_id points to a valid sesion filename if response.session_id: - if regex_session_id.match(response.session_id): + if not regex_session_id.match(response.session_id): + response.session_id = None + else: response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) - else: - response.session_id = None - # do not try load the data from file is these was data in cookie - if response.session_id and not session_cookie_data: - # os.path.exists(response.session_filename): - try: - response.session_file = \ - open(response.session_filename, 'rb+') try: + response.session_file = \ + open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True self.update(cPickle.load(response.session_file)) response.session_file.seek(0) - oc = response.session_filename.split('/')[-1]\ - .split('-')[0] - if check_client and client != oc: + oc = response.session_filename.split('/')[-1].split('-')[0] + if check_client and response.session_client != oc: raise Exception("cookie attack") except: response.session_id = None - finally: - pass - #This causes admin login to break. Must find out why. - #self._close(response) - except: - response.session_file = None if not response.session_id: uuid = web2py_uuid() - response.session_id = '%s-%s' % (client, uuid) + response.session_id = '%s-%s' % (response.session_client, uuid) + separate = separate and (lambda session_name: session_name[-2:]) if separate: prefix = separate(response.session_id) - response.session_id = '%s/%s' % \ - (prefix, response.session_id) + response.session_id = '%s/%s' % (prefix, response.session_id) response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) response.session_new = True + # else the session goes in db - else: - response.session_storage_type = 'db' + elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) + # if had a session on file alreday, close it (yes, can happen) if response.session_file: self._close(response) + # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: - # in principle this could work without GAE request.tickets_db = db - if masterapp == request.application: - table_migrate = migrate - else: - table_migrate = False + table_migrate = (masterapp == request.application) tname = tablename + '_' + masterapp table = db.get(tname, None) Field = db.Field @@ -899,46 +810,111 @@ class Session(Storage): migrate=table_migrate, ) table = db[tname] # to allow for lazy table - try: - - # Get session data out of the database - (record_id, unique_key) = response.session_id.split(':') - if record_id == '0': - raise Exception('record_id == 0') - # Select from database - if not session_cookie_data: - rows = db(table.id == record_id).select() - # Make sure the session data exists in the database - if len(rows) == 0 or rows[0].unique_key != unique_key: - raise Exception('No record') - # rows[0].update_record(locked=True) - # Unpickle the data - session_data = cPickle.loads(rows[0].session_data) - self.update(session_data) - except Exception: - record_id = None - unique_key = web2py_uuid() - session_data = {} - response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_table = table - response.session_db_record_id = record_id - response.session_db_unique_key = unique_key - # keep tablename parameter for use in session renew - response.session_table_name = tablename + if response.session_id: + # Get session data out of the database + try: + (record_id, unique_key) = response.session_id.split(':') + record_id = long(record_id) + except (TypeError,ValueError): + record_id = None + + # Select from database + if record_id: + row = table(record_id) #,unique_key=unique_key) + # Make sure the session data exists in the database + if row: + # rows[0].update_record(locked=True) + # Unpickle the data + session_data = cPickle.loads(row.session_data) + self.update(session_data) + else: + record_id = None + if record_id: + response.session_id = '%s:%s' % (record_id, unique_key) + response.session_db_unique_key = unique_key + response.session_db_record_id = record_id + else: + response.session_id = None + response.session_new = True + + if self.flash: + (response.flash, self.flash) = (self.flash, None) + + def renew(self, clear_session=False): + + if clear_session: + self.clear() + + request = current.request + response = current.response + session = response.session + masterapp = response.session_masterapp + cookies = request.cookies + + if response.session_storage_type == 'cookie': + return + + # if the session goes in file + if response.session_storage_type == 'file': + self._close(response) + uuid = web2py_uuid() + response.session_id = '%s-%s' % (response.session_client, uuid) + separate = (lambda s: s[-2:]) if session and response.session_id[2:3]=="/" else None + if separate: + prefix = separate(response.session_id) + response.session_id = '%s/%s' % \ + (prefix, response.session_id) + response.session_filename = \ + os.path.join(up(request.folder), masterapp, + 'sessions', response.session_id) + response.session_new = True + + # else the session goes in db + elif response.session_storage_type == 'db': + table = response.session_db_table + + # verify that session_id exists + if response.session_file: + self._close(response) + if response.session_new: + return + # Get session data out of the database + (record_id, unique_key) = response.session_id.split(':') + + if record_id.isdigit() and long(record_id)>1: + new_unique_key = web2py_uuid() + rows = db(table.id==record_id)(table.unique_key==unique_key)\ + .update(unique_key=new_unique_key) + else: + rows = None + if rows: + response.session_id = '%s:%s' % (record_id, unique_key) + response.session_db_record_id = record_id + response.session_db_unique_key = new_unique_key + else: + response.session_new = True + + def save_session_id_cookie(self): + request = current.request + response = current.response + session = response.session + masterapp = response.session_masterapp + cookies = request.cookies rcookies = response.cookies - rcookies[response.session_id_name] = response.session_id - rcookies[response.session_id_name]['path'] = '/' - if cookie_expires: - rcookies[response.session_id_name][ - 'expires'] = cookie_expires.strftime(FMT) + # if not cookie_key, but session_data_name in cookies # expire session_data_name from cookies - if session_cookie_data: + if response.session_data_name in cookies: rcookies[response.session_data_name] = 'expired' rcookies[response.session_data_name]['path'] = '/' rcookies[response.session_data_name]['expires'] = PAST - if self.flash: - (response.flash, self.flash) = (self.flash, None) + if response.session_id: + rcookies[response.session_id_name] = response.session_id + rcookies[response.session_id_name]['path'] = '/' + if response.session_cookie_expires: + rcookies[response.session_id_name]['expires'] = \ + response.session_cookie_expires.strftime(FMT) def clear(self): previous_session_hash = self.pop('_session_hash', None) @@ -970,10 +946,13 @@ class Session(Storage): self._forget = True def _try_store_in_cookie(self, request, response): - if response.session_storage_type != 'cookie': + if self._forget or self._unchanged(): return False name = response.session_data_name - value = secure_dumps(dict(self), response.session_cookie_key, compression_level=response.session_cookie_compression_level) + compression_level = response.session_cookie_compression_level + value = secure_dumps(dict(self), + response.session_cookie_key, + compression_level=compression_level) expires = response.session_cookie_expires rcookies = response.cookies rcookies.pop(name, None) @@ -1000,37 +979,44 @@ class Session(Storage): # don't save if file-based sessions, # no session id, or session being forgotten # or no changes to session - if response.session_storage_type != 'db' or not response.session_id \ - or self._forget or self._unchanged(): + + if not response.session_db_table or self._forget or self._unchanged(): + if (not response.session_db_table and + global_settings.db_sessions is not True and + response.session_masterapp in global_settings.db_sessions): + global_settings.db_sessions.remove(response.session_masterapp) return False table = response.session_db_table record_id = response.session_db_record_id - unique_key = response.session_db_unique_key + if response.session_new: + unique_key = web2py_uuid() + else: + unique_key = response.session_db_unique_key dd = dict(locked=False, - client_ip=request.client.replace(':', '.'), + client_ip=response.session_client, modified_datetime=request.now, session_data=cPickle.dumps(dict(self)), unique_key=unique_key) if record_id: - table._db(table.id == record_id).update(**dd) - else: + table(record_id).update_record(**dd) + if not record_id: record_id = table.insert(**dd) + response.session_id = '%s:%s' % (record_id, unique_key) + response.session_db_unique_key = unique_key + response.session_db_record_id = record_id - cookies, session_id_name = response.cookies, response.session_id_name - cookies[session_id_name] = '%s:%s' % (record_id, unique_key) - cookies[session_id_name]['path'] = '/' + self.save_session_id_cookie() return True def _try_store_in_cookie_or_file(self, request, response): - return \ - self._try_store_in_cookie(request, response) or \ - self._try_store_in_file(request, response) + if response.session_storage_type == 'file': + return self._try_store_in_file(request, response) + if response.session_storage_type == 'cookie': + return self._try_store_in_cookie(request, response) def _try_store_in_file(self, request, response): - if response.session_storage_type != 'file': - return False try: if not response.session_id or self._forget or self._unchanged(): return False @@ -1042,12 +1028,13 @@ class Session(Storage): response.session_file = open(response.session_filename, 'wb') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True - if response.session_file: cPickle.dump(dict(self), response.session_file) response.session_file.truncate() finally: self._close(response) + + self.save_session_id_cookie() return True def _unlock(self, response): diff --git a/gluon/html.py b/gluon/html.py index 61a5bee8..da092a33 100644 --- a/gluon/html.py +++ b/gluon/html.py @@ -1075,10 +1075,8 @@ class DIV(XmlComponent): matches = [] # check if the component has an attribute with the same # value as provided - check = True tag = getattr(self, 'tag').replace('/', '') - if args and tag not in args: - check = False + check = not (args and tag not in args) for (key, value) in kargs.iteritems(): if key not in ['first_only', 'replace', 'find_text']: if isinstance(value, (str, int)): @@ -1109,24 +1107,28 @@ class DIV(XmlComponent): def replace_component(i): if replace is None: del self[i] - elif callable(replace): - self[i] = replace(self[i]) + return i else: - self[i] = replace + self[i] = replace(self[i]) if callable(replace) else replace + return i+1 # loop the components if find_text or find_components: - for i, c in enumerate(self.components): + i = 0 + while i 0 or + if (len(self.options.ssl_certificate) > 0 or len(self.options.ssl_private_key) > 0): proto = 'https' else: @@ -503,7 +503,7 @@ class web2pyDialog(object): password, pid_filename=options.pid_filename, log_filename=options.log_filename, - profiler_filename=options.profiler_filename, + profiler_dir=options.profiler_dir, ssl_certificate=options.ssl_certificate, ssl_private_key=options.ssl_private_key, ssl_ca_certificate=options.ssl_ca_certificate, @@ -865,9 +865,9 @@ def console(): parser.add_option('-F', '--profiler', - dest='profiler_filename', + dest='profiler_dir', default=None, - help='profiler filename') + help='profiler dir') parser.add_option('-t', '--taskbar', @@ -914,7 +914,7 @@ def console(): dest='run_system_tests', default=False, help=msg) - + msg = ('adds coverage reporting (needs --run_system_tests), ' 'python 2.7 and the coverage module installed. ' 'You can alter the default path setting the environmental ' @@ -925,7 +925,7 @@ def console(): dest='with_coverage', default=False, help=msg) - + if '-A' in sys.argv: k = sys.argv.index('-A') elif '--args' in sys.argv: @@ -1117,7 +1117,8 @@ def start(cron=True): if not options.args is None: sys.argv[:] = options.args run(options.shell, plain=options.plain, bpython=options.bpython, - import_models=options.import_models, startfile=options.run, cronjob=options.cronjob) + import_models=options.import_models, startfile=options.run, + cronjob=options.cronjob) return # ## if -C start cron run (extcron) and exit @@ -1267,7 +1268,7 @@ end tell password=options.password, pid_filename=options.pid_filename, log_filename=options.log_filename, - profiler_filename=options.profiler_filename, + profiler_dir=options.profiler_dir, ssl_certificate=options.ssl_certificate, ssl_private_key=options.ssl_private_key, ssl_ca_certificate=options.ssl_ca_certificate, diff --git a/scgihandler.py b/scgihandler.py index c7f51cfd..2c3fb03c 100755 --- a/scgihandler.py +++ b/scgihandler.py @@ -61,7 +61,7 @@ wsgiapp = WSGIFilterMiddleware(gluon.main.wsgibase, GzipWSGIFilter) if LOGGING: application = gluon.main.appfactory(wsgiapp=wsgiapp, logfilename='httpserver.log', - profilerfilename=None) + profiler_dir=None) else: application = wsgiapp diff --git a/wsgihandler.py b/wsgihandler.py index e98c45e4..eadf2ab1 100644 --- a/wsgihandler.py +++ b/wsgihandler.py @@ -35,7 +35,7 @@ import gluon.main if LOGGING: application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase, logfilename='httpserver.log', - profilerfilename=None) + profiler_dir=None) else: application = gluon.main.wsgibase