Merge ssh://github.com/web2py/web2py

VERSION

@@ -1 +1 @@
-Version 2.6.0-development+timestamp.2013.08.09.16.07.32
+Version 2.6.0-development+timestamp.2013.08.21.04.28.27

app.example.yaml

@@ -25,13 +25,13 @@ handlers:
 # the parametric router's language logic. 
 # You cannot use them together.
 
-- url: /(.+?)/[^_]*\/?static/_\d.\d.\d\/?(.+)
+- url: /(.+?)/static/_\d.\d.\d\/(.+)
   static_files: applications/\1/static/\2
   upload: applications/(.+?)/static/(.+)
   secure: optional
   expiration: "365d"
 
-- url: /(.+?)/[^_]*\/?static/?(.+)
+- url: /(.+?)/static/(.+)
   static_files: applications/\1/static/\2
   upload: applications/(.+?)/static/(.+)
   secure: optional

applications/admin/controllers/default.py

@@ -296,8 +296,8 @@ def site():
         apps = [f for f in apps if f in FILTER_APPS]
 
     apps = sorted(apps, lambda a, b: cmp(a.upper(), b.upper()))
-
+    myplatform = platform.python_version()
-    return dict(app=None, apps=apps, myversion=myversion,
+    return dict(app=None, apps=apps, myversion=myversion, myplatform=myplatform,
                 form_create=form_create, form_update=form_update)
 
 

applications/admin/views/default/site.html

@@ -78,7 +78,8 @@
     <h4>{{=T("Version")}}</h4>
     <p>
       <tt>{{=myversion}}</tt><br/>
-      ({{=T("Running on %s", request.env.server_software)}})
+      {{running_on = T("Running on %s", request.env.server_software or 'Unknown')}}
+      ({{="%s, Python %s" % (running_on, myplatform)}})
     </p>
     <p id="check_version" class="row-buttons">
       {{if session.check_version:}}

fcgihandler.py

@@ -42,7 +42,7 @@ import gluon.contrib.gateways.fcgi as fcgi
 if LOGGING:
     application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase,
                                         logfilename='httpserver.log',
-                                        profilerfilename=None)
+                                        profiler_dir=None)
 else:
     application = gluon.main.wsgibase
 

gluon/contrib/fpdf/html.py

@@ -33,7 +33,6 @@ class HTML2FPDF(HTMLParser):
         self.href = ''
         self.align = ''
         self.page_links = {}
-        self.font_list = ("times","courier", "helvetica")
         self.font = None
         self.font_stack = [] 
         self.pdf = pdf
@@ -55,6 +54,7 @@ class HTML2FPDF(HTMLParser):
         self.thead = None
         self.tfoot = None
         self.theader_out = self.tfooter_out = False
+        self.hsize = dict(h1=2, h2=1.5, h3=1.17, h4=1, h5=0.83, h6=0.67)
         
     def width2mm(self, length):
         if length[-1]=='%':
@@ -88,7 +88,7 @@ class HTML2FPDF(HTMLParser):
             else:
                 self.set_style('B',True)
                 border = border or 'B'
-                align = 'C'
+                align = self.td.get('align', 'C')[0].upper()
             bgcolor = hex2dec(self.td.get('bgcolor', self.tr.get('bgcolor', '')))
             # parsing table header/footer (drawn later):
             if self.thead is not None:
@@ -179,8 +179,8 @@ class HTML2FPDF(HTMLParser):
             self.pdf.ln(5)
             if attrs:
                 if attrs: self.align = attrs.get('align')
-        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
+        if tag in self.hsize:
-            k = (2, 1.5, 1.17, 1, 0.83, 0.67)[int(tag[1])]
+            k = self.hsize[tag]
             self.pdf.ln(5*k)
             self.pdf.set_text_color(150,0,0)
             self.pdf.set_font_size(12 * k)
@@ -219,7 +219,7 @@ class HTML2FPDF(HTMLParser):
                 self.color = hex2dec(attrs['color'])
                 self.set_text_color(*color)
                 self.color = color
-            if 'face' in attrs and attrs['face'].lower() in self.font_list:
+            if 'face' in attrs:
                 face = attrs.get('face').lower()
                 self.pdf.set_font(face)
                 self.font_face = face

gluon/contrib/fpdf/template.py

@@ -110,8 +110,8 @@ class Template:
             pdf.set_auto_page_break(False,margin=0)
 
             for element in sorted(self.elements,key=lambda x: x['priority']):
-                #print "dib",element['type'], element['name'], element['x1'], element['y1'], element['x2'], element['y2']
+                # make a copy of the element:
-                element = element.copy()
+                element = dict(element)
                 element['text'] = self.texts[pg].get(element['name'].lower(), element['text'])
                 if 'rotate' in element:
                     pdf.rotate(element['rotate'], element['x1'], element['y1'])

gluon/contrib/memdb.py

@@ -299,6 +299,11 @@ class Table(DALStorage):
     def __str__(self):
         return self._tablename
 
+    def __call__(self, id):
+        return self.get(id)
+
+    def __getitem__(self,id):
+        return self.get(id)
 
 class Expression(object):
 
@@ -502,9 +507,8 @@ class Query(object):
                 'Query: right side of filter must be a value or entity')
         if isinstance(left, Field) and left.name == 'id':
             if op == '=':
-                self.get_one = \
+                self.get_one = QueryException(
-                    QueryException(tablename=left._tablename,
+                    tablename=left._tablename, id=long(right or 0))
-                                   id=long(right))
                 return
             else:
                 raise SyntaxError('only equality by id is supported')

gluon/contrib/redis_session.py

@@ -168,7 +168,7 @@ class MockQuery(object):
     def select(self):
         if self.op == 'eq' and self.field == 'id' and self.value:
             #means that someone wants to retrieve the key self.value
-            key = self.keyprefix + ':' + self.value
+            key = self.keyprefix + ':' + str(self.value)
             if self.with_lock:
                 acquire_lock(self.db, key + ':lock', self.value)
             rtn = self.db.hgetall(key)
@@ -198,7 +198,7 @@ class MockQuery(object):
     def update(self, **kwargs):
         #means that the session has been found and needs an update
         if self.op == 'eq' and self.field == 'id' and self.value:
-            key = "%s:%s" % (self.keyprefix, self.value)
+            key = self.keyprefix + ':' + str(self.value)
             with self.db.pipeline() as pipe:
                 pipe.hmset(key, kwargs)
                 if self.session_expiry:

gluon/dal.py

@@ -684,6 +684,11 @@ class BaseAdapter(ConnectionPool):
             return None
         return isinstance(exception, self.driver.OperationalError)
 
+    def isProgrammingError(self,exception):
+        if not hasattr(self.driver, "ProgrammingError"):
+            return None
+        return isinstance(exception, self.driver.ProgrammingError)
+
     def id_query(self, table):
         return table._id != None
 
@@ -1263,17 +1268,12 @@ class BaseAdapter(ConnectionPool):
         return '(%s OR %s)' % (self.expand(first), self.expand(second))
 
     def BELONGS(self, first, second):
-       if isinstance(second, str):
+        if isinstance(second, str):
-           return '(%s IN (%s))' % (self.expand(first), second[:-1])
+            return '(%s IN (%s))' % (self.expand(first), second[:-1])
-       if not second:
+        if not second:
-           return '(1=0)'
+            return '(1=0)'
-       if isinstance(second, (list,tuple,frozenset)):
+        items = ','.join(self.expand(item, first.type) for item in second)
-           second = set(second) # remove duplicates, make mutable
+        return '(%s IN (%s))' % (self.expand(first), items)
-       if isinstance(second, set) and None in second:
-           second.remove(None)
-           return self.OR(self.EQ(first, None), self.BELONGS(first, second))
-       items = ','.join(self.expand(item, first.type) for item in second)
-       return '(%s IN (%s))' % (self.expand(first), items)
 
     def REGEXP(self, first, second):
         "regular expression operator"
@@ -1778,13 +1778,18 @@ class BaseAdapter(ConnectionPool):
         return list(tables)
 
     def commit(self):
-        if self.connection: return self.connection.commit()
+        if self.connection:
+            return self.connection.commit()
 
     def rollback(self):
-        if self.connection: return self.connection.rollback()
+        if self.connection:
+            return self.connection.rollback()
 
     def close_connection(self):
-        if self.connection: return self.connection.close()
+        if self.connection: 
+            r = self.connection.close()
+            self.connection = None
+            return r
 
     def distributed_transaction_begin(self, key):
         return
@@ -4333,7 +4338,8 @@ class DatabaseStoredFile:
             if db.executesql(query):
                 return True
         except Exception, e:
-            if not db._adapter.isOperationalError(e):
+            if not (db._adapter.isOperationalError(e) or
+                    db._adapter.isProgrammingError(e)):
                 raise
             # no web2py_filesystem found?
             tb = traceback.format_exc()
@@ -5835,7 +5841,7 @@ class IMAPAdapter(NoSQLAdapter):
     uid         string
     answered    boolean        Flag
     created     date
-    content     list:string    A list of text or html parts
+    content     list:string    A list of dict text or html parts
     to          string
     cc          string
     bcc         string
@@ -5977,8 +5983,9 @@ class IMAPAdapter(NoSQLAdapter):
 
         """ MESSAGE is an identifier for sequence number"""
 
-        self.flags = ['\\Deleted', '\\Draft', '\\Flagged',
+        self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft',
-                      '\\Recent', '\\Seen', '\\Answered']
+                      'flagged': '\\Flagged', 'recent': '\\Recent',
+                      'seen': '\\Seen', 'answered': '\\Answered'}
         self.search_fields = {
             'id': 'MESSAGE', 'created': 'DATE',
             'uid': 'UID', 'sender': 'FROM',
@@ -6201,7 +6208,7 @@ class IMAPAdapter(NoSQLAdapter):
         return tablename
 
     def is_flag(self, flag):
-        if self.search_fields.get(flag, None) in self.flags:
+        if self.search_fields.get(flag, None) in self.flags.values():
             return True
         else:
             return False
@@ -6233,7 +6240,7 @@ class IMAPAdapter(NoSQLAdapter):
                             Field("uid", "string", writable=False),
                             Field("answered", "boolean"),
                             Field("created", "datetime", writable=False),
-                            Field("content", "list:string", writable=False),
+                            Field("content", list, writable=False),
                             Field("to", "string", writable=False),
                             Field("cc", "string", writable=False),
                             Field("bcc", "string", writable=False),
@@ -6450,23 +6457,23 @@ class IMAPAdapter(NoSQLAdapter):
                 maintype = part.get_content_maintype()
                 if ("%s.attachments" % tablename in colnames) or \
                    ("%s.content" % tablename in colnames):
-                    if "%s.attachments" % tablename in colnames:
+                    payload = part.get_payload(decode=True)
-                        if not ("text" in maintype):
+                    if payload:
-                            payload = part.get_payload(decode=True)
+                        filename = part.get_filename()
-                            if payload:
+                        values = {"mime": part.get_content_type()}
-                                attachment = {
+                        if ((filename or not "text" in maintype) and
-                                    "payload": payload,
+                            ("%s.attachments" % tablename in colnames)):
-                                    "filename": part.get_filename(),
+                            values.update({"payload": payload,
-                                    "encoding": part.get_content_charset(),
+                                "filename": filename,
-                                    "mime": part.get_content_type(),
+                                "encoding": part.get_content_charset(),
-                                    "disposition": part["Content-Disposition"]}
+                                "disposition": part["Content-Disposition"]})
-                                attachments.append(attachment)
+                            attachments.append(values)
-                    if "%s.content" % tablename in colnames:
+                        elif (("text" in maintype) and
-                        payload = part.get_payload(decode=True)
+                              ("%s.content" % tablename in colnames)):
-                        part_charset = self.get_charset(part)
+                            values.update({"text": self.encode_text(payload,
-                        if "text" in maintype:
+                                               self.get_charset(part))})
-                            if payload:
+                            content.append(values)
-                                content.append(self.encode_text(payload, part_charset))
+
                 if "%s.size" % tablename in colnames:
                     if part is not None:
                         size += len(str(part))
@@ -6488,6 +6495,71 @@ class IMAPAdapter(NoSQLAdapter):
         processor = attributes.get('processor',self.parse)
         return processor(imapqry_array, fields, colnames)
 
+    def _insert(self, table, fields):
+        def add_payload(message, obj):
+            payload = Message()
+            charset = obj.get("encoding", "utf-8")
+            payload.set_type(obj.get("mime", None))
+            payload.set_charset(charset)
+            if "text" in obj:
+                payload.set_payload(obj["text"])
+            elif "payload" in obj:
+                payload.set_payload(obj["payload"])
+            if "filename" in obj and obj["filename"]:
+                payload.add_header("Content-Disposition",
+                    "attachment", filename=obj["filename"])
+            message.attach(payload)
+
+        mailbox = table.mailbox
+        d = dict(((k.name, v) for k, v in fields))
+        date_time = (d.get("created", datetime.datetime.now())).timetuple()
+        if len(d) > 0:
+            message = d.get("email", None)
+            attachments = d.get("attachments", [])
+            content = d.get("content", [])
+            flags = " ".join(["\\%s" % flag.capitalize() for flag in
+                     ("answered", "deleted", "draft", "flagged",
+                      "recent", "seen") if d.get(flag, False)])
+            if not message:
+                from email.message import Message
+                mime = d.get("mime", None)
+                charset = d.get("encoding", None)
+                message = Message()
+                message["from"] = d.get("sender", "")
+                message["subject"] = d.get("subject", "")
+                if mime:
+                    message.set_type(mime)
+                if charset:
+                    message.set_charset(charset)
+                for item in ("to", "cc", "bcc"):
+                    value = d.get(item, "")
+                    if isinstance(value, basestring):
+                        message[item] = value
+                    else:
+                        message[item] = ";".join([i for i in
+                            value])
+                if not message.is_multipart():
+                    if isinstance(content, basestring):
+                        message.set_payload(content)
+                    elif len(content) > 0:
+                        message.set_payload(content[0]["text"])
+                else:
+                    [add_payload(message, c) for c in content]
+                    [add_payload(message, a) for a in attachments]
+                message = message.as_string()
+            return (mailbox, flags, date_time, message)
+        else:
+            raise NotImplementedError("IMAP empty insert is not implemented")
+
+    def insert(self, table, fields):
+        values = self._insert(table, fields)
+        result, data = self.connection.append(*values)
+        if result == "OK":
+            uid = int(re.findall("\d+", str(data))[-1])
+            return self.db(table.uid==uid).select(table.id).first().id
+        else:
+            raise Exception("IMAP message append failed: %s" % data)
+
     def _update(self, tablename, query, fields, commit=False):
         # TODO: the adapter should implement an .expand method
         commands = list()
@@ -6721,7 +6793,7 @@ class IMAPAdapter(NoSQLAdapter):
             elif name == "DATE":
                 result = "ON %s" % self.convert_date(second)
 
-            elif name in self.flags:
+            elif name in self.flags.values():
                 if second:
                     result = "%s" % (name.upper()[1:])
                 else:
@@ -7180,7 +7252,7 @@ class DAL(object):
 
        or
 
-       db = DAL({"uri": ..., "items": ...}) # experimental
+       db = DAL(**{"uri": ..., "tables": [...]...}) # experimental
 
        db.define_table('tablename', Field('fieldname1'),
                                     Field('fieldname2'))
@@ -7295,8 +7367,9 @@ class DAL(object):
                  migrate_enabled=True, fake_migrate_all=False,
                  decode_credentials=False, driver_args=None,
                  adapter_args=None, attempts=5, auto_import=False,
-                 bigint_id=False,debug=False,lazy_tables=False,
+                 bigint_id=False, debug=False, lazy_tables=False,
-                 db_uid=None, do_connect=True, after_connection=None):
+                 db_uid=None, do_connect=True,
+                 after_connection=None, tables=None):
         """
         Creates a new Database Abstraction Layer instance.
 
@@ -7308,7 +7381,7 @@ class DAL(object):
                 experimental: you can specify a dictionary as uri
                 parameter i.e. with
                 db = DAL({"uri": "sqlite://storage.sqlite",
-                          "items": {...}, ...})
+                          "tables": {...}, ...})
 
                 for an example of dict input you can check the output
                 of the scaffolding db model with
@@ -7352,18 +7425,6 @@ class DAL(object):
         :lazy_tables (defaults to False): delay table definition until table access
         :after_connection (defaults to None): a callable that will be execute after the connection
         """
-
-        items = None
-        if isinstance(uri, dict):
-            if "items" in uri:
-                items = uri.pop("items")
-            try:
-                newuri = uri.pop("uri")
-            except KeyError:
-                newuri = DEFAULT_URI
-            locals().update(uri)
-            uri = newuri
-
         if uri == '<zombie>' and db_uid is not None: return
         if not decode_credentials:
             credential_decoder = lambda cred: cred
@@ -7456,31 +7517,20 @@ class DAL(object):
         self._fake_migrate = fake_migrate
         self._migrate_enabled = migrate_enabled
         self._fake_migrate_all = fake_migrate_all
-        if auto_import or items:
+        if auto_import or tables:
             self.import_table_definitions(adapter.folder,
-                                          items=items)
+                                          tables=tables)
 
     @property
     def tables(self):
         return self._tables
 
     def import_table_definitions(self, path, migrate=False,
-                                 fake_migrate=False, items=None):
+                                 fake_migrate=False, tables=None):
         pattern = pjoin(path,self._uri_hash+'_*.table')
-        if items:
+        if tables:
-            for tablename, table in items.iteritems():
+            for table in tables:
-                # TODO: read all field/table options
+                self.define_table(**table)
-                fields = []
-                # remove unsupported/illegal Table arguments
-                [table.pop(name) for name in ("name", "fields") if
-                 name in table]
-                if "items" in table:
-                    for fieldname, field in table.pop("items").iteritems():
-                        # remove unsupported/illegal Field arguments
-                        [field.pop(key) for key in ("requires", "name",
-                         "compute", "colname") if key in field]
-                        fields.append(Field(str(fieldname), **field))
-                self.define_table(str(tablename), *fields, **table)
         else:
             for filename in glob.glob(pattern):
                 tfile = self._adapter.file_open(filename, 'r')
@@ -7778,8 +7828,14 @@ def index():
         ):
         if not fields and 'fields' in args:
             fields = args.get('fields',())
-        if not isinstance(tablename,str):
+        if not isinstance(tablename, str):
-            raise SyntaxError("missing table name")
+            if isinstance(tablename, unicode):
+                try:
+                    tablename = str(tablename)
+                except UnicodeEncodeError:
+                    raise SyntaxError("invalid unicode table name")
+            else:
+                raise SyntaxError("missing table name")
         elif hasattr(self,tablename) or tablename in self.tables:
             if not args.get('redefine',False):
                 raise SyntaxError('table already defined: %s' % tablename)
@@ -7843,48 +7899,40 @@ def index():
         if on_define: on_define(table)
         return table
 
-    def as_dict(self, flat=False, sanitize=True, field_options=True):
+    def as_dict(self, flat=False, sanitize=True):
-        dbname = db_uid = uri = None
+        db_uid = uri = None
         if not sanitize:
-            uri, dbname, db_uid = (self._uri, self._dbname, self._db_uid)
+            uri, db_uid = (self._uri, self._db_uid)
-        db_as_dict = dict(items={}, tables=[], uri=uri, dbname=dbname,
+        db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid,
-                          db_uid=db_uid,
+                          **dict([(k, getattr(self, "_" + k, None))
-                          **dict([(k, getattr(self, "_" + k)) for
+                          for k in 'pool_size','folder','db_codec',
-                          k in 'pool_size','folder','db_codec',
                           'check_reserved','migrate','fake_migrate',
                           'migrate_enabled','fake_migrate_all',
                           'decode_credentials','driver_args',
                           'adapter_args', 'attempts',
                           'bigint_id','debug','lazy_tables',
                           'do_connect']))
-
         for table in self:
-            tablename = str(table)
+            db_as_dict["tables"].append(table.as_dict(flat=flat,
-            db_as_dict["tables"].append(tablename)
+                                        sanitize=sanitize))
-            db_as_dict["items"][tablename] = table.as_dict(flat=flat,
-                                         sanitize=sanitize,
-                                         field_options=field_options)
         return db_as_dict
 
-    def as_xml(self, sanitize=True, field_options=True):
+    def as_xml(self, sanitize=True):
         if not have_serializers:
             raise ImportError("No xml serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         field_options=field_options)
         return serializers.xml(d)
 
-    def as_json(self, sanitize=True, field_options=True):
+    def as_json(self, sanitize=True):
         if not have_serializers:
             raise ImportError("No json serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         field_options=field_options)
         return serializers.json(d)
 
-    def as_yaml(self, sanitize=True, field_options=True):
+    def as_yaml(self, sanitize=True):
         if not have_serializers:
             raise ImportError("No YAML serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         field_options=field_options)
         return serializers.yaml(d)
 
     def __contains__(self, tablename):
@@ -8252,7 +8300,9 @@ class Table(object):
             if len(self._primarykey)==1:
                 self._id = [f for f in fields if isinstance(f,Field) \
                                 and f.name==self._primarykey[0]][0]
-        elif not [f for f in fields if isinstance(f,Field) and f.type=='id']:
+        elif not [f for f in fields if (isinstance(f,Field) and
+                  f.type=='id') or (isinstance(f, dict) and
+                  f.get("type", None)=="id")]:
             field = Field('id', 'id')
             newfields.append(field)
             fieldnames.add('id')
@@ -8270,8 +8320,7 @@ class Table(object):
                 if field.db is not None:
                     field = copy.copy(field)                
                 include_new(field)
-            elif isinstance(field, dict) and 'fieldname' and \
+            elif isinstance(field, dict) and not field['fieldname'] in fieldnames:
-                    not field['fieldname'] in fieldnames:
                 include_new(Field(**field))
             elif isinstance(field, Table):
                 table = field
@@ -8826,9 +8875,8 @@ class Table(object):
                 if id_map and cid is not None:
                     id_map_self[long(line[cid])] = new_id
 
-    def as_dict(self, flat=False, sanitize=True, field_options=True):
+    def as_dict(self, flat=False, sanitize=True):
-        tablename = str(self)
+        table_as_dict = dict(tablename=str(self), fields=[],
-        table_as_dict = dict(name=tablename, items={}, fields=[],
         sequence_name=self._sequence_name,
         trigger_name=self._trigger_name,
         common_filter=self._common_filter, format=self._format,
@@ -8836,31 +8884,26 @@ class Table(object):
 
         for field in self:
             if (field.readable or field.writable) or (not sanitize):
-                table_as_dict["fields"].append(field.name)
+                table_as_dict["fields"].append(field.as_dict(
-                table_as_dict["items"][field.name] = \
+                    flat=flat, sanitize=sanitize))
-                    field.as_dict(flat=flat, sanitize=sanitize,
-                                  options=field_options)
         return table_as_dict
 
-    def as_xml(self, sanitize=True, field_options=True):
+    def as_xml(self, sanitize=True):
         if not have_serializers:
             raise ImportError("No xml serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         field_options=field_options)
         return serializers.xml(d)
 
-    def as_json(self, sanitize=True, field_options=True):
+    def as_json(self, sanitize=True):
         if not have_serializers:
             raise ImportError("No json serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         field_options=field_options)
         return serializers.json(d)
 
-    def as_yaml(self, sanitize=True, field_options=True):
+    def as_yaml(self, sanitize=True):
         if not have_serializers:
             raise ImportError("No YAML serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         field_options=field_options)
         return serializers.yaml(d)
 
     def with_alias(self, alias):
@@ -9070,7 +9113,7 @@ class Expression(object):
         db = self.db
         return Query(db, db._adapter.REGEXP, self, value)
 
-    def belongs(self, *value):
+    def belongs(self, *value, **kwattr):
         """
         Accepts the following inputs:
            field.belongs(1,2)
@@ -9085,6 +9128,11 @@ class Expression(object):
             value = value[0]
         if isinstance(value,Query):
             value = db(value)._select(value.first._table._id)
+        elif not isinstance(value, basestring):
+            value = set(value)
+            if kwattr.get('null') and None in value:
+                value.remove(None)
+                return (self == None) | Query(db, db._adapter.BELONGS, self, value)
         return Query(db, db._adapter.BELONGS, self, value)
 
     def startswith(self, value):
@@ -9336,8 +9384,13 @@ class Field(Expression):
         self.op = None
         self.first = None
         self.second = None
+        if isinstance(fieldname, unicode):
+            try:
+                fieldname = str(fieldname)
+            except UnicodeEncodeError:
+                raise SyntaxError('Field: invalid unicode field name')
         self.name = fieldname = cleanup(fieldname)
-        if not isinstance(fieldname,str) or hasattr(Table,fieldname) or \
+        if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \
                 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
             raise SyntaxError('Field: invalid field name: %s' % fieldname)
         self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
@@ -9537,113 +9590,61 @@ class Field(Expression):
     def count(self, distinct=None):
         return Expression(self.db, self.db._adapter.COUNT, self, distinct, 'integer')
 
-    def as_dict(self, flat=False, sanitize=True, options=True):
+    def as_dict(self, flat=False, sanitize=True):
-
+        attrs = ("name", 'authorize', 'represent', 'ondelete',
-        attrs = ('type', 'length', 'default', 'required',
+        'custom_store', 'autodelete', 'custom_retrieve',
-                 'ondelete', 'notnull', 'unique', 'uploadfield',
+        'filter_out', 'uploadseparate', 'widget', 'uploadfs',
-                 'widget', 'label', 'comment', 'writable', 'readable',
+        'update', 'custom_delete', 'uploadfield', 'uploadfolder',
-                 'update', 'authorize', 'autodelete', 'represent',
+        'custom_qualifier', 'unique', 'writable', 'compute',
-                 'uploadfolder', 'uploadseparate', 'uploadfs',
+        'map_none', 'default', 'type', 'required', 'readable',
-                 'compute', 'custom_store', 'custom_retrieve',
+        'requires', 'comment', 'label', 'length', 'notnull',
-                 'custom_retrieve_file_properties', 'custom_delete',
+        'custom_retrieve_file_properties', 'filter_in')
-                 'filter_in', 'filter_out', 'custom_qualifier',
+        serializable = (int, long, basestring, float, tuple,
-                 'map_none', 'name')
+                        bool, type(None))
-
-        SERIALIZABLE_TYPES = (int, long, basestring, dict, list,
-                              float, tuple, bool, type(None))
 
         def flatten(obj):
-            if flat:
+            if isinstance(obj, dict):
-                if isinstance(obj, flatten.__class__):
+                return dict((flatten(k), flatten(v)) for k, v in
-                    return str(type(obj))
+                             obj.items())
-                elif isinstance(obj, type):
+            elif isinstance(obj, (tuple, list, set)):
-                    try:
+                return [flatten(v) for v in obj]
-                        return str(obj).split("'")[1]
+            elif isinstance(obj, serializable):
-                    except IndexError:
+                return obj
-                        return str(obj)
+            elif isinstance(obj, (datetime.datetime,
-                elif not isinstance(obj, SERIALIZABLE_TYPES):
+                                  datetime.date, datetime.time)):
-                    return str(obj)
+                return str(obj)
-                elif isinstance(obj, dict):
+            else:
-                    newobj = dict()
-                    for k, v in obj.items():
-                        newobj[k] = flatten(v)
-                    return newobj
-                elif isinstance(obj, (list, tuple, set)):
-                    return [flatten(v) for v in obj]
-                else:
-                    return obj
-            elif isinstance(obj, (dict, set)):
-                return obj.copy()
-            else: return obj
-
-        def filter_requires(t, r, options=True):
-            if sanitize and any([keyword in str(t).upper() for
-                                 keyword in ("CRYPT", "IS_STRONG")]):
                 return None
 
-            if not isinstance(r, dict):
+        d = dict()
-                if options and hasattr(r, "options"):
+        if not (sanitize and not (self.readable or self.writable)):
-                    if callable(r.options):
+            for attr in attrs:
-                        r.options()
+               if flat:
-                newr = r.__dict__.copy()
+                   d.update({attr: flatten(getattr(self, attr))})
-            else:
+               else:
-                newr = r.copy()
+                   d.update({attr: getattr(self, attr)})
-
+            d["fieldname"] = d.pop("name")
-            # remove options if not required
-            if not options and newr.has_key("labels"):
-                [newr.update({key:None}) for key in
-                 ("labels", "theset") if (key in newr)]
-
-            for k, v in newr.items():
-                if k == "other":
-                    if isinstance(v, dict):
-                        otype, other = v.popitem()
-                    else:
-                        otype = flatten(type(v))
-                        other = v
-                    newr[k] = {otype: filter_requires(otype, other,
-                                                      options=options)}
-                else:
-                    newr[k] = flatten(v)
-            return newr
-
-        if isinstance(self.requires, (tuple, list, set)):
-            requires = dict([(flatten(type(r)),
-                             filter_requires(type(r), r,
-                                             options=options)) for
-                             r in self.requires])
-        else:
-            requires = {flatten(type(self.requires)):
-                        filter_requires(type(self.requires),
-                            self.requires, options=options)}
-
-        d = dict(colname="%s.%s" % (self.tablename, self.name),
-                 requires=requires)
-        d.update([(attr, flatten(getattr(self, attr))) for attr in attrs])
         return d
 
-    def as_xml(self, sanitize=True, options=True):
+    def as_xml(self, sanitize=True):
         if have_serializers:
             xml = serializers.xml
         else:
             raise ImportError("No xml serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         options=options)
         return xml(d)
 
-    def as_json(self, sanitize=True, options=True):
+    def as_json(self, sanitize=True):
         if have_serializers:
             json = serializers.json
         else:
             raise ImportError("No json serializers available")
-        d = self.as_dict(flat=True, sanitize=sanitize,
+        d = self.as_dict(flat=True, sanitize=sanitize)
-                         options=options)
         return json(d)
 
-    def as_yaml(self, sanitize=True, options=True):
+    def as_yaml(self, sanitize=True):
         if have_serializers:
-            d = self.as_dict(flat=True, sanitize=sanitize,
+            d = self.as_dict(flat=True, sanitize=sanitize)
-                             options=options)
             return serializers.yaml(d)
         else:
             raise ImportError("No YAML serializers available")

gluon/globals.py

@@ -310,10 +310,9 @@ class Request(Storage):
                 self.is_restful = True
                 method = _self.env.request_method
                 if len(_self.args) and '.' in _self.args[-1]:
-                    _self.args[-
+                    _self.args[-1], _, self.extension = self.args[-1].rpartition('.')
-                               1], _self.extension = _self.args[-1].rsplit('.', 1)
                     current.response.headers['Content-Type'] = \
-                        contenttype(_self.extension.lower())
+                        contenttype('.' + _self.extension.lower())
                 if not method in ['GET', 'POST', 'DELETE', 'PUT']:
                     raise HTTP(400, "invalid method")
                 rest_action = _action().get(method, None)
@@ -651,115 +650,34 @@ class Response(Storage):
 
 
 class Session(Storage):
-
     """
     defines the session object and the default values of its members (None)
+
+        response.session_storage_type   : 'file', 'db', or 'cookie'
+        response.session_cookie_compression_level :
+        response.session_cookie_expires : cookie expiration
+        response.session_cookie_key     : for encrypted sessions in cookies
+        response.session_id             : a number or None if no session
+        response.session_id_name        :
+        response.session_locked         :
+        response.session_masterapp      :
+        response.session_new            : a new session obj is being created
+
+    if session in cookie:
+
+        response.session_data_name      : name of the cookie for session data
+
+    if session in db:
+
+        response.session_db_record_id   :
+        response.session_db_table       :
+        response.session_db_unique_key  :
+
+    if session in file:
+
+        response.session_file           :
+        response.session_filename       :
     """
-    def renew(
-        self,
-        request=None,
-        response=None,
-        db=None,
-        tablename='web2py_session',
-        masterapp=None,
-        clear_session=False
-    ):
-        if request is None:
-            request = current.request
-        if response is None:
-            response = current.response
-
-        #check if session is separate
-        separate = None
-        if response.session and response.session_id[2:3] == "/":
-            separate = lambda session_name: session_name[-2:]
-
-        self._unlock(response)
-
-        if not masterapp:
-            masterapp = request.application
-
-        # Load session data from cookie
-        cookies = request.cookies
-
-        # check if there is a session_id in cookies
-        if response.session_id_name in cookies:
-            response.session_id = \
-                cookies[response.session_id_name].value
-        else:
-            response.session_id = None
-
-        # if the session goes in file
-        if response.session_storage_type == 'file':
-            if global_settings.db_sessions is True \
-                    or masterapp in global_settings.db_sessions:
-                return
-
-            client = request.client and request.client.replace(':', '.')
-
-            uuid = web2py_uuid()
-            response.session_id = '%s-%s' % (client, uuid)
-            if separate:
-                prefix = separate(response.session_id)
-                response.session_id = '%s/%s' % \
-                    (prefix, response.session_id)
-            response.session_filename = \
-                os.path.join(up(request.folder), masterapp,
-                             'sessions', response.session_id)
-            response.session_new = True
-
-        # else the session goes in db
-        elif response.session_storage_type == 'db':
-            # verify that session_id exists
-            if not response.session_id:
-                return
-
-            # verify if tablename was set or used in connect
-            if response.session_table_name and tablename == 'web2py_session':
-                tablename = response.session_table_name
-
-            if global_settings.db_sessions is not True:
-                global_settings.db_sessions.add(masterapp)
-
-            if response.session_file:
-                self._close(response)
-            if settings.global_settings.web2py_runtime_gae:
-                # in principle this could work without GAE
-                request.tickets_db = db
-
-            tname = tablename + '_' + masterapp
-
-            if not db:
-                raise Exception('No database parameter passed: "db=database"')
-
-            table = db.get(tname, None)
-            if table is None:
-                raise Exception('No session to renew')
-
-            # Get session data out of the database
-            (record_id, unique_key) = response.session_id.split(':')
-            if record_id == '0':
-                raise Exception('record_id == 0')
-            # Select from database
-            row = db(table.id == record_id).select()
-            row = row and row[0] or None
-            # Make sure the session data exists in the database
-            if not row or row.unique_key != unique_key:
-                raise Exception('No record')
-
-            unique_key = web2py_uuid()
-            db(table.id == record_id).update(unique_key=unique_key)
-            response.session_id = '%s:%s' % (record_id, unique_key)
-            response.session_db_table = table
-            response.session_db_record_id = record_id
-            response.session_db_unique_key = unique_key
-
-        rcookies = response.cookies
-        if response.session_id_name:
-            rcookies[response.session_id_name] = response.session_id
-            rcookies[response.session_id_name]['path'] = '/'
-        if clear_session:
-            self.clear()
 
     def connect(
         self,
@@ -780,109 +698,102 @@ class Session(Storage):
         and it is used to determine a session prefix.
         separate can be True and it is set to session_name[-2:]
         """
-        if request is None:
+        request = request or current.request
-            request = current.request
+        response = response or current.response
-        if response is None:
+        masterapp = masterapp or request.application
-            response = current.response
+        cookies = request.cookies
-        if separate is True:
+
-            separate = lambda session_name: session_name[-2:]
         self._unlock(response)
-        if not masterapp:
+
-            masterapp = request.application
+        response.session_masterapp = masterapp
         response.session_id_name = 'session_id_%s' % masterapp.lower()
         response.session_data_name = 'session_data_%s' % masterapp.lower()
         response.session_cookie_expires = cookie_expires
-
+        response.session_client = str(request.client).replace(':', '.')
-        # Load session data from cookie
+        response.session_cookie_key = cookie_key
-        cookies = request.cookies
+        response.session_cookie_compression_level = compression_level
 
         # check if there is a session_id in cookies
-        if response.session_id_name in cookies:
+        try:
-            response.session_id = \
+            response.session_id = cookies[response.session_id_name].value
-                cookies[response.session_id_name].value
+        except KeyError:
-        else:
             response.session_id = None
 
-        # check if there is session data in cookies
-        if response.session_data_name in cookies:
-            session_cookie_data = cookies[response.session_data_name].value
-        else:
-            session_cookie_data = None
-
         # if we are supposed to use cookie based session data
         if cookie_key:
             response.session_storage_type = 'cookie'
-            response.session_cookie_key = cookie_key
+        elif db:
-            response.session_cookie_compression_level = compression_level
+            response.session_storage_type = 'db'
+        else:
+            response.session_storage_type = 'file'
+            # why do we do this?
+            # because connect may be called twice, by web2py and in models.
+            # the first time there is no db yet so it should do nothing
+            if (global_settings.db_sessions is True or
+                masterapp in global_settings.db_sessions):
+                return
+
+        if response.session_storage_type == 'cookie':
+            # check if there is session data in cookies
+            if response.session_data_name in cookies:
+                session_cookie_data = cookies[response.session_data_name].value
+            else:
+                session_cookie_data = None
             if session_cookie_data:
                 data = secure_loads(session_cookie_data, cookie_key,
                                     compression_level=compression_level)
                 if data:
                     self.update(data)
+            response.session_id = True
+
         # else if we are supposed to use file based sessions
-        elif not db:
+        elif response.session_storage_type == 'file':
-            response.session_storage_type = 'file'
-            if global_settings.db_sessions is True \
-                    or masterapp in global_settings.db_sessions:
-                return
             response.session_new = False
-            client = request.client and request.client.replace(':', '.')
+            response.session_file = None
+            # check if the session_id points to a valid sesion filename
             if response.session_id:
-                if regex_session_id.match(response.session_id):
+                if not regex_session_id.match(response.session_id):
+                    response.session_id = None
+                else:
                     response.session_filename = \
                         os.path.join(up(request.folder), masterapp,
                                      'sessions', response.session_id)
-                else:
-                    response.session_id = None
-            # do not try load the data from file is these was data in cookie
-            if response.session_id and not session_cookie_data:
-                # os.path.exists(response.session_filename):
-                try:
-                    response.session_file = \
-                        open(response.session_filename, 'rb+')
                     try:
+                        response.session_file = \
+                            open(response.session_filename, 'rb+')
                         portalocker.lock(response.session_file,
                                          portalocker.LOCK_EX)
                         response.session_locked = True
                         self.update(cPickle.load(response.session_file))
                         response.session_file.seek(0)
-                        oc = response.session_filename.split('/')[-1]\
+                        oc = response.session_filename.split('/')[-1].split('-')[0]
-                            .split('-')[0]
+                        if check_client and response.session_client != oc:
-                        if check_client and client != oc:
                             raise Exception("cookie attack")
                     except:
                         response.session_id = None
-                    finally:
-                        pass
-                        #This causes admin login to break. Must find out why.
-                        #self._close(response)
-                except:
-                    response.session_file = None
             if not response.session_id:
                 uuid = web2py_uuid()
-                response.session_id = '%s-%s' % (client, uuid)
+                response.session_id = '%s-%s' % (response.session_client, uuid)
+                separate = separate and (lambda session_name: session_name[-2:])
                 if separate:
                     prefix = separate(response.session_id)
-                    response.session_id = '%s/%s' % \
+                    response.session_id = '%s/%s' % (prefix, response.session_id)
-                        (prefix, response.session_id)
                 response.session_filename = \
                     os.path.join(up(request.folder), masterapp,
                                  'sessions', response.session_id)
                 response.session_new = True
+
         # else the session goes in db
-        else:
+        elif response.session_storage_type == 'db':
-            response.session_storage_type = 'db'
             if global_settings.db_sessions is not True:
                 global_settings.db_sessions.add(masterapp)
+            # if had a session on file alreday, close it (yes, can happen)
             if response.session_file:
                 self._close(response)
+            # if on GAE tickets go also in DB
             if settings.global_settings.web2py_runtime_gae:
-                # in principle this could work without GAE
                 request.tickets_db = db
-            if masterapp == request.application:
+            table_migrate = (masterapp == request.application)
-                table_migrate = migrate
-            else:
-                table_migrate = False
             tname = tablename + '_' + masterapp
             table = db.get(tname, None)
             Field = db.Field
@@ -899,46 +810,111 @@ class Session(Storage):
                     migrate=table_migrate,
                 )
                 table = db[tname]  # to allow for lazy table
-            try:
-
-                # Get session data out of the database
-                (record_id, unique_key) = response.session_id.split(':')
-                if record_id == '0':
-                    raise Exception('record_id == 0')
-                        # Select from database
-                if not session_cookie_data:
-                    rows = db(table.id == record_id).select()
-                    # Make sure the session data exists in the database
-                    if len(rows) == 0 or rows[0].unique_key != unique_key:
-                        raise Exception('No record')
-                    # rows[0].update_record(locked=True)
-                    # Unpickle the data
-                    session_data = cPickle.loads(rows[0].session_data)
-                    self.update(session_data)
-            except Exception:
-                record_id = None
-                unique_key = web2py_uuid()
-                session_data = {}
-            response.session_id = '%s:%s' % (record_id, unique_key)
             response.session_db_table = table
-            response.session_db_record_id = record_id
+            if response.session_id:
-            response.session_db_unique_key = unique_key
+                # Get session data out of the database
-            # keep tablename parameter for use in session renew
+                try:
-            response.session_table_name = tablename
+                    (record_id, unique_key) = response.session_id.split(':')
+                    record_id = long(record_id)
+                except (TypeError,ValueError):
+                    record_id = None
+
+                # Select from database
+                if record_id:
+                    row = table(record_id) #,unique_key=unique_key)
+                    # Make sure the session data exists in the database
+                    if row:
+                        # rows[0].update_record(locked=True)
+                        # Unpickle the data
+                        session_data = cPickle.loads(row.session_data)
+                        self.update(session_data)
+                    else:
+                        record_id = None
+                if record_id:
+                    response.session_id = '%s:%s' % (record_id, unique_key)
+                    response.session_db_unique_key = unique_key
+                    response.session_db_record_id = record_id
+                else:
+                    response.session_id = None
+                    response.session_new = True
+
+        if self.flash:
+            (response.flash, self.flash) = (self.flash, None)
+
+    def renew(self, clear_session=False):
+
+        if clear_session:
+            self.clear()
+
+        request = current.request
+        response = current.response
+        session = response.session
+        masterapp = response.session_masterapp
+        cookies = request.cookies
+
+        if response.session_storage_type == 'cookie':
+            return
+
+        # if the session goes in file
+        if response.session_storage_type == 'file':
+            self._close(response)
+            uuid = web2py_uuid()
+            response.session_id = '%s-%s' % (response.session_client, uuid)
+            separate = (lambda s: s[-2:]) if session and response.session_id[2:3]=="/" else None
+            if separate:
+                prefix = separate(response.session_id)
+                response.session_id = '%s/%s' % \
+                    (prefix, response.session_id)
+            response.session_filename = \
+                os.path.join(up(request.folder), masterapp,
+                             'sessions', response.session_id)
+            response.session_new = True
+
+        # else the session goes in db
+        elif response.session_storage_type == 'db':
+            table = response.session_db_table
+
+            # verify that session_id exists
+            if response.session_file:
+                self._close(response)
+            if response.session_new:
+                return
+            # Get session data out of the database
+            (record_id, unique_key) = response.session_id.split(':')
+
+            if record_id.isdigit() and long(record_id)>1:
+                new_unique_key = web2py_uuid()
+                rows = db(table.id==record_id)(table.unique_key==unique_key)\
+                    .update(unique_key=new_unique_key)
+            else:
+                rows = None
+            if rows:
+                response.session_id = '%s:%s' % (record_id, unique_key)
+                response.session_db_record_id = record_id
+                response.session_db_unique_key = new_unique_key
+            else:
+                response.session_new = True
+
+    def save_session_id_cookie(self):
+        request = current.request
+        response = current.response
+        session = response.session
+        masterapp = response.session_masterapp
+        cookies = request.cookies
         rcookies = response.cookies
-        rcookies[response.session_id_name] = response.session_id
+
-        rcookies[response.session_id_name]['path'] = '/'
-        if cookie_expires:
-            rcookies[response.session_id_name][
-                'expires'] = cookie_expires.strftime(FMT)
         # if not cookie_key, but session_data_name in cookies
         # expire session_data_name from cookies
-        if session_cookie_data:
+        if response.session_data_name in cookies:
             rcookies[response.session_data_name] = 'expired'
             rcookies[response.session_data_name]['path'] = '/'
             rcookies[response.session_data_name]['expires'] = PAST
-        if self.flash:
+        if response.session_id:
-            (response.flash, self.flash) = (self.flash, None)
+            rcookies[response.session_id_name] = response.session_id
+            rcookies[response.session_id_name]['path'] = '/'
+            if response.session_cookie_expires:
+                rcookies[response.session_id_name]['expires'] = \
+                    response.session_cookie_expires.strftime(FMT)
 
     def clear(self):
         previous_session_hash = self.pop('_session_hash', None)
@@ -970,10 +946,13 @@ class Session(Storage):
         self._forget = True
 
     def _try_store_in_cookie(self, request, response):
-        if response.session_storage_type != 'cookie':
+        if self._forget or self._unchanged():
             return False
         name = response.session_data_name
-        value = secure_dumps(dict(self), response.session_cookie_key, compression_level=response.session_cookie_compression_level)
+        compression_level = response.session_cookie_compression_level
+        value = secure_dumps(dict(self),
+                             response.session_cookie_key,
+                             compression_level=compression_level)
         expires = response.session_cookie_expires
         rcookies = response.cookies
         rcookies.pop(name, None)
@@ -1000,37 +979,44 @@ class Session(Storage):
         # don't save if file-based sessions,
         # no session id, or session being forgotten
         # or no changes to session
-        if response.session_storage_type != 'db' or not response.session_id \
+
-                or self._forget or self._unchanged():
+        if not response.session_db_table or self._forget or self._unchanged():
+            if (not response.session_db_table and
+                global_settings.db_sessions is not True and
+                response.session_masterapp in global_settings.db_sessions):
+                global_settings.db_sessions.remove(response.session_masterapp)
             return False
 
         table = response.session_db_table
         record_id = response.session_db_record_id
-        unique_key = response.session_db_unique_key
+        if response.session_new:
+            unique_key = web2py_uuid()
+        else:
+            unique_key = response.session_db_unique_key
 
         dd = dict(locked=False,
-                  client_ip=request.client.replace(':', '.'),
+                  client_ip=response.session_client,
                   modified_datetime=request.now,
                   session_data=cPickle.dumps(dict(self)),
                   unique_key=unique_key)
         if record_id:
-            table._db(table.id == record_id).update(**dd)
+            table(record_id).update_record(**dd)
-        else:
+        if not record_id:
             record_id = table.insert(**dd)
+            response.session_id = '%s:%s' % (record_id, unique_key)
+            response.session_db_unique_key = unique_key
+            response.session_db_record_id = record_id
 
-        cookies, session_id_name = response.cookies, response.session_id_name
+        self.save_session_id_cookie()
-        cookies[session_id_name] = '%s:%s' % (record_id, unique_key)
-        cookies[session_id_name]['path'] = '/'
         return True
 
     def _try_store_in_cookie_or_file(self, request, response):
-        return \
+        if response.session_storage_type == 'file':
-            self._try_store_in_cookie(request, response) or \
+            return self._try_store_in_file(request, response)
-            self._try_store_in_file(request, response)
+        if response.session_storage_type == 'cookie':
+            return self._try_store_in_cookie(request, response)
 
     def _try_store_in_file(self, request, response):
-        if response.session_storage_type != 'file':
-            return False
         try:
             if not response.session_id or self._forget or self._unchanged():
                 return False
@@ -1042,12 +1028,13 @@ class Session(Storage):
                 response.session_file = open(response.session_filename, 'wb')
                 portalocker.lock(response.session_file, portalocker.LOCK_EX)
                 response.session_locked = True
-
             if response.session_file:
                 cPickle.dump(dict(self), response.session_file)
                 response.session_file.truncate()
         finally:
             self._close(response)
+
+        self.save_session_id_cookie()
         return True
 
     def _unlock(self, response):

gluon/html.py

@@ -1075,10 +1075,8 @@ class DIV(XmlComponent):
         matches = []
         # check if the component has an attribute with the same
         # value as provided
-        check = True
         tag = getattr(self, 'tag').replace('/', '')
-        if args and tag not in args:
+        check = not (args and tag not in args)
-            check = False
         for (key, value) in kargs.iteritems():
             if key not in ['first_only', 'replace', 'find_text']:
                 if isinstance(value, (str, int)):
@@ -1109,24 +1107,28 @@ class DIV(XmlComponent):
         def replace_component(i):
             if replace is None:
                 del self[i]
-            elif callable(replace):
+                return i
-                self[i] = replace(self[i])
             else:
-                self[i] = replace
+                self[i] = replace(self[i]) if callable(replace) else replace
+                return i+1
         # loop the components
         if find_text or find_components:
-            for i, c in enumerate(self.components):
+            i = 0
+            while i<len(self.components):
+                c = self[i]
+                j = i+1
                 if check and find_text and isinstance(c, str) and \
                         ((is_regex and find_text.search(c)) or (str(find_text) in c)):
-                    replace_component(i)
+                    j = replace_component(i)
-                if find_components and isinstance(c, XmlComponent):
+                elif find_components and isinstance(c, XmlComponent):
                     child_matches = c.elements(*args, **kargs)
                     if len(child_matches):
                         if not find_text and replace is not False and child_matches[0] is c:
-                            replace_component(i)
+                            j = replace_component(i)
                         if first_only:
                             return child_matches
                         matches.extend(child_matches)
+                i = j
         return matches
 
     def element(self, *args, **kargs):

gluon/http.py

@@ -44,6 +44,7 @@ defined_status = {
     416: 'REQUESTED RANGE NOT SATISFIABLE',
     417: 'EXPECTATION FAILED',
     422: 'UNPROCESSABLE ENTITY',
+    451: 'UNAVAILABLE FOR LEGAL REASONS', # http://www.451unavailable.org/
     500: 'INTERNAL SERVER ERROR',
     501: 'NOT IMPLEMENTED',
     502: 'BAD GATEWAY',

gluon/main.py

@@ -26,6 +26,7 @@ import signal
 import socket
 import random
 import urllib2
+import string
 
 
 try:
@@ -40,6 +41,7 @@ from thread import allocate_lock
 
 from fileutils import abspath, write_file
 from settings import global_settings
+from utils import web2py_uuid
 from admin import add_path_first, create_missing_folders, create_missing_app_folders
 from globals import current
 
@@ -379,7 +381,7 @@ def wsgibase(environ, responder):
                 # ##################################################
                 # access the requested application
                 # ##################################################
-                
+
                 disabled = pjoin(request.folder, 'DISABLED')
                 if not exists(request.folder):
                     if app == rwthread.routes.default_application \
@@ -396,7 +398,7 @@ def wsgibase(environ, responder):
                                    % 'invalid request',
                                    web2py_error='invalid application')
                 elif request.is_local and exists(disabled):
-                    data = dict([item.strip() for item in line.split(':',1)] 
+                    data = dict([item.strip() for item in line.split(':',1)]
                                 for line in open(disabled) if line.strip())
                     if data.get('disabled','True').lower() != 'false':
                         if 'redirect' in data:
@@ -605,7 +607,7 @@ def save_password(password, port):
 
 def appfactory(wsgiapp=wsgibase,
                logfilename='httpserver.log',
-               profilerfilename='profiler.log'):
+               profiler_dir=None):
     """
     generates a wsgi application that does logging and profiling and calls
     wsgibase
@@ -616,9 +618,22 @@ def appfactory(wsgiapp=wsgibase,
             [, profilerfilename='profiler.log']]])
 
     """
-    if profilerfilename and exists(profilerfilename):
+
-        os.unlink(profilerfilename)
+    if profiler_dir:
-    locker = allocate_lock()
+        profiler_dir = abspath(profiler_dir)
+        logger.warn('profiler is on. will use dir %s', profiler_dir)
+        if not os.path.isdir(profiler_dir):
+            try:
+                os.makedirs(profiler_dir)
+            except:
+                raise BaseException, "Can't create dir %s" % profiler_dir
+        filepath = pjoin(profiler_dir, 'wtest')
+        try:
+            filehandle = open( filepath, 'w' )
+            filehandle.close()
+            os.unlink(filepath)
+        except IOError:
+            raise BaseException, "Unable to write to dir %s" % profiler_dir
 
     def app_with_logging(environ, responder):
         """
@@ -636,25 +651,17 @@ def appfactory(wsgiapp=wsgibase,
 
         time_in = time.time()
         ret = [0]
-        if not profilerfilename:
+        if not profiler_dir:
             ret[0] = wsgiapp(environ, responder2)
         else:
             import cProfile
-            import pstats
+            prof = cProfile.Profile()
-            logger.warn('profiler is on. this makes web2py slower and serial')
+            prof.enable()
+            ret[0] = wsgiapp(environ, responder2)
+            prof.disable()
+            destfile = pjoin(profiler_dir, "req_%s.prof" % web2py_uuid())
+            prof.dump_stats(destfile)
 
-            locker.acquire()
-            cProfile.runctx('ret[0] = wsgiapp(environ, responder2)',
-                            globals(), locals(), profilerfilename + '.tmp')
-            stat = pstats.Stats(profilerfilename + '.tmp')
-            stat.stream = cStringIO.StringIO()
-            stat.strip_dirs().sort_stats("time").print_stats(80)
-            profile_out = stat.stream.getvalue()
-            profile_file = open(profilerfilename, 'a')
-            profile_file.write('%s\n%s\n%s\n%s\n\n' %
-                               ('=' * 60, environ['PATH_INFO'], '=' * 60, profile_out))
-            profile_file.close()
-            locker.release()
         try:
             line = '%s, %s, %s, %s, %s, %s, %f\n' % (
                 environ['REMOTE_ADDR'],
@@ -677,7 +684,6 @@ def appfactory(wsgiapp=wsgibase,
 
     return app_with_logging
 
-
 class HttpServer(object):
     """
     the web2py web server (Rocket)
@@ -690,7 +696,7 @@ class HttpServer(object):
         password='',
         pid_filename='httpserver.pid',
         log_filename='httpserver.log',
-        profiler_filename=None,
+        profiler_dir=None,
         ssl_certificate=None,
         ssl_private_key=None,
         ssl_ca_certificate=None,
@@ -755,7 +761,7 @@ class HttpServer(object):
             logger.info('SSL is ON')
         app_info = {'wsgi_app': appfactory(wsgibase,
                                            log_filename,
-                                           profiler_filename)}
+                                           profiler_dir)}
 
         self.server = rocket.Rocket(interfaces or tuple(sock_list),
                                     method='wsgi',

gluon/sqlhtml.py

@@ -1750,7 +1750,7 @@ class SQLFORM(FORM):
              oncreate=None,
              onupdate=None,
              ondelete=None,
-             sorter_icons=(XML('↑'), XML('↓')),
+             sorter_icons=(XML('▲'), XML('▼')),
              ui = 'web2py',
              showbuttontext=True,
              _class="web2py_grid",
@@ -2209,9 +2209,8 @@ class SQLFORM(FORM):
                     elif key == order[1:]:
                         marker = sorter_icons[1]
                 else:
-                    print 'a', key, ordermatch
                     if key == ordermatch:
-                        key, marker = '~' + order, sorter_icons[0]
+                        key, marker = '~' + ordermatch, sorter_icons[0]
                     elif key == ordermatch[1:]:
                         marker = sorter_icons[1]
                 header = A(header, marker, _href=url(vars=dict(

gluon/tests/test_dal.py

@@ -701,13 +701,12 @@ class TestDALDictImportExport(unittest.TestCase):
         assert isinstance(dbdict, dict)
         uri = dbdict["uri"]
         assert isinstance(uri, basestring) and uri
-        assert len(dbdict["items"]) == 2
+        assert len(dbdict["tables"]) == 2
-        assert len(dbdict["items"]["person"]["items"]) == 3
+        assert len(dbdict["tables"][0]["fields"]) == 3
-        assert dbdict["items"]["person"]["items"]["name"]["type"] == db.person.name.type
+        assert dbdict["tables"][0]["fields"][1]["type"] == db.person.name.type
-        assert dbdict["items"]["person"]["items"]["name"]["default"] == db.person.name.default
+        assert dbdict["tables"][0]["fields"][1]["default"] == db.person.name.default
-        assert dbdict
 
-        db2 = DAL(dbdict, check_reserved=['all'])
+        db2 = DAL(**dbdict)
         assert len(db.tables) == len(db2.tables)
         assert hasattr(db2, "pet") and isinstance(db2.pet, Table)
         assert hasattr(db2.pet, "friend") and isinstance(db2.pet.friend, Field)
@@ -725,7 +724,7 @@ class TestDALDictImportExport(unittest.TestCase):
             unicode_keys = True
             if sys.version < "2.6.5":
                 unicode_keys = False
-            db3 = DAL(serializers.loads_json(dbjson,
+            db3 = DAL(**serializers.loads_json(dbjson,
                           unicode_keys=unicode_keys))
             assert hasattr(db3, "person") and hasattr(db3.person, "uuid") and\
             db3.person.uuid.type == db.person.uuid.type
@@ -736,18 +735,19 @@ class TestDALDictImportExport(unittest.TestCase):
 
         mpfc = "Monty Python's Flying Circus"
         dbdict4 = {"uri": DEFAULT_URI,
-                   "items":{"staff":{"items": {"name":
+                   "tables":[{"tablename": "staff",
-                                                   {"default":"Michael"},
+                              "fields": [{"fieldname": "name",
-                                               "food":
+                                          "default":"Michael"},
-                                                   {"default":"Spam"},
+                                         {"fieldname": "food",
-                                               "tvshow":
+                                          "default":"Spam"},
-                                                   {"type": "reference tvshow"}
+                                         {"fieldname": "tvshow",
-                                               }},
+                                          "type": "reference tvshow"}]},
-                            "tvshow":{"items": {"name":
+                             {"tablename": "tvshow",
-                                                   {"default":mpfc},
+                              "fields": [{"fieldname": "name",
-                                              "rating":
+                                          "default":mpfc},
-                                                   {"type":"double"}}}}}
+                                         {"fieldname": "rating",
-        db4 = DAL(dbdict4, check_reserved=['all'])
+                                          "type":"double"}]}]}
+        db4 = DAL(**dbdict4)
         assert "staff" in db4.tables
         assert "name" in db4.staff
         assert db4.tvshow.rating.type == "double"
@@ -761,20 +761,19 @@ class TestDALDictImportExport(unittest.TestCase):
         db4.commit()
 
         dbdict5 = {"uri": DEFAULT_URI}
-        db5 = DAL(dbdict5, check_reserved=['all'])
+        db5 = DAL(**dbdict5)
         assert db5.tables in ([], None)
         assert not (str(db5) in ("", None))
 
         dbdict6 = {"uri": DEFAULT_URI,
-                   "items":{"staff":{},
+                   "tables":[{"tablename": "staff"},
-                            "tvshow":{"items": {"name": {},
+                             {"tablename": "tvshow",
-                                              "rating":
+                              "fields": [{"fieldname": "name"},
-                                                 {"type":"double"}
+                                         {"fieldname": "rating", "type":"double"}
-                                                 }
+                                        ]
-                                }
+                             }]
-                            }
+                  }
-                    }
+        db6 = DAL(**dbdict6)
-        db6 = DAL(dbdict6, check_reserved=['all'])
 
         assert len(db6["staff"].fields) == 1
         assert "name" in db6["tvshow"].fields
@@ -782,15 +781,12 @@ class TestDALDictImportExport(unittest.TestCase):
         assert db6.staff.insert() is not None
         assert db6(db6.staff).select().first().id == 1
 
-
         db6.staff.drop()
         db6.tvshow.drop()
         db6.commit()
 
 
-
-
-
 if __name__ == '__main__':
     unittest.main()
     tearDownModule()
+

gluon/tools.py

@@ -1868,11 +1868,7 @@ class Auth(object):
             for key, value in user.items():
                 if callable(value) or key=='password':
                     delattr(user,key)
-        sessdb = current.response.session_db_table and current.response.session_db_table._db or None
+        current.session.renew(clear_session=not self.settings.keep_session_onlogin)
-        current.session.renew(
-            clear_session=not self.settings.keep_session_onlogin,
-            db=sessdb
-            )
         current.session.auth = Storage(
             user = user,
             last_visit=current.request.now,
@@ -2304,10 +2300,7 @@ class Auth(object):
 
         current.session.auth = None
         current.session.flash = self.messages.logged_out
-        sessdb = current.response.session_db_table and current.response.session_db_table._db or None
+        current.session.renew(clear_session=not self.settings.keep_session_onlogout)
-        current.session.renew(
-            clear_session=not self.settings.keep_session_onlogout,
-            db=sessdb)
         if not next is None:
             redirect(next)
 
@@ -2642,9 +2635,10 @@ class Auth(object):
                 redirect(self.url(args=request.args))
             password = self.random_password()
             passfield = self.settings.password_field
-            d = dict(
+            d = {
-                passfield=str(table_user[passfield].validate(password)[0]),
+                passfield: str(table_user[passfield].validate(password)[0]),
-                registration_key='')
+                'registration_key': ''
+                }
             user.update_record(**d)
             if self.settings.mailer and \
                self.settings.mailer.send(to=form.vars.email,

gluon/widget.py

@@ -66,9 +66,9 @@ def run_system_tests(options):
                     coverage_config = os.environ.get(
                         "COVERAGE_PROCESS_START",
                         os.path.join('gluon', 'tests', 'coverage.ini'))
-                        
+
-                    call_args = ['coverage', 'run', '--rcfile=%s' % 
+                    call_args = ['coverage', 'run', '--rcfile=%s' %
-                                 coverage_config, 
+                                 coverage_config,
                                  '-m', 'unittest', '-v', 'gluon.tests']
                 except:
                     sys.stderr.write('Coverage was not installed, skipping\n')
@@ -159,7 +159,7 @@ def presentation(root):
         # Prevent garbage collection of img
         pnl.image = img
 
-    def add_label(text='Change Me', font_size=12, 
+    def add_label(text='Change Me', font_size=12,
                   foreground='#195866', height=1):
         return Tkinter.Label(
             master=canvas,
@@ -339,7 +339,7 @@ class web2pyDialog(object):
         if start:
             #the widget takes care of starting the scheduler
             if self.options.scheduler and self.options.with_scheduler:
-                apps = [app.strip() for app 
+                apps = [app.strip() for app
                         in self.options.scheduler.split(',')
                         if app in available_apps]
         for app in apps:
@@ -431,7 +431,7 @@ class web2pyDialog(object):
             url = self.url + arq
             self.pagesmenu.add_command(
                 label=url, command=lambda u=url: start_browser(u))
-            
+
     def quit(self, justHide=False):
         """ Finish the program execution """
         if justHide:
@@ -484,7 +484,7 @@ class web2pyDialog(object):
             return self.error('invalid port number')
 
         # Check for non default value for ssl inputs
-        if (len(self.options.ssl_certificate) > 0 or 
+        if (len(self.options.ssl_certificate) > 0 or
             len(self.options.ssl_private_key) > 0):
             proto = 'https'
         else:
@@ -503,7 +503,7 @@ class web2pyDialog(object):
                 password,
                 pid_filename=options.pid_filename,
                 log_filename=options.log_filename,
-                profiler_filename=options.profiler_filename,
+                profiler_dir=options.profiler_dir,
                 ssl_certificate=options.ssl_certificate,
                 ssl_private_key=options.ssl_private_key,
                 ssl_ca_certificate=options.ssl_ca_certificate,
@@ -865,9 +865,9 @@ def console():
 
     parser.add_option('-F',
                       '--profiler',
-                      dest='profiler_filename',
+                      dest='profiler_dir',
                       default=None,
-                      help='profiler filename')
+                      help='profiler dir')
 
     parser.add_option('-t',
                       '--taskbar',
@@ -914,7 +914,7 @@ def console():
                       dest='run_system_tests',
                       default=False,
                       help=msg)
-    
+
     msg = ('adds coverage reporting (needs --run_system_tests), '
            'python 2.7 and the coverage module installed. '
            'You can alter the default path setting the environmental '
@@ -925,7 +925,7 @@ def console():
                       dest='with_coverage',
                       default=False,
                       help=msg)
-                      
+
     if '-A' in sys.argv:
         k = sys.argv.index('-A')
     elif '--args' in sys.argv:
@@ -1117,7 +1117,8 @@ def start(cron=True):
         if not options.args is None:
             sys.argv[:] = options.args
         run(options.shell, plain=options.plain, bpython=options.bpython,
-            import_models=options.import_models, startfile=options.run, cronjob=options.cronjob)
+            import_models=options.import_models, startfile=options.run,
+            cronjob=options.cronjob)
         return
 
     # ## if -C start cron run (extcron) and exit
@@ -1267,7 +1268,7 @@ end tell
                              password=options.password,
                              pid_filename=options.pid_filename,
                              log_filename=options.log_filename,
-                             profiler_filename=options.profiler_filename,
+                             profiler_dir=options.profiler_dir,
                              ssl_certificate=options.ssl_certificate,
                              ssl_private_key=options.ssl_private_key,
                              ssl_ca_certificate=options.ssl_ca_certificate,

scgihandler.py

@@ -61,7 +61,7 @@ wsgiapp = WSGIFilterMiddleware(gluon.main.wsgibase, GzipWSGIFilter)
 if LOGGING:
     application = gluon.main.appfactory(wsgiapp=wsgiapp,
                                         logfilename='httpserver.log',
-                                        profilerfilename=None)
+                                        profiler_dir=None)
 else:
     application = wsgiapp
 

wsgihandler.py

@@ -35,7 +35,7 @@ import gluon.main
 if LOGGING:
     application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase,
                                         logfilename='httpserver.log',
-                                        profilerfilename=None)
+                                        profiler_dir=None)
 else:
     application = gluon.main.wsgibase