Merge github.com:web2py/web2py

2012-09-02 21:42:06 +02:00
parent ddb61d75ca 7b655465bb
commit 9a5e8172d6
69 changed files with 1289 additions and 701 deletions
@@ -23,6 +23,7 @@
 - Support for Google App Engine projections, thanks Christian
 - Field(... 'upload', default=path) now accepts a path to a local file as default value, if user does not upload a file. Relative path looks inside current application folder, thanks Marin
 - executesql(...,fields=,columns=) allows parsing of results in Rows, thanks Anthony
+- Rows.find(lambda row: bool(), limitby=(0,1))

 ### Auth improvements

@@ -80,6 +81,7 @@

 ### Other Improvements 

+- gluon/contrib/webclient.py makes it easy to create functional tests for app
 - DIV(..).elements(...replace=...), thanks Anthony
 - new layout based on Twitter Bootstrap
 - New generic views: generic.ics (Mac Mail Calendar) and generic.map (Google Maps)
@@ -29,7 +29,7 @@ update:
 	wget -O gluon/contrib/simplejsonrpc.py http://rad2py.googlecode.com/hg/ide2py/simplejsonrpc.py
 	echo "remember that pymysql was tweaked"
 src:
-	echo 'Version 2.00.1 ('`date +%Y-%m-%d\ %H:%M:%S`') rc4' > VERSION
+	echo 'Version 2.0.6 ('`date +%Y-%m-%d\ %H:%M:%S`') stable' > VERSION
 	### rm -f all junk files
 	make clean
 	### clean up baisc apps
@@ -127,5 +127,5 @@ push:
 tag:
 	git tag -l '$(S)'
 	hg tag -l '$(S)'
-	make commit
+	make commit S='$(S)'
 	make push
@@ -1 +1 @@
-Version 2.00.1 (2012-08-29 07:27:23) rc4
+Version 2.0.6 (2012-09-02 12:26:12) stable
@@ -51,12 +51,12 @@ def log_progress(app,mode='EDIT',filename=None,progress=0):
    progress_file = os.path.join(apath(app, r=request), 'progress.log')
    now = str(request.now)[:19]
    if not os.path.exists(progress_file):
-        open(progress_file,'w').write('[%s] START\n' % now)
+        safe_open(progress_file,'w').write('[%s] START\n' % now)
    if filename:
-        open(progress_file,'a').write('[%s] %s %s: %s\n' % (now,mode,filename,progress))
+        safe_open(progress_file,'a').write('[%s] %s %s: %s\n' % (now,mode,filename,progress))

 def safe_open(a,b):
-    if DEMO_MODE and 'w' in b:
+    if DEMO_MODE and ('w' in b or 'a' in b):
        class tmp:
            def write(self,data): pass
        return tmp()
@@ -140,8 +140,8 @@ def check_version():
        return SPAN('You should upgrade to version %s' % version_number)
    else:
        return sp_button(URL('upgrade_web2py'), T('upgrade now')) \
-          + XML(' <strong class="upgrade_version">%s</strong>' % version_number)
-
+            + XML(' <strong class="upgrade_version">%s.%s.%s</strong>' \
+                      % version_number[:3])

 def logout():
    """ Logout handler """
@@ -455,11 +455,13 @@ def delete():
 def enable():
    app = get_app()
    filename = os.path.join(apath(app, r=request),'DISABLED')
-    if os.path.exists(filename):
+    if is_gae:
+        return SPAN(T('Not supported'),_style='color:yellow')
+    elif os.path.exists(filename):
        os.unlink(filename)
        return SPAN(T('Disable'),_style='color:green')
    else:
-        open(filename,'wb').write(time.ctime())
+        safe_open(filename,'wb').write(time.ctime())
        return SPAN(T('Enable'),_style='color:red')

 def peek():
@@ -33,6 +33,7 @@
 'created by': 'created by',
 'crontab': 'crontab',
 'currently running': 'currently running',
+'currently saved or': 'currently saved or',
 'database administration': 'database administration',
 'Debug': 'Debug',
 'defines tables': 'defines tables',
@@ -43,16 +44,22 @@
 'Detailed traceback description': 'Detailed traceback description',
 'direction: ltr': 'direction: ltr',
 'Disable': 'Disable',
+'docs': 'docs',
 'download layouts': 'download layouts',
 'download plugins': 'download plugins',
 'Edit': 'Edit',
 'Edit application': 'Edit application',
+'edit views:': 'edit views:',
+'Editing file "%s"': 'Editing file "%s"',
 'Error snapshot': 'Error snapshot',
 'Error ticket': 'Error ticket',
 'Errors': 'Errors',
 'Exception instance attributes': 'Exception instance attributes',
 'exposes': 'exposes',
+'exposes:': 'exposes:',
 'extends': 'extends',
+'file does not exist': 'file does not exist',
+'file saved on %s': 'file saved on %s',
 'filter': 'filter',
 'Frames': 'Frames',
 'Get from URL:': 'Get from URL:',
@@ -63,8 +70,12 @@
 'inspect attributes': 'inspect attributes',
 'Install': 'Install',
 'Installed applications': 'Installed applications',
+'invalid password.': 'invalid password.',
+'Key bindings': 'Key bindings',
+'Language files (static strings) updated': 'Language files (static strings) updated',
 'languages': 'languages',
 'Languages': 'Languages',
+'Last saved on:': 'Last saved on:',
 'loading...': 'loading...',
 'locals': 'locals',
 'Login': 'Login',
@@ -72,12 +83,13 @@
 'Logout': 'Logout',
 'Models': 'Models',
 'models': 'models',
-'modules': 'modules',
 'Modules': 'Modules',
+'modules': 'modules',
 'New application wizard': 'New application wizard',
 'New simple application': 'New simple application',
 'Overwrite installed app': 'Overwrite installed app',
 'Pack all': 'Pack all',
+'Peeking at file': 'Peeking at file',
 'Plugins': 'Plugins',
 'plugins': 'plugins',
 'Plural-Forms:': 'Plural-Forms:',
@@ -87,11 +99,17 @@
 'Reload routes': 'Reload routes',
 'request': 'request',
 'response': 'response',
+'restore': 'restore',
+'revert': 'revert',
 'rules are not defined': 'rules are not defined',
 'rules:': 'rules:',
 "Run tests in this file (to run all files, you may also use the button labelled 'test')": "Run tests in this file (to run all files, you may also use the button labelled 'test')",
 'Running on %s': 'Running on %s',
+'Save': 'Save',
+'Save via Ajax': 'Save via Ajax',
+'Saved file hash:': 'Saved file hash:',
 'session': 'session',
+'session expired': 'session expired',
 'shell': 'shell',
 'Site': 'Site',
 'Start wizard': 'Start wizard',
@@ -108,7 +126,9 @@
 'These files are served without processing, your images go here': 'These files are served without processing, your images go here',
 'Ticket ID': 'Ticket ID',
 'Ticket Missing': 'Ticket Missing',
+'to  previous version.': 'to  previous version.',
 'To create a plugin, name a file/folder plugin_[name]': 'To create a plugin, name a file/folder plugin_[name]',
+'toggle breakpoint': 'toggle breakpoint',
 'Traceback': 'Traceback',
 'Translation strings for the application': 'Translation strings for the application',
 'Uninstall': 'Uninstall',
@@ -12,6 +12,9 @@ if request.env.web2py_runtime_gae:
    session_db = DAL('gae')
    session.connect(request, response, db=session_db)
    hosts = (http_host, )
+    is_gae = True
+else:
+    is_gae = False

 if request.env.http_x_forwarded_for or request.is_https:
    session.secure()
@@ -27,7 +30,7 @@ try:
        raise HTTP(200, T('admin disabled because no admin password'))
 except IOError:
    import gluon.fileutils
-    if request.env.web2py_runtime_gae:
+    if is_gae:
        if gluon.fileutils.check_credentials(request):
            session.authorized = True
            session.last_time = time.time()
@@ -223,16 +223,12 @@ for c in controllers: controller_functions+=[c[:-3]+'/%s.html'%x for x in functi
            {{=editpluralsfile('languages',pfile,dict(nplurals=p[0]))}}
          </span>
          <span class="file">
-            {{=peekfile('languages',pfile,dict(id=id))}},
+            {{=peekfile('languages',pfile,dict(id=id))}}
          </span>
        {{else:}}
-          <b>{{=T("are not used yet")}}</b>,
+          <b>{{=T("are not used yet")}}</b>
        {{pass}}
      {{pass}}
-      {{=T("rules:")}}
-        <span class="file{{=' error' if p[3]!='ok' else ''}}">
-          {{=peekfile('gluon/contrib/rules', p[2], dict(app=app, id=id), p[3] if p[3]!='ok' else None)}}
-        </span>
    {{pass}}
    )
    </td>
@@ -4,13 +4,119 @@
 '!langname!': 'English (US)',
 '%s %%(shop)': '%s %%(shop)',
 '%s %%(shop[0])': '%s %%(shop[0])',
+'%s %%{quark[0]}': '%s %%{quark[0]}',
 '%s %%{shop[0]}': '%s %%{shop[0]}',
 '%s %%{shop}': '%s %%{shop}',
+'%Y-%m-%d': '%Y-%m-%d',
 '%Y-%m-%d %H:%M:%S': '%Y-%m-%d %H:%M:%S',
 '@markmin\x01**Hello World**': '**Hello World**',
+'About': 'About',
+'Access Control': 'Access Control',
+'Administrative Interface': 'Administrative Interface',
+'Ajax Recipes': 'Ajax Recipes',
+'Are you sure you want to delete this object?': 'Are you sure you want to delete this object?',
+'Buy this book': 'Buy this book',
+'Cannot be empty': 'Cannot be empty',
+'Check to delete': 'Check to delete',
+'Client IP': 'Client IP',
+'Community': 'Community',
+'Components and Plugins': 'Components and Plugins',
+'Controller': 'Controller',
+'Copyright': 'Copyright',
+'Created By': 'Created By',
+'Created On': 'Created On',
+'customize me!': 'customize me!',
+'Database': 'Database',
+'DB Model': 'DB Model',
+'Demo': 'Demo',
+'Deployment Recipes': 'Deployment Recipes',
+'Description': 'Description',
+'Documentation': 'Documentation',
+"Don't know what to do?": "Don't know what to do?",
+'Download': 'Download',
+'E-mail': 'E-mail',
+'Email and SMS': 'Email and SMS',
 'enter an integer between %(min)g and %(max)g': 'enter an integer between %(min)g and %(max)g',
 'enter date and time as %(format)s': 'enter date and time as %(format)s',
+'Errors': 'Errors',
+'FAQ': 'FAQ',
+'First name': 'First name',
+'Forms and Validators': 'Forms and Validators',
+'Free Applications': 'Free Applications',
+'Group %(group_id)s created': 'Group %(group_id)s created',
+'Group ID': 'Group ID',
+'Group uniquely assigned to user %(id)s': 'Group uniquely assigned to user %(id)s',
+'Groups': 'Groups',
 'Hello World': 'Hello World',
 'Hello World  ## comment': 'Hello World  ',
 'Hello World## comment': 'Hello World',
+'Home': 'Home',
+'How did you get here?': 'How did you get here?',
+'Introduction': 'Introduction',
+'Invalid email': 'Invalid email',
+'Is Active': 'Is Active',
+'Last name': 'Last name',
+'Layout': 'Layout',
+'Layout Plugins': 'Layout Plugins',
+'Layouts': 'Layouts',
+'Live Chat': 'Live Chat',
+'Logged in': 'Logged in',
+'Logged out': 'Logged out',
+'Login': 'Login',
+'Logout': 'Logout',
+'Lost Password': 'Lost Password',
+'Lost password?': 'Lost password?',
+'Menu Model': 'Menu Model',
+'Modified By': 'Modified By',
+'Modified On': 'Modified On',
+'My Sites': 'My Sites',
+'Name': 'Name',
+'Object or table name': 'Object or table name',
+'Online examples': 'Online examples',
+'Origin': 'Origin',
+'Other Plugins': 'Other Plugins',
+'Other Recipes': 'Other Recipes',
+'Overview': 'Overview',
+'Password': 'Password',
+"Password fields don't match": "Password fields don't match",
+'please input your password again': 'please input your password again',
+'Plugins': 'Plugins',
+'Powered by': 'Powered by',
+'Preface': 'Preface',
+'Profile': 'Profile',
+'Python': 'Python',
+'Quick Examples': 'Quick Examples',
+'Recipes': 'Recipes',
+'Record ID': 'Record ID',
+'Register': 'Register',
+'Registration identifier': 'Registration identifier',
+'Registration key': 'Registration key',
+'Registration successful': 'Registration successful',
+'Remember me (for 30 days)': 'Remember me (for 30 days)',
+'Reset Password key': 'Reset Password key',
+'Role': 'Role',
+'Semantic': 'Semantic',
+'Services': 'Services',
+'Stylesheet': 'Stylesheet',
+'Support': 'Support',
+'The Core': 'The Core',
+'The output of the file is a dictionary that was rendered by the view %s': 'The output of the file is a dictionary that was rendered by the view %s',
+'The Views': 'The Views',
+'This App': 'This App',
+'Timestamp': 'Timestamp',
+'Twitter': 'Twitter',
+'User %(id)s Logged-in': 'User %(id)s Logged-in',
+'User %(id)s Logged-out': 'User %(id)s Logged-out',
+'User %(id)s Registered': 'User %(id)s Registered',
+'User ID': 'User ID',
+'value already in database or empty': 'value already in database or empty',
+'Verify Password': 'Verify Password',
+'Videos': 'Videos',
+'View': 'View',
+'Welcome': 'Welcome',
+'Welcome to web2py!': 'Welcome to web2py!',
+'Which called the function %s located in the file %s': 'Which called the function %s located in the file %s',
+'You are successfully running web2py': 'You are successfully running web2py',
+'You can modify this application and adapt it to your needs': 'You can modify this application and adapt it to your needs',
+'You visited the url %s': 'You visited the url %s',
 }
@@ -6,6 +6,7 @@
 'is': ['are'],
 'man': ['men'],
 'person': ['people'],
+'quark': ['quarks'],
 'shop': ['shops'],
 'this': ['these'],
 'was': ['were'],
@@ -40,7 +40,6 @@ __all__ = ['Cache', 'lazy_cache']

 DEFAULT_TIME_EXPIRE = 300

-
 class CacheAbstract(object):
    """
    Abstract class for cache implementations.
@@ -14,6 +14,7 @@ FOR INTERNAL USE ONLY

 import os
 import thread
+import logging
 from fileutils import read_file

 cfs = {}  # for speed-up
@@ -36,13 +37,13 @@ def getcfs(key, filename, filter=None):
    try:
        t = os.stat(filename).st_mtime
    except OSError:
-        return filter()
+        return filter() if callable(filter) else ''
    cfs_lock.acquire()
    item = cfs.get(key, None)
    cfs_lock.release()
    if item and item[0] == t:
        return item[1]
-    if not filter:
+    if not callable(filter):
        data = read_file(filename)
    else:
        data = filter()
@@ -91,6 +91,15 @@ def _TEST():
 _TEST()
 """

+CACHED_REGEXES = {}
+
+def re_compile(regex):
+    try:
+        return CACHED_REGEXES[regex]
+    except KeyError:
+        compiled_regex = CACHED_REGEXES[regex] = re.compile(regex)
+        return compiled_regex
+
 class mybuiltin(object):
    """
    NOTE could simple use a dict and populate it,
@@ -355,7 +364,11 @@ def build_environment(request, response, session, store_current=True):
    environment.update((k,getattr(v, k)) for k in v.__all__)
    if not request.env:
        request.env = Storage()
-
+    # Enable standard conditional models (i.e., /*.py, /[controller]/*.py, and
+    # /[controller]/[function]/*.py)
+    response.models_to_run = [r'^\w+\.py$', r'^%s/\w+\.py$' % request.controller,
+        r'^%s/%s/\w+\.py$' % (request.controller, request.function)]
+        
    t = environment['T'] = translator(request)
    c = environment['cache'] = Cache(request)
    if store_current:
@@ -485,9 +498,13 @@ def run_models_in(environment):
        path = pjoin(folder, 'models')
        models = listdir(path, '^\w+\.py$',0,sort=False)
        compiled=False
-    paths = (path, pjoin(path,c), pjoin(path,c,f))
+    n = len(path) + 1
    for model in models:
-        if not os.path.split(model)[0] in paths and c!='appadmin':
+        regex = environment['response'].models_to_run
+        if isinstance(regex, list):
+            regex = re_compile('|'.join(regex))
+        file = model[n:].replace(os.path.sep, '/').replace('.pyc', '.py')
+        if not regex.search(file) and c!= 'appadmin':
            continue
        elif compiled:
            code = read_pyc(model)
@@ -580,10 +597,13 @@ def run_view_in(environment):
    folder = request.folder
    path = pjoin(folder, 'compiled')
    badv = 'invalid view (%s)' % view
-    patterns = response.generic_patterns or []
-    regex = re.compile('|'.join(map(fnmatch.translate, patterns)))
-    short_action =  '%(controller)s/%(function)s.%(extension)s' % request
-    allow_generic = patterns and regex.search(short_action)
+    if response.generic_patterns:
+        patterns = response.generic_patterns
+        regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
+        short_action =  '%(controller)s/%(function)s.%(extension)s' % request
+        allow_generic = regex.search(short_action)
+    else:
+        allow_generic = False
    if not isinstance(view, str):
        ccode = parse_template(view, pjoin(folder, 'views'),
                               context=environment)
@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # created by Massimo Di Pierro
 # recreated by Vladyslav Kozlovskyy
@@ -44,8 +44,8 @@ from markmin2pdf import markmin2pdf # requires pdflatex
 print markmin2pdf(m)
 ``
 ====================
-# This is a test block with new features:
-
+# This is a test block
+  with new features:
 This is a blockquote with
 a list with tables in it:
 -----------
@@ -71,7 +71,9 @@ a list with tables in it:
 -----------:blockquoteclass[blockquoteid]

 This this a new paragraph
-with a table. Table has header, footer, sections, odd and even rows:
+with a followed table.
+Table has header, footer, sections,
+odd and even rows:
 -------------------------------
 **Title 1**|**Title 2**|**Title 3**
 ==============================
@@ -98,6 +100,8 @@ Now lists can be multilevel:
  You can continue item text on
  next strings

+. paragraph in an item
+
 ++. Ordered item 1 of sublevel 2 with
    a paragraph (paragraph can start
    with point after plus or minus
@@ -143,13 +147,13 @@ line 1
  line 2
     line 3
 ``
-++++. Yet another item with code block:
-``
++++. Yet another item with code block (we need to indent \`\` to add code block as part of item):
+ ``
  line 1
 line 2
  line 3
 ``
-This item finishes with this paragraph.
+ This item finishes with this paragraph.

 ... Item in sublevel 3 can be continued with paragraphs.

@@ -201,7 +205,7 @@ We wanted a markup language with the following requirements:
 - support table, ul, ol, code
 - support html5 video and audio elements (html serialization only)
 - can align images and resize them
- can specify class for tables and code elements
+- can specify class for tables, blockquotes and code elements
 - can add anchors
 - does not use _ for markup (since it creates odd behavior)
 - automatically links urls
@@ -420,6 +424,23 @@ generates

 (the ``!`!`...`!`!:custom`` block is rendered by the ``custom=lambda`` function passed to ``render``).

+### Line breaks
+
+``[[NEWLINE]]`` tag is used to break lines:
+``
+#### Multiline [[NEWLINE]]
+   title
+paragraph [[NEWLINE]]
+with breaks[[NEWLINE]]in it
+``
+generates:
+
+#### Multiline [[NEWLINE]]
+   title
+paragraph [[NEWLINE]]
+with breaks[[NEWLINE]]in it
+
+
 ### Html5 support

 Markmin also supports the <video> and <audio> html5 tags using the notation:
@@ -430,7 +451,7 @@ Markmin also supports the <video> and <audio> html5 tags using the notation:
 [[message [title] link video]]
 [[message [title] link audio]]
 ``
-where ``message`` will be shown in brousers without HTML5 video/audio tags support.
+where ``message`` will be shown in browsers without HTML5 video/audio tags support.

 ### Latex and other extensions

@@ -512,18 +533,18 @@ DISABLED_META = '\x08'
 LATEX = '<img src="http://chart.apis.google.com/chart?cht=tx&chl=%s" />'
 regex_URL=re.compile(r'@/(?P<a>\w*)/(?P<c>\w*)/(?P<f>\w*(\.\w+)?)(/(?P<args>[\w\.\-/]+))?')
 regex_env=re.compile(r'@\{(?P<a>[\w\-\.]+?)(\:(?P<b>.*?))?\}')
-regex_expand_meta = re.compile('('+META+'|'+DISABLED_META+')')
+regex_expand_meta = re.compile('('+META+'|'+DISABLED_META+'|````)')
 regex_dd=re.compile(r'\$\$(?P<latex>.*?)\$\$')
-regex_code = re.compile('('+META+'|'+DISABLED_META+r')|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',re.S)
+regex_code = re.compile('('+META+'|'+DISABLED_META+r'|````)|(``(?P<t>.+?)``(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[^\]]*)\])?)?)',re.S)
 regex_strong=re.compile(r'\*\*(?P<t>[^\s*]+( +[^\s*]+)*)\*\*')
 regex_del=re.compile(r'~~(?P<t>[^\s*]+( +[^\s*]+)*)~~')
 regex_em=re.compile(r"''(?P<t>[^\s']+(?: +[^\s']+)*)''")
 regex_num=re.compile(r"^\s*[+-]?((\d+(\.\d*)?)|\.\d+)([eE][+-]?[0-9]+)?\s*$")
-regex_list=re.compile('^(?:(#{1,6}|\.+ |\++ |\++\. |\-+ |\-+\. )\s*)?(.*)$')
+regex_list=re.compile('^(?:(?:(#{1,6})|(?:(\.+|\++|\-+)(\.)?))\s+)?(.*)$')
 regex_bq_headline=re.compile('^(?:(\.+|\++|\-+)(\.)?\s+)?(-{3}-*)$')
 regex_tq=re.compile('^(-{3}-*)(?::(?P<c>[a-zA-Z][_a-zA-Z\-\d]*)(?:\[(?P<p>[a-zA-Z][_a-zA-Z\-\d]*)\])?)?$')
 regex_proto = re.compile(r'(?<!["\w>/=])(?P<p>\w+):(?P<k>\w+://[\w\d\-+=?%&/:.]+)', re.M)
-regex_auto = re.compile(r'(?<!["\w>/=])(?P<k>\w+://[^\s\'\"\]\}\)]+)',re.M)
+regex_auto = re.compile(r'(?<!["\w>/=])(?P<k>\w+://[\w\d\-+_=?%&/:.]+)',re.M)
 regex_link=re.compile(r'('+LINK+r')|\[\[(?P<s>.+?)\]\]')
 regex_link_level2=re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?(?:\s+(?P<p>popup))?\s*$')
 regex_media_level2=re.compile(r'^(?P<t>\S.*?)?(?:\s+\[(?P<a>.+?)\])?(?:\s+(?P<k>\S+))?\s+(?P<p>img|IMG|left|right|center|video|audio)(?:\s+(?P<w>\d+px))?\s*$')
@@ -579,7 +600,8 @@ def render(text,
           autolinks='default',
           protolinks='default',
           class_prefix='',
-           id_prefix='markmin_'):
+           id_prefix='markmin_',
+           pretty_print=False):
    """
    Arguments:
    - text is the text to be processed
@@ -598,7 +620,7 @@ def render(text,
    - class_prefix is a prefix for ALL classes in markmin text. E.g. if class_prefix='my_'
      then for ``test``:cls class will be changed to "my_cls" (default value is '')
    - id_prefix is prefix for ALL ids in markmin text (default value is 'markmin_'). E.g.:
-        -- [[id]] will be converted to <div class="anchor" id="markmin_id"></div>
+        -- [[id]] will be converted to <a name="markmin_id"></a>
        -- [[link #id]] will be converted to <a href="#markmin_id">link</a>
        -- ``test``:cls[id] will be converted to <code class="cls" id="markmin_id">test</code>

@@ -791,9 +813,18 @@ def render(text,
    >>> render('[[id1 [span **messag** in ''markmin''] ]] ... [[**link** to id [link\\\'s title] #mark1]]')
    '<p><div class="anchor" id="markmin_id1">span <strong>messag</strong> in markmin</div> ... <a href="#markmin_mark1" title="link\\\'s title"><strong>link</strong> to id</a></p>'

+    >>> render('# Multiline[[NEWLINE]]\\n title\\nParagraph[[NEWLINE]]\\nwith breaks[[NEWLINE]]\\nin it')
+    '<h1>Multiline<br /> title</h1><p>Paragraph<br /> with breaks<br /> in it</p>'
+
+    >>> render("anchor with name 'NEWLINE': [[NEWLINE [ ] ]]")
+    '<p>anchor with name \\'NEWLINE\\': <div class="anchor" id="markmin_NEWLINE"></div></p>'
+
+    >>> render("anchor with name 'NEWLINE': [[NEWLINE [newline] ]]")
+    '<p>anchor with name \\'NEWLINE\\': <div class="anchor" id="markmin_NEWLINE">newline</div></p>'
    """
    if autolinks=="default": autolinks = autolinks_simple
    if protolinks=="default": protolinks = protolinks_simple
+    pp='\n' if pretty_print else ''
    text = str(text or '')
    text = regex_backslash.sub(lambda m: m.group(1).translate(ttab_in), text)

@@ -801,11 +832,22 @@ def render(text,
        # this is experimental @{function/args}
        # turns into a digitally signed URL
        def u1(match,URL=URL):
-            a,c,f,args = match.group('a','c','f','args')
+            a,c,f,args = match.group('a','c','f','args')                        
            return URL(a=a or None,c=c or None,f = f or None,
                       args=args.split('/'), scheme=True, host=True)
        text = regex_URL.sub(u1,text)

+    if environment:
+        def u2(match, environment=environment):
+            f = environment.get(match.group('a'), match.group(0))
+            if callable(f):
+                try:
+                    f = f(match.group('b'))
+                except Exception, e:
+                    f = 'ERROR: %s' % e
+            return str(f)
+        text = regex_env.sub(u2, text)
+
    if latex == 'google':
        text = regex_dd.sub('``\g<latex>``:latex ', text)

@@ -816,9 +858,12 @@ def render(text,
    segments = []
    def mark_code(m):
        g = m.group(0)
-        if m.group() in ( META, DISABLED_META ):
+        if g in (META, DISABLED_META ):
            segments.append((None, None, None, g))
            return m.group()
+        elif g == '````':
+            segments.append((None, None, None, ''))
+            return m.group()
        else:
            c = m.group('c') or ''
            p = m.group('p') or ''
@@ -849,13 +894,13 @@ def render(text,
    #############################################################
    # normalize spaces
    #############################################################
-    strings=[t.strip() for t in text.split('\n')]
+    strings=text.split('\n')

    def parse_title(t, s): #out, lev, etags, tag, s):
        hlevel=str(len(t))
        out.extend(etags[::-1])
        out.append("<h%s>%s"%(hlevel,s))
-        etags[:]=["</h%s>"%hlevel]
+        etags[:]=["</h%s>%s"%(hlevel,pp)]
        lev=0
        ltags[:]=[]
        tlev[:]=[]
@@ -880,8 +925,8 @@ def render(text,
                out.append(etags.pop())
                ltags.pop()
            for i in xrange(lent-lev):
-                out.append('<'+tag+'>')
-                etags.append('</'+tag+'>')
+                out.append('<'+tag+'>'+pp)
+                etags.append('</'+tag+'>'+pp)
                lev+=1
                ltags.append(lev)
                tlev.append(tag)
@@ -892,8 +937,8 @@ def render(text,
                    ltags.pop()
                    out.append(etags.pop())
                tlev[-1]=tag
-                out.append('<'+tag+'>')
-                etags.append('</'+tag+'>')
+                out.append('<'+tag+'>'+pp)
+                etags.append('</'+tag+'>'+pp)
                ltags.append(lev)
            else:
                if ltags.count(lev)>1:
@@ -901,7 +946,7 @@ def render(text,
                    ltags.pop()
        mtag='l'
        out.append('<li>')
-        etags.append('</li>')
+        etags.append('</li>'+pp)
        ltags.append(lev)
        if s[:1] == '-':
            (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
@@ -954,7 +999,7 @@ def render(text,
           return (s, mtag, lineno)

        lineno+=1
-        s = strings[lineno]
+        s = strings[lineno].strip()
        if s:
            if '|' in s:
                # table
@@ -967,7 +1012,7 @@ def render(text,

                # parse table:
                while lineno < strings_len:
-                    s = strings[lineno]
+                    s = strings[lineno].strip()
                    if s[:1] == '=':
                        if s.count('=')==len(s) and len(s)>3:  # header or footer
                            if not thead: # if thead list is empty:
@@ -994,7 +1039,7 @@ def render(text,
                                                  if regex_num.match(f)
                                                  else '',
                                               f.strip()
-                                              ) for f in s.split('|')])+'</tr>')
+                                              ) for f in s.split('|')])+'</tr>'+pp)
                    rownum+=1
                    lineno+=1

@@ -1002,15 +1047,15 @@ def render(text,
                t_id  = ' id="%s%s"'%(id_prefix, t_id) if t_id else ''
                s = ''
                if thead:
-                    s += '<thead>'+''.join([l for l in thead])+'</thead>'
+                    s += '<thead>'+pp+''.join([l for l in thead])+'</thead>'+pp
                if not tbody: # tbody strings are in tout list
                    tbody = tout
                    tout = []
                if tbody: # if tbody list is not empty:
-                    s += '<tbody>'+''.join([l for l in tbody])+'</tbody>'
+                    s += '<tbody>'+pp+''.join([l for l in tbody])+'</tbody>'+pp
                if tout: # tfoot is not empty:
-                    s += '<tfoot>'+''.join([l for l in tout])+'</tfoot>'
-                s = '<table%s%s>%s</table>' % (t_cls, t_id, s)
+                    s += '<tfoot>'+pp+''.join([l for l in tout])+'</tfoot>'+pp
+                s = '<table%s%s>%s%s</table>%s' % (t_cls, t_id, pp, s, pp)
                mtag='t'
            else:
                # parse blockquote:
@@ -1021,7 +1066,7 @@ def render(text,

                # search blockquote closing line:
                while lineno < strings_len:
-                    s = strings[lineno]
+                    s = strings[lineno].strip()
                    if not t_mode:
                        m = regex_tq.match(s)
                        if m:
@@ -1031,7 +1076,7 @@ def render(text,
                               break

                        if regex_bq_headline.match(s):
-                            if lineno+1 < strings_len and strings[lineno+1]:
+                            if lineno+1 < strings_len and strings[lineno+1].strip():
                                    t_mode = True
                            lineno+=1
                            continue
@@ -1044,7 +1089,7 @@ def render(text,

                t_cls = ' class="%s%s"'%(class_prefix,t_cls) if t_cls and t_cls != 'id' else ''
                t_id  = ' id="%s%s"'%(id_prefix,t_id) if t_id else ''
-                s = '<blockquote%s%s>%s</blockquote>' \
+                s = '<blockquote%s%s>%s</blockquote>%s' \
                         % (t_cls,
                            t_id,
                            render('\n'.join(strings[bq_begin:lineno]),
@@ -1057,7 +1102,9 @@ def render(text,
                                   autolinks,
                                   protolinks,
                                   class_prefix,
-                                   id_prefix)
+                                   id_prefix,
+                                   pretty_print),
+                            pp
                           )
                mtag='q'
        else:
@@ -1068,53 +1115,58 @@ def render(text,

    if sep == 'p':
      pbeg = "<p>"
-      pend = "</p>"
+      pend = "</p>"+pp
      br = ''
    else:
      pbeg = pend = ''
-      br = "<br />" if sep=='br' else ''
+      br = "<br />"+pp if sep=='br' else ''

-    lev = 0 # рівень вкладеності списків
-    c0 = '' # перший символ поточного рядка
-    out = [] # результуючий список рядків
-    etags = [] # завершуючі таги
-    ltags = [] # номер рівня відповідний завершуючому тагу
-    tlev = [] # таг рівня ('ul' або 'ol')
-    mtag = '' # marked tag (~last tag) ('l','.','h','p','t'). Used for set <br/>
-              # and for avoid <p></p> around tables and blockquotes
+    lev = 0    # nesting level of lists
+    c0 = ''    # first character of current line
+    out = []   # list of processed lines
+    etags = [] # trailing tags
+    ltags = [] # level# correspondent to trailing tag
+    tlev = []  # list of tags for each level ('ul' or 'ol')
+    mtag = ''  # marked tag (~last tag) ('l','.','h','p','t'). Used to set <br/>
+               # and to avoid <p></p> around tables and blockquotes
    lineno = 0
    strings_len = len(strings)
    while lineno < strings_len:
-        s = strings[lineno]
+        s0 = strings[lineno][:1]
+        s = strings[lineno].strip()
        """ #     +     -     .             ---------------------
            ##    ++    --    ..   -------  field | field | field  <-title
            ###   +++   ---   ...  quote    =====================
            ####  ++++  ----  .... -------  field | field | field  <-body
            ##### +++++ ----- .....         ---------------------:class[id]
        """
-        pc0=c0 # перший символ попереднього рядка
+        pc0=c0 # first character of previous line
        c0=s[:1]
        if c0: # for non empty strings
            if c0 in "#+-.": # first character is one of: # + - .
-                match = regex_list.search(s)
-                (t,p,s) = match.group(1), None, match.group(2)
-                t = (t or '').strip()
-                if t.endswith('.'): t, p = t[:-1], '.'
-                # t - tag ("###", "+++", "---", "...")
+                (t1,t2,p,ss) = regex_list.findall(s)[0]
+                # t1 - tag ("###")
+                # t2 - tag ("+++", "---", "...")
                # p - paragraph point ('.')->for "++." or "--."
-                # s - other part of string
-                if t:
+                # ss - other part of string
+                if t1 or t2:
                    # headers and lists:
                    if c0 == '#': # headers
-                        (lev, mtag) = parse_title(t, s)
+                        (lev, mtag) = parse_title(t1, ss)
+                        lineno+=1
+                        continue
                    elif c0 == '+': # ordered list
-                        (lev, mtag, lineno)= parse_list(t, p, s, 'ol', lev, mtag, lineno)
+                        (lev, mtag, lineno)= parse_list(t2, p, ss, 'ol', lev, mtag, lineno)
+                        lineno+=1
+                        continue
                    elif c0 == '-': # unordered list
-                        (lev, mtag, lineno) = parse_list(t, p, s, 'ul', lev, mtag, lineno)
-                    else: # c0 == '.' # paragraph in lists
-                        (lev, mtag, lineno) = parse_point(t, s, lev, mtag, lineno)
-                    lineno+=1
-                    continue
+                        (lev, mtag, lineno) = parse_list(t2, p, ss, 'ul', lev, mtag, lineno)
+                        lineno+=1
+                        continue
+                    elif lev>0: # and c0 == '.' # paragraph in lists
+                        (lev, mtag, lineno) = parse_point(t2, ss, lev, mtag, lineno)
+                        lineno+=1
+                        continue
                else:
                    if c0 == '-': # table or blockquote?
                        (s, mtag, lineno) = parse_table_or_blockquote(s, mtag, lineno)
@@ -1123,7 +1175,7 @@ def render(text,
                # new paragraph
                pc0=''

-            if pc0 == '':
+            if pc0 == '' or (mtag != 'p' and s0 not in (' ','\t')):
                # paragraph
                out.extend(etags[::-1])
                etags=[]
@@ -1171,12 +1223,12 @@ def render(text,
        style = p_begin = p_end = ''
        if p == 'center':
            p_begin = '<p style="text-align:center">'
-            p_end = '</p>'
+            p_end = '</p>'+pp
        elif p in ('left','right'):
            style = ' style="float:%s"' % p
        if p in ('video','audio'):
            t = render(t, {}, {}, 'br', URL, environment, latex,
-                       autolinks, protolinks, class_prefix, id_prefix)
+                       autolinks, protolinks, class_prefix, id_prefix, pretty_print)
            return '<%(p)s controls="controls"%(title)s%(width)s><source src="%(k)s" />%(t)s</%(p)s>' \
                    % dict(p=p, title=title, width=width, k=k, t=t)
        alt = ' alt="%s"'%escape(t).replace(META, DISABLED_META) if t else ''
@@ -1197,13 +1249,16 @@ def render(text,
            title = ' title="%s"' % a.replace(META, DISABLED_META) if a else ''
            target = ' target="_blank"' if p == 'popup' else ''
            t = render(t, {}, {}, 'br', URL, environment, latex, autolinks,
-                       protolinks, class_prefix, id_prefix) if t else k
+                       protolinks, class_prefix, id_prefix, pretty_print) if t else k
            return '<a href="%(k)s"%(title)s%(target)s>%(t)s</a>' \
                   % dict(k=k, title=title, target=target, t=t)
+        if t == 'NEWLINE' and not a:
+            return '<br />'+pp
        return '<div class="anchor" id="%s">%s</div>' % (escape(id_prefix+t),
-                                        render(a, {},{},'br', URL,
-                                               environment, latex, autolinks,
-                                               protolinks, class_prefix, id_prefix))
+                                                         render(a, {},{},'br', URL,
+                                                                environment, latex, autolinks,
+                                                                protolinks, class_prefix,
+                                                                id_prefix, pretty_print))

    parts = text.split(LINK)
    text = parts[0]
@@ -1230,9 +1285,9 @@ def render(text,
            if code[:1]=='\n': code=code[1:]
            if code[-1:]=='\n': code=code[:-1]
            if p:
-                return extra[b](code,p)
+                return str(extra[b](code,p))
            else:
-                return extra[b](code)
+                return str(extra[b](code))
        elif b=='cite':
            return '['+','.join('<a href="#%s" class="%s">%s</a>' \
                  % (d,b,d) \
@@ -1241,41 +1296,37 @@ def render(text,
            return LATEX % code.replace('"','\"').replace('\n',' ')
        elif b in html_colors:
            return '<span style="color: %s">%s</span>' \
-                  % (b, render(code,{},{},'br',URL,environment,latex,autolinks, protolinks))
+                  % (b, render(code, {}, {}, 'br', URL, environment, latex,
+                      autolinks, protolinks, class_prefix, id_prefix, pretty_print))
        elif b in ('c', 'color') and p:
             c=p.split(':')
             fg='color: %s;' % c[0] if c[0] else ''
             bg='background-color: %s;' % c[1] if len(c)>1 and c[1] else ''
             return '<span style="%s%s">%s</span>' \
-                 % (fg, bg, render(code,{},{},'br', URL, environment, latex, autolinks, protolinks))
+                 % (fg, bg, render(code, {}, {}, 'br', URL, environment, latex,
+                    autolinks, protolinks, class_prefix, id_prefix, pretty_print))
        cls = ' class="%s%s"'%(class_prefix,b) if b and b != 'id' else ''
        id  = ' id="%s%s"'%(id_prefix,escape(p)) if p else ''
        beg=(code[:1]=='\n')
        end=[None,-1][code[-1:]=='\n']
        if beg and end:
-            return '<pre><code%s%s>%s</code></pre>' % (cls, id, escape(code[1:-1]))
+            return '<pre><code%s%s>%s</code></pre>%s' % (cls, id, escape(code[1:-1]), pp)
        return '<code%s%s>%s</code>' % (cls, id, escape(code[beg:end]))
-
    text = regex_expand_meta.sub(expand_meta, text)
    text = text.translate(ttab_out)
-
-    if environment:
-        def u2(match, environment=environment):
-            f = environment.get(match.group('a'), match.group(0))
-            if callable(f):
-                try:
-                    f = f(match.group('b')) 
-                except Exception, e:
-                    f = 'ERROR: %s' % e
-            return str(f)
-        text = regex_env.sub(u2, text)
-
    return text

 def markmin2html(text, extra={}, allowed={}, sep='p', 
-                 autolinks='default',protolinks='default'):                 
+                 autolinks='default', protolinks='default',
+                 class_prefix='', id_prefix='markmin_', pretty_print=False):                 
    return render(text, extra, allowed, sep, 
-                  autolinks=autolinks, protolinks=protolinks)
+                  autolinks=autolinks, protolinks=protolinks,
+                  class_prefix=class_prefix, id_prefix=id_prefix,
+                  pretty_print=pretty_print)
+
+def run_doctests():
+    import doctest
+    doctest.testmod()

 if __name__ == '__main__':
    import sys
@@ -1312,7 +1363,9 @@ if __name__ == '__main__':
                pre { background-color: #E0E0E0; padding: 5px; }
              </style>""")[1:]

-        print html % dict(title="Markmin markup language", style=style, body=markmin2html(__doc__))
+        print html % dict(title="Markmin markup language",
+                          style=style,
+                          body=markmin2html(__doc__, pretty_print=True))
    elif sys.argv[1:2] == ['-t']:
        from timeit import Timer
        loops=1000
@@ -1338,7 +1391,8 @@ if __name__ == '__main__':
            else:
                markmin_style = ""

-            print html % dict(title=sys.argv[1], style=markmin_style, body=markmin2html(markmin_text))
+            print html % dict(title=sys.argv[1], style=markmin_style,
+                              body=markmin2html(markmin_text, pretty_print=True))
        finally:
            fargv.close()

@@ -1348,4 +1402,4 @@ if __name__ == '__main__':
        print "       -t  - timeit __doc__ (for testing purpuse only)"
        print "       file.markmin  [file.css] - process file.markmin + built in file.css (optional)"
        print "       file.markmin  [@path_to/css] - process file.markmin + link path_to/css (optional)"
-        doctest.testmod()
+        run_doctests()
@@ -0,0 +1 @@
+
@@ -0,0 +1,205 @@
+"""
+Developed by Massimo Di Pierro
+Released under the web2py license (LGPL)
+
+It an interface on top of urllib2 which simplifies scripting of http requests
+mostly for testing purposes
+
+- customizable
+- supports basic auth
+- supports cookies 
+- supports session cookies (tested with web2py sessions)
+- detects broken session
+- detects web2py form postbacks and handles formname and formkey
+- detects web2py tickets
+
+Some examples at the bottom.
+"""
+
+import re
+import time
+import urllib
+import urllib2
+
+DEFAULT_HEADERS = {
+    'user-agent': 'Mozilla/4.0', # some servers are picky
+    'accept-language': 'en',
+    }
+
+FORM_REGEX = re.compile('(\<input name\="_formkey" type\="hidden" value\="(?P<formkey>.+?)" \/\>)?\<input name\="_formname" type\="hidden" value\="(?P<formname>.+?)" \/\>')
+
+SESSION_REGEX = 'session_id_(?P<name>.+)'
+
+class WebClient(object):
+
+    def __init__(self,
+                 app = '',
+                 postbacks = True,
+                 default_headers = DEFAULT_HEADERS,
+                 session_regex = SESSION_REGEX):
+        self.app = app
+        self.postbacks = postbacks
+        self.forms = {}
+        self.history = []
+        self.cookies = {}
+        self.default_headers = default_headers
+        self.sessions = {}
+        self.session_regex = session_regex and re.compile(session_regex)
+
+    def get(self,url,cookies=None,headers=None,auth=None):
+        return self.post(url,data=None,cookies=cookies,headers=headers)
+
+    def post(self,url,data=None,cookies=None,headers=None,auth=None):
+        self.url = self.app+url
+        
+        # if this POST form requires a postback do it
+        if data and '_formname' in data and self.postbacks and \
+                self.history and self.history[-1][1]!=self.url:
+            # to bypass the web2py CSRF need to get formkey
+            # before submitting the form
+            self.get(url,cookies=cookies,headers=headers,auth=auth)
+
+        # unless cookies are specified, recycle cookies
+        if cookies is None:
+            cookies = self.cookies
+        cookies = cookies or {}
+        headers = headers or {}
+
+        # if required do basic auth
+        if auth:
+            auth_handler = urllib2.HTTPBasicAuthHandler()
+            auth_handler.add_password(**auth)
+            opener = urllib2.build_opener(auth_handler)
+        else:
+            opener = urllib2.build_opener()
+
+        # copy headers from dict to list of key,value
+        headers_list = []
+        for key,value in self.default_headers.iteritems():
+            if not key in headers:
+                headers[key] = value
+        for key,value in headers.iteritems():
+            if isinstance(value,(list,tuple)):
+                for v in value: headers_list.append((key,v))
+            else:
+                headers_list.append((key,value))
+
+        # move cookies to headers
+        for key,value in cookies.iteritems():
+            headers_list.append(('Cookie','%s=%s' % (key,value)))
+
+        # add headers to request
+        for key,value in headers_list:
+            opener.addheaders.append((key,str(value)))
+
+        # assume everything is ok and make http request
+        error = None
+        try:
+            if data is not None:
+                self.method = 'POST'
+
+                # if there is only one form, set _formname automatically
+                if not '_formname' in data and len(self.forms)==1:
+                    data['_formname'] = self.forms.keys()[0]
+
+                # if there is no formkey but it is known, set it
+                if '_formname' in data and not '_formkey' in data and \
+                        data['_formname'] in self.forms:
+                    data['_formkey'] = self.forms[data['_formname']]
+
+                # time the POST request
+                data = urllib.urlencode(data)
+                t0 = time.time()
+                self.response = opener.open(self.url,data)
+                self.time = time.time()-t0
+            else:
+                self.method = 'GET'
+
+                # time the GET request
+                t0 = time.time()
+                self.response = opener.open(self.url)
+                self.time = time.time()-t0
+        except urllib2.HTTPError, error:
+            # catch HTTP errors
+            self.time = time.time()-t0
+            self.response = error
+
+        self.status = self.response.getcode()
+        self.text = self.response.read()
+        self.headers = dict(self.response.headers)
+
+        # treat web2py tickets as special types of errors
+        if error is not None:
+            if 'web2py_error' in self.headers:
+                raise RuntimeError, self.headers['web2py_error']
+            else:
+                raise error
+
+        # parse headers into cookies
+        if 'set-cookie' in self.headers:
+            self.cookies = dict(
+                item[:item.find(';')].split('=') for item in \
+                    self.headers['set-cookie'].split(','))
+        else:
+            self.cookies = {}
+
+        # check is a new session id has been issued, symptom of broken session
+        if self.session_regex is not None:
+            for cookie, value in self.cookies.iteritems():
+                match = self.session_regex.match(cookie)
+                if match:
+                    name = match.group('name')
+                    if name in self.sessions and self.sessions[name]!=value:
+                        raise RuntimeError, 'Broken sessions %s' % name
+                    self.sessions[name] = value
+
+        # find all forms and formkeys in page
+        self.forms = {}
+        for match in FORM_REGEX.finditer(self.text):
+            self.forms[match.group('formname')] = match.group('formkey')
+
+        # log this request
+        self.history.append((self.method,self.url,self.status,self.time))
+
+def test_web2py_registration_and_login():
+    # from gluon.contrib.webclient import WebClient
+    # start a web2py instance for testing
+
+    client = WebClient('http://127.0.0.1:8000/welcome/default/')
+    client.get('index')
+
+    # register
+    data = dict(first_name = 'Homer',
+                last_name = 'Simpson',
+                email = 'homer@web2py.com',
+                password = 'test',
+                password_two = 'test',
+                _formname = 'register')
+    client.post('user/register',data = data)
+
+    # logout
+    client.get('user/logout')
+
+    # login
+    data = dict(email='homer@web2py.com',
+                password='test',
+                _formname = 'login')
+    client.post('user/login',data = data)
+
+    # check registration and login were successful
+    client.get('user/profile')
+    assert 'Welcome Homer' in client.text
+
+    # print some variables
+    print '\nsessions:\n',client.sessions
+    print '\nheaders:\n',client.headers
+    print '\ncookies:\n',client.cookies
+    print '\nforms:\n',client.forms
+    print
+    for method, url, status, t in client.history:
+        print method, url, status, t
+
+if __name__ == '__main__':
+    test_web2py_registration_and_login()
+
+
@@ -76,7 +76,8 @@ class HTTP(BaseException):
            self.headers['Set-Cookie'] = [
                str(cookie)[11:] for cookie in cookies.values()]

-    def to(self, responder):
+    def to(self, responder, env=None):
+        env = env or {}
        status = self.status
        headers = self.headers
        if status in defined_status:
@@ -100,7 +101,9 @@ class HTTP(BaseException):
            else:
                rheaders.append((k, str(v)))
        responder(status, rheaders)
-        if isinstance(body,str):
+        if env.get('request_method','')=='HEAD':
+            return ['']
+        elif isinstance(body,str):
            return [body]
        elif hasattr(body, '__iter__'):
            return body
@@ -12,13 +12,14 @@ Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine)

 import os
 import re
+import pkgutil
 from utf8 import Utf8
 from cgi import escape
 import portalocker
 import logging
 import marshal
 import copy_reg
-from fileutils import abspath, listdir
+from fileutils import listdir
 import settings
 from cfs import getcfs
 from thread import allocate_lock
@@ -31,6 +32,8 @@ __all__ = ['translator', 'findT', 'update_all_languages']
 ospath = os.path
 ostat = os.stat
 osep = os.sep
+pjoin = os.path.join
+pdirname = os.path.dirname
 isdir = os.path.isdir
 is_gae = settings.global_settings.web2py_runtime_gae

@@ -80,7 +83,6 @@ regex_backslash = re.compile(r"\\([\\{}%])")
 regex_plural = re.compile('%({.+?})')
 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)\)}$')  # %%{word(varname or number)}
 regex_plural_tuple = re.compile('^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word}
-regex_plural_rules = re.compile('^plural_rules-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')

 # UTF8 helper functions
 def upper_fun(s):
@@ -213,47 +215,30 @@ def read_possible_languages(appdir):
        langs['en'] = ('en', 'English', 0)
    return langs

-def read_global_plural_rules(filename):
-    """
-    retrieve plural rules from rules/*plural_rules-lang*.py file.
-
-    args:
-        filename (str): plural_rules filename
-
-    returns:
-        (nplurals, get_plural_id, construct_plural_form, status)
-        e.g.: (3, <function>, <function>, ok)
-    """
-    env = {}
-    data = portalocker.read_locked(filename)
-    try:
-        exec(data) in env
-        status='ok'
-    except Exception, e:
-        status='Syntax error in %s (%s)' % (filename, e)
-        logging.error(status)
-    nplurals = env.get('nplurals', DEFAULT_NPLURALS)
-    get_plural_id = env.get('get_plural_id', DEFAULT_GET_PLURAL_ID)
-    construct_plural_form = env.get('construct_plural_form',
-                                    DEFAULT_CONSTRUCTOR_PLURAL_FORM)
-    return (nplurals, get_plural_id, construct_plural_form, status)
-
-
 def read_possible_plurals():
    """
    create list of all possible plural rules files
    result is cached to increase speed
    """
-    pdir = abspath('gluon','contrib','rules')
-    plurals = {}
-    # scan rules directory for plural_rules-*.py files:
-    for pname in os.listdir(pdir):
-        if not isdir(pname) and regex_plural_rules.match(pname):
-            lang = pname[13:-3]
-            fname = ospath.join(pdir, pname)
-            n, f1, f2, status = read_global_plural_rules(fname)
-            if status == 'ok':
-                plurals[lang] = (lang, n, f1, f2, pname)
+    try:
+        import gluon.contrib.plural_rules as package
+        plurals = {}
+        for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
+            if len(modname)==2:
+                module = __import__(package.__name__+'.'+modname)
+                lang = modname
+                pname = modname+'.py'
+                nplurals = getattr(module,'nplurals', DEFAULT_NPLURALS)
+                get_plural_id = getattr(
+                    module,'get_plural_id', 
+                    DEFAULT_GET_PLURAL_ID)
+                construct_plural_form = getattr(
+                    module,'construct_plural_form',
+                    DEFAULT_CONSTRUCTOR_PLURAL_FORM)
+                plurals[lang] = (lang, nplurals, get_plural_id,
+                                 construct_plural_form, pname)
+    except ImportError:
+        logging.warn('Unable to import plural rules')
    plurals['default'] = ('default',
                          DEFAULT_NPLURALS,
                          DEFAULT_GET_PLURAL_ID,
@@ -429,7 +414,7 @@ class translator(object):
        self.request = request
        self.folder = request.folder
        self.langpath = ospath.join(self.folder,'languages')
-        self.filenames = set(os.listdir(self.langpath))        
+        self.filenames = set(os.listdir(self.langpath))
        self.http_accept_language = request.env.http_accept_language
        # self.cache                        # filled in self.force()
        # self.accepted_language = None     # filled in self.force()
@@ -442,7 +427,6 @@ class translator(object):
        # self.plural_file = None           # filled in self.force()
        # self.plural_dict = None           # filled in self.force()
        # self.plural_status = None         # filled in self.force()
-
        self.requested_languages = \
            self.force(self.http_accept_language)
        self.lazy = True
@@ -506,7 +490,7 @@ class translator(object):
        if int(n)==1:
            return word
        elif word:
-            id = min(int(n)-1,1) # self.get_plural_id(abs(int(n)))
+            id = self.get_plural_id(abs(int(n)))
            # id = 0 first plural form
            # id = 1 second plural form
            # etc.
@@ -816,8 +800,8 @@ def findT(path, language='en'):
    """
    must be run by the admin app
    """
-    filename = ospath.join(path, 'languages', language + '.py')
-    sentences = read_dict(filename)
+    lang_file = ospath.join(path, 'languages', language + '.py')
+    sentences = read_dict(lang_file)
    mp = ospath.join(path, 'models')
    cp = ospath.join(path, 'controllers')
    vp = ospath.join(path, 'views')
@@ -846,8 +830,9 @@ def findT(path, language='en'):
            'en' if language in ('default', 'en') else language)
    if not '!langname!' in sentences:
        sentences['!langname!'] = (
-            'English' if language in ('default', 'en') else sentences['!langcode!'])
-    write_dict(filename, sentences)
+            'English' if language in ('default', 'en')
+            else sentences['!langcode!'])
+    write_dict(lang_file, sentences)

 ### important to allow safe session.flash=T(....)
 def lazyT_unpickle(data):
@@ -30,7 +30,7 @@ import string
 import urllib2
 from thread import allocate_lock

-from fileutils import abspath, write_file, parse_version
+from fileutils import abspath, write_file, parse_version, copystream
 from settings import global_settings
 from admin import add_path_first, create_missing_folders, create_missing_app_folders
 from globals import current
@@ -84,7 +84,6 @@ from http import HTTP, redirect
 from globals import Request, Response, Session
 from compileapp import build_environment, run_models_in, \
    run_controller_in, run_view_in
-from fileutils import copystream, parse_version
 from contenttype import contenttype
 from dal import BaseAdapter
 from settings import global_settings
@@ -384,7 +383,7 @@ def wsgibase(environ, responder):
                # ##################################################

                eget = environ.get
-                if not eget('PATH_INFO',None) and eget('REQUEST_URI',None):
+                if not eget('PATH_INFO') and eget('REQUEST_URI'):
                    # for fcgi, get path_info and 
                    # query_string from request_uri
                    items = environ['REQUEST_URI'].split('?')
@@ -393,9 +392,14 @@ def wsgibase(environ, responder):
                        environ['QUERY_STRING'] = items[1]
                    else:
                        environ['QUERY_STRING'] = ''
-                if not eget('HTTP_HOST',None):
+                elif not eget('REQUEST_URI'):
+                    if eget('QUERY_STRING'):
+                        environ['REQUEST_URI'] = eget('PATH_INFO') + '?' + eget('QUERY_STRING')
+                    else:
+                        environ['REQUEST_URI'] = eget('PATH_INFO')
+                if not eget('HTTP_HOST'):
                    environ['HTTP_HOST'] = \
-                        eget('SERVER_NAME')+':'+eget('SERVER_PORT')
+                        eget('SERVER_NAME') + ':' + eget('SERVER_PORT')
                    

                (static_file, environ) = url_in(request, environ)
@@ -526,7 +530,8 @@ def wsgibase(environ, responder):

            except HTTP, http_response:
                if static_file:
-                    return http_response.to(responder)
+                    return http_response.to(responder,env=env)
+                        

                if request.body:
                    request.body.close()
@@ -636,7 +641,7 @@ def wsgibase(environ, responder):
        return wsgibase(new_environ,responder)
    if global_settings.web2py_crontype == 'soft':
        newcron.softcron(global_settings.applications_parent).start()
-    return http_response.to(responder)
+    return http_response.to(responder,env=env)


 def save_password(password, port):
@@ -38,6 +38,7 @@ thread = threading.local()  # thread-local storage for routing params
 regex_at = re.compile(r'(?<!\\)\$[a-zA-Z]\w*')
 regex_anything = re.compile(r'(?<!\\)\$anything')
 regex_redirect = re.compile(r'(\d+)->(.*)')
+regex_full_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)')

 def _router_default():
    "return new copy of default base router"
@@ -695,11 +696,14 @@ def regex_filter_out(url, e=None):
    return url


-def filter_url(url, method='get', remote='0.0.0.0', out=False, app=False, lang=None,
-        domain=(None,None), env=False, scheme=None, host=None, port=None):
-    "doctest/unittest interface to regex_filter_in() and regex_filter_out()"
-    regex_url = re.compile(r'^(?P<scheme>http|https|HTTP|HTTPS)\://(?P<host>[^/]*)(?P<uri>.*)')
-    match = regex_url.match(url)
+def filter_url(url, method='get', remote='0.0.0.0', 
+               out=False, app=False, lang=None,
+               domain=(None,None), env=False, scheme=None, 
+               host=None, port=None):
+    """
+    doctest/unittest interface to regex_filter_in() and regex_filter_out()
+    """
+    match = regex_full_url.match(url)
    urlscheme = match.group('scheme').lower()
    urlhost = match.group('host').lower()
    uri = match.group('uri')
@@ -994,6 +998,7 @@ class MapUrlIn(object):
            static_file = pjoin(self.request.env.applications_parent,
                                'applications', self.application,
                                'static', file)
+        self.extension = None
        log_rewrite("route: static=%s" % static_file)
        return static_file

@@ -1053,7 +1058,7 @@ class MapUrlIn(object):
        if self.map_hyphen:
            uri = uri.replace('_', '-')
            app = app.replace('_', '-')
-        if self.extension != 'html':
+        if self.extension and self.extension != 'html':
            uri += '.' + self.extension
        if self.language:
            uri = '/%s%s' % (self.language, uri)
@@ -1271,19 +1276,14 @@ def map_url_in(request, env, app=False):
    # handle mapping of lang/static to static/lang in externally-rewritten URLs
    # in case we have to handle them ourselves
    if map.languages and map.map_static is False and map.arg0 == 'static' and map.args(1) in map.languages:
-        if 'es' in map.languages:
-            print 'handle static/lang %s' % map.args(1)
        map.map_controller()
        map.map_language()
    else:
-        if 'es' in map.languages:
-            print 'NO handle static/lang %s' % map.args(1)
        map.map_language()
        map.map_controller()
    static_file = map.map_static()
-    if 'es' in map.languages:
-        print 'static_file=%s' % static_file
    if static_file:
+        map.update_request()
        return (static_file, map.env)
    map.map_function()
    map.validate_args()
@@ -88,7 +88,7 @@ except:
    from simplejson import loads, dumps


-from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB
+from gluon import DAL, Field, IS_NOT_EMPTY, IS_IN_SET, IS_NOT_IN_DB, IS_INT_IN_RANGE
 from gluon.utils import web2py_uuid


@@ -262,7 +262,8 @@ class MetaScheduler(threading.Thread):

            start = time.time()

-            while p.is_alive() and (time.time()-start < task.timeout):
+            while p.is_alive() and (
+                not task.timeout or time.time()-start < task.timeout):
                if tout:
                    try:
                        logging.debug(' partial output saved')
@@ -453,15 +454,20 @@ class Scheduler(MetaScheduler):
            Field('args','text',default='[]',requires=TYPE(list)),
            Field('vars','text',default='{}',requires=TYPE(dict)),
            Field('enabled','boolean',default=True),
-            Field('start_time','datetime',default=now),
+            Field('start_time','datetime',default=now, requires=IS_NOT_EMPTY()),
            Field('next_run_time','datetime',default=now),
            Field('stop_time','datetime'),
-            Field('repeats','integer',default=1,comment="0=unlimited"),
-            Field('retry_failed', 'integer', default=0, comment="-1=unlimited"),
-            Field('period','integer',default=60,comment='seconds'),
-            Field('timeout','integer',default=60,comment='seconds'),
+            Field('repeats','integer',default=1,comment="0=unlimited",
+                  requires=IS_INT_IN_RANGE(0, None)),
+            Field('retry_failed', 'integer', default=0, comment="-1=unlimited",
+                  requires=IS_INT_IN_RANGE(-1, None)),
+            Field('period','integer',default=60,comment='seconds',
+                  requires=IS_INT_IN_RANGE(0, None)),
+            Field('timeout','integer',default=60,comment='seconds',
+                  requires=IS_INT_IN_RANGE(0, None)),
            Field('sync_output', 'integer', default=0,
-                  comment="update output every n sec: 0=never"),
+                  comment="update output every n sec: 0=never",
+                  requires=IS_INT_IN_RANGE(0, None)),
            Field('times_run','integer',default=0,writable=False),
            Field('times_failed','integer',default=0,writable=False),
            Field('last_run_time','datetime',writable=False,readable=False),
@@ -869,4 +875,3 @@ def main():

 if __name__=='__main__':
    main()
-
@@ -1,8 +1,8 @@
 # this file exists for backward compatibility

-__all__ = ['DAL','Field','drivers']
+__all__ = ['DAL','Field','DRIVERS']

-from dal import DAL, Field, Table, Query, Set, Expression, Row, Rows, drivers, BaseAdapter, SQLField, SQLTable, SQLXorable, SQLQuery, SQLSet, SQLRows, SQLStorage, SQLDB, GQLDB, SQLALL, SQLCustomType
+from dal import DAL, Field, Table, Query, Set, Expression, Row, Rows, DRIVERS, BaseAdapter, SQLField, SQLTable, SQLXorable, SQLQuery, SQLSet, SQLRows, SQLStorage, SQLDB, GQLDB, SQLALL, SQLCustomType



@@ -24,7 +24,7 @@ from html import FORM, INPUT, LABEL, OPTION, SELECT, BUTTON
 from html import TABLE, THEAD, TBODY, TR, TD, TH, STYLE
 from html import URL, truncate_string, FIELDSET
 from dal import DAL, Field, Table, Row, CALLABLETYPES, smart_query, \
-    bar_encode, regex_table_field, Reference
+    bar_encode, Reference, REGEX_TABLE_DOT_FIELD
 from storage import Storage
 from utils import md5_hash
 from validators import IS_EMPTY_OR, IS_NOT_EMPTY, IS_LIST_OF, IS_DATE, \
@@ -1042,20 +1042,19 @@ class SQLFORM(FORM):
        # build a link
        if record and linkto:
            db = linkto.split('/')[-1]
-            for (rtable, rfield) in table._referenced_by:
+            for rfld in table._referenced_by:
                if keyed:
-                    rfld = table._db[rtable][rfield]
                    query = urllib.quote('%s.%s==%s' % (db,rfld,record[rfld.type[10:].split('.')[1]]))
                else:
-                    query = urllib.quote('%s.%s==%s' % (db,table._db[rtable][rfield],record[self.id_field_name]))
-                lname = olname = '%s.%s' % (rtable, rfield)
+                    query = urllib.quote('%s.%s==%s' % (db,rfld,record[self.id_field_name]))
+                lname = olname = '%s.%s' % (rfld.tablename, rfld.name)
                if ofields and not olname in ofields:
                    continue
                if labels and lname in labels:
                    lname = labels[lname]
                widget = A(lname,
                           _class='reference',
-                           _href='%s/%s?query=%s' % (linkto, rtable, query))
+                           _href='%s/%s?query=%s' % (linkto, rfld.tablename, query))
                xfields.append((olname.replace('.', '__')+SQLFORM.ID_ROW_SUFFIX,
                                '',widget,col3.get(olname,'')))
                self.custom.linkto[olname.replace('.', '__')] = widget
@@ -1657,7 +1656,7 @@ class SQLFORM(FORM):
        if user_signature:
            if (args != request.args and user_signature and \
                    not URL.verify(request,user_signature=user_signature)) or \
-                    (not session.auth.user and \
+                    (not (session.auth and session.auth.user) and \
                         ('edit' in request.args or \
                              'create' in request.args or \
                              'delete' in request.args)):
@@ -1827,14 +1826,15 @@ class SQLFORM(FORM):
                    try:
                        dbset = dbset(SQLFORM.build_query(
                                fields,request.vars.get('keywords','')))
-                        rows = dbset.select()
+                        rows = dbset.select(cacheable=True)
                    except Exception, e:
                        response.flash = T('Internal Error')
                        rows = []
                else:
-                    rows = dbset.select()
+                    rows = dbset.select(cacheable=True)
            else:
-                rows = dbset.select(left=left,orderby=orderby,*columns)
+                rows = dbset.select(left=left,orderby=orderby,
+                                    cacheable=True*columns)

            if export_type in exportManager:
                value = exportManager[export_type]
@@ -1893,7 +1893,8 @@ class SQLFORM(FORM):
        try:
            if left or groupby:
                c = 'count(*)'
-                nrows = dbset.select(c,left=left,groupby=groupby).first()[c]
+                nrows = dbset.select(c,left=left,cacheable=True,
+                                     groupby=groupby).first()[c]
            else:
                nrows = dbset.count()
        except:
@@ -1977,7 +1978,9 @@ class SQLFORM(FORM):

        try:
            table_fields = [f for f in fields if f._tablename in tablenames]
-            rows = dbset.select(left=left,orderby=orderby,groupby=groupby,limitby=limitby,*table_fields)
+            rows = dbset.select(left=left,orderby=orderby,
+                                groupby=groupby,limitby=limitby,
+                                cacheable=True,*table_fields)
        except SyntaxError:
            rows = None
            error = T("Query Not Supported")
@@ -2187,12 +2190,14 @@ class SQLFORM(FORM):
                        LI(A(T(db[referee]._plural),
                             _class=trap_class(),
                             _href=url()),
-                           SPAN(divider,_class='divider'),_class='w2p_grid_breadcrumb_elem'))
+                           SPAN(divider,_class='divider'),
+                           _class='w2p_grid_breadcrumb_elem'))
                    if kwargs.get('details',True):
                        breadcrumbs.append(
                            LI(A(name,_class=trap_class(),
                                 _href=url(args=['view',referee,id])),
-                               SPAN(divider,_class='divider'),_class='w2p_grid_breadcrumb_elem'))
+                               SPAN(divider,_class='divider'),
+                               _class='w2p_grid_breadcrumb_elem'))
                    nargs+=2
                else:
                    break
@@ -2218,16 +2223,18 @@ class SQLFORM(FORM):
                    del kwargs[key]
        check = {}
        id_field_name = table._id.name
-        for tablename,fieldname in table._referenced_by:
-            if db[tablename][fieldname].readable:
-                check[tablename] = check.get(tablename,[])+[fieldname]
+        for rfield in table._referenced_by:
+            if rfield.readable:
+                check[rfield.tablename] = \
+                    check.get(rfield.tablename,[])+[rfield.name]
        for tablename in sorted(check):
            linked_fieldnames = check[tablename]
            tb = db[tablename]
            multiple_links = len(linked_fieldnames)>1
            for fieldname in linked_fieldnames:
                if linked_tables is None or tablename in linked_tables:
-                    t = T(tb._plural) if not multiple_links else T(tb._plural+'('+fieldname+')')
+                    t = T(tb._plural) if not multiple_links else \
+                        T(tb._plural+'('+fieldname+')')
                    args0 = tablename+'.'+fieldname
                    links.append(
                        lambda row,t=t,nargs=nargs,args0=args0:\
@@ -2561,7 +2568,7 @@ class ExportClass(object):
        for record in self.rows:
            row = []
            for col in self.rows.colnames:
-                if not regex_table_field.match(col):
+                if not REGEX_TABLE_DOT_FIELD.match(col):
                    row.append(record._extra[col])
                else:
                    (t, f) = col.split('.')
@@ -136,7 +136,7 @@ class StorageList(Storage):
    like Storage but missing elements default to [] instead of None
    """
    def __getitem__(self,key):
-        return self.__gteattr__(key)
+        return self.__getattr__(key)
    def __getattr__(self, key):
        if key in self:
            return getattr(self,key)
@@ -177,7 +177,8 @@ class Content(BlockNode):
        if isinstance(other, (list, tuple)):
            # Must reverse so the order stays the same.
            other.reverse()
-            (self._insert(item, index) for item in other)
+            for item in other:
+                self._insert(item, index)
        else:
            self._insert(other, index)

@@ -8,3 +8,6 @@ from test_routes import *
 from test_storage import *
 from test_template import *
 from test_utils import *
+from test_contribs import *
+from test_markmin import *
+# from test_web import *
@@ -28,8 +28,10 @@ else
 	elif [ "$1" = "doctest" ]; then
 		# this has to run in gluon's parent; needs work
 		#
-		# the problem is that doctests run this way have a very different environment,
-		# apparently due to imports that don't happen in the normal course of running
+		# the problem is that doctests run this way 
+	        # have a very different environment,
+		# apparently due to imports that don't happen 
+                # in the normal course of running
 		# doctest via __main__.
 		#
 		echo doctest not supported >&2
@@ -70,3 +70,5 @@ if __name__ == '__main__':
    setUpModule()       # pre-python-2.7
    unittest.main()
    tearDownModule()
+
+
@@ -14,7 +14,7 @@ else:
 from utils import md5_hash
 import contrib.fpdf as fpdf
 import contrib.pyfpdf as pyfpdf
-    
+

 class TestContribs(unittest.TestCase):
    """ Tests the contrib package """
@@ -28,14 +28,15 @@ class TestContribs(unittest.TestCase):
        pdf = fpdf.FPDF()
        pdf.add_page()
        pdf.compress = False
-        pdf.set_font('Arial', '',14)  
+        pdf.set_font('Arial', '',14)
        pdf.ln(10)
        pdf.write(5, 'hello world')
        pdf_out = pdf.output('', 'S')
-        
+
        self.assertTrue(fpdf.FPDF_VERSION in pdf_out, 'version string')
        self.assertTrue('hello world' in pdf_out, 'sample message')


 if __name__ == '__main__':
    unittest.main()
+
@@ -176,9 +176,9 @@ class TestTable(unittest.TestCase):
        self.assertRaises(SyntaxError, Table, None, 'test', None)

        persons = Table(None, 'persons',
-                        Field('firstname','string'), 
+                        Field('firstname','string'),
                        Field('lastname', 'string'))
-        
+
        # Does it have the correct fields?

        self.assert_(set(persons.fields).issuperset(set(['firstname',
@@ -411,12 +411,14 @@ class TestMinMaxSum(unittest.TestCase):
        self.assertEqual(db.t.insert(a=3), 3)
        s = db.t.a.min()
        self.assertEqual(db(db.t.id > 0).select(s)[0]._extra[s], 1)
+        self.assertEqual(db(db.t.id > 0).select(s).first()[s], 1)
+        self.assertEqual(db().select(s).first()[s], 1)
        s = db.t.a.max()
-        self.assertEqual(db(db.t.id > 0).select(s)[0]._extra[s], 3)
+        self.assertEqual(db().select(s).first()[s], 3)
        s = db.t.a.sum()
-        self.assertEqual(db(db.t.id > 0).select(s)[0]._extra[s], 6)
+        self.assertEqual(db().select(s).first()[s], 6)
        s = db.t.a.count()
-        self.assertEqual(db(db.t.id > 0).select(s)[0]._extra[s], 3)
+        self.assertEqual(db().select(s).first()[s], 3)
        db.t.drop()


@@ -515,6 +517,20 @@ class TestVirtualFields(unittest.TestCase):
        db.t.drop()
        db.commit()

+class TestComputedFields(unittest.TestCase):
+
+    def testRun(self):
+        db = DAL('sqlite:memory:')
+        db.define_table('t',
+                        Field('a'),
+                        Field('b',default='x'),
+                        Field('c',compute=lambda r: r.a+r.b))
+        db.commit()
+        id = db.t.insert(a="z")
+        self.assertEqual(db.t[id].c,'zx')
+        db.t.drop()
+        db.commit()
+
 class TestImportExportFields(unittest.TestCase):

    def testRun(self):
@@ -533,7 +549,7 @@ class TestImportExportFields(unittest.TestCase):
        db(db.pet).delete()
        db(db.person).delete()
        stream = cStringIO.StringIO(stream.getvalue())
-        db.import_from_csv_file(stream)        
+        db.import_from_csv_file(stream)
        assert db(db.person.id==db.pet.friend)(db.person.name==db.pet.name).count()==10
        db.pet.drop()
        db.person.drop()
@@ -555,7 +571,7 @@ class TestImportExportUuidFields(unittest.TestCase):
        stream = cStringIO.StringIO()
        db.export_to_csv_file(stream)
        stream = cStringIO.StringIO(stream.getvalue())
-        db.import_from_csv_file(stream)        
+        db.import_from_csv_file(stream)
        assert db(db.person).count()==10
        assert db(db.person.id==db.pet.friend)(db.person.name==db.pet.name).count()==20
        db.pet.drop()
@@ -565,3 +581,4 @@ class TestImportExportUuidFields(unittest.TestCase):
 if __name__ == '__main__':
    unittest.main()
    tearDownModule()
+
@@ -27,7 +27,7 @@ class TestBareHelpers(unittest.TestCase):

    def testHR(self):
        self.assertEqual(HR(_a='1', _b='2').xml(), '<hr a="1" b="2" />')
-        
+
    def testIMG(self):
        self.assertEqual(IMG(_a='1', _b='2').xml(),
                         '<img a="1" b="2" />')
@@ -209,3 +209,4 @@ class TestBareHelpers(unittest.TestCase):

 if __name__ == '__main__':
    unittest.main()
+
@@ -642,3 +642,4 @@ class TestUnicode(unittest.TestCase):

 if __name__ == '__main__':
    unittest.main()
+
@@ -32,7 +32,7 @@ try:
        return True

    class TestLanguagesParallel(unittest.TestCase):
-    
+
        def setUp(self):
            self.filename = tempfile.mktemp()
            contents = dict()
@@ -46,7 +46,7 @@ try:
                os.remove(self.filename)
            except:
                pass
-    
+
        def test_reads_and_writes(self):
            readwriters = 10
            pool = multiprocessing.Pool(processes = readwriters)
@@ -56,17 +56,17 @@ try:


    class TestTranslations(unittest.TestCase):
-    
-        def setUp(self):            
+
+        def setUp(self):
            self.request = Storage()
            self.request.folder = 'applications/welcome'
            self.request.env = Storage()
            self.request.env.http_accept_language = 'en'
-            
-            
+
+
        def tearDown(self):
            pass
-    
+
        def test_plain(self):
            T = languages.translator(self.request)
            self.assertEqual(str(T('Hello World')),
@@ -81,6 +81,10 @@ try:
                             '1 shop')
            self.assertEqual(str(T('%s %%{shop[0]}', 2)),
                             '2 shops')
+            self.assertEqual(str(T('%s %%{quark[0]}', 1)),
+                             '1 quark')
+            self.assertEqual(str(T('%s %%{quark[0]}', 2)),
+                             '2 quarks')
            self.assertEqual(str(T.M('**Hello World**')),
                             '<strong>Hello World</strong>')
            T.force('it')
@@ -89,6 +93,7 @@ try:

 except ImportError:
    logging.warning("Skipped test case, no multiprocessing module.")
-        
+
 if __name__ == '__main__':
    unittest.main()
+
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Unit tests for running web2py
+"""
+import sys
+import os
+if os.path.isdir('gluon'):
+    sys.path.append(os.path.realpath('gluon'))
+else:
+    sys.path.append(os.path.realpath('../'))
+
+import unittest
+from gluon.contrib.markmin.markmin2html import run_doctests
+
+class TestMarkmin(unittest.TestCase):
+    def testMarkmin(self):
+        run_doctests()
+
+if __name__ == '__main__':
+    unittest.main()
+
@@ -54,7 +54,7 @@ def setUpModule():
        routes = open(abspath('applications', 'examples', 'routes.py'), 'w')
        routes.write("routers=dict(examples=dict(default_function='exdef'))")
        routes.close()
-        
+
        #  create language files for examples app
        for lang in ('en', 'it'):
            os.mkdir(abspath('applications', 'examples', 'static', lang))
@@ -92,11 +92,11 @@ class TestRouter(unittest.TestCase):
        self.assertRaises(SyntaxError, load, rdict=dict(BASE=dict(), app=dict(default_application="name")))
        try:
            # 2.7+ only
-            self.assertRaisesRegexp(SyntaxError, "invalid syntax", 
+            self.assertRaisesRegexp(SyntaxError, "invalid syntax",
                load, data='x:y')
-            self.assertRaisesRegexp(SyntaxError, "unknown key", 
+            self.assertRaisesRegexp(SyntaxError, "unknown key",
                load, rdict=dict(BASE=dict(badkey="value")))
-            self.assertRaisesRegexp(SyntaxError, "BASE-only key", 
+            self.assertRaisesRegexp(SyntaxError, "BASE-only key",
                load, rdict=dict(BASE=dict(), app=dict(default_application="name")))
        except AttributeError:
            pass
@@ -133,9 +133,9 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('http://domain.com/admin/default/abc', out=True), '/admin/abc')

    def test_router_specific(self):
-        """ 
-        Test app-specific routes.py 
-        
+        """
+        Test app-specific routes.py
+
        Note that make_apptree above created applications/examples/routes.py with a default_function.
        """
        load(rdict=dict())
@@ -164,7 +164,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('http://domain.com/welcome/default/index/arg1', out=True), '/index/arg1')
        self.assertEqual(filter_url('http://domain.com/welcome/default/abc', out=True), '/abc')
        self.assertEqual(filter_url('http://domain.com/welcome/default/admin', out=True), '/default/admin')
-        self.assertEqual(filter_url('http://domain.com/welcome/static/abc', out=True), 
+        self.assertEqual(filter_url('http://domain.com/welcome/static/abc', out=True),
            '/welcome/static/abc')
        self.assertEqual(filter_url('http://domain.com/welcome/appadmin/index', out=True), '/appadmin')
        self.assertEqual(filter_url('http://domain.com/welcome/appadmin/abc', out=True), '/appadmin/abc')
@@ -183,7 +183,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('http://domain.com/welcome/default/index', out=True), '/default')
        self.assertEqual(filter_url('http://domain.com/welcome/default/index/arg1', out=True), '/default/index/arg1')
        self.assertEqual(filter_url('http://domain.com/welcome/default/abc', out=True), '/default/abc')
-        self.assertEqual(filter_url('http://domain.com/welcome/static/abc', out=True), 
+        self.assertEqual(filter_url('http://domain.com/welcome/static/abc', out=True),
            '/welcome/static/abc')
        self.assertEqual(filter_url('http://domain.com/welcome/appadmin/index', out=True), '/appadmin')
        self.assertEqual(filter_url('http://domain.com/welcome/appadmin/abc', out=True), '/appadmin/abc')
@@ -386,7 +386,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('http://domain2.com/f2'), '/welcome/default/f2')
        self.assertEqual(filter_url('http://domain2.com/other/f3'), '/welcome/other/f3')

-        
+
    def test_router_domains(self):
        '''
        Test URLs that map domains
@@ -429,7 +429,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('http://domain1.com/abc.css'), '/app1/c1/abc.css')
        self.assertEqual(filter_url('http://domain1.com/index/abc'), "/app1/c1/index ['abc']")
        self.assertEqual(filter_url('http://domain2.com/app1'), "/app2a/c2a/app1")
-        
+
        self.assertEqual(filter_url('https://domain1.com/app1/ctr/fcn', domain=('app1',None), out=True), "/ctr/fcn")
        self.assertEqual(filter_url('https://www.domain1.com/app1/ctr/fcn', domain=('app1',None), out=True), "/ctr/fcn")

@@ -534,7 +534,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('https://domain.com/init/ctr/index', out=True), "/ctr")
        self.assertEqual(filter_url('http://domain.com/init/default/fcn?query', out=True), "/fcn?query")
        self.assertEqual(filter_url('http://domain.com/init/default/fcn#anchor', out=True), "/fcn#anchor")
-        self.assertEqual(filter_url('http://domain.com/init/default/fcn?query#anchor', out=True), 
+        self.assertEqual(filter_url('http://domain.com/init/default/fcn?query#anchor', out=True),
            "/fcn?query#anchor")

        router_out['BASE']['map_static'] = True
@@ -576,11 +576,11 @@ class TestRouter(unittest.TestCase):
            ),
        )
        load(rdict=router_functions)
-        
+
        # outbound
        self.assertEqual(str(URL(a='init', c='default', f='f', args=['arg1'])), "/init/f/arg1")
        self.assertEqual(str(URL(a='init', c='default', f='index', args=['arg1'])), "/init/index/arg1")
-        
+
        self.assertEqual(str(URL(a='app', c='default', f='index', args=['arg1'])), "/arg1")
        self.assertEqual(str(URL(a='app', c='default', f='user', args=['arg1'])), "/user/arg1")
        self.assertEqual(str(URL(a='app', c='default', f='user', args=['index'])), "/user/index")
@@ -631,7 +631,7 @@ class TestRouter(unittest.TestCase):
        )

        load(rdict=router_functions)
-        
+
        # outbound
        self.assertEqual(str(URL(a='init', c='default', f='index', args=['arg1'])), "/arg1")
        self.assertEqual(str(URL(a='init', c='default', f='user', args=['arg1'])), "/user/arg1")
@@ -673,18 +673,18 @@ class TestRouter(unittest.TestCase):
        )
        load(rdict=router_hyphen)
        self.assertEqual(filter_url('http://domain.com/init/default/fcn_1', out=True), "/fcn_1")
-        self.assertEqual(filter_url('http://domain.com/static/filename-with_underscore'), 
+        self.assertEqual(filter_url('http://domain.com/static/filename-with_underscore'),
            "%s/applications/init/static/filename-with_underscore" % root)
-        self.assertEqual(filter_url('http://domain.com/init/static/filename-with_underscore', out=True), 
+        self.assertEqual(filter_url('http://domain.com/init/static/filename-with_underscore', out=True),
            "/init/static/filename-with_underscore")

-        self.assertEqual(filter_url('http://domain.com/app2/fcn_1'), 
+        self.assertEqual(filter_url('http://domain.com/app2/fcn_1'),
            "/app2/default/fcn_1")
-        self.assertEqual(filter_url('http://domain.com/app2/ctr/fcn_1', domain=('app2',None), out=True), 
+        self.assertEqual(filter_url('http://domain.com/app2/ctr/fcn_1', domain=('app2',None), out=True),
            "/ctr/fcn_1")
-        self.assertEqual(filter_url('http://domain.com/app2/static/filename-with_underscore', domain=('app2',None), out=True), 
+        self.assertEqual(filter_url('http://domain.com/app2/static/filename-with_underscore', domain=('app2',None), out=True),
            "/app2/static/filename-with_underscore")
-        self.assertEqual(filter_url('http://domain.com/app2/static/filename-with_underscore'), 
+        self.assertEqual(filter_url('http://domain.com/app2/static/filename-with_underscore'),
            "%s/applications/app2/static/filename-with_underscore" % root)

        self.assertEqual(str(URL(a='init', c='default', f='a_b')), "/a_b")
@@ -731,7 +731,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('https://domain.com/admin/static/file', lang='it', out=True), "/admin/it/static/file")
        self.assertEqual(filter_url('https://domain.com/admin/static/file', lang='it-it', out=True), "/admin/it-it/static/file")
        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='it', out=True), "/welcome/ctr/fcn")
-        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='es', out=True), "/welcome/ctr/fcn")        
+        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='es', out=True), "/welcome/ctr/fcn")

        router_lang['admin']['map_static'] = True
        load(rdict=router_lang)
@@ -742,7 +742,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('https://domain.com/admin/static/file', lang='it', out=True), "/it/static/file")
        self.assertEqual(filter_url('https://domain.com/admin/static/file', lang='it-it', out=True), "/it-it/static/file")
        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='it', out=True), "/welcome/ctr/fcn")
-        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='es', out=True), "/welcome/ctr/fcn")        
+        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='es', out=True), "/welcome/ctr/fcn")

        router_lang['admin']['map_static'] = False
        router_lang['examples']['map_static'] = False
@@ -754,7 +754,7 @@ class TestRouter(unittest.TestCase):
        self.assertEqual(filter_url('https://domain.com/admin/static/file', lang='it', out=True), "/admin/static/it/file")
        self.assertEqual(filter_url('https://domain.com/admin/static/file', lang='it-it', out=True), "/admin/static/it-it/file")
        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='it', out=True), "/welcome/ctr/fcn")
-        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='es', out=True), "/welcome/ctr/fcn")        
+        self.assertEqual(filter_url('https://domain.com/welcome/ctr/fcn', lang='es', out=True), "/welcome/ctr/fcn")
        self.assertEqual(filter_url('http://domain.com/static/file'), "%s/applications/admin/static/file" % root)
        self.assertEqual(filter_url('http://domain.com/en/static/file'), "%s/applications/admin/static/file" % root)
        self.assertEqual(filter_url('http://domain.com/examples/en/static/file'), "%s/applications/examples/static/en/file" % root)
@@ -865,21 +865,21 @@ class TestRouter(unittest.TestCase):
        Test URL args parsing/generation
        '''
        load(rdict=dict())
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1'),
            "/init/default/f ['arg1']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1/'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1/'),
            "/init/default/f ['arg1']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//'),
            "/init/default/f ['arg1', '']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f//arg1'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f//arg1'),
            "/init/default/f ['', 'arg1']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1/arg2'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1/arg2'),
            "/init/default/f ['arg1', 'arg2']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//arg2'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//arg2'),
            "/init/default/f ['arg1', '', 'arg2']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//arg3/'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//arg3/'),
            "/init/default/f ['arg1', '', 'arg3']")
-        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//arg3//'), 
+        self.assertEqual(filter_url('http://domain.com/init/default/f/arg1//arg3//'),
            "/init/default/f ['arg1', '', 'arg3', '']")

        self.assertEqual(filter_url('http://domain.com/init/default/f', out=True), "/f")
@@ -903,12 +903,12 @@ class TestRouter(unittest.TestCase):
        load(rdict=dict())
        self.assertEqual(str(URL(a='a', c='c', f='f', anchor='anchor')), "/a/c/f#anchor")
        args = ['a1', 'a2']
-        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, anchor='anchor')), 
+        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, anchor='anchor')),
            "/a/c/f/a1/a2#anchor")
        vars = dict(v1=1, v2=2)
-        self.assertEqual(str(URL(a='a', c='c', f='f', vars=vars, anchor='anchor')), 
+        self.assertEqual(str(URL(a='a', c='c', f='f', vars=vars, anchor='anchor')),
            "/a/c/f?v1=1&v2=2#anchor")
-        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, vars=vars, anchor='anchor')), 
+        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, vars=vars, anchor='anchor')),
            "/a/c/f/a1/a2?v1=1&v2=2#anchor")
        self.assertEqual(str(URL(a='init', c='default', f='index')),
            "/")
@@ -938,11 +938,11 @@ class TestRouter(unittest.TestCase):
            ),
        )
        load(rdict=router_path_prefix)
-        self.assertEqual(str(URL(a='a1', c='c1a', f='f')), 
+        self.assertEqual(str(URL(a='a1', c='c1a', f='f')),
            "/path/to/apps/c1a/f")
-        self.assertEqual(str(URL(a='a2', c='c', f='f')), 
+        self.assertEqual(str(URL(a='a2', c='c', f='f')),
            "/path/to/apps/a2/c/f")
-        self.assertEqual(str(URL(a='a2', c='c2', f='f')), 
+        self.assertEqual(str(URL(a='a2', c='c2', f='f')),
            "/path/to/apps/a2/c2/f")
        self.assertEqual(filter_url('http://domain.com/a1/'), "/a1/default/index")
        self.assertEqual(filter_url('http://domain.com/path/to/apps/a1/'), "/a1/default/index")
@@ -958,35 +958,35 @@ class TestRouter(unittest.TestCase):
        r.env.http_host = 'domain.com'
        r.env.wsgi_url_scheme = 'httpx' # distinguish incoming scheme
        self.assertEqual(str(URL(r=r, a='a', c='c', f='f')), "/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com')),
            "httpx://host.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False)),
            "/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https')),
            "https://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss')),
            "wss://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https', host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https', host=True)),
            "https://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host='host.com')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host='host.com')),
            "httpx://host.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host='host.com')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host='host.com')),
            "httpx://host.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', port=1234)),
            "httpx://domain.com:1234/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, port=1234)),
            "httpx://domain.com:1234/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com', port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com', port=1234)),
            "httpx://host.com:1234/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss', host='host.com', port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss', host='host.com', port=1234)),
            "wss://host.com:1234/a/c/f")

    def test_request_uri(self):
@@ -995,15 +995,15 @@ class TestRouter(unittest.TestCase):
        '''
        load(rdict=dict())

-        self.assertEqual(filter_url('http://domain.com/abc', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/abc', env=True).request_uri,
            '/init/default/abc')
-        self.assertEqual(filter_url('http://domain.com/abc?def', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/abc?def', env=True).request_uri,
            '/init/default/abc?def')
-        self.assertEqual(filter_url('http://domain.com/index/abc', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/index/abc', env=True).request_uri,
            "/init/default/index/abc")
-        self.assertEqual(filter_url('http://domain.com/abc/def', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/abc/def', env=True).request_uri,
            "/init/default/abc/def")
-        self.assertEqual(filter_url('http://domain.com/index/a%20bc', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/index/a%20bc', env=True).request_uri,
            "/init/default/index/a%20bc")

    def test_request_collide(self):
@@ -1022,7 +1022,7 @@ class TestRouter(unittest.TestCase):
            ),
        )
        load(rdict=router_collide)
-        
+
        # basic inbound
        self.assertEqual(filter_url('http://ex.domain.com'), '/examples/default/exdef')
        self.assertEqual(filter_url('http://ad.domain.com'), '/admin/default/index')
@@ -1053,3 +1053,4 @@ if __name__ == '__main__':
    setUpModule()       # pre-2.7
    unittest.main()
    tearDownModule()
+
@@ -112,9 +112,9 @@ routes_in = (
        self.assertEqual(filter_url('http://localhost:8000/service/person/create?var1=val1'), "/app/default/call ['json', 'create'] ?model=person&var1=val1")

    def test_routes_specific(self):
-        """ 
-        Test app-specific routes.py 
-        
+        """
+        Test app-specific routes.py
+
        Note that make_apptree above created applications/examples/routes.py with a default_function.
        """
        data = r'''
@@ -190,59 +190,59 @@ default_application = 'defapp'
        Test URL args parsing/generation
        '''
        data = r'''routes_in = [
-    ('/robots.txt', '/welcome/static/robots.txt'), 
-    ('/favicon.ico', '/welcome/static/favicon.ico'), 
-    ('/admin$anything', '/admin$anything'), 
-    ('.*:https?://(.*\\.)?domain1.com:$method /', '/app1/default'), 
-    ('.*:https?://(.*\\.)?domain1.com:$method /static/$anything', '/app1/static/$anything'), 
-    ('.*:https?://(.*\\.)?domain1.com:$method /appadmin/$anything', '/app1/appadmin/$anything'), 
-    ('.*:https?://(.*\\.)?domain1.com:$method /$anything', '/app1/default/$anything'), 
-    ('.*:https?://(.*\\.)?domain2.com:$method /', '/app2/default'), 
-    ('.*:https?://(.*\\.)?domain2.com:$method /static/$anything', '/app2/static/$anything'), 
-    ('.*:https?://(.*\\.)?domain2.com:$method /appadmin/$anything', '/app2/appadmin/$anything'), 
-    ('.*:https?://(.*\\.)?domain2.com:$method /$anything', '/app2/default/$anything'), 
-    ('.*:https?://(.*\\.)?domain3.com:$method /', '/app3/defcon3'), 
-    ('.*:https?://(.*\\.)?domain3.com:$method /static/$anything', '/app3/static/$anything'), 
-    ('.*:https?://(.*\\.)?domain3.com:$method /appadmin/$anything', '/app3/appadmin/$anything'), 
+    ('/robots.txt', '/welcome/static/robots.txt'),
+    ('/favicon.ico', '/welcome/static/favicon.ico'),
+    ('/admin$anything', '/admin$anything'),
+    ('.*:https?://(.*\\.)?domain1.com:$method /', '/app1/default'),
+    ('.*:https?://(.*\\.)?domain1.com:$method /static/$anything', '/app1/static/$anything'),
+    ('.*:https?://(.*\\.)?domain1.com:$method /appadmin/$anything', '/app1/appadmin/$anything'),
+    ('.*:https?://(.*\\.)?domain1.com:$method /$anything', '/app1/default/$anything'),
+    ('.*:https?://(.*\\.)?domain2.com:$method /', '/app2/default'),
+    ('.*:https?://(.*\\.)?domain2.com:$method /static/$anything', '/app2/static/$anything'),
+    ('.*:https?://(.*\\.)?domain2.com:$method /appadmin/$anything', '/app2/appadmin/$anything'),
+    ('.*:https?://(.*\\.)?domain2.com:$method /$anything', '/app2/default/$anything'),
+    ('.*:https?://(.*\\.)?domain3.com:$method /', '/app3/defcon3'),
+    ('.*:https?://(.*\\.)?domain3.com:$method /static/$anything', '/app3/static/$anything'),
+    ('.*:https?://(.*\\.)?domain3.com:$method /appadmin/$anything', '/app3/appadmin/$anything'),
    ('.*:https?://(.*\\.)?domain3.com:$method /$anything', '/app3/defcon3/$anything'),
-    ('/', '/welcome/default'), 
-    ('/welcome/default/$anything', '/welcome/default/$anything'), 
-    ('/welcome/$anything', '/welcome/default/$anything'), 
-    ('/static/$anything', '/welcome/static/$anything'), 
-    ('/appadmin/$anything', '/welcome/appadmin/$anything'), 
-    ('/$anything', '/welcome/default/$anything'), 
+    ('/', '/welcome/default'),
+    ('/welcome/default/$anything', '/welcome/default/$anything'),
+    ('/welcome/$anything', '/welcome/default/$anything'),
+    ('/static/$anything', '/welcome/static/$anything'),
+    ('/appadmin/$anything', '/welcome/appadmin/$anything'),
+    ('/$anything', '/welcome/default/$anything'),
    ]
 routes_out = [
-    ('/welcome/static/$anything', '/static/$anything'), 
-    ('/welcome/appadmin/$anything', '/appadmin/$anything'), 
-    ('/welcome/default/$anything', '/$anything'), 
-    ('/app1/static/$anything', '/static/$anything'), 
-    ('/app1/appadmin/$anything', '/appadmin/$anything'), 
-    ('/app1/default/$anything', '/$anything'), 
-    ('/app2/static/$anything', '/static/$anything'), 
-    ('/app2/appadmin/$anything', '/appadmin/$anything'), 
-    ('/app2/default/$anything', '/$anything'), 
-    ('/app3/static/$anything', '/static/$anything'), 
-    ('/app3/appadmin/$anything', '/appadmin/$anything'), 
+    ('/welcome/static/$anything', '/static/$anything'),
+    ('/welcome/appadmin/$anything', '/appadmin/$anything'),
+    ('/welcome/default/$anything', '/$anything'),
+    ('/app1/static/$anything', '/static/$anything'),
+    ('/app1/appadmin/$anything', '/appadmin/$anything'),
+    ('/app1/default/$anything', '/$anything'),
+    ('/app2/static/$anything', '/static/$anything'),
+    ('/app2/appadmin/$anything', '/appadmin/$anything'),
+    ('/app2/default/$anything', '/$anything'),
+    ('/app3/static/$anything', '/static/$anything'),
+    ('/app3/appadmin/$anything', '/appadmin/$anything'),
    ('/app3/defcon3/$anything', '/$anything')
    ]
 '''
        load(data=data)
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1'),
            "/welcome/default/f ['arg1']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1/'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1/'),
            "/welcome/default/f ['arg1']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//'),
            "/welcome/default/f ['arg1', '']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f//arg1'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f//arg1'),
            "/welcome/default/f ['', 'arg1']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1/arg2'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1/arg2'),
            "/welcome/default/f ['arg1', 'arg2']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//arg2'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//arg2'),
            "/welcome/default/f ['arg1', '', 'arg2']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//arg3/'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//arg3/'),
            "/welcome/default/f ['arg1', '', 'arg3']")
-        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//arg3//'), 
+        self.assertEqual(filter_url('http://domain.com/welcome/default/f/arg1//arg3//'),
            "/welcome/default/f ['arg1', '', 'arg3', '']")

        self.assertEqual(filter_url('http://domain.com/welcome/default/f', out=True), "/f")
@@ -263,16 +263,16 @@ routes_out = [
        load(data='')
        self.assertEqual(str(URL(a='a', c='c', f='f', anchor='anchor')), "/a/c/f#anchor")
        args = ['a1', 'a2']
-        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, anchor='anchor')), 
+        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, anchor='anchor')),
            "/a/c/f/a1/a2#anchor")
        vars = dict(v1=1, v2=2)
-        self.assertEqual(str(URL(a='a', c='c', f='f', vars=vars, anchor='anchor')), 
+        self.assertEqual(str(URL(a='a', c='c', f='f', vars=vars, anchor='anchor')),
            "/a/c/f?v1=1&v2=2#anchor")
-        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, vars=vars, anchor='anchor')), 
+        self.assertEqual(str(URL(a='a', c='c', f='f', args=args, vars=vars, anchor='anchor')),
            "/a/c/f/a1/a2?v1=1&v2=2#anchor")

        data = r'''routes_out = [
-            ('/init/default/index', '/'), 
+            ('/init/default/index', '/'),
        ]'''
        load(data=data)
        self.assertEqual(str(URL(a='init', c='default', f='index')),
@@ -281,7 +281,7 @@ routes_out = [
            "/init/default/index#anchor")

        data = r'''routes_out = [
-            (r'/init/default/index(?P<anchor>(#.*)?)', r'/\g<anchor>'), 
+            (r'/init/default/index(?P<anchor>(#.*)?)', r'/\g<anchor>'),
        ]'''
        load(data=data)
        self.assertEqual(str(URL(a='init', c='default', f='index')),
@@ -290,7 +290,7 @@ routes_out = [
            "/#anchor")

        data = r'''routes_out = [
-            (r'/init/default/index(?P<qa>([?#].*)?)', r'/\g<qa>'), 
+            (r'/init/default/index(?P<qa>([?#].*)?)', r'/\g<qa>'),
        ]'''
        load(data=data)
        self.assertEqual(str(URL(a='init', c='default', f='index')),
@@ -313,35 +313,35 @@ routes_out = [
        r.env.http_host = 'domain.com'
        r.env.wsgi_url_scheme = 'httpx' # distinguish incoming scheme
        self.assertEqual(str(URL(r=r, a='a', c='c', f='f')), "/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com')),
            "httpx://host.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False)),
            "/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https')),
            "https://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss')),
            "wss://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https', host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='https', host=True)),
            "https://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host=True)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host=True)),
            "httpx://domain.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host='host.com')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, host='host.com')),
            "httpx://host.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host='host.com')), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=False, host='host.com')),
            "httpx://host.com/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', port=1234)),
            "httpx://domain.com:1234/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme=True, port=1234)),
            "httpx://domain.com:1234/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com', port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', host='host.com', port=1234)),
            "httpx://host.com:1234/a/c/f")
-        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss', host='host.com', port=1234)), 
+        self.assertEqual(str(URL(r=r, a='a', c='c', f='f', scheme='wss', host='host.com', port=1234)),
            "wss://host.com:1234/a/c/f")

    def test_request_uri(self):
@@ -349,18 +349,18 @@ routes_out = [
        Test REQUEST_URI in env
        '''
        data = r'''routes_in = [
-    ('/abc', '/init/default/abc'), 
-    ('/index/$anything', '/init/default/index/$anything'), 
+    ('/abc', '/init/default/abc'),
+    ('/index/$anything', '/init/default/index/$anything'),
    ]
 '''
        load(data=data)
-        self.assertEqual(filter_url('http://domain.com/abc', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/abc', env=True).request_uri,
            '/init/default/abc')
-        self.assertEqual(filter_url('http://domain.com/abc?def', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/abc?def', env=True).request_uri,
            '/init/default/abc?def')
-        self.assertEqual(filter_url('http://domain.com/index/abc', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/index/abc', env=True).request_uri,
            "/init/default/index/abc")
-        self.assertEqual(filter_url('http://domain.com/index/a%20bc', env=True).request_uri, 
+        self.assertEqual(filter_url('http://domain.com/index/a%20bc', env=True).request_uri,
            "/init/default/index/a bc")


@@ -368,3 +368,4 @@ if __name__ == '__main__':
    setUpModule()       # pre-2.7
    unittest.main()
    tearDownModule()
+
@@ -21,38 +21,38 @@ class TestStorage(unittest.TestCase):
        """ Tests Storage attribute handling """

        s = Storage(a=1)
-        
+
        self.assertEqual(s.a, 1)
        self.assertEqual(s['a'], 1)
        self.assertEqual(s.b, None)
-        
+
        s.b = 2
        self.assertEqual(s.a, 1)
        self.assertEqual(s['a'], 1)
        self.assertEqual(s.b, 2)
        self.assertEqual(s['b'], 2)
-        
+
        s['c'] = 3
        self.assertEqual(s.c, 3)
        self.assertEqual(s['c'], 3)

        s.d = list()
        self.assertTrue(s.d is s['d'])
-        
-        
+
+
    def test_store_none(self):
        """ Test Storage store-None handling
            s.key = None deletes an item
            s['key'] = None sets the item to None
        """
-        
+
        s = Storage(a=1)
-        
+
        self.assertTrue('a' in s)
        self.assertFalse('b' in s)
        s.a = None
        # self.assertFalse('a' in s) # how about this?
-        
+
        s.a = 1
        self.assertTrue('a' in s)
        s['a'] = None
@@ -62,12 +62,12 @@ class TestStorage(unittest.TestCase):

    def test_item(self):
        """ Tests Storage item handling """
-        
+
        s = Storage()
-        
+
        self.assertEqual(s.d, None)
        self.assertEqual(s['d'], None)
-        #self.assertRaises(KeyError, lambda x: s[x], 'd')   # old Storage        
+        #self.assertRaises(KeyError, lambda x: s[x], 'd')   # old Storage
        s.a = 1
        s['a'] = None
        self.assertEquals(s.a, None)
@@ -76,3 +76,4 @@ class TestStorage(unittest.TestCase):

 if __name__ == '__main__':
    unittest.main()
+
@@ -58,3 +58,4 @@ class TestVirtualFields(unittest.TestCase):

 if __name__ == '__main__':
    unittest.main()
+
@@ -25,3 +25,4 @@ class TestUtils(unittest.TestCase):

 if __name__ == '__main__':
    unittest.main()
+
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Unit tests for running web2py
+"""
+import sys
+import os
+if os.path.isdir('gluon'):
+    sys.path.append(os.path.realpath('gluon'))
+else:
+    sys.path.append(os.path.realpath('../'))
+
+import unittest
+from gluon.contrib.webclient import WebClient
+
+class TestWeb(unittest.TestCase):
+    def testWebClient(self):
+        client = WebClient('http://127.0.0.1:8000/welcome/default/')
+
+        client.get('index')
+
+        # register
+        data = dict(first_name = 'Homer',
+                    last_name = 'Simpson',
+                    email = 'homer@web2py.com',
+                    password = 'test',
+                    password_two = 'test',
+                    _formname = 'register')
+        client.post('user/register',data = data)
+
+        # logout
+        client.get('user/logout')
+
+        # login again
+        data = dict(email='homer@web2py.com',
+                    password='test',
+                    _formname = 'login')
+        client.post('user/login',data = data)
+
+        # check registration and login were successful
+        client.get('index')
+        self.assertTrue('Welcome Homer' in client.text)
+
+        client = WebClient('http://127.0.0.1:8000/admin/default/')
+        client.post('index',data=dict(password='hello'))
+        client.get('site')
+        client.get('design/welcome')
+
+if __name__ == '__main__':
+    unittest.main()
+
@@ -1060,6 +1060,7 @@ class Auth(object):
        request = current.request
        session = current.session
        auth = session.auth
+        self.use_username = None # None means postpone detection
        self.user_groups = auth and auth.user_groups or {}
        if auth and auth.last_visit and auth.last_visit + \
                datetime.timedelta(days=0, seconds=auth.expiration) > request.now:
@@ -1257,6 +1258,9 @@ class Auth(object):
            if not 'register' in self.settings.actions_disabled:
                bar.insert(-1, s2)
                bar.insert(-1, register)
+            if self.use_username is None:
+                # should always be false if auth.define_tables() is called
+                self.use_username = 'username' in self.table_user().fields
            if self.use_username and \
                    not 'retrieve_username' in self.settings.actions_disabled:
                bar.insert(-1, s2)
@@ -1612,12 +1616,12 @@ class Auth(object):
        checks = []
        # make a guess about who this user is
        for fieldname in ['registration_id','username','email']:
-            if fieldname in table_user.fields() and keys.get(fieldname,None):
+            if fieldname in table_user.fields() and \
+                    keys.get(fieldname,None):
                checks.append(fieldname)
                value = keys[fieldname]
-                user = user or table_user._db(
-                    (table_user.registration_id==value)|
-                    (table_user[fieldname]==value)).select().first()
+                user = table_user(**{fieldname:value})
+                if user: break
        if not checks:
            return None
        if not 'registration_id' in keys:
@@ -1376,6 +1376,7 @@ class IS_GENERIC_URL(Validator):

    """

+
    def __init__(
        self,
        error_message='enter a valid URL',
@@ -1402,6 +1403,9 @@ class IS_GENERIC_URL(Validator):
                "prepend_scheme='%s' is not in allowed_schemes=%s" \
                % (self.prepend_scheme, self.allowed_schemes)

+    GENERIC_URL = re.compile(r"%[^0-9A-Fa-f]{2}|%[^0-9A-Fa-f][0-9A-Fa-f]|%[0-9A-Fa-f][^0-9A-Fa-f]|%$|%[0-9A-Fa-f]$|%[^0-9A-Fa-f]$")
+    GENERIC_URL_VALID = re.compile(r"[A-Za-z0-9;/?:@&=+$,\-_\.!~*'\(\)%#]+$")
+
    def __call__(self, value):
        """
        :param value: a string, the URL to validate
@@ -1411,12 +1415,9 @@ class IS_GENERIC_URL(Validator):
        """
        try:
            # if the URL does not misuse the '%' character
-            if not re.compile(
-                r"%[^0-9A-Fa-f]{2}|%[^0-9A-Fa-f][0-9A-Fa-f]|%[0-9A-Fa-f][^0-9A-Fa-f]|%$|%[0-9A-Fa-f]$|%[^0-9A-Fa-f]$"
-                              ).search(value):
+            if not self.GENERIC_URL.search(value):
                # if the URL is only composed of valid characters
-                if re.compile(
-                    r"[A-Za-z0-9;/?:@&=+$,\-_\.!~*'\(\)%#]+$").match(value):
+                if self.GENERIC_URL_VALID.match(value):
                    # Then split up the URL into its components and check on
                    # the scheme
                    scheme = url_split_regex.match(value).group(2)
@@ -1432,11 +1433,10 @@ class IS_GENERIC_URL(Validator):
                        # ports, check to see if adding a valid scheme fixes
                        # the problem (but only do this if it doesn't have
                        # one already!)
-                        if not re.compile('://').search(value) and None\
-                             in self.allowed_schemes:
+                        if value.find('://')<0 and None in self.allowed_schemes:
                            schemeToUse = self.prepend_scheme or 'http'
-                            prependTest = self.__call__(schemeToUse
-                                     + '://' + value)
+                            prependTest = self.__call__(
+                                schemeToUse + '://' + value)
                            # if the prepend test succeeded
                            if prependTest[1] is None:
                                # if prepending in the output is enabled
@@ -1791,6 +1791,9 @@ class IS_HTTP_URL(Validator):

    """

+    GENERIC_VALID_IP = re.compile("([\w.!~*'|;:&=+$,-]+@)?\d+\.\d+\.\d+\.\d+(:\d*)*$")
+    GENERIC_VALID_DOMAIN = re.compile("([\w.!~*'|;:&=+$,-]+@)?(([A-Za-z0-9]+[A-Za-z0-9\-]*[A-Za-z0-9]+\.)*([A-Za-z0-9]+\.)*)*([A-Za-z]+[A-Za-z0-9\-]*[A-Za-z0-9]+)\.?(:\d*)*$")
+
    def __init__(
        self,
        error_message='enter a valid URL',
@@ -1843,16 +1846,12 @@ class IS_HTTP_URL(Validator):
                # if there is an authority component
                if authority:
                    # if authority is a valid IP address
-                    if re.compile(
-                        "([\w.!~*'|;:&=+$,-]+@)?\d+\.\d+\.\d+\.\d+(:\d*)*$").match(authority):
+                    if self.GENERIC_VALID_IP.match(authority):
                        # Then this HTTP URL is valid
                        return (value, None)
                    else:
                        # else if authority is a valid domain name
-                        domainMatch = \
-                            re.compile(
-                                "([\w.!~*'|;:&=+$,-]+@)?(([A-Za-z0-9]+[A-Za-z0-9\-]*[A-Za-z0-9]+\.)*([A-Za-z0-9]+\.)*)*([A-Za-z]+[A-Za-z0-9\-]*[A-Za-z0-9]+)\.?(:\d*)*$"
-                                ).match(authority)
+                        domainMatch = self.GENERIC_VALID_DOMAIN.match(authority)
                        if domainMatch:
                            # if the top-level domain really exists
                            if domainMatch.group(5).lower()\
@@ -1865,13 +1864,13 @@ class IS_HTTP_URL(Validator):
                    path = componentsMatch.group(5)
                    # relative case: if this is a valid path (if it starts with
                    # a slash)
-                    if re.compile('/').match(path):
+                    if path.startswith('/'):
                        # Then this HTTP URL is valid
                        return (value, None)
                    else:
                        # abbreviated case: if we haven't already, prepend a
                        # scheme and see if it fixes the problem
-                        if not re.compile('://').search(value):
+                        if value.find('://')<0:
                            schemeToUse = self.prepend_scheme or 'http'
                            prependTest = self.__call__(schemeToUse
                                     + '://' + value)
@@ -2521,9 +2520,11 @@ class CLEANUP(Validator):

    removes special characters on validation
    """
+    REGEX_CLEANUP = re.compile('[^\x09\x0a\x0d\x20-\x7e]')

-    def __init__(self, regex='[^\x09\x0a\x0d\x20-\x7e]'):
-        self.regex = re.compile(regex)
+    def __init__(self, regex=None):
+        self.regex = self.REGEX_CLEANUP if regex is None \
+            else re.compile(regex)

    def __call__(self, value):
        v = self.regex.sub('',str(value).strip())
@@ -2790,11 +2791,13 @@ class IS_STRONG(object):

 class IS_IN_SUBSET(IS_IN_SET):

+    REGEX_W = re.compile('\w+')
+
    def __init__(self, *a, **b):
        IS_IN_SET.__init__(self, *a, **b)

    def __call__(self, value):
-        values = re.compile("\w+").findall(str(value))
+        values = self.REGEX_W.findall(str(value))
        failures = [x for x in values if IS_IN_SET.__call__(self, x)[1]]
        if failures:
            return (value, translate(self.error_message))
@@ -33,8 +33,9 @@ try:
    import Tkinter, tkMessageBox
    import contrib.taskbar_widget
    from winservice import web2py_windows_service_handler
+    have_winservice = True
 except:
-    pass
+    have_winservice = False


 try:
@@ -1003,9 +1004,9 @@ def start(cron=True):
        print ProgramAuthor
        print ProgramVersion

-    from dal import drivers
+    from dal import DRIVERS
    if not options.nobanner:
-        print 'Database drivers available: %s' % ', '.join(drivers)
+        print 'Database drivers available: %s' % ', '.join(DRIVERS)


    # ## if -L load options from options.config file
@@ -1080,8 +1081,12 @@ def start(cron=True):
    # ## if -W install/start/stop web2py as service
    if options.winservice:
        if os.name == 'nt':
-            web2py_windows_service_handler(['', options.winservice],
-                    options.config)
+            if have_winservice:
+                web2py_windows_service_handler(['', options.winservice],
+                                               options.config)
+            else:
+                print 'Error: Missing python module winservice'
+                sys.exit(1)
        else:
            print 'Error: Windows services not supported on this platform'
            sys.exit(1)
@@ -27,12 +27,14 @@
 #  set by the setLevel call, the [logger_myapp] section, and the [handler_...]
 #  section. For example, you will not see DEBUG messages unless all three are
 #  set to DEBUG.
+#
+#  Available levels: DEBUG INFO WARNING ERROR CRITICAL

 [loggers]
 keys=root,rocket,markdown,web2py,rewrite,cron,app,welcome

 [handlers]
-keys=consoleHandler,messageBoxHandler
+keys=consoleHandler,messageBoxHandler,rotatingFileHandler
 #keys=consoleHandler,rotatingFileHandler
 #keys=osxSysLogHandler

@@ -41,50 +43,53 @@ keys=simpleFormatter

 [logger_root]
 level=WARNING
-handlers=consoleHandler
+handlers=consoleHandler,rotatingFileHandler

 [logger_web2py]
 level=WARNING
-handlers=consoleHandler
+handlers=consoleHandler,rotatingFileHandler
 qualname=web2py
 propagate=0

+#  URL rewrite logging (routes.py)
+#  See also the logging parameter in routes.py
+#
 [logger_rewrite]
 level=WARNING
 qualname=web2py.rewrite
-handlers=consoleHandler
+handlers=consoleHandler,rotatingFileHandler
 propagate=0

 [logger_cron]
 level=WARNING
 qualname=web2py.cron
-handlers=consoleHandler
+handlers=consoleHandler,rotatingFileHandler
 propagate=0

 # generic app handler
 [logger_app]
 level=WARNING
 qualname=web2py.app
-handlers=consoleHandler
+handlers=consoleHandler,rotatingFileHandler
 propagate=0

 # welcome app handler
 [logger_welcome]
 level=WARNING
-qualname=web2py.app.welcome
+qualname=web2py.app.welcome,rotatingFileHandler
 handlers=consoleHandler
 propagate=0

 # loggers for legacy getLogger calls: Rocket and markdown
 [logger_rocket]
 level=WARNING
-handlers=consoleHandler,messageBoxHandler
+handlers=consoleHandler,messageBoxHandler,rotatingFileHandler
 qualname=Rocket
 propagate=0

 [logger_markdown]
 level=WARNING
-handlers=consoleHandler
+handlers=consoleHandler,rotatingFileHandler
 qualname=markdown
 propagate=0

@@ -106,7 +111,7 @@ args=()
 #
 [handler_rotatingFileHandler]
 class=handlers.RotatingFileHandler
-level=INFO
+level=DEBUG
 formatter=simpleFormatter
 args=("logs/web2py.log", "a", 1000000, 5)

@@ -26,10 +26,8 @@ administrator_email = 'you@localhost'

 while 1:
    for file in os.listdir(path):
-        filename = os.path.join(path, file)
-
        if not ALLOW_DUPLICATES:
-            fileobj = open(filename, 'r')
+            fileobj = open(file, 'r')
            try:
                file_data = fileobj.read()
            finally:
@@ -42,10 +40,10 @@ while 1:
            hashes[key] = 1

        error = RestrictedError()
-        error.load(request, request.application, filename)
+        error.load(request, request.application, file)

        mail.send(to=administrator_email, subject='new web2py ticket', message=error.traceback)

-        os.unlink(filename)
+        os.unlink(os.path.join(path, file))
    time.sleep(SLEEP_MINUTES * 60)

@@ -9,10 +9,10 @@ if '__file__' in globals():
 elif hasattr(sys, 'frozen'):
    path = os.path.dirname(os.path.abspath(sys.executable)) # for py2exe
 else: #should never happen
-     path = os.getcwd()
+    path = os.getcwd()
 os.chdir(path)

-sys.path = [path]+[p for p in sys.path if not p==path]
+sys.path = [path]+[p for p in sys.path if not p == path]

 # import gluon.import_all ##### This should be uncommented for py2exe.py
 import gluon.widget