diff --git a/VERSION b/VERSION
index e28b3572..6d25a1c3 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-Version 2.6.0-development+timestamp.2013.08.06.05.12.03
+Version 2.6.0-development+timestamp.2013.08.06.06.45.13
diff --git a/gluon/globals.py b/gluon/globals.py
index 39117a22..9f214837 100644
--- a/gluon/globals.py
+++ b/gluon/globals.py
@@ -36,6 +36,11 @@ import os
import sys
import traceback
import threading
+import cgi
+import copy
+import tempfile
+from cache import CacheInRam
+from fileutils import copystream
FMT = '%a, %d-%b-%Y %H:%M:%S PST'
PAST = 'Sat, 1-Jan-1971 00:00:00'
@@ -47,6 +52,15 @@ try:
except ImportError:
have_minify = False
+
+try:
+ import simplejson as sj #external installed library
+except:
+ try:
+ import json as sj #standard installed library
+ except:
+ import contrib.simplejson as sj #pure python library
+
regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$')
__all__ = ['Request', 'Response', 'Session']
@@ -62,6 +76,52 @@ css_inline = ''
js_inline = ''
+def copystream_progress(request, chunk_size=10 ** 5):
+ """
+ copies request.env.wsgi_input into request.body
+ and stores progress upload status in cache_ram
+ X-Progress-ID:length and X-Progress-ID:uploaded
+ """
+ env = request.env
+ if not env.content_length:
+ return cStringIO.StringIO()
+ source = env.wsgi_input
+ try:
+ size = int(env.content_length)
+ except ValueError:
+ raise HTTP(400, "Invalid Content-Length header")
+ try: # Android requires this
+ dest = tempfile.NamedTemporaryFile()
+ except NotImplementedError: # and GAE this
+ dest = tempfile.TemporaryFile()
+ if not 'X-Progress-ID' in request.vars:
+ copystream(source, dest, size, chunk_size)
+ return dest
+ cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID']
+ cache_ram = CacheInRam(request) # same as cache.ram because meta_storage
+ cache_ram(cache_key + ':length', lambda: size, 0)
+ cache_ram(cache_key + ':uploaded', lambda: 0, 0)
+ while size > 0:
+ if size < chunk_size:
+ data = source.read(size)
+ cache_ram.increment(cache_key + ':uploaded', size)
+ else:
+ data = source.read(chunk_size)
+ cache_ram.increment(cache_key + ':uploaded', chunk_size)
+ length = len(data)
+ if length > size:
+ (data, length) = (data[:size], size)
+ size -= length
+ if length == 0:
+ break
+ dest.write(data)
+ if length < chunk_size:
+ break
+ dest.seek(0)
+ cache_ram(cache_key + ':length', None)
+ cache_ram(cache_key + ':uploaded', None)
+ return dest
+
class Request(Storage):
"""
@@ -81,13 +141,15 @@ class Request(Storage):
- restful()
"""
- def __init__(self):
+ def __init__(self, env):
Storage.__init__(self)
- self.env = Storage()
+ self.env = Storage(env)
+ self.env.web2py_path = global_settings.applications_parent
+ self.env.update(global_settings)
self.cookies = Cookie.SimpleCookie()
- self.get_vars = Storage()
- self.post_vars = Storage()
- self.vars = Storage()
+ self._get_vars = None
+ self._post_vars = None
+ self._vars = None
self.folder = None
self.application = None
self.function = None
@@ -100,6 +162,105 @@ class Request(Storage):
self.is_local = False
self.global_settings = settings.global_settings
+ def parse_get_vars(self):
+ query_string = self.env.get('QUERY_STRING','')
+ dget = cgi.parse_qs(query_string, keep_blank_values=1)
+ get_vars = self._get_vars = Storage(dget)
+ for (key, value) in get_vars.iteritems():
+ if isinstance(value,list) and len(value)==1:
+ get_vars[key] = value[0]
+
+ def parse_post_vars(self):
+ env = self.env
+ post_vars = self._post_vars = Storage()
+ try:
+ self.body = body = copystream_progress(self)
+ except IOError:
+ raise HTTP(400, "Bad Request - HTTP body is incomplete")
+
+ #if content-type is application/json, we must read the body
+ is_json = env.get('content_type', '')[:16] == 'application/json'
+
+ if is_json:
+ try:
+ json_vars = sj.load(body)
+ except:
+ # incoherent request bodies can still be parsed "ad-hoc"
+ json_vars = {}
+ pass
+ # update vars and get_vars with what was posted as json
+ if isinstance(json_vars, dict):
+ post_vars.update(json_vars)
+
+ body.seek(0)
+
+ # parse POST variables on POST, PUT, BOTH only in post_vars
+ if (body and
+ env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH') and
+ not is_json):
+ dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1)
+ post_vars.update(dpost)
+ # The same detection used by FieldStorage to detect multipart POSTs
+ is_multipart = dpost.type[:10] == 'multipart/'
+ body.seek(0)
+
+ def listify(a):
+ return (not isinstance(a, list) and [a]) or a
+ try:
+ keys = sorted(dpost)
+ except TypeError:
+ keys = []
+ for key in keys:
+ if key is None:
+ continue # not sure why cgi.FieldStorage returns None key
+ dpk = dpost[key]
+ # if an element is not a file replace it with its value else leave it alone
+ if isinstance(dpk, list):
+ value = []
+ for _dpk in dpk:
+ if not _dpk.filename:
+ value.append(_dpk.value)
+ else:
+ value.append(_dpk)
+ elif not dpk.filename:
+ value = dpk.value
+ else:
+ value = dpk
+ pvalue = listify(value)
+ if len(pvalue):
+ post_vars[key] = (len(pvalue) > 1 and pvalue) or pvalue[0]
+
+ def parse_all_vars(self):
+ self._vars = copy.copy(self.get_vars)
+ for key,value in self.post_vars.iteritems():
+ if not key in self._vars:
+ self._vars[key] = value
+ else:
+ if not isinstance(self._vars[key],list):
+ self._vars[key] = [self._vars[key]]
+ self._vars[key] += value if isinstance(value,list) else [value]
+
+ @property
+ def get_vars(self):
+ "lazily parse the query string into get_vars"
+ if self._get_vars is None:
+ self.parse_get_vars()
+ return self._get_vars
+
+ @property
+ def post_vars(self):
+ "lazily parse the body into post_vars"
+ if self._post_vars is None:
+ self.parse_post_vars()
+ return self._post_vars
+
+ @property
+ def vars(self):
+ "lazily parse all get_vars and post_vars to fill vars"
+ if self._vars is None:
+ self.parse_all_vars()
+ return self._vars
+
def compute_uuid(self):
self.uuid = '%s/%s.%s.%s' % (
self.application,
diff --git a/gluon/main.py b/gluon/main.py
index 2c56fe3f..53f426b4 100644
--- a/gluon/main.py
+++ b/gluon/main.py
@@ -14,7 +14,6 @@ Contains:
if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS
import gc
-import cgi
import cStringIO
import Cookie
import os
@@ -25,11 +24,10 @@ import time
import datetime
import signal
import socket
-import tempfile
import random
-import string
import urllib2
+
try:
import simplejson as sj #external installed library
except:
@@ -40,7 +38,7 @@ except:
from thread import allocate_lock
-from fileutils import abspath, write_file, parse_version, copystream
+from fileutils import abspath, write_file
from settings import global_settings
from admin import add_path_first, create_missing_folders, create_missing_app_folders
from globals import current
@@ -97,9 +95,7 @@ from compileapp import build_environment, run_models_in, \
run_controller_in, run_view_in
from contenttype import contenttype
from dal import BaseAdapter
-from settings import global_settings
from validators import CRYPT
-from cache import CacheInRam
from html import URL, xmlescape
from utils import is_valid_ip_address, getipaddrinfo
from rewrite import load, url_in, THREAD_LOCAL as rwthread, \
@@ -143,10 +139,11 @@ def get_client(env):
first tries 'http_x_forwarded_for', secondly 'remote_addr'
if all fails, assume '127.0.0.1' or '::1' (running locally)
"""
- g = regex_client.search(env.get('http_x_forwarded_for', ''))
+ eget = env.get
+ g = regex_client.search(eget('http_x_forwarded_for', ''))
client = (g.group() or '').split(',')[0] if g else None
if client in (None, '', 'unknown'):
- g = regex_client.search(env.get('remote_addr', ''))
+ g = regex_client.search(eget('remote_addr', ''))
if g:
client = g.group()
elif env.http_host.startswith('['): # IPv6
@@ -158,51 +155,6 @@ def get_client(env):
return client
-def copystream_progress(request, chunk_size=10 ** 5):
- """
- copies request.env.wsgi_input into request.body
- and stores progress upload status in cache_ram
- X-Progress-ID:length and X-Progress-ID:uploaded
- """
- env = request.env
- if not env.content_length:
- return cStringIO.StringIO()
- source = env.wsgi_input
- try:
- size = int(env.content_length)
- except ValueError:
- raise HTTP(400, "Invalid Content-Length header")
- try: # Android requires this
- dest = tempfile.NamedTemporaryFile()
- except NotImplementedError: # and GAE this
- dest = tempfile.TemporaryFile()
- if not 'X-Progress-ID' in request.vars:
- copystream(source, dest, size, chunk_size)
- return dest
- cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID']
- cache_ram = CacheInRam(request) # same as cache.ram because meta_storage
- cache_ram(cache_key + ':length', lambda: size, 0)
- cache_ram(cache_key + ':uploaded', lambda: 0, 0)
- while size > 0:
- if size < chunk_size:
- data = source.read(size)
- cache_ram.increment(cache_key + ':uploaded', size)
- else:
- data = source.read(chunk_size)
- cache_ram.increment(cache_key + ':uploaded', chunk_size)
- length = len(data)
- if length > size:
- (data, length) = (data[:size], size)
- size -= length
- if length == 0:
- break
- dest.write(data)
- if length < chunk_size:
- break
- dest.seek(0)
- cache_ram(cache_key + ':length', None)
- cache_ram(cache_key + ':uploaded', None)
- return dest
def serve_controller(request, response, session):
@@ -282,10 +234,10 @@ class LazyWSGI(object):
def start_response(self,status='200', headers=[], exec_info=None):
"""
in controller you can use::
-
+
- request.wsgi.environ
- request.wsgi.start_response
-
+
to call third party WSGI applications
"""
self.response.status = str(status).split(' ', 1)[0]
@@ -295,116 +247,27 @@ class LazyWSGI(object):
def middleware(self,*middleware_apps):
"""
In you controller use::
-
+
@request.wsgi.middleware(middleware1, middleware2, ...)
-
+
to decorate actions with WSGI middleware. actions must return strings.
uses a simulated environment so it may have weird behavior in some cases
"""
def middleware(f):
def app(environ, start_response):
data = f()
- start_response(self.response.status,
+ start_response(self.response.status,
self.response.headers.items())
if isinstance(data, list):
return data
return [data]
for item in middleware_apps:
app = item(app)
- def caller(app):
+ def caller(app):
return app(self.environ, self.start_response)
return lambda caller=caller, app=app: caller(app)
return middleware
-ISLE25 = sys.version_info[1] <= 5
-
-def parse_get_post_vars(request, environ):
-
- # always parse variables in URL for GET, POST, PUT, DELETE, etc. in get_vars
- env = request.env
- dget = cgi.parse_qsl(env.query_string or '', keep_blank_values=1)
- for (key, value) in dget:
- if key in request.get_vars:
- if isinstance(request.get_vars[key], list):
- request.get_vars[key] += [value]
- else:
- request.get_vars[key] = [request.get_vars[key]] + [value]
- else:
- request.get_vars[key] = value
- request.vars[key] = request.get_vars[key]
-
-
- try:
- request.body = body = copystream_progress(request)
- except IOError:
- raise HTTP(400, "Bad Request - HTTP body is incomplete")
-
- #if content-type is application/json, we must read the body
- is_json = env.get('http_content_type', '')[:16] == 'application/json'
-
-
- if is_json:
- try:
- json_vars = sj.load(body)
- body.seek(0)
- except:
- # incoherent request bodies can still be parsed "ad-hoc"
- json_vars = {}
- pass
- # update vars and get_vars with what was posted as json
- if isinstance(json_vars,dict):
- request.get_vars.update(json_vars)
- request.vars.update(json_vars)
-
-
- # parse POST variables on POST, PUT, BOTH only in post_vars
- if (body and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
- dpost = cgi.FieldStorage(fp=body, environ=environ, keep_blank_values=1)
- # The same detection used by FieldStorage to detect multipart POSTs
- is_multipart = dpost.type[:10] == 'multipart/'
- body.seek(0)
-
-
- def listify(a):
- return (not isinstance(a, list) and [a]) or a
- try:
- keys = sorted(dpost)
- except TypeError:
- keys = []
- for key in keys:
- if key is None:
- continue # not sure why cgi.FieldStorage returns None key
- dpk = dpost[key]
- # if en element is not a file replace it with its value else leave it alone
- if isinstance(dpk, list):
- value = []
- for _dpk in dpk:
- if not _dpk.filename:
- value.append(_dpk.value)
- else:
- value.append(_dpk)
- elif not dpk.filename:
- value = dpk.value
- else:
- value = dpk
- pvalue = listify(value)
- if key in request.vars:
- gvalue = listify(request.vars[key])
- if ISLE25:
- value = pvalue + gvalue
- elif is_multipart:
- pvalue = pvalue[len(gvalue):]
- else:
- pvalue = pvalue[:-len(gvalue)]
- request.vars[key] = value
- if len(pvalue):
- request.post_vars[key] = (len(pvalue) >
- 1 and pvalue) or pvalue[0]
- if is_json and isinstance(json_vars,dict):
- # update post_vars with what was posted as json
- request.post_vars.update(json_vars)
-
-
def wsgibase(environ, responder):
"""
this is the gluon wsgi application. the first function called when a page
@@ -435,15 +298,15 @@ def wsgibase(environ, responder):
[a-zA-Z0-9_]
- file and sub may also contain '-', '=', '.' and '/'
"""
-
+ eget = environ.get
current.__dict__.clear()
- request = Request()
+ request = Request(environ)
response = Response()
session = Session()
env = request.env
- env.web2py_path = global_settings.applications_parent
+ #env.web2py_path = global_settings.applications_parent
env.web2py_version = web2py_version
- env.update(global_settings)
+ #env.update(global_settings)
static_file = False
try:
try:
@@ -462,8 +325,7 @@ def wsgibase(environ, responder):
response.status = env.web2py_status_code or response.status
if static_file:
- if environ.get('QUERY_STRING', '').startswith(
- 'attachment'):
+ if eget('QUERY_STRING', '').startswith('attachment'):
response.headers['Content-Disposition'] \
= 'attachment'
if version:
@@ -472,6 +334,7 @@ def wsgibase(environ, responder):
'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
response.stream(static_file, request=request)
+
# ##################################################
# fill in request items
# ##################################################
@@ -485,7 +348,7 @@ def wsgibase(environ, responder):
local_hosts.add(socket.gethostname())
local_hosts.add(fqdn)
local_hosts.update([
- addrinfo[4][0] for addrinfo
+ addrinfo[4][0] for addrinfo
in getipaddrinfo(fqdn)])
if env.server_name:
local_hosts.add(env.server_name)
@@ -508,7 +371,8 @@ def wsgibase(environ, responder):
is_local = env.remote_addr in local_hosts,
is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
- or env.https == 'on')
+ or env.https == 'on'
+ )
request.compute_uuid() # requires client
request.url = environ['PATH_INFO']
@@ -544,7 +408,7 @@ def wsgibase(environ, responder):
# get the GET and POST data
# ##################################################
- parse_get_post_vars(request, environ)
+ #parse_get_post_vars(request, environ)
# ##################################################
# expose wsgi hooks for convenience
diff --git a/gluon/shell.py b/gluon/shell.py
index 54a52f47..92d59291 100644
--- a/gluon/shell.py
+++ b/gluon/shell.py
@@ -38,7 +38,7 @@ def enable_autocomplete_and_history(adir,env):
except ImportError:
pass
else:
- readline.parse_and_bind("bind ^I rl_complete"
+ readline.parse_and_bind("bind ^I rl_complete"
if sys.platform == 'darwin'
else "tab: complete")
history_file = os.path.join(adir,'.pythonhistory')
@@ -71,7 +71,7 @@ def exec_environment(
"""
if request is None:
- request = Request()
+ request = Request({})
if response is None:
response = Response()
if session is None:
@@ -116,7 +116,7 @@ def env(
web2py environment.
"""
- request = Request()
+ request = Request({})
response = Response()
session = Session()
request.application = a