dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

web3py backport of lazy get_vars,post_vars and vars, thanks Niphlod (may need more testing)

Browse Source
This commit is contained in:
mdipierro
2013-08-06 06:46:07 -05:00
parent 349aa6c0a5
commit b0e9140508
4 changed files with 191 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 2.6.0-development+timestamp.2013.08.06.05.12.03
Version 2.6.0-development+timestamp.2013.08.06.06.45.13
gluon/globals.py
+166 -5
View File
@@ -36,6 +36,11 @@ import os
import sys
import traceback
import threading
import cgi
import copy
import tempfile
from cache import CacheInRam
from fileutils import copystream
FMT = '%a, %d-%b-%Y %H:%M:%S PST'
PAST = 'Sat, 1-Jan-1971 00:00:00'
@@ -47,6 +52,15 @@ try:
except ImportError:
have_minify = False
try:
import simplejson as sj #external installed library
except:
try:
import json as sj #standard installed library
except:
import contrib.simplejson as sj #pure python library
regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$')
__all__ = ['Request', 'Response', 'Session']
@@ -62,6 +76,52 @@ css_inline = '<style type="text/css">\n%s\n</style>'
js_inline = '<script type="text/javascript">\n%s\n</script>'
def copystream_progress(request, chunk_size=10 ** 5):
"""
copies request.env.wsgi_input into request.body
and stores progress upload status in cache_ram
X-Progress-ID:length and X-Progress-ID:uploaded
"""
env = request.env
if not env.content_length:
return cStringIO.StringIO()
source = env.wsgi_input
try:
size = int(env.content_length)
except ValueError:
raise HTTP(400, "Invalid Content-Length header")
try: # Android requires this
dest = tempfile.NamedTemporaryFile()
except NotImplementedError: # and GAE this
dest = tempfile.TemporaryFile()
if not 'X-Progress-ID' in request.vars:
copystream(source, dest, size, chunk_size)
return dest
cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID']
cache_ram = CacheInRam(request) # same as cache.ram because meta_storage
cache_ram(cache_key + ':length', lambda: size, 0)
cache_ram(cache_key + ':uploaded', lambda: 0, 0)
while size > 0:
if size < chunk_size:
data = source.read(size)
cache_ram.increment(cache_key + ':uploaded', size)
else:
data = source.read(chunk_size)
cache_ram.increment(cache_key + ':uploaded', chunk_size)
length = len(data)
if length > size:
(data, length) = (data[:size], size)
size -= length
if length == 0:
break
dest.write(data)
if length < chunk_size:
break
dest.seek(0)
cache_ram(cache_key + ':length', None)
cache_ram(cache_key + ':uploaded', None)
return dest
class Request(Storage):
"""
@@ -81,13 +141,15 @@ class Request(Storage):
- restful()
"""
def __init__(self):
def __init__(self, env):
Storage.__init__(self)
self.env = Storage()
self.env = Storage(env)
self.env.web2py_path = global_settings.applications_parent
self.env.update(global_settings)
self.cookies = Cookie.SimpleCookie()
self.get_vars = Storage()
self.post_vars = Storage()
self.vars = Storage()
self._get_vars = None
self._post_vars = None
self._vars = None
self.folder = None
self.application = None
self.function = None
@@ -100,6 +162,105 @@ class Request(Storage):
self.is_local = False
self.global_settings = settings.global_settings
def parse_get_vars(self):
query_string = self.env.get('QUERY_STRING','')
dget = cgi.parse_qs(query_string, keep_blank_values=1)
get_vars = self._get_vars = Storage(dget)
for (key, value) in get_vars.iteritems():
if isinstance(value,list) and len(value)==1:
get_vars[key] = value[0]
def parse_post_vars(self):
env = self.env
post_vars = self._post_vars = Storage()
try:
self.body = body = copystream_progress(self)
except IOError:
raise HTTP(400, "Bad Request - HTTP body is incomplete")
#if content-type is application/json, we must read the body
is_json = env.get('content_type', '')[:16] == 'application/json'
if is_json:
try:
json_vars = sj.load(body)
except:
# incoherent request bodies can still be parsed "ad-hoc"
json_vars = {}
pass
# update vars and get_vars with what was posted as json
if isinstance(json_vars, dict):
post_vars.update(json_vars)
body.seek(0)
# parse POST variables on POST, PUT, BOTH only in post_vars
if (body and
env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH') and
not is_json):
dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1)
post_vars.update(dpost)
# The same detection used by FieldStorage to detect multipart POSTs
is_multipart = dpost.type[:10] == 'multipart/'
body.seek(0)
def listify(a):
return (not isinstance(a, list) and [a]) or a
try:
keys = sorted(dpost)
except TypeError:
keys = []
for key in keys:
if key is None:
continue # not sure why cgi.FieldStorage returns None key
dpk = dpost[key]
# if an element is not a file replace it with its value else leave it alone
if isinstance(dpk, list):
value = []
for _dpk in dpk:
if not _dpk.filename:
value.append(_dpk.value)
else:
value.append(_dpk)
elif not dpk.filename:
value = dpk.value
else:
value = dpk
pvalue = listify(value)
if len(pvalue):
post_vars[key] = (len(pvalue) > 1 and pvalue) or pvalue[0]
def parse_all_vars(self):
self._vars = copy.copy(self.get_vars)
for key,value in self.post_vars.iteritems():
if not key in self._vars:
self._vars[key] = value
else:
if not isinstance(self._vars[key],list):
self._vars[key] = [self._vars[key]]
self._vars[key] += value if isinstance(value,list) else [value]
@property
def get_vars(self):
"lazily parse the query string into get_vars"
if self._get_vars is None:
self.parse_get_vars()
return self._get_vars
@property
def post_vars(self):
"lazily parse the body into post_vars"
if self._post_vars is None:
self.parse_post_vars()
return self._post_vars
@property
def vars(self):
"lazily parse all get_vars and post_vars to fill vars"
if self._vars is None:
self.parse_all_vars()
return self._vars
def compute_uuid(self):
self.uuid = '%s/%s.%s.%s' % (
self.application,
gluon/main.py
+21 -157
View File
@@ -14,7 +14,6 @@ Contains:
if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS
import gc
import cgi
import cStringIO
import Cookie
import os
@@ -25,11 +24,10 @@ import time
import datetime
import signal
import socket
import tempfile
import random
import string
import urllib2
try:
import simplejson as sj #external installed library
except:
@@ -40,7 +38,7 @@ except:
from thread import allocate_lock
from fileutils import abspath, write_file, parse_version, copystream
from fileutils import abspath, write_file
from settings import global_settings
from admin import add_path_first, create_missing_folders, create_missing_app_folders
from globals import current
@@ -97,9 +95,7 @@ from compileapp import build_environment, run_models_in, \
run_controller_in, run_view_in
from contenttype import contenttype
from dal import BaseAdapter
from settings import global_settings
from validators import CRYPT
from cache import CacheInRam
from html import URL, xmlescape
from utils import is_valid_ip_address, getipaddrinfo
from rewrite import load, url_in, THREAD_LOCAL as rwthread, \
@@ -143,10 +139,11 @@ def get_client(env):
first tries 'http_x_forwarded_for', secondly 'remote_addr'
if all fails, assume '127.0.0.1' or '::1' (running locally)
"""
g = regex_client.search(env.get('http_x_forwarded_for', ''))
eget = env.get
g = regex_client.search(eget('http_x_forwarded_for', ''))
client = (g.group() or '').split(',')[0] if g else None
if client in (None, '', 'unknown'):
g = regex_client.search(env.get('remote_addr', ''))
g = regex_client.search(eget('remote_addr', ''))
if g:
client = g.group()
elif env.http_host.startswith('['): # IPv6
@@ -158,51 +155,6 @@ def get_client(env):
return client
def copystream_progress(request, chunk_size=10 ** 5):
"""
copies request.env.wsgi_input into request.body
and stores progress upload status in cache_ram
X-Progress-ID:length and X-Progress-ID:uploaded
"""
env = request.env
if not env.content_length:
return cStringIO.StringIO()
source = env.wsgi_input
try:
size = int(env.content_length)
except ValueError:
raise HTTP(400, "Invalid Content-Length header")
try: # Android requires this
dest = tempfile.NamedTemporaryFile()
except NotImplementedError: # and GAE this
dest = tempfile.TemporaryFile()
if not 'X-Progress-ID' in request.vars:
copystream(source, dest, size, chunk_size)
return dest
cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID']
cache_ram = CacheInRam(request) # same as cache.ram because meta_storage
cache_ram(cache_key + ':length', lambda: size, 0)
cache_ram(cache_key + ':uploaded', lambda: 0, 0)
while size > 0:
if size < chunk_size:
data = source.read(size)
cache_ram.increment(cache_key + ':uploaded', size)
else:
data = source.read(chunk_size)
cache_ram.increment(cache_key + ':uploaded', chunk_size)
length = len(data)
if length > size:
(data, length) = (data[:size], size)
size -= length
if length == 0:
break
dest.write(data)
if length < chunk_size:
break
dest.seek(0)
cache_ram(cache_key + ':length', None)
cache_ram(cache_key + ':uploaded', None)
return dest
def serve_controller(request, response, session):
@@ -282,10 +234,10 @@ class LazyWSGI(object):
def start_response(self,status='200', headers=[], exec_info=None):
"""
in controller you can use::
- request.wsgi.environ
- request.wsgi.start_response
to call third party WSGI applications
"""
self.response.status = str(status).split(' ', 1)[0]
@@ -295,116 +247,27 @@ class LazyWSGI(object):
def middleware(self,*middleware_apps):
"""
In you controller use::
@request.wsgi.middleware(middleware1, middleware2, ...)
to decorate actions with WSGI middleware. actions must return strings.
uses a simulated environment so it may have weird behavior in some cases
"""
def middleware(f):
def app(environ, start_response):
data = f()
start_response(self.response.status,
start_response(self.response.status,
self.response.headers.items())
if isinstance(data, list):
return data
return [data]
for item in middleware_apps:
app = item(app)
def caller(app):
def caller(app):
return app(self.environ, self.start_response)
return lambda caller=caller, app=app: caller(app)
return middleware
ISLE25 = sys.version_info[1] <= 5
def parse_get_post_vars(request, environ):
# always parse variables in URL for GET, POST, PUT, DELETE, etc. in get_vars
env = request.env
dget = cgi.parse_qsl(env.query_string or '', keep_blank_values=1)
for (key, value) in dget:
if key in request.get_vars:
if isinstance(request.get_vars[key], list):
request.get_vars[key] += [value]
else:
request.get_vars[key] = [request.get_vars[key]] + [value]
else:
request.get_vars[key] = value
request.vars[key] = request.get_vars[key]
try:
request.body = body = copystream_progress(request)
except IOError:
raise HTTP(400, "Bad Request - HTTP body is incomplete")
#if content-type is application/json, we must read the body
is_json = env.get('http_content_type', '')[:16] == 'application/json'
if is_json:
try:
json_vars = sj.load(body)
body.seek(0)
except:
# incoherent request bodies can still be parsed "ad-hoc"
json_vars = {}
pass
# update vars and get_vars with what was posted as json
if isinstance(json_vars,dict):
request.get_vars.update(json_vars)
request.vars.update(json_vars)
# parse POST variables on POST, PUT, BOTH only in post_vars
if (body and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
dpost = cgi.FieldStorage(fp=body, environ=environ, keep_blank_values=1)
# The same detection used by FieldStorage to detect multipart POSTs
is_multipart = dpost.type[:10] == 'multipart/'
body.seek(0)
def listify(a):
return (not isinstance(a, list) and [a]) or a
try:
keys = sorted(dpost)
except TypeError:
keys = []
for key in keys:
if key is None:
continue # not sure why cgi.FieldStorage returns None key
dpk = dpost[key]
# if en element is not a file replace it with its value else leave it alone
if isinstance(dpk, list):
value = []
for _dpk in dpk:
if not _dpk.filename:
value.append(_dpk.value)
else:
value.append(_dpk)
elif not dpk.filename:
value = dpk.value
else:
value = dpk
pvalue = listify(value)
if key in request.vars:
gvalue = listify(request.vars[key])
if ISLE25:
value = pvalue + gvalue
elif is_multipart:
pvalue = pvalue[len(gvalue):]
else:
pvalue = pvalue[:-len(gvalue)]
request.vars[key] = value
if len(pvalue):
request.post_vars[key] = (len(pvalue) >
1 and pvalue) or pvalue[0]
if is_json and isinstance(json_vars,dict):
# update post_vars with what was posted as json
request.post_vars.update(json_vars)
def wsgibase(environ, responder):
"""
this is the gluon wsgi application. the first function called when a page
@@ -435,15 +298,15 @@ def wsgibase(environ, responder):
[a-zA-Z0-9_]
- file and sub may also contain '-', '=', '.' and '/'
"""
eget = environ.get
current.__dict__.clear()
request = Request()
request = Request(environ)
response = Response()
session = Session()
env = request.env
env.web2py_path = global_settings.applications_parent
#env.web2py_path = global_settings.applications_parent
env.web2py_version = web2py_version
env.update(global_settings)
#env.update(global_settings)
static_file = False
try:
try:
@@ -462,8 +325,7 @@ def wsgibase(environ, responder):
response.status = env.web2py_status_code or response.status
if static_file:
if environ.get('QUERY_STRING', '').startswith(
'attachment'):
if eget('QUERY_STRING', '').startswith('attachment'):
response.headers['Content-Disposition'] \
= 'attachment'
if version:
@@ -472,6 +334,7 @@ def wsgibase(environ, responder):
'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
response.stream(static_file, request=request)
# ##################################################
# fill in request items
# ##################################################
@@ -485,7 +348,7 @@ def wsgibase(environ, responder):
local_hosts.add(socket.gethostname())
local_hosts.add(fqdn)
local_hosts.update([
addrinfo[4][0] for addrinfo
addrinfo[4][0] for addrinfo
in getipaddrinfo(fqdn)])
if env.server_name:
local_hosts.add(env.server_name)
@@ -508,7 +371,8 @@ def wsgibase(environ, responder):
is_local = env.remote_addr in local_hosts,
is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
or env.https == 'on')
or env.https == 'on'
)
request.compute_uuid() # requires client
request.url = environ['PATH_INFO']
@@ -544,7 +408,7 @@ def wsgibase(environ, responder):
# get the GET and POST data
# ##################################################
parse_get_post_vars(request, environ)
#parse_get_post_vars(request, environ)
# ##################################################
# expose wsgi hooks for convenience
gluon/shell.py
+3 -3
View File
@@ -38,7 +38,7 @@ def enable_autocomplete_and_history(adir,env):
except ImportError:
pass
else:
readline.parse_and_bind("bind ^I rl_complete"
readline.parse_and_bind("bind ^I rl_complete"
if sys.platform == 'darwin'
else "tab: complete")
history_file = os.path.join(adir,'.pythonhistory')
@@ -71,7 +71,7 @@ def exec_environment(
"""
if request is None:
request = Request()
request = Request({})
if response is None:
response = Response()
if session is None:
@@ -116,7 +116,7 @@ def env(
web2py environment.
"""
request = Request()
request = Request({})
response = Response()
session = Session()
request.application = a