From b0e914050830a1e85d80e4bdbc34205f02fc6f24 Mon Sep 17 00:00:00 2001 From: mdipierro Date: Tue, 6 Aug 2013 06:46:07 -0500 Subject: [PATCH] web3py backport of lazy get_vars,post_vars and vars, thanks Niphlod (may need more testing) --- VERSION | 2 +- gluon/globals.py | 171 +++++++++++++++++++++++++++++++++++++++++++-- gluon/main.py | 178 ++++++----------------------------------------- gluon/shell.py | 6 +- 4 files changed, 191 insertions(+), 166 deletions(-) diff --git a/VERSION b/VERSION index e28b3572..6d25a1c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 2.6.0-development+timestamp.2013.08.06.05.12.03 +Version 2.6.0-development+timestamp.2013.08.06.06.45.13 diff --git a/gluon/globals.py b/gluon/globals.py index 39117a22..9f214837 100644 --- a/gluon/globals.py +++ b/gluon/globals.py @@ -36,6 +36,11 @@ import os import sys import traceback import threading +import cgi +import copy +import tempfile +from cache import CacheInRam +from fileutils import copystream FMT = '%a, %d-%b-%Y %H:%M:%S PST' PAST = 'Sat, 1-Jan-1971 00:00:00' @@ -47,6 +52,15 @@ try: except ImportError: have_minify = False + +try: + import simplejson as sj #external installed library +except: + try: + import json as sj #standard installed library + except: + import contrib.simplejson as sj #pure python library + regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$') __all__ = ['Request', 'Response', 'Session'] @@ -62,6 +76,52 @@ css_inline = '' js_inline = '' +def copystream_progress(request, chunk_size=10 ** 5): + """ + copies request.env.wsgi_input into request.body + and stores progress upload status in cache_ram + X-Progress-ID:length and X-Progress-ID:uploaded + """ + env = request.env + if not env.content_length: + return cStringIO.StringIO() + source = env.wsgi_input + try: + size = int(env.content_length) + except ValueError: + raise HTTP(400, "Invalid Content-Length header") + try: # Android requires this + dest = tempfile.NamedTemporaryFile() + except NotImplementedError: # and GAE this + dest = tempfile.TemporaryFile() + if not 'X-Progress-ID' in request.vars: + copystream(source, dest, size, chunk_size) + return dest + cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID'] + cache_ram = CacheInRam(request) # same as cache.ram because meta_storage + cache_ram(cache_key + ':length', lambda: size, 0) + cache_ram(cache_key + ':uploaded', lambda: 0, 0) + while size > 0: + if size < chunk_size: + data = source.read(size) + cache_ram.increment(cache_key + ':uploaded', size) + else: + data = source.read(chunk_size) + cache_ram.increment(cache_key + ':uploaded', chunk_size) + length = len(data) + if length > size: + (data, length) = (data[:size], size) + size -= length + if length == 0: + break + dest.write(data) + if length < chunk_size: + break + dest.seek(0) + cache_ram(cache_key + ':length', None) + cache_ram(cache_key + ':uploaded', None) + return dest + class Request(Storage): """ @@ -81,13 +141,15 @@ class Request(Storage): - restful() """ - def __init__(self): + def __init__(self, env): Storage.__init__(self) - self.env = Storage() + self.env = Storage(env) + self.env.web2py_path = global_settings.applications_parent + self.env.update(global_settings) self.cookies = Cookie.SimpleCookie() - self.get_vars = Storage() - self.post_vars = Storage() - self.vars = Storage() + self._get_vars = None + self._post_vars = None + self._vars = None self.folder = None self.application = None self.function = None @@ -100,6 +162,105 @@ class Request(Storage): self.is_local = False self.global_settings = settings.global_settings + def parse_get_vars(self): + query_string = self.env.get('QUERY_STRING','') + dget = cgi.parse_qs(query_string, keep_blank_values=1) + get_vars = self._get_vars = Storage(dget) + for (key, value) in get_vars.iteritems(): + if isinstance(value,list) and len(value)==1: + get_vars[key] = value[0] + + def parse_post_vars(self): + env = self.env + post_vars = self._post_vars = Storage() + try: + self.body = body = copystream_progress(self) + except IOError: + raise HTTP(400, "Bad Request - HTTP body is incomplete") + + #if content-type is application/json, we must read the body + is_json = env.get('content_type', '')[:16] == 'application/json' + + if is_json: + try: + json_vars = sj.load(body) + except: + # incoherent request bodies can still be parsed "ad-hoc" + json_vars = {} + pass + # update vars and get_vars with what was posted as json + if isinstance(json_vars, dict): + post_vars.update(json_vars) + + body.seek(0) + + # parse POST variables on POST, PUT, BOTH only in post_vars + if (body and + env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH') and + not is_json): + dpost = cgi.FieldStorage(fp=body, environ=env, keep_blank_values=1) + post_vars.update(dpost) + # The same detection used by FieldStorage to detect multipart POSTs + is_multipart = dpost.type[:10] == 'multipart/' + body.seek(0) + + def listify(a): + return (not isinstance(a, list) and [a]) or a + try: + keys = sorted(dpost) + except TypeError: + keys = [] + for key in keys: + if key is None: + continue # not sure why cgi.FieldStorage returns None key + dpk = dpost[key] + # if an element is not a file replace it with its value else leave it alone + if isinstance(dpk, list): + value = [] + for _dpk in dpk: + if not _dpk.filename: + value.append(_dpk.value) + else: + value.append(_dpk) + elif not dpk.filename: + value = dpk.value + else: + value = dpk + pvalue = listify(value) + if len(pvalue): + post_vars[key] = (len(pvalue) > 1 and pvalue) or pvalue[0] + + def parse_all_vars(self): + self._vars = copy.copy(self.get_vars) + for key,value in self.post_vars.iteritems(): + if not key in self._vars: + self._vars[key] = value + else: + if not isinstance(self._vars[key],list): + self._vars[key] = [self._vars[key]] + self._vars[key] += value if isinstance(value,list) else [value] + + @property + def get_vars(self): + "lazily parse the query string into get_vars" + if self._get_vars is None: + self.parse_get_vars() + return self._get_vars + + @property + def post_vars(self): + "lazily parse the body into post_vars" + if self._post_vars is None: + self.parse_post_vars() + return self._post_vars + + @property + def vars(self): + "lazily parse all get_vars and post_vars to fill vars" + if self._vars is None: + self.parse_all_vars() + return self._vars + def compute_uuid(self): self.uuid = '%s/%s.%s.%s' % ( self.application, diff --git a/gluon/main.py b/gluon/main.py index 2c56fe3f..53f426b4 100644 --- a/gluon/main.py +++ b/gluon/main.py @@ -14,7 +14,6 @@ Contains: if False: import import_all # DO NOT REMOVE PART OF FREEZE PROCESS import gc -import cgi import cStringIO import Cookie import os @@ -25,11 +24,10 @@ import time import datetime import signal import socket -import tempfile import random -import string import urllib2 + try: import simplejson as sj #external installed library except: @@ -40,7 +38,7 @@ except: from thread import allocate_lock -from fileutils import abspath, write_file, parse_version, copystream +from fileutils import abspath, write_file from settings import global_settings from admin import add_path_first, create_missing_folders, create_missing_app_folders from globals import current @@ -97,9 +95,7 @@ from compileapp import build_environment, run_models_in, \ run_controller_in, run_view_in from contenttype import contenttype from dal import BaseAdapter -from settings import global_settings from validators import CRYPT -from cache import CacheInRam from html import URL, xmlescape from utils import is_valid_ip_address, getipaddrinfo from rewrite import load, url_in, THREAD_LOCAL as rwthread, \ @@ -143,10 +139,11 @@ def get_client(env): first tries 'http_x_forwarded_for', secondly 'remote_addr' if all fails, assume '127.0.0.1' or '::1' (running locally) """ - g = regex_client.search(env.get('http_x_forwarded_for', '')) + eget = env.get + g = regex_client.search(eget('http_x_forwarded_for', '')) client = (g.group() or '').split(',')[0] if g else None if client in (None, '', 'unknown'): - g = regex_client.search(env.get('remote_addr', '')) + g = regex_client.search(eget('remote_addr', '')) if g: client = g.group() elif env.http_host.startswith('['): # IPv6 @@ -158,51 +155,6 @@ def get_client(env): return client -def copystream_progress(request, chunk_size=10 ** 5): - """ - copies request.env.wsgi_input into request.body - and stores progress upload status in cache_ram - X-Progress-ID:length and X-Progress-ID:uploaded - """ - env = request.env - if not env.content_length: - return cStringIO.StringIO() - source = env.wsgi_input - try: - size = int(env.content_length) - except ValueError: - raise HTTP(400, "Invalid Content-Length header") - try: # Android requires this - dest = tempfile.NamedTemporaryFile() - except NotImplementedError: # and GAE this - dest = tempfile.TemporaryFile() - if not 'X-Progress-ID' in request.vars: - copystream(source, dest, size, chunk_size) - return dest - cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID'] - cache_ram = CacheInRam(request) # same as cache.ram because meta_storage - cache_ram(cache_key + ':length', lambda: size, 0) - cache_ram(cache_key + ':uploaded', lambda: 0, 0) - while size > 0: - if size < chunk_size: - data = source.read(size) - cache_ram.increment(cache_key + ':uploaded', size) - else: - data = source.read(chunk_size) - cache_ram.increment(cache_key + ':uploaded', chunk_size) - length = len(data) - if length > size: - (data, length) = (data[:size], size) - size -= length - if length == 0: - break - dest.write(data) - if length < chunk_size: - break - dest.seek(0) - cache_ram(cache_key + ':length', None) - cache_ram(cache_key + ':uploaded', None) - return dest def serve_controller(request, response, session): @@ -282,10 +234,10 @@ class LazyWSGI(object): def start_response(self,status='200', headers=[], exec_info=None): """ in controller you can use:: - + - request.wsgi.environ - request.wsgi.start_response - + to call third party WSGI applications """ self.response.status = str(status).split(' ', 1)[0] @@ -295,116 +247,27 @@ class LazyWSGI(object): def middleware(self,*middleware_apps): """ In you controller use:: - + @request.wsgi.middleware(middleware1, middleware2, ...) - + to decorate actions with WSGI middleware. actions must return strings. uses a simulated environment so it may have weird behavior in some cases """ def middleware(f): def app(environ, start_response): data = f() - start_response(self.response.status, + start_response(self.response.status, self.response.headers.items()) if isinstance(data, list): return data return [data] for item in middleware_apps: app = item(app) - def caller(app): + def caller(app): return app(self.environ, self.start_response) return lambda caller=caller, app=app: caller(app) return middleware -ISLE25 = sys.version_info[1] <= 5 - -def parse_get_post_vars(request, environ): - - # always parse variables in URL for GET, POST, PUT, DELETE, etc. in get_vars - env = request.env - dget = cgi.parse_qsl(env.query_string or '', keep_blank_values=1) - for (key, value) in dget: - if key in request.get_vars: - if isinstance(request.get_vars[key], list): - request.get_vars[key] += [value] - else: - request.get_vars[key] = [request.get_vars[key]] + [value] - else: - request.get_vars[key] = value - request.vars[key] = request.get_vars[key] - - - try: - request.body = body = copystream_progress(request) - except IOError: - raise HTTP(400, "Bad Request - HTTP body is incomplete") - - #if content-type is application/json, we must read the body - is_json = env.get('http_content_type', '')[:16] == 'application/json' - - - if is_json: - try: - json_vars = sj.load(body) - body.seek(0) - except: - # incoherent request bodies can still be parsed "ad-hoc" - json_vars = {} - pass - # update vars and get_vars with what was posted as json - if isinstance(json_vars,dict): - request.get_vars.update(json_vars) - request.vars.update(json_vars) - - - # parse POST variables on POST, PUT, BOTH only in post_vars - if (body and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')): - dpost = cgi.FieldStorage(fp=body, environ=environ, keep_blank_values=1) - # The same detection used by FieldStorage to detect multipart POSTs - is_multipart = dpost.type[:10] == 'multipart/' - body.seek(0) - - - def listify(a): - return (not isinstance(a, list) and [a]) or a - try: - keys = sorted(dpost) - except TypeError: - keys = [] - for key in keys: - if key is None: - continue # not sure why cgi.FieldStorage returns None key - dpk = dpost[key] - # if en element is not a file replace it with its value else leave it alone - if isinstance(dpk, list): - value = [] - for _dpk in dpk: - if not _dpk.filename: - value.append(_dpk.value) - else: - value.append(_dpk) - elif not dpk.filename: - value = dpk.value - else: - value = dpk - pvalue = listify(value) - if key in request.vars: - gvalue = listify(request.vars[key]) - if ISLE25: - value = pvalue + gvalue - elif is_multipart: - pvalue = pvalue[len(gvalue):] - else: - pvalue = pvalue[:-len(gvalue)] - request.vars[key] = value - if len(pvalue): - request.post_vars[key] = (len(pvalue) > - 1 and pvalue) or pvalue[0] - if is_json and isinstance(json_vars,dict): - # update post_vars with what was posted as json - request.post_vars.update(json_vars) - - def wsgibase(environ, responder): """ this is the gluon wsgi application. the first function called when a page @@ -435,15 +298,15 @@ def wsgibase(environ, responder): [a-zA-Z0-9_] - file and sub may also contain '-', '=', '.' and '/' """ - + eget = environ.get current.__dict__.clear() - request = Request() + request = Request(environ) response = Response() session = Session() env = request.env - env.web2py_path = global_settings.applications_parent + #env.web2py_path = global_settings.applications_parent env.web2py_version = web2py_version - env.update(global_settings) + #env.update(global_settings) static_file = False try: try: @@ -462,8 +325,7 @@ def wsgibase(environ, responder): response.status = env.web2py_status_code or response.status if static_file: - if environ.get('QUERY_STRING', '').startswith( - 'attachment'): + if eget('QUERY_STRING', '').startswith('attachment'): response.headers['Content-Disposition'] \ = 'attachment' if version: @@ -472,6 +334,7 @@ def wsgibase(environ, responder): 'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT' response.stream(static_file, request=request) + # ################################################## # fill in request items # ################################################## @@ -485,7 +348,7 @@ def wsgibase(environ, responder): local_hosts.add(socket.gethostname()) local_hosts.add(fqdn) local_hosts.update([ - addrinfo[4][0] for addrinfo + addrinfo[4][0] for addrinfo in getipaddrinfo(fqdn)]) if env.server_name: local_hosts.add(env.server_name) @@ -508,7 +371,8 @@ def wsgibase(environ, responder): is_local = env.remote_addr in local_hosts, is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \ request.env.http_x_forwarded_proto in HTTPS_SCHEMES \ - or env.https == 'on') + or env.https == 'on' + ) request.compute_uuid() # requires client request.url = environ['PATH_INFO'] @@ -544,7 +408,7 @@ def wsgibase(environ, responder): # get the GET and POST data # ################################################## - parse_get_post_vars(request, environ) + #parse_get_post_vars(request, environ) # ################################################## # expose wsgi hooks for convenience diff --git a/gluon/shell.py b/gluon/shell.py index 54a52f47..92d59291 100644 --- a/gluon/shell.py +++ b/gluon/shell.py @@ -38,7 +38,7 @@ def enable_autocomplete_and_history(adir,env): except ImportError: pass else: - readline.parse_and_bind("bind ^I rl_complete" + readline.parse_and_bind("bind ^I rl_complete" if sys.platform == 'darwin' else "tab: complete") history_file = os.path.join(adir,'.pythonhistory') @@ -71,7 +71,7 @@ def exec_environment( """ if request is None: - request = Request() + request = Request({}) if response is None: response = Response() if session is None: @@ -116,7 +116,7 @@ def env( web2py environment. """ - request = Request() + request = Request({}) response = Response() session = Session() request.application = a