Flask update

libs/werkzeug/__init__.py

@@ -19,7 +19,7 @@ import sys
 
 
 # the version.  Usually set automatically by a script.
-__version__ = '0.8-dev'
+__version__ = '0.9-dev'
 
 
 # This import magic raises concerns quite often which is why the implementation

libs/werkzeug/contrib/cache.py

@@ -65,9 +65,14 @@ except ImportError:
     from md5 import new as md5
 from itertools import izip
 from time import time
-from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL
 from werkzeug.posixemulation import rename
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+
 def _items(mappingorseq):
     """Wrapper for efficient iteration over mappings represented by dicts
     or sequences::
@@ -82,6 +87,7 @@ def _items(mappingorseq):
     return mappingorseq.iteritems() if hasattr(mappingorseq, 'iteritems') \
         else mappingorseq
 
+
 class BaseCache(object):
     """Baseclass for the cache systems.  All the cache systems implement this
     API or a superset of it.
@@ -242,20 +248,22 @@ class SimpleCache(BaseCache):
         now = time()
         expires, value = self._cache.get(key, (0, None))
         if expires > time():
-            return loads(value)
+            return pickle.loads(value)
 
     def set(self, key, value, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
         self._prune()
-        self._cache[key] = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
+        self._cache[key] = (time() + timeout, pickle.dumps(value,
+            pickle.HIGHEST_PROTOCOL))
 
     def add(self, key, value, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
         if len(self._cache) > self._threshold:
             self._prune()
-        item = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
+        item = (time() + timeout, pickle.dumps(value,
+            pickle.HIGHEST_PROTOCOL))
         self._cache.setdefault(key, item)
 
     def delete(self, key):
@@ -267,9 +275,10 @@ _test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match
 class MemcachedCache(BaseCache):
     """A cache that uses memcached as backend.
 
-    The first argument can either be a list or tuple of server addresses
-    in which case Werkzeug tries to import the memcache module and connect
-    to it, or an object that resembles the API of a :class:`memcache.Client`.
+    The first argument can either be an object that resembles the API of a
+    :class:`memcache.Client` or a tuple/list of server addresses. In the
+    event that a tuple/list is passed, Werkzeug tries to import the best
+    available memcache library.
 
     Implementation notes:  This cache backend works around some limitations in
     memcached to simplify the interface.  For example unicode keys are encoded
@@ -289,47 +298,19 @@ class MemcachedCache(BaseCache):
                        different prefix.
     """
 
-    def __init__(self, servers, default_timeout=300, key_prefix=None):
+    def __init__(self, servers=None, default_timeout=300, key_prefix=None):
         BaseCache.__init__(self, default_timeout)
-        if isinstance(servers, (list, tuple)):
-            is_cmemcached = is_cmemcache = is_pylibmc = False
-            try:
-                import cmemcached as memcache
-                is_cmemcached = True
-            except ImportError:
-                try:
-                    import cmemcache as memcache
-                    is_cmemcache = True
-                except ImportError:
-                    try:
-                        import memcache
-                        is_cmemcache = False
-                        is_pylibmc = False
-                    except ImportError:
-                        try:
-                            import pylibmc as memcache
-                            is_cmemcache = False
-                            is_pylibmc = True
-                        except ImportError:
-                            raise RuntimeError('no memcache module found')
-
-            if is_cmemcache:
-                # cmemcache has a bug that debuglog is not defined for the
-                # client.  Whenever pickle fails you get a weird
-                # AttributeError.
-                client = memcache.Client(map(str, servers))
-                try:
-                    client.debuglog = lambda *a: None
-                except Exception:
-                    pass
-            elif is_pylibmc or is_cmemcached:
-                client = memcache.Client(servers, False)
-            else:
-                client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
+        if servers is None or isinstance(servers, (list, tuple)):
+            if servers is None:
+                servers = ['127.0.0.1:11211']
+            self._client = self.import_preferred_memcache_lib(servers)
+            if self._client is None:
+                raise RuntimeError('no memcache module found')
         else:
-            client = servers
+            # NOTE: servers is actually an already initialized memcache
+            # client.
+            self._client = servers
 
-        self._client = client
         self.key_prefix = key_prefix
 
     def get(self, key):
@@ -356,9 +337,6 @@ class MemcachedCache(BaseCache):
                 encoded_key = self.key_prefix + encoded_key
             if _test_memcached_key(key):
                 key_mapping[encoded_key] = key
-        # the keys call here is important because otherwise cmemcache
-        # does ugly things.  What exactly I don't know, I think it does
-        # Py_DECREF but quite frankly I don't care.
         d = rv = self._client.get_multi(key_mapping.keys())
         if have_encoded_keys or self.key_prefix:
             rv = {}
@@ -440,23 +418,32 @@ class MemcachedCache(BaseCache):
             key = self.key_prefix + key
         self._client.decr(key, delta)
 
+    def import_preferred_memcache_lib(self, servers):
+        """Returns an initialized memcache client.  Used by the constructor."""
+        try:
+            import pylibmc
+        except ImportError:
+            pass
+        else:
+            return pylibmc.Client(servers)
 
-class GAEMemcachedCache(MemcachedCache):
-    """Connects to the Google appengine memcached Cache.
+        try:
+            from google.appengine.api import memcache
+        except ImportError:
+            pass
+        else:
+            return memcache.Client()
 
-    :param default_timeout: the default timeout that is used if no timeout is
-                            specified on :meth:`~BaseCache.set`.
-    :param key_prefix: a prefix that is added before all keys.  This makes it
-                       possible to use the same memcached server for different
-                       applications.  Keep in mind that
-                       :meth:`~BaseCache.clear` will also clear keys with a
-                       different prefix.
-    """
+        try:
+            import memcache
+        except ImportError:
+            pass
+        else:
+            return memcache.Client(servers)
 
-    def __init__(self, default_timeout=300, key_prefix=None):
-        from google.appengine.api import memcache
-        MemcachedCache.__init__(self, memcache.Client(),
-                                default_timeout, key_prefix)
+
+# backwards compatibility
+GAEMemcachedCache = MemcachedCache
 
 
 class RedisCache(BaseCache):
@@ -470,64 +457,109 @@ class RedisCache(BaseCache):
 
     .. versionadded:: 0.7
 
+    .. versionadded:: 0.8
+       `key_prefix` was added.
+
+    .. versionchanged:: 0.8
+       This cache backend now properly serializes objects.
+
     :param host: address of the Redis server or an object which API is
                  compatible with the official Python Redis client (redis-py).
     :param port: port number on which Redis server listens for connections
     :param default_timeout: the default timeout that is used if no timeout is
                             specified on :meth:`~BaseCache.set`.
-
+    :param key_prefix: A prefix that should be added to all keys.
     """
-    def __init__(self, host='localhost', port=6379, default_timeout=300):
+
+    def __init__(self, host='localhost', port=6379, password=None,
+                 default_timeout=300, key_prefix=None):
         BaseCache.__init__(self, default_timeout)
         if isinstance(host, basestring):
             try:
                 import redis
             except ImportError:
                 raise RuntimeError('no redis module found')
-            self._client = redis.Redis(host=host, port=port)
+            self._client = redis.Redis(host=host, port=port, password=password)
         else:
             self._client = host
+        self.key_prefix = key_prefix or ''
+
+    def dump_object(self, value):
+        """Dumps an object into a string for redis.  By default it serializes
+        integers as regular string and pickle dumps everything else.
+        """
+        if isinstance(value, (int, long)):
+            return str(value)
+        return '!' + pickle.dumps(value)
+
+    def load_object(self, value):
+        """The reversal of :meth:`dump_object`.  This might be callde with
+        None.
+        """
+        if value is None:
+            return None
+        if value.startswith('!'):
+            return pickle.loads(value[1:])
+        try:
+            return int(value)
+        except ValueError:
+            # before 0.8 we did not have serialization.  Still support that.
+            return value
 
     def get(self, key):
-        return self._client.get(key)
+        return self.load_object(self._client.get(self.key_prefix + key))
 
     def get_many(self, *keys):
-        return self._client.mget(keys)
+        if self.key_prefix:
+            keys = [self.key_prefix + key for key in keys]
+        return [self.load_object(x) for x in self._client.mget(keys)]
 
     def set(self, key, value, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
-        self._client.setex(key, value, timeout)
+        dump = self.dump_object(value)
+        self._client.setex(self.key_prefix + key, dump, timeout)
 
     def add(self, key, value, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
-        added = self._client.setnx(key, value)
+        dump = self.dump_object(value)
+        added = self._client.setnx(self.key_prefix + key, dump)
         if added:
-            self._client.expire(key, timeout)
+            self._client.expire(self.key_prefix + key, timeout)
 
     def set_many(self, mapping, timeout=None):
         if timeout is None:
             timeout = self.default_timeout
         pipe = self._client.pipeline()
         for key, value in _items(mapping):
-            pipe.setex(key, value, timeout)
+            dump = self.dump_object(value)
+            pipe.setex(self.key_prefix + key, dump, timeout)
         pipe.execute()
 
     def delete(self, key):
-        self._client.delete(key)
+        self._client.delete(self.key_prefix + key)
 
     def delete_many(self, *keys):
+        if not keys:
+            return
+        if self.key_prefix:
+            keys = [self.key_prefix + key for key in keys]
         self._client.delete(*keys)
 
     def clear(self):
-        self._client.flushdb()
+        if self.key_prefix:
+            keys = self._client.keys(self.key_prefix + '*')
+            if keys:
+                self._client.delete(*keys)
+        else:
+            self._client.flushdb()
 
     def inc(self, key, delta=1):
-        return self._client.incr(key, delta)
+        return self._client.incr(self.key_prefix + key, delta)
 
     def dec(self, key, delta=1):
-        return self._client.decr(key, delta)
+        return self._client.decr(self.key_prefix + key, delta)
 
 
 class FileSystemCache(BaseCache):
@@ -571,7 +603,7 @@ class FileSystemCache(BaseCache):
                 try:
                     try:
                         f = open(fname, 'rb')
-                        expires = load(f)
+                        expires = pickle.load(f)
                         remove = expires <= now or idx % 3 == 0
                     finally:
                         if f is not None:
@@ -600,8 +632,8 @@ class FileSystemCache(BaseCache):
         try:
             f = open(filename, 'rb')
             try:
-                if load(f) >= time():
-                    return load(f)
+                if pickle.load(f) >= time():
+                    return pickle.load(f)
             finally:
                 f.close()
             os.remove(filename)
@@ -623,8 +655,8 @@ class FileSystemCache(BaseCache):
                                        dir=self._path)
             f = os.fdopen(fd, 'wb')
             try:
-                dump(int(time() + timeout), f, 1)
-                dump(value, f, HIGHEST_PROTOCOL)
+                pickle.dump(int(time() + timeout), f, 1)
+                pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
             finally:
                 f.close()
             rename(tmp, filename)
@@ -637,4 +669,3 @@ class FileSystemCache(BaseCache):
             os.remove(self._get_filename(key))
         except (IOError, OSError):
             pass
-

libs/werkzeug/contrib/fixers.py

@@ -95,18 +95,33 @@ class ProxyFix(object):
     def __init__(self, app):
         self.app = app
 
+    def get_remote_addr(self, forwarded_for):
+        """Selects the new remote addr from the given list of ips in
+        X-Forwarded-For.  By default the first one is picked.
+
+        .. versionadded:: 0.8
+        """
+        if forwarded_for:
+            return forwarded_for[0]
+
     def __call__(self, environ, start_response):
         getter = environ.get
+        forwarded_proto = getter('HTTP_X_FORWARDED_PROTO', '')
         forwarded_for = getter('HTTP_X_FORWARDED_FOR', '').split(',')
         forwarded_host = getter('HTTP_X_FORWARDED_HOST', '')
         environ.update({
-            'werkzeug.proxy_fix.orig_remote_addr':  getter('REMOTE_ADDR'),
-            'werkzeug.proxy_fix.orig_http_host':    getter('HTTP_HOST')
+            'werkzeug.proxy_fix.orig_wsgi_url_scheme':  getter('wsgi.url_scheme'),
+            'werkzeug.proxy_fix.orig_remote_addr':      getter('REMOTE_ADDR'),
+            'werkzeug.proxy_fix.orig_http_host':        getter('HTTP_HOST')
         })
-        if forwarded_for:
-            environ['REMOTE_ADDR'] = forwarded_for[0].strip()
+        forwarded_for = [x for x in [x.strip() for x in forwarded_for] if x]
+        remote_addr = self.get_remote_addr(forwarded_for)
+        if remote_addr is not None:
+            environ['REMOTE_ADDR'] = remote_addr
         if forwarded_host:
             environ['HTTP_HOST'] = forwarded_host
+        if forwarded_proto:
+            environ['wsgi.url_scheme'] = forwarded_proto
         return self.app(environ, start_response)
 
 

libs/werkzeug/contrib/iterio.py

@@ -127,25 +127,22 @@ class IterI(IterIO):
         if greenlet is None:
             raise RuntimeError('IterI requires greenlet support')
         stream = object.__new__(cls)
-        stream.__init__(greenlet.getcurrent())
+        stream._parent = greenlet.getcurrent()
+        stream._buffer = []
+        stream.closed = False
+        stream.pos = 0
 
         def run():
             func(stream)
             stream.flush()
 
-        g = greenlet(run, stream._parent)
+        g = greenlet.greenlet(run, stream._parent)
         while 1:
             rv = g.switch()
             if not rv:
                 return
             yield rv[0]
 
-    def __init__(self, parent):
-        self._parent = parent
-        self._buffer = []
-        self.closed = False
-        self.pos = 0
-
     def close(self):
         if not self.closed:
             self.closed = True
@@ -171,13 +168,12 @@ class IterO(IterIO):
     """Iter output.  Wrap an iterator and give it a stream like interface."""
 
     def __new__(cls, gen):
-        return object.__new__(cls)
-
-    def __init__(self, gen):
+        self = object.__new__(cls)
         self._gen = gen
         self._buf = ''
         self.closed = False
         self.pos = 0
+        return self
 
     def __iter__(self):
         return self

libs/werkzeug/contrib/securecookie.py

@@ -88,7 +88,6 @@ r"""
     :copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
-import sys
 import cPickle as pickle
 from hmac import new as hmac
 from time import time
@@ -98,21 +97,7 @@ from werkzeug.contrib.sessions import ModificationTrackingDict
 from werkzeug.security import safe_str_cmp
 
 
-# rather ugly way to import the correct hash method.  Because
-# hmac either accepts modules with a new method (sha, md5 etc.)
-# or a hashlib factory function we have to figure out what to
-# pass to it.  If we have 2.5 or higher (so not 2.4 with a
-# custom hashlib) we import from hashlib and fail if it does
-# not exist (have seen that in old OS X versions).
-# in all other cases the now deprecated sha module is used.
-_default_hash = None
-if sys.version_info >= (2, 5):
-    try:
-        from hashlib import sha1 as _default_hash
-    except ImportError:
-        pass
-if _default_hash is None:
-    import sha as _default_hash
+from hashlib import sha1 as _default_hash
 
 
 class UnquoteError(Exception):

libs/werkzeug/contrib/sessions.py

@@ -58,10 +58,7 @@ import tempfile
 from os import path
 from time import time
 from random import random
-try:
-    from hashlib import sha1
-except ImportError:
-    from sha import new as sha1
+from hashlib import sha1
 from cPickle import dump, load, HIGHEST_PROTOCOL
 
 from werkzeug.datastructures import CallbackDict

libs/werkzeug/datastructures.py

@@ -50,6 +50,14 @@ class ImmutableListMixin(object):
     :private:
     """
 
+    _hash_cache = None
+
+    def __hash__(self):
+        if self._hash_cache is not None:
+            return self._hash_cache
+        rv = self._hash_cache = hash(tuple(self))
+        return rv
+
     def __reduce_ex__(self, protocol):
         return type(self), (list(self),)
 
@@ -107,6 +115,8 @@ class ImmutableDictMixin(object):
 
     :private:
     """
+    _hash_cache = None
+
     @classmethod
     def fromkeys(cls, keys, value=None):
         instance = super(cls, cls).__new__(cls)
@@ -116,6 +126,15 @@ class ImmutableDictMixin(object):
     def __reduce_ex__(self, protocol):
         return type(self), (dict(self),)
 
+    def _iter_hashitems(self):
+        return self.iteritems()
+
+    def __hash__(self):
+        if self._hash_cache is not None:
+            return self._hash_cache
+        rv = self._hash_cache = hash(frozenset(self._iter_hashitems()))
+        return rv
+
     def setdefault(self, key, default=None):
         is_immutable(self)
 
@@ -149,6 +168,9 @@ class ImmutableMultiDictMixin(ImmutableDictMixin):
     def __reduce_ex__(self, protocol):
         return type(self), (self.items(multi=True),)
 
+    def _iter_hashitems(self):
+        return self.iteritems(multi=True)
+
     def add(self, key, value):
         is_immutable(self)
 
@@ -561,6 +583,9 @@ class MultiDict(TypeConversionDict):
         except KeyError, e:
             raise BadRequestKeyError(str(e))
 
+    def __copy__(self):
+        return self.copy()
+
     def __repr__(self):
         return '%s(%r)' % (self.__class__.__name__, self.items(multi=True))
 
@@ -1032,8 +1057,14 @@ class Headers(object):
         """
         if kw:
             _value = _options_header_vkw(_value, kw)
+        self._validate_value(_value)
         self._list.append((_key, _value))
 
+    def _validate_value(self, value):
+        if isinstance(value, basestring) and ('\n' in value or '\r' in value):
+            raise ValueError('Detected newline in header value.  This is '
+                'a potential security problem')
+
     def add_header(self, _key, _value, **_kw):
         """Add a new header tuple to the list.
 
@@ -1063,18 +1094,21 @@ class Headers(object):
         """
         if kw:
             _value = _options_header_vkw(_value, kw)
+        self._validate_value(_value)
         if not self._list:
-            return self.add(_key, _value)
-        lc_key = _key.lower()
-        for idx, (old_key, old_value) in enumerate(self._list):
-            if old_key.lower() == lc_key:
+            self._list.append((_key, _value))
+            return
+        listiter = iter(self._list)
+        ikey = _key.lower()
+        for idx, (old_key, old_value) in enumerate(listiter):
+            if old_key.lower() == ikey:
                 # replace first ocurrence
                 self._list[idx] = (_key, _value)
                 break
         else:
-            return self.add(_key, _value)
-        self._list[idx + 1:] = [(k, v) for k, v in self._list[idx + 1:]
-                                if k.lower() != lc_key]
+            self._list.append((_key, _value))
+            return
+        self._list[idx + 1:] = [t for t in listiter if t[0].lower() != ikey]
 
     def setdefault(self, key, value):
         """Returns the value for the key if it is in the dict, otherwise it
@@ -1092,6 +1126,7 @@ class Headers(object):
     def __setitem__(self, key, value):
         """Like :meth:`set` but also supports index/slice based setting."""
         if isinstance(key, (slice, int, long)):
+            self._validate_value(value)
             self._list[key] = value
         else:
             self.set(key, value)
@@ -1102,14 +1137,8 @@ class Headers(object):
 
         :return: list
         """
-        result = []
-        for k, v in self:
-            if isinstance(v, unicode):
-                v = v.encode(charset)
-            else:
-                v = str(v)
-            result.append((k, v))
-        return result
+        return [(k, isinstance(v, unicode) and v.encode(charset) or str(v))
+                for k, v in self]
 
     def copy(self):
         return self.__class__(self._list)
@@ -1133,7 +1162,9 @@ class Headers(object):
 
 
 class ImmutableHeadersMixin(object):
-    """Makes a :class:`Headers` immutable.
+    """Makes a :class:`Headers` immutable.  We do not mark them as
+    hashable though since the only usecase for this datastructure
+    in Werkzeug is a view on a mutable structure.
 
     .. versionadded:: 0.5
 
@@ -1426,6 +1457,9 @@ class ImmutableOrderedMultiDict(ImmutableMultiDictMixin, OrderedMultiDict):
     .. versionadded:: 0.6
     """
 
+    def _iter_hashitems(self):
+        return enumerate(self.iteritems(multi=True))
+
     def copy(self):
         """Return a shallow mutable copy of this object.  Keep in mind that
         the standard library's :func:`copy` function is a no-op for this class
@@ -2045,7 +2079,7 @@ class ETags(object):
         return etag in self._strong
 
     def __nonzero__(self):
-        return bool(self.star_tag or self._strong)
+        return bool(self.star_tag or self._strong or self._weak)
 
     def __str__(self):
         return self.to_header()
@@ -2422,7 +2456,7 @@ class FileStorage(object):
     """
 
     def __init__(self, stream=None, filename=None, name=None,
-                 content_type='application/octet-stream', content_length=-1,
+                 content_type=None, content_length=None,
                  headers=None):
         self.name = name
         self.stream = stream or _empty_stream
@@ -2437,17 +2471,29 @@ class FileStorage(object):
                 filename = None
 
         self.filename = filename
-        self.content_type = content_type
-        self.content_length = content_length
         if headers is None:
             headers = Headers()
         self.headers = headers
+        if content_type is not None:
+            headers['Content-Type'] = content_type
+        if content_length is not None:
+            headers['Content-Length'] = str(content_length)
 
     def _parse_content_type(self):
         if not hasattr(self, '_parsed_content_type'):
             self._parsed_content_type = \
                 parse_options_header(self.content_type)
 
+    @property
+    def content_type(self):
+        """The file's content type.  Usually not available"""
+        return self.headers.get('content-type')
+
+    @property
+    def content_length(self):
+        """The file's content length.  Usually not available"""
+        return int(self.headers.get('content-length') or 0)
+
     @property
     def mimetype(self):
         """Like :attr:`content_type` but without parameters (eg, without

libs/werkzeug/exceptions.py

@@ -179,6 +179,21 @@ class BadRequest(HTTPException):
     )
 
 
+class ClientDisconnected(BadRequest):
+    """Internal exception that is raised if Werkzeug detects a disconnected
+    client.  Since the client is already gone at that point attempting to
+    send the error message to the client might not work and might ultimately
+    result in another exception in the server.  Mainly this is here so that
+    it is silenced by default as far as Werkzeug is concerned.
+
+    Since disconnections cannot be reliably detected and are unspecified
+    by WSGI to a large extend this might or might not be raised if a client
+    is gone.
+
+    .. versionadded:: 0.8
+    """
+
+
 class Unauthorized(HTTPException):
     """*401* `Unauthorized`
 

libs/werkzeug/formparser.py

@@ -13,9 +13,10 @@ import re
 from cStringIO import StringIO
 from tempfile import TemporaryFile
 from itertools import chain, repeat
+from functools import update_wrapper
 
 from werkzeug._internal import _decode_unicode, _empty_stream
-from werkzeug.urls import url_decode
+from werkzeug.urls import url_decode_stream
 from werkzeug.wsgi import LimitedStream, make_line_iter
 from werkzeug.exceptions import RequestEntityTooLarge
 from werkzeug.datastructures import Headers, FileStorage, MultiDict
@@ -54,8 +55,7 @@ def parse_form_data(environ, stream_factory=None, charset='utf-8',
     mimetype is unknown the input stream is wrapped and returned as first
     argument, else the stream is empty.
 
-    This function does not raise exceptions, even if the input data is
-    malformed.
+    This is a shortcut for the common usage of :class:`FormDataParser`.
 
     Have a look at :ref:`dealing-with-request-data` for more details.
 
@@ -86,56 +86,140 @@ def parse_form_data(environ, stream_factory=None, charset='utf-8',
     :param silent: If set to False parsing errors will not be caught.
     :return: A tuple in the form ``(stream, form, files)``.
     """
-    content_type, extra = parse_options_header(environ.get('CONTENT_TYPE', ''))
-    try:
-        content_length = int(environ['CONTENT_LENGTH'])
-    except (KeyError, ValueError):
-        content_length = 0
+    return FormDataParser(stream_factory, charset, errors,
+                          max_form_memory_size, max_content_length,
+                          cls, silent).parse_from_environ(environ)
 
-    if cls is None:
-        cls = MultiDict
 
-    if max_content_length is not None and content_length > max_content_length:
-        raise RequestEntityTooLarge()
-
-    stream = _empty_stream
-    files = ()
-
-    if content_type == 'multipart/form-data':
+def exhaust_stream(f):
+    """Helper decorator for methods that exhausts the stream on return."""
+    def wrapper(self, stream, *args, **kwargs):
         try:
-            form, files = parse_multipart(environ['wsgi.input'],
-                                          extra.get('boundary'),
-                                          content_length, stream_factory,
-                                          charset, errors,
-                                          max_form_memory_size=max_form_memory_size)
-        except ValueError, e:
-            if not silent:
-                raise
-            form = cls()
-        else:
-            form = cls(form)
-    elif content_type == 'application/x-www-form-urlencoded' or \
-         content_type == 'application/x-url-encoded':
-        if max_form_memory_size is not None and \
-           content_length > max_form_memory_size:
-            raise RequestEntityTooLarge()
-        form = url_decode(environ['wsgi.input'].read(content_length),
-                          charset, errors=errors, cls=cls)
-    else:
-        form = cls()
-        stream = LimitedStream(environ['wsgi.input'], content_length)
-
-    return stream, form, cls(files)
+            return f(self, stream, *args, **kwargs)
+        finally:
+            stream.exhaust()
+    return update_wrapper(wrapper, f)
 
 
-def _fix_ie_filename(filename):
-    """Internet Explorer 6 transmits the full file name if a file is
-    uploaded.  This function strips the full path if it thinks the
-    filename is Windows-like absolute.
+class FormDataParser(object):
+    """This class implements parsing of form data for Werkzeug.  By itself
+    it can parse multipart and url encoded form data.  It can be subclasses
+    and extended but for most mimetypes it is a better idea to use the
+    untouched stream and expose it as separate attributes on a request
+    object.
+
+    .. versionadded:: 0.8
+
+    :param stream_factory: An optional callable that returns a new read and
+                           writeable file descriptor.  This callable works
+                           the same as :meth:`~BaseResponse._get_file_stream`.
+    :param charset: The character set for URL and url encoded form data.
+    :param errors: The encoding error behavior.
+    :param max_form_memory_size: the maximum number of bytes to be accepted for
+                           in-memory stored form data.  If the data
+                           exceeds the value specified an
+                           :exc:`~exceptions.RequestURITooLarge`
+                           exception is raised.
+    :param max_content_length: If this is provided and the transmitted data
+                               is longer than this value an
+                               :exc:`~exceptions.RequestEntityTooLarge`
+                               exception is raised.
+    :param cls: an optional dict class to use.  If this is not specified
+                       or `None` the default :class:`MultiDict` is used.
+    :param silent: If set to False parsing errors will not be caught.
     """
-    if filename[1:3] == ':\\' or filename[:2] == '\\\\':
-        return filename.split('\\')[-1]
-    return filename
+
+    def __init__(self, stream_factory=None, charset='utf-8',
+                 errors='replace', max_form_memory_size=None,
+                 max_content_length=None, cls=None,
+                 silent=True):
+        if stream_factory is None:
+            stream_factory = default_stream_factory
+        self.stream_factory = stream_factory
+        self.charset = charset
+        self.errors = errors
+        self.max_form_memory_size = max_form_memory_size
+        self.max_content_length = max_content_length
+        if cls is None:
+            cls = MultiDict
+        self.cls = cls
+        self.silent = silent
+
+    def get_parse_func(self, mimetype, options):
+        return self.parse_functions.get(mimetype)
+
+    def parse_from_environ(self, environ):
+        """Parses the information from the environment as form data.
+
+        :param environ: the WSGI environment to be used for parsing.
+        :return: A tuple in the form ``(stream, form, files)``.
+        """
+        content_type = environ.get('CONTENT_TYPE', '')
+        mimetype, options = parse_options_header(content_type)
+        try:
+            content_length = int(environ['CONTENT_LENGTH'])
+        except (KeyError, ValueError):
+            content_length = 0
+        stream = environ['wsgi.input']
+        return self.parse(stream, mimetype, content_length, options)
+
+    def parse(self, stream, mimetype, content_length, options=None):
+        """Parses the information from the given stream, mimetype,
+        content length and mimetype parameters.
+
+        :param stream: an input stream
+        :param mimetype: the mimetype of the data
+        :param content_length: the content length of the incoming data
+        :param options: optional mimetype parameters (used for
+                        the multipart boundary for instance)
+        :return: A tuple in the form ``(stream, form, files)``.
+        """
+        if self.max_content_length is not None and \
+           content_length > self.max_content_length:
+            raise RequestEntityTooLarge()
+        if options is None:
+            options = {}
+        input_stream = LimitedStream(stream, content_length)
+
+        parse_func = self.get_parse_func(mimetype, options)
+        if parse_func is not None:
+            try:
+                return parse_func(self, input_stream, mimetype,
+                                  content_length, options)
+            except ValueError:
+                if not self.silent:
+                    raise
+        return input_stream, self.cls(), self.cls()
+
+    @exhaust_stream
+    def _parse_multipart(self, stream, mimetype, content_length, options):
+        parser = MultiPartParser(self.stream_factory, self.charset, self.errors,
+                                 max_form_memory_size=self.max_form_memory_size,
+                                 cls=self.cls)
+        form, files = parser.parse(stream, options.get('boundary'),
+                                   content_length)
+        return _empty_stream, form, files
+
+    @exhaust_stream
+    def _parse_urlencoded(self, stream, mimetype, content_length, options):
+        if self.max_form_memory_size is not None and \
+           content_length > self.max_form_memory_size:
+            raise RequestEntityTooLarge()
+        form = url_decode_stream(stream, self.charset,
+                                 errors=self.errors, cls=self.cls)
+        return _empty_stream, form, self.cls()
+
+    #: mapping of mimetypes to parsing functions
+    parse_functions = {
+        'multipart/form-data':                  _parse_multipart,
+        'application/x-www-form-urlencoded':    _parse_urlencoded,
+        'application/x-url-encoded':            _parse_urlencoded
+    }
+
+
+def is_valid_multipart_boundary(boundary):
+    """Checks if the string given is a valid multipart boundary."""
+    return _multipart_boundary_re.match(boundary) is not None
 
 
 def _line_parse(line):
@@ -149,97 +233,152 @@ def _line_parse(line):
     return line, False
 
 
-def _find_terminator(iterator):
-    """The terminator might have some additional newlines before it.
-    There is at least one application that sends additional newlines
-    before headers (the python setuptools package).
+def parse_multipart_headers(iterable):
+    """Parses multipart headers from an iterable that yields lines (including
+    the trailing newline symbol.  The iterable has to be newline terminated:
+
+    >>> parse_multipart_headers(['Foo: Bar\r\n', 'Test: Blub\r\n',
+    ...                          '\r\n', 'More data'])
+    Headers([('Foo', 'Bar'), ('Test', 'Blub')])
+
+    :param iterable: iterable of strings that are newline terminated
     """
-    for line in iterator:
+    result = []
+    for line in iterable:
+        line, line_terminated = _line_parse(line)
+        if not line_terminated:
+            raise ValueError('unexpected end of line in multipart header')
         if not line:
             break
-        line = line.strip()
-        if line:
-            return line
-    return ''
+        elif line[0] in ' \t' and result:
+            key, value = result[-1]
+            result[-1] = (key, value + '\n ' + line[1:])
+        else:
+            parts = line.split(':', 1)
+            if len(parts) == 2:
+                result.append((parts[0].strip(), parts[1].strip()))
+
+    # we link the list to the headers, no need to create a copy, the
+    # list was not shared anyways.
+    return Headers.linked(result)
 
 
-def is_valid_multipart_boundary(boundary):
-    """Checks if the string given is a valid multipart boundary."""
-    return _multipart_boundary_re.match(boundary) is not None
+class MultiPartParser(object):
 
+    def __init__(self, stream_factory=None, charset='utf-8', errors='replace',
+                 max_form_memory_size=None, cls=None, buffer_size=10 * 1024):
+        self.stream_factory = stream_factory
+        self.charset = charset
+        self.errors = errors
+        self.max_form_memory_size = max_form_memory_size
+        if stream_factory is None:
+            stream_factory = default_stream_factory
+        if cls is None:
+            cls = MultiDict
+        self.cls = cls
 
-def parse_multipart(file, boundary, content_length, stream_factory=None,
-                    charset='utf-8', errors='replace', buffer_size=10 * 1024,
-                    max_form_memory_size=None):
-    """Parse a multipart/form-data stream.  This is invoked by
-    :func:`utils.parse_form_data` if the content type matches.  Currently it
-    exists for internal usage only, but could be exposed as separate
-    function if it turns out to be useful and if we consider the API stable.
-    """
-    # XXX: this function does not support multipart/mixed.  I don't know of
-    #      any browser that supports this, but it should be implemented
-    #      nonetheless.
+        # make sure the buffer size is divisible by four so that we can base64
+        # decode chunk by chunk
+        assert buffer_size % 4 == 0, 'buffer size has to be divisible by 4'
+        # also the buffer size has to be at least 1024 bytes long or long headers
+        # will freak out the system
+        assert buffer_size >= 1024, 'buffer size has to be at least 1KB'
 
-    # make sure the buffer size is divisible by four so that we can base64
-    # decode chunk by chunk
-    assert buffer_size % 4 == 0, 'buffer size has to be divisible by 4'
-    # also the buffer size has to be at least 1024 bytes long or long headers
-    # will freak out the system
-    assert buffer_size >= 1024, 'buffer size has to be at least 1KB'
+        self.buffer_size = buffer_size
 
-    if stream_factory is None:
-        stream_factory = default_stream_factory
+    def _fix_ie_filename(self, filename):
+        """Internet Explorer 6 transmits the full file name if a file is
+        uploaded.  This function strips the full path if it thinks the
+        filename is Windows-like absolute.
+        """
+        if filename[1:3] == ':\\' or filename[:2] == '\\\\':
+            return filename.split('\\')[-1]
+        return filename
 
-    if not boundary:
-        raise ValueError('Missing boundary')
-    if not is_valid_multipart_boundary(boundary):
-        raise ValueError('Invalid boundary: %s' % boundary)
-    if len(boundary) > buffer_size: # pragma: no cover
-        # this should never happen because we check for a minimum size
-        # of 1024 and boundaries may not be longer than 200.  The only
-        # situation when this happen is for non debug builds where
-        # the assert i skipped.
-        raise ValueError('Boundary longer than buffer size')
+    def _find_terminator(self, iterator):
+        """The terminator might have some additional newlines before it.
+        There is at least one application that sends additional newlines
+        before headers (the python setuptools package).
+        """
+        for line in iterator:
+            if not line:
+                break
+            line = line.strip()
+            if line:
+                return line
+        return ''
 
-    total_content_length = content_length
-    next_part = '--' + boundary
-    last_part = next_part + '--'
+    def fail(self, message):
+        raise ValueError(message)
 
-    form = []
-    files = []
-    in_memory = 0
+    def get_part_encoding(self, headers):
+        transfer_encoding = headers.get('content-transfer-encoding')
+        if transfer_encoding is not None and \
+           transfer_encoding in _supported_multipart_encodings:
+            return transfer_encoding
 
-    # convert the file into a limited stream with iteration capabilities
-    file = LimitedStream(file, content_length)
-    iterator = chain(make_line_iter(file, buffer_size=buffer_size),
-                     _empty_string_iter)
+    def get_part_charset(self, headers):
+        # Figure out input charset for current part
+        content_type = headers.get('content-type')
+        if content_type:
+            mimetype, ct_params = parse_options_header(content_type)
+            return ct_params.get('charset', self.charset)
+        return self.charset
 
-    try:
-        terminator = _find_terminator(iterator)
+    def start_file_streaming(self, filename, headers, total_content_length):
+        filename = _decode_unicode(filename, self.charset, self.errors)
+        filename = self._fix_ie_filename(filename)
+        content_type = headers.get('content_type')
+        try:
+            content_length = int(headers['content-length'])
+        except (KeyError, ValueError):
+            content_length = 0
+        container = self.stream_factory(total_content_length, content_type,
+                                        filename, content_length)
+        return filename, container
+
+    def in_memory_threshold_reached(self, bytes):
+        raise RequestEntityTooLarge()
+
+    def validate_boundary(self, boundary):
+        if not boundary:
+            self.fail('Missing boundary')
+        if not is_valid_multipart_boundary(boundary):
+            self.fail('Invalid boundary: %s' % boundary)
+        if len(boundary) > self.buffer_size: # pragma: no cover
+            # this should never happen because we check for a minimum size
+            # of 1024 and boundaries may not be longer than 200.  The only
+            # situation when this happen is for non debug builds where
+            # the assert i skipped.
+            self.fail('Boundary longer than buffer size')
+
+    def parse(self, file, boundary, content_length):
+        next_part = '--' + boundary
+        last_part = next_part + '--'
+
+        form = []
+        files = []
+        in_memory = 0
+
+        iterator = chain(make_line_iter(file, limit=content_length,
+                                        buffer_size=self.buffer_size),
+                         _empty_string_iter)
+
+        terminator = self._find_terminator(iterator)
         if terminator != next_part:
-            raise ValueError('Expected boundary at start of multipart data')
+            self.fail('Expected boundary at start of multipart data')
 
         while terminator != last_part:
             headers = parse_multipart_headers(iterator)
+
             disposition = headers.get('content-disposition')
             if disposition is None:
-                raise ValueError('Missing Content-Disposition header')
+                self.fail('Missing Content-Disposition header')
             disposition, extra = parse_options_header(disposition)
+            transfer_encoding = self.get_part_encoding(headers)
             name = extra.get('name')
-
-            transfer_encoding = headers.get('content-transfer-encoding')
-            try_decode = transfer_encoding is not None and \
-                         transfer_encoding in _supported_multipart_encodings
-
             filename = extra.get('filename')
-
-            # Figure out input charset for current part
-            content_type = headers.get('content-type')
-            if content_type:
-                mimetype, ct_params = parse_options_header(content_type)
-                part_charset = ct_params.get("charset", charset)
-            else:
-                part_charset = charset
+            part_charset = self.get_part_charset(headers)
 
             # if no content type is given we stream into memory.  A list is
             # used as a temporary container.
@@ -247,41 +386,32 @@ def parse_multipart(file, boundary, content_length, stream_factory=None,
                 is_file = False
                 container = []
                 _write = container.append
-                guard_memory = max_form_memory_size is not None
+                guard_memory = self.max_form_memory_size is not None
 
             # otherwise we parse the rest of the headers and ask the stream
             # factory for something we can write in.
             else:
                 is_file = True
                 guard_memory = False
-                if filename is not None:
-                    filename = _fix_ie_filename(_decode_unicode(filename,
-                                                                charset,
-                                                                errors))
-                try:
-                    content_length = int(headers['content-length'])
-                except (KeyError, ValueError):
-                    content_length = 0
-                container = stream_factory(total_content_length, content_type,
-                                           filename, content_length)
+                filename, container = self.start_file_streaming(
+                    filename, headers, content_length)
                 _write = container.write
 
             buf = ''
             for line in iterator:
                 if not line:
-                    raise ValueError('unexpected end of stream')
+                    self.fail('unexpected end of stream')
 
                 if line[:2] == '--':
                     terminator = line.rstrip()
                     if terminator in (next_part, last_part):
                         break
 
-                if try_decode:
+                if transfer_encoding is not None:
                     try:
                         line = line.decode(transfer_encoding)
                     except Exception:
-                        raise ValueError('could not decode transfer '
-                                         'encoded chunk')
+                        self.fail('could not decode transfer encoded chunk')
 
                 # we have something in the buffer from the last iteration.
                 # this is usually a newline delimiter.
@@ -311,9 +441,8 @@ def parse_multipart(file, boundary, content_length, stream_factory=None,
                 # there is too much data in memory.
                 if guard_memory:
                     in_memory += len(line)
-                    if in_memory > max_form_memory_size:
-                        from werkzeug.exceptions import RequestEntityTooLarge
-                        raise RequestEntityTooLarge()
+                    if in_memory > self.max_form_memory_size:
+                        self.in_memory_threshold_reached(in_memory)
             else: # pragma: no cover
                 raise ValueError('unexpected end of part')
 
@@ -326,37 +455,9 @@ def parse_multipart(file, boundary, content_length, stream_factory=None,
             if is_file:
                 container.seek(0)
                 files.append((name, FileStorage(container, filename, name,
-                                                content_type,
-                                                content_length, headers)))
+                                                headers=headers)))
             else:
                 form.append((name, _decode_unicode(''.join(container),
-                                                   part_charset, errors)))
-    finally:
-        # make sure the whole input stream is read
-        file.exhaust()
+                                                   part_charset, self.errors)))
 
-    return form, files
-
-
-def parse_multipart_headers(iterable):
-    """Parses multipart headers from an iterable that yields lines (including
-    the trailing newline symbol.
-    """
-    result = []
-    for line in iterable:
-        line, line_terminated = _line_parse(line)
-        if not line_terminated:
-            raise ValueError('unexpected end of line in multipart header')
-        if not line:
-            break
-        elif line[0] in ' \t' and result:
-            key, value = result[-1]
-            result[-1] = (key, value + '\n ' + line[1:])
-        else:
-            parts = line.split(':', 1)
-            if len(parts) == 2:
-                result.append((parts[0].strip(), parts[1].strip()))
-
-    # we link the list to the headers, no need to create a copy, the
-    # list was not shared anyways.
-    return Headers.linked(result)
+        return self.cls(form), self.cls(files)

libs/werkzeug/http.py

@@ -50,7 +50,7 @@ _entity_headers = frozenset([
     'content-location', 'content-md5', 'content-range', 'content-type',
     'expires', 'last-modified'
 ])
-_hop_by_pop_headers = frozenset([
+_hop_by_hop_headers = frozenset([
     'connection', 'keep-alive', 'proxy-authenticate',
     'proxy-authorization', 'te', 'trailers', 'transfer-encoding',
     'upgrade'
@@ -357,10 +357,12 @@ def parse_authorization_header(value):
                                        'password': password})
     elif auth_type == 'digest':
         auth_map = parse_dict_header(auth_info)
-        for key in 'username', 'realm', 'nonce', 'uri', 'nc', 'cnonce', \
-                   'response':
+        for key in 'username', 'realm', 'nonce', 'uri', 'response':
             if not key in auth_map:
                 return
+        if 'qop' in auth_map:
+            if not auth_map.get('nc') or not auth_map.get('cnonce'):
+                return
         return Authorization('digest', auth_map)
 
 
@@ -708,7 +710,7 @@ def is_hop_by_hop_header(header):
     :param header: the header to test.
     :return: `True` if it's an entity header, `False` otherwise.
     """
-    return header.lower() in _hop_by_pop_headers
+    return header.lower() in _hop_by_hop_headers
 
 
 def parse_cookie(header, charset='utf-8', errors='replace',

libs/werkzeug/local.py

@@ -8,26 +8,19 @@
     :copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
-try:
-    from greenlet import getcurrent as get_current_greenlet
-except ImportError: # pragma: no cover
-    get_current_greenlet = int
-try:
-    from thread import get_ident as get_current_thread, allocate_lock
-except ImportError: # pragma: no cover
-    from dummy_thread import get_ident as get_current_thread, allocate_lock
-
 from werkzeug.wsgi import ClosingIterator
 from werkzeug._internal import _patch_wrapper
 
-
-# get the best ident function.  if greenlets are not installed we can
-# safely just use the builtin thread function and save a python methodcall
-# and the cost of calculating a hash.
-if get_current_greenlet is int: # pragma: no cover
-    get_ident = get_current_thread
-else:
-    get_ident = lambda: (get_current_thread(), get_current_greenlet())
+# since each thread has its own greenlet we can just use those as identifiers
+# for the context.  If greenlets are not available we fall back to the
+# current thread ident.
+try:
+    from greenlet import getcurrent as get_ident
+except ImportError: # pragma: no cover
+    try:
+        from thread import get_ident
+    except ImportError: # pragma: no cover
+        from dummy_thread import get_ident
 
 
 def release_local(local):
@@ -54,15 +47,14 @@ def release_local(local):
 
 
 class Local(object):
-    __slots__ = ('__storage__', '__lock__', '__ident_func__')
+    __slots__ = ('__storage__', '__ident_func__')
 
     def __init__(self):
         object.__setattr__(self, '__storage__', {})
-        object.__setattr__(self, '__lock__', allocate_lock())
         object.__setattr__(self, '__ident_func__', get_ident)
 
     def __iter__(self):
-        return self.__storage__.iteritems()
+        return iter(self.__storage__.items())
 
     def __call__(self, proxy):
         """Create a proxy for a name."""
@@ -79,15 +71,11 @@ class Local(object):
 
     def __setattr__(self, name, value):
         ident = self.__ident_func__()
-        self.__lock__.acquire()
+        storage = self.__storage__
         try:
-            storage = self.__storage__
-            if ident in storage:
-                storage[ident][name] = value
-            else:
-                storage[ident] = {name: value}
-        finally:
-            self.__lock__.release()
+            storage[ident][name] = value
+        except KeyError:
+            storage[ident] = {name: value}
 
     def __delattr__(self, name):
         try:
@@ -125,7 +113,6 @@ class LocalStack(object):
 
     def __init__(self):
         self._local = Local()
-        self._lock = allocate_lock()
 
     def __release_local__(self):
         self._local.__release_local__()
@@ -147,32 +134,24 @@ class LocalStack(object):
 
     def push(self, obj):
         """Pushes a new item to the stack"""
-        self._lock.acquire()
-        try:
-            rv = getattr(self._local, 'stack', None)
-            if rv is None:
-                self._local.stack = rv = []
-            rv.append(obj)
-            return rv
-        finally:
-            self._lock.release()
+        rv = getattr(self._local, 'stack', None)
+        if rv is None:
+            self._local.stack = rv = []
+        rv.append(obj)
+        return rv
 
     def pop(self):
         """Removes the topmost item from the stack, will return the
         old value or `None` if the stack was already empty.
         """
-        self._lock.acquire()
-        try:
-            stack = getattr(self._local, 'stack', None)
-            if stack is None:
-                return None
-            elif len(stack) == 1:
-                release_local(self._local)
-                return stack[-1]
-            else:
-                return stack.pop()
-        finally:
-            self._lock.release()
+        stack = getattr(self._local, 'stack', None)
+        if stack is None:
+            return None
+        elif len(stack) == 1:
+            release_local(self._local)
+            return stack[-1]
+        else:
+            return stack.pop()
 
     @property
     def top(self):

libs/werkzeug/routing.py

@@ -100,7 +100,7 @@ import posixpath
 from pprint import pformat
 from urlparse import urljoin
 
-from werkzeug.urls import url_encode, url_decode, url_quote
+from werkzeug.urls import url_encode, url_quote
 from werkzeug.utils import redirect, format_string
 from werkzeug.exceptions import HTTPException, NotFound, MethodNotAllowed
 from werkzeug._internal import _get_environ
@@ -115,7 +115,7 @@ _rule_re = re.compile(r'''
         (?:\((?P<args>.*?)\))?                  # converter arguments
         \:                                      # variable delimiter
     )?
-    (?P<variable>[a-zA-Z][a-zA-Z0-9_]*)         # variable name
+    (?P<variable>[a-zA-Z_][a-zA-Z0-9_]*)        # variable name
     >
 ''', re.VERBOSE)
 _simple_rule_re = re.compile(r'<([^>]+)>')
@@ -240,7 +240,7 @@ class RequestRedirect(HTTPException, RoutingException):
         self.new_url = new_url
 
     def get_response(self, environ):
-        return redirect(self.new_url, 301)
+        return redirect(self.new_url, self.code)
 
 
 class RequestSlash(RoutingException):
@@ -715,7 +715,7 @@ class Rule(RuleFactory):
                     return
                 processed.add(data)
             else:
-                add(data)
+                add(url_quote(data, self.map.charset, safe='/:|+'))
         domain_part, url = (u''.join(tmp)).split('|', 1)
 
         if append_unknown:
@@ -1112,6 +1112,7 @@ class Map(object):
         .. versionadded:: 0.8
            `query_args` can now also be a string.
         """
+        server_name = server_name.lower()
         if self.host_matching:
             if subdomain is not None:
                 raise RuntimeError('host matching enabled and a '
@@ -1120,6 +1121,8 @@ class Map(object):
             subdomain = self.default_subdomain
         if script_name is None:
             script_name = '/'
+        if isinstance(server_name, unicode):
+            server_name = server_name.encode('idna')
         return MapAdapter(self, server_name, script_name, subdomain,
                           url_scheme, path_info, default_method, query_args)
 
@@ -1149,6 +1152,10 @@ class Map(object):
             parameter that did not have any effect.  It was removed because
             of that.
 
+        .. versionchanged:: 0.8
+           This will no longer raise a ValueError when an unexpected server
+           name was passed.
+
         :param environ: a WSGI environment.
         :param server_name: an optional server name hint (see above).
         :param subdomain: optionally the current subdomain (see above).
@@ -1163,6 +1170,7 @@ class Map(object):
                    in (('https', '443'), ('http', '80')):
                     server_name += ':' + environ['SERVER_PORT']
         elif subdomain is None and not self.host_matching:
+            server_name = server_name.lower()
             if 'HTTP_HOST' in environ:
                 wsgi_server_name = environ.get('HTTP_HOST')
             else:
@@ -1170,15 +1178,19 @@ class Map(object):
                 if (environ['wsgi.url_scheme'], environ['SERVER_PORT']) not \
                    in (('https', '443'), ('http', '80')):
                     wsgi_server_name += ':' + environ['SERVER_PORT']
+            wsgi_server_name = wsgi_server_name.lower()
             cur_server_name = wsgi_server_name.split('.')
             real_server_name = server_name.split('.')
             offset = -len(real_server_name)
             if cur_server_name[offset:] != real_server_name:
-                raise ValueError('the server name provided (%r) does not '
-                                 'match the server name from the WSGI '
-                                 'environment (%r)' %
-                                 (server_name, wsgi_server_name))
-            subdomain = '.'.join(filter(None, cur_server_name[:offset]))
+                # This can happen even with valid configs if the server was
+                # accesssed directly by IP address under some situations.
+                # Instead of raising an exception like in Werkzeug 0.7 or
+                # earlier we go by an invalid subdomain which will result
+                # in a 404 error on matching.
+                subdomain = '<invalid>'
+            else:
+                subdomain = '.'.join(filter(None, cur_server_name[:offset]))
         return Map.bind(self, server_name, environ.get('SCRIPT_NAME'),
                         subdomain, environ['wsgi.url_scheme'],
                         environ['REQUEST_METHOD'], environ.get('PATH_INFO'),
@@ -1491,7 +1503,8 @@ class MapAdapter(object):
             self.url_scheme,
             self.get_host(domain_part),
             posixpath.join(self.script_name[:-1].lstrip('/'),
-                           url_quote(path_info.lstrip('/'), self.map.charset)),
+                           url_quote(path_info.lstrip('/'), self.map.charset,
+                                     safe='/:|+')),
             suffix
         ))
 

libs/werkzeug/security.py

@@ -10,27 +10,12 @@
 """
 import os
 import hmac
-import string
 import posixpath
 from itertools import izip
 from random import SystemRandom
 
-# because the API of hmac changed with the introduction of the
-# new hashlib module, we have to support both.  This sets up a
-# mapping to the digest factory functions and the digest modules
-# (or factory functions with changed API)
-try:
-    from hashlib import sha1, md5
-    _hash_funcs = _hash_mods = {'sha1': sha1, 'md5': md5}
-    _sha1_mod = sha1
-    _md5_mod = md5
-except ImportError:
-    import sha as _sha1_mod, md5 as _md5_mod
-    _hash_mods = {'sha1': _sha1_mod, 'md5': _md5_mod}
-    _hash_funcs = {'sha1': _sha1_mod.new, 'md5': _md5_mod.new}
 
-
-SALT_CHARS = string.letters + string.digits
+SALT_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
 
 
 _sys_rng = SystemRandom()
@@ -38,6 +23,20 @@ _os_alt_seps = list(sep for sep in [os.path.sep, os.path.altsep]
                     if sep not in (None, '/'))
 
 
+def _find_hashlib_algorithms():
+    import hashlib
+    algos = getattr(hashlib, 'algorithms', None)
+    if algos is None:
+        algos = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
+    rv = {}
+    for algo in algos:
+        func = getattr(hashlib, algo, None)
+        if func is not None:
+            rv[algo] = func
+    return rv
+_hash_funcs = _find_hashlib_algorithms()
+
+
 def safe_str_cmp(a, b):
     """This function compares strings in somewhat constant time.  This
     requires that the length of at least one string is known in advance.
@@ -69,11 +68,11 @@ def _hash_internal(method, salt, password):
     if method == 'plain':
         return password
     if salt:
-        if method not in _hash_mods:
+        if method not in _hash_funcs:
             return None
         if isinstance(salt, unicode):
             salt = salt.encode('utf-8')
-        h = hmac.new(salt, None, _hash_mods[method])
+        h = hmac.new(salt, None, _hash_funcs[method])
     else:
         if method not in _hash_funcs:
             return None
@@ -98,7 +97,7 @@ def generate_password_hash(password, method='sha1', salt_length=8):
     is used, hmac is used internally to salt the password.
 
     :param password: the password to hash
-    :param method: the hash method to use (``'md5'`` or ``'sha1'``)
+    :param method: the hash method to use (one that hashlib supports)
     :param salt_length: the lengt of the salt in letters
     """
     salt = method != 'plain' and gen_salt(salt_length) or ''

libs/werkzeug/serving.py

@@ -35,6 +35,8 @@
     :copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
+from __future__ import with_statement
+
 import os
 import socket
 import sys
@@ -177,6 +179,7 @@ class WSGIRequestHandler(BaseHTTPRequestHandler, object):
 
     def handle(self):
         """Handles a request ignoring dropped connections."""
+        rv = None
         try:
             rv = BaseHTTPRequestHandler.handle(self)
         except (socket.error, socket.timeout), e:
@@ -192,9 +195,11 @@ class WSGIRequestHandler(BaseHTTPRequestHandler, object):
         """A horrible, horrible way to kill the server for Python 2.6 and
         later.  It's the best we can do.
         """
+        # Windows does not provide SIGKILL, go with SIGTERM then.
+        sig = getattr(signal, 'SIGKILL', signal.SIGTERM)
         # reloader active
         if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
-            os.kill(os.getpid(), signal.SIGKILL)
+            os.kill(os.getpid(), sig)
         # python 2.7
         self.server._BaseServer__shutdown_request = True
         # python 2.6
@@ -247,10 +252,13 @@ class WSGIRequestHandler(BaseHTTPRequestHandler, object):
 BaseRequestHandler = WSGIRequestHandler
 
 
-def generate_adhoc_ssl_context():
-    """Generates an adhoc SSL context for the development server."""
+def generate_adhoc_ssl_pair(cn=None):
     from random import random
-    from OpenSSL import crypto, SSL
+    from OpenSSL import crypto
+
+    # pretty damn sure that this is not actually accepted by anyone
+    if cn is None:
+        cn = '*'
 
     cert = crypto.X509()
     cert.set_serial_number(int(random() * sys.maxint))
@@ -258,7 +266,7 @@ def generate_adhoc_ssl_context():
     cert.gmtime_adj_notAfter(60 * 60 * 24 * 365)
 
     subject = cert.get_subject()
-    subject.CN = '*'
+    subject.CN = cn
     subject.O = 'Dummy Certificate'
 
     issuer = cert.get_issuer()
@@ -270,10 +278,59 @@ def generate_adhoc_ssl_context():
     cert.set_pubkey(pkey)
     cert.sign(pkey, 'md5')
 
+    return cert, pkey
+
+
+def make_ssl_devcert(base_path, host=None, cn=None):
+    """Creates an SSL key for development.  This should be used instead of
+    the ``'adhoc'`` key which generates a new cert on each server start.
+    It accepts a path for where it should store the key and cert and
+    either a host or CN.  If a host is given it will use the CN
+    ``*.host/CN=host``.
+
+    For more information see :func:`run_simple`.
+
+    .. versionadded:: 0.9
+
+    :param base_path: the path to the certificate and key.  The extension
+                      ``.crt`` is added for the certificate, ``.key`` is
+                      added for the key.
+    :param host: the name of the host.  This can be used as an alternative
+                 for the `cn`.
+    :param cn: the `CN` to use.
+    """
+    from OpenSSL import crypto
+    if host is not None:
+        cn = '*.%s/CN=%s' % (host, host)
+    cert, pkey = generate_adhoc_ssl_pair(cn=cn)
+
+    cert_file = base_path + '.crt'
+    pkey_file = base_path + '.key'
+
+    with open(cert_file, 'w') as f:
+        f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
+    with open(pkey_file, 'w') as f:
+        f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
+
+    return cert_file, pkey_file
+
+
+def generate_adhoc_ssl_context():
+    """Generates an adhoc SSL context for the development server."""
+    from OpenSSL import SSL
+    pkey, cert = generate_adhoc_ssl_pair()
     ctx = SSL.Context(SSL.SSLv23_METHOD)
     ctx.use_privatekey(pkey)
     ctx.use_certificate(cert)
+    return ctx
 
+
+def load_ssl_context(cert_file, pkey_file):
+    """Loads an SSL context from a certificate and private key file."""
+    from OpenSSL import SSL
+    ctx = SSL.Context(SSL.SSLv23_METHOD)
+    ctx.use_certificate_file(cert_file)
+    ctx.use_privatekey_file(pkey_file)
     return ctx
 
 
@@ -297,6 +354,9 @@ class _SSLConnectionFix(object):
     def __getattr__(self, attrib):
         return getattr(self._con, attrib)
 
+    def shutdown(self, arg=None):
+        self._con.shutdown()
+
 
 def select_ip_version(host, port):
     """Returns AF_INET4 or AF_INET6 depending on where to connect to."""
@@ -339,6 +399,8 @@ class BaseWSGIServer(HTTPServer, object):
             except ImportError:
                 raise TypeError('SSL is not available if the OpenSSL '
                                 'library is not installed.')
+            if isinstance(ssl_context, tuple):
+                ssl_context = load_ssl_context(*ssl_context)
             if ssl_context == 'adhoc':
                 ssl_context = generate_adhoc_ssl_context()
             self.socket = tsafe.Connection(ssl_context, self.socket)
@@ -405,7 +467,23 @@ def make_server(host, port, app=None, threaded=False, processes=1,
                               passthrough_errors, ssl_context)
 
 
-def reloader_loop(extra_files=None, interval=1):
+def _iter_module_files():
+    for module in sys.modules.values():
+        filename = getattr(module, '__file__', None)
+        if filename:
+            old = None
+            while not os.path.isfile(filename):
+                old = filename
+                filename = os.path.dirname(filename)
+                if filename == old:
+                    break
+            else:
+                if filename[-4:] in ('.pyc', '.pyo'):
+                    filename = filename[:-1]
+                yield filename
+
+
+def _reloader_stat_loop(extra_files=None, interval=1):
     """When this function is run from the main thread, it will force other
     threads to exit when any modules currently loaded change.
 
@@ -414,32 +492,10 @@ def reloader_loop(extra_files=None, interval=1):
 
     :param extra_files: a list of additional files it should watch.
     """
-    def iter_module_files():
-        for module in sys.modules.values():
-            filename = getattr(module, '__file__', None)
-            if filename:
-                old = None
-                while not os.path.isfile(filename):
-                    old = filename
-                    filename = os.path.dirname(filename)
-                    if filename == old:
-                        break
-                else:
-                    if filename[-4:] in ('.pyc', '.pyo'):
-                        filename = filename[:-1]
-                    yield filename
-
-    fnames = []
-    fnames.extend(iter_module_files())
-    fnames.extend(extra_files or ())
-
-    reloader(fnames, interval=interval)
-
-
-def _reloader_stat_loop(fnames, interval=1):
+    from itertools import chain
     mtimes = {}
     while 1:
-        for filename in fnames:
+        for filename in chain(_iter_module_files(), extra_files or ()):
             try:
                 mtime = os.stat(filename).st_mtime
             except OSError:
@@ -455,7 +511,7 @@ def _reloader_stat_loop(fnames, interval=1):
         time.sleep(interval)
 
 
-def _reloader_inotify(fnames, interval=None):
+def _reloader_inotify(extra_files=None, interval=None):
     # Mutated by inotify loop when changes occur.
     changed = [False]
 
@@ -478,13 +534,16 @@ def _reloader_inotify(fnames, interval=None):
         _log('info', ' * Detected change in %r, reloading' % event.path)
         changed[:] = [True]
 
-    for fname in fnames:
+    for fname in extra_files or ():
         wm.add_watch(fname, mask, signal_changed)
 
     # ... And now we wait...
     notif = Notifier(wm)
     try:
         while not changed[0]:
+            # always reiterate through sys.modules, adding them
+            for fname in _iter_module_files():
+                wm.add_watch(fname, mask, signal_changed)
             notif.process_events()
             if notif.check_events(timeout=interval):
                 notif.read_events()
@@ -497,7 +556,7 @@ def _reloader_inotify(fnames, interval=None):
 # currently we always use the stat loop reloader for the simple reason
 # that the inotify one does not respond to added files properly.  Also
 # it's quite buggy and the API is a mess.
-reloader = _reloader_stat_loop
+reloader_loop = _reloader_stat_loop
 
 
 def restart_with_reloader():
@@ -555,6 +614,10 @@ def run_simple(hostname, port, application, use_reloader=False,
     .. versionadded:: 0.6
        support for SSL was added.
 
+    .. versionadded:: 0.8
+       Added support for automatically loading a SSL context from certificate
+       file and private key.
+
     :param hostname: The host for the application.  eg: ``'localhost'``
     :param port: The port for the server.  eg: ``8080``
     :param application: the WSGI application to execute
@@ -582,7 +645,8 @@ def run_simple(hostname, port, application, use_reloader=False,
                                This means that the server will die on errors but
                                it can be useful to hook debuggers in (pdb etc.)
     :param ssl_context: an SSL context for the connection. Either an OpenSSL
-                        context, the string ``'adhoc'`` if the server should
+                        context, a tuple in the form ``(cert_file, pkey_file)``,
+                        the string ``'adhoc'`` if the server should
                         automatically create one, or `None` to disable SSL
                         (which is the default).
     """
@@ -607,7 +671,8 @@ def run_simple(hostname, port, application, use_reloader=False,
     if use_reloader:
         # Create and destroy a socket so that any exceptions are raised before
         # we spawn a separate Python interpreter and lose this ability.
-        test_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        address_family = select_ip_version(hostname, port)
+        test_socket = socket.socket(address_family, socket.SOCK_STREAM)
         test_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         test_socket.bind((hostname, port))
         test_socket.close()

libs/werkzeug/test.py

@@ -474,7 +474,10 @@ class EnvironBuilder(object):
         return 80
 
     def __del__(self):
-        self.close()
+        try:
+            self.close()
+        except Exception:
+            pass
 
     def close(self):
         """Closes all files.  If you put real :class:`file` objects into the
@@ -600,14 +603,11 @@ class Client(object):
     def __init__(self, application, response_wrapper=None, use_cookies=True,
                  allow_subdomain_redirects=False):
         self.application = application
-        if response_wrapper is None:
-            response_wrapper = lambda a, s, h: (a, s, h)
         self.response_wrapper = response_wrapper
         if use_cookies:
             self.cookie_jar = _TestCookieJar()
         else:
             self.cookie_jar = None
-        self.redirect_client = None
         self.allow_subdomain_redirects = allow_subdomain_redirects
 
     def set_cookie(self, server_name, key, value='', max_age=None,
@@ -629,6 +629,46 @@ class Client(object):
         self.set_cookie(server_name, key, expires=0, max_age=0,
                         path=path, domain=domain)
 
+    def run_wsgi_app(self, environ, buffered=False):
+        """Runs the wrapped WSGI app with the given environment."""
+        if self.cookie_jar is not None:
+            self.cookie_jar.inject_wsgi(environ)
+        rv = run_wsgi_app(self.application, environ, buffered=buffered)
+        if self.cookie_jar is not None:
+            self.cookie_jar.extract_wsgi(environ, rv[2])
+        return rv
+
+    def resolve_redirect(self, response, new_location, environ, buffered=False):
+        """Resolves a single redirect and triggers the request again
+        directly on this redirect client.
+        """
+        scheme, netloc, script_root, qs, anchor = urlparse.urlsplit(new_location)
+        base_url = urlparse.urlunsplit((scheme, netloc, '', '', '')).rstrip('/') + '/'
+
+        cur_server_name = netloc.split(':', 1)[0].split('.')
+        real_server_name = get_host(environ).rsplit(':', 1)[0].split('.')
+
+        if self.allow_subdomain_redirects:
+            allowed = cur_server_name[-len(real_server_name):] == real_server_name
+        else:
+            allowed = cur_server_name == real_server_name
+
+        if not allowed:
+            raise RuntimeError('%r does not support redirect to '
+                               'external targets' % self.__class__)
+
+        # For redirect handling we temporarily disable the response
+        # wrapper.  This is not threadsafe but not a real concern
+        # since the test client must not be shared anyways.
+        old_response_wrapper = self.response_wrapper
+        self.response_wrapper = None
+        try:
+            return self.open(path=script_root, base_url=base_url,
+                             query_string=qs, as_tuple=True,
+                             buffered=buffered)
+        finally:
+            self.response_wrapper = old_response_wrapper
+
     def open(self, *args, **kwargs):
         """Takes the same arguments as the :class:`EnvironBuilder` class with
         some additions:  You can provide a :class:`EnvironBuilder` or a WSGI
@@ -670,61 +710,25 @@ class Client(object):
             finally:
                 builder.close()
 
-        if self.cookie_jar is not None:
-            self.cookie_jar.inject_wsgi(environ)
-        rv = run_wsgi_app(self.application, environ, buffered=buffered)
-        if self.cookie_jar is not None:
-            self.cookie_jar.extract_wsgi(environ, rv[2])
+        response = self.run_wsgi_app(environ, buffered=buffered)
 
         # handle redirects
         redirect_chain = []
-        status_code = int(rv[1].split(None, 1)[0])
-        while status_code in (301, 302, 303, 305, 307) and follow_redirects:
-            if not self.redirect_client:
-                # assume that we're not using the user defined response wrapper
-                # so that we don't need any ugly hacks to get the status
-                # code from the response.
-                self.redirect_client = Client(self.application)
-                self.redirect_client.cookie_jar = self.cookie_jar
+        while 1:
+            status_code = int(response[1].split(None, 1)[0])
+            if status_code not in (301, 302, 303, 305, 307) \
+               or not follow_redirects:
+                break
+            new_location = Headers.linked(response[2])['location']
+            new_redirect_entry = (new_location, status_code)
+            if new_redirect_entry in redirect_chain:
+                raise ClientRedirectError('loop detected')
+            redirect_chain.append(new_redirect_entry)
+            environ, response = self.resolve_redirect(response, new_location,
+                                                      environ, buffered=buffered)
 
-            redirect = dict(rv[2])['Location']
-
-            scheme, netloc, script_root, qs, anchor = urlparse.urlsplit(redirect)
-            base_url = urlparse.urlunsplit((scheme, netloc, '', '', '')).rstrip('/') + '/'
-
-            cur_server_name = netloc.split(':', 1)[0].split('.')
-            real_server_name = get_host(environ).split(':', 1)[0].split('.')
-
-            if self.allow_subdomain_redirects:
-                allowed = cur_server_name[-len(real_server_name):] == real_server_name
-            else:
-                allowed = cur_server_name == real_server_name
-
-            if not allowed:
-                raise RuntimeError('%r does not support redirect to '
-                                   'external targets' % self.__class__)
-
-            redirect_chain.append((redirect, status_code))
-
-            # the redirect request should be a new request, and not be based on
-            # the old request
-
-            redirect_kwargs = {
-                'path':             script_root,
-                'base_url':         base_url,
-                'query_string':     qs,
-                'as_tuple':         True,
-                'buffered':         buffered,
-                'follow_redirects': False,
-            }
-            environ, rv = self.redirect_client.open(**redirect_kwargs)
-            status_code = int(rv[1].split(None, 1)[0])
-
-            # Prevent loops
-            if redirect_chain[-1] in redirect_chain[:-1]:
-                raise ClientRedirectError("loop detected")
-
-        response = self.response_wrapper(*rv)
+        if self.response_wrapper is not None:
+            response = self.response_wrapper(*response)
         if as_tuple:
             return environ, response
         return response

libs/werkzeug/urls.py

@@ -12,6 +12,7 @@ import urlparse
 
 from werkzeug._internal import _decode_unicode
 from werkzeug.datastructures import MultiDict, iter_multi_items
+from werkzeug.wsgi import make_chunk_iter
 
 
 #: list of characters that are always safe in URLs.
@@ -237,20 +238,62 @@ def url_decode(s, charset='utf-8', decode_keys=False, include_empty=True,
     """
     if cls is None:
         cls = MultiDict
-    result = []
-    for pair in str(s).split(separator):
+    return cls(_url_decode_impl(str(s).split(separator), charset, decode_keys,
+                                include_empty, errors))
+
+
+def url_decode_stream(stream, charset='utf-8', decode_keys=False,
+                      include_empty=True, errors='replace', separator='&',
+                      cls=None, limit=None, return_iterator=False):
+    """Works like :func:`url_decode` but decodes a stream.  The behavior
+    of stream and limit follows functions like
+    :func:`~werkzeug.wsgi.make_line_iter`.  The generator of pairs is
+    directly fed to the `cls` so you can consume the data while it's
+    parsed.
+
+    .. versionadded:: 0.8
+
+    :param stream: a stream with the encoded querystring
+    :param charset: the charset of the query string.
+    :param decode_keys: set to `True` if you want the keys to be decoded
+                        as well.
+    :param include_empty: Set to `False` if you don't want empty values to
+                          appear in the dict.
+    :param errors: the decoding error behavior.
+    :param separator: the pair separator to be used, defaults to ``&``
+    :param cls: an optional dict class to use.  If this is not specified
+                       or `None` the default :class:`MultiDict` is used.
+    :param limit: the content length of the URL data.  Not necessary if
+                  a limited stream is provided.
+    :param return_iterator: if set to `True` the `cls` argument is ignored
+                            and an iterator over all decoded pairs is
+                            returned
+    """
+    if return_iterator:
+        cls = lambda x: x
+    elif cls is None:
+        cls = MultiDict
+    pair_iter = make_chunk_iter(stream, separator, limit)
+    return cls(_url_decode_impl(pair_iter, charset, decode_keys,
+                                include_empty, errors))
+
+
+def _url_decode_impl(pair_iter, charset, decode_keys, include_empty,
+                     errors):
+    for pair in pair_iter:
         if not pair:
             continue
         if '=' in pair:
             key, value = pair.split('=', 1)
         else:
+            if not include_empty:
+                continue
             key = pair
             value = ''
         key = _unquote_plus(key)
         if decode_keys:
             key = _decode_unicode(key, charset, errors)
-        result.append((key, url_unquote_plus(value, charset, errors)))
-    return cls(result)
+        yield key, url_unquote_plus(value, charset, errors)
 
 
 def url_encode(obj, charset='utf-8', encode_keys=False, sort=False, key=None,
@@ -274,10 +317,41 @@ def url_encode(obj, charset='utf-8', encode_keys=False, sort=False, key=None,
     :param key: an optional function to be used for sorting.  For more details
                 check out the :func:`sorted` documentation.
     """
+    return separator.join(_url_encode_impl(obj, charset, encode_keys, sort, key))
+
+
+def url_encode_stream(obj, stream=None, charset='utf-8', encode_keys=False,
+                      sort=False, key=None, separator='&'):
+    """Like :meth:`url_encode` but writes the results to a stream
+    object.  If the stream is `None` a generator over all encoded
+    pairs is returned.
+
+    .. versionadded:: 0.8
+
+    :param obj: the object to encode into a query string.
+    :param stream: a stream to write the encoded object into or `None` if
+                   an iterator over the encoded pairs should be returned.  In
+                   that case the separator argument is ignored.
+    :param charset: the charset of the query string.
+    :param encode_keys: set to `True` if you have unicode keys.
+    :param sort: set to `True` if you want parameters to be sorted by `key`.
+    :param separator: the separator to be used for the pairs.
+    :param key: an optional function to be used for sorting.  For more details
+                check out the :func:`sorted` documentation.
+    """
+    gen = _url_encode_impl(obj, charset, encode_keys, sort, key)
+    if stream is None:
+        return gen
+    for idx, chunk in enumerate(gen):
+        if idx:
+            stream.write(separator)
+        stream.write(chunk)
+
+
+def _url_encode_impl(obj, charset, encode_keys, sort, key):
     iterable = iter_multi_items(obj)
     if sort:
         iterable = sorted(iterable, key=key)
-    tmp = []
     for key, value in iterable:
         if value is None:
             continue
@@ -289,9 +363,7 @@ def url_encode(obj, charset='utf-8', encode_keys=False, sort=False, key=None,
             value = value.encode(charset)
         else:
             value = str(value)
-        tmp.append('%s=%s' % (_quote(key),
-                              _quote_plus(value)))
-    return separator.join(tmp)
+        yield '%s=%s' % (_quote(key), _quote_plus(value))
 
 
 def url_quote(s, charset='utf-8', safe='/:'):

libs/werkzeug/utils.py

@@ -189,10 +189,10 @@ class HTMLBuilder(object):
                     buffer += '>'
                 return buffer
             buffer += '>'
-            
+
             children_as_string = ''.join([unicode(x) for x in children
                                          if x is not None])
-            
+
             if children_as_string:
                 if tag in self._plaintext_elements:
                     children_as_string = escape(children_as_string)
@@ -352,7 +352,6 @@ def redirect(location, code=302):
     :param location: the location the response should redirect to.
     :param code: the redirect status code. defaults to 302.
     """
-    assert code in (201, 301, 302, 303, 305, 307), 'invalid code'
     from werkzeug.wrappers import BaseResponse
     display_location = location
     if isinstance(location, unicode):
@@ -392,6 +391,9 @@ def import_string(import_name, silent=False):
 
     If `silent` is True the return value will be `None` if the import fails.
 
+    For better debugging we recommend the new :func:`import_module`
+    function to be used instead.
+
     :param import_name: the dotted name for the object to import.
     :param silent: if set to `True` import errors are ignored and
                    `None` is returned instead.
@@ -584,7 +586,7 @@ class ImportStringError(ImportError):
             name += (name and '.') + part
             imported = import_string(name, silent=True)
             if imported:
-                tracked.append((name, imported.__file__))
+                tracked.append((name, getattr(imported, '__file__', None)))
             else:
                 track = ['- %r found in %r.' % (n, i) for n, i in tracked]
                 track.append('- %r not found.' % name)

libs/werkzeug/wrappers.py

@@ -32,7 +32,7 @@ from werkzeug.http import HTTP_STATUS_CODES, \
      parse_if_range_header, parse_cookie, dump_cookie, \
      parse_range_header, parse_content_range_header, dump_header
 from werkzeug.urls import url_decode, iri_to_uri
-from werkzeug.formparser import parse_form_data, default_stream_factory
+from werkzeug.formparser import FormDataParser, default_stream_factory
 from werkzeug.utils import cached_property, environ_property, \
      header_property, get_content_type
 from werkzeug.wsgi import get_current_url, get_host, LimitedStream, \
@@ -167,6 +167,10 @@ class BaseRequest(object):
     #: .. versionadded:: 0.6
     dict_storage_class = ImmutableTypeConversionDict
 
+    #: The form data parser that shoud be used.  Can be replaced to customize
+    #: the form date parsing.
+    form_data_parser_class = FormDataParser
+
     def __init__(self, environ, populate_request=True, shallow=False):
         self.environ = environ
         if populate_request and not shallow:
@@ -246,7 +250,7 @@ class BaseRequest(object):
         return _patch_wrapper(f, lambda *a: f(*a[:-2]+(cls(a[-2]),))(*a[-2:]))
 
     def _get_file_stream(self, total_content_length, content_type, filename=None,
-                         content_length=None):
+                        content_length=None):
         """Called to get a stream for the file upload.
 
         This must provide a file-like class with `read()`, `readline()`
@@ -257,11 +261,6 @@ class BaseRequest(object):
         provide a content length for the files only the total content
         length matters.
 
-        .. versionchanged:: 0.5
-           Previously this function was not passed any arguments.  In 0.5 older
-           functions not accepting any arguments are still supported for
-           backwards compatibility.
-
         :param total_content_length: the total content length of all the
                                      data in the request combined.  This value
                                      is guaranteed to be there.
@@ -274,6 +273,29 @@ class BaseRequest(object):
         return default_stream_factory(total_content_length, content_type,
                                       filename, content_length)
 
+    @property
+    def want_form_data_parsed(self):
+        """Returns True if the request method is ``POST``, ``PUT`` or
+        ``PATCH``.  Can be overriden to support other HTTP methods that
+        should carry form data.
+
+        .. versionadded:: 0.8
+        """
+        return self.environ['REQUEST_METHOD'] in ('POST', 'PUT', 'PATCH')
+
+    def make_form_data_parser(self):
+        """Creates the form data parser.  Instanciates the
+        :attr:`form_data_parser_class` with some parameters.
+
+        .. versionadded:: 0.8
+        """
+        return self.form_data_parser_class(self._get_file_stream,
+                                           self.charset,
+                                           self.encoding_errors,
+                                           self.max_form_memory_size,
+                                           self.max_content_length,
+                                           self.parameter_storage_class)
+
     def _load_form_data(self):
         """Method used internally to retrieve submitted data.  After calling
         this sets `form` and `files` on the request object to multi dicts
@@ -291,16 +313,9 @@ class BaseRequest(object):
                                'that, set `shallow` to False.')
         data = None
         stream = _empty_stream
-        if self.environ['REQUEST_METHOD'] in ('POST', 'PUT', 'PATCH'):
-            try:
-                data = parse_form_data(self.environ, self._get_file_stream,
-                                       self.charset, self.encoding_errors,
-                                       self.max_form_memory_size,
-                                       self.max_content_length,
-                                       cls=self.parameter_storage_class,
-                                       silent=False)
-            except ValueError, e:
-                self._form_parsing_failed(e)
+        if self.want_form_data_parsed:
+            parser = self.make_form_data_parser()
+            data = parser.parse_from_environ(self.environ)
         else:
             # if we have a content length header we are able to properly
             # guard the incoming stream, no matter what request method is
@@ -319,17 +334,6 @@ class BaseRequest(object):
         d = self.__dict__
         d['stream'], d['form'], d['files'] = data
 
-    def _form_parsing_failed(self, error):
-        """Called if parsing of form data failed.  This is currently only
-        invoked for failed multipart uploads.  By default this method does
-        nothing.
-
-        :param error: a `ValueError` object with a message why the
-                      parsing failed.
-
-        .. versionadded:: 0.5.1
-        """
-
     @cached_property
     def stream(self):
         """The parsed stream if the submitted data was not multipart or
@@ -601,6 +605,18 @@ class BaseResponse(object):
     #:    your code to the name change.
     implicit_sequence_conversion = True
 
+    #: Should this response object correct the location header to be RFC
+    #: conformant?  This is true by default.
+    #:
+    #: .. versionadded:: 0.8
+    autocorrect_location_header = True
+
+    #: Should this response object automatically set the content-length
+    #: header if possible?  This is true by default.
+    #:
+    #: .. versionadded:: 0.8
+    automatically_set_content_length = True
+
     def __init__(self, response=None, status=None, headers=None,
                  mimetype=None, content_type=None, direct_passthrough=False):
         if isinstance(headers, Headers):
@@ -721,7 +737,7 @@ class BaseResponse(object):
         except KeyError:
             self._status = '%d UNKNOWN' % code
     status_code = property(_get_status_code, _set_status_code,
-                           'The HTTP Status code as number')
+                           doc='The HTTP Status code as number')
     del _get_status_code, _set_status_code
 
     def _get_status(self):
@@ -732,7 +748,7 @@ class BaseResponse(object):
             self._status_code = int(self._status.split(None, 1)[0])
         except ValueError:
             self._status_code = 0
-    status = property(_get_status, _set_status, 'The HTTP Status code')
+    status = property(_get_status, _set_status, doc='The HTTP Status code')
     del _get_status, _set_status
 
     def _get_data(self):
@@ -746,11 +762,13 @@ class BaseResponse(object):
         self._ensure_sequence()
         return ''.join(self.iter_encoded())
     def _set_data(self, value):
-        # if an unicode string is set, it's encoded directly.  this allows
-        # us to guess the content length automatically in `get_wsgi_headers`.
+        # if an unicode string is set, it's encoded directly so that we
+        # can set the content length
         if isinstance(value, unicode):
             value = value.encode(self.charset)
         self.response = [value]
+        if self.automatically_set_content_length:
+            self.headers['Content-Length'] = str(len(value))
     data = property(_get_data, _set_data, doc=_get_data.__doc__)
     del _get_data, _set_data
 
@@ -940,6 +958,7 @@ class BaseResponse(object):
         location = None
         content_location = None
         content_length = None
+        status = self.status_code
 
         # iterate over the headers to find all values in one go.  Because
         # get_wsgi_headers is used each response that gives us a tiny
@@ -955,12 +974,16 @@ class BaseResponse(object):
 
         # make sure the location header is an absolute URL
         if location is not None:
+            old_location = location
             if isinstance(location, unicode):
                 location = iri_to_uri(location)
-            headers['Location'] = urlparse.urljoin(
-                get_current_url(environ, root_only=True),
-                location
-            )
+            if self.autocorrect_location_header:
+                location = urlparse.urljoin(
+                    get_current_url(environ, root_only=True),
+                    location
+                )
+            if location != old_location:
+                headers['Location'] = location
 
         # make sure the content location is a URL
         if content_location is not None and \
@@ -971,9 +994,9 @@ class BaseResponse(object):
         # Also update content_length accordingly so that the automatic
         # content length detection does not trigger in the following
         # code.
-        if 100 <= self.status_code < 200 or self.status_code == 204:
+        if 100 <= status < 200 or status == 204:
             headers['Content-Length'] = content_length = '0'
-        elif self.status_code == 304:
+        elif status == 304:
             remove_entity_headers(headers)
 
         # if we can determine the content length automatically, we
@@ -981,8 +1004,8 @@ class BaseResponse(object):
         # flattening the iterator or encoding of unicode strings in
         # the response.  We however should not do that if we have a 304
         # response.
-        if self.is_sequence and content_length is None and \
-           self.status_code != 304:
+        if self.automatically_set_content_length and \
+           self.is_sequence and content_length is None and status != 304:
             try:
                 content_length = sum(len(str(x)) for x in self.response)
             except UnicodeError:
@@ -1008,8 +1031,9 @@ class BaseResponse(object):
         :param environ: the WSGI environment of the request.
         :return: a response iterable.
         """
+        status = self.status_code
         if environ['REQUEST_METHOD'] == 'HEAD' or \
-           100 <= self.status_code < 200 or self.status_code in (204, 304):
+           100 <= status < 200 or status in (204, 304):
             return ()
         if self.direct_passthrough:
             if __debug__:
@@ -1244,7 +1268,7 @@ class ETagResponseMixin(object):
             # wsgiref.
             if 'date' not in self.headers:
                 self.headers['Date'] = http_date()
-            if 'content-length' in self.headers:
+            if 'content-length' not in self.headers:
                 self.headers['Content-Length'] = len(self.data)
             if not is_resource_modified(environ, self.headers.get('etag'), None,
                                         self.headers.get('last-modified')):

libs/werkzeug/wsgi.py

@@ -8,14 +8,17 @@
     :copyright: (c) 2011 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
+import re
 import os
 import urllib
 import urlparse
 import posixpath
 import mimetypes
+from itertools import chain, repeat
 from zlib import adler32
 from time import time, mktime
 from datetime import datetime
+from functools import partial
 
 from werkzeug._internal import _patch_wrapper
 from werkzeug.http import is_resource_modified, http_date
@@ -570,6 +573,22 @@ class FileWrapper(object):
         raise StopIteration()
 
 
+def make_limited_stream(stream, limit):
+    """Makes a stream limited."""
+    if not isinstance(stream, LimitedStream):
+        if limit is None:
+            raise TypeError('stream not limited and no limit provided.')
+        stream = LimitedStream(stream, limit)
+    return stream
+
+
+def make_chunk_iter_func(stream, limit, buffer_size):
+    """Helper for the line and chunk iter functions."""
+    if hasattr(stream, 'read'):
+        return partial(make_limited_stream(stream, limit).read, buffer_size)
+    return iter(chain(stream, repeat(''))).next
+
+
 def make_line_iter(stream, limit=None, buffer_size=10 * 1024):
     """Safely iterates line-based over an input stream.  If the input stream
     is not a :class:`LimitedStream` the `limit` parameter is mandatory.
@@ -583,40 +602,85 @@ def make_line_iter(stream, limit=None, buffer_size=10 * 1024):
     If you need line-by-line processing it's strongly recommended to iterate
     over the input stream using this helper function.
 
-    :param stream: the stream to iterate over.
+    .. versionchanged:: 0.8
+       This function now ensures that the limit was reached.
+
+    .. versionadded:: 0.9
+       added support for iterators as input stream.
+
+    :param stream: the stream or iterate to iterate over.
     :param limit: the limit in bytes for the stream.  (Usually
                   content length.  Not necessary if the `stream`
                   is a :class:`LimitedStream`.
     :param buffer_size: The optional buffer size.
     """
-    if not isinstance(stream, LimitedStream):
-        if limit is None:
-            raise TypeError('stream not limited and no limit provided.')
-        stream = LimitedStream(stream, limit)
-    _read = stream.read
+    def _iter_basic_lines():
+        _read = make_chunk_iter_func(stream, limit, buffer_size)
+        buffer = []
+        while 1:
+            new_data = _read()
+            if not new_data:
+                break
+            new_buf = []
+            for item in chain(buffer, new_data.splitlines(True)):
+                new_buf.append(item)
+                if item and item[-1:] in '\r\n':
+                    yield ''.join(new_buf)
+                    new_buf = []
+            buffer = new_buf
+        if buffer:
+            yield ''.join(buffer)
+
+    # This hackery is necessary to merge 'foo\r' and '\n' into one item
+    # of 'foo\r\n' if we were unlucky and we hit a chunk boundary.
+    previous = ''
+    for item in _iter_basic_lines():
+        if item == '\n' and previous[-1:] == '\r':
+            previous += '\n'
+            item = ''
+        if previous:
+            yield previous
+        previous = item
+    if previous:
+        yield previous
+
+
+def make_chunk_iter(stream, separator, limit=None, buffer_size=10 * 1024):
+    """Works like :func:`make_line_iter` but accepts a separator
+    which divides chunks.  If you want newline based processing
+    you should use :func:`make_limited_stream` instead as it
+    supports arbitrary newline markers.
+
+    .. versionadded:: 0.8
+
+    .. versionadded:: 0.9
+       added support for iterators as input stream.
+
+    :param stream: the stream or iterate to iterate over.
+    :param separator: the separator that divides chunks.
+    :param limit: the limit in bytes for the stream.  (Usually
+                  content length.  Not necessary if the `stream`
+                  is a :class:`LimitedStream`.
+    :param buffer_size: The optional buffer size.
+    """
+    _read = make_chunk_iter_func(stream, limit, buffer_size)
+    _split = re.compile(r'(%s)' % re.escape(separator)).split
     buffer = []
     while 1:
-        if len(buffer) > 1:
-            yield buffer.pop()
-            continue
-
-        # we reverse the chunks because popping from the last
-        # position of the list is O(1) and the number of chunks
-        # read will be quite large for binary files.
-        chunks = _read(buffer_size).splitlines(True)
-        chunks.reverse()
-
-        first_chunk = buffer and buffer[0] or ''
-        if chunks:
-            if first_chunk.endswith('\n') or first_chunk.endswith('\r'):
-                yield first_chunk
-                first_chunk = ''
-            first_chunk += chunks.pop()
-        if not first_chunk:
-            return
-
-        buffer = chunks
-        yield first_chunk
+        new_data = _read()
+        if not new_data:
+            break
+        chunks = _split(new_data)
+        new_buf = []
+        for item in chain(buffer, chunks):
+            if item == separator:
+                yield ''.join(new_buf)
+                new_buf = []
+            else:
+                new_buf.append(item)
+        buffer = new_buf
+    if buffer:
+        yield ''.join(buffer)
 
 
 class LimitedStream(object):
@@ -688,14 +752,21 @@ class LimitedStream(object):
         """This is called when the stream tries to read past the limit.
         The return value of this function is returned from the reading
         function.
-
-        Per default this raises a :exc:`~werkzeug.exceptions.BadRequest`.
         """
         if self.silent:
             return ''
         from werkzeug.exceptions import BadRequest
         raise BadRequest('input stream exhausted')
 
+    def on_disconnect(self):
+        """What should happen if a disconnect is detected?  The return
+        value of this function is returned from read functions in case
+        the client went away.  By default a
+        :exc:`~werkzeug.exceptions.ClientDisconnected` exception is raised.
+        """
+        from werkzeug.exceptions import ClientDisconnected
+        raise ClientDisconnected()
+
     def exhaust(self, chunk_size=1024 * 16):
         """Exhaust the stream.  This consumes all the data left until the
         limit is reached.
@@ -718,9 +789,15 @@ class LimitedStream(object):
         """
         if self._pos >= self.limit:
             return self.on_exhausted()
-        if size is None:
+        if size is None or size == -1:  # -1 is for consistence with file
             size = self.limit
-        read = self._read(min(self.limit - self._pos, size))
+        to_read = min(self.limit - self._pos, size)
+        try:
+            read = self._read(to_read)
+        except (IOError, ValueError):
+            return self.on_disconnect()
+        if to_read and len(read) != to_read:
+            return self.on_disconnect()
         self._pos += len(read)
         return read
 
@@ -732,7 +809,12 @@ class LimitedStream(object):
             size = self.limit - self._pos
         else:
             size = min(size, self.limit - self._pos)
-        line = self._readline(size)
+        try:
+            line = self._readline(size)
+        except (ValueError, IOError):
+            return self.on_disconnect()
+        if size and not line:
+            return self.on_disconnect()
         self._pos += len(line)
         return line
 
@@ -758,6 +840,13 @@ class LimitedStream(object):
                 last_pos = self._pos
         return result
 
+    def tell(self):
+        """Returns the position of the stream.
+
+        .. versionadded:: 0.9
+        """
+        return self._pos
+
     def next(self):
         line = self.readline()
         if line is None: