updated user_agent_parser to 1.7.8, fix webclient

gluon/_compat.py

@@ -30,6 +30,7 @@ if PY2:
     from string import maketrans
     from types import ClassType
     import cgi
+    import cookielib
     reduce = reduce
     hashlib_md5 = hashlib.md5
     iterkeys = lambda d: d.iterkeys()
@@ -94,6 +95,7 @@ else:
     from email.charset import Charset, add_charset, QP as charset_QP
     from urllib.request import FancyURLopener, urlopen
     from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode
+    from http import cookiejar as cookielib
     import html
     hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
     iterkeys = lambda d: iter(d.keys())

gluon/contrib/webclient.py

@@ -16,11 +16,9 @@ mostly for testing purposes
 Some examples at the bottom.
 """
 from __future__ import print_function
+from gluon._compat import urllib2, cookielib, iteritems, to_native, urlencode, to_bytes
 import re
 import time
-import urllib
-import urllib2
-import cookielib
 
 
 DEFAULT_HEADERS = {
@@ -85,10 +83,10 @@ class WebClient(object):
 
         # copy headers from dict to list of key,value
         headers_list = []
-        for key, value in self.default_headers.iteritems():
+        for key, value in iteritems(self.default_headers):
             if not key in headers:
                 headers[key] = value
-        for key, value in headers.iteritems():
+        for key, value in iteritems(headers):
             if isinstance(value, (list, tuple)):
                 for v in value:
                     headers_list.append((key, v))
@@ -96,7 +94,7 @@ class WebClient(object):
                 headers_list.append((key, value))
 
         # move cookies to headers
-        for key, value in cookies.iteritems():
+        for key, value in iteritems(cookies):
             headers_list.append(('Cookie', '%s=%s' % (key, value)))
 
         # add headers to request
@@ -120,24 +118,25 @@ class WebClient(object):
                     data['_formkey'] = self.forms[data['_formname']]
 
                 # time the POST request
-                data = urllib.urlencode(data, doseq=True)
+                data = urlencode(data, doseq=True)
             else:
                 self.method = 'GET' if method=='auto' else method
                 data = None
             t0 = time.time()
-            self.response = opener.open(self.url, data)
+            self.response = opener.open(self.url, to_bytes(data))
             self.time = time.time() - t0
-        except urllib2.HTTPError as error:
+        except urllib2.HTTPError as er:
+            error = er
             # catch HTTP errors
             self.time = time.time() - t0
-            self.response = error
+            self.response = er
 
         if hasattr(self.response, 'getcode'):
             self.status = self.response.getcode()
         else:#python2.5
             self.status = None
 
-        self.text = self.response.read()
+        self.text = to_native(self.response.read())
         self.headers = dict(self.response.headers)
 
         # treat web2py tickets as special types of errors
@@ -147,6 +146,9 @@ class WebClient(object):
             else:
                 raise error
 
+        if 'Set-Cookie' in self.headers:    #PY3
+            self.headers['set-cookie'] = self.headers['Set-Cookie']
+
         # parse headers into cookies
         self.cookies = {}
         if 'set-cookie' in self.headers:
@@ -156,7 +158,7 @@ class WebClient(object):
 
         # check is a new session id has been issued, symptom of broken session
         if self.session_regex is not None:
-            for cookie, value in self.cookies.iteritems():
+            for cookie, value in iteritems(self.cookies):
                 match = self.session_regex.match(cookie)
                 if match:
                     name = match.group('name')
@@ -166,7 +168,7 @@ class WebClient(object):
 
         # find all forms and formkeys in page
         self.forms = {}
-        for match in FORM_REGEX.finditer(self.text):
+        for match in FORM_REGEX.finditer(to_native(self.text)):
             self.forms[match.group('formname')] = match.group('formkey')
 
         # log this request