dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create reusable url opener

Browse Source
This commit is contained in:
Ruud
2014-01-05 22:17:16 +01:00
parent c23b014cff
commit 5ef0c52277
1 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/plugins/base.py
+12 -11
View File
@@ -39,6 +39,7 @@ class Plugin(object):
http_time_between_calls = 0 http_time_between_calls = 0
http_failed_request = {} http_failed_request = {}
http_failed_disabled = {} http_failed_disabled = {}
http_opener = None
def __new__(typ, *args, **kwargs): def __new__(typ, *args, **kwargs):
new_plugin = super(Plugin, typ).__new__(typ) new_plugin = super(Plugin, typ).__new__(typ)
@@ -137,6 +138,14 @@ class Plugin(object):
headers['Connection'] = headers.get('Connection', 'keep-alive') headers['Connection'] = headers.get('Connection', 'keep-alive')
headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')
# Create opener
if not opener:
if not self.http_opener:
cookies = cookielib.CookieJar()
self.http_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies))
opener = self.http_opener
# Don't try for failed requests # Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0: if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > (time.time() - 900): if self.http_failed_disabled[host] > (time.time() - 900):
@@ -153,18 +162,13 @@ class Plugin(object):
try: try:
# Make sure opener has the correct headers # Make sure opener has the correct headers
if opener: opener.add_headers = headers
opener.add_headers = headers
if multipart: if multipart:
log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
request = urllib2.Request(url, params, headers) request = urllib2.Request(url, params, headers)
if opener: opener.add_handler(MultipartPostHandler())
opener.add_handler(MultipartPostHandler())
else:
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler)
response = opener.open(request, timeout = timeout) response = opener.open(request, timeout = timeout)
else: else:
@@ -177,10 +181,7 @@ class Plugin(object):
request = urllib2.Request(url, data, headers) request = urllib2.Request(url, data, headers)
if opener: response = opener.open(request, timeout = timeout)
response = opener.open(request, timeout = timeout)
else:
response = urllib2.urlopen(request, timeout = timeout)
# unzip if needed # unzip if needed
if response.info().get('Content-Encoding') == 'gzip': if response.info().get('Content-Encoding') == 'gzip':