Send referer with scheme

This commit is contained in:
Ruud
2013-06-02 14:22:59 +02:00
parent dc20b68a37
commit 95d146fea2

View File

@@ -105,13 +105,14 @@ class Plugin(object):
if not params: params = {}
# Fill in some headers
headers['Referer'] = headers.get('Referer', urlparse(url).hostname)
headers['Host'] = headers.get('Host', urlparse(url).hostname)
parsed_url = urlparse(url)
host = parsed_url.hostname
headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host))
headers['Host'] = headers.get('Host', host)
headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
host = urlparse(url).hostname
# Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > (time.time() - 900):