Send referer with scheme
This commit is contained in:
@@ -105,13 +105,14 @@ class Plugin(object):
|
|||||||
if not params: params = {}
|
if not params: params = {}
|
||||||
|
|
||||||
# Fill in some headers
|
# Fill in some headers
|
||||||
headers['Referer'] = headers.get('Referer', urlparse(url).hostname)
|
parsed_url = urlparse(url)
|
||||||
headers['Host'] = headers.get('Host', urlparse(url).hostname)
|
host = parsed_url.hostname
|
||||||
|
|
||||||
|
headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host))
|
||||||
|
headers['Host'] = headers.get('Host', host)
|
||||||
headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
|
headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
|
||||||
headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
|
headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
|
||||||
|
|
||||||
host = urlparse(url).hostname
|
|
||||||
|
|
||||||
# Don't try for failed requests
|
# Don't try for failed requests
|
||||||
if self.http_failed_disabled.get(host, 0) > 0:
|
if self.http_failed_disabled.get(host, 0) > 0:
|
||||||
if self.http_failed_disabled[host] > (time.time() - 900):
|
if self.http_failed_disabled[host] > (time.time() - 900):
|
||||||
|
|||||||
Reference in New Issue
Block a user