Add info for selectred TPB domain on start of request
This commit is contained in:
Janez Troha
@@ -15,22 +15,18 @@ import time
|
|||||||
|
|
||||||
class TPBProxy(object):
|
class TPBProxy(object):
|
||||||
|
|
||||||
""" TPBProxy deals with failed or blocked TPB proxys
|
""" TPBProxy deals with failed or blocked TPB proxys.
|
||||||
it works as Round-robin balancer if user seleced
|
It works as round-robin balancer, if user seleced
|
||||||
or default domain becomes unavaliable.
|
or default domain becomes unavaliable.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
list = [
|
list = [
|
||||||
('(Sweden) thepiratebay.se', 'http://thepiratebay.se'),
|
('(Sweden) thepiratebay.se', 'http://thepiratebay.se'),
|
||||||
('(Sweden) tpb.ipredator.se (ssl)', 'https://tpb.ipredator.se'
|
('(Sweden) tpb.ipredator.se (ssl)', 'https://tpb.ipredator.se'),
|
||||||
),
|
|
||||||
('(Germany) depiraatbaai.be', 'http://depiraatbaai.be'),
|
('(Germany) depiraatbaai.be', 'http://depiraatbaai.be'),
|
||||||
('(UK) piratereverse.info (ssl)', 'https://piratereverse.info'
|
('(UK) piratereverse.info (ssl)', 'https://piratereverse.info'),
|
||||||
),
|
('(UK) tpb.pirateparty.org.uk (ssl)', 'https://tpb.pirateparty.org.uk'),
|
||||||
('(UK) tpb.pirateparty.org.uk (ssl)',
|
('(Netherlands) argumentomteemigreren.nl', 'http://argumentomteemigreren.nl'),
|
||||||
'https://tpb.pirateparty.org.uk'),
|
|
||||||
('(Netherlands) argumentomteemigreren.nl',
|
|
||||||
'http://argumentomteemigreren.nl'),
|
|
||||||
('(direct) 194.71.107.80', 'http://194.71.107.80'),
|
('(direct) 194.71.107.80', 'http://194.71.107.80'),
|
||||||
('(direct) 194.71.107.81', 'http://194.71.107.81'),
|
('(direct) 194.71.107.81', 'http://194.71.107.81'),
|
||||||
('(direct) 194.71.107.82', 'http://194.71.107.82'),
|
('(direct) 194.71.107.82', 'http://194.71.107.82'),
|
||||||
@@ -42,8 +38,8 @@ class TPBProxy(object):
|
|||||||
|
|
||||||
# compare lists and user/default value, exclude filter
|
# compare lists and user/default value, exclude filter
|
||||||
|
|
||||||
unused = [item for item in TPBProxy.list if item
|
unused = [item for item in TPBProxy.list if item not in http_failed_disabled and current
|
||||||
not in http_failed_disabled and current not in item]
|
not in item]
|
||||||
|
|
||||||
if len(unused) > 0:
|
if len(unused) > 0:
|
||||||
|
|
||||||
@@ -71,29 +67,35 @@ class ThePirateBay(TorrentProvider):
|
|||||||
]), ([202], ['dvdr'])]
|
]), ([202], ['dvdr'])]
|
||||||
|
|
||||||
cat_backup_id = 200
|
cat_backup_id = 200
|
||||||
|
disable_provider = False
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(ThePirateBay, self).__init__()
|
super(ThePirateBay, self).__init__()
|
||||||
self.urls = {'test': self.api_domain(),
|
self.urls = {'test': self.api_domain(), 'detail': '%s/torrent/%s',
|
||||||
'detail': '%s/torrent/%s',
|
|
||||||
'search': '%s/search/%s/0/7/%d'}
|
'search': '%s/search/%s/0/7/%d'}
|
||||||
|
|
||||||
def api_domain(self, url=''):
|
def api_domain(self, url=''):
|
||||||
|
|
||||||
# default domain
|
# default domain
|
||||||
|
|
||||||
domain = ('http://thepiratebay.se', self.conf('domain_for_tpb'
|
domain = self.conf('domain_for_tpb', default='http://thepiratebay.se')
|
||||||
))[self.conf('domain_for_tpb') != None]
|
self.log.info('Selected domain for this request: %s', domain)
|
||||||
|
|
||||||
host = urlparse(domain).hostname
|
host = urlparse(domain).hostname
|
||||||
|
|
||||||
# Clear disabled list for default or user selected host if time expired
|
# Clear disabled list for default or user selected host if time expired
|
||||||
|
|
||||||
if self.http_failed_disabled.get(host, 0) > 0:
|
if self.http_failed_disabled.get(host, 0) > 0:
|
||||||
if self.http_failed_disabled[host] > time.time() - 900:
|
if self.http_failed_disabled[host] > time.time() - 900:
|
||||||
|
|
||||||
# get new random domain
|
# get new random domain
|
||||||
domain = TPBProxy.get_proxy(self.http_failed_disabled, domain)
|
|
||||||
|
try:
|
||||||
|
domain = TPBProxy.get_proxy(self.http_failed_disabled, domain)
|
||||||
|
except Exception, err:
|
||||||
|
self.disable_provider = True
|
||||||
|
self.log.error(err)
|
||||||
else:
|
else:
|
||||||
|
|
||||||
del self.http_failed_request[host]
|
del self.http_failed_request[host]
|
||||||
del self.http_failed_disabled[host]
|
del self.http_failed_disabled[host]
|
||||||
|
|
||||||
@@ -102,15 +104,15 @@ class ThePirateBay(TorrentProvider):
|
|||||||
def search(self, movie, quality):
|
def search(self, movie, quality):
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
if self.isDisabled():
|
if self.isDisabled() or self.disable_provider:
|
||||||
return results
|
return results
|
||||||
|
|
||||||
cache_key = 'thepiratebay.%s.%s' % (movie['library'
|
cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'
|
||||||
]['identifier'], quality.get('identifier'))
|
))
|
||||||
search_url = self.urls['search'] % (self.api_domain(),
|
search_url = self.urls['search'] % (self.api_domain(),
|
||||||
self.for_search(getTitle(movie['library']) + ' '
|
self.for_search(getTitle(movie['library']) + ' '
|
||||||
+ quality['identifier']),
|
+ quality['identifier']),
|
||||||
self.getCatId(quality['identifier'])[0])
|
self.getCatId(quality['identifier'])[0])
|
||||||
self.log.info('searchUrl: %s', search_url)
|
self.log.info('searchUrl: %s', search_url)
|
||||||
data = self.getCache(cache_key, search_url)
|
data = self.getCache(cache_key, search_url)
|
||||||
|
|
||||||
@@ -122,8 +124,7 @@ class ThePirateBay(TorrentProvider):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
soup = BeautifulSoup(data)
|
soup = BeautifulSoup(data)
|
||||||
results_table = soup.find('table',
|
results_table = soup.find('table', attrs={'id': 'searchResult'})
|
||||||
attrs={'id': 'searchResult'})
|
|
||||||
entries = results_table.find_all('tr')
|
entries = results_table.find_all('tr')
|
||||||
for result in entries[1:]:
|
for result in entries[1:]:
|
||||||
link = result.find(href=re.compile('torrent\/\d+\/'))
|
link = result.find(href=re.compile('torrent\/\d+\/'))
|
||||||
@@ -131,8 +132,7 @@ class ThePirateBay(TorrentProvider):
|
|||||||
|
|
||||||
# Uploaded 06-28 02:27, Size 1.37 GiB,
|
# Uploaded 06-28 02:27, Size 1.37 GiB,
|
||||||
|
|
||||||
size = re.search('Size (?P<size>.+),',
|
size = re.search('Size (?P<size>.+),', unicode(result.select('font.detDesc'
|
||||||
unicode(result.select('font.detDesc'
|
|
||||||
)[0])).group('size')
|
)[0])).group('size')
|
||||||
if link and download:
|
if link and download:
|
||||||
new = {
|
new = {
|
||||||
@@ -142,42 +142,29 @@ class ThePirateBay(TorrentProvider):
|
|||||||
'provider': self.getName(),
|
'provider': self.getName(),
|
||||||
}
|
}
|
||||||
|
|
||||||
trusted = (0, 10)[result.find('img',
|
trusted = (0, 10)[result.find('img', alt=re.compile('Trusted')) != None]
|
||||||
alt=re.compile('Trusted')) != None]
|
vip = (0, 20)[result.find('img', alt=re.compile('VIP')) != None]
|
||||||
vip = (0, 20)[result.find('img',
|
confirmed = (0, 30)[result.find('img', alt=re.compile('Helpers')) != None]
|
||||||
alt=re.compile('VIP')) != None]
|
moderated = (0, 50)[result.find('img', alt=re.compile('Moderator')) != None]
|
||||||
confirmed = (0, 30)[result.find('img',
|
is_imdb = self.imdb_match(self.api_domain(link['href']), movie['library'
|
||||||
alt=re.compile('Helpers')) != None]
|
]['identifier'])
|
||||||
moderated = (0, 50)[result.find('img',
|
|
||||||
alt=re.compile('Moderator')) != None]
|
|
||||||
is_imdb = \
|
|
||||||
self.imdb_match(self.api_domain(link['href']),
|
|
||||||
movie['library']['identifier'])
|
|
||||||
|
|
||||||
self.log.info('Name: %s', link.string)
|
self.log.info('Name: %s', link.string)
|
||||||
self.log.info('Seeders: %s', result.find_all('td'
|
self.log.info('Seeders: %s', result.find_all('td')[2].string)
|
||||||
)[2].string)
|
self.log.info('Leechers: %s', result.find_all('td')[3].string)
|
||||||
self.log.info('Leechers: %s', result.find_all('td'
|
|
||||||
)[3].string)
|
|
||||||
self.log.info('Size: %s', size)
|
self.log.info('Size: %s', size)
|
||||||
self.log.info('Score(trusted + vip + moderated): %d'
|
self.log.info('Score(trusted + vip + moderated): %d', confirmed + trusted + vip
|
||||||
, confirmed + trusted + vip
|
|
||||||
+ moderated)
|
+ moderated)
|
||||||
|
|
||||||
new['name'] = link.string
|
new['name'] = link.string
|
||||||
new['id'] = re.search('/(?P<id>\d+)/', link['href'
|
new['id'] = re.search('/(?P<id>\d+)/', link['href']).group('id')
|
||||||
]).group('id')
|
|
||||||
new['url'] = self.api_domain(link['href'])
|
new['url'] = self.api_domain(link['href'])
|
||||||
new['magnet'] = download['href']
|
new['magnet'] = download['href']
|
||||||
new['size'] = self.parseSize(size)
|
new['size'] = self.parseSize(size)
|
||||||
new['seeders'] = int(result.find_all('td'
|
new['seeders'] = int(result.find_all('td')[2].string)
|
||||||
)[2].string)
|
new['leechers'] = int(result.find_all('td')[3].string)
|
||||||
new['leechers'] = int(result.find_all('td'
|
new['extra_score'] = lambda x: confirmed + trusted + vip + moderated
|
||||||
)[3].string)
|
new['score'] = fireEvent('score.calculate', new, movie, single=True)
|
||||||
new['extra_score'] = lambda x: confirmed + trusted \
|
|
||||||
+ vip + moderated
|
|
||||||
new['score'] = fireEvent('score.calculate', new,
|
|
||||||
movie, single=True)
|
|
||||||
|
|
||||||
is_correct_movie = fireEvent(
|
is_correct_movie = fireEvent(
|
||||||
'searcher.correct_movie',
|
'searcher.correct_movie',
|
||||||
|
|||||||
Reference in New Issue
Block a user