dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Torrent provider cleanup

Browse Source
This commit is contained in:
Ruud
2012-07-08 12:21:22 +02:00
parent 1018a7dd32
commit 0dfefef0c5
13 changed files with 311 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/downloaders/transmission/__init__.py
+1
View File
@@ -34,6 +34,7 @@ config = [{
{
'name': 'paused',
'type': 'bool',
'default': False,
'description': 'Add the torrent paused.',
},
{
couchpotato/core/downloaders/transmission/main.py
+74 -117
View File
@@ -1,98 +1,124 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from base64 import b64encode
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import isInt
from couchpotato.core.logger import CPLog
import urllib2
import httplib
import json
import re
import urllib2
log = CPLog(__name__)
class Transmission(Downloader):
type = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
def download(self, data, movie, manual = False, filedata = None):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return
log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type')))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
# Set parameters for Transmission
params = {
'paused': self.conf('paused', default = 0),
'download-dir': self.conf('directory', default = None)
}
torrent_params = {
'seedRatioLimit': self.conf('ratio'),
'seedRatioMode': (0 if self.conf('ratio') else 1)
}
if not filedata and data.get('type') == 'torrent':
log.error('Failed sending torrent, no data')
return False
# Send request to Transmission
try:
trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password'))
if data.get('type') == 'torrent_magnet':
remote_torrent = trpc.add_torrent_uri(data.get('url'), arguments = params)
else:
remote_torrent = trpc.add_torrent_file(b64encode(filedata), arguments = params)
# Change settings of added torrents
trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params)
return True
except Exception, err:
log.error('Failed to change settings for transfer: %s', err)
return False
class TransmissionRPC(object):
"""TransmissionRPC lite library"""
log = CPLog(__name__)
def __init__(
self,
host='localhost',
port=9091,
username=None,
password=None,
):
def __init__(self, host = 'localhost', port = 9091, username = None, password = None):
super(TransmissionRPC, self).__init__()
self.url = 'http://' + host + ':' + str(port) \
+ '/transmission/rpc'
self.url = 'http://' + host + ':' + str(port) + '/transmission/rpc'
self.tag = 0
self.session_id = 0
self.session = {}
if username and password:
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None, uri=self.url,
user=username, passwd=password)
opener = \
urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager),
urllib2.HTTPDigestAuthHandler(password_manager))
opener.addheaders = [('User-agent',
'couchpotato-transmission-client/1.0')]
password_manager.add_password(realm = None, uri = self.url, user = username, passwd = password)
opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager), urllib2.HTTPDigestAuthHandler(password_manager))
opener.addheaders = [('User-agent', 'couchpotato-transmission-client/1.0')]
urllib2.install_opener(opener)
elif username or password:
self.log.debug('User or password missing, not using authentication.'
)
log.debug('User or password missing, not using authentication.')
self.session = self.get_session()
def _request(self, ojson):
self.tag += 1
headers = {'x-transmission-session-id': str(self.session_id)}
request = urllib2.Request(self.url,
json.dumps(ojson).encode('utf-8'),
headers)
request = urllib2.Request(self.url, json.dumps(ojson).encode('utf-8'), headers)
try:
open_request = urllib2.urlopen(request)
response = json.loads(open_request.read())
self.log.debug('response: '
+ str(json.dumps(response).encode('utf-8')))
log.debug('response: %s', json.dumps(response))
if response['result'] == 'success':
self.log.debug(u'Transmission action successfull')
log.debug('Transmission action successfull')
return response['arguments']
else:
self.log.debug('Unknown failure sending command to Transmission. Return text is: '
+ response['result'])
log.debug('Unknown failure sending command to Transmission. Return text is: %s', response['result'])
return False
except httplib.InvalidURL, err:
self.log.error(u'Invalid Transmission host, check your config %s'
% err)
log.error('Invalid Transmission host, check your config %s', err)
return False
except urllib2.HTTPError, err:
if err.code == 401:
self.log.error(u'Invalid Transmission Username or Password, check your config'
)
log.error('Invalid Transmission Username or Password, check your config')
return False
elif err.code == 409:
msg = str(err.read())
try:
self.session_id = \
re.search('X-Transmission-Session-Id:\s*(\w+)',
msg).group(1)
self.log.debug('X-Transmission-Session-Id: '
+ self.session_id)
re.search('X-Transmission-Session-Id:\s*(\w+)', msg).group(1)
log.debug('X-Transmission-Session-Id: %s', self.session_id)
# #resend request with the updated header
return self._request(ojson)
except:
self.log.error(u'Unable to get Transmission Session-Id %s'
% err)
log.error('Unable to get Transmission Session-Id %s', err)
else:
self.log.error(u'TransmissionRPC HTTPError: %s' % err)
log.error('TransmissionRPC HTTPError: %s', err)
except urllib2.URLError, err:
self.log.error(u'Unable to connect to Transmission %s' % err)
log.error('Unable to connect to Transmission %s', err)
def get_session(self):
post_data = {'method': 'session-get', 'tag': self.tag}
@@ -100,84 +126,15 @@ class TransmissionRPC(object):
def add_torrent_uri(self, torrent, arguments):
arguments['filename'] = torrent
post_data = {'arguments': arguments, 'method': 'torrent-add',
'tag': self.tag}
post_data = {'arguments': arguments, 'method': 'torrent-add', 'tag': self.tag}
return self._request(post_data)
def add_torrent_file(self, torrent, arguments):
arguments['metainfo'] = torrent
post_data = {'arguments': arguments, 'method': 'torrent-add',
'tag': self.tag}
post_data = {'arguments': arguments, 'method': 'torrent-add', 'tag': self.tag}
return self._request(post_data)
def set_torrent(self, torrent_id, arguments):
arguments['ids'] = torrent_id
post_data = {'arguments': arguments, 'method': 'torrent-set',
'tag': self.tag}
post_data = {'arguments': arguments, 'method': 'torrent-set', 'tag': self.tag}
return self._request(post_data)
class Transmission(Downloader):
type = ['torrent', 'magnet']
log = CPLog(__name__)
def download(
self,
data,
movie,
manual=False,
filedata=None,
):
print data
if self.isDisabled(manual) \
or not self.isCorrectType(data.get('type')):
return
self.log.debug('Sending "%s" to Transmission.', data.get('name'))
self.log.debug('Type "%s" to Transmission.', data.get('type'))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
self.log.error('Config properties are not filled in correctly, port is missing.'
)
return False
# Set parameters for Transmission
params = {'paused': self.conf('paused', default=0),
'download-dir': self.conf('directory', default=None)}
torrent_params = {'seedRatioLimit': self.conf('ratio'),
'seedRatioMode': (0 if self.conf('ratio'
) else 1)}
if not filedata or data.get('type') != 'magnet':
self.log.error('Failed sending torrent, no data')
# Send request to Transmission
try:
trpc = TransmissionRPC(host[0], port=host[1],
username=self.conf('username'),
password=self.conf('password'))
if data.get('type') == 'magnet' or data.get('magnet') \
!= None:
remote_torrent = trpc.add_torrent_uri(data.get('magnet'),
arguments=params)
else:
remote_torrent = \
trpc.add_torrent_file(b64encode(filedata),
arguments=params)
# Change settings of added torrents
trpc.set_torrent(remote_torrent['torrent-added']['hashString'
], torrent_params)
return True
except Exception, err:
self.log.error('Failed to change settings for transfer: %s', err)
return False
couchpotato/core/plugins/release/main.py
+1 -1
View File
@@ -143,7 +143,7 @@ class Release(Plugin):
item[info.identifier] = info.value
# Get matching provider
provider = fireEvent('provider.belongs_to', item['url'], single = True)
provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True)
item['download'] = provider.download
fireEvent('searcher.download', data = item, movie = rel.movie.to_dict({
couchpotato/core/plugins/score/scores.py
-1
View File
@@ -1,7 +1,6 @@
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.plugins.scanner.main import Scanner
from couchpotato.environment import Env
import re
couchpotato/core/plugins/searcher/main.py
+4 -1
View File
@@ -29,6 +29,9 @@ class Searcher(Plugin):
# Schedule cronjob
fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute'))
addEvent('app.load', self.all_movies)
def all_movies(self):
if self.in_progress:
@@ -193,7 +196,7 @@ class Searcher(Plugin):
if filedata is 'try_next':
return filedata
successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True, filedata = filedata)
successful = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True)
if successful:
couchpotato/core/providers/base.py
+4 -20
View File
@@ -65,8 +65,11 @@ class YarrProvider(Provider):
def search(self, movie, quality):
return []
def belongsTo(self, url, host = None):
def belongsTo(self, url, provider = None, host = None):
try:
if provider and provider == self.getName():
return self
hostname = urlparse(url).hostname
if host and hostname in host:
return self
@@ -108,24 +111,5 @@ class YarrProvider(Provider):
return [self.cat_backup_id]
def imdb_match(self, url, imdb_id):
""" Searches for imdb_id in url of webpage """
log.info('Finding if imbd_id(%s) is found in url: %s' % (imdb_id, url))
try:
data = self.urlopen(url)
except:
log.error('Failed to open %s.' % url)
return False
imdb_id_alt = re.sub('tt[0]*', 'tt', imdb_id)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdb_id in data or 'imdb.com/title/' \
+ imdb_id_alt in data:
return True
return False
def for_search(self, string):
""" Prepare string for search, removing all characters that might confuse search engine"""
return quote_plus(simplifyString(string))
def found(self, new):
log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new)
couchpotato/core/providers/torrent/base.py
+8 -6
View File
@@ -1,4 +1,4 @@
from couchpotato.core.helpers.variable import getImdb
from couchpotato.core.helpers.variable import getImdb, md5
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import YarrProvider
import cookielib
@@ -19,7 +19,8 @@ class TorrentProvider(YarrProvider):
if url[:4] == 'http':
try:
data = self.urlopen(url)
cache_key = md5(url)
data = self.getCache(cache_key, url)
except IOError:
log.error('Failed to open %s.', url)
return False
@@ -34,7 +35,7 @@ class TorrentProvider(YarrProvider):
cookiejar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
urllib2.install_opener(opener)
f = opener.open(self.urls['login'], self.getLoginParam())
f = opener.open(self.urls['login'], self.getLoginParams())
f.read()
f.close()
self.login_opener = opener
@@ -44,12 +45,13 @@ class TorrentProvider(YarrProvider):
return False
def download(self, url = '', nzb_id = ''):
def loginDownload(self, url = '', nzb_id = ''):
try:
if not self.login_opener and not self.login():
log.error('Failed downloading from %s', self.getName())
return self.urlopen(url, opener = self.login_opener)
except:
log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return ''
couchpotato/core/providers/torrent/publichd/main.py
+72 -83
View File
@@ -1,116 +1,105 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from urlparse import parse_qs
import re
import traceback
log = CPLog(__name__)
class PublicHD(TorrentProvider):
log = CPLog(__name__)
urls = {
'test': 'http://publichd.eu',
'download': 'http://publichd.eu/%s',
'detail': 'http://publichd.eu/index.php?page=torrent-details&id=%s',
'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d',
}
'search': 'http://publichd.eu/index.php',
}
cat_ids = [([2], ['720p']), ([5], ['1080p']), ([16], ['brrip']),
([16], ['bd50'])]
cat_ids = [
([9], ['bd50']),
([5], ['1080p']),
([2], ['720p']),
([15, 16], ['brrip']),
]
cat_backup_id = 0
http_time_between_calls = 0
def search(self, movie, quality):
results = []
if self.isDisabled() and quality['hd'] != True:
if self.isDisabled() or quality['hd'] != True:
return results
cache_key = 'publichd.%s.%s' % (movie['library']['identifier'],
quality.get('identifier'))
search_url = self.urls['search'] \
% (self.for_search(getTitle(movie['library'])
+ ' ' + quality['identifier']),
self.getCatId(quality['identifier'])[0])
self.log.info('searchUrl: %s', search_url)
data = self.getCache(cache_key, search_url)
if not data:
self.log.error('Failed to get data from %s.', search_url)
return results
params = tryUrlencode({
'page':'torrents',
'search': getTitle(movie['library']) + ' ' + quality['identifier'],
'active': 1,
'category': self.getCatId(quality['identifier'])[0]
})
url = '%s?%s' % (self.urls['search'], params)
try:
soup = BeautifulSoup(data)
cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url)
results_table = soup.find('table',
attrs={'id': 'bgtorrlist2'})
entries = results_table.find_all('tr')
for result in entries[2:len(entries) - 1]:
info_url = result.find(href=re.compile('torrent-details'
))
download = result.find(href=re.compile('\.torrent'))
if data:
if info_url and download:
new = {
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
try:
soup = BeautifulSoup(data)
results_table = soup.find('table', attrs = {'id': 'bgtorrlist2'})
entries = results_table.find_all('tr')
for result in entries[2:len(entries) - 1]:
info_url = result.find(href = re.compile('torrent-details'))
download = result.find(href = re.compile('\.torrent'))
if info_url and download:
url = parse_qs(info_url['href'])
new = {
'id': url['id'][0],
'name': info_url.string,
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'download': self.download,
'url': self.urls['download'] % download['href'],
'detail_url': self.urls['detail'] % url['id'][0],
'size': self.parseSize(result.find_all('td')[7].string),
'seeders': tryInt(result.find_all('td')[4].string),
'leechers': tryInt(result.find_all('td')[5].string),
'get_more_info': self.getMoreInfo
}
self.log.debug('Name: %s', result.find_all('td'
)[1].string)
self.log.debug('Seeders: %s', result.find_all('td'
)[4].string)
self.log.debug('Leaches: %s', result.find_all('td'
)[5].string)
self.log.debug('Size: %s', result.find_all('td'
)[7].string)
url = parse_qs(info_url['href'])
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single_category = False, single = True)
new['name'] = info_url.string
new['id'] = url['id'][0]
new['url'] = self.urls['download'] % download['href'
]
new['size'] = self.parseSize(result.find_all('td'
)[7].string)
new['seeders'] = int(result.find_all('td'
)[4].string)
new['leechers'] = int(result.find_all('td'
)[5].string)
if is_correct_movie:
results.append(new)
self.found(new)
new['score'] = fireEvent('score.calculate', new,
movie, single=True)
is_imdb = self.imdb_match(self.urls['detail']
% new['id'], movie['library']['identifier'])
is_correct_movie = fireEvent(
'searcher.correct_movie',
nzb=new,
movie=movie,
quality=quality,
imdb_results=is_imdb,
single_category=False,
single=True,
)
return results
if is_correct_movie:
new['download'] = self.download
results.append(new)
self.found(new)
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return results
except Exception, err:
self.log.debug(err)
self.log.info('Error occured during parsing! Passing only processed entries'
)
return results
return []
def download(self, url='', nzb_id=''):
self.log.info('Downloading: %s', url)
torrent = self.urlopen(url)
return torrent
def getMoreInfo(self, item):
full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
html = BeautifulSoup(full_description)
nfo_pre = html.find('div', attrs = {'id':'torrmain'})
description = toUnicode(nfo_pre.text) if nfo_pre else ''
item['description'] = description
return item
couchpotato/core/providers/torrent/sceneaccess/main.py
+24 -14
View File
@@ -1,12 +1,11 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode, \
toUnicode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from urllib import quote_plus
import traceback
import urllib
log = CPLog(__name__)
@@ -35,15 +34,16 @@ class SceneAccess(TorrentProvider):
if self.isDisabled():
return results
url = self.urls['search'] % (
self.getCatId(quality['identifier'])[0],
self.getCatId(quality['identifier'])[0]
)
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
arguments = tryUrlencode({
'search': q,
})
url = "%s&%s" % (self.urls['search'], arguments)
url = url % (
self.getCatId(quality['identifier'])[0],
self.getCatId(quality['identifier'])[0]
)
url = "%s&%s" % (url, arguments)
# Do login for the cookies
if not self.login_opener and not self.login():
@@ -63,26 +63,27 @@ class SceneAccess(TorrentProvider):
link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a')
url = result.find('td', attrs = {'class' : 'td_dl'}).find('a')
leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a')
id = link['href'].replace('details?id=', '')
new = {
'id': link['href'].replace('details?id=', ''),
'id': id,
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'name': link['title'],
'url': self.urls['download'] % url['href'],
'detail_url': self.urls['detail'] % id,
'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]),
'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string),
'leechers': tryInt(leechers.string) if leechers else 0,
'download': self.download,
'download': self.loginDownload,
'get_more_info': self.getMoreInfo,
}
imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier'])
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = imdb_results, single_category = False, single = True)
imdb_results = False, single_category = False, single = True)
if is_correct_movie:
results.append(new)
@@ -94,9 +95,18 @@ class SceneAccess(TorrentProvider):
return []
def getLoginParams(self, params):
def getLoginParams(self):
return tryUrlencode({
'username': self.conf('username'),
'password': self.conf('password'),
'submit': 'come on in',
})
def getMoreInfo(self, item):
full_description = self.getCache('sceneaccess.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
html = BeautifulSoup(full_description)
nfo_pre = html.find('div', attrs = {'id':'details_table'})
description = toUnicode(nfo_pre.text) if nfo_pre else ''
item['description'] = description
return item
couchpotato/core/providers/torrent/scenehd/main.py
+1 -2
View File
@@ -5,7 +5,6 @@ from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
import traceback
import urllib
log = CPLog(__name__)
@@ -72,7 +71,7 @@ class SceneHD(TorrentProvider):
'seeders': tryInt(all_cells[10].find('a').string),
'leechers': tryInt(leechers),
'url': self.urls['download'] % id,
'download': self.download,
'download': self.loginDownload,
}
imdb_link = all_cells[1].find('a')
couchpotato/core/providers/torrent/thepiratebay/__init__.py
+22 -21
View File
@@ -1,26 +1,27 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from main import ThePirateBay
from main import TPBProxy
def start():
return ThePirateBay()
config = [{'name': 'ThePirateBay', 'groups': [{
'tab': 'searcher',
'subtab': 'providers',
'name': 'ThePirateBay',
'description': 'The world\'s largest bittorrent tracker.',
'options': [{'name': 'enabled', 'type': 'enabler',
'default': False}, {
'name': 'domain_for_tpb',
'label': 'Proxy server',
'default': 'http://thepiratebay.se',
'description': 'Default domain for requests',
'type': 'dropdown',
'values': TPBProxy.list,
}],
}]}]
config = [{
'name': 'thepiratebay',
'groups': [{
'tab': 'searcher',
'subtab': 'providers',
'name': 'ThePirateBay',
'description': 'The world\'s largest bittorrent tracker.',
'options': [
{
'name': 'enabled',
'type': 'enabler',
'default': False
},
{
'name': 'domain',
'advanced': True,
'label': 'Proxy server',
'description': 'Domain for requests, keep empty to let CouchPotato pick.',
}
],
}]
}]
couchpotato/core/providers/torrent/thepiratebay/main.py
+99 -146
View File
@@ -1,189 +1,142 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.variable import getTitle, tryInt, cleanHost
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from random import sample as random
from urlparse import urlparse
from couchpotato.environment import Env
from urllib import quote_plus
import re
import time
import traceback
class TPBProxy(object):
""" TPBProxy deals with failed or blocked TPB proxys.
It works as round-robin balancer, if user seleced
or default domain becomes unavaliable.
"""
list = [
('(Sweden) thepiratebay.se', 'http://thepiratebay.se'),
('(Sweden) tpb.ipredator.se (ssl)', 'https://tpb.ipredator.se'),
('(Germany) depiraatbaai.be', 'http://depiraatbaai.be'),
('(UK) piratereverse.info (ssl)', 'https://piratereverse.info'),
('(UK) tpb.pirateparty.org.uk (ssl)', 'https://tpb.pirateparty.org.uk'),
('(Netherlands) argumentomteemigreren.nl', 'http://argumentomteemigreren.nl'),
('(direct) 194.71.107.80', 'http://194.71.107.80'),
('(direct) 194.71.107.81', 'http://194.71.107.81'),
('(direct) 194.71.107.82', 'http://194.71.107.82'),
('(direct) 194.71.107.83', 'http://194.71.107.83'),
]
@staticmethod
def get_proxy(http_failed_disabled=None, current=None):
# compare lists and user/default value, exclude filter
unused = [item for item in TPBProxy.list if item not in http_failed_disabled and current
not in item]
if len(unused) > 0:
# only return uri
return random(unused, 1)[0][1]
else:
# this should disable provider for some time
raise Exception('All ThePirateBay proxies are exhausted')
log = CPLog(__name__)
class ThePirateBay(TorrentProvider):
log = CPLog(__name__)
cat_ids = [([207], ['720p', '1080p']), ([201], [
'cam',
'ts',
'dvdrip',
'tc',
'r5',
'scr',
'brrip',
]), ([202], ['dvdr'])]
urls = {
'detail': '%s/torrent/%s',
'search': '%s/search/%s/0/7/%d'
}
cat_ids = [
([207], ['720p', '1080p']),
([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']),
([202], ['dvdr'])
]
cat_backup_id = 200
disable_provider = False
http_time_between_calls = 0
proxy_list = [
'https://thepiratebay.se',
'https://tpb.ipredator.se',
'https://depiraatbaai.be',
'https://piratereverse.info',
'https://tpb.pirateparty.org.uk',
'https://argumentomteemigreren.nl',
]
def __init__(self):
self.domain = self.conf('domain')
super(ThePirateBay, self).__init__()
self.urls = {'test': self.api_domain(), 'detail': '%s/torrent/%s',
'search': '%s/search/%s/0/7/%d'}
def api_domain(self, url=''):
def getDomain(self, url = ''):
# default domain
if not self.domain:
for proxy in self.proxy_list:
domain = self.conf('domain_for_tpb', default='http://thepiratebay.se')
self.log.info('Selected domain for this request: %s', domain)
host = urlparse(domain).hostname
# Clear disabled list for default or user selected host if time expired
if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > time.time() - 900:
# get new random domain
prop_name = 'tpb_proxy.%s' % proxy
last_check = float(Env.prop(prop_name, default = 0))
if last_check > time.time() - 1209600:
continue
data = ''
try:
domain = TPBProxy.get_proxy(self.http_failed_disabled, domain)
except Exception, err:
self.disable_provider = True
self.log.error(err)
else:
data = self.urlopen(proxy, timeout = 3)
except:
log.debug('Failed tpb proxy %s', proxy)
del self.http_failed_request[host]
del self.http_failed_disabled[host]
if 'title="Pirate Search"' in data:
log.debug('Using proxy: %s', proxy)
self.domain = proxy
break
return domain + url
Env.prop(prop_name, time.time())
if not self.domain:
log.error('No TPB proxies left, please add one in settings, or let us know which one to add on the forum.')
return None
return cleanHost(self.domain).rstrip('/') + url
def search(self, movie, quality):
results = []
if self.isDisabled() or self.disable_provider:
if self.isDisabled() or not self.getDomain():
return results
cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'
))
search_url = self.urls['search'] % (self.api_domain(),
self.for_search(getTitle(movie['library']) + ' '
+ quality['identifier']),
self.getCatId(quality['identifier'])[0])
self.log.info('searchUrl: %s', search_url)
cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
search_url = self.urls['search'] % (self.getDomain(), quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getCache(cache_key, search_url)
# print data
if data:
try:
soup = BeautifulSoup(data)
results_table = soup.find('table', attrs = {'id': 'searchResult'})
entries = results_table.find_all('tr')
for result in entries[1:]:
link = result.find(href = re.compile('torrent\/\d+\/'))
download = result.find(href = re.compile('magnet:'))
if not data:
self.log.error('Failed to get data from %s.', search_url)
return results
size = re.search('Size (?P<size>.+),', unicode(result.select('font.detDesc')[0])).group('size')
if link and download:
try:
soup = BeautifulSoup(data)
results_table = soup.find('table', attrs={'id': 'searchResult'})
entries = results_table.find_all('tr')
for result in entries[1:]:
link = result.find(href=re.compile('torrent\/\d+\/'))
download = result.find(href=re.compile('magnet:'))
def extra_score(item):
trusted = (0, 10)[result.find('img', alt = re.compile('Trusted')) != None]
vip = (0, 20)[result.find('img', alt = re.compile('VIP')) != None]
confirmed = (0, 30)[result.find('img', alt = re.compile('Helpers')) != None]
moderated = (0, 50)[result.find('img', alt = re.compile('Moderator')) != None]
# Uploaded 06-28 02:27, Size 1.37 GiB,
return confirmed + trusted + vip + moderated
size = re.search('Size (?P<size>.+),', unicode(result.select('font.detDesc'
)[0])).group('size')
if link and download:
new = {
'type': 'magnet',
'check_nzb': False,
'description': '',
'provider': self.getName(),
new = {
'id': re.search('/(?P<id>\d+)/', link['href']).group('id'),
'type': 'torrent_magnet',
'name': link.string,
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': download['href'],
'detail_url': self.getDomain(link['href']),
'size': self.parseSize(size),
'seeders': tryInt(result.find_all('td')[2].string),
'leechers': tryInt(result.find_all('td')[3].string),
'extra_score': extra_score,
'get_more_info': self.getMoreInfo
}
trusted = (0, 10)[result.find('img', alt=re.compile('Trusted')) != None]
vip = (0, 20)[result.find('img', alt=re.compile('VIP')) != None]
confirmed = (0, 30)[result.find('img', alt=re.compile('Helpers')) != None]
moderated = (0, 50)[result.find('img', alt=re.compile('Moderator')) != None]
is_imdb = self.imdb_match(self.api_domain(link['href']), movie['library'
]['identifier'])
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single_category = False, single = True)
self.log.info('Name: %s', link.string)
self.log.info('Seeders: %s', result.find_all('td')[2].string)
self.log.info('Leechers: %s', result.find_all('td')[3].string)
self.log.info('Size: %s', size)
self.log.info('Score(trusted + vip + moderated): %d', confirmed + trusted + vip
+ moderated)
if is_correct_movie:
results.append(new)
self.found(new)
new['name'] = link.string
new['id'] = re.search('/(?P<id>\d+)/', link['href']).group('id')
new['url'] = self.api_domain(link['href'])
new['magnet'] = download['href']
new['size'] = self.parseSize(size)
new['seeders'] = int(result.find_all('td')[2].string)
new['leechers'] = int(result.find_all('td')[3].string)
new['extra_score'] = lambda x: confirmed + trusted + vip + moderated
new['score'] = fireEvent('score.calculate', new, movie, single=True)
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
is_correct_movie = fireEvent(
'searcher.correct_movie',
nzb=new,
movie=movie,
quality=quality,
imdb_results=is_imdb,
single_category=False,
single=True,
)
return []
if is_correct_movie:
results.append(new)
self.found(new)
def getMoreInfo(self, item):
full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
html = BeautifulSoup(full_description)
nfo_pre = html.find('div', attrs = {'class':'nfo'})
description = toUnicode(nfo_pre.text) if nfo_pre else ''
return results
except Exception, error:
self.log.debug(error)
return results
def download(self, url='', nzb_id=''):
return url
item['description'] = description
return item
couchpotato/core/providers/torrent/torrentleech/main.py
+1 -1
View File
@@ -67,7 +67,7 @@ class TorrentLeech(TorrentProvider):
'description': '',
'provider': self.getName(),
'url': self.urls['download'] % url['href'],
'download': self.download,
'download': self.loginDownload,
'size': self.parseSize(result.find_all('td')[4].string),
'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string),
'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string),