diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index f72dea6b..189fcd91 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -34,6 +34,7 @@ config = [{ { 'name': 'paused', 'type': 'bool', + 'default': False, 'description': 'Add the torrent paused.', }, { diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 2ab24d14..1819f310 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -1,98 +1,124 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - from base64 import b64encode from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog - -import urllib2 import httplib import json import re +import urllib2 + +log = CPLog(__name__) + + +class Transmission(Downloader): + + type = ['torrent', 'torrent_magnet'] + log = CPLog(__name__) + + def download(self, data, movie, manual = False, filedata = None): + + if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): + return + + log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) + + # Load host from config and split out port. + host = self.conf('host').split(':') + if not isInt(host[1]): + log.error('Config properties are not filled in correctly, port is missing.') + return False + + # Set parameters for Transmission + params = { + 'paused': self.conf('paused', default = 0), + 'download-dir': self.conf('directory', default = None) + } + + torrent_params = { + 'seedRatioLimit': self.conf('ratio'), + 'seedRatioMode': (0 if self.conf('ratio') else 1) + } + + if not filedata and data.get('type') == 'torrent': + log.error('Failed sending torrent, no data') + return False + + # Send request to Transmission + try: + trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + if data.get('type') == 'torrent_magnet': + remote_torrent = trpc.add_torrent_uri(data.get('url'), arguments = params) + else: + remote_torrent = trpc.add_torrent_file(b64encode(filedata), arguments = params) + + # Change settings of added torrents + trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) + + return True + except Exception, err: + log.error('Failed to change settings for transfer: %s', err) + return False class TransmissionRPC(object): """TransmissionRPC lite library""" - log = CPLog(__name__) - - def __init__( - self, - host='localhost', - port=9091, - username=None, - password=None, - ): + def __init__(self, host = 'localhost', port = 9091, username = None, password = None): super(TransmissionRPC, self).__init__() - self.url = 'http://' + host + ':' + str(port) \ - + '/transmission/rpc' + + self.url = 'http://' + host + ':' + str(port) + '/transmission/rpc' self.tag = 0 self.session_id = 0 self.session = {} if username and password: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() - password_manager.add_password(realm=None, uri=self.url, - user=username, passwd=password) - opener = \ - urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager), - urllib2.HTTPDigestAuthHandler(password_manager)) - opener.addheaders = [('User-agent', - 'couchpotato-transmission-client/1.0')] + password_manager.add_password(realm = None, uri = self.url, user = username, passwd = password) + opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager), urllib2.HTTPDigestAuthHandler(password_manager)) + opener.addheaders = [('User-agent', 'couchpotato-transmission-client/1.0')] urllib2.install_opener(opener) elif username or password: - self.log.debug('User or password missing, not using authentication.' - ) + log.debug('User or password missing, not using authentication.') self.session = self.get_session() def _request(self, ojson): self.tag += 1 headers = {'x-transmission-session-id': str(self.session_id)} - request = urllib2.Request(self.url, - json.dumps(ojson).encode('utf-8'), - headers) + request = urllib2.Request(self.url, json.dumps(ojson).encode('utf-8'), headers) try: open_request = urllib2.urlopen(request) response = json.loads(open_request.read()) - self.log.debug('response: ' - + str(json.dumps(response).encode('utf-8'))) + log.debug('response: %s', json.dumps(response)) if response['result'] == 'success': - self.log.debug(u'Transmission action successfull') + log.debug('Transmission action successfull') return response['arguments'] else: - self.log.debug('Unknown failure sending command to Transmission. Return text is: ' - + response['result']) + log.debug('Unknown failure sending command to Transmission. Return text is: %s', response['result']) return False except httplib.InvalidURL, err: - self.log.error(u'Invalid Transmission host, check your config %s' - % err) + log.error('Invalid Transmission host, check your config %s', err) return False except urllib2.HTTPError, err: if err.code == 401: - self.log.error(u'Invalid Transmission Username or Password, check your config' - ) + log.error('Invalid Transmission Username or Password, check your config') return False elif err.code == 409: msg = str(err.read()) try: self.session_id = \ - re.search('X-Transmission-Session-Id:\s*(\w+)', - msg).group(1) - self.log.debug('X-Transmission-Session-Id: ' - + self.session_id) + re.search('X-Transmission-Session-Id:\s*(\w+)', msg).group(1) + log.debug('X-Transmission-Session-Id: %s', self.session_id) # #resend request with the updated header return self._request(ojson) except: - self.log.error(u'Unable to get Transmission Session-Id %s' - % err) + log.error('Unable to get Transmission Session-Id %s', err) else: - self.log.error(u'TransmissionRPC HTTPError: %s' % err) + log.error('TransmissionRPC HTTPError: %s', err) except urllib2.URLError, err: - self.log.error(u'Unable to connect to Transmission %s' % err) + log.error('Unable to connect to Transmission %s', err) def get_session(self): post_data = {'method': 'session-get', 'tag': self.tag} @@ -100,84 +126,15 @@ class TransmissionRPC(object): def add_torrent_uri(self, torrent, arguments): arguments['filename'] = torrent - post_data = {'arguments': arguments, 'method': 'torrent-add', - 'tag': self.tag} + post_data = {'arguments': arguments, 'method': 'torrent-add', 'tag': self.tag} return self._request(post_data) def add_torrent_file(self, torrent, arguments): arguments['metainfo'] = torrent - post_data = {'arguments': arguments, 'method': 'torrent-add', - 'tag': self.tag} + post_data = {'arguments': arguments, 'method': 'torrent-add', 'tag': self.tag} return self._request(post_data) def set_torrent(self, torrent_id, arguments): arguments['ids'] = torrent_id - post_data = {'arguments': arguments, 'method': 'torrent-set', - 'tag': self.tag} + post_data = {'arguments': arguments, 'method': 'torrent-set', 'tag': self.tag} return self._request(post_data) - - -class Transmission(Downloader): - - type = ['torrent', 'magnet'] - log = CPLog(__name__) - - def download( - self, - data, - movie, - manual=False, - filedata=None, - ): - - print data - if self.isDisabled(manual) \ - or not self.isCorrectType(data.get('type')): - return - - self.log.debug('Sending "%s" to Transmission.', data.get('name')) - self.log.debug('Type "%s" to Transmission.', data.get('type')) - - # Load host from config and split out port. - - host = self.conf('host').split(':') - if not isInt(host[1]): - self.log.error('Config properties are not filled in correctly, port is missing.' - ) - return False - - # Set parameters for Transmission - - params = {'paused': self.conf('paused', default=0), - 'download-dir': self.conf('directory', default=None)} - - torrent_params = {'seedRatioLimit': self.conf('ratio'), - 'seedRatioMode': (0 if self.conf('ratio' - ) else 1)} - - if not filedata or data.get('type') != 'magnet': - self.log.error('Failed sending torrent, no data') - - # Send request to Transmission - - try: - trpc = TransmissionRPC(host[0], port=host[1], - username=self.conf('username'), - password=self.conf('password')) - if data.get('type') == 'magnet' or data.get('magnet') \ - != None: - remote_torrent = trpc.add_torrent_uri(data.get('magnet'), - arguments=params) - else: - remote_torrent = \ - trpc.add_torrent_file(b64encode(filedata), - arguments=params) - - # Change settings of added torrents - - trpc.set_torrent(remote_torrent['torrent-added']['hashString' - ], torrent_params) - return True - except Exception, err: - self.log.error('Failed to change settings for transfer: %s', err) - return False diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 1fc11d53..10de0541 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -143,7 +143,7 @@ class Release(Plugin): item[info.identifier] = info.value # Get matching provider - provider = fireEvent('provider.belongs_to', item['url'], single = True) + provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True) item['download'] = provider.download fireEvent('searcher.download', data = item, movie = rel.movie.to_dict({ diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index ddff25b1..385e3a95 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -1,7 +1,6 @@ from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import simplifyString from couchpotato.core.helpers.variable import tryInt -from couchpotato.core.plugins.scanner.main import Scanner from couchpotato.environment import Env import re diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 18e89320..9ebdec66 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -29,6 +29,9 @@ class Searcher(Plugin): # Schedule cronjob fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + + addEvent('app.load', self.all_movies) + def all_movies(self): if self.in_progress: @@ -193,7 +196,7 @@ class Searcher(Plugin): if filedata is 'try_next': return filedata - successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True, filedata = filedata) + successful = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True) if successful: diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index d4a35442..d25377b4 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -65,8 +65,11 @@ class YarrProvider(Provider): def search(self, movie, quality): return [] - def belongsTo(self, url, host = None): + def belongsTo(self, url, provider = None, host = None): try: + if provider and provider == self.getName(): + return self + hostname = urlparse(url).hostname if host and hostname in host: return self @@ -108,24 +111,5 @@ class YarrProvider(Provider): return [self.cat_backup_id] - def imdb_match(self, url, imdb_id): - """ Searches for imdb_id in url of webpage """ - log.info('Finding if imbd_id(%s) is found in url: %s' % (imdb_id, url)) - try: - data = self.urlopen(url) - except: - log.error('Failed to open %s.' % url) - return False - imdb_id_alt = re.sub('tt[0]*', 'tt', imdb_id) - data = unicode(data, errors='ignore') - if 'imdb.com/title/' + imdb_id in data or 'imdb.com/title/' \ - + imdb_id_alt in data: - return True - return False - - def for_search(self, string): - """ Prepare string for search, removing all characters that might confuse search engine""" - return quote_plus(simplifyString(string)) - def found(self, new): log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new) diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index 04d191bc..fc2d35e2 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -1,4 +1,4 @@ -from couchpotato.core.helpers.variable import getImdb +from couchpotato.core.helpers.variable import getImdb, md5 from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import YarrProvider import cookielib @@ -19,7 +19,8 @@ class TorrentProvider(YarrProvider): if url[:4] == 'http': try: - data = self.urlopen(url) + cache_key = md5(url) + data = self.getCache(cache_key, url) except IOError: log.error('Failed to open %s.', url) return False @@ -34,7 +35,7 @@ class TorrentProvider(YarrProvider): cookiejar = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) urllib2.install_opener(opener) - f = opener.open(self.urls['login'], self.getLoginParam()) + f = opener.open(self.urls['login'], self.getLoginParams()) f.read() f.close() self.login_opener = opener @@ -44,12 +45,13 @@ class TorrentProvider(YarrProvider): return False - def download(self, url = '', nzb_id = ''): - + def loginDownload(self, url = '', nzb_id = ''): try: if not self.login_opener and not self.login(): log.error('Failed downloading from %s', self.getName()) - return self.urlopen(url, opener = self.login_opener) except: log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc())) + + def getLoginParams(self): + return '' diff --git a/couchpotato/core/providers/torrent/publichd/main.py b/couchpotato/core/providers/torrent/publichd/main.py index 673ca303..c962f935 100644 --- a/couchpotato/core/providers/torrent/publichd/main.py +++ b/couchpotato/core/providers/torrent/publichd/main.py @@ -1,116 +1,105 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode +from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider from urlparse import parse_qs - import re +import traceback + +log = CPLog(__name__) class PublicHD(TorrentProvider): - log = CPLog(__name__) urls = { 'test': 'http://publichd.eu', 'download': 'http://publichd.eu/%s', 'detail': 'http://publichd.eu/index.php?page=torrent-details&id=%s', - 'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d', - } + 'search': 'http://publichd.eu/index.php', + } - cat_ids = [([2], ['720p']), ([5], ['1080p']), ([16], ['brrip']), - ([16], ['bd50'])] + cat_ids = [ + ([9], ['bd50']), + ([5], ['1080p']), + ([2], ['720p']), + ([15, 16], ['brrip']), + ] cat_backup_id = 0 + http_time_between_calls = 0 def search(self, movie, quality): results = [] - if self.isDisabled() and quality['hd'] != True: + + if self.isDisabled() or quality['hd'] != True: return results - cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], - quality.get('identifier')) - search_url = self.urls['search'] \ - % (self.for_search(getTitle(movie['library']) - + ' ' + quality['identifier']), - self.getCatId(quality['identifier'])[0]) - self.log.info('searchUrl: %s', search_url) - data = self.getCache(cache_key, search_url) - if not data: - self.log.error('Failed to get data from %s.', search_url) - return results + params = tryUrlencode({ + 'page':'torrents', + 'search': getTitle(movie['library']) + ' ' + quality['identifier'], + 'active': 1, + 'category': self.getCatId(quality['identifier'])[0] + }) + url = '%s?%s' % (self.urls['search'], params) - try: - soup = BeautifulSoup(data) + cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + data = self.getCache(cache_key, url) - results_table = soup.find('table', - attrs={'id': 'bgtorrlist2'}) - entries = results_table.find_all('tr') - for result in entries[2:len(entries) - 1]: - info_url = result.find(href=re.compile('torrent-details' - )) - download = result.find(href=re.compile('\.torrent')) + if data: - if info_url and download: - new = { - 'type': 'torrent', - 'check_nzb': False, - 'description': '', - 'provider': self.getName(), + try: + soup = BeautifulSoup(data) + + results_table = soup.find('table', attrs = {'id': 'bgtorrlist2'}) + entries = results_table.find_all('tr') + + for result in entries[2:len(entries) - 1]: + info_url = result.find(href = re.compile('torrent-details')) + download = result.find(href = re.compile('\.torrent')) + + if info_url and download: + + url = parse_qs(info_url['href']) + + new = { + 'id': url['id'][0], + 'name': info_url.string, + 'type': 'torrent', + 'check_nzb': False, + 'description': '', + 'provider': self.getName(), + 'download': self.download, + 'url': self.urls['download'] % download['href'], + 'detail_url': self.urls['detail'] % url['id'][0], + 'size': self.parseSize(result.find_all('td')[7].string), + 'seeders': tryInt(result.find_all('td')[4].string), + 'leechers': tryInt(result.find_all('td')[5].string), + 'get_more_info': self.getMoreInfo } - self.log.debug('Name: %s', result.find_all('td' - )[1].string) - self.log.debug('Seeders: %s', result.find_all('td' - )[4].string) - self.log.debug('Leaches: %s', result.find_all('td' - )[5].string) - self.log.debug('Size: %s', result.find_all('td' - )[7].string) - url = parse_qs(info_url['href']) + new['score'] = fireEvent('score.calculate', new, movie, single = True) + is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, + imdb_results = False, single_category = False, single = True) - new['name'] = info_url.string - new['id'] = url['id'][0] - new['url'] = self.urls['download'] % download['href' - ] - new['size'] = self.parseSize(result.find_all('td' - )[7].string) - new['seeders'] = int(result.find_all('td' - )[4].string) - new['leechers'] = int(result.find_all('td' - )[5].string) + if is_correct_movie: + results.append(new) + self.found(new) - new['score'] = fireEvent('score.calculate', new, - movie, single=True) - is_imdb = self.imdb_match(self.urls['detail'] - % new['id'], movie['library']['identifier']) - is_correct_movie = fireEvent( - 'searcher.correct_movie', - nzb=new, - movie=movie, - quality=quality, - imdb_results=is_imdb, - single_category=False, - single=True, - ) + return results - if is_correct_movie: - new['download'] = self.download - results.append(new) - self.found(new) + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) - return results - except Exception, err: - self.log.debug(err) - self.log.info('Error occured during parsing! Passing only processed entries' - ) - return results + return [] - def download(self, url='', nzb_id=''): - self.log.info('Downloading: %s', url) - torrent = self.urlopen(url) - return torrent + def getMoreInfo(self, item): + full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) + html = BeautifulSoup(full_description) + nfo_pre = html.find('div', attrs = {'id':'torrmain'}) + description = toUnicode(nfo_pre.text) if nfo_pre else '' + + item['description'] = description + return item diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 7ba92cbd..20542941 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -1,12 +1,11 @@ from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode +from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode, \ + toUnicode from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider -from urllib import quote_plus import traceback -import urllib log = CPLog(__name__) @@ -35,15 +34,16 @@ class SceneAccess(TorrentProvider): if self.isDisabled(): return results + url = self.urls['search'] % ( + self.getCatId(quality['identifier'])[0], + self.getCatId(quality['identifier'])[0] + ) + q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier')) arguments = tryUrlencode({ 'search': q, }) - url = "%s&%s" % (self.urls['search'], arguments) - url = url % ( - self.getCatId(quality['identifier'])[0], - self.getCatId(quality['identifier'])[0] - ) + url = "%s&%s" % (url, arguments) # Do login for the cookies if not self.login_opener and not self.login(): @@ -63,26 +63,27 @@ class SceneAccess(TorrentProvider): link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a') url = result.find('td', attrs = {'class' : 'td_dl'}).find('a') leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a') + id = link['href'].replace('details?id=', '') new = { - 'id': link['href'].replace('details?id=', ''), + 'id': id, 'type': 'torrent', 'check_nzb': False, 'description': '', 'provider': self.getName(), 'name': link['title'], 'url': self.urls['download'] % url['href'], + 'detail_url': self.urls['detail'] % id, 'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]), 'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string), 'leechers': tryInt(leechers.string) if leechers else 0, - 'download': self.download, + 'download': self.loginDownload, + 'get_more_info': self.getMoreInfo, } - imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier']) - new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = imdb_results, single_category = False, single = True) + imdb_results = False, single_category = False, single = True) if is_correct_movie: results.append(new) @@ -94,9 +95,18 @@ class SceneAccess(TorrentProvider): return [] - def getLoginParams(self, params): + def getLoginParams(self): return tryUrlencode({ 'username': self.conf('username'), 'password': self.conf('password'), 'submit': 'come on in', }) + + def getMoreInfo(self, item): + full_description = self.getCache('sceneaccess.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) + html = BeautifulSoup(full_description) + nfo_pre = html.find('div', attrs = {'id':'details_table'}) + description = toUnicode(nfo_pre.text) if nfo_pre else '' + + item['description'] = description + return item diff --git a/couchpotato/core/providers/torrent/scenehd/main.py b/couchpotato/core/providers/torrent/scenehd/main.py index d7fb7f54..89d847d2 100644 --- a/couchpotato/core/providers/torrent/scenehd/main.py +++ b/couchpotato/core/providers/torrent/scenehd/main.py @@ -5,7 +5,6 @@ from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider import traceback -import urllib log = CPLog(__name__) @@ -72,7 +71,7 @@ class SceneHD(TorrentProvider): 'seeders': tryInt(all_cells[10].find('a').string), 'leechers': tryInt(leechers), 'url': self.urls['download'] % id, - 'download': self.download, + 'download': self.loginDownload, } imdb_link = all_cells[1].find('a') diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index d8e05600..8aa49111 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -1,26 +1,27 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - from main import ThePirateBay -from main import TPBProxy - def start(): return ThePirateBay() - -config = [{'name': 'ThePirateBay', 'groups': [{ - 'tab': 'searcher', - 'subtab': 'providers', - 'name': 'ThePirateBay', - 'description': 'The world\'s largest bittorrent tracker.', - 'options': [{'name': 'enabled', 'type': 'enabler', - 'default': False}, { - 'name': 'domain_for_tpb', - 'label': 'Proxy server', - 'default': 'http://thepiratebay.se', - 'description': 'Default domain for requests', - 'type': 'dropdown', - 'values': TPBProxy.list, - }], - }]}] +config = [{ + 'name': 'thepiratebay', + 'groups': [{ + 'tab': 'searcher', + 'subtab': 'providers', + 'name': 'ThePirateBay', + 'description': 'The world\'s largest bittorrent tracker.', + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': False + }, + { + 'name': 'domain', + 'advanced': True, + 'label': 'Proxy server', + 'description': 'Domain for requests, keep empty to let CouchPotato pick.', + } + ], + }] +}] diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index 0fbc25d8..cdb86a75 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -1,189 +1,142 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import getTitle, tryInt, cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider -from random import sample as random -from urlparse import urlparse - +from couchpotato.environment import Env +from urllib import quote_plus import re import time +import traceback - -class TPBProxy(object): - - """ TPBProxy deals with failed or blocked TPB proxys. - It works as round-robin balancer, if user seleced - or default domain becomes unavaliable. - """ - - list = [ - ('(Sweden) thepiratebay.se', 'http://thepiratebay.se'), - ('(Sweden) tpb.ipredator.se (ssl)', 'https://tpb.ipredator.se'), - ('(Germany) depiraatbaai.be', 'http://depiraatbaai.be'), - ('(UK) piratereverse.info (ssl)', 'https://piratereverse.info'), - ('(UK) tpb.pirateparty.org.uk (ssl)', 'https://tpb.pirateparty.org.uk'), - ('(Netherlands) argumentomteemigreren.nl', 'http://argumentomteemigreren.nl'), - ('(direct) 194.71.107.80', 'http://194.71.107.80'), - ('(direct) 194.71.107.81', 'http://194.71.107.81'), - ('(direct) 194.71.107.82', 'http://194.71.107.82'), - ('(direct) 194.71.107.83', 'http://194.71.107.83'), - ] - - @staticmethod - def get_proxy(http_failed_disabled=None, current=None): - - # compare lists and user/default value, exclude filter - - unused = [item for item in TPBProxy.list if item not in http_failed_disabled and current - not in item] - - if len(unused) > 0: - - # only return uri - - return random(unused, 1)[0][1] - else: - - # this should disable provider for some time - - raise Exception('All ThePirateBay proxies are exhausted') +log = CPLog(__name__) class ThePirateBay(TorrentProvider): - log = CPLog(__name__) - cat_ids = [([207], ['720p', '1080p']), ([201], [ - 'cam', - 'ts', - 'dvdrip', - 'tc', - 'r5', - 'scr', - 'brrip', - ]), ([202], ['dvdr'])] + urls = { + 'detail': '%s/torrent/%s', + 'search': '%s/search/%s/0/7/%d' + } + + cat_ids = [ + ([207], ['720p', '1080p']), + ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']), + ([202], ['dvdr']) + ] cat_backup_id = 200 disable_provider = False + http_time_between_calls = 0 + + proxy_list = [ + 'https://thepiratebay.se', + 'https://tpb.ipredator.se', + 'https://depiraatbaai.be', + 'https://piratereverse.info', + 'https://tpb.pirateparty.org.uk', + 'https://argumentomteemigreren.nl', + ] def __init__(self): + self.domain = self.conf('domain') super(ThePirateBay, self).__init__() - self.urls = {'test': self.api_domain(), 'detail': '%s/torrent/%s', - 'search': '%s/search/%s/0/7/%d'} - def api_domain(self, url=''): + def getDomain(self, url = ''): - # default domain + if not self.domain: + for proxy in self.proxy_list: - domain = self.conf('domain_for_tpb', default='http://thepiratebay.se') - self.log.info('Selected domain for this request: %s', domain) - host = urlparse(domain).hostname - - # Clear disabled list for default or user selected host if time expired - - if self.http_failed_disabled.get(host, 0) > 0: - if self.http_failed_disabled[host] > time.time() - 900: - - # get new random domain + prop_name = 'tpb_proxy.%s' % proxy + last_check = float(Env.prop(prop_name, default = 0)) + if last_check > time.time() - 1209600: + continue + data = '' try: - domain = TPBProxy.get_proxy(self.http_failed_disabled, domain) - except Exception, err: - self.disable_provider = True - self.log.error(err) - else: + data = self.urlopen(proxy, timeout = 3) + except: + log.debug('Failed tpb proxy %s', proxy) - del self.http_failed_request[host] - del self.http_failed_disabled[host] + if 'title="Pirate Search"' in data: + log.debug('Using proxy: %s', proxy) + self.domain = proxy + break - return domain + url + Env.prop(prop_name, time.time()) + + if not self.domain: + log.error('No TPB proxies left, please add one in settings, or let us know which one to add on the forum.') + return None + + return cleanHost(self.domain).rstrip('/') + url def search(self, movie, quality): results = [] - if self.isDisabled() or self.disable_provider: + if self.isDisabled() or not self.getDomain(): return results - cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier' - )) - search_url = self.urls['search'] % (self.api_domain(), - self.for_search(getTitle(movie['library']) + ' ' - + quality['identifier']), - self.getCatId(quality['identifier'])[0]) - self.log.info('searchUrl: %s', search_url) + cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + search_url = self.urls['search'] % (self.getDomain(), quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) data = self.getCache(cache_key, search_url) - # print data + if data: + try: + soup = BeautifulSoup(data) + results_table = soup.find('table', attrs = {'id': 'searchResult'}) + entries = results_table.find_all('tr') + for result in entries[1:]: + link = result.find(href = re.compile('torrent\/\d+\/')) + download = result.find(href = re.compile('magnet:')) - if not data: - self.log.error('Failed to get data from %s.', search_url) - return results + size = re.search('Size (?P.+),', unicode(result.select('font.detDesc')[0])).group('size') + if link and download: - try: - soup = BeautifulSoup(data) - results_table = soup.find('table', attrs={'id': 'searchResult'}) - entries = results_table.find_all('tr') - for result in entries[1:]: - link = result.find(href=re.compile('torrent\/\d+\/')) - download = result.find(href=re.compile('magnet:')) + def extra_score(item): + trusted = (0, 10)[result.find('img', alt = re.compile('Trusted')) != None] + vip = (0, 20)[result.find('img', alt = re.compile('VIP')) != None] + confirmed = (0, 30)[result.find('img', alt = re.compile('Helpers')) != None] + moderated = (0, 50)[result.find('img', alt = re.compile('Moderator')) != None] - # Uploaded 06-28 02:27, Size 1.37 GiB, + return confirmed + trusted + vip + moderated - size = re.search('Size (?P.+),', unicode(result.select('font.detDesc' - )[0])).group('size') - if link and download: - new = { - 'type': 'magnet', - 'check_nzb': False, - 'description': '', - 'provider': self.getName(), + new = { + 'id': re.search('/(?P\d+)/', link['href']).group('id'), + 'type': 'torrent_magnet', + 'name': link.string, + 'check_nzb': False, + 'description': '', + 'provider': self.getName(), + 'url': download['href'], + 'detail_url': self.getDomain(link['href']), + 'size': self.parseSize(size), + 'seeders': tryInt(result.find_all('td')[2].string), + 'leechers': tryInt(result.find_all('td')[3].string), + 'extra_score': extra_score, + 'get_more_info': self.getMoreInfo } - trusted = (0, 10)[result.find('img', alt=re.compile('Trusted')) != None] - vip = (0, 20)[result.find('img', alt=re.compile('VIP')) != None] - confirmed = (0, 30)[result.find('img', alt=re.compile('Helpers')) != None] - moderated = (0, 50)[result.find('img', alt=re.compile('Moderator')) != None] - is_imdb = self.imdb_match(self.api_domain(link['href']), movie['library' - ]['identifier']) + new['score'] = fireEvent('score.calculate', new, movie, single = True) + is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, + imdb_results = False, single_category = False, single = True) - self.log.info('Name: %s', link.string) - self.log.info('Seeders: %s', result.find_all('td')[2].string) - self.log.info('Leechers: %s', result.find_all('td')[3].string) - self.log.info('Size: %s', size) - self.log.info('Score(trusted + vip + moderated): %d', confirmed + trusted + vip - + moderated) + if is_correct_movie: + results.append(new) + self.found(new) - new['name'] = link.string - new['id'] = re.search('/(?P\d+)/', link['href']).group('id') - new['url'] = self.api_domain(link['href']) - new['magnet'] = download['href'] - new['size'] = self.parseSize(size) - new['seeders'] = int(result.find_all('td')[2].string) - new['leechers'] = int(result.find_all('td')[3].string) - new['extra_score'] = lambda x: confirmed + trusted + vip + moderated - new['score'] = fireEvent('score.calculate', new, movie, single=True) + return results + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) - is_correct_movie = fireEvent( - 'searcher.correct_movie', - nzb=new, - movie=movie, - quality=quality, - imdb_results=is_imdb, - single_category=False, - single=True, - ) + return [] - if is_correct_movie: - results.append(new) - self.found(new) + def getMoreInfo(self, item): + full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) + html = BeautifulSoup(full_description) + nfo_pre = html.find('div', attrs = {'class':'nfo'}) + description = toUnicode(nfo_pre.text) if nfo_pre else '' - return results - except Exception, error: - self.log.debug(error) - return results - - def download(self, url='', nzb_id=''): - return url + item['description'] = description + return item diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 6691b6ff..e106a5a6 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -67,7 +67,7 @@ class TorrentLeech(TorrentProvider): 'description': '', 'provider': self.getName(), 'url': self.urls['download'] % url['href'], - 'download': self.download, + 'download': self.loginDownload, 'size': self.parseSize(result.find_all('td')[4].string), 'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string), 'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string),