From a60b227e819dea93110032dee743ef679d46ce05 Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Tue, 3 Jul 2012 12:32:12 +0200 Subject: [PATCH] for_search, imdb_match added to class --- couchpotato/core/providers/base.py | 22 ++++ .../core/providers/torrent/publichd/main.py | 94 ++++++--------- .../providers/torrent/thepiratebay/main.py | 107 ++++++++---------- 3 files changed, 103 insertions(+), 120 deletions(-) diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 0397784c..d4a35442 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -3,6 +3,9 @@ from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env from urlparse import urlparse +from urllib import quote_plus +from couchpotato.core.helpers.encoding import simplifyString + import re import time @@ -105,5 +108,24 @@ class YarrProvider(Provider): return [self.cat_backup_id] + def imdb_match(self, url, imdb_id): + """ Searches for imdb_id in url of webpage """ + log.info('Finding if imbd_id(%s) is found in url: %s' % (imdb_id, url)) + try: + data = self.urlopen(url) + except: + log.error('Failed to open %s.' % url) + return False + imdb_id_alt = re.sub('tt[0]*', 'tt', imdb_id) + data = unicode(data, errors='ignore') + if 'imdb.com/title/' + imdb_id in data or 'imdb.com/title/' \ + + imdb_id_alt in data: + return True + return False + + def for_search(self, string): + """ Prepare string for search, removing all characters that might confuse search engine""" + return quote_plus(simplifyString(string)) + def found(self, new): log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new) diff --git a/couchpotato/core/providers/torrent/publichd/main.py b/couchpotato/core/providers/torrent/publichd/main.py index cbd7efa4..673ca303 100644 --- a/couchpotato/core/providers/torrent/publichd/main.py +++ b/couchpotato/core/providers/torrent/publichd/main.py @@ -6,15 +6,14 @@ from couchpotato.core.event import fireEvent from couchpotato.core.helpers.variable import getTitle from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider -import re from urlparse import parse_qs -from urllib import quote_plus -import urllib2 -log = CPLog(__name__) + +import re class PublicHD(TorrentProvider): + log = CPLog(__name__) urls = { 'test': 'http://publichd.eu', 'download': 'http://publichd.eu/%s', @@ -22,37 +21,35 @@ class PublicHD(TorrentProvider): 'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d', } - cat_ids = [([2], ['720p']), ([5], ['1080p']), ([15], ['bdrip']), - ([16], ['brrip']), ([16], ['blue-ray'])] + cat_ids = [([2], ['720p']), ([5], ['1080p']), ([16], ['brrip']), + ([16], ['bd50'])] cat_backup_id = 0 def search(self, movie, quality): results = [] - if self.isDisabled(): + if self.isDisabled() and quality['hd'] != True: return results - movie_name = re.sub("\W", ' ', getTitle(movie['library'])) - movie_name = re.sub(' ', ' ', movie_name) - log.info('Cleaned Name: %s', movie_name) cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(movie_name + ' ' - + quality['identifier']), - self.getCatId(quality['identifier'])[0]) - log.info('searchUrl: %s', searchUrl) - data = self.getCache(cache_key, searchUrl) + search_url = self.urls['search'] \ + % (self.for_search(getTitle(movie['library']) + + ' ' + quality['identifier']), + self.getCatId(quality['identifier'])[0]) + self.log.info('searchUrl: %s', search_url) + data = self.getCache(cache_key, search_url) if not data: - log.error('Failed to get data from %s.', searchUrl) + self.log.error('Failed to get data from %s.', search_url) return results try: soup = BeautifulSoup(data) - resultsTable = soup.find('table', attrs={'id': 'bgtorrlist2' - }) - entries = resultsTable.findAll('tr') + results_table = soup.find('table', + attrs={'id': 'bgtorrlist2'}) + entries = results_table.find_all('tr') for result in entries[2:len(entries) - 1]: info_url = result.find(href=re.compile('torrent-details' )) @@ -65,12 +62,14 @@ class PublicHD(TorrentProvider): 'description': '', 'provider': self.getName(), } - log.info('Name: %s', result.findAll('td')[1].string) - log.info('Seeders: %s', result.findAll('td' - )[4].string) - log.info('Leaches: %s', result.findAll('td' - )[5].string) - log.info('Size: %s', result.findAll('td')[7].string) + self.log.debug('Name: %s', result.find_all('td' + )[1].string) + self.log.debug('Seeders: %s', result.find_all('td' + )[4].string) + self.log.debug('Leaches: %s', result.find_all('td' + )[5].string) + self.log.debug('Size: %s', result.find_all('td' + )[7].string) url = parse_qs(info_url['href']) @@ -78,22 +77,23 @@ class PublicHD(TorrentProvider): new['id'] = url['id'][0] new['url'] = self.urls['download'] % download['href' ] - new['size'] = self.parseSize(result.findAll('td' + new['size'] = self.parseSize(result.find_all('td' )[7].string) - new['seeders'] = int(result.findAll('td')[4].string) - new['leechers'] = int(result.findAll('td' + new['seeders'] = int(result.find_all('td' + )[4].string) + new['leechers'] = int(result.find_all('td' )[5].string) - new['imdbid'] = movie['library']['identifier'] - new['extra_score'] = self.extra_score new['score'] = fireEvent('score.calculate', new, movie, single=True) + is_imdb = self.imdb_match(self.urls['detail'] + % new['id'], movie['library']['identifier']) is_correct_movie = fireEvent( 'searcher.correct_movie', nzb=new, movie=movie, quality=quality, - imdb_results=True, + imdb_results=is_imdb, single_category=False, single=True, ) @@ -104,35 +104,13 @@ class PublicHD(TorrentProvider): self.found(new) return results - except Exception, e: - log.debug(e) - log.info('Error occured during parsing! Passing only processed entries' - ) + except Exception, err: + self.log.debug(err) + self.log.info('Error occured during parsing! Passing only processed entries' + ) return results - def extra_score(self, torrent): - url = self.urls['detail'] % torrent['id'] - log.info('extra_score: %s', url) - imdbId = torrent['imdbid'] - return self.imdbMatch(url, imdbId) - - def imdbMatch(self, url, imdbId): - log.info('imdbMatch: %s', url) - try: - data = urllib2.urlopen(url).read() - pass - except IOError: - log.error('Failed to open %s.' % url) - return '' - - imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) - data = unicode(data, errors='ignore') - if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' \ - + imdbIdAlt in data: - return 50 - return 0 - def download(self, url='', nzb_id=''): - log.info('Downloading: %s', url) + self.log.info('Downloading: %s', url) torrent = self.urlopen(url) return torrent diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index 885b4c60..bae79551 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -6,14 +6,13 @@ from couchpotato.core.event import fireEvent from couchpotato.core.helpers.variable import getTitle from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider + import re -from urllib import quote_plus -import urllib2 -log = CPLog(__name__) class ThePirateBay(TorrentProvider): + log = CPLog(__name__) cat_ids = [([207], ['720p', '1080p']), ([201], [ 'cam', 'ts', @@ -28,10 +27,13 @@ class ThePirateBay(TorrentProvider): def __init__(self): super(ThePirateBay, self).__init__() - self.urls = {"test": self.getAPIurl(), 'detail': '%s/torrent/%s', 'search': '%s/search/%s/0/7/%d'} + self.urls = {'test': self.getapiurl(), + 'detail': '%s/torrent/%s', + 'search': '%s/search/%s/0/7/%d'} - def getAPIurl(self, url=""): - return (("http://thepiratebay.se", self.conf('domain_for_tpb'))[self.conf('domain_for_tpb') != None]) + url + def getapiurl(self, url=''): + return ('http://thepiratebay.se', self.conf('domain_for_tpb' + ))[self.conf('domain_for_tpb') != None] + url def search(self, movie, quality): @@ -39,34 +41,35 @@ class ThePirateBay(TorrentProvider): if self.isDisabled(): return results - movie_name = re.sub("\W", ' ', getTitle(movie['library'])) - movie_name = re.sub(' ', ' ', movie_name) - - log.info('API url: %s', self.getAPIurl()) - log.info('Cleaned Name: %s', movie_name) - cache_key = 'thepiratebay.%s.%s' % (movie['library' ]['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (self.getAPIurl(), - quote_plus(movie_name + ' ' + quality['identifier']), + search_url = self.urls['search'] % (self.getapiurl(), + self.for_search(getTitle(movie['library']) + + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) - log.info('searchUrl: %s', searchUrl) - data = self.getCache(cache_key, searchUrl) - #print data + self.log.info('searchUrl: %s', search_url) + data = self.getCache(cache_key, search_url) + + # print data + if not data: - log.error('Failed to get data from %s.', searchUrl) + self.log.error('Failed to get data from %s.', search_url) return results try: soup = BeautifulSoup(data) - resultsTable = soup.find('table', + results_table = soup.find('table', attrs={'id': 'searchResult'}) - entries = resultsTable.findAll('tr') + entries = results_table.find_all('tr') for result in entries[1:]: link = result.find(href=re.compile('torrent\/\d+\/')) download = result.find(href=re.compile('magnet:')) - #Uploaded 06-28 02:27, Size 1.37 GiB, - size = re.search('Size (?P.+),', unicode(result.select("font.detDesc")[0])).group("size") + + # Uploaded 06-28 02:27, Size 1.37 GiB, + + size = re.search('Size (?P.+),', + unicode(result.select('font.detDesc' + )[0])).group('size') if link and download: new = { 'type': 'magnet', @@ -81,39 +84,39 @@ class ThePirateBay(TorrentProvider): alt=re.compile('VIP')) != None] moderated = (0, 50)[result.find('img', alt=re.compile('Moderator')) != None] + is_imdb = self.imdb_match(self.getapiurl(link['href' + ]), movie['library']['identifier']) - log.info('Name: %s', link.string) - log.info('Seeders: %s', result.findAll('td' - )[2].string) - log.info('Leechers: %s', result.findAll('td' - )[3].string) - log.info('Size: %s', size) - log.info('Score(trusted + vip + moderated): %d', - trusted + vip + moderated) + self.log.info('Name: %s', link.string) + self.log.info('Seeders: %s', result.find_all('td' + )[2].string) + self.log.info('Leechers: %s', result.find_all('td' + )[3].string) + self.log.info('Size: %s', size) + self.log.info('Score(trusted + vip + moderated): %d' + , trusted + vip + moderated) new['name'] = link.string new['id'] = re.search('/(?P\d+)/', link['href' ]).group('id') - new['url'] = self.getAPIurl(link['href']) - new['magnet'] = unicode(download['href']) # forcing of storing full magnet data + new['url'] = self.getapiurl(link['href']) + new['magnet'] = download['href'] new['size'] = self.parseSize(size) - new['seeders'] = int(result.findAll('td')[2].string) - new['leechers'] = int(result.findAll('td' + new['seeders'] = int(result.find_all('td' + )[2].string) + new['leechers'] = int(result.find_all('td' )[3].string) - - new['TPB_score'] = trusted + vip + moderated - new['extra_score'] = self.extra_score + new['extra_score'] = lambda : trusted + vip \ + + moderated new['score'] = fireEvent('score.calculate', new, movie, single=True) - isImdb = self.imdbMatch(self.getAPIurl(link['href']), movie['library']['identifier']) - is_correct_movie = fireEvent( 'searcher.correct_movie', nzb=new, movie=movie, quality=quality, - imdb_results=isImdb, + imdb_results=is_imdb, single_category=False, single=True, ) @@ -123,29 +126,9 @@ class ThePirateBay(TorrentProvider): self.found(new) return results - except Exception, e: - log.debug(e) - log.info('Error occured during parsing! Passing only processed entries' - ) + except Exception, error: + self.log.debug(error) return results - def extra_score(self, torrent): - return torrent["TPB_score"] - - def imdbMatch(self, url, imdbId): - log.info('imdbMatch: %s', url) - try: - data = urllib2.urlopen(url).read() - pass - except: - log.error('Failed to open %s.' % url) - return False - imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) - data = unicode(data, errors='ignore') - if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' \ - + imdbIdAlt in data: - return True - return False - def download(self, url='', nzb_id=''): return url