diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 67f36a8b..3d1682cb 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -95,7 +95,7 @@ class Plugin(object): return False # http request - def urlopen(self, url, timeout = 30, params = {}, headers = {}, multipart = False, show_error = True): + def urlopen(self, url, timeout = 30, params = {}, headers = {}, opener = None, multipart = False, show_error = True): # Fill in some headers if not headers.get('Referer'): @@ -130,7 +130,10 @@ class Plugin(object): data = tryUrlencode(params) if len(params) > 0 else None request = urllib2.Request(url, data, headers) - data = urllib2.urlopen(request, timeout = timeout).read() + if opener: + data = opener.open(request, timeout = timeout).read() + else: + data = urllib2.urlopen(request, timeout = timeout).read() self.http_failed_request[host] = 0 except IOError: @@ -215,18 +218,7 @@ class Plugin(object): cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] - opener = None - if kwargs.get('opener'): - opener = kwargs.get('opener') - del kwargs['opener'] - - if opener: - log.info('Opening url: %s', url) - f = opener.open(url) - data = f.read() - f.close() - else: - data = self.urlopen(url, **kwargs) + data = self.urlopen(url, **kwargs) if data: self.setCache(cache_key, data, timeout = cache_timeout) diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index 89f68bee..3d6384c0 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -10,6 +10,7 @@ from couchpotato.environment import Env from dateutil.parser import parse import re import time +import traceback import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -100,8 +101,8 @@ class NzbIndex(NZBProvider, RSS): self.found(new) return results - except SyntaxError: - log.error('Failed to parse XML response from NZBMatrix.com') + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) return results diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index a91af44a..04d191bc 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -1,31 +1,55 @@ -from couchpotato.core.providers.base import YarrProvider +from couchpotato.core.helpers.variable import getImdb from couchpotato.core.logger import CPLog -import urllib2 +from couchpotato.core.providers.base import YarrProvider import cookielib +import traceback +import urllib2 log = CPLog(__name__) class TorrentProvider(YarrProvider): - type = 'torrent' - def login(self, params): - + type = 'torrent' + login_opener = None + + def imdbMatch(self, url, imdbId): + if getImdb(url) == imdbId: + return True + + if url[:4] == 'http': + try: + data = self.urlopen(url) + except IOError: + log.error('Failed to open %s.', url) + return False + + return getImdb(data) == imdbId + + return False + + def login(self): + try: cookiejar = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) urllib2.install_opener(opener) - f = opener.open(self.urls['login'], params) - loginData = f.read() + f = opener.open(self.urls['login'], self.getLoginParam()) + f.read() f.close() - - except: - log.error('Failed to login.') - - return opener - + self.login_opener = opener + return True + except: + log.error('Failed to login %s: %s', (self.getName(), traceback.format_exc())) + + return False + def download(self, url = '', nzb_id = ''): - loginParams = self.getLoginParams() - self.login(params = loginParams) - torrent = self.urlopen(url) - return torrent + + try: + if not self.login_opener and not self.login(): + log.error('Failed downloading from %s', self.getName()) + + return self.urlopen(url, opener = self.login_opener) + except: + log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc())) diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 4f413f46..7ba92cbd 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -1,17 +1,12 @@ from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import tryInt, getTitle +from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode +from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider -import StringIO -import gzip -import re +from urllib import quote_plus import traceback import urllib -import urllib2 -import cookielib -from urllib import quote_plus -from urllib2 import URLError log = CPLog(__name__) @@ -22,7 +17,7 @@ class SceneAccess(TorrentProvider): 'test': 'https://www.sceneaccess.eu/', 'login' : 'https://www.sceneaccess.eu/login', 'detail': 'https://www.sceneaccess.eu/details?id=%s', - 'search': 'https://www.sceneaccess.eu/browse?search=%s&method=2&c%d=%d', + 'search': 'https://www.sceneaccess.eu/browse?method=2&c%d=%d', 'download': 'https://www.sceneaccess.eu/%s', } @@ -33,10 +28,6 @@ class SceneAccess(TorrentProvider): ] http_time_between_calls = 1 #seconds - - def getLoginParams(self): - loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in')) - return loginParams def search(self, movie, quality): @@ -44,78 +35,68 @@ class SceneAccess(TorrentProvider): if self.isDisabled(): return results - cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0]) - loginParams = self.getLoginParams() + q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier')) + arguments = tryUrlencode({ + 'search': q, + }) + url = "%s&%s" % (self.urls['search'], arguments) + url = url % ( + self.getCatId(quality['identifier'])[0], + self.getCatId(quality['identifier'])[0] + ) - opener = self.login(params = loginParams) - if not opener: - log.info("Couldn't login at SceneAccess") + # Do login for the cookies + if not self.login_opener and not self.login(): return results - data = self.getCache(cache_key, searchUrl, opener = opener) + cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + data = self.getCache(cache_key, url, opener = self.login_opener) if data: html = BeautifulSoup(data) - - else: - log.info("No results found at SceneAccess") - try: - resultsTable = html.find('table', attrs = {'id' : 'torrents-table'}) - entries = resultsTable.findAll('tr', attrs = {'class' : 'tt_row'}) - for result in entries: - new = { - 'type': 'torrent', - 'check_nzb': False, - 'description': '', - 'provider': self.getName(), - } - - link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a') - new['name'] = link['title'] - new['id'] = link['href'].replace('details?id=', '') - url = result.find('td', attrs = {'class' : 'td_dl'}).find('a') - new['url'] = self.urls['download'] % url['href'] - new['size'] = self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]) - new['seeders'] = int(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string) - leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a') - if leechers: - new['leechers'] = int(leechers.string) - else: - new['leechers'] = 0 - - details = self.urls['detail'] % new['id'] - imdb_results = self.imdbMatch(details, movie['library']['identifier']) + try: + resultsTable = html.find('table', attrs = {'id' : 'torrents-table'}) + entries = resultsTable.findAll('tr', attrs = {'class' : 'tt_row'}) + for result in entries: - new['score'] = fireEvent('score.calculate', new, movie, single = True) - is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = imdb_results, single_category = False, single = True) + link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a') + url = result.find('td', attrs = {'class' : 'td_dl'}).find('a') + leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a') - if is_correct_movie: - new['download'] = self.download - results.append(new) - self.found(new) - return results - - except: - log.info("No results found at SceneAccess") - return [] + new = { + 'id': link['href'].replace('details?id=', ''), + 'type': 'torrent', + 'check_nzb': False, + 'description': '', + 'provider': self.getName(), + 'name': link['title'], + 'url': self.urls['download'] % url['href'], + 'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]), + 'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string), + 'leechers': tryInt(leechers.string) if leechers else 0, + 'download': self.download, + } + imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier']) - def imdbMatch(self, url, imdbId): - try: - data = urllib2.urlopen(url).read() - pass - except IOError: - log.error('Failed to open %s.' % url) - return False + new['score'] = fireEvent('score.calculate', new, movie, single = True) + is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, + imdb_results = imdb_results, single_category = False, single = True) - html = BeautifulSoup(data) - imdbDiv = html.find('span', attrs = {'class':'i_link'}) - imdbDiv = str(imdbDiv).decode("utf-8", "replace") - imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) + if is_correct_movie: + results.append(new) + self.found(new) - if 'imdb.com/title/' + imdbId in imdbDiv or 'imdb.com/title/' + imdbIdAlt in imdbDiv: - return True - return False + return results + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + def getLoginParams(self, params): + return tryUrlencode({ + 'username': self.conf('username'), + 'password': self.conf('password'), + 'submit': 'come on in', + }) diff --git a/couchpotato/core/providers/torrent/scenehd/main.py b/couchpotato/core/providers/torrent/scenehd/main.py index 443c7ad5..d7fb7f54 100644 --- a/couchpotato/core/providers/torrent/scenehd/main.py +++ b/couchpotato/core/providers/torrent/scenehd/main.py @@ -1,18 +1,11 @@ from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import tryInt, getTitle +from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode +from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider -import StringIO -import gzip -import re import traceback import urllib -import urllib2 -import cookielib -from urllib import quote_plus -from urllib2 import URLError - log = CPLog(__name__) @@ -23,15 +16,11 @@ class SceneHD(TorrentProvider): 'test': 'http://scenehd.org/', 'login' : 'http://scenehd.org/takelogin.php', 'detail': 'http://scenehd.org/details.php?id=%s', - 'search': 'http://scenehd.org/browse.php?ajax&search=%s', + 'search': 'http://scenehd.org/browse.php?ajax', 'download': 'http://scenehd.org/download.php?id=%s', } - - http_time_between_calls = 1 #seconds - def getLoginParams(self): - loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes')) - return loginParams + http_time_between_calls = 1 #seconds def search(self, movie, quality): @@ -39,73 +28,75 @@ class SceneHD(TorrentProvider): if self.isDisabled(): return results - cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier'])) - loginParams = self.getLoginParams() + q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier')) + arguments = tryUrlencode({ + 'search': q, + }) + url = "%s&%s" % (self.urls['search'], arguments) - opener = self.login(params = loginParams) - if not opener: - log.error("Couldn't login at SceneHD") + # Cookie login + if not self.login_opener and not self.login(): return results - data = self.getCache(cache_key, searchUrl, opener = opener) + cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + data = self.getCache(cache_key, url, opener = self.login_opener) if data: html = BeautifulSoup(data) - - else: - log.info("No results found at SceneHD") - - try: - resultsTable = html.findAll('table')[6] - entries = resultsTable.findAll('tr') - for result in entries[1:]: - new = { - 'type': 'torrent', - 'check_nzb': False, - 'description': '', - 'provider': self.getName(), - } - allCells = result.findAll('td') - new['size'] = self.parseSize(allCells[7].string.replace('GiB', 'GB')) - new['seeders'] = allCells[10].find('a').string - leechers = allCells[11].find('a') - if leechers: - new['leechers'] = leechers.string - else: - new['leechers'] = allCells[11].string - - detailLink = allCells[2].find('a') - details = detailLink['href'] - new['id'] = details.replace('details.php?id=', '') - new['name'] = detailLink['title'] - - imdbLink = allCells[1].find('a') - imdb_results = False + try: + resultsTable = html.find_all('table')[6] + entries = resultsTable.find_all('tr') + for result in entries[1:]: - if imdbLink: - imdbFound = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/') - imdb_results = self.imdbMatch(imdbFound, movie['library']['identifier']) - - new['url'] = self.urls['download'] % new['id'] - new['score'] = fireEvent('score.calculate', new, movie, single = True) - is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = imdb_results, single_category = False, single = True) + all_cells = result.find_all('td') - if is_correct_movie: - new['download'] = self.download - results.append(new) - self.found(new) - return results - - except: - log.info("No results found at SceneHD") - return [] + detail_link = all_cells[2].find('a') + details = detail_link['href'] + id = details.replace('details.php?id=', '') - def imdbMatch(self, imdbFound, imdbId): - imdbIdAlt = re.sub('tt[0]*', 'tt', imdbFound) - if imdbFound == imdbId or imdbIdAlt == imdbId: - return True - return False + leechers = all_cells[11].find('a') + if leechers: + leechers = leechers.string + else: + leechers = all_cells[11].string + new = { + 'id': id, + 'name': detail_link['title'], + 'type': 'torrent', + 'check_nzb': False, + 'description': '', + 'provider': self.getName(), + 'size': self.parseSize(all_cells[7].string), + 'seeders': tryInt(all_cells[10].find('a').string), + 'leechers': tryInt(leechers), + 'url': self.urls['download'] % id, + 'download': self.download, + } + + imdb_link = all_cells[1].find('a') + imdb_results = self.imdbMatch(imdb_link['href'], movie['library']['identifier']) if imdb_link else False + + new['score'] = fireEvent('score.calculate', new, movie, single = True) + is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, + imdb_results = imdb_results, single_category = False, single = True) + + if is_correct_movie: + results.append(new) + self.found(new) + + return results + + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + + def getLoginParams(self, params): + return tryUrlencode({ + 'username': self.conf('username'), + 'password': self.conf('password'), + 'ssl': 'yes', + }) diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 3614ea7c..6691b6ff 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -1,18 +1,11 @@ from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import tryInt, getTitle +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider -import StringIO -import gzip -import re -import traceback -import urllib -import urllib2 -import cookielib from urllib import quote_plus -from urllib2 import URLError -import sys +import traceback log = CPLog(__name__) @@ -39,10 +32,6 @@ class TorrentLeech(TorrentProvider): ] http_time_between_calls = 1 #seconds - - def getLoginParams(self): - loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit')) - return loginParams def search(self, movie, quality): @@ -50,71 +39,60 @@ class TorrentLeech(TorrentProvider): if self.isDisabled(): return results - cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) - loginParams = self.getLoginParams() - - opener = self.login(params = loginParams) - if not opener: - log.info("Couldn't login at Torrentleech") + # Cookie login + if not self.login_opener and not self.login(): return results - data = self.getCache(cache_key, searchUrl, opener = opener) + cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + url = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':', '') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) + data = self.getCache(cache_key, url, opener = self.login_opener) if data: html = BeautifulSoup(data) - - else: - log.info("No results found at Torrentleech") - try: - resultsTable = html.find('table', attrs = {'id' : 'torrenttable'}) - entries = resultsTable.findAll('tr') - for result in entries[1:]: - new = { - 'type': 'torrent', - 'check_nzb': False, - 'description': '', - 'provider': self.getName(), - } - - link = result.find('td', attrs = {'class' : 'name'}).find('a') - new['name'] = link.string - new['id'] = link['href'].replace('/torrent/', '') - url = result.find('td', attrs = {'class' : 'quickdownload'}).find('a') - new['url'] = self.urls['download'] % url['href'] - new['size'] = self.parseSize(result.findAll('td')[4].string) - new['seeders'] = int(result.find('td', attrs = {'class' : 'seeders'}).string) - new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string) - - details = self.urls['detail'] % new['id'] - imdb_results = self.imdbMatch(details, movie['library']['identifier']) - - new['score'] = fireEvent('score.calculate', new, movie, single = True) - is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = imdb_results, single_category = False, single = True) + try: + result_table = html.find('table', attrs = {'id' : 'torrenttable'}) + entries = result_table.find_all('tr') - if is_correct_movie: - new['download'] = self.download - results.append(new) - self.found(new) - return results - - except: - log.info("No results found at TorrentLeech") - return [] + for result in entries[1:]: + link = result.find('td', attrs = {'class' : 'name'}).find('a') + url = result.find('td', attrs = {'class' : 'quickdownload'}).find('a') - def imdbMatch(self, url, imdbId): - try: - data = urllib2.urlopen(url).read() - pass - except IOError: - log.error('Failed to open %s.' % url) - return False + new = { + 'id': link['href'].replace('/torrent/', ''), + 'name': link.string, + 'type': 'torrent', + 'check_nzb': False, + 'description': '', + 'provider': self.getName(), + 'url': self.urls['download'] % url['href'], + 'download': self.download, + 'size': self.parseSize(result.find_all('td')[4].string), + 'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string), + 'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string), + } - imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) - data = unicode(data, errors='ignore') - if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' + imdbIdAlt in data: - return True - return False + imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier']) + + new['score'] = fireEvent('score.calculate', new, movie, single = True) + is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, + imdb_results = imdb_results, single_category = False, single = True) + + if is_correct_movie: + results.append(new) + self.found(new) + + return results + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + return [] + + def getLoginParams(self): + return tryUrlencode({ + 'username': self.conf('username'), + 'password': self.conf('password'), + 'remember_me': 'on', + 'login': 'submit', + })