dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup torrent providers

Browse Source
This commit is contained in:
Ruud
2012-12-25 14:56:27 +01:00
parent dbbbbb2f84
commit 5ab0d7a97b
11 changed files with 114 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/providers/nzb/ftdworld/main.py
-2
View File
@@ -86,8 +86,6 @@ class FTDWorld(NZBProvider):
'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0,
})
return results
except:
log.error('Failed to parse HTML response from FTDWorld')
couchpotato/core/providers/nzb/newznab/main.py
+2 -2
View File
@@ -35,12 +35,12 @@ class Newznab(NZBProvider, RSS):
results = ResultList(self, movie, quality, imdb_result = True)
for host in hosts:
result = self.singleSearch(host, movie, quality)
result = self._search(host, movie, quality)
results.extend(result)
return results
def singleSearch(self, host, movie, quality):
def _search(self, host, movie, quality):
if self.isDisabled(host):
return []
couchpotato/core/providers/nzb/omgwtfnzbs/main.py
-1
View File
@@ -9,7 +9,6 @@ from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
from urlparse import urlparse, parse_qs
import time
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
couchpotato/core/providers/torrent/base.py
+6
View File
@@ -24,3 +24,9 @@ class TorrentProvider(YarrProvider):
return getImdb(data) == imdbId
return False
class TorrentMagnetProvider(TorrentProvider):
type = 'torrent_magnet'
download = None
couchpotato/core/providers/torrent/kickasstorrents/main.py
+34 -54
View File
@@ -1,16 +1,15 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentMagnetProvider
import re
import traceback
log = CPLog(__name__)
class KickAssTorrents(TorrentProvider):
class KickAssTorrents(TorrentMagnetProvider):
urls = {
'test': 'https://kat.ph/',
@@ -32,14 +31,13 @@ class KickAssTorrents(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
title = simplifyString(getTitle(movie['library'])).replace(' ', '-')
results = ResultList(self, movie, quality, imdb_results = True)
data = self.getHTMLData(self.urls['search'] % ('m', movie['library']['identifier'].replace('tt', '')))
cache_key = 'kickasstorrents.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, self.urls['search'] % (title, movie['library']['identifier'].replace('tt', '')))
if data:
cat_ids = self.getCatId(quality['identifier'])
@@ -53,57 +51,39 @@ class KickAssTorrents(TorrentProvider):
continue
try:
for temp in result.find_all('tr'):
if temp['class'] is 'firstr' or not temp.get('id'):
continue
try:
for temp in result.find_all('tr'):
if temp['class'] is 'firstr' or not temp.get('id'):
continue
new = {}
new = {
'type': 'torrent_magnet',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'score': 0,
}
nr = 0
for td in temp.find_all('td'):
column_name = table_order[nr]
if column_name:
nr = 0
for td in temp.find_all('td'):
column_name = table_order[nr]
if column_name:
if column_name is 'name':
link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
new['id'] = temp.get('id')[-8:]
new['name'] = link.text
new['url'] = td.find('a', 'imagnet')['href']
new['detail_url'] = self.urls['detail'] % link['href'][1:]
new['score'] = 20 if td.find('a', 'iverif') else 0
elif column_name is 'size':
new['size'] = self.parseSize(td.text)
elif column_name is 'age':
new['age'] = self.ageToDays(td.text)
elif column_name is 'seeds':
new['seeders'] = tryInt(td.text)
elif column_name is 'leechers':
new['leechers'] = tryInt(td.text)
if column_name is 'name':
link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
new['id'] = temp.get('id')[-8:]
new['name'] = link.text
new['url'] = td.find('a', 'imagnet')['href']
new['detail_url'] = self.urls['detail'] % link['href'][1:]
new['score'] = 20 if td.find('a', 'iverif') else 0
elif column_name is 'size':
new['size'] = self.parseSize(td.text)
elif column_name is 'age':
new['age'] = self.ageToDays(td.text)
elif column_name is 'seeds':
new['seeders'] = tryInt(td.text)
elif column_name is 'leechers':
new['leechers'] = tryInt(td.text)
nr += 1
nr += 1
new['score'] += fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie',
nzb = new, movie = movie, quality = quality,
imdb_results = True, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
except:
log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
results.append(new)
except:
pass
log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
return results
except AttributeError:
log.debug('No search results found.')
couchpotato/core/providers/torrent/passthepopcorn/main.py
+8 -22
View File
@@ -2,6 +2,7 @@ from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt, mergeDicts
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
from dateutil.parser import parse
import cookielib
@@ -67,10 +68,10 @@ class PassThePopcorn(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
results = ResultList(self, movie, quality, imdb_results = True)
movie_title = getTitle(movie['library'])
quality_id = quality['identifier']
@@ -118,7 +119,6 @@ class PassThePopcorn(TorrentProvider):
if 'Scene' in torrent and torrent['Scene']:
torrentdesc += ' Scene'
if 'RemasterTitle' in torrent and torrent['RemasterTitle']:
# eliminate odd characters...
torrentdesc += self.htmlToASCII(' %s' % torrent['RemasterTitle'])
torrentdesc += ' (%s)' % quality_id
@@ -127,38 +127,24 @@ class PassThePopcorn(TorrentProvider):
def extra_check(item):
return self.torrentMeetsQualitySpec(item, type)
def extra_score(item):
return 50 if torrent['GoldenPopcorn'] else 0
new = {
results.append({
'id': torrent_id,
'type': 'torrent',
'provider': self.getName(),
'name': torrent_name,
'description': '',
'url': '%s?action=download&id=%d&authkey=%s&torrent_pass=%s' % (self.urls['torrent'], torrent_id, authkey, passkey),
'detail_url': self.urls['detail'] % torrent_id,
'date': tryInt(time.mktime(parse(torrent['UploadTime']).timetuple())),
'size': tryInt(torrent['Size']) / (1024 * 1024),
'provider': self.getName(),
'seeders': tryInt(torrent['Seeders']),
'leechers': tryInt(torrent['Leechers']),
'extra_score': extra_score,
'score': 50 if torrent['GoldenPopcorn'] else 0,
'extra_check': extra_check,
'download': self.loginDownload,
}
})
new['score'] = fireEvent('score.calculate', new, movie, single = True)
if fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality):
results.append(new)
self.found(new)
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def login(self):
couchpotato/core/providers/torrent/publichd/main.py
+10 -26
View File
@@ -1,9 +1,9 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentMagnetProvider
from urlparse import parse_qs
import re
import traceback
@@ -11,7 +11,7 @@ import traceback
log = CPLog(__name__)
class PublicHD(TorrentProvider):
class PublicHD(TorrentMagnetProvider):
urls = {
'test': 'https://publichd.se',
@@ -22,20 +22,18 @@ class PublicHD(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled() or not quality.get('hd', False):
return results
return []
results = ResultList(self, movie, quality)
params = tryUrlencode({
'page':'torrents',
'search': '%s %s' % (getTitle(movie['library']), movie['library']['year']),
'active': 1,
})
url = '%s?%s' % (self.urls['search'], params)
cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url)
data = self.getHTMLData('%s?%s' % (self.urls['search'], params))
if data:
@@ -53,35 +51,21 @@ class PublicHD(TorrentProvider):
url = parse_qs(info_url['href'])
new = {
results.append({
'id': url['id'][0],
'name': info_url.string,
'type': 'torrent_magnet',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': download['href'],
'detail_url': self.urls['detail'] % url['id'][0],
'size': self.parseSize(result.find_all('td')[7].string),
'seeders': tryInt(result.find_all('td')[4].string),
'leechers': tryInt(result.find_all('td')[5].string),
'get_more_info': self.getMoreInfo
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
})
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getMoreInfo(self, item):
full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
couchpotato/core/providers/torrent/sceneaccess/main.py
+12 -25
View File
@@ -1,8 +1,8 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
import traceback
@@ -29,9 +29,8 @@ class SceneAccess(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
url = self.urls['search'] % (
self.getCatId(quality['identifier'])[0],
@@ -47,10 +46,11 @@ class SceneAccess(TorrentProvider):
# Do login for the cookies
if not self.login_opener and not self.login():
return results
return []
cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url, opener = self.login_opener)
results = ResultList(self, movie, quality, imdb_results = True)
data = self.getHTMLData(url, opener = self.login_opener)
if data:
html = BeautifulSoup(data)
@@ -66,37 +66,24 @@ class SceneAccess(TorrentProvider):
link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a')
url = result.find('td', attrs = {'class' : 'td_dl'}).find('a')
leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a')
id = link['href'].replace('details?id=', '')
torrent_id = link['href'].replace('details?id=', '')
new = {
'id': id,
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
results.append({
'id': torrent_id,
'name': link['title'],
'url': self.urls['download'] % url['href'],
'detail_url': self.urls['detail'] % id,
'detail_url': self.urls['detail'] % torrent_id,
'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]),
'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string),
'leechers': tryInt(leechers.string) if leechers else 0,
'download': self.loginDownload,
'get_more_info': self.getMoreInfo,
}
})
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getLoginParams(self):
return tryUrlencode({
couchpotato/core/providers/torrent/scenehd/main.py
+13 -29
View File
@@ -1,8 +1,8 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
import traceback
@@ -23,9 +23,8 @@ class SceneHD(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
arguments = tryUrlencode({
@@ -35,10 +34,11 @@ class SceneHD(TorrentProvider):
# Cookie login
if not self.login_opener and not self.login():
return results
return []
cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url, opener = self.login_opener)
results = ResultList(self, movie, quality, imdb_results = True)
data = self.getHTMLData(url, opener = self.login_opener)
if data:
html = BeautifulSoup(data)
@@ -52,7 +52,7 @@ class SceneHD(TorrentProvider):
detail_link = all_cells[2].find('a')
details = detail_link['href']
id = details.replace('details.php?id=', '')
torrent_id = details.replace('details.php?id=', '')
leechers = all_cells[11].find('a')
if leechers:
@@ -60,37 +60,21 @@ class SceneHD(TorrentProvider):
else:
leechers = all_cells[11].string
new = {
'id': id,
results.append({
'id': torrent_id,
'name': detail_link['title'],
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'size': self.parseSize(all_cells[7].string),
'seeders': tryInt(all_cells[10].find('a').string),
'leechers': tryInt(leechers),
'url': self.urls['download'] % id,
'url': self.urls['download'] % torrent_id,
'download': self.loginDownload,
}
imdb_link = all_cells[1].find('a')
imdb_results = self.imdbMatch(imdb_link['href'], movie['library']['identifier']) if imdb_link else False
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = imdb_results, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
'description': all_cells[1].find('a')['href'],
})
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getLoginParams(self, params):
couchpotato/core/providers/torrent/thepiratebay/main.py
+13 -26
View File
@@ -1,11 +1,10 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt, cleanHost
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentMagnetProvider
from couchpotato.environment import Env
from urllib import quote_plus
import re
import time
import traceback
@@ -13,7 +12,7 @@ import traceback
log = CPLog(__name__)
class ThePirateBay(TorrentProvider):
class ThePirateBay(TorrentMagnetProvider):
urls = {
'detail': '%s/torrent/%s',
@@ -76,13 +75,14 @@ class ThePirateBay(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled() or not self.getDomain():
return results
return []
cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
search_url = self.urls['search'] % (self.getDomain(), quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getCache(cache_key, search_url)
results = ResultList(self, movie, quality)
search_url = self.urls['search'] % (self.getDomain(), tryUrlencode(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(search_url)
if data:
try:
@@ -90,7 +90,7 @@ class ThePirateBay(TorrentProvider):
results_table = soup.find('table', attrs = {'id': 'searchResult'})
if not results_table:
return results
return []
entries = results_table.find_all('tr')
for result in entries[2:]:
@@ -112,13 +112,9 @@ class ThePirateBay(TorrentProvider):
return confirmed + trusted + vip + moderated
new = {
results.append({
'id': re.search('/(?P<id>\d+)/', link['href']).group('id'),
'type': 'torrent_magnet',
'name': link.string,
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': download['href'],
'detail_url': self.getDomain(link['href']),
'size': self.parseSize(size),
@@ -126,17 +122,8 @@ class ThePirateBay(TorrentProvider):
'leechers': tryInt(result.find_all('td')[3].string),
'extra_score': extra_score,
'get_more_info': self.getMoreInfo
}
})
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
couchpotato/core/providers/torrent/torrentleech/main.py
+16 -32
View File
@@ -1,10 +1,9 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
from urllib import quote_plus
import traceback
@@ -14,11 +13,11 @@ log = CPLog(__name__)
class TorrentLeech(TorrentProvider):
urls = {
'test' : 'http://torrentleech.org/',
'login' : 'http://torrentleech.org/user/account/login/',
'detail' : 'http://torrentleech.org/torrent/%s',
'search' : 'http://torrentleech.org/torrents/browse/index/query/%s/categories/%d',
'download' : 'http://torrentleech.org%s',
'test' : 'http://www.torrentleech.org/',
'login' : 'http://www.torrentleech.org/user/account/login/',
'detail' : 'http://www.torrentleech.org/torrent/%s',
'search' : 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
'download' : 'http://www.torrentleech.org%s',
}
cat_ids = [
@@ -36,17 +35,17 @@ class TorrentLeech(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
# Cookie login
if not self.login_opener and not self.login():
return results
return []
cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
url = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':', '') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getCache(cache_key, url, opener = self.login_opener)
results = ResultList(self, movie, quality)
url = self.urls['search'] % (tryUrlencode(getTitle(movie['library']).replace(':', '') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(url, opener = self.login_opener)
if data:
html = BeautifulSoup(data)
@@ -54,7 +53,7 @@ class TorrentLeech(TorrentProvider):
try:
result_table = html.find('table', attrs = {'id' : 'torrenttable'})
if not result_table:
return results
return []
entries = result_table.find_all('tr')
@@ -64,36 +63,21 @@ class TorrentLeech(TorrentProvider):
url = result.find('td', attrs = {'class' : 'quickdownload'}).find('a')
details = result.find('td', attrs = {'class' : 'name'}).find('a')
new = {
results.append({
'id': link['href'].replace('/torrent/', ''),
'name': link.string,
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': self.urls['download'] % url['href'],
'detail_url': self.urls['download'] % details['href'],
'download': self.loginDownload,
'size': self.parseSize(result.find_all('td')[4].string),
'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string),
'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string),
}
})
imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier'])
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = imdb_results, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getLoginParams(self):
return tryUrlencode({