dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

for_search, imdb_match added to class

Browse Source
This commit is contained in:
Janez Troha
2012-07-03 12:32:12 +02:00
parent dfd522d7b9
commit a60b227e81
3 changed files with 103 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/providers/base.py
+22
View File
@@ -3,6 +3,9 @@ from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from urlparse import urlparse
from urllib import quote_plus
from couchpotato.core.helpers.encoding import simplifyString
import re
import time
@@ -105,5 +108,24 @@ class YarrProvider(Provider):
return [self.cat_backup_id]
def imdb_match(self, url, imdb_id):
""" Searches for imdb_id in url of webpage """
log.info('Finding if imbd_id(%s) is found in url: %s' % (imdb_id, url))
try:
data = self.urlopen(url)
except:
log.error('Failed to open %s.' % url)
return False
imdb_id_alt = re.sub('tt[0]*', 'tt', imdb_id)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdb_id in data or 'imdb.com/title/' \
+ imdb_id_alt in data:
return True
return False
def for_search(self, string):
""" Prepare string for search, removing all characters that might confuse search engine"""
return quote_plus(simplifyString(string))
def found(self, new):
log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new)
couchpotato/core/providers/torrent/publichd/main.py
+36 -58
View File
@@ -6,15 +6,14 @@ from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
import re
from urlparse import parse_qs
from urllib import quote_plus
import urllib2
log = CPLog(__name__)
import re
class PublicHD(TorrentProvider):
log = CPLog(__name__)
urls = {
'test': 'http://publichd.eu',
'download': 'http://publichd.eu/%s',
@@ -22,37 +21,35 @@ class PublicHD(TorrentProvider):
'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d',
}
cat_ids = [([2], ['720p']), ([5], ['1080p']), ([15], ['bdrip']),
([16], ['brrip']), ([16], ['blue-ray'])]
cat_ids = [([2], ['720p']), ([5], ['1080p']), ([16], ['brrip']),
([16], ['bd50'])]
cat_backup_id = 0
def search(self, movie, quality):
results = []
if self.isDisabled():
if self.isDisabled() and quality['hd'] != True:
return results
movie_name = re.sub("\W", ' ', getTitle(movie['library']))
movie_name = re.sub(' ', ' ', movie_name)
log.info('Cleaned Name: %s', movie_name)
cache_key = 'publichd.%s.%s' % (movie['library']['identifier'],
quality.get('identifier'))
searchUrl = self.urls['search'] % (quote_plus(movie_name + ' '
+ quality['identifier']),
self.getCatId(quality['identifier'])[0])
log.info('searchUrl: %s', searchUrl)
data = self.getCache(cache_key, searchUrl)
search_url = self.urls['search'] \
% (self.for_search(getTitle(movie['library'])
+ ' ' + quality['identifier']),
self.getCatId(quality['identifier'])[0])
self.log.info('searchUrl: %s', search_url)
data = self.getCache(cache_key, search_url)
if not data:
log.error('Failed to get data from %s.', searchUrl)
self.log.error('Failed to get data from %s.', search_url)
return results
try:
soup = BeautifulSoup(data)
resultsTable = soup.find('table', attrs={'id': 'bgtorrlist2'
})
entries = resultsTable.findAll('tr')
results_table = soup.find('table',
attrs={'id': 'bgtorrlist2'})
entries = results_table.find_all('tr')
for result in entries[2:len(entries) - 1]:
info_url = result.find(href=re.compile('torrent-details'
))
@@ -65,12 +62,14 @@ class PublicHD(TorrentProvider):
'description': '',
'provider': self.getName(),
}
log.info('Name: %s', result.findAll('td')[1].string)
log.info('Seeders: %s', result.findAll('td'
)[4].string)
log.info('Leaches: %s', result.findAll('td'
)[5].string)
log.info('Size: %s', result.findAll('td')[7].string)
self.log.debug('Name: %s', result.find_all('td'
)[1].string)
self.log.debug('Seeders: %s', result.find_all('td'
)[4].string)
self.log.debug('Leaches: %s', result.find_all('td'
)[5].string)
self.log.debug('Size: %s', result.find_all('td'
)[7].string)
url = parse_qs(info_url['href'])
@@ -78,22 +77,23 @@ class PublicHD(TorrentProvider):
new['id'] = url['id'][0]
new['url'] = self.urls['download'] % download['href'
]
new['size'] = self.parseSize(result.findAll('td'
new['size'] = self.parseSize(result.find_all('td'
)[7].string)
new['seeders'] = int(result.findAll('td')[4].string)
new['leechers'] = int(result.findAll('td'
new['seeders'] = int(result.find_all('td'
)[4].string)
new['leechers'] = int(result.find_all('td'
)[5].string)
new['imdbid'] = movie['library']['identifier']
new['extra_score'] = self.extra_score
new['score'] = fireEvent('score.calculate', new,
movie, single=True)
is_imdb = self.imdb_match(self.urls['detail']
% new['id'], movie['library']['identifier'])
is_correct_movie = fireEvent(
'searcher.correct_movie',
nzb=new,
movie=movie,
quality=quality,
imdb_results=True,
imdb_results=is_imdb,
single_category=False,
single=True,
)
@@ -104,35 +104,13 @@ class PublicHD(TorrentProvider):
self.found(new)
return results
except Exception, e:
log.debug(e)
log.info('Error occured during parsing! Passing only processed entries'
)
except Exception, err:
self.log.debug(err)
self.log.info('Error occured during parsing! Passing only processed entries'
)
return results
def extra_score(self, torrent):
url = self.urls['detail'] % torrent['id']
log.info('extra_score: %s', url)
imdbId = torrent['imdbid']
return self.imdbMatch(url, imdbId)
def imdbMatch(self, url, imdbId):
log.info('imdbMatch: %s', url)
try:
data = urllib2.urlopen(url).read()
pass
except IOError:
log.error('Failed to open %s.' % url)
return ''
imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' \
+ imdbIdAlt in data:
return 50
return 0
def download(self, url='', nzb_id=''):
log.info('Downloading: %s', url)
self.log.info('Downloading: %s', url)
torrent = self.urlopen(url)
return torrent
couchpotato/core/providers/torrent/thepiratebay/main.py
+45 -62
View File
@@ -6,14 +6,13 @@ from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
import re
from urllib import quote_plus
import urllib2
log = CPLog(__name__)
class ThePirateBay(TorrentProvider):
log = CPLog(__name__)
cat_ids = [([207], ['720p', '1080p']), ([201], [
'cam',
'ts',
@@ -28,10 +27,13 @@ class ThePirateBay(TorrentProvider):
def __init__(self):
super(ThePirateBay, self).__init__()
self.urls = {"test": self.getAPIurl(), 'detail': '%s/torrent/%s', 'search': '%s/search/%s/0/7/%d'}
self.urls = {'test': self.getapiurl(),
'detail': '%s/torrent/%s',
'search': '%s/search/%s/0/7/%d'}
def getAPIurl(self, url=""):
return (("http://thepiratebay.se", self.conf('domain_for_tpb'))[self.conf('domain_for_tpb') != None]) + url
def getapiurl(self, url=''):
return ('http://thepiratebay.se', self.conf('domain_for_tpb'
))[self.conf('domain_for_tpb') != None] + url
def search(self, movie, quality):
@@ -39,34 +41,35 @@ class ThePirateBay(TorrentProvider):
if self.isDisabled():
return results
movie_name = re.sub("\W", ' ', getTitle(movie['library']))
movie_name = re.sub(' ', ' ', movie_name)
log.info('API url: %s', self.getAPIurl())
log.info('Cleaned Name: %s', movie_name)
cache_key = 'thepiratebay.%s.%s' % (movie['library'
]['identifier'], quality.get('identifier'))
searchUrl = self.urls['search'] % (self.getAPIurl(),
quote_plus(movie_name + ' ' + quality['identifier']),
search_url = self.urls['search'] % (self.getapiurl(),
self.for_search(getTitle(movie['library'])
+ ' ' + quality['identifier']),
self.getCatId(quality['identifier'])[0])
log.info('searchUrl: %s', searchUrl)
data = self.getCache(cache_key, searchUrl)
#print data
self.log.info('searchUrl: %s', search_url)
data = self.getCache(cache_key, search_url)
# print data
if not data:
log.error('Failed to get data from %s.', searchUrl)
self.log.error('Failed to get data from %s.', search_url)
return results
try:
soup = BeautifulSoup(data)
resultsTable = soup.find('table',
results_table = soup.find('table',
attrs={'id': 'searchResult'})
entries = resultsTable.findAll('tr')
entries = results_table.find_all('tr')
for result in entries[1:]:
link = result.find(href=re.compile('torrent\/\d+\/'))
download = result.find(href=re.compile('magnet:'))
#Uploaded 06-28 02:27, Size 1.37 GiB,
size = re.search('Size (?P<size>.+),', unicode(result.select("font.detDesc")[0])).group("size")
# Uploaded 06-28 02:27, Size 1.37 GiB,
size = re.search('Size (?P<size>.+),',
unicode(result.select('font.detDesc'
)[0])).group('size')
if link and download:
new = {
'type': 'magnet',
@@ -81,39 +84,39 @@ class ThePirateBay(TorrentProvider):
alt=re.compile('VIP')) != None]
moderated = (0, 50)[result.find('img',
alt=re.compile('Moderator')) != None]
is_imdb = self.imdb_match(self.getapiurl(link['href'
]), movie['library']['identifier'])
log.info('Name: %s', link.string)
log.info('Seeders: %s', result.findAll('td'
)[2].string)
log.info('Leechers: %s', result.findAll('td'
)[3].string)
log.info('Size: %s', size)
log.info('Score(trusted + vip + moderated): %d',
trusted + vip + moderated)
self.log.info('Name: %s', link.string)
self.log.info('Seeders: %s', result.find_all('td'
)[2].string)
self.log.info('Leechers: %s', result.find_all('td'
)[3].string)
self.log.info('Size: %s', size)
self.log.info('Score(trusted + vip + moderated): %d'
, trusted + vip + moderated)
new['name'] = link.string
new['id'] = re.search('/(?P<id>\d+)/', link['href'
]).group('id')
new['url'] = self.getAPIurl(link['href'])
new['magnet'] = unicode(download['href']) # forcing of storing full magnet data
new['url'] = self.getapiurl(link['href'])
new['magnet'] = download['href']
new['size'] = self.parseSize(size)
new['seeders'] = int(result.findAll('td')[2].string)
new['leechers'] = int(result.findAll('td'
new['seeders'] = int(result.find_all('td'
)[2].string)
new['leechers'] = int(result.find_all('td'
)[3].string)
new['TPB_score'] = trusted + vip + moderated
new['extra_score'] = self.extra_score
new['extra_score'] = lambda : trusted + vip \
+ moderated
new['score'] = fireEvent('score.calculate', new,
movie, single=True)
isImdb = self.imdbMatch(self.getAPIurl(link['href']), movie['library']['identifier'])
is_correct_movie = fireEvent(
'searcher.correct_movie',
nzb=new,
movie=movie,
quality=quality,
imdb_results=isImdb,
imdb_results=is_imdb,
single_category=False,
single=True,
)
@@ -123,29 +126,9 @@ class ThePirateBay(TorrentProvider):
self.found(new)
return results
except Exception, e:
log.debug(e)
log.info('Error occured during parsing! Passing only processed entries'
)
except Exception, error:
self.log.debug(error)
return results
def extra_score(self, torrent):
return torrent["TPB_score"]
def imdbMatch(self, url, imdbId):
log.info('imdbMatch: %s', url)
try:
data = urllib2.urlopen(url).read()
pass
except:
log.error('Failed to open %s.' % url)
return False
imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' \
+ imdbIdAlt in data:
return True
return False
def download(self, url='', nzb_id=''):
return url