dinislage/CouchPotatoServer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

for_search, imdb_match added to class

Browse Source
This commit is contained in:
Janez Troha committed 2012-07-03 12:32:12 +02:00
1 parent dfd522d7b9
commit a60b227e81
3 files changed
+103 -120
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

couchpotato/core/providers/base.py
+22
View File
@@ -3,6 +3,9 @@ from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env from couchpotato.environment import Env
from urlparse import urlparse from urlparse import urlparse
from urllib import quote_plus
from couchpotato.core.helpers.encoding import simplifyString
import re import re
import time import time
@@ -105,5 +108,24 @@ class YarrProvider(Provider):
return [self.cat_backup_id] return [self.cat_backup_id]
def imdb_match(self, url, imdb_id):
""" Searches for imdb_id in url of webpage """
log.info('Finding if imbd_id(%s) is found in url: %s' % (imdb_id, url))
try:
data = self.urlopen(url)
except:
log.error('Failed to open %s.' % url)
return False
imdb_id_alt = re.sub('tt[0]*', 'tt', imdb_id)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdb_id in data or 'imdb.com/title/' \
+ imdb_id_alt in data:
return True
return False
def for_search(self, string):
""" Prepare string for search, removing all characters that might confuse search engine"""
return quote_plus(simplifyString(string))
def found(self, new): def found(self, new):
log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new) log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new)
couchpotato/core/providers/torrent/publichd/main.py
+36 -58
View File
@@ -6,15 +6,14 @@ from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import getTitle from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider from couchpotato.core.providers.torrent.base import TorrentProvider
import re
from urlparse import parse_qs from urlparse import parse_qs
from urllib import quote_plus
import urllib2 import re
log = CPLog(__name__)
class PublicHD(TorrentProvider): class PublicHD(TorrentProvider):
log = CPLog(__name__)
urls = { urls = {
'test': 'http://publichd.eu', 'test': 'http://publichd.eu',
'download': 'http://publichd.eu/%s', 'download': 'http://publichd.eu/%s',
@@ -22,37 +21,35 @@ class PublicHD(TorrentProvider):
'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d', 'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d',
} }
cat_ids = [([2], ['720p']), ([5], ['1080p']), ([15], ['bdrip']), cat_ids = [([2], ['720p']), ([5], ['1080p']), ([16], ['brrip']),
([16], ['brrip']), ([16], ['blue-ray'])] ([16], ['bd50'])]
cat_backup_id = 0 cat_backup_id = 0
def search(self, movie, quality): def search(self, movie, quality):
results = [] results = []
if self.isDisabled(): if self.isDisabled() and quality['hd'] != True:
return results return results
movie_name = re.sub("\W", ' ', getTitle(movie['library']))
movie_name = re.sub(' ', ' ', movie_name)
log.info('Cleaned Name: %s', movie_name)
cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], cache_key = 'publichd.%s.%s' % (movie['library']['identifier'],
quality.get('identifier')) quality.get('identifier'))
searchUrl = self.urls['search'] % (quote_plus(movie_name + ' ' search_url = self.urls['search'] \
+ quality['identifier']), % (self.for_search(getTitle(movie['library'])
self.getCatId(quality['identifier'])[0]) + ' ' + quality['identifier']),
log.info('searchUrl: %s', searchUrl) self.getCatId(quality['identifier'])[0])
data = self.getCache(cache_key, searchUrl) self.log.info('searchUrl: %s', search_url)
data = self.getCache(cache_key, search_url)
if not data: if not data:
log.error('Failed to get data from %s.', searchUrl) self.log.error('Failed to get data from %s.', search_url)
return results return results
try: try:
soup = BeautifulSoup(data) soup = BeautifulSoup(data)
resultsTable = soup.find('table', attrs={'id': 'bgtorrlist2' results_table = soup.find('table',
}) attrs={'id': 'bgtorrlist2'})
entries = resultsTable.findAll('tr') entries = results_table.find_all('tr')
for result in entries[2:len(entries) - 1]: for result in entries[2:len(entries) - 1]:
info_url = result.find(href=re.compile('torrent-details' info_url = result.find(href=re.compile('torrent-details'
)) ))
@@ -65,12 +62,14 @@ class PublicHD(TorrentProvider):
'description': '', 'description': '',
'provider': self.getName(), 'provider': self.getName(),
} }
log.info('Name: %s', result.findAll('td')[1].string) self.log.debug('Name: %s', result.find_all('td'
log.info('Seeders: %s', result.findAll('td' )[1].string)
)[4].string) self.log.debug('Seeders: %s', result.find_all('td'
log.info('Leaches: %s', result.findAll('td' )[4].string)
)[5].string) self.log.debug('Leaches: %s', result.find_all('td'
log.info('Size: %s', result.findAll('td')[7].string) )[5].string)
self.log.debug('Size: %s', result.find_all('td'
)[7].string)
url = parse_qs(info_url['href']) url = parse_qs(info_url['href'])
@@ -78,22 +77,23 @@ class PublicHD(TorrentProvider):
new['id'] = url['id'][0] new['id'] = url['id'][0]
new['url'] = self.urls['download'] % download['href' new['url'] = self.urls['download'] % download['href'
] ]
new['size'] = self.parseSize(result.findAll('td' new['size'] = self.parseSize(result.find_all('td'
)[7].string) )[7].string)
new['seeders'] = int(result.findAll('td')[4].string) new['seeders'] = int(result.find_all('td'
new['leechers'] = int(result.findAll('td' )[4].string)
new['leechers'] = int(result.find_all('td'
)[5].string) )[5].string)
new['imdbid'] = movie['library']['identifier']
new['extra_score'] = self.extra_score
new['score'] = fireEvent('score.calculate', new, new['score'] = fireEvent('score.calculate', new,
movie, single=True) movie, single=True)
is_imdb = self.imdb_match(self.urls['detail']
% new['id'], movie['library']['identifier'])
is_correct_movie = fireEvent( is_correct_movie = fireEvent(
'searcher.correct_movie', 'searcher.correct_movie',
nzb=new, nzb=new,
movie=movie, movie=movie,
quality=quality, quality=quality,
imdb_results=True, imdb_results=is_imdb,
single_category=False, single_category=False,
single=True, single=True,
) )
@@ -104,35 +104,13 @@ class PublicHD(TorrentProvider):
self.found(new) self.found(new)
return results return results
except Exception, e: except Exception, err:
log.debug(e) self.log.debug(err)
log.info('Error occured during parsing! Passing only processed entries' self.log.info('Error occured during parsing! Passing only processed entries'
) )
return results return results
def extra_score(self, torrent):
url = self.urls['detail'] % torrent['id']
log.info('extra_score: %s', url)
imdbId = torrent['imdbid']
return self.imdbMatch(url, imdbId)
def imdbMatch(self, url, imdbId):
log.info('imdbMatch: %s', url)
try:
data = urllib2.urlopen(url).read()
pass
except IOError:
log.error('Failed to open %s.' % url)
return ''
imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' \
+ imdbIdAlt in data:
return 50
return 0
def download(self, url='', nzb_id=''): def download(self, url='', nzb_id=''):
log.info('Downloading: %s', url) self.log.info('Downloading: %s', url)
torrent = self.urlopen(url) torrent = self.urlopen(url)
return torrent return torrent
couchpotato/core/providers/torrent/thepiratebay/main.py
+45 -62
View File
@@ -6,14 +6,13 @@ from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import getTitle from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider from couchpotato.core.providers.torrent.base import TorrentProvider
import re import re
from urllib import quote_plus
import urllib2
log = CPLog(__name__)
class ThePirateBay(TorrentProvider): class ThePirateBay(TorrentProvider):
log = CPLog(__name__)
cat_ids = [([207], ['720p', '1080p']), ([201], [ cat_ids = [([207], ['720p', '1080p']), ([201], [
'cam', 'cam',
'ts', 'ts',
@@ -28,10 +27,13 @@ class ThePirateBay(TorrentProvider):
def __init__(self): def __init__(self):
super(ThePirateBay, self).__init__() super(ThePirateBay, self).__init__()
self.urls = {"test": self.getAPIurl(), 'detail': '%s/torrent/%s', 'search': '%s/search/%s/0/7/%d'} self.urls = {'test': self.getapiurl(),
'detail': '%s/torrent/%s',
'search': '%s/search/%s/0/7/%d'}
def getAPIurl(self, url=""): def getapiurl(self, url=''):
return (("http://thepiratebay.se", self.conf('domain_for_tpb'))[self.conf('domain_for_tpb') != None]) + url return ('http://thepiratebay.se', self.conf('domain_for_tpb'
))[self.conf('domain_for_tpb') != None] + url
def search(self, movie, quality): def search(self, movie, quality):
@@ -39,34 +41,35 @@ class ThePirateBay(TorrentProvider):
if self.isDisabled(): if self.isDisabled():
return results return results
movie_name = re.sub("\W", ' ', getTitle(movie['library']))
movie_name = re.sub(' ', ' ', movie_name)
log.info('API url: %s', self.getAPIurl())
log.info('Cleaned Name: %s', movie_name)
cache_key = 'thepiratebay.%s.%s' % (movie['library' cache_key = 'thepiratebay.%s.%s' % (movie['library'
]['identifier'], quality.get('identifier')) ]['identifier'], quality.get('identifier'))
searchUrl = self.urls['search'] % (self.getAPIurl(), search_url = self.urls['search'] % (self.getapiurl(),
quote_plus(movie_name + ' ' + quality['identifier']), self.for_search(getTitle(movie['library'])
+ ' ' + quality['identifier']),
self.getCatId(quality['identifier'])[0]) self.getCatId(quality['identifier'])[0])
log.info('searchUrl: %s', searchUrl) self.log.info('searchUrl: %s', search_url)
data = self.getCache(cache_key, searchUrl) data = self.getCache(cache_key, search_url)
#print data
# print data
if not data: if not data:
log.error('Failed to get data from %s.', searchUrl) self.log.error('Failed to get data from %s.', search_url)
return results return results
try: try:
soup = BeautifulSoup(data) soup = BeautifulSoup(data)
resultsTable = soup.find('table', results_table = soup.find('table',
attrs={'id': 'searchResult'}) attrs={'id': 'searchResult'})
entries = resultsTable.findAll('tr') entries = results_table.find_all('tr')
for result in entries[1:]: for result in entries[1:]:
link = result.find(href=re.compile('torrent\/\d+\/')) link = result.find(href=re.compile('torrent\/\d+\/'))
download = result.find(href=re.compile('magnet:')) download = result.find(href=re.compile('magnet:'))
#Uploaded 06-28 02:27, Size 1.37 GiB,
size = re.search('Size (?P<size>.+),', unicode(result.select("font.detDesc")[0])).group("size") # Uploaded 06-28 02:27, Size 1.37 GiB,
size = re.search('Size (?P<size>.+),',
unicode(result.select('font.detDesc'
)[0])).group('size')
if link and download: if link and download:
new = { new = {
'type': 'magnet', 'type': 'magnet',
@@ -81,39 +84,39 @@ class ThePirateBay(TorrentProvider):
alt=re.compile('VIP')) != None] alt=re.compile('VIP')) != None]
moderated = (0, 50)[result.find('img', moderated = (0, 50)[result.find('img',
alt=re.compile('Moderator')) != None] alt=re.compile('Moderator')) != None]
is_imdb = self.imdb_match(self.getapiurl(link['href'
]), movie['library']['identifier'])
log.info('Name: %s', link.string) self.log.info('Name: %s', link.string)
log.info('Seeders: %s', result.findAll('td' self.log.info('Seeders: %s', result.find_all('td'
)[2].string) )[2].string)
log.info('Leechers: %s', result.findAll('td' self.log.info('Leechers: %s', result.find_all('td'
)[3].string) )[3].string)
log.info('Size: %s', size) self.log.info('Size: %s', size)
log.info('Score(trusted + vip + moderated): %d', self.log.info('Score(trusted + vip + moderated): %d'
trusted + vip + moderated) , trusted + vip + moderated)
new['name'] = link.string new['name'] = link.string
new['id'] = re.search('/(?P<id>\d+)/', link['href' new['id'] = re.search('/(?P<id>\d+)/', link['href'
]).group('id') ]).group('id')
new['url'] = self.getAPIurl(link['href']) new['url'] = self.getapiurl(link['href'])
new['magnet'] = unicode(download['href']) # forcing of storing full magnet data new['magnet'] = download['href']
new['size'] = self.parseSize(size) new['size'] = self.parseSize(size)
new['seeders'] = int(result.findAll('td')[2].string) new['seeders'] = int(result.find_all('td'
new['leechers'] = int(result.findAll('td' )[2].string)
new['leechers'] = int(result.find_all('td'
)[3].string) )[3].string)
new['extra_score'] = lambda : trusted + vip \
new['TPB_score'] = trusted + vip + moderated + moderated
new['extra_score'] = self.extra_score
new['score'] = fireEvent('score.calculate', new, new['score'] = fireEvent('score.calculate', new,
movie, single=True) movie, single=True)
isImdb = self.imdbMatch(self.getAPIurl(link['href']), movie['library']['identifier'])
is_correct_movie = fireEvent( is_correct_movie = fireEvent(
'searcher.correct_movie', 'searcher.correct_movie',
nzb=new, nzb=new,
movie=movie, movie=movie,
quality=quality, quality=quality,
imdb_results=isImdb, imdb_results=is_imdb,
single_category=False, single_category=False,
single=True, single=True,
) )
@@ -123,29 +126,9 @@ class ThePirateBay(TorrentProvider):
self.found(new) self.found(new)
return results return results
except Exception, e: except Exception, error:
log.debug(e) self.log.debug(error)
log.info('Error occured during parsing! Passing only processed entries'
)
return results return results
def extra_score(self, torrent):
return torrent["TPB_score"]
def imdbMatch(self, url, imdbId):
log.info('imdbMatch: %s', url)
try:
data = urllib2.urlopen(url).read()
pass
except:
log.error('Failed to open %s.' % url)
return False
imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
data = unicode(data, errors='ignore')
if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' \
+ imdbIdAlt in data:
return True
return False
def download(self, url='', nzb_id=''): def download(self, url='', nzb_id=''):
return url return url