dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve name searching. closes #1137

Browse Source
This commit is contained in:
Ruud
2012-12-17 18:22:12 +01:00
parent a57ba9026d
commit 65570ba479
9 changed files with 95 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/helpers/variable.py
+11
View File
@@ -1,3 +1,4 @@
from couchpotato.core.helpers.encoding import simplifyString, toSafeString
from couchpotato.core.logger import CPLog
import hashlib
import os.path
@@ -153,6 +154,16 @@ def getTitle(library_dict):
log.error('Could not get title for library item: %s', library_dict)
return None
def possibleTitles(raw_title):
titles = []
titles.append(toSafeString(raw_title).lower())
titles.append(raw_title.lower())
titles.append(simplifyString(raw_title))
return list(set(titles))
def randomString(size = 8, chars = string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for x in range(size))
couchpotato/core/plugins/base.py
+2 -2
View File
@@ -3,7 +3,7 @@ from couchpotato import addView
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import tryUrlencode, simplifyString, ss, \
toSafeString
from couchpotato.core.helpers.variable import getExt
from couchpotato.core.helpers.variable import getExt, md5
from couchpotato.core.logger import CPLog
from couchpotato.environment import Env
from flask.templating import render_template_string
@@ -222,7 +222,7 @@ class Plugin(object):
def getCache(self, cache_key, url = None, **kwargs):
cache_key = simplifyString(cache_key)
cache_key = md5(cache_key)
cache = Env.get('cache').get(cache_key)
if cache:
if not Env.get('dev'): log.debug('Getting cache %s', cache_key)
couchpotato/core/plugins/searcher/main.py
+12 -10
View File
@@ -3,7 +3,8 @@ from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import simplifyString, toUnicode
from couchpotato.core.helpers.request import jsonified, getParam
from couchpotato.core.helpers.variable import md5, getTitle, splitString
from couchpotato.core.helpers.variable import md5, getTitle, splitString, \
possibleTitles
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Movie, Release, ReleaseInfo
@@ -365,17 +366,18 @@ class Searcher(Plugin):
if self.checkIMDB([nzb['description']], movie['library']['identifier']):
return True
for movie_title in movie['library']['titles']:
movie_words = re.split('\W+', simplifyString(movie_title['title']))
for raw_title in movie['library']['titles']:
for movie_title in possibleTitles(raw_title['title']):
movie_words = re.split('\W+', simplifyString(movie_title))
if self.correctName(nzb['name'], movie_title['title']):
# if no IMDB link, at least check year range 1
if len(movie_words) > 2 and self.correctYear([nzb['name']], movie['library']['year'], 1):
return True
if self.correctName(nzb['name'], movie_title):
# if no IMDB link, at least check year range 1
if len(movie_words) > 2 and self.correctYear([nzb['name']], movie['library']['year'], 1):
return True
# if no IMDB link, at least check year
if len(movie_words) <= 2 and self.correctYear([nzb['name']], movie['library']['year'], 0):
return True
# if no IMDB link, at least check year
if len(movie_words) <= 2 and self.correctYear([nzb['name']], movie['library']['year'], 0):
return True
log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'" % (nzb['name'], movie_name, movie['library']['year']))
return False
couchpotato/core/providers/base.py
+14 -1
View File
@@ -1,5 +1,6 @@
from couchpotato.core.event import addEvent
from couchpotato.core.helpers.variable import tryFloat
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryFloat, getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
@@ -155,3 +156,15 @@ class YarrProvider(Provider):
new['provider_extra'] = ', %s' % new['provider_extra']
log.info('Found: score(%(score)s) on %(provider)s%(provider_extra)s: %(name)s', new)
def removeDuplicateResults(self, results):
result_ids = []
new_results = []
for result in results:
if result['id'] not in result_ids:
new_results.append(result)
result_ids.append(result['id'])
return new_results
couchpotato/core/providers/nzb/ftdworld/main.py
+15 -8
View File
@@ -1,8 +1,7 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.variable import tryInt, possibleTitles, getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
@@ -22,7 +21,7 @@ class FTDWorld(NZBProvider):
'login': 'http://ftdworld.net/index.php',
}
http_time_between_calls = 1 #seconds
http_time_between_calls = 3 #seconds
cat_ids = [
([4, 11], ['dvdr']),
@@ -33,11 +32,19 @@ class FTDWorld(NZBProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
q = '%s %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'])
results = []
for title in possibleTitles(getTitle(movie['library'])):
results.extend(self._search(title, movie, quality))
return self.removeDuplicateResults(results)
def _search(self, title, movie, quality):
results = []
q = '"%s" %s' % (title, movie['library']['year'])
params = {
'ctitle': q,
@@ -81,7 +88,7 @@ class FTDWorld(NZBProvider):
'download': self.loginDownload,
'detail_url': self.urls['detail'] % nzb_id,
'description': '',
'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3),
'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0,
}
is_correct_movie = fireEvent('searcher.correct_movie',
couchpotato/core/providers/nzb/nzbclub/main.py
+13 -7
View File
@@ -1,12 +1,10 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
import time
import xml.etree.ElementTree as XMLTree
@@ -24,11 +22,19 @@ class NZBClub(NZBProvider, RSS):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
results = []
for title in possibleTitles(getTitle(movie['library'])):
results.extend(self._search(title, movie, quality))
return self.removeDuplicateResults(results)
def _search(self, title, movie, quality):
results = []
q = '"%s %s" %s' % (title, movie['library']['year'], quality.get('identifier'))
params = {
'q': q,
couchpotato/core/providers/nzb/nzbindex/main.py
+15 -8
View File
@@ -1,9 +1,8 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
@@ -27,11 +26,19 @@ class NzbIndex(NZBProvider, RSS):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
results = []
for title in possibleTitles(getTitle(movie['library'])):
results.extend(self._search(title, movie, quality))
return self.removeDuplicateResults(results)
def _search(self, title, movie, quality):
results = []
q = '"%s" %s %s' % (title, movie['library']['year'], quality.get('identifier'))
arguments = tryUrlencode({
'q': q,
'age': Env.setting('retention', 'nzb'),
@@ -45,9 +52,9 @@ class NzbIndex(NZBProvider, RSS):
})
url = "%s?%s" % (self.urls['api'], arguments)
cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], q)
data = self.getCache(cache_key, url)
if data:
try:
try:
couchpotato/core/providers/nzb/nzbsrus/main.py
+1 -2
View File
@@ -46,8 +46,7 @@ class Nzbsrus(NZBProvider, RSS):
url = "%s&%s&%s" % (self.urls['search'], arguments , cat_id_string)
cache_key = 'nzbsrus_1.%s.%s' % (movie['library'].get('identifier'), cat_id_string)
single_cat = True
cache_key = 'nzbsrus.%s.%s' % (movie['library'].get('identifier'), cat_id_string)
data = self.getCache(cache_key, url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
if data:
couchpotato/core/providers/nzb/omgwtfnzbs/main.py
+12 -6
View File
@@ -1,9 +1,8 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.helpers.variable import tryInt, getTitle, possibleTitles
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
@@ -33,12 +32,19 @@ class OMGWTFNZBs(NZBProvider, RSS):
def search(self, movie, quality):
pre_releases = fireEvent('quality.pre_releases', single = True)
if self.isDisabled() or quality['identifier'] in pre_releases:
return []
results = []
if self.isDisabled() or quality['identifier'] in pre_releases:
return results
for title in possibleTitles(getTitle(movie['library'])):
results.extend(self._search(title, movie, quality))
q = '%s %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'])
return self.removeDuplicateResults(results)
def _search(self, title, movie, quality):
results = []
q = '%s %s' % (title, movie['library']['year'])
params = {
'search': q,