diff --git a/CouchPotato.py b/CouchPotato.py index 375a1d41..364d77a4 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -29,7 +29,7 @@ class Loader(object): # Get options via arg from couchpotato.runner import getOptions - self.options = getOptions(base_path, sys.argv[1:]) + self.options = getOptions(sys.argv[1:]) # Load settings settings = Env.get('settings') diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index 1ca7d878..ca45ceb0 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -90,7 +90,11 @@ class Core(Plugin): def shutdown(): self.initShutdown() - IOLoop.current().add_callback(shutdown) + + if IOLoop.current()._closing: + shutdown() + else: + IOLoop.current().add_callback(shutdown) return 'shutdown' @@ -139,7 +143,8 @@ class Core(Plugin): log.debug('Safe to shutdown/restart') try: - IOLoop.current().stop() + if not IOLoop.current()._closing: + IOLoop.current().stop() except RuntimeError: pass except: diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index cde17eb2..9ebecce2 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -10,7 +10,7 @@ from threading import RLock from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync -from couchpotato.core.helpers.encoding import ss +from couchpotato.core.helpers.encoding import ss, sp from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -182,7 +182,7 @@ class BaseUpdater(Plugin): def deletePyc(self, only_excess = True, show_logs = True): - for root, dirs, files in scandir.walk(ss(Env.get('app_dir'))): + for root, dirs, files in scandir.walk(Env.get('app_dir')): pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) py_files = set(filter(lambda filename: filename.endswith('.py'), files)) @@ -322,8 +322,9 @@ class SourceUpdater(BaseUpdater): return False def replaceWith(self, path): - app_dir = ss(Env.get('app_dir')) - data_dir = ss(Env.get('data_dir')) + path = sp(path) + app_dir = Env.get('app_dir') + data_dir = Env.get('data_dir') # Get list of files we want to overwrite self.deletePyc() diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 532befae..f9607137 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -28,6 +28,7 @@ class Database(object): addEvent('database.setup_index', self.setupIndex) addEvent('app.migrate', self.migrate) + addEvent('app.after_shutdown', self.close) def getDB(self): @@ -37,6 +38,9 @@ class Database(object): return self.db + def close(self, **kwargs): + self.getDB().close() + def setupIndex(self, index_name, klass): self.indexes.append(index_name) @@ -412,7 +416,10 @@ class Database(object): empty_info = True rel['info'] = {} - quality = quality_link[rel.get('quality_id')] + quality = quality_link.get(rel.get('quality_id')) + if not quality: + continue + release_status = statuses.get(rel.get('status_id')).get('identifier') if rel['info'].get('download_id'): diff --git a/couchpotato/core/downloaders/rtorrent_.py b/couchpotato/core/downloaders/rtorrent_.py index cccecb90..5ea3a4de 100644 --- a/couchpotato/core/downloaders/rtorrent_.py +++ b/couchpotato/core/downloaders/rtorrent_.py @@ -154,19 +154,13 @@ class rTorrent(DownloaderBase): return False def getTorrentStatus(self, torrent): - if torrent.hashing or torrent.hash_checking or torrent.message: - return 'busy' - if not torrent.complete: return 'busy' - if not torrent.open: - return 'completed' - - if torrent.state and torrent.active: + if torrent.open: return 'seeding' - return 'busy' + return 'completed' def getAllDownloadStatus(self, ids): log.debug('Checking rTorrent download status.') @@ -244,7 +238,7 @@ class rTorrent(DownloaderBase): if torrent.is_multi_file() and torrent.directory.endswith(torrent.name): # Remove empty directories bottom up try: - for path, _, _ in scandir.walk(torrent.directory, topdown = False): + for path, _, _ in scandir.walk(sp(torrent.directory), topdown = False): os.rmdir(path) except OSError: log.info('Directory "%s" contains extra files, unable to remove', torrent.directory) diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index ff6a3e59..41591683 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -78,7 +78,7 @@ def sp(path, *args): # Replace *NIX ambiguous '//' at the beginning of a path with '/' (crashes guessit) path = re.sub('^//', '/', path) - return toUnicode(path) + return path def ek(original, *args): diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index c491c016..f0abd28c 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -25,7 +25,6 @@ class MediaBase(Plugin): def onComplete(): try: - db = get_db() media = fireEvent('media.get', media_id, single = True) event_name = '%s.searcher.single' % media.get('type') diff --git a/couchpotato/core/media/_base/media/index.py b/couchpotato/core/media/_base/media/index.py index d0b52c38..fd1affca 100644 --- a/couchpotato/core/media/_base/media/index.py +++ b/couchpotato/core/media/_base/media/index.py @@ -99,7 +99,7 @@ from couchpotato.core.helpers.encoding import simplifyString""" class TitleIndex(TreeBasedIndex): - _version = 2 + _version = 3 custom_header = """from CodernityDB.tree_index import TreeBasedIndex from string import ascii_letters @@ -123,7 +123,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString""" nr_prefix = '' if title and len(title) > 0 and title[0] in ascii_letters else '#' title = simplifyString(title) - for prefix in ['the ']: + for prefix in ['the ', 'an ', 'a ']: if prefix == title[:len(prefix)]: title = title[len(prefix):] break @@ -132,7 +132,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString""" class StartsWithIndex(TreeBasedIndex): - _version = 2 + _version = 3 custom_header = """from CodernityDB.tree_index import TreeBasedIndex from string import ascii_letters @@ -153,7 +153,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString""" title = toUnicode(title) title = simplifyString(title) - for prefix in ['the ']: + for prefix in ['the ', 'an ', 'a ']: if prefix == title[:len(prefix)]: title = title[len(prefix):] break diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 29a98fd1..beb9dd02 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -107,8 +107,7 @@ class MediaPlugin(MediaBase): def handler(): fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id)) - if handler: - return handler + return handler except: log.error('Refresh handler for non existing media: %s', traceback.format_exc()) @@ -361,13 +360,18 @@ class MediaPlugin(MediaBase): media = db.get('id', media_id) if media: deleted = False + + media_releases = fireEvent('release.for_media', media['_id'], single = True) + if delete_from == 'all': + # Delete connected releases + for release in media_releases: + db.delete(release) + db.delete(media) deleted = True else: - media_releases = fireEvent('release.for_media', media['_id'], single = True) - total_releases = len(media_releases) total_deleted = 0 new_media_status = None diff --git a/couchpotato/core/media/_base/providers/base.py b/couchpotato/core/media/_base/providers/base.py index 4476bc49..b13b279d 100644 --- a/couchpotato/core/media/_base/providers/base.py +++ b/couchpotato/core/media/_base/providers/base.py @@ -88,10 +88,14 @@ class Provider(Plugin): if data and len(data) > 0: try: - data = XMLTree.fromstring(ss(data)) + data = XMLTree.fromstring(data) return self.getElements(data, item_path) except: - log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + try: + data = XMLTree.fromstring(ss(data)) + return self.getElements(data, item_path) + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) return [] @@ -200,7 +204,7 @@ class YarrProvider(Provider): self._search(media, quality, results) # Search possible titles else: - media_title = fireEvent('library.query', media, single = True) + media_title = fireEvent('library.query', media, include_year = False, single = True) for title in possibleTitles(media_title): self._searchOnTitle(title, media, quality, results) @@ -298,7 +302,7 @@ class ResultList(list): old_score = new_result['score'] new_result['score'] = int(old_score * is_correct_weight) - log.info('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', ( + log.info2('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', ( is_correct_weight, old_score, new_result['score'] diff --git a/couchpotato/core/media/_base/providers/nzb/binsearch.py b/couchpotato/core/media/_base/providers/nzb/binsearch.py index 90778af8..84c7b314 100644 --- a/couchpotato/core/media/_base/providers/nzb/binsearch.py +++ b/couchpotato/core/media/_base/providers/nzb/binsearch.py @@ -2,7 +2,7 @@ import re import traceback from bs4 import BeautifulSoup -from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.helpers.variable import tryInt, simplifyString from couchpotato.core.logger import CPLog from couchpotato.core.media._base.providers.nzb.base import NZBProvider @@ -50,8 +50,8 @@ class Base(NZBProvider): def extra_check(item): parts = re.search('available:.(?P\d+)./.(?P\d+)', info.text) - total = tryInt(parts.group('total')) - parts = tryInt(parts.group('parts')) + total = float(tryInt(parts.group('total'))) + parts = float(tryInt(parts.group('parts'))) if (total / parts) < 1 and ((total / parts) < 0.95 or ((total / parts) >= 0.95 and not ('par2' in info.text.lower() or 'pa3' in info.text.lower()))): log.info2('Wrong: \'%s\', not complete: %s out of %s', (item['name'], parts, total)) @@ -65,7 +65,7 @@ class Base(NZBProvider): results.append({ 'id': nzb_id, - 'name': title.text, + 'name': simplifyString(title.text), 'age': tryInt(age), 'size': self.parseSize(size_match.group('size')), 'url': self.urls['download'] % nzb_id, diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 4430ac26..04915776 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -2,6 +2,7 @@ from urllib2 import HTTPError from urlparse import urlparse import time import traceback +import re import urllib2 from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode @@ -20,10 +21,11 @@ log = CPLog(__name__) class Base(NZBProvider, RSS): urls = { - 'detail': 'details&id=%s', + 'detail': 'details/%s', 'download': 't=get&id=%s' } + passwords_regex = 'password|wachtwoord' limits_reached = {} http_time_between_calls = 1 # Seconds @@ -43,10 +45,8 @@ class Base(NZBProvider, RSS): def _searchOnHost(self, host, media, quality, results): - query = self.buildUrl(media, host['api_key']) - + query = self.buildUrl(media, host) url = '%s&%s' % (self.getUrl(host['host']), query) - nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) for nzb in nzbs: @@ -79,6 +79,23 @@ class Base(NZBProvider, RSS): if spotter: name_extra = spotter + description = '' + if "@spot.net" in nzb_id: + try: + # Get details for extended description to retrieve passwords + query = self.buildDetailsUrl(nzb_id, host['api_key']) + url = '%s&%s' % (self.getUrl(host['host']), query) + nzb_details = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})[0] + + description = self.getTextElement(nzb_details, 'description') + + # Extract a password from the description + password = re.search('(?:' + self.passwords_regex + ')(?: *)(?:\:|\=)(?: *)(.*?)\|\n|$', description, flags = re.I).group(1) + if password: + name = name + ' {{%s}}' % password.strip() + except: + log.debug('Error getting details of "%s": %s', (name, traceback.format_exc())) + results.append({ 'id': nzb_id, 'provider_extra': urlparse(host['host']).hostname or host['host'], @@ -87,8 +104,9 @@ class Base(NZBProvider, RSS): 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024, 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host), - 'detail_url': '%sdetails/%s' % (cleanHost(host['host']), tryUrlencode(nzb_id)), + 'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id), 'content': self.getTextElement(nzb, 'description'), + 'description': description, 'score': host['extra_score'], }) @@ -191,6 +209,15 @@ class Base(NZBProvider, RSS): return 'try_next' + def buildDetailsUrl(self, nzb_id, api_key): + query = tryUrlencode({ + 't': 'details', + 'id': nzb_id, + 'apikey': api_key, + }) + return query + + config = [{ 'name': 'newznab', @@ -217,7 +244,7 @@ config = [{ }, { 'name': 'host', - 'default': 'api.nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', + 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', 'description': 'The hostname of your newznab provider', }, { diff --git a/couchpotato/core/media/_base/providers/torrent/base.py b/couchpotato/core/media/_base/providers/torrent/base.py index 2ed496c5..9f5f2890 100644 --- a/couchpotato/core/media/_base/providers/torrent/base.py +++ b/couchpotato/core/media/_base/providers/torrent/base.py @@ -44,7 +44,8 @@ class TorrentProvider(YarrProvider): prop_name = 'proxy.%s' % proxy last_check = float(Env.prop(prop_name, default = 0)) - if last_check > time.time() - 1209600: + + if last_check > time.time() - 86400: continue data = '' diff --git a/couchpotato/core/media/_base/providers/torrent/bithdtv.py b/couchpotato/core/media/_base/providers/torrent/bithdtv.py index 94aafcd1..bb12c66a 100644 --- a/couchpotato/core/media/_base/providers/torrent/bithdtv.py +++ b/couchpotato/core/media/_base/providers/torrent/bithdtv.py @@ -25,7 +25,7 @@ class Base(TorrentProvider): def _search(self, media, quality, results): - query = self.buildUrl(media) + query = self.buildUrl(media, quality) url = "%s&%s" % (self.urls['search'], query) diff --git a/couchpotato/core/media/_base/providers/torrent/bitsoup.py b/couchpotato/core/media/_base/providers/torrent/bitsoup.py index 1c762b77..f32e79e7 100644 --- a/couchpotato/core/media/_base/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/_base/providers/torrent/bitsoup.py @@ -15,7 +15,7 @@ class Base(TorrentProvider): 'test': 'https://www.bitsoup.me/', 'login': 'https://www.bitsoup.me/takelogin.php', 'login_check': 'https://www.bitsoup.me/my.php', - 'search': 'https://www.bitsoup.me/browse.php?', + 'search': 'https://www.bitsoup.me/browse.php?%s', 'baseurl': 'https://www.bitsoup.me/%s', } diff --git a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py index 534dc926..a711502a 100644 --- a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py @@ -65,12 +65,13 @@ class Base(TorrentMagnetProvider): if column_name: if column_name == 'name': - link = td.find('div', {'class': 'torrentname'}).find_all('a')[1] - new['id'] = temp.get('id')[-8:] + link = td.find('div', {'class': 'torrentname'}).find_all('a')[2] + new['id'] = temp.get('id')[-7:] new['name'] = link.text new['url'] = td.find('a', 'imagnet')['href'] new['detail_url'] = self.urls['detail'] % (self.getDomain(), link['href'][1:]) - new['score'] = 20 if td.find('a', 'iverif') else 0 + new['verified'] = True if td.find('a', 'iverify') else False + new['score'] = 100 if new['verified'] else 0 elif column_name is 'size': new['size'] = self.parseSize(td.text) elif column_name is 'age': @@ -82,6 +83,10 @@ class Base(TorrentMagnetProvider): nr += 1 + # Only store verified torrents + if self.conf('only_verified') and not new['verified']: + continue + results.append(new) except: log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc()) @@ -151,6 +156,13 @@ config = [{ 'default': 40, 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, + { + 'name': 'only_verified', + 'advanced': True, + 'type': 'bool', + 'default': False, + 'description': 'Only search for verified releases.' + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/media/_base/providers/torrent/sceneaccess.py b/couchpotato/core/media/_base/providers/torrent/sceneaccess.py index 2622e318..e488b63b 100644 --- a/couchpotato/core/media/_base/providers/torrent/sceneaccess.py +++ b/couchpotato/core/media/_base/providers/torrent/sceneaccess.py @@ -24,9 +24,9 @@ class Base(TorrentProvider): http_time_between_calls = 1 # Seconds - def _search(self, media, quality, results): + def _searchOnTitle(self, title, media, quality, results): - url = self.buildUrl(media, quality) + url = self.buildUrl(title, media, quality) data = self.getHTMLData(url) if data: diff --git a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py index d401e6b3..b840fc71 100644 --- a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py +++ b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py @@ -65,7 +65,7 @@ class Base(TorrentMagnetProvider): pass entries = results_table.find_all('tr') - for result in entries[2:]: + for result in entries[1:]: link = result.find(href = re.compile('torrent\/\d+\/')) download = result.find(href = re.compile('magnet:')) @@ -109,7 +109,11 @@ class Base(TorrentMagnetProvider): full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) html = BeautifulSoup(full_description) nfo_pre = html.find('div', attrs = {'class': 'nfo'}) - description = toUnicode(nfo_pre.text) if nfo_pre else '' + description = '' + try: + description = toUnicode(nfo_pre.text) + except: + pass item['description'] = description return item diff --git a/couchpotato/core/media/_base/providers/torrent/torrentday.py b/couchpotato/core/media/_base/providers/torrent/torrentday.py index bcc0593b..a4eb3899 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentday.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentday.py @@ -1,3 +1,4 @@ +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media._base.providers.torrent.base import TorrentProvider @@ -18,9 +19,9 @@ class Base(TorrentProvider): http_time_between_calls = 1 # Seconds - def _search(self, media, quality, results): + def _searchOnTitle(self, title, media, quality, results): - query = self.buildUrl(media) + query = '"%s" %s' % (title, media['info']['year']) data = { '/browse.php?': None, diff --git a/couchpotato/core/media/_base/providers/torrent/yify.py b/couchpotato/core/media/_base/providers/torrent/yify.py index a6941004..82413fc5 100644 --- a/couchpotato/core/media/_base/providers/torrent/yify.py +++ b/couchpotato/core/media/_base/providers/torrent/yify.py @@ -35,7 +35,11 @@ class Base(TorrentProvider): def _search(self, movie, quality, results): - search_url = self.urls['search'] % (self.getDomain(), getIdentifier(movie), quality['identifier']) + domain = self.getDomain() + if not domain: + return + + search_url = self.urls['search'] % (domain, getIdentifier(movie), quality['identifier']) data = self.getJsonData(search_url) @@ -43,21 +47,19 @@ class Base(TorrentProvider): try: for result in data.get('MovieList'): - try: - title = result['TorrentUrl'].split('/')[-1][:-8].replace('_', '.').strip('._') - title = title.replace('.-.', '-') - title = title.replace('..', '.') - except: - continue + if result['Quality'] and result['Quality'] not in result['MovieTitle']: + title = result['MovieTitle'] + ' BrRip ' + result['Quality'] + else: + title = result['MovieTitle'] + ' BrRip' results.append({ 'id': result['MovieID'], 'name': title, 'url': result['TorrentMagnetUrl'], - 'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']), + 'detail_url': self.urls['detail'] % (domain, result['MovieID']), 'size': self.parseSize(result['Size']), 'seeders': tryInt(result['TorrentSeeds']), - 'leechers': tryInt(result['TorrentPeers']) + 'leechers': tryInt(result['TorrentPeers']), }) except: diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index fc07ded2..4e8dae2e 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -87,31 +87,23 @@ class Searcher(SearcherBase): def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = None): if not preferred_quality: preferred_quality = {} - name = nzb['name'] - size = nzb.get('size', 0) - nzb_words = re.split('\W+', simplifyString(name)) - - qualities = fireEvent('quality.all', single = True) - found = {} - for quality in qualities: - # Main in words - if quality['identifier'] in nzb_words: - found[quality['identifier']] = True - - # Alt in words - if list(set(nzb_words) & set(quality['alternative'])): - found[quality['identifier']] = True # Try guessing via quality tags - guess = fireEvent('quality.guess', [nzb.get('name')], single = True) + guess = fireEvent('quality.guess', files = [nzb.get('name')], size = nzb.get('size', None), single = True) if guess: found[guess['identifier']] = True # Hack for older movies that don't contain quality tag + name = nzb['name'] + size = nzb.get('size', 0) + year_name = fireEvent('scanner.name_year', name, single = True) if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): - if size > 3000: # Assume dvdr + if size > 20000: # Assume bd50 + log.info('Quality was missing in name, assuming it\'s a BR-Disk based on the size: %s', size) + found['bd50'] = True + elif size > 3000: # Assume dvdr log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', size) found['dvdr'] = True else: # Assume dvdrip @@ -123,7 +115,10 @@ class Searcher(SearcherBase): if found.get(allowed): del found[allowed] - return not (found.get(preferred_quality['identifier']) and len(found) == 1) + if found.get(preferred_quality['identifier']) and len(found) == 1: + return False + + return found def correct3D(self, nzb, preferred_quality = None): if not preferred_quality: preferred_quality = {} diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index a783cd1e..860ec965 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -139,7 +139,7 @@ class MovieBase(MovieTypeBase): # Clean snatched history for release in fireEvent('release.for_media', m['_id'], single = True): - if release.get('status') in ['downloaded', 'snatched', 'done']: + if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if params.get('ignore_previous', False): release['status'] = 'ignored' db.update(release) diff --git a/couchpotato/core/media/movie/_base/static/list.js b/couchpotato/core/media/movie/_base/static/list.js index 9310ce1a..323a22ae 100644 --- a/couchpotato/core/media/movie/_base/static/list.js +++ b/couchpotato/core/media/movie/_base/static/list.js @@ -373,6 +373,7 @@ var MovieList = new Class({ (e).preventDefault(); this.set('text', 'Deleting..'); Api.request('media.delete', { + 'method': 'post', 'data': { 'id': ids.join(','), 'delete_from': self.options.identifier @@ -413,6 +414,7 @@ var MovieList = new Class({ var ids = self.getSelectedMovies(); Api.request('movie.edit', { + 'method': 'post', 'data': { 'id': ids.join(','), 'profile_id': self.mass_edit_quality.get('value') @@ -426,6 +428,7 @@ var MovieList = new Class({ var ids = self.getSelectedMovies(); Api.request('media.refresh', { + 'method': 'post', 'data': { 'id': ids.join(',') } diff --git a/couchpotato/core/media/movie/_base/static/2_manage.js b/couchpotato/core/media/movie/_base/static/manage.js similarity index 99% rename from couchpotato/core/media/movie/_base/static/2_manage.js rename to couchpotato/core/media/movie/_base/static/manage.js index 9383c72f..295983cb 100644 --- a/couchpotato/core/media/movie/_base/static/2_manage.js +++ b/couchpotato/core/media/movie/_base/static/manage.js @@ -2,6 +2,7 @@ Page.Movies.Manage = new Class({ Extends: PageBase, + order: 20, name: 'manage', title: 'Do stuff to your existing movies!', diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index 4d9dcccb..26ee2421 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -78,7 +78,7 @@ MA.IMDB = new Class({ create: function(){ var self = this; - self.id = self.movie.get('imdb') || self.movie.get('identifier'); + self.id = self.movie.getIdentifier ? self.movie.getIdentifier() : self.get('imdb'); self.el = new Element('a.imdb', { 'title': 'Go to the IMDB page of ' + self.getTitle(), @@ -684,7 +684,7 @@ MA.Readd = new Class({ var movie_done = self.movie.data.status == 'done'; if(self.movie.data.releases && !movie_done) var snatched = self.movie.data.releases.filter(function(release){ - return release.status && (release.status == 'snatched' || release.status == 'downloaded' || release.status == 'done'); + return release.status && (release.status == 'snatched' || release.status == 'seeding' || release.status == 'downloaded' || release.status == 'done'); }).length; if(movie_done || snatched && snatched > 0) @@ -703,7 +703,7 @@ MA.Readd = new Class({ Api.request('movie.add', { 'data': { - 'identifier': self.movie.get('identifier'), + 'identifier': self.movie.getIdentifier(), 'ignore_previous': 1 } }); diff --git a/couchpotato/core/media/movie/_base/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css index 6718dc3a..05fcddf4 100644 --- a/couchpotato/core/media/movie/_base/static/movie.css +++ b/couchpotato/core/media/movie/_base/static/movie.css @@ -123,6 +123,7 @@ .movies.thumbs_list .movie { width: 16.66667%; height: auto; + min-height: 200px; display: inline-block; margin: 0; padding: 0; @@ -133,6 +134,7 @@ @media all and (max-width: 800px) { .movies.thumbs_list .movie { width: 25%; + min-height: 100px; } } diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index 25d5e08f..1a547f2f 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -250,6 +250,10 @@ var Movie = new Class({ getUnprefixedTitle: function(t){ if(t.substr(0, 4).toLowerCase() == 'the ') t = t.substr(4) + ', The'; + else if(t.substr(0, 3).toLowerCase() == 'an ') + t = t.substr(3) + ', An'; + else if(t.substr(0, 2).toLowerCase() == 'a ') + t = t.substr(2) + ', A'; return t; }, @@ -296,6 +300,17 @@ var Movie = new Class({ self.el.removeClass(self.view+'_view') }, + getIdentifier: function(){ + var self = this; + + try { + return self.get('identifiers').imdb; + } + catch (e){ } + + return self.get('imdb'); + }, + get: function(attr){ return this.data[attr] || this.data.info[attr] }, diff --git a/couchpotato/core/media/movie/_base/static/1_wanted.js b/couchpotato/core/media/movie/_base/static/wanted.js similarity index 99% rename from couchpotato/core/media/movie/_base/static/1_wanted.js rename to couchpotato/core/media/movie/_base/static/wanted.js index eee156db..0f74cd13 100644 --- a/couchpotato/core/media/movie/_base/static/1_wanted.js +++ b/couchpotato/core/media/movie/_base/static/wanted.js @@ -2,6 +2,7 @@ Page.Movies.Wanted = new Class({ Extends: PageBase, + order: 10, name: 'wanted', title: 'Gimmy gimmy gimmy!', folder_browser: null, diff --git a/couchpotato/core/media/movie/charts/__init__.py b/couchpotato/core/media/movie/charts/__init__.py index 9c673751..361da51a 100644 --- a/couchpotato/core/media/movie/charts/__init__.py +++ b/couchpotato/core/media/movie/charts/__init__.py @@ -28,6 +28,20 @@ config = [{ 'advanced': True, 'description': '(hours)', }, + { + 'name': 'hide_wanted', + 'default': False, + 'type': 'bool', + 'advanced': True, + 'description': 'Hide the chart movies that are already in your wanted list.', + }, + { + 'name': 'hide_library', + 'default': False, + 'type': 'bool', + 'advanced': True, + 'description': 'Hide the chart movies that are already in your library.', + }, ], }, ], diff --git a/couchpotato/core/media/movie/charts/main.py b/couchpotato/core/media/movie/charts/main.py index 71002752..58133d83 100644 --- a/couchpotato/core/media/movie/charts/main.py +++ b/couchpotato/core/media/movie/charts/main.py @@ -36,7 +36,6 @@ class Charts(Plugin): 'charts': charts } - def updateViewCache(self): if self.update_in_progress: @@ -49,6 +48,9 @@ class Charts(Plugin): try: self.update_in_progress = True charts = fireEvent('automation.get_chart_list', merge = True) + for chart in charts: + chart['hide_wanted'] = self.conf('hide_wanted') + chart['hide_library'] = self.conf('hide_library') self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12))) except: log.error('Failed refreshing charts') diff --git a/couchpotato/core/media/movie/charts/static/charts.css b/couchpotato/core/media/movie/charts/static/charts.css index 610ac153..ce8ff0ab 100644 --- a/couchpotato/core/media/movie/charts/static/charts.css +++ b/couchpotato/core/media/movie/charts/static/charts.css @@ -11,6 +11,17 @@ display: inline-block; width: 50%; vertical-align: top; + max-height: 510px; + overflow: hidden; + scrollbar-base-color: #4e5969; +} + +.charts .chart:hover { + overflow-y: auto; +} + +.charts .chart .media_result.hidden { + display: none; } .charts .refresh { diff --git a/couchpotato/core/media/movie/charts/static/charts.js b/couchpotato/core/media/movie/charts/static/charts.js index 079eabf6..559046f0 100644 --- a/couchpotato/core/media/movie/charts/static/charts.js +++ b/couchpotato/core/media/movie/charts/static/charts.js @@ -89,7 +89,7 @@ var Charts = new Class({ } }); - var in_database_class = movie.in_wanted ? 'chart_in_wanted' : (movie.in_library ? 'chart_in_library' : ''), + var in_database_class = (chart.hide_wanted && movie.in_wanted) ? 'hidden' : (movie.in_wanted ? 'chart_in_wanted' : ((chart.hide_library && movie.in_library) ? 'hidden': (movie.in_library ? 'chart_in_library' : ''))), in_database_title = movie.in_wanted ? 'Movie in wanted list' : (movie.in_library ? 'Movie in library' : ''); m.el diff --git a/couchpotato/core/media/movie/providers/automation/bluray.py b/couchpotato/core/media/movie/providers/automation/bluray.py index 55114fd3..0501c601 100644 --- a/couchpotato/core/media/movie/providers/automation/bluray.py +++ b/couchpotato/core/media/movie/providers/automation/bluray.py @@ -1,4 +1,5 @@ from bs4 import BeautifulSoup +from couchpotato import fireEvent from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog @@ -82,6 +83,7 @@ class Bluray(Automation, RSS): def getChartList(self): # Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id) movie_list = {'name': 'Blu-ray.com - New Releases', 'url': self.display_url, 'order': self.chart_order, 'list': []} + movie_ids = [] max_items = int(self.conf('max_items', section='charts', default=5)) rss_movies = self.getRSSData(self.rss_url) @@ -95,6 +97,15 @@ class Bluray(Automation, RSS): movie = self.search(name, year) if movie: + + if movie.get('imdb') in movie_ids: + continue + + is_movie = fireEvent('movie.is_movie', identifier = movie.get('imdb'), single = True) + if not is_movie: + continue + + movie_ids.append(movie.get('imdb')) movie_list['list'].append( movie ) if len(movie_list['list']) >= max_items: break diff --git a/couchpotato/core/media/movie/providers/automation/imdb.py b/couchpotato/core/media/movie/providers/automation/imdb.py index 0f2e2940..b52816a8 100644 --- a/couchpotato/core/media/movie/providers/automation/imdb.py +++ b/couchpotato/core/media/movie/providers/automation/imdb.py @@ -100,20 +100,28 @@ class IMDBAutomation(IMDBBase): enabled_option = 'automation_providers_enabled' - chart_urls = { - 'theater': 'http://www.imdb.com/movies-in-theaters/', - 'top250': 'http://www.imdb.com/chart/top', - 'boxoffice': 'http://www.imdb.com/chart/', - } - chart_names = { - 'theater': 'IMDB - Movies in Theaters', - 'top250': 'IMDB - Top 250 Movies', - 'boxoffice': 'IMDB - Box Office', - } - chart_order = { - 'theater': 2, - 'top250': 4, - 'boxoffice': 3, + charts = { + 'theater': { + 'order': 1, + 'name': 'IMDB - Movies in Theaters', + 'url': 'http://www.imdb.com/movies-in-theaters/', + }, + 'boxoffice': { + 'order': 2, + 'name': 'IMDB - Box Office', + 'url': 'http://www.imdb.com/chart/', + }, + 'rentals': { + 'order': 3, + 'name': 'IMDB - Top DVD rentals', + 'url': 'http://m.imdb.com/boxoffice_json', + 'type': 'json', + }, + 'top250': { + 'order': 4, + 'name': 'IMDB - Top 250 Movies', + 'url': 'http://www.imdb.com/chart/top', + }, } first_table = ['boxoffice'] @@ -122,23 +130,30 @@ class IMDBAutomation(IMDBBase): movies = [] - for url in self.chart_urls: - if self.conf('automation_charts_%s' % url): - data = self.getHTMLData(self.chart_urls[url]) + for name in self.charts: + chart = self.charts[name] + url = chart.get('url') + + if self.conf('automation_charts_%s' % name): + data = self.getHTMLData(url) + if data: - html = BeautifulSoup(data) - try: - result_div = html.find('div', attrs = {'id': 'main'}) + html = BeautifulSoup(data) - try: - if url in self.first_table: - table = result_div.find('table') - result_div = table if table else result_div - except: - pass + if chart.get('type', 'html') == 'html': + result_div = html.find('div', attrs = {'id': 'main'}) - imdb_ids = getImdb(str(result_div), multiple = True) + try: + if url in self.first_table: + table = result_div.find('table') + result_div = table if table else result_div + except: + pass + + imdb_ids = getImdb(str(result_div), multiple = True) + else: + imdb_ids = getImdb(str(data), multiple = True) for imdb_id in imdb_ids: info = self.getInfo(imdb_id) @@ -153,42 +168,56 @@ class IMDBAutomation(IMDBBase): return movies - def getChartList(self): + # Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id) movie_lists = [] - max_items = int(self.conf('max_items', section='charts', default=5)) + max_items = int(self.conf('max_items', section = 'charts', default=5)) - for url in self.chart_urls: - if self.conf('chart_display_%s' % url): - movie_list = {'name': self.chart_names[url], 'url': self.chart_urls[url], 'order': self.chart_order[url], 'list': []} - data = self.getHTMLData(self.chart_urls[url]) + for name in self.charts: + chart = self.charts[name].copy() + url = chart.get('url') + + if self.conf('chart_display_%s' % name): + + chart['list'] = [] + + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) try: - result_div = html.find('div', attrs = {'id': 'main'}) - try: - if url in self.first_table: - table = result_div.find('table') - result_div = table if table else result_div - except: - pass + if chart.get('type', 'html') == 'html': + result_div = html.find('div', attrs = {'id': 'main'}) - imdb_ids = getImdb(str(result_div), multiple = True) + try: + if url in self.first_table: + table = result_div.find('table') + result_div = table if table else result_div + except: + pass + + imdb_ids = getImdb(str(result_div), multiple = True) + else: + imdb_ids = getImdb(str(data), multiple = True) for imdb_id in imdb_ids[0:max_items]: + + is_movie = fireEvent('movie.is_movie', identifier = imdb_id, single = True) + if not is_movie: + continue + info = self.getInfo(imdb_id) - movie_list['list'].append(info) + chart['list'].append(info) if self.shuttingDown(): break except: log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc())) - if movie_list['list']: - movie_lists.append(movie_list) + if chart['list']: + movie_lists.append(chart) return movie_lists @@ -240,12 +269,19 @@ config = [{ 'description': 'New Movies In-Theaters chart', 'default': True, }, + { + 'name': 'automation_charts_rentals', + 'type': 'bool', + 'label': 'DVD Rentals', + 'description': 'Top DVD rentals chart', + 'default': True, + }, { 'name': 'automation_charts_top250', 'type': 'bool', 'label': 'TOP 250', 'description': 'IMDB TOP 250 chart', - 'default': True, + 'default': False, }, { 'name': 'automation_charts_boxoffice', @@ -282,6 +318,13 @@ config = [{ 'description': 'IMDB TOP 250 chart', 'default': False, }, + { + 'name': 'chart_display_rentals', + 'type': 'bool', + 'label': 'DVD Rentals', + 'description': 'Top DVD rentals chart', + 'default': True, + }, { 'name': 'chart_display_boxoffice', 'type': 'bool', diff --git a/couchpotato/core/media/movie/providers/nzb/newznab.py b/couchpotato/core/media/movie/providers/nzb/newznab.py index 9783f8d5..fc94acbf 100644 --- a/couchpotato/core/media/movie/providers/nzb/newznab.py +++ b/couchpotato/core/media/movie/providers/nzb/newznab.py @@ -11,11 +11,16 @@ autoload = 'Newznab' class Newznab(MovieProvider, Base): - def buildUrl(self, media, api_key): + def buildUrl(self, media, host): + query = tryUrlencode({ 't': 'movie', 'imdbid': getIdentifier(media).replace('tt', ''), - 'apikey': api_key, + 'apikey': host['api_key'], 'extended': 1 }) + + if len(host.get('custom_tag', '')) > 0: + query = '%s&%s' % (query, host.get('custom_tag')) + return query diff --git a/couchpotato/core/media/movie/providers/torrent/bithdtv.py b/couchpotato/core/media/movie/providers/torrent/bithdtv.py index e7b16af9..da6954c8 100644 --- a/couchpotato/core/media/movie/providers/torrent/bithdtv.py +++ b/couchpotato/core/media/movie/providers/torrent/bithdtv.py @@ -10,10 +10,14 @@ autoload = 'BiTHDTV' class BiTHDTV(MovieProvider, Base): + cat_ids = [ + ([2], ['bd50']), + ] + cat_backup_id = 7 # Movies - def buildUrl(self, media): + def buildUrl(self, media, quality): query = tryUrlencode({ 'search': fireEvent('library.query', media, single = True), - 'cat': 7 # Movie cat + 'cat': self.getCatId(quality)[0] }) return query diff --git a/couchpotato/core/media/movie/providers/torrent/iptorrents.py b/couchpotato/core/media/movie/providers/torrent/iptorrents.py index f06aadfc..89aeee80 100644 --- a/couchpotato/core/media/movie/providers/torrent/iptorrents.py +++ b/couchpotato/core/media/movie/providers/torrent/iptorrents.py @@ -18,6 +18,6 @@ class IPTorrents(MovieProvider, Base): ] def buildUrl(self, title, media, quality): - query = '%s %s' % (title.replace(':', ''), media['info']['year']) + query = '"%s" %s' % (title.replace(':', ''), media['info']['year']) return self._buildUrl(query, quality) diff --git a/couchpotato/core/media/movie/providers/torrent/publichd.py b/couchpotato/core/media/movie/providers/torrent/publichd.py index ce54c468..9fed4619 100644 --- a/couchpotato/core/media/movie/providers/torrent/publichd.py +++ b/couchpotato/core/media/movie/providers/torrent/publichd.py @@ -11,4 +11,4 @@ autoload = 'PublicHD' class PublicHD(MovieProvider, Base): def buildUrl(self, media): - return fireEvent('library.query', media, single = True) + return fireEvent('library.query', media, single = True).replace(':', '') diff --git a/couchpotato/core/media/movie/providers/torrent/sceneaccess.py b/couchpotato/core/media/movie/providers/torrent/sceneaccess.py index dee4b32c..99157d73 100644 --- a/couchpotato/core/media/movie/providers/torrent/sceneaccess.py +++ b/couchpotato/core/media/movie/providers/torrent/sceneaccess.py @@ -17,13 +17,13 @@ class SceneAccess(MovieProvider, Base): ([8], ['dvdr']), ] - def buildUrl(self, media, quality): + def buildUrl(self, title, media, quality): cat_id = self.getCatId(quality)[0] url = self.urls['search'] % (cat_id, cat_id) arguments = tryUrlencode({ - 'search': fireEvent('library.query', media, single = True), - 'method': 3, + 'search': '%s %s' % (title, media['info']['year']), + 'method': 2, }) query = "%s&%s" % (url, arguments) diff --git a/couchpotato/core/media/movie/providers/torrent/thepiratebay.py b/couchpotato/core/media/movie/providers/torrent/thepiratebay.py index f804d709..0dc8313d 100644 --- a/couchpotato/core/media/movie/providers/torrent/thepiratebay.py +++ b/couchpotato/core/media/movie/providers/torrent/thepiratebay.py @@ -13,7 +13,7 @@ class ThePirateBay(MovieProvider, Base): cat_ids = [ ([209], ['3d']), - ([207], ['720p', '1080p']), + ([207], ['720p', '1080p', 'bd50']), ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), ([201, 207], ['brrip']), ([202], ['dvdr']) diff --git a/couchpotato/core/media/movie/providers/torrent/torrentday.py b/couchpotato/core/media/movie/providers/torrent/torrentday.py index 872a7d17..768d3043 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentday.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentday.py @@ -1,5 +1,4 @@ from couchpotato.core.logger import CPLog -from couchpotato.core.event import fireEvent from couchpotato.core.media._base.providers.torrent.torrentday import Base from couchpotato.core.media.movie.providers.base import MovieProvider @@ -16,6 +15,3 @@ class TorrentDay(MovieProvider, Base): ([3], ['dvdr']), ([5], ['bd50']), ] - - def buildUrl(self, media): - return fireEvent('library.query', media, single = True) diff --git a/couchpotato/core/media/movie/providers/torrent/torrentshack.py b/couchpotato/core/media/movie/providers/torrent/torrentshack.py index 85e420dd..01eb6d6a 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentshack.py @@ -31,6 +31,6 @@ class TorrentShack(MovieProvider, Base): def buildUrl(self, media, quality): query = (tryUrlencode(fireEvent('library.query', media, single = True)), - self.getCatId(quality)[0], - self.getSceneOnly()) + self.getSceneOnly(), + self.getCatId(quality)[0]) return query diff --git a/couchpotato/core/media/movie/providers/userscript/filmstarts.py b/couchpotato/core/media/movie/providers/userscript/filmstarts.py new file mode 100644 index 00000000..59027e03 --- /dev/null +++ b/couchpotato/core/media/movie/providers/userscript/filmstarts.py @@ -0,0 +1,30 @@ +from bs4 import BeautifulSoup +from couchpotato.core.media._base.providers.userscript.base import UserscriptBase + +autoload = 'Filmstarts' + + +class Filmstarts(UserscriptBase): + + includes = ['*://www.filmstarts.de/kritiken/*'] + + def getMovie(self, url): + try: + data = self.getUrl(url) + except: + return + + html = BeautifulSoup(data) + table = html.find("table", attrs={"class": "table table-standard thead-standard table-striped_2 fs11"}) + + if table.find(text='Originaltitel'): + # Get original film title from the table specified above + name = table.find("div", text="Originaltitel").parent.parent.parent.td.text + else: + # If none is available get the title from the meta data + name = html.find("meta", {"property":"og:title"})['content'] + + # Year of production is not available in the meta data, so get it from the table + year = table.find("tr", text="Produktionsjahr").parent.parent.parent.td.text + + return self.search(name, year) \ No newline at end of file diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 68a76ade..a5b1f477 100644 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -126,6 +126,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): release_dates = fireEvent('movie.update_release_dates', movie['_id'], merge = True) found_releases = [] + previous_releases = movie.get('releases', []) too_early_to_search = [] default_title = getTitle(movie) @@ -139,16 +140,15 @@ class MovieSearcher(SearcherBase, MovieTypeBase): db = get_db() profile = db.get('id', movie['profile_id']) - quality_order = fireEvent('quality.order', single = True) - ret = False index = 0 for q_identifier in profile.get('qualities'): quality_custom = { + 'index': index, 'quality': q_identifier, 'finish': profile['finish'][index], - 'wait_for': profile['wait_for'][index], + 'wait_for': tryInt(profile['wait_for'][index]), '3d': profile['3d'][index] if profile.get('3d') else False } @@ -162,43 +162,51 @@ class MovieSearcher(SearcherBase, MovieTypeBase): # See if better quality is available for release in movie.get('releases', []): - if quality_order.index(release['quality']) <= quality_order.index(q_identifier) and release['status'] not in ['available', 'ignored', 'failed']: - has_better_quality += 1 + if release['status'] not in ['available', 'ignored', 'failed']: + is_higher = fireEvent('quality.ishigher', \ + {'identifier': q_identifier, 'is_3d': quality_custom.get('3d', 0)}, \ + {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, \ + profile, single = True) + if is_higher != 'higher': + has_better_quality += 1 # Don't search for quality lower then already available. - if has_better_quality is 0: - - quality = fireEvent('quality.single', identifier = q_identifier, single = True) - log.info('Search for %s in %s', (default_title, quality['label'])) - - # Extend quality with profile customs - quality['custom'] = quality_custom - - results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or [] - if len(results) == 0: - log.debug('Nothing found for %s in %s', (default_title, quality['label'])) - - # Check if movie isn't deleted while searching - if not fireEvent('media.get', movie.get('_id'), single = True): - break - - # Add them to this movie releases list - found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True) - - # Try find a valid result and download it - if fireEvent('release.try_download_result', results, movie, quality_custom, manual, single = True): - ret = True - - # Remove releases that aren't found anymore - for release in movie.get('releases', []): - if release.get('status') == 'available' and release.get('identifier') not in found_releases: - fireEvent('release.delete', release.get('_id'), single = True) - - else: + if has_better_quality > 0: log.info('Better quality (%s) already available or snatched for %s', (q_identifier, default_title)) fireEvent('media.restatus', movie['_id']) break + quality = fireEvent('quality.single', identifier = q_identifier, single = True) + log.info('Search for %s in %s', (default_title, quality['label'])) + + # Extend quality with profile customs + quality['custom'] = quality_custom + + results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or [] + if len(results) == 0: + log.debug('Nothing found for %s in %s', (default_title, quality['label'])) + + # Check if movie isn't deleted while searching + if not fireEvent('media.get', movie.get('_id'), single = True): + break + + # Add them to this movie releases list + found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True) + + # Try find a valid result and download it + if fireEvent('release.try_download_result', results, movie, quality_custom, manual, single = True): + ret = True + + # Remove releases that aren't found anymore + temp_previous_releases = [] + for release in previous_releases: + if release.get('status') == 'available' and release.get('identifier') not in found_releases: + fireEvent('release.delete', release.get('_id'), single = True) + else: + temp_previous_releases.append(release) + previous_releases = temp_previous_releases + del temp_previous_releases + # Break if CP wants to shut down if self.shuttingDown() or ret: break @@ -230,8 +238,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase): preferred_quality = quality if quality else fireEvent('quality.single', identifier = quality['identifier'], single = True) # Contains lower quality string - if fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True): - log.info2('Wrong: %s, looking for %s', (nzb['name'], quality['label'])) + contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True) + if contains_other != False: + log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality')) return False # Contains lower quality string @@ -288,7 +297,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): now_year = date.today().year now_month = date.today().month - if (year is None or year < now_year - 1) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)): + if (year is None or year < now_year - 1 or (year <= now_year - 1 and now_month > 4)) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)): return True else: diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 13eb7caf..7cc4d1c6 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -113,7 +113,7 @@ class Plugin(object): fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f), f) def createFile(self, path, content, binary = False): - path = ss(path) + path = sp(path) self.makeDir(os.path.dirname(path)) @@ -131,7 +131,7 @@ class Plugin(object): os.remove(path) def makeDir(self, path): - path = ss(path) + path = sp(path) try: if not os.path.isdir(path): os.makedirs(path, Env.getPermission('folder')) diff --git a/couchpotato/core/plugins/dashboard.py b/couchpotato/core/plugins/dashboard.py index 7a26b2a0..776f24ec 100644 --- a/couchpotato/core/plugins/dashboard.py +++ b/couchpotato/core/plugins/dashboard.py @@ -79,7 +79,20 @@ class Dashboard(Plugin): # Don't list older movies if ((not late and (media['info']['year'] >= now_year - 1) and (not eta.get('dvd') and not eta.get('theater') or eta.get('dvd') and eta.get('dvd') > (now - 2419200))) or (late and (media['info']['year'] < now_year - 1 or (eta.get('dvd', 0) > 0 or eta.get('theater')) and eta.get('dvd') < (now - 2419200)))): - medias.append(media) + + add = True + + # Check if it doesn't have any releases + if late: + media['releases'] = fireEvent('release.for_media', media['_id'], single = True) + + for release in media.get('releases'): + if release.get('status') in ['snatched', 'available', 'seeding', 'downloaded']: + add = False + break + + if add: + medias.append(media) if len(medias) >= limit: break diff --git a/couchpotato/core/plugins/log/main.py b/couchpotato/core/plugins/log/main.py index 2565398e..5206acd6 100644 --- a/couchpotato/core/plugins/log/main.py +++ b/couchpotato/core/plugins/log/main.py @@ -1,9 +1,10 @@ import os +import re import traceback from couchpotato.api import addApiView from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.helpers.variable import tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -22,7 +23,11 @@ class Logging(Plugin): }, 'return': {'type': 'object', 'example': """{ 'success': True, - 'log': string, //Log file + 'log': [{ + 'time': '03-12 09:12:59', + 'type': 'INFO', + 'message': 'Log message' + }, ..], //Log file 'total': int, //Total log files available }"""} }) @@ -34,7 +39,11 @@ class Logging(Plugin): }, 'return': {'type': 'object', 'example': """{ 'success': True, - 'log': string, //Log file + 'log': [{ + 'time': '03-12 09:12:59', + 'type': 'INFO', + 'message': 'Log message' + }, ..] }"""} }) addApiView('logging.clear', self.clear, docs = { @@ -71,16 +80,18 @@ class Logging(Plugin): if current_path: f = open(current_path, 'r') log_content = f.read() + logs = self.toList(log_content) return { 'success': True, - 'log': toUnicode(log_content), + 'log': logs, 'total': total, } - def partial(self, type = 'all', lines = 30, **kwargs): + def partial(self, type = 'all', lines = 30, offset = 0, **kwargs): total_lines = tryInt(lines) + offset = tryInt(offset) log_lines = [] @@ -93,28 +104,57 @@ class Logging(Plugin): break f = open(path, 'r') - reversed_lines = toUnicode(f.read()).split('[0m\n') - reversed_lines.reverse() + log_content = toUnicode(f.read()) + raw_lines = self.toList(log_content) + raw_lines.reverse() brk = False - for line in reversed_lines: + for line in raw_lines: - if type == 'all' or '%s ' % type.upper() in line: + if type == 'all' or line.get('type') == type.upper(): log_lines.append(line) - if len(log_lines) >= total_lines: + if len(log_lines) >= (total_lines + offset): brk = True break if brk: break + log_lines = log_lines[offset:] log_lines.reverse() + return { 'success': True, - 'log': '[0m\n'.join(log_lines), + 'log': log_lines, } + def toList(self, log_content = ''): + + logs_raw = toUnicode(log_content).split('[0m\n') + + logs = [] + for log in logs_raw: + split = splitString(log, '\x1b') + if split: + try: + date, time, log_type = splitString(split[0], ' ') + timestamp = '%s %s' % (date, time) + except: + timestamp = 'UNKNOWN' + log_type = 'UNKNOWN' + + message = ''.join(split[1]) if len(split) > 1 else split[0] + message = re.sub('\[\d+m\[', '[', message) + + logs.append({ + 'time': timestamp, + 'type': log_type, + 'message': message + }) + + return logs + def clear(self, **kwargs): for x in range(0, 50): diff --git a/couchpotato/core/plugins/log/static/log.css b/couchpotato/core/plugins/log/static/log.css index b706835a..8a809e32 100644 --- a/couchpotato/core/plugins/log/static/log.css +++ b/couchpotato/core/plugins/log/static/log.css @@ -16,10 +16,14 @@ display: inline-block; padding: 5px 10px; margin: 0; + } + + .page.log .nav li.select, + .page.log .nav li.clear { cursor: pointer; } - .page.log .nav li:hover:not(.active) { + .page.log .nav li:hover:not(.active, .filter) { background: rgba(255, 255, 255, 0.1); } @@ -39,6 +43,19 @@ } } + .page.log .nav li.hint { + text-align: center; + width: 400px; + left: 50%; + margin-left: -200px; + font-style: italic; + font-size: 11px; + position: absolute; + right: 20px; + opacity: .5; + bottom: 5px; + } + .page.log .loading { text-align: center; font-size: 20px; @@ -50,28 +67,133 @@ overflow: hidden; line-height: 150%; font-size: 11px; - font-family: Lucida Console, Monaco, Nimbus Mono L, monospace, serif; + color: #FFF; } - .page.log .container .error { - color: #FFA4A4; - white-space: pre-wrap; + .page.log .container select { + vertical-align: top; } - .page.log .container .debug { color: lightgrey; } .page.log .container .time { clear: both; color: lightgrey; - padding: 3px 0; font-size: 10px; - border-top: 1px solid rgba(255, 255, 255, 0.2); + border-top: 1px solid rgba(255, 255, 255, 0.1); position: relative; + overflow: hidden; + padding: 0 3px; + font-family: Lucida Console, Monaco, Nimbus Mono L, monospace, serif; } - - .page.log .container .time:last-child { display: none; } - + .page.log .container .time.highlight { + background: rgba(255, 255, 255, 0.1); + } .page.log .container .time span { - float: right; - width: 86%; + padding: 5px 0 3px; + display: inline-block; + vertical-align: middle; } + .page.log[data-filter=INFO] .error, + .page.log[data-filter=INFO] .debug, + .page.log[data-filter=ERROR] .debug, + .page.log[data-filter=ERROR] .info, + .page.log[data-filter=DEBUG] .info, + .page.log[data-filter=DEBUG] .error { + display: none; + } + + .page.log .container .type { + margin-left: 10px; + } + + .page.log .container .message { + float: right; + width: 86%; + white-space: pre-wrap; + } + + .page.log .container .error { color: #FFA4A4; } + .page.log .container .debug span { opacity: .6; } + +.do_report { + position: absolute; + padding: 10px; +} + +.page.log .report { + position: fixed; + width: 100%; + height: 100%; + background: rgba(0,0,0,.7); + left: 0; + top: 0; + z-index: 99999; + font-size: 14px; +} + + .page.log .report .button { + display: inline-block; + margin: 10px 0; + padding: 10px; + } + + .page.log .report .bug { + width: 800px; + height: 80%; + position: absolute; + left: 50%; + top: 50%; + margin: 0 0 0 -400px; + transform: translate(0, -50%); + } + + .page.log .report .bug textarea { + display: block; + width: 100%; + background: #FFF; + padding: 20px; + overflow: auto; + color: #666; + height: 70%; + font-size: 12px; + } + +.page.log .container .time ::-webkit-selection { + background-color: #000; + color: #FFF; +} + +.page.log .container .time ::-moz-selection { + background-color: #000; + color: #FFF; +} + +.page.log .container .time ::-ms-selection { + background-color: #000; + color: #FFF; +} + +.page.log .container .time.highlight ::selection { + background-color: transparent; + color: inherit; +} + +.page.log .container .time.highlight ::-webkit-selection { + background-color: transparent; + color: inherit; +} + +.page.log .container .time.highlight ::-moz-selection { + background-color: transparent; + color: inherit; +} + +.page.log .container .time.highlight ::-ms-selection { + background-color: transparent; + color: inherit; +} + +.page.log .container .time.highlight ::selection { + background-color: transparent; + color: inherit; +} diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index c5effeba..d39a20a9 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -2,80 +2,281 @@ Page.Log = new Class({ Extends: PageBase, + order: 60, name: 'log', title: 'Show recent logs.', has_tab: false, - indexAction: function(){ + log_items: [], + report_text: '\ +### Steps to reproduce:\n\ +1. ..\n\ +2. ..\n\ +\n\ +### Information:\n\ +Movie(s) I have this with: ...\n\ +Quality of the movie being searched: ...\n\ +Providers I use: ...\n\ +Version of CouchPotato: ...\n\ +Running on: ...\n\ +\n\ +### Logs:\n\ +```\n{issue}```', + + indexAction: function () { var self = this; self.getLogs(0); }, - getLogs: function(nr){ + getLogs: function (nr) { var self = this; - if(self.log) self.log.destroy(); + if (self.log) self.log.destroy(); self.log = new Element('div.container.loading', { - 'text': 'loading...' + 'text': 'loading...', + 'events': { + 'mouseup:relay(.time)': function(e){ + self.showSelectionButton.delay(100, self, e); + } + } }).inject(self.el); Api.request('logging.get', { 'data': { 'nr': nr }, - 'onComplete': function(json){ - self.log.set('html', self.addColors(json.log)); + 'onComplete': function (json) { + self.log.set('text', ''); + self.log_items = self.createLogElements(json.log); + self.log.adopt(self.log_items); self.log.removeClass('loading'); - new Fx.Scroll(window, {'duration': 0}).toBottom(); + var nav = new Element('ul.nav', { + 'events': { + 'click:relay(li.select)': function (e, el) { + self.getLogs(parseInt(el.get('text')) - 1); + } + } + }); - var nav = new Element('ul.nav').inject(self.log, 'top'); - for (var i = 0; i <= json.total; i++) { - new Element('li', { - 'text': i+1, - 'class': nr == i ? 'active': '', + // Type selection + new Element('li.filter').grab( + new Element('select', { 'events': { - 'click': function(e){ - self.getLogs(e.target.get('text')-1); + 'change': function () { + var type_filter = this.getSelected()[0].get('value'); + self.el.set('data-filter', type_filter); + self.scrollToBottom(); } } + }).adopt( + new Element('option', {'value': 'ALL', 'text': 'Show all logs'}), + new Element('option', {'value': 'INFO', 'text': 'Show only INFO'}), + new Element('option', {'value': 'DEBUG', 'text': 'Show only DEBUG'}), + new Element('option', {'value': 'ERROR', 'text': 'Show only ERROR'}) + ) + ).inject(nav); + + // Selections + for (var i = 0; i <= json.total; i++) { + new Element('li', { + 'text': i + 1, + 'class': 'select ' + (nr == i ? 'active' : '') }).inject(nav); } - new Element('li', { + // Clear button + new Element('li.clear', { 'text': 'clear', 'events': { - 'click': function(){ + 'click': function () { Api.request('logging.clear', { - 'onComplete': function(){ + 'onComplete': function () { self.getLogs(0); } }); } } - }).inject(nav) + }).inject(nav); + + // Hint + new Element('li.hint', { + 'text': 'Select multiple lines & report an issue' + }).inject(nav); + + // Add to page + nav.inject(self.log, 'top'); + + self.scrollToBottom(); } }); }, - addColors: function(text){ + createLogElements: function (logs) { - text = text - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/\u001b\[31m/gi, '') - .replace(/\u001b\[36m/gi, '') - .replace(/\u001b\[33m/gi, '') - .replace(/\u001b\[0m\n/gi, '
') - .replace(/\u001b\[0m/gi, ''); + var elements = []; - return '
' + text + '
'; + logs.each(function (log) { + elements.include(new Element('div', { + 'class': 'time ' + log.type.toLowerCase() + }).adopt( + new Element('span', { + 'text': log.time + }), + new Element('span.type', { + 'text': log.type + }), + new Element('span.message', { + 'text': log.message + }) + )) + }); + + return elements; + }, + + scrollToBottom: function () { + new Fx.Scroll(window, {'duration': 0}).toBottom(); + }, + + showSelectionButton: function(e){ + var self = this, + selection = self.getSelected(), + start_node = selection.anchorNode, + parent_start = start_node.parentNode.getParent('.time'), + end_node = selection.focusNode.parentNode.getParent('.time'), + text = ''; + + var remove_button = function(){ + self.log.getElements('.highlight').removeClass('highlight'); + if(self.do_report) + self.do_report.destroy(); + document.body.removeEvent('click', remove_button); + }; + remove_button(); + + if(parent_start) + start_node = parent_start; + + var index = { + 'start': self.log_items.indexOf(start_node), + 'end': self.log_items.indexOf(end_node) + }; + + if(index.start > index.end){ + index = { + 'start': index.end, + 'end': index.start + }; + } + + var nodes = self.log_items.slice(index.start, index.end + 1); + + nodes.each(function(node, nr){ + node.addClass('highlight'); + node.getElements('span').each(function(span){ + text += self.spaceFill(span.get('text') + ' ', 6); + }); + text += '\n'; + }); + + self.do_report = new Element('a.do_report.button', { + 'text': 'Report issue', + 'styles': { + 'top': e.page.y, + 'left': e.page.x + }, + 'events': { + 'click': function(e){ + (e).stop(); + + self.showReport(text); + } + } + }).inject(document.body); + + setTimeout(function(){ + document.body.addEvent('click', remove_button); + }, 0); + + }, + + showReport: function(text){ + var self = this, + body = self.report_text.replace('{issue}', text); + + var overlay = new Element('div.report', { + 'method': 'post', + 'events': { + 'click': function(e){ + overlay.destroy(); + } + } + }).grab( + new Element('div.bug', { + 'events': { + 'click': function(e){ + (e).stopPropagation(); + } + } + }).adopt( + new Element('h1', { + 'text': 'Report a bug' + }), + new Element('span').adopt( + new Element('span', { + 'text': 'Read ' + }), + new Element('a.button', { + 'target': '_blank', + 'text': 'the contributing guide', + 'href': 'https://github.com/RuudBurger/CouchPotatoServer/blob/develop/contributing.md' + }), + new Element('span', { + 'text': ' before posting, then copy the text below' + }) + ), + new Element('textarea', { + 'text': body, + 'events': { + 'click': function(){ + this.select(); + } + } + }), + new Element('a.button', { + 'target': '_blank', + 'text': 'Create a new issue on GitHub with the text above', + 'href': 'https://github.com/RuudBurger/CouchPotatoServer/issues/new?body=' + (body.length < 2000 ? encodeURIComponent(body) : 'Paste the text here') + }) + ) + ); + + overlay.inject(self.log); + }, + + getSelected: function(){ + if (window.getSelection) + return window.getSelection(); + else if (document.getSelection) + return document.getSelection(); + else { + var selection = document.selection && document.selection.createRange(); + if (selection.text) + return selection.text; + } + return false; + + }, + + spaceFill: function( number, width ){ + if ( number.toString().length >= width ) + return number; + return ( new Array( width ).join( ' ' ) + number.toString() ).substr( -width ); } }); diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index f6b0ddd7..ab29e72d 100644 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -120,7 +120,7 @@ class Manage(Plugin): if self.conf('cleanup') and full and not self.shuttingDown(): # Get movies with done status - total_movies, done_movies = fireEvent('media.list', types = 'movie', status = 'done', single = True) + total_movies, done_movies = fireEvent('media.list', types = 'movie', status = 'done', release_status = 'done', status_or = True, single = True) for done_movie in done_movies: if getIdentifier(done_movie) not in added_identifiers: @@ -131,12 +131,16 @@ class Manage(Plugin): for release in releases: if release.get('files'): + brk = False for file_type in release.get('files', {}): for release_file in release['files'][file_type]: # Remove release not available anymore if not os.path.isfile(sp(release_file)): fireEvent('release.clean', release['_id']) + brk = True break + if brk: + break # Check if there are duplicate releases (different quality) use the last one, delete the rest if len(releases) > 1: @@ -147,10 +151,12 @@ class Manage(Plugin): already_used = used_files.get(release_file) if already_used: + # delete current one if already_used.get('last_edit', 0) < release.get('last_edit', 0): - fireEvent('release.delete', release['_id'], single = True) # delete current one + fireEvent('release.delete', release['_id'], single = True) + # delete previous one else: - fireEvent('release.delete', already_used['_id'], single = True) # delete previous one + fireEvent('release.delete', already_used['_id'], single = True) break else: used_files[release_file] = release diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index a7fdcd41..8dc59336 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -34,7 +34,7 @@ class ProfilePlugin(Plugin): }) addEvent('app.initialize', self.fill, priority = 90) - addEvent('app.load', self.forceDefaults) + addEvent('app.load', self.forceDefaults, priority = 110) def forceDefaults(self): @@ -87,7 +87,7 @@ class ProfilePlugin(Plugin): order = 0 for type in kwargs.get('types', []): profile['qualities'].append(type.get('quality')) - profile['wait_for'].append(tryInt(type.get('wait_for'))) + profile['wait_for'].append(tryInt(kwargs.get('wait_for', 0))) profile['finish'].append((tryInt(type.get('finish')) == 1) if order > 0 else True) profile['3d'].append(tryInt(type.get('3d'))) order += 1 diff --git a/couchpotato/core/plugins/profile/static/profile.js b/couchpotato/core/plugins/profile/static/profile.js index 8441b6a8..c62b137c 100644 --- a/couchpotato/core/plugins/profile/static/profile.js +++ b/couchpotato/core/plugins/profile/static/profile.js @@ -41,7 +41,7 @@ var Profile = new Class({ new Element('span', {'text':'Wait'}), new Element('input.inlay.xsmall', { 'type':'text', - 'value': data.types && data.types.length > 0 ? data.types[0].wait_for : 0 + 'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0 }), new Element('span', {'text':'day(s) for a better quality.'}) ), @@ -63,8 +63,7 @@ var Profile = new Class({ data.types.include({ 'quality': quality, 'finish': data.finish[nr] || false, - '3d': data['3d'] ? data['3d'][nr] || false : false, - 'wait_for': data.wait_for[nr] || 0 + '3d': data['3d'] ? data['3d'][nr] || false : false }) }); } @@ -126,8 +125,7 @@ var Profile = new Class({ data.types.include({ 'quality': type.getElement('select').get('value'), 'finish': +type.getElement('input.finish[type=checkbox]').checked, - '3d': +type.getElement('input.3d[type=checkbox]').checked, - 'wait_for': 0 + '3d': +type.getElement('input.3d[type=checkbox]').checked }); }); @@ -340,8 +338,7 @@ Profile.Type = new Class({ return { 'quality': self.qualities.get('value'), 'finish': +self.finish.checked, - '3d': +self['3d'].checked, - 'wait_for': 0 + '3d': +self['3d'].checked } }, diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index c27815ed..e52e3d57 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -21,11 +21,11 @@ class QualityPlugin(Plugin): } qualities = [ - {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, + {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, - {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r')]}, + {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, @@ -35,9 +35,9 @@ class QualityPlugin(Plugin): ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] threed_tags = { - 'hsbs': [('half', 'sbs')], - 'fsbs': [('full', 'sbs')], - '3d': [], + 'sbs': [('half', 'sbs'), 'hsbs', ('full', 'sbs'), 'fsbs'], + 'ou': [('half', 'ou'), 'hou', ('full', 'ou'), 'fou'], + '3d': ['2d3d', '3d2d'], } cached_qualities = None @@ -49,6 +49,8 @@ class QualityPlugin(Plugin): addEvent('quality.guess', self.guess) addEvent('quality.pre_releases', self.preReleases) addEvent('quality.order', self.getOrder) + addEvent('quality.ishigher', self.isHigher) + addEvent('quality.isfinish', self.isFinish) addApiView('quality.size.save', self.saveSize) addApiView('quality.list', self.allView, docs = { @@ -177,7 +179,7 @@ class QualityPlugin(Plugin): return False - def guess(self, files, extra = None): + def guess(self, files, extra = None, size = None): if not extra: extra = {} # Create hash for cache @@ -205,15 +207,20 @@ class QualityPlugin(Plugin): self.calcScore(score, quality, contains_score, threedscore) + # Evaluate score based on size + for quality in qualities: + size_score = self.guessSizeScore(quality, size = size) + self.calcScore(score, quality, size_score, penalty = False) + # Try again with loose testing for quality in qualities: loose_score = self.guessLooseScore(quality, extra = extra) - self.calcScore(score, quality, loose_score) + self.calcScore(score, quality, loose_score, penalty = False) - # Return nothing if all scores are 0 + # Return nothing if all scores are <= 0 has_non_zero = 0 for s in score: - if score[s] > 0: + if score[s]['score'] > 0: has_non_zero += 1 if not has_non_zero: @@ -281,7 +288,7 @@ class QualityPlugin(Plugin): return 1, key if list(set([key]) & set(words)): - log.debug('Found %s in %s', (tag, cur_file)) + log.debug('Found %s in %s', (key, cur_file)) return 1, key return 0, None @@ -308,7 +315,20 @@ class QualityPlugin(Plugin): return score - def calcScore(self, score, quality, add_score, threedscore = (0, None)): + + def guessSizeScore(self, quality, size = None): + + score = 0 + + if size: + + if tryInt(quality['size_min']) <= tryInt(size) <= tryInt(quality['size_max']): + log.debug('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], quality['size_min'], size, quality['size_max'])) + score += 5 + + return score + + def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = True): score[quality['identifier']]['score'] += add_score @@ -325,30 +345,79 @@ class QualityPlugin(Plugin): for q in self.qualities: self.cached_order[q.get('identifier')] = self.qualities.index(q) - if add_score != 0: + if penalty and add_score != 0: for allow in quality.get('allow', []): score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 + # Give panelty for all lower qualities + for q in self.qualities[self.order.index(quality.get('identifier'))+1:]: + score[q.get('identifier')]['score'] -= 1 + + def isFinish(self, quality, profile): + if not isinstance(profile, dict) or not profile.get('qualities'): + return False + + try: + quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0] + return profile['finish'][quality_order] + except: + return False + + def isHigher(self, quality, compare_with, profile = None): + if not isinstance(profile, dict) or not profile.get('qualities'): + profile = {'qualities': self.order} + + # Try to find quality in profile, if not found: a quality we do not want is lower than anything else + try: + quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0] + except: + log.debug('Quality %s not found in profile identifiers %s', (quality['identifier'] + (' 3D' if quality.get('is_3d', 0) else ''), \ + [identifier + ('3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])])) + return 'lower' + + # Try to find compare quality in profile, if not found: anything is higher than a not wanted quality + try: + compare_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == compare_with['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(compare_with.get('is_3d', 0))][0] + except: + log.debug('Compare quality %s not found in profile identifiers %s', (compare_with['identifier'] + (' 3D' if compare_with.get('is_3d', 0) else ''), \ + [identifier + (' 3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])])) + return 'higher' + + # Note to self: a lower number means higher quality + if quality_order > compare_order: + return 'lower' + elif quality_order == compare_order: + return 'equal' + else: + return 'higher' + def doTest(self): tests = { - 'Movie Name (1999)-DVD-Rip.avi': 'dvdrip', - 'Movie Name 1999 720p Bluray.mkv': '720p', - 'Movie Name 1999 BR-Rip 720p.avi': 'brrip', - 'Movie Name 1999 720p Web Rip.avi': 'scr', - 'Movie Name 1999 Web DL.avi': 'brrip', - 'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr', - 'Movie.Name.1999.DVDRip-Group': 'dvdrip', - 'Movie.Name.1999.DVD-Rip-Group': 'dvdrip', - 'Movie.Name.1999.DVD-R-Group': 'dvdr', - 'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': '720p', - 'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': '1080p', - 'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': '1080p', + 'Movie Name (1999)-DVD-Rip.avi': {'size': 700, 'quality': 'dvdrip'}, + 'Movie Name 1999 720p Bluray.mkv': {'size': 4200, 'quality': '720p'}, + 'Movie Name 1999 BR-Rip 720p.avi': {'size': 1000, 'quality': 'brrip'}, + 'Movie Name 1999 720p Web Rip.avi': {'size': 1200, 'quality': 'scr'}, + 'Movie Name 1999 Web DL.avi': {'size': 800, 'quality': 'brrip'}, + 'Movie.Name.1999.1080p.WEBRip.H264-Group': {'size': 1500, 'quality': 'scr'}, + 'Movie.Name.1999.DVDRip-Group': {'size': 750, 'quality': 'dvdrip'}, + 'Movie.Name.1999.DVD-Rip-Group': {'size': 700, 'quality': 'dvdrip'}, + 'Movie.Name.1999.DVD-R-Group': {'size': 4500, 'quality': 'dvdr'}, + 'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': {'size': 5500, 'quality': '720p'}, + 'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': {'size': 8500, 'extra': {'resolution_width': 1920, 'resolution_height': 1080} , 'quality': '1080p'}, + 'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': {'size': 8000, 'quality': '1080p'}, + 'Movie.Name.2013.BR-Disk-Group.iso': {'size': 48000, 'quality': 'bd50'}, + 'Movie.Name.2013.2D+3D.BR-Disk-Group.iso': {'size': 52000, 'quality': 'bd50', 'is_3d': True}, + 'Movie.Rising.Name.Girl.2011.NTSC.DVD9-GroupDVD': {'size': 7200, 'quality': 'dvdr'}, + 'Movie Name (2013) 2D + 3D': {'size': 49000, 'quality': 'bd50', 'is_3d': True}, + 'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'}, + 'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'}, } correct = 0 for name in tests: - success = self.guess([name]).get('identifier') == tests[name] + test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None)) or {} + success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False) if not success: log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier'))) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 72a0a502..8af26fd2 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -3,10 +3,11 @@ import os import time import traceback +from CodernityDB.database import RecordDeleted from couchpotato import md5, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent -from couchpotato.core.helpers.encoding import ss, toUnicode +from couchpotato.core.helpers.encoding import ss, toUnicode, sp from couchpotato.core.helpers.variable import getTitle from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -58,7 +59,7 @@ class Release(Plugin): # Clean releases that didn't have activity in the last week addEvent('app.load', self.cleanDone) - fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 4) + fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 12) def cleanDone(self): log.debug('Removing releases from dashboard') @@ -68,6 +69,27 @@ class Release(Plugin): db = get_db() + # Get (and remove) parentless releases + releases = db.all('release', with_doc = True) + media_exist = [] + for release in releases: + if release.get('key') in media_exist: + continue + + try: + db.get('id', release.get('key')) + media_exist.append(release.get('key')) + except RecordDeleted: + db.delete(release['doc']) + log.debug('Deleted orphaned release: %s', release['doc']) + except: + log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc()) + + del media_exist + + # Reindex statuses + db.reindex_index('media_status') + # get movies last_edit more than a week ago medias = fireEvent('media.with_status', 'done', single = True) @@ -107,6 +129,7 @@ class Release(Plugin): 'media_id': media['_id'], 'identifier': release_identifier, 'quality': group['meta_data']['quality'].get('identifier'), + 'is_3d': group['meta_data']['quality'].get('is_3d', 0), 'last_edit': int(time.time()), 'status': 'done' } @@ -157,15 +180,20 @@ class Release(Plugin): try: db = get_db() rel = db.get('id', release_id) + raw_files = rel.get('files') - if len(rel.get('files')) == 0: + if len(raw_files) == 0: self.delete(rel['_id']) else: - files = [] - for release_file in rel.get('files'): - if os.path.isfile(ss(release_file['path'])): - files.append(release_file) + files = {} + for file_type in raw_files: + + for release_file in raw_files.get(file_type, []): + if os.path.isfile(sp(release_file)): + if file_type not in files: + files[file_type] = [] + files[file_type].append(release_file) rel['files'] = files db.update(rel) @@ -315,10 +343,12 @@ class Release(Plugin): def tryDownloadResult(self, results, media, quality_custom, manual = False): + wait_for = False + let_through = False + filtered_results = [] + + # If a single release comes through the "wait for", let through all for rel in results: - if not quality_custom.get('finish', False) and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0): - log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for'), rel['name'])) - continue if rel['status'] in ['ignored', 'failed']: log.info('Ignored: %s', rel['name']) @@ -328,13 +358,30 @@ class Release(Plugin): log.info('Ignored, score to low: %s', rel['name']) continue + rel['wait_for'] = False + if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0): + rel['wait_for'] = True + else: + let_through = True + + filtered_results.append(rel) + + # Loop through filtered results + for rel in filtered_results: + + # Only wait if not a single release is old enough + if rel.get('wait_for') and not let_through: + log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for') - rel.get('age'), rel['name'])) + wait_for = True + continue + downloaded = fireEvent('release.download', data = rel, media = media, manual = manual, single = True) if downloaded is True: return True elif downloaded != 'try_next': break - return False + return wait_for def createFromSearch(self, search_results, media, quality): @@ -406,7 +453,7 @@ class Release(Plugin): rel = db.get('id', release_id) if rel and rel.get('status') != status: - release_name = rel.get('name') + release_name = rel['info'].get('name') if rel.get('files'): for file_type in rel.get('files', {}): if file_type == 'movie': diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 3830b97d..0b56ca33 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -112,16 +112,13 @@ class Renamer(Plugin): return if not base_folder: - base_folder = self.conf('from') + base_folder = sp(self.conf('from')) from_folder = sp(self.conf('from')) to_folder = sp(self.conf('to')) # Get media folder to process - media_folder = release_download.get('folder') - - # Quality order for calculation quality priority - quality_order = fireEvent('quality.order', single = True) + media_folder = sp(release_download.get('folder')) # Get all folders that should not be processed no_process = [to_folder] @@ -149,9 +146,9 @@ class Renamer(Plugin): # Update to the from folder if len(release_download.get('files', [])) == 1: - new_media_folder = from_folder + new_media_folder = sp(from_folder) else: - new_media_folder = os.path.join(from_folder, os.path.basename(media_folder)) + new_media_folder = sp(os.path.join(from_folder, os.path.basename(media_folder))) if not os.path.isdir(new_media_folder): log.error('The provided media folder %s does not exist and could also not be found in the \'from\' folder.', media_folder) @@ -290,8 +287,10 @@ class Renamer(Plugin): # Put 'The' at the end name_the = movie_name - if movie_name[:4].lower() == 'the ': - name_the = movie_name[4:] + ', The' + for prefix in ['the ', 'an ', 'a ']: + if prefix == movie_name[:len(prefix)].lower(): + name_the = movie_name[len(prefix):] + ', ' + prefix.strip().capitalize() + break replacements = { 'ext': 'mkv', @@ -312,9 +311,15 @@ class Renamer(Plugin): 'cd': '', 'cd_nr': '', 'mpaa': media['info'].get('mpaa', ''), + 'mpaa_only': media['info'].get('mpaa', ''), 'category': category_label, + '3d': '3D' if group['meta_data']['quality'].get('is_3d', 0) else '', + '3d_type': group['meta_data'].get('3d_type'), } + if replacements['mpaa_only'] not in ('G', 'PG', 'PG-13', 'R', 'NC-17'): + replacements['mpaa_only'] = 'Not Rated' + for file_type in group['files']: # Move nfo depending on settings @@ -410,8 +415,12 @@ class Renamer(Plugin): # Don't add language if multiple languages in 1 subtitle file if len(sub_langs) == 1: - sub_name = sub_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext'])) - rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) + sub_suffix = '%s.%s' % (sub_langs[0], replacements['ext']) + + # Don't add language to subtitle file it it's already there + if not sub_name.endswith(sub_suffix): + sub_name = sub_name.replace(replacements['ext'], sub_suffix) + rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) rename_files = mergeDicts(rename_files, rename_extras) @@ -438,17 +447,15 @@ class Renamer(Plugin): remove_leftovers = True # Mark movie "done" once it's found the quality with the finish check + profile = None try: if media.get('status') == 'active' and media.get('profile_id'): profile = db.get('id', media['profile_id']) - if group['meta_data']['quality']['identifier'] in profile.get('qualities', []): - nr = profile['qualities'].index(group['meta_data']['quality']['identifier']) - finish = profile['finish'][nr] - if finish: - mdia = db.get('id', media['_id']) - mdia['status'] = 'done' - mdia['last_edit'] = int(time.time()) - db.update(mdia) + if fireEvent('quality.isfinish', group['meta_data']['quality'], profile, single = True): + mdia = db.get('id', media['_id']) + mdia['status'] = 'done' + mdia['last_edit'] = int(time.time()) + db.update(mdia) except Exception as e: log.error('Failed marking movie finished: %s', (traceback.format_exc())) @@ -459,18 +466,19 @@ class Renamer(Plugin): # When a release already exists if release.get('status') == 'done': - release_order = quality_order.index(release['quality']) - group_quality_order = quality_order.index(group['meta_data']['quality']['identifier']) + # This is where CP removes older, lesser quality releases or releases that are not wanted anymore + is_higher = fireEvent('quality.ishigher', \ + group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, profile, single = True) - # This is where CP removes older, lesser quality releases - if release_order > group_quality_order: - log.info('Removing lesser quality %s for %s.', (media_title, release.get('quality'))) + if is_higher == 'higher': + log.info('Removing lesser or not wanted quality %s for %s.', (media_title, release.get('quality'))) for file_type in release.get('files', {}): for release_file in release['files'][file_type]: remove_files.append(release_file) remove_releases.append(release) + # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc - elif release_order == group_quality_order: + elif is_higher == 'equal': log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (media_title, release.get('quality'))) for file_type in release.get('files', {}): for release_file in release['files'][file_type]: @@ -652,11 +660,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif isinstance(release_download, dict): # Tag download_files if they are known if release_download.get('files', []): - tag_files = release_download.get('files', []) + tag_files = [filename for filename in release_download.get('files', []) if os.path.exists(filename)] # Tag all files in release folder elif release_download['folder']: - for root, folders, names in scandir.walk(release_download['folder']): + for root, folders, names in scandir.walk(sp(release_download['folder'])): tag_files.extend([os.path.join(root, name) for name in names]) for filename in tag_files: @@ -680,13 +688,13 @@ Remove it if you want it to be renamed (again, or at least let it try again) if isinstance(group, dict): tag_files = [sorted(list(group['files']['movie']))[0]] - folder = group['parentdir'] + folder = sp(group['parentdir']) if not group.get('dirname') or not os.path.isdir(folder): return False elif isinstance(release_download, dict): - folder = release_download['folder'] + folder = sp(release_download['folder']) if not os.path.isdir(folder): return False @@ -720,7 +728,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if not release_download: return False - folder = release_download['folder'] + folder = sp(release_download['folder']) if not os.path.isdir(folder): return False @@ -749,7 +757,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) return False def moveFile(self, old, dest, forcemove = False): - dest = ss(dest) + dest = sp(dest) try: if forcemove or self.conf('file_action') not in ['copy', 'link']: try: @@ -822,7 +830,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) def replaceDoubles(self, string): replaces = [ - ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), + ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), (' \\\\', '\\\\'), (' /', '/'), ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ] @@ -1054,6 +1062,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) release_download.update({ 'imdb_id': getIdentifier(media), 'quality': rls['quality'], + 'is_3d': rls['is_3d'], 'protocol': rls.get('info', {}).get('protocol') or rls.get('info', {}).get('type'), 'release_id': rls['_id'], }) @@ -1193,6 +1202,8 @@ rename_options = { 'first': 'First letter (M)', 'quality': 'Quality (720p)', 'quality_type': '(HD) or (SD)', + '3d': '3D', + '3d_type': '3D Type (Full SBS)', 'video': 'Video (x264)', 'audio': 'Audio (DTS)', 'group': 'Releasegroup name', @@ -1205,7 +1216,8 @@ rename_options = { 'imdb_id': 'IMDB id (tt0123456)', 'cd': 'CD number (cd1)', 'cd_nr': 'Just the cd nr. (1)', - 'mpaa': 'MPAA Rating', + 'mpaa': 'MPAA or other certification', + 'mpaa_only': 'MPAA only certification (G|PG|PG-13|R|NC-17|Not Rated)', 'category': 'Category label', }, } diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 6b7c7b0f..b8eca4c4 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -6,7 +6,7 @@ import traceback from couchpotato import get_db from couchpotato.core.event import fireEvent, addEvent -from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp +from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp, ss from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \ splitString, getIdentifier from couchpotato.core.logger import CPLog @@ -40,6 +40,17 @@ class Scanner(Plugin): 'trailer': ['mov', 'mp4', 'flv'] } + threed_types = { + 'Half SBS': [('half', 'sbs'), ('h', 'sbs'), 'hsbs'], + 'Full SBS': [('full', 'sbs'), ('f', 'sbs'), 'fsbs'], + 'SBS': ['sbs'], + 'Half OU': [('half', 'ou'), ('h', 'ou'), 'hou'], + 'Full OU': [('full', 'ou'), ('h', 'ou'), 'fou'], + 'OU': ['ou'], + 'Frame Packed': ['mvc', ('complete', 'bluray')], + '3D': ['3d'] + } + file_types = { 'subtitle': ('subtitle', 'subtitle'), 'subtitle_extra': ('subtitle', 'subtitle_extra'), @@ -60,33 +71,43 @@ class Scanner(Plugin): } codecs = { - 'audio': ['dts', 'ac3', 'ac3d', 'mp3'], - 'video': ['x264', 'h264', 'divx', 'xvid'] + 'audio': ['DTS', 'AC3', 'AC3D', 'MP3'], + 'video': ['x264', 'H264', 'DivX', 'Xvid'] + } + + resolutions = { + '1080p': {'resolution_width': 1920, 'resolution_height': 1080, 'aspect': 1.78}, + '1080i': {'resolution_width': 1920, 'resolution_height': 1080, 'aspect': 1.78}, + '720p': {'resolution_width': 1280, 'resolution_height': 720, 'aspect': 1.78}, + '720i': {'resolution_width': 1280, 'resolution_height': 720, 'aspect': 1.78}, + '480p': {'resolution_width': 640, 'resolution_height': 480, 'aspect': 1.33}, + '480i': {'resolution_width': 640, 'resolution_height': 480, 'aspect': 1.33}, + 'default': {'resolution_width': 0, 'resolution_height': 0, 'aspect': 1}, } audio_codec_map = { - 0x2000: 'ac3', - 0x2001: 'dts', - 0x0055: 'mp3', - 0x0050: 'mp2', - 0x0001: 'pcm', - 0x003: 'pcm', - 0x77a1: 'tta1', - 0x5756: 'wav', - 0x6750: 'vorbis', - 0xF1AC: 'flac', - 0x00ff: 'aac', + 0x2000: 'AC3', + 0x2001: 'DTS', + 0x0055: 'MP3', + 0x0050: 'MP2', + 0x0001: 'PCM', + 0x003: 'WAV', + 0x77a1: 'TTA1', + 0x5756: 'WAV', + 0x6750: 'Vorbis', + 0xF1AC: 'FLAC', + 0x00ff: 'AAC', } source_media = { - 'bluray': ['bluray', 'blu-ray', 'brrip', 'br-rip'], - 'hddvd': ['hddvd', 'hd-dvd'], - 'dvd': ['dvd'], - 'hdtv': ['hdtv'] + 'Blu-ray': ['bluray', 'blu-ray', 'brrip', 'br-rip'], + 'HD DVD': ['hddvd', 'hd-dvd'], + 'DVD': ['dvd'], + 'HDTV': ['hdtv'] } - clean = '[ _\,\.\(\)\[\]\-]?(3d|hsbs|sbs|extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \ - '|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)' + clean = '([ _\,\.\(\)\[\]\-]|^)(3d|hsbs|sbs|ou|extended.cut|directors.cut|french|fr|swedisch|sw|danish|dutch|nl|swesub|subs|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \ + '|hdtvrip|webdl|web.dl|webrip|web.rip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|hc|\[.*\])(?=[ _\,\.\(\)\[\]\-]|$)' multipart_regex = [ '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 @@ -130,7 +151,7 @@ class Scanner(Plugin): try: files = [] for root, dirs, walk_files in scandir.walk(folder, followlinks=True): - files.extend([sp(os.path.join(root, filename)) for filename in walk_files]) + files.extend([sp(os.path.join(sp(root), ss(filename))) for filename in walk_files]) # Break if CP wants to shut down if self.shuttingDown(): @@ -164,7 +185,7 @@ class Scanner(Plugin): identifiers = [identifier] # Identifier with quality - quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'} + quality = fireEvent('quality.guess', files = [file_path], size = self.getFileSize(file_path), single = True) if not is_dvd_file else {'identifier':'dvdr'} if quality: identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] @@ -431,28 +452,39 @@ class Scanner(Plugin): for cur_file in files: if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files - meta = self.getMeta(cur_file) + if not data.get('audio'): # Only get metadata from first media file + meta = self.getMeta(cur_file) - try: - data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video'])) - data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio'])) - data['resolution_width'] = meta.get('resolution_width', 720) - data['resolution_height'] = meta.get('resolution_height', 480) - data['audio_channels'] = meta.get('audio_channels', 2.0) - data['aspect'] = round(float(meta.get('resolution_width', 720)) / meta.get('resolution_height', 480), 2) - except: - log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc())) - pass + try: + data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video'])) + data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio'])) + data['audio_channels'] = meta.get('audio_channels', 2.0) + if meta.get('resolution_width'): + data['resolution_width'] = meta.get('resolution_width') + data['resolution_height'] = meta.get('resolution_height') + data['aspect'] = round(float(meta.get('resolution_width')) / meta.get('resolution_height', 1), 2) + else: + data.update(self.getResolution(cur_file)) + except: + log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc())) + pass - if data.get('audio'): break + data['size'] = data.get('size', 0) + self.getFileSize(cur_file) - # Use the quality guess first, if that failes use the quality we wanted to download data['quality'] = None + quality = fireEvent('quality.guess', size = data['size'], files = files, extra = data, single = True) + + # Use the quality that we snatched but check if it matches our guess if release_download and release_download.get('quality'): data['quality'] = fireEvent('quality.single', release_download.get('quality'), single = True) + data['quality']['is_3d'] = release_download.get('is_3d', 0) + if data['quality']['identifier'] != quality['identifier']: + log.info('Different quality snatched than detected for %s: %s vs. %s. Assuming snatched quality is correct.', (files[0], data['quality']['identifier'], quality['identifier'])) + if data['quality']['is_3d'] != quality['is_3d']: + log.info('Different 3d snatched than detected for %s: %s vs. %s. Assuming snatched 3d is correct.', (files[0], data['quality']['is_3d'], quality['is_3d'])) if not data['quality']: - data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True) + data['quality'] = quality if not data['quality']: data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) @@ -462,16 +494,32 @@ class Scanner(Plugin): filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) data['group'] = self.getGroup(filename[len(folder):]) data['source'] = self.getSourceMedia(filename) - + if data['quality'].get('is_3d', 0): + data['3d_type'] = self.get3dType(filename) return data + def get3dType(self, filename): + filename = ss(filename) + + words = re.split('\W+', filename.lower()) + + for key in self.threed_types: + tags = self.threed_types.get(key, []) + + for tag in tags: + if (isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words)) or (isinstance(tag, (str, unicode)) and ss(tag.lower()) in words): + log.debug('Found %s in %s', (tag, filename)) + return key + + return '' + def getMeta(self, filename): try: p = enzyme.parse(filename) # Video codec - vc = ('h264' if p.video[0].codec == 'AVC1' else p.video[0].codec).lower() + vc = ('H264' if p.video[0].codec == 'AVC1' else p.video[0].codec) # Audio codec ac = p.audio[0].codec @@ -708,19 +756,26 @@ class Scanner(Plugin): if not file_size: file_size = [] try: - return (file_size.get('min', 0) * 1048576) < os.path.getsize(file) < (file_size.get('max', 100000) * 1048576) + return file_size.get('min', 0) < self.getFileSize(file) < file_size.get('max', 100000) except: log.error('Couldn\'t get filesize of %s.', file) return False - def createStringIdentifier(self, file_path, folder = '', exclude_filename = False): + def getFileSize(self, file): + try: + return os.path.getsize(file) / 1024 / 1024 + except: + return None - year = self.findYear(file_path) + def createStringIdentifier(self, file_path, folder = '', exclude_filename = False): identifier = file_path.replace(folder, '').lstrip(os.path.sep) # root folder identifier = os.path.splitext(identifier)[0] # ext + # Make sure the identifier is lower case as all regex is with lower case tags + identifier = identifier.lower() + try: path_split = splitString(identifier, os.path.sep) identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename @@ -735,8 +790,13 @@ class Scanner(Plugin): # remove cptag identifier = self.removeCPTag(identifier) - # groups, release tags, scenename cleaner, regex isn't correct - identifier = re.sub(self.clean, '::', simplifyString(identifier)).strip(':') + # simplify the string + identifier = simplifyString(identifier) + + year = self.findYear(file_path) + + # groups, release tags, scenename cleaner + identifier = re.sub(self.clean, '::', identifier).strip(':') # Year if year and identifier[:4] != year: @@ -785,6 +845,14 @@ class Scanner(Plugin): except: return '' + def getResolution(self, filename): + try: + for key in self.resolutions: + if key in filename.lower() and key != 'default': + return self.resolutions[key] + except: + return self.resolutions['default'] + def getGroup(self, file): try: match = re.findall('\-([A-Z0-9]+)[\.\/]', file, re.I) diff --git a/couchpotato/core/plugins/subtitle.py b/couchpotato/core/plugins/subtitle.py index 9fd7ef16..fdb640b1 100644 --- a/couchpotato/core/plugins/subtitle.py +++ b/couchpotato/core/plugins/subtitle.py @@ -32,7 +32,7 @@ class Subtitle(Plugin): for lang in self.getLanguages(): if lang not in available_languages: - download = subliminal.download_subtitles(files, multi = True, force = False, languages = [lang], services = self.services, cache_dir = Env.get('cache_dir')) + download = subliminal.download_subtitles(files, multi = True, force = self.conf('force'), languages = [lang], services = self.services, cache_dir = Env.get('cache_dir')) for subtitle in download: downloaded.extend(download[subtitle]) @@ -72,6 +72,14 @@ config = [{ 'name': 'languages', 'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at on Wikipedia'), }, + { + 'advanced': True, + 'name': 'force', + 'label': 'Force', + 'description': ('Force download all languages (including embedded).', 'This will also overwrite all existing subtitles.'), + 'default': False, + 'type': 'bool', + }, ], }, ], diff --git a/couchpotato/core/plugins/userscript/static/userscript.js b/couchpotato/core/plugins/userscript/static/userscript.js index 71543d51..c52dbee2 100644 --- a/couchpotato/core/plugins/userscript/static/userscript.js +++ b/couchpotato/core/plugins/userscript/static/userscript.js @@ -2,6 +2,7 @@ Page.Userscript = new Class({ Extends: PageBase, + order: 80, name: 'userscript', has_tab: false, diff --git a/couchpotato/core/plugins/wizard/static/wizard.js b/couchpotato/core/plugins/wizard/static/wizard.js index b43e3985..fdd7b743 100644 --- a/couchpotato/core/plugins/wizard/static/wizard.js +++ b/couchpotato/core/plugins/wizard/static/wizard.js @@ -2,6 +2,7 @@ Page.Wizard = new Class({ Extends: Page.Settings, + order: 70, name: 'wizard', has_tab: false, wizard_only: true, diff --git a/couchpotato/core/settings.py b/couchpotato/core/settings.py index 16c827c8..a0bcd743 100644 --- a/couchpotato/core/settings.py +++ b/couchpotato/core/settings.py @@ -234,7 +234,7 @@ class Settings(object): propert = db.get('property', identifier, with_doc = True) prop = propert['doc']['value'] except: - self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0))) + pass # self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0))) return prop diff --git a/couchpotato/runner.py b/couchpotato/runner.py index dcdac5c8..d22f8fc8 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -10,20 +10,20 @@ import warnings import re import tarfile -from CodernityDB.database_thread_safe import ThreadSafeDatabase +from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase from argparse import ArgumentParser from cache import FileSystemCache from couchpotato import KeyHandler, LoginHandler, LogoutHandler from couchpotato.api import NonBlockHandler, ApiHandler from couchpotato.core.event import fireEventAsync, fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import getDataDir, tryInt from scandir import scandir from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler -def getOptions(base_path, args): +def getOptions(args): # Options parser = ArgumentParser(prog = 'CouchPotato.py') @@ -86,15 +86,15 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En Env.set('encoding', encoding) # Do db stuff - db_path = toUnicode(os.path.join(data_dir, 'database')) + db_path = sp(os.path.join(data_dir, 'database')) # Check if database exists - db = ThreadSafeDatabase(db_path) + db = SuperThreadSafeDatabase(db_path) db_exists = db.exists() if db_exists: # Backup before start and cleanup old backups - backup_path = toUnicode(os.path.join(data_dir, 'db_backup')) + backup_path = sp(os.path.join(data_dir, 'db_backup')) backup_count = 5 existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) @@ -114,7 +114,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En os.remove(os.path.join(backup_path, eb[1])) # Create new backup - new_backup = toUnicode(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) + new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) zipf = tarfile.open(new_backup, 'w:gz') for root, dirs, files in scandir.walk(db_path): for zfilename in files: @@ -128,12 +128,12 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En db.create() # Register environment settings - Env.set('app_dir', toUnicode(base_path)) - Env.set('data_dir', toUnicode(data_dir)) - Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log'))) + Env.set('app_dir', sp(base_path)) + Env.set('data_dir', sp(data_dir)) + Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) - Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache'))) - Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python')))) + Env.set('cache_dir', sp(os.path.join(data_dir, 'cache'))) + Env.set('cache', FileSystemCache(sp(os.path.join(Env.get('cache_dir'), 'python')))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) @@ -245,13 +245,13 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ - ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) + ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') - loader.preload(root = toUnicode(base_path)) + loader.preload(root = sp(base_path)) loader.run() # Fill database with needed stuff @@ -262,8 +262,14 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Go go go! from tornado.ioloop import IOLoop + from tornado.autoreload import add_reload_hook loop = IOLoop.current() + # Reload hook + def test(): + fireEvent('app.shutdown') + add_reload_hook(test) + # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index f4462772..8a8f1e7c 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -1,4 +1,4 @@ -var CouchPotato = new Class({ +var CouchPotato = new Class({ Implements: [Events, Options], @@ -137,19 +137,34 @@ createPages: function(){ var self = this; + var pages = []; Object.each(Page, function(page_class, class_name){ var pg = new Page[class_name](self, {}); self.pages[class_name] = pg; - self.fireEvent('load'+class_name); - - $(pg).inject(self.pages); + pages.include({ + 'order': pg.order, + 'name': class_name, + 'class': pg + }); }); + pages.stableSort(self.sortPageByOrder).each(function(page){ + page['class'].load(); + self.fireEvent('load'+page.name); + $(page['class']).inject(self.content); + }); + + delete pages; + self.fireEvent('load'); }, + sortPageByOrder: function(a, b){ + return (a.order || 100) - (b.order || 100) + }, + openPage: function(url) { var self = this; diff --git a/couchpotato/static/scripts/page.js b/couchpotato/static/scripts/page.js index 58ba5acd..57b8b108 100644 --- a/couchpotato/static/scripts/page.js +++ b/couchpotato/static/scripts/page.js @@ -6,6 +6,7 @@ var PageBase = new Class({ }, + order: 1, has_tab: true, name: '', @@ -16,6 +17,10 @@ var PageBase = new Class({ // Create main page container self.el = new Element('div.page.'+self.name); + }, + + load: function(){ + var self = this; // Create tab for page if(self.has_tab){ @@ -26,6 +31,7 @@ var PageBase = new Class({ 'text': self.name.capitalize() }); } + }, open: function(action, params){ diff --git a/couchpotato/static/scripts/page/home.js b/couchpotato/static/scripts/page/home.js index bf435c14..7cde7d2a 100644 --- a/couchpotato/static/scripts/page/home.js +++ b/couchpotato/static/scripts/page/home.js @@ -104,7 +104,7 @@ Page.Home = new Class({ // Make all thumbnails the same size self.soon_list.addEvent('loaded', function(){ - var images = $(self.soon_list).getElements('.poster'), + var images = $(self.soon_list).getElements('.poster, .no_thumbnail'), timer, highest = 100; diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index 09155486..de5ffcbc 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -2,6 +2,7 @@ Page.Settings = new Class({ Extends: PageBase, + order: 50, name: 'settings', title: 'Change settings.', wizard_only: false, @@ -112,7 +113,7 @@ Page.Settings = new Class({ }, sortByOrder: function(a, b){ - return (a.order || 100) - (b.order || 100) + return (a.order || 100) - (b.order || 100) }, create: function(json){