From 1de457fa8d2d7ba6e489738d65d95fb6d865449a Mon Sep 17 00:00:00 2001 From: Troy Olson Date: Wed, 3 Sep 2014 12:33:04 -0700 Subject: [PATCH 01/22] Update iptorrents.py Adds another URL parameter when searching on iptorrents for brrip. This allows it to find brrip that are also classified as 720p/1080p. See this thread for info: https://couchpota.to/forum/viewtopic.php?f=5&t=4032&sid=7892abbaaca9dad8bd3cc27cafb7fd33 And this prior pull request for more info: https://github.com/RuudBurger/CouchPotatoServer/pull/3696 --- couchpotato/core/media/movie/providers/torrent/iptorrents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/torrent/iptorrents.py b/couchpotato/core/media/movie/providers/torrent/iptorrents.py index 89aeee80..1c75feb7 100644 --- a/couchpotato/core/media/movie/providers/torrent/iptorrents.py +++ b/couchpotato/core/media/movie/providers/torrent/iptorrents.py @@ -13,7 +13,7 @@ class IPTorrents(MovieProvider, Base): ([87], ['3d']), ([48], ['720p', '1080p', 'bd50']), ([72], ['cam', 'ts', 'tc', 'r5', 'scr']), - ([7], ['dvdrip', 'brrip']), + ([7,48], ['dvdrip', 'brrip']), ([6], ['dvdr']), ] From bc6d19700429cf5a6aab611018be4fb55aef5087 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:40:54 +0200 Subject: [PATCH 02/22] Don't score identifier quality.guess twice --- couchpotato/core/plugins/quality/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index baeca670..1c836146 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -293,10 +293,6 @@ class QualityPlugin(Plugin): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) score += points.get(tag_type) - if list(set(qualities) & set(words)): - log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) - score += points.get(tag_type) - # Check extention for ext in quality.get('ext', []): if ext == extension: From ab253f90302e64ee09ca6b37358da0ad338bc78a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:41:59 +0200 Subject: [PATCH 03/22] Add quality test --- couchpotato/core/plugins/quality/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 1c836146..f5d890a3 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -482,6 +482,8 @@ class QualityPlugin(Plugin): 'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}}, 'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}}, 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, + 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, + 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, } correct = 0 From 4879bc62516167cfab16a21cb818ce65c3cf1c06 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:42:45 +0200 Subject: [PATCH 04/22] Move hdtv & hdrip to alternative for brrips --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index f5d890a3..3daac01a 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -26,7 +26,7 @@ class QualityPlugin(Plugin): {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, From 4f646094b5dcdc03f759cd1e637a24bdba6322fd Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:50:08 +0200 Subject: [PATCH 05/22] Add quality test --- couchpotato/core/plugins/quality/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 3daac01a..b5d82386 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -484,6 +484,7 @@ class QualityPlugin(Plugin): 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, + 'Movie Name.2014.720p.WEBRip.x264.AC3-MAJESTiC': {'size': 3000, 'quality': 'scr'}, } correct = 0 From af8806e2928dacb7df638127fbfd9071fd14565c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Sep 2014 21:50:20 +0200 Subject: [PATCH 06/22] Move webrip to scr alternative --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index b5d82386..352cf797 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -29,7 +29,7 @@ class QualityPlugin(Plugin): {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, + {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []}, {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]}, {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]}, {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]}, From ca24bf031c05faf83e5d8cfbaa17c75afa22fb28 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 8 Sep 2014 19:21:31 +0200 Subject: [PATCH 07/22] Change quality test --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 352cf797..f3fbefef 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -484,7 +484,7 @@ class QualityPlugin(Plugin): 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'}, 'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'}, - 'Movie Name.2014.720p.WEBRip.x264.AC3-MAJESTiC': {'size': 3000, 'quality': 'scr'}, + 'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'}, } correct = 0 From fa054b6b3419595ecbe30560011fba96fb5f81fd Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 13:55:44 +0200 Subject: [PATCH 08/22] Migration: Don't fail on missing release file --- couchpotato/core/database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 10ae26c2..09f21059 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -579,7 +579,10 @@ class Database(object): continue for f in release_files: - rfile = all_files[f.get('file_id')] + rfile = all_files.get(f.get('file_id')) + if not rfile: + continue + file_type = type_by_id.get(rfile.get('type_id')).get('identifier') if not release['files'].get(file_type): From 0358378cae45e76be1accfba32db8e37abc151bf Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 15:16:32 +0200 Subject: [PATCH 09/22] Fix marshal data corrupted documents --- couchpotato/core/plugins/release/main.py | 29 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 31c3e61b..b8417c3b 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -70,28 +70,45 @@ class Release(Plugin): db = get_db() # Get (and remove) parentless releases - releases = db.all('release', with_doc = True) + releases = db.all('release', with_doc = False) media_exist = [] + reindex = 0 for release in releases: if release.get('key') in media_exist: continue try: + + try: + doc = db.get('id', release.get('_id')) + except RecordDeleted: + reindex += 1 + continue + db.get('id', release.get('key')) media_exist.append(release.get('key')) try: - if release['doc'].get('status') == 'ignore': - release['doc']['status'] = 'ignored' - db.update(release['doc']) + if doc.get('status') == 'ignore': + doc['status'] = 'ignored' + db.update(doc) except: log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) + except ValueError: + log.debug('Deleted corrupted document "%s": %s', (release.get('key'), traceback.format_exc(0))) + corrupted = db.get('id', release.get('key'), with_storage = False) + db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + reindex += 1 except RecordDeleted: - db.delete(release['doc']) - log.debug('Deleted orphaned release: %s', release['doc']) + db.delete(doc) + log.debug('Deleted orphaned release: %s', doc) + reindex += 1 except: log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc()) + if reindex > 0: + db.reindex() + del media_exist # get movies last_edit more than a week ago From 80df57f2b625231dfd76538f16428a303f7621ff Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 22:00:54 +0200 Subject: [PATCH 10/22] Delete corrupted documents --- couchpotato/core/database.py | 12 ++++++++++++ couchpotato/core/media/_base/media/main.py | 4 +++- couchpotato/core/plugins/release/main.py | 6 ++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 09f21059..37841bf6 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -32,6 +32,7 @@ class Database(object): addEvent('database.setup.after', self.startup_compact) addEvent('database.setup_index', self.setupIndex) + addEvent('database.delete_corrupted', self.deleteCorrupted) addEvent('app.migrate', self.migrate) addEvent('app.after_shutdown', self.close) @@ -147,6 +148,17 @@ class Database(object): return results + def deleteCorrupted(self, _id, traceback_error = ''): + + db = self.getDB() + + try: + log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error)) + corrupted = db.get('id', _id, with_storage = False) + db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + except: + log.debug('Failed deleting corrupted: %s', traceback.format_exc()) + def reindex(self, **kwargs): success = True diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index d5e54d73..522d9fbb 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -178,8 +178,10 @@ class MediaPlugin(MediaBase): continue yield doc - except RecordNotFound: + except (RecordDeleted, RecordNotFound): log.debug('Record not found, skipping: %s', ms['_id']) + except (ValueError, EOFError): + fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0)) else: yield ms diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index b8417c3b..cc92b9f0 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -95,9 +95,7 @@ class Release(Plugin): except: log.error('Failed fixing mis-status tag: %s', traceback.format_exc()) except ValueError: - log.debug('Deleted corrupted document "%s": %s', (release.get('key'), traceback.format_exc(0))) - corrupted = db.get('id', release.get('key'), with_storage = False) - db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None) + fireEvent('database.delete_corrupted', release.get('key'), traceback_error = traceback.format_exc(0)) reindex += 1 except RecordDeleted: db.delete(doc) @@ -112,7 +110,7 @@ class Release(Plugin): del media_exist # get movies last_edit more than a week ago - medias = fireEvent('media.with_status', ['done','active'], single = True) + medias = fireEvent('media.with_status', ['done', 'active'], single = True) for media in medias: if media.get('last_edit', 0) > (now - week): From 002ce4d4e1283e6a1782cec718dc75c8407048a9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 22:05:54 +0200 Subject: [PATCH 11/22] Ignore deleted media documents --- couchpotato/core/media/__init__.py | 12 ++++++------ couchpotato/core/media/_base/media/main.py | 4 ++++ couchpotato/core/media/movie/_base/main.py | 3 +++ couchpotato/core/media/movie/searcher.py | 8 +++++--- couchpotato/core/plugins/automation.py | 3 ++- couchpotato/core/plugins/dashboard.py | 7 ++++++- couchpotato/core/plugins/manage.py | 3 ++- 7 files changed, 28 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 4e319fc3..549ed0d7 100755 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -26,9 +26,9 @@ class MediaBase(Plugin): def onComplete(): try: media = fireEvent('media.get', media_id, single = True) - event_name = '%s.searcher.single' % media.get('type') - - fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) + if media: + event_name = '%s.searcher.single' % media.get('type') + fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) @@ -39,9 +39,9 @@ class MediaBase(Plugin): def notifyFront(): try: media = fireEvent('media.get', media_id, single = True) - event_name = '%s.update' % media.get('type') - - fireEvent('notify.frontend', type = event_name, data = media) + if media: + event_name = '%s.update' % media.get('type') + fireEvent('notify.frontend', type = event_name, data = media) except: log.error('Failed creating onComplete: %s', traceback.format_exc()) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 522d9fbb..ccee1fed 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -282,6 +282,10 @@ class MediaPlugin(MediaBase): media = fireEvent('media.get', media_id, single = True) + # Skip if no media has been found + if not media: + continue + # Merge releases with movie dict medias.append(media) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 0dc39e56..2757a920 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -179,6 +179,9 @@ class MovieBase(MovieTypeBase): db.delete(rel) movie_dict = fireEvent('media.get', m['_id'], single = True) + if not movie_dict: + log.debug('Failed adding media, can\'t find it anymore') + return False if do_search and search_after: onComplete = self.createOnComplete(m['_id']) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 97181aee..7ebaaf54 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -89,6 +89,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): for media_id in medias: media = fireEvent('media.get', media_id, single = True) + if not media: continue try: self.single(media, search_protocols, manual = manual) @@ -388,9 +389,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): rel['status'] = 'ignored' db.update(rel) - movie_dict = fireEvent('media.get', media_id, single = True) - log.info('Trying next release for: %s', getTitle(movie_dict)) - self.single(movie_dict, manual = manual, force_download = force_download) + media = fireEvent('media.get', media_id, single = True) + if media: + log.info('Trying next release for: %s', getTitle(media)) + self.single(media, manual = manual, force_download = force_download) return True diff --git a/couchpotato/core/plugins/automation.py b/couchpotato/core/plugins/automation.py index 39d7c9e7..e98a00a6 100644 --- a/couchpotato/core/plugins/automation.py +++ b/couchpotato/core/plugins/automation.py @@ -46,7 +46,8 @@ class Automation(Plugin): break movie_dict = fireEvent('media.get', movie_id, single = True) - fireEvent('movie.searcher.single', movie_dict) + if movie_dict: + fireEvent('movie.searcher.single', movie_dict) return True diff --git a/couchpotato/core/plugins/dashboard.py b/couchpotato/core/plugins/dashboard.py index d4af7ad3..afead443 100644 --- a/couchpotato/core/plugins/dashboard.py +++ b/couchpotato/core/plugins/dashboard.py @@ -1,5 +1,6 @@ import random as rndm import time +from CodernityDB.database import RecordDeleted from couchpotato import get_db from couchpotato.api import addApiView @@ -58,7 +59,11 @@ class Dashboard(Plugin): rndm.shuffle(active_ids) for media_id in active_ids: - media = db.get('id', media_id) + try: + media = db.get('id', media_id) + except RecordDeleted: + log.debug('Record already deleted: %s', media_id) + continue pp = profile_pre.get(media.get('profile_id')) if not pp: continue diff --git a/couchpotato/core/plugins/manage.py b/couchpotato/core/plugins/manage.py index 132f7ec0..75c550bf 100755 --- a/couchpotato/core/plugins/manage.py +++ b/couchpotato/core/plugins/manage.py @@ -234,7 +234,8 @@ class Manage(Plugin): total = self.in_progress[folder]['total'] movie_dict = fireEvent('media.get', identifier, single = True) - fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) + if movie_dict: + fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict)) return afterUpdate From f7eeaf3eda2b4144ebf3ec90c4b7488325519445 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 16 Sep 2014 22:44:33 +0200 Subject: [PATCH 12/22] Locking mechanism --- couchpotato/core/event.py | 2 +- couchpotato/core/media/movie/_base/main.py | 8 ++++++-- couchpotato/core/plugins/base.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 7246cde1..35818e7e 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs): else: - e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock()) + e = Event(name = name, threads = 10, exc_info = True, traceback = True) for event in events[name]: e.handle(event['handler'], priority = event['priority']) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 2757a920..1863d897 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -271,6 +271,10 @@ class MovieBase(MovieTypeBase): if self.shuttingDown(): return + lock_key = 'media.get.%s' % media_id if media_id else identifier + self.acquireLock(lock_key) + + media = {} try: db = get_db() @@ -319,11 +323,11 @@ class MovieBase(MovieTypeBase): self.getPoster(media, image_urls) db.update(media) - return media except: log.error('Failed update media: %s', traceback.format_exc()) - return {} + self.releaseLock(lock_key) + return media def updateReleaseDate(self, media_id): """ diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2c7f9e13..d9308f4d 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -1,3 +1,4 @@ +import threading from urllib import quote from urlparse import urlparse import glob @@ -35,6 +36,8 @@ class Plugin(object): _needs_shutdown = False _running = None + _locks = {} + user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0' http_last_use = {} http_time_between_calls = 0 @@ -393,3 +396,19 @@ class Plugin(object): def isEnabled(self): return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None + + def acquireLock(self, key): + + lock = self._locks.get(key) + if not lock: + self._locks[key] = threading.RLock() + + log.debug('Acquiring lock: %s', key) + self._locks.get(key).acquire() + + def releaseLock(self, key): + + lock = self._locks.get(key) + if lock: + log.debug('Releasing lock: %s', key) + self._locks.get(key).release() From 4a9452672a77d14c6c0c8def313050b1d645b0eb Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 12:59:24 +0200 Subject: [PATCH 13/22] Use status_code to stop requesting url --- couchpotato/core/media/_base/providers/nzb/newznab.py | 7 ++++--- couchpotato/core/plugins/base.py | 9 ++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 87ecc752..62b787d8 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -187,11 +187,12 @@ class Base(NZBProvider, RSS): self.limits_reached[host] = False return data except HTTPError as e: - if e.response.status_code == 503: + sc = e.response.status_code + if sc in [503, 429]: response = e.read().lower() - if 'maximum api' in response or 'download limit' in response: + if sc == 429 or 'maximum api' in response or 'download limit' in response: if not self.limits_reached.get(host): - log.error('Limit reached for newznab provider: %s', host) + log.error('Limit reached / to many requests for newznab provider: %s', host) self.limits_reached[host] = time.time() return 'try_next' diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2c7f9e13..ecc77b30 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -198,6 +198,7 @@ class Plugin(object): del self.http_failed_disabled[host] self.wait(host) + status_code = None try: kwargs = { @@ -212,6 +213,7 @@ class Plugin(object): log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data')) response = r.request(method, url, **kwargs) + status_code = response.status_code if response.status_code == requests.codes.ok: data = response.content else: @@ -224,6 +226,11 @@ class Plugin(object): # Save failed requests by hosts try: + + # To many requests + if status_code == 429: + self.http_failed_request[host] = time.time() + if not self.http_failed_request.get(host): self.http_failed_request[host] = 1 else: @@ -255,7 +262,7 @@ class Plugin(object): if wait > 0: log.debug('Waiting for %s, %d seconds', (self.getName(), wait)) - time.sleep(wait) + time.sleep(min(wait, 30)) def beforeCall(self, handler): self.isRunning('%s.%s' % (self.getName(), handler.__name__)) From 76322c0145335f2e46575b543f1220e026ae6e1c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 21:50:41 +0200 Subject: [PATCH 14/22] Don't save data in notification --- couchpotato/core/notifications/core/main.py | 9 +++++++-- .../core/notifications/core/static/notification.js | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index fdc837ad..c5f47933 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -155,9 +155,14 @@ class CoreNotifier(Notification): n = { '_t': 'notification', 'time': int(time.time()), - 'message': toUnicode(message), - 'data': data + 'message': toUnicode(message) } + + if data.get('sticky'): + n['sticky'] = True + if data.get('important'): + n['important'] = True + db.insert(n) self.frontend(type = listener, data = n) diff --git a/couchpotato/core/notifications/core/static/notification.js b/couchpotato/core/notifications/core/static/notification.js index 93bfa15d..b388b289 100644 --- a/couchpotato/core/notifications/core/static/notification.js +++ b/couchpotato/core/notifications/core/static/notification.js @@ -50,7 +50,7 @@ var NotificationBase = new Class({ , 'top'); self.notifications.include(result); - if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){ + if((result.important !== undefined || result.sticky !== undefined) && !result.read){ var sticky = true; App.trigger('message', [result.message, sticky, result]) } @@ -72,7 +72,7 @@ var NotificationBase = new Class({ if(!force_ids) { var rn = self.notifications.filter(function(n){ - return !n.read && n.data.important === undefined + return !n.read && n.important === undefined }); var ids = []; From 4c198f71168dd5e128a4cbbd09eb083858b65e26 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 22:51:27 +0200 Subject: [PATCH 15/22] Ignore RecordDeleted on notification getter fix #3888 --- couchpotato/core/notifications/core/main.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index c5f47933..fa8e9a7e 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -3,6 +3,7 @@ import threading import time import traceback import uuid +from CodernityDB.database import RecordDeleted from couchpotato import get_db from couchpotato.api import addApiView, addNonBlockApiView @@ -270,11 +271,16 @@ class CoreNotifier(Notification): if init: db = get_db() - notifications = db.all('notification', with_doc = True) + notifications = db.all('notification') for n in notifications: - if n['doc'].get('time') > (time.time() - 604800): - messages.append(n['doc']) + + try: + doc = db.get('id', n.get('_id')) + if doc.get('time') > (time.time() - 604800): + messages.append(doc) + except RecordDeleted: + pass return { 'success': True, From f74b837faa072c4b0762e22ea107f317a8571b4c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 23:05:01 +0200 Subject: [PATCH 16/22] Ignore RecordDeleted in release for media call --- couchpotato/core/plugins/release/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index cc92b9f0..13ba5d4c 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -543,11 +543,15 @@ class Release(Plugin): def forMedia(self, media_id): db = get_db() - raw_releases = list(db.get_many('release', media_id, with_doc = True)) + raw_releases = db.get_many('release', media_id) releases = [] for r in raw_releases: - releases.append(r['doc']) + try: + doc = db.get('id', r.get('_id')) + releases.append(doc) + except RecordDeleted: + pass releases = sorted(releases, key = lambda k: k.get('info', {}).get('score', 0), reverse = True) From 035b99bc8abe8cfc0f37e524cf1e806d9eb42718 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 17 Sep 2014 23:06:59 +0200 Subject: [PATCH 17/22] Don't use event when not needed --- couchpotato/core/plugins/release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 13ba5d4c..0385e226 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -116,7 +116,7 @@ class Release(Plugin): if media.get('last_edit', 0) > (now - week): continue - for rel in fireEvent('release.for_media', media['_id'], single = True): + for rel in self.forMedia(media['_id']): # Remove all available releases if rel['status'] in ['available']: From c948216e33476339b2687a92362d76526049b514 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 14:17:00 +0200 Subject: [PATCH 18/22] I need to watch more Sesame Street.. 26 letters in the alphabet + # for numbers is 27.. --- couchpotato/core/media/_base/media/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index ccee1fed..4bcd820e 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -379,7 +379,7 @@ class MediaPlugin(MediaBase): if x['_id'] in media_ids: chars.add(x['key']) - if len(chars) == 25: + if len(chars) == 27: break return list(chars) From 5fc9d7182c75dffe10afe1df47ed71ce5c8ad9a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 14:26:34 +0200 Subject: [PATCH 19/22] Hide urllib3 error closes #3887 --- couchpotato/runner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 5d3f62b0..1c6f5779 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -19,6 +19,7 @@ from couchpotato.core.event import fireEventAsync, fireEvent from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace import requests +from requests.packages.urllib3 import disable_warnings from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler @@ -174,6 +175,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En for logger_name in ['gntp']: logging.getLogger(logger_name).setLevel(logging.WARNING) + # Disable SSL warning + disable_warnings() + # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon From 1b724b5606933daff50e0375137d9981a82ea4dd Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 16:04:21 +0200 Subject: [PATCH 20/22] Media got tagged with ignored, instead of release --- couchpotato/core/media/movie/searcher.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 7ebaaf54..4940e7c9 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -382,12 +382,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): def tryNextRelease(self, media_id, manual = False, force_download = False): try: + db = get_db() - rels = fireEvent('media.with_status', ['snatched', 'done'], single = True) + rels = fireEvent('release.for_media', media_id, single = True) for rel in rels: - rel['status'] = 'ignored' - db.update(rel) + if rel.get('status') in ['snatched', 'done']: + rel['status'] = 'ignored' + db.update(rel) media = fireEvent('media.get', media_id, single = True) if media: From a3a8a820fedcf6d730b3b36b7aadd1c1dd61deb2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 16:49:44 +0200 Subject: [PATCH 21/22] release.update_status not triggered on frontend --- couchpotato/core/media/movie/_base/main.py | 3 +-- .../media/movie/_base/static/movie.actions.js | 15 +++++++++++---- .../core/media/movie/_base/static/movie.js | 14 +++++++++++--- couchpotato/core/media/movie/searcher.py | 4 +--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 1863d897..75e10c7d 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -150,8 +150,7 @@ class MovieBase(MovieTypeBase): for release in fireEvent('release.for_media', m['_id'], single = True): if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if params.get('ignore_previous', False): - release['status'] = 'ignored' - db.update(release) + fireEvent('release.update_status', m['_id'], status = 'ignored') else: fireEvent('release.delete', release['_id'], single = True) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index ff71f31d..09a998f3 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -115,8 +115,15 @@ MA.Release = new Class({ self.releases = null; if(self.options_container){ - self.options_container.destroy(); - self.options_container = null; + // Releases are currently displayed + if(self.options_container.isDisplayed()){ + self.options_container.destroy(); + self.createReleases(); + } + else { + self.options_container.destroy(); + self.options_container = null; + } } }); @@ -131,10 +138,10 @@ MA.Release = new Class({ }, - createReleases: function(){ + createReleases: function(refresh){ var self = this; - if(!self.options_container){ + if(!self.options_container || refresh){ self.options_container = new Element('div.options').grab( self.release_container = new Element('div.releases.table') ); diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index 669546b8..e2977a14 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -54,13 +54,21 @@ var Movie = new Class({ // Reload when releases have updated self.global_events['release.update_status'] = function(notification){ var data = notification.data; - if(data && self.data._id == data.movie_id){ + if(data && self.data._id == data.media_id){ if(!self.data.releases) self.data.releases = []; - self.data.releases.push({'quality': data.quality, 'status': data.status}); - self.updateReleases(); + var updated = false; + self.data.releases.each(function(release){ + if(release._id == data._id){ + release['status'] = data.status; + updated = true; + } + }); + + if(updated) + self.updateReleases(); } }; diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 4940e7c9..50b2d446 100755 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -383,13 +383,11 @@ class MovieSearcher(SearcherBase, MovieTypeBase): try: - db = get_db() rels = fireEvent('release.for_media', media_id, single = True) for rel in rels: if rel.get('status') in ['snatched', 'done']: - rel['status'] = 'ignored' - db.update(rel) + fireEvent('release.update_status', rel.get('_id'), status = 'ignored') media = fireEvent('media.get', media_id, single = True) if media: From 052d64eb396e3d4caf000df5e722c1d89162433c Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 17:45:55 +0200 Subject: [PATCH 22/22] Force restatus on ignored movies --- couchpotato/core/media/_base/media/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 4bcd820e..24b01241 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -77,6 +77,7 @@ class MediaPlugin(MediaBase): addEvent('app.load', self.addSingleListView, priority = 100) addEvent('app.load', self.addSingleCharView, priority = 100) addEvent('app.load', self.addSingleDeleteView, priority = 100) + addEvent('app.load', self.cleanupFaults) addEvent('media.get', self.get) addEvent('media.with_status', self.withStatus) @@ -87,6 +88,12 @@ class MediaPlugin(MediaBase): addEvent('media.tag', self.tag) addEvent('media.untag', self.unTag) + # Wrongly tagged media files + def cleanupFaults(self): + medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) + for media in medias: + self.restatus(media.get('_id')) + def refresh(self, id = '', **kwargs): handlers = [] ids = splitString(id)