Merge branch 'develop'

couchpotato/core/database.py

@@ -32,6 +32,7 @@ class Database(object):
 
         addEvent('database.setup.after', self.startup_compact)
         addEvent('database.setup_index', self.setupIndex)
+        addEvent('database.delete_corrupted', self.deleteCorrupted)
 
         addEvent('app.migrate', self.migrate)
         addEvent('app.after_shutdown', self.close)
@@ -147,6 +148,17 @@ class Database(object):
 
         return results
 
+    def deleteCorrupted(self, _id, traceback_error = ''):
+
+        db = self.getDB()
+
+        try:
+            log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error))
+            corrupted = db.get('id', _id, with_storage = False)
+            db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
+        except:
+            log.debug('Failed deleting corrupted: %s', traceback.format_exc())
+
     def reindex(self, **kwargs):
 
         success = True
@@ -579,7 +591,10 @@ class Database(object):
                             continue
 
                         for f in release_files:
-                            rfile = all_files[f.get('file_id')]
+                            rfile = all_files.get(f.get('file_id'))
+                            if not rfile:
+                                continue
+
                             file_type = type_by_id.get(rfile.get('type_id')).get('identifier')
 
                             if not release['files'].get(file_type):

couchpotato/core/event.py

@@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs):
 
         else:
 
-            e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock())
+            e = Event(name = name, threads = 10, exc_info = True, traceback = True)
 
             for event in events[name]:
                 e.handle(event['handler'], priority = event['priority'])

couchpotato/core/media/__init__.py

@@ -26,9 +26,9 @@ class MediaBase(Plugin):
         def onComplete():
             try:
                 media = fireEvent('media.get', media_id, single = True)
-                event_name = '%s.searcher.single' % media.get('type')
+                if media:
-
+                    event_name = '%s.searcher.single' % media.get('type')
-                fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True)
+                    fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True)
             except:
                 log.error('Failed creating onComplete: %s', traceback.format_exc())
 
@@ -39,9 +39,9 @@ class MediaBase(Plugin):
         def notifyFront():
             try:
                 media = fireEvent('media.get', media_id, single = True)
-                event_name = '%s.update' % media.get('type')
+                if media:
-
+                    event_name = '%s.update' % media.get('type')
-                fireEvent('notify.frontend', type = event_name, data = media)
+                    fireEvent('notify.frontend', type = event_name, data = media)
             except:
                 log.error('Failed creating onComplete: %s', traceback.format_exc())
 

couchpotato/core/media/_base/media/main.py

@@ -77,6 +77,7 @@ class MediaPlugin(MediaBase):
         addEvent('app.load', self.addSingleListView, priority = 100)
         addEvent('app.load', self.addSingleCharView, priority = 100)
         addEvent('app.load', self.addSingleDeleteView, priority = 100)
+        addEvent('app.load', self.cleanupFaults)
 
         addEvent('media.get', self.get)
         addEvent('media.with_status', self.withStatus)
@@ -87,6 +88,12 @@ class MediaPlugin(MediaBase):
         addEvent('media.tag', self.tag)
         addEvent('media.untag', self.unTag)
 
+    # Wrongly tagged media files
+    def cleanupFaults(self):
+        medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True)
+        for media in medias:
+            self.restatus(media.get('_id'))
+
     def refresh(self, id = '', **kwargs):
         handlers = []
         ids = splitString(id)
@@ -178,8 +185,10 @@ class MediaPlugin(MediaBase):
                             continue
 
                         yield doc
-                    except RecordNotFound:
+                    except (RecordDeleted, RecordNotFound):
                         log.debug('Record not found, skipping: %s', ms['_id'])
+                    except (ValueError, EOFError):
+                        fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0))
                 else:
                     yield ms
 
@@ -280,6 +289,10 @@ class MediaPlugin(MediaBase):
 
             media = fireEvent('media.get', media_id, single = True)
 
+            # Skip if no media has been found
+            if not media:
+                continue
+
             # Merge releases with movie dict
             medias.append(media)
 
@@ -373,7 +386,7 @@ class MediaPlugin(MediaBase):
             if x['_id'] in media_ids:
                 chars.add(x['key'])
 
-            if len(chars) == 25:
+            if len(chars) == 27:
                 break
 
         return list(chars)

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -187,11 +187,12 @@ class Base(NZBProvider, RSS):
             self.limits_reached[host] = False
             return data
         except HTTPError as e:
-            if e.response.status_code == 503:
+            sc = e.response.status_code
+            if sc in [503, 429]:
                 response = e.read().lower()
-                if 'maximum api' in response or 'download limit' in response:
+                if sc == 429 or 'maximum api' in response or 'download limit' in response:
                     if not self.limits_reached.get(host):
-                        log.error('Limit reached for newznab provider: %s', host)
+                        log.error('Limit reached / to many requests for newznab provider: %s', host)
                     self.limits_reached[host] = time.time()
                     return 'try_next'
 

couchpotato/core/media/movie/_base/main.py

@@ -150,8 +150,7 @@ class MovieBase(MovieTypeBase):
                 for release in fireEvent('release.for_media', m['_id'], single = True):
                     if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']:
                         if params.get('ignore_previous', False):
-                            release['status'] = 'ignored'
+                            fireEvent('release.update_status', m['_id'], status = 'ignored')
-                            db.update(release)
                         else:
                             fireEvent('release.delete', release['_id'], single = True)
 
@@ -179,6 +178,9 @@ class MovieBase(MovieTypeBase):
                     db.delete(rel)
 
             movie_dict = fireEvent('media.get', m['_id'], single = True)
+            if not movie_dict:
+                log.debug('Failed adding media, can\'t find it anymore')
+                return False
 
             if do_search and search_after:
                 onComplete = self.createOnComplete(m['_id'])
@@ -268,6 +270,10 @@ class MovieBase(MovieTypeBase):
         if self.shuttingDown():
             return
 
+        lock_key = 'media.get.%s' % media_id if media_id else identifier
+        self.acquireLock(lock_key)
+
+        media = {}
         try:
             db = get_db()
 
@@ -316,11 +322,11 @@ class MovieBase(MovieTypeBase):
             self.getPoster(media, image_urls)
 
             db.update(media)
-            return media
         except:
             log.error('Failed update media: %s', traceback.format_exc())
 
-        return {}
+        self.releaseLock(lock_key)
+        return media
 
     def updateReleaseDate(self, media_id):
         """

couchpotato/core/media/movie/_base/static/movie.actions.js

@@ -115,8 +115,15 @@ MA.Release = new Class({
 
 			self.releases = null;
 			if(self.options_container){
-				self.options_container.destroy();
+				// Releases are currently displayed
-				self.options_container = null;
+				if(self.options_container.isDisplayed()){
+					self.options_container.destroy();
+					self.createReleases();
+				}
+				else {
+					self.options_container.destroy();
+					self.options_container = null;
+				}
 			}
 		});
 
@@ -131,10 +138,10 @@ MA.Release = new Class({
 
 	},
 
-	createReleases: function(){
+	createReleases: function(refresh){
 		var self = this;
 
-		if(!self.options_container){
+		if(!self.options_container || refresh){
 			self.options_container = new Element('div.options').grab(
 				self.release_container = new Element('div.releases.table')
 			);

couchpotato/core/media/movie/_base/static/movie.js

@@ -54,13 +54,21 @@ var Movie = new Class({
 		// Reload when releases have updated
 		self.global_events['release.update_status'] = function(notification){
 			var data = notification.data;
-			if(data && self.data._id == data.movie_id){
+			if(data && self.data._id == data.media_id){
 
 				if(!self.data.releases)
 					self.data.releases = [];
 
-				self.data.releases.push({'quality': data.quality, 'status': data.status});
+				var updated = false;
-				self.updateReleases();
+				self.data.releases.each(function(release){
+					if(release._id == data._id){
+						release['status'] = data.status;
+						updated = true;
+					}
+				});
+
+				if(updated)
+					self.updateReleases();
 			}
 		};
 

couchpotato/core/media/movie/providers/torrent/iptorrents.py

@@ -13,7 +13,7 @@ class IPTorrents(MovieProvider, Base):
         ([87], ['3d']),
         ([48], ['720p', '1080p', 'bd50']),
         ([72], ['cam', 'ts', 'tc', 'r5', 'scr']),
-        ([7], ['dvdrip', 'brrip']),
+        ([7,48], ['dvdrip', 'brrip']),
         ([6], ['dvdr']),
     ]
 

couchpotato/core/media/movie/searcher.py

@@ -89,6 +89,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
             for media_id in medias:
 
                 media = fireEvent('media.get', media_id, single = True)
+                if not media: continue
 
                 try:
                     self.single(media, search_protocols, manual = manual)
@@ -381,16 +382,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
     def tryNextRelease(self, media_id, manual = False, force_download = False):
 
         try:
-            db = get_db()
+
-            rels = fireEvent('media.with_status', ['snatched', 'done'], single = True)
+            rels = fireEvent('release.for_media', media_id, single = True)
 
             for rel in rels:
-                rel['status'] = 'ignored'
+                if rel.get('status') in ['snatched', 'done']:
-                db.update(rel)
+                    fireEvent('release.update_status', rel.get('_id'), status = 'ignored')
 
-            movie_dict = fireEvent('media.get', media_id, single = True)
+            media = fireEvent('media.get', media_id, single = True)
-            log.info('Trying next release for: %s', getTitle(movie_dict))
+            if media:
-            self.single(movie_dict, manual = manual, force_download = force_download)
+                log.info('Trying next release for: %s', getTitle(media))
+                self.single(media, manual = manual, force_download = force_download)
 
             return True
 

couchpotato/core/notifications/core/main.py

@@ -3,6 +3,7 @@ import threading
 import time
 import traceback
 import uuid
+from CodernityDB.database import RecordDeleted
 
 from couchpotato import get_db
 from couchpotato.api import addApiView, addNonBlockApiView
@@ -155,9 +156,14 @@ class CoreNotifier(Notification):
             n = {
                 '_t': 'notification',
                 'time': int(time.time()),
-                'message': toUnicode(message),
+                'message': toUnicode(message)
-                'data': data
             }
+
+            if data.get('sticky'):
+                n['sticky'] = True
+            if data.get('important'):
+                n['important'] = True
+
             db.insert(n)
 
             self.frontend(type = listener, data = n)
@@ -265,11 +271,16 @@ class CoreNotifier(Notification):
         if init:
             db = get_db()
 
-            notifications = db.all('notification', with_doc = True)
+            notifications = db.all('notification')
 
             for n in notifications:
-                if n['doc'].get('time') > (time.time() - 604800):
+
-                    messages.append(n['doc'])
+                try:
+                    doc = db.get('id', n.get('_id'))
+                    if doc.get('time') > (time.time() - 604800):
+                        messages.append(doc)
+                except RecordDeleted:
+                    pass
 
         return {
             'success': True,

couchpotato/core/notifications/core/static/notification.js

@@ -50,7 +50,7 @@ var NotificationBase = new Class({
 		, 'top');
 		self.notifications.include(result);
 
-		if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){
+		if((result.important !== undefined || result.sticky !== undefined) && !result.read){
 			var sticky = true;
 			App.trigger('message', [result.message, sticky, result])
 		}
@@ -72,7 +72,7 @@ var NotificationBase = new Class({
 
 		if(!force_ids) {
 			var rn = self.notifications.filter(function(n){
-				return !n.read && n.data.important === undefined
+				return !n.read && n.important === undefined
 			});
 
 			var ids = [];

couchpotato/core/plugins/automation.py

@@ -46,7 +46,8 @@ class Automation(Plugin):
                 break
 
             movie_dict = fireEvent('media.get', movie_id, single = True)
-            fireEvent('movie.searcher.single', movie_dict)
+            if movie_dict:
+                fireEvent('movie.searcher.single', movie_dict)
 
         return True
 

couchpotato/core/plugins/base.py

@@ -1,3 +1,4 @@
+import threading
 from urllib import quote
 from urlparse import urlparse
 import glob
@@ -35,6 +36,8 @@ class Plugin(object):
     _needs_shutdown = False
     _running = None
 
+    _locks = {}
+
     user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0'
     http_last_use = {}
     http_time_between_calls = 0
@@ -198,6 +201,7 @@ class Plugin(object):
                 del self.http_failed_disabled[host]
 
         self.wait(host)
+        status_code = None
         try:
 
             kwargs = {
@@ -212,6 +216,7 @@ class Plugin(object):
             log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data'))
             response = r.request(method, url, **kwargs)
 
+            status_code = response.status_code
             if response.status_code == requests.codes.ok:
                 data = response.content
             else:
@@ -224,6 +229,11 @@ class Plugin(object):
 
             # Save failed requests by hosts
             try:
+
+                # To many requests
+                if status_code == 429:
+                    self.http_failed_request[host] = time.time()
+
                 if not self.http_failed_request.get(host):
                     self.http_failed_request[host] = 1
                 else:
@@ -255,7 +265,7 @@ class Plugin(object):
 
             if wait > 0:
                 log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
-                time.sleep(wait)
+                time.sleep(min(wait, 30))
 
     def beforeCall(self, handler):
         self.isRunning('%s.%s' % (self.getName(), handler.__name__))
@@ -393,3 +403,19 @@ class Plugin(object):
 
     def isEnabled(self):
         return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None
+
+    def acquireLock(self, key):
+
+        lock = self._locks.get(key)
+        if not lock:
+            self._locks[key] = threading.RLock()
+
+        log.debug('Acquiring lock: %s', key)
+        self._locks.get(key).acquire()
+
+    def releaseLock(self, key):
+
+        lock = self._locks.get(key)
+        if lock:
+            log.debug('Releasing lock: %s', key)
+            self._locks.get(key).release()

couchpotato/core/plugins/dashboard.py

@@ -1,5 +1,6 @@
 import random as rndm
 import time
+from CodernityDB.database import RecordDeleted
 
 from couchpotato import get_db
 from couchpotato.api import addApiView
@@ -58,7 +59,11 @@ class Dashboard(Plugin):
                 rndm.shuffle(active_ids)
 
             for media_id in active_ids:
-                media = db.get('id', media_id)
+                try:
+                    media = db.get('id', media_id)
+                except RecordDeleted:
+                    log.debug('Record already deleted: %s', media_id)
+                    continue
 
                 pp = profile_pre.get(media.get('profile_id'))
                 if not pp: continue

couchpotato/core/plugins/manage.py

@@ -234,7 +234,8 @@ class Manage(Plugin):
             total = self.in_progress[folder]['total']
             movie_dict = fireEvent('media.get', identifier, single = True)
 
-            fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict))
+            if movie_dict:
+                fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict))
 
         return afterUpdate
 

couchpotato/core/plugins/quality/main.py

@@ -26,10 +26,10 @@ class QualityPlugin(Plugin):
         {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
         {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']},
         {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
-        {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
+        {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]},
         {'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
         {'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
-        {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
+        {'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []},
         {'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]},
         {'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]},
         {'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]},
@@ -293,10 +293,6 @@ class QualityPlugin(Plugin):
                     log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
                     score += points.get(tag_type)
 
-            if list(set(qualities) & set(words)):
-                log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
-                score += points.get(tag_type)
-
         # Check extention
         for ext in quality.get('ext', []):
             if ext == extension:
@@ -486,6 +482,9 @@ class QualityPlugin(Plugin):
             'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}},
             'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}},
             'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
+            'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
+            'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'},
+            'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'},
         }
 
         correct = 0

couchpotato/core/plugins/release/main.py

@@ -70,38 +70,53 @@ class Release(Plugin):
         db = get_db()
 
         # Get (and remove) parentless releases
-        releases = db.all('release', with_doc = True)
+        releases = db.all('release', with_doc = False)
         media_exist = []
+        reindex = 0
         for release in releases:
             if release.get('key') in media_exist:
                 continue
 
             try:
+
+                try:
+                    doc = db.get('id', release.get('_id'))
+                except RecordDeleted:
+                    reindex += 1
+                    continue
+
                 db.get('id', release.get('key'))
                 media_exist.append(release.get('key'))
 
                 try:
-                    if release['doc'].get('status') == 'ignore':
+                    if doc.get('status') == 'ignore':
-                        release['doc']['status'] = 'ignored'
+                        doc['status'] = 'ignored'
-                        db.update(release['doc'])
+                        db.update(doc)
                 except:
                     log.error('Failed fixing mis-status tag: %s', traceback.format_exc())
+            except ValueError:
+                fireEvent('database.delete_corrupted', release.get('key'), traceback_error = traceback.format_exc(0))
+                reindex += 1
             except RecordDeleted:
-                db.delete(release['doc'])
+                db.delete(doc)
-                log.debug('Deleted orphaned release: %s', release['doc'])
+                log.debug('Deleted orphaned release: %s', doc)
+                reindex += 1
             except:
                 log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc())
 
+        if reindex > 0:
+            db.reindex()
+
         del media_exist
 
         # get movies last_edit more than a week ago
-        medias = fireEvent('media.with_status', ['done','active'], single = True)
+        medias = fireEvent('media.with_status', ['done', 'active'], single = True)
 
         for media in medias:
             if media.get('last_edit', 0) > (now - week):
                 continue
 
-            for rel in fireEvent('release.for_media', media['_id'], single = True):
+            for rel in self.forMedia(media['_id']):
 
                 # Remove all available releases
                 if rel['status'] in ['available']:
@@ -528,11 +543,15 @@ class Release(Plugin):
     def forMedia(self, media_id):
 
         db = get_db()
-        raw_releases = list(db.get_many('release', media_id, with_doc = True))
+        raw_releases = db.get_many('release', media_id)
 
         releases = []
         for r in raw_releases:
-            releases.append(r['doc'])
+            try:
+                doc = db.get('id', r.get('_id'))
+                releases.append(doc)
+            except RecordDeleted:
+                pass
 
         releases = sorted(releases, key = lambda k: k.get('info', {}).get('score', 0), reverse = True)
 

couchpotato/runner.py

@@ -19,6 +19,7 @@ from couchpotato.core.event import fireEventAsync, fireEvent
 from couchpotato.core.helpers.encoding import sp
 from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace
 import requests
+from requests.packages.urllib3 import disable_warnings
 from tornado.httpserver import HTTPServer
 from tornado.web import Application, StaticFileHandler, RedirectHandler
 
@@ -174,6 +175,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     for logger_name in ['gntp']:
         logging.getLogger(logger_name).setLevel(logging.WARNING)
 
+    # Disable SSL warning
+    disable_warnings()
+
     # Use reloader
     reloader = debug is True and development and not Env.get('desktop') and not options.daemon