dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Ruud
2014-09-18 17:48:04 +02:00
parent d6a264aaed 052d64eb39
commit 37c6bc7612
19 changed files with 177 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
couchpotato/core/database.py
View File

@@ -32,6 +32,7 @@ class Database(object):
addEvent('database.setup.after', self.startup_compact)
addEvent('database.setup_index', self.setupIndex)
addEvent('database.delete_corrupted', self.deleteCorrupted)
addEvent('app.migrate', self.migrate)
addEvent('app.after_shutdown', self.close)
@@ -147,6 +148,17 @@ class Database(object):
return results
def deleteCorrupted(self, _id, traceback_error = ''):
db = self.getDB()
try:
log.debug('Deleted corrupted document "%s": %s', (_id, traceback_error))
corrupted = db.get('id', _id, with_storage = False)
db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
except:
log.debug('Failed deleting corrupted: %s', traceback.format_exc())
def reindex(self, **kwargs):
success = True
@@ -579,7 +591,10 @@ class Database(object):
continue
for f in release_files:
rfile = all_files[f.get('file_id')]
rfile = all_files.get(f.get('file_id'))
if not rfile:
continue
file_type = type_by_id.get(rfile.get('type_id')).get('identifier')
if not release['files'].get(file_type):

2
couchpotato/core/event.py
View File

@@ -90,7 +90,7 @@ def fireEvent(name, *args, **kwargs):
else:
e = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock())
e = Event(name = name, threads = 10, exc_info = True, traceback = True)
for event in events[name]:
e.handle(event['handler'], priority = event['priority'])

12
couchpotato/core/media/__init__.py
View File

@@ -26,9 +26,9 @@ class MediaBase(Plugin):
def onComplete():
try:
media = fireEvent('media.get', media_id, single = True)
event_name = '%s.searcher.single' % media.get('type')
fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True)
if media:
event_name = '%s.searcher.single' % media.get('type')
fireEventAsync(event_name, media, on_complete = self.createNotifyFront(media_id), manual = True)
except:
log.error('Failed creating onComplete: %s', traceback.format_exc())
@@ -39,9 +39,9 @@ class MediaBase(Plugin):
def notifyFront():
try:
media = fireEvent('media.get', media_id, single = True)
event_name = '%s.update' % media.get('type')
fireEvent('notify.frontend', type = event_name, data = media)
if media:
event_name = '%s.update' % media.get('type')
fireEvent('notify.frontend', type = event_name, data = media)
except:
log.error('Failed creating onComplete: %s', traceback.format_exc())

17
couchpotato/core/media/_base/media/main.py
View File

@@ -77,6 +77,7 @@ class MediaPlugin(MediaBase):
addEvent('app.load', self.addSingleListView, priority = 100)
addEvent('app.load', self.addSingleCharView, priority = 100)
addEvent('app.load', self.addSingleDeleteView, priority = 100)
addEvent('app.load', self.cleanupFaults)
addEvent('media.get', self.get)
addEvent('media.with_status', self.withStatus)
@@ -87,6 +88,12 @@ class MediaPlugin(MediaBase):
addEvent('media.tag', self.tag)
addEvent('media.untag', self.unTag)
# Wrongly tagged media files
def cleanupFaults(self):
medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True)
for media in medias:
self.restatus(media.get('_id'))
def refresh(self, id = '', **kwargs):
handlers = []
ids = splitString(id)
@@ -178,8 +185,10 @@ class MediaPlugin(MediaBase):
continue
yield doc
except RecordNotFound:
except (RecordDeleted, RecordNotFound):
log.debug('Record not found, skipping: %s', ms['_id'])
except (ValueError, EOFError):
fireEvent('database.delete_corrupted', ms.get('_id'), traceback_error = traceback.format_exc(0))
else:
yield ms
@@ -280,6 +289,10 @@ class MediaPlugin(MediaBase):
media = fireEvent('media.get', media_id, single = True)
# Skip if no media has been found
if not media:
continue
# Merge releases with movie dict
medias.append(media)
@@ -373,7 +386,7 @@ class MediaPlugin(MediaBase):
if x['_id'] in media_ids:
chars.add(x['key'])
if len(chars) == 25:
if len(chars) == 27:
break
return list(chars)

7
couchpotato/core/media/_base/providers/nzb/newznab.py
View File

@@ -187,11 +187,12 @@ class Base(NZBProvider, RSS):
self.limits_reached[host] = False
return data
except HTTPError as e:
if e.response.status_code == 503:
sc = e.response.status_code
if sc in [503, 429]:
response = e.read().lower()
if 'maximum api' in response or 'download limit' in response:
if sc == 429 or 'maximum api' in response or 'download limit' in response:
if not self.limits_reached.get(host):
log.error('Limit reached for newznab provider: %s', host)
log.error('Limit reached / to many requests for newznab provider: %s', host)
self.limits_reached[host] = time.time()
return 'try_next'

14
couchpotato/core/media/movie/_base/main.py
View File

@@ -150,8 +150,7 @@ class MovieBase(MovieTypeBase):
for release in fireEvent('release.for_media', m['_id'], single = True):
if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']:
if params.get('ignore_previous', False):
release['status'] = 'ignored'
db.update(release)
fireEvent('release.update_status', m['_id'], status = 'ignored')
else:
fireEvent('release.delete', release['_id'], single = True)
@@ -179,6 +178,9 @@ class MovieBase(MovieTypeBase):
db.delete(rel)
movie_dict = fireEvent('media.get', m['_id'], single = True)
if not movie_dict:
log.debug('Failed adding media, can\'t find it anymore')
return False
if do_search and search_after:
onComplete = self.createOnComplete(m['_id'])
@@ -268,6 +270,10 @@ class MovieBase(MovieTypeBase):
if self.shuttingDown():
return
lock_key = 'media.get.%s' % media_id if media_id else identifier
self.acquireLock(lock_key)
media = {}
try:
db = get_db()
@@ -316,11 +322,11 @@ class MovieBase(MovieTypeBase):
self.getPoster(media, image_urls)
db.update(media)
return media
except:
log.error('Failed update media: %s', traceback.format_exc())
return {}
self.releaseLock(lock_key)
return media
def updateReleaseDate(self, media_id):
"""

15
couchpotato/core/media/movie/_base/static/movie.actions.js
View File

@@ -115,8 +115,15 @@ MA.Release = new Class({
self.releases = null;
if(self.options_container){
self.options_container.destroy();
self.options_container = null;
// Releases are currently displayed
if(self.options_container.isDisplayed()){
self.options_container.destroy();
self.createReleases();
}
else {
self.options_container.destroy();
self.options_container = null;
}
}
});
@@ -131,10 +138,10 @@ MA.Release = new Class({
},
createReleases: function(){
createReleases: function(refresh){
var self = this;
if(!self.options_container){
if(!self.options_container || refresh){
self.options_container = new Element('div.options').grab(
self.release_container = new Element('div.releases.table')
);

14
couchpotato/core/media/movie/_base/static/movie.js
View File

@@ -54,13 +54,21 @@ var Movie = new Class({
// Reload when releases have updated
self.global_events['release.update_status'] = function(notification){
var data = notification.data;
if(data && self.data._id == data.movie_id){
if(data && self.data._id == data.media_id){
if(!self.data.releases)
self.data.releases = [];
self.data.releases.push({'quality': data.quality, 'status': data.status});
self.updateReleases();
var updated = false;
self.data.releases.each(function(release){
if(release._id == data._id){
release['status'] = data.status;
updated = true;
}
});
if(updated)
self.updateReleases();
}
};

2
couchpotato/core/media/movie/providers/torrent/iptorrents.py
View File

@@ -13,7 +13,7 @@ class IPTorrents(MovieProvider, Base):
([87], ['3d']),
([48], ['720p', '1080p', 'bd50']),
([72], ['cam', 'ts', 'tc', 'r5', 'scr']),
([7], ['dvdrip', 'brrip']),
([7,48], ['dvdrip', 'brrip']),
([6], ['dvdr']),
]

16
couchpotato/core/media/movie/searcher.py
View File

@@ -89,6 +89,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
for media_id in medias:
media = fireEvent('media.get', media_id, single = True)
if not media: continue
try:
self.single(media, search_protocols, manual = manual)
@@ -381,16 +382,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
def tryNextRelease(self, media_id, manual = False, force_download = False):
try:
db = get_db()
rels = fireEvent('media.with_status', ['snatched', 'done'], single = True)
rels = fireEvent('release.for_media', media_id, single = True)
for rel in rels:
rel['status'] = 'ignored'
db.update(rel)
if rel.get('status') in ['snatched', 'done']:
fireEvent('release.update_status', rel.get('_id'), status = 'ignored')
movie_dict = fireEvent('media.get', media_id, single = True)
log.info('Trying next release for: %s', getTitle(movie_dict))
self.single(movie_dict, manual = manual, force_download = force_download)
media = fireEvent('media.get', media_id, single = True)
if media:
log.info('Trying next release for: %s', getTitle(media))
self.single(media, manual = manual, force_download = force_download)
return True

21
couchpotato/core/notifications/core/main.py
View File

@@ -3,6 +3,7 @@ import threading
import time
import traceback
import uuid
from CodernityDB.database import RecordDeleted
from couchpotato import get_db
from couchpotato.api import addApiView, addNonBlockApiView
@@ -155,9 +156,14 @@ class CoreNotifier(Notification):
n = {
'_t': 'notification',
'time': int(time.time()),
'message': toUnicode(message),
'data': data
'message': toUnicode(message)
}
if data.get('sticky'):
n['sticky'] = True
if data.get('important'):
n['important'] = True
db.insert(n)
self.frontend(type = listener, data = n)
@@ -265,11 +271,16 @@ class CoreNotifier(Notification):
if init:
db = get_db()
notifications = db.all('notification', with_doc = True)
notifications = db.all('notification')
for n in notifications:
if n['doc'].get('time') > (time.time() - 604800):
messages.append(n['doc'])
try:
doc = db.get('id', n.get('_id'))
if doc.get('time') > (time.time() - 604800):
messages.append(doc)
except RecordDeleted:
pass
return {
'success': True,

4
couchpotato/core/notifications/core/static/notification.js
View File

@@ -50,7 +50,7 @@ var NotificationBase = new Class({
, 'top');
self.notifications.include(result);
if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){
if((result.important !== undefined || result.sticky !== undefined) && !result.read){
var sticky = true;
App.trigger('message', [result.message, sticky, result])
}
@@ -72,7 +72,7 @@ var NotificationBase = new Class({
if(!force_ids) {
var rn = self.notifications.filter(function(n){
return !n.read && n.data.important === undefined
return !n.read && n.important === undefined
});
var ids = [];

3
couchpotato/core/plugins/automation.py
View File

@@ -46,7 +46,8 @@ class Automation(Plugin):
break
movie_dict = fireEvent('media.get', movie_id, single = True)
fireEvent('movie.searcher.single', movie_dict)
if movie_dict:
fireEvent('movie.searcher.single', movie_dict)
return True

28
couchpotato/core/plugins/base.py
View File

@@ -1,3 +1,4 @@
import threading
from urllib import quote
from urlparse import urlparse
import glob
@@ -35,6 +36,8 @@ class Plugin(object):
_needs_shutdown = False
_running = None
_locks = {}
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0'
http_last_use = {}
http_time_between_calls = 0
@@ -198,6 +201,7 @@ class Plugin(object):
del self.http_failed_disabled[host]
self.wait(host)
status_code = None
try:
kwargs = {
@@ -212,6 +216,7 @@ class Plugin(object):
log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data'))
response = r.request(method, url, **kwargs)
status_code = response.status_code
if response.status_code == requests.codes.ok:
data = response.content
else:
@@ -224,6 +229,11 @@ class Plugin(object):
# Save failed requests by hosts
try:
# To many requests
if status_code == 429:
self.http_failed_request[host] = time.time()
if not self.http_failed_request.get(host):
self.http_failed_request[host] = 1
else:
@@ -255,7 +265,7 @@ class Plugin(object):
if wait > 0:
log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
time.sleep(wait)
time.sleep(min(wait, 30))
def beforeCall(self, handler):
self.isRunning('%s.%s' % (self.getName(), handler.__name__))
@@ -393,3 +403,19 @@ class Plugin(object):
def isEnabled(self):
return self.conf(self.enabled_option) or self.conf(self.enabled_option) is None
def acquireLock(self, key):
lock = self._locks.get(key)
if not lock:
self._locks[key] = threading.RLock()
log.debug('Acquiring lock: %s', key)
self._locks.get(key).acquire()
def releaseLock(self, key):
lock = self._locks.get(key)
if lock:
log.debug('Releasing lock: %s', key)
self._locks.get(key).release()

7
couchpotato/core/plugins/dashboard.py
View File

@@ -1,5 +1,6 @@
import random as rndm
import time
from CodernityDB.database import RecordDeleted
from couchpotato import get_db
from couchpotato.api import addApiView
@@ -58,7 +59,11 @@ class Dashboard(Plugin):
rndm.shuffle(active_ids)
for media_id in active_ids:
media = db.get('id', media_id)
try:
media = db.get('id', media_id)
except RecordDeleted:
log.debug('Record already deleted: %s', media_id)
continue
pp = profile_pre.get(media.get('profile_id'))
if not pp: continue

3
couchpotato/core/plugins/manage.py
View File

@@ -234,7 +234,8 @@ class Manage(Plugin):
total = self.in_progress[folder]['total']
movie_dict = fireEvent('media.get', identifier, single = True)
fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict))
if movie_dict:
fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict))
return afterUpdate

11
couchpotato/core/plugins/quality/main.py
View File

@@ -26,10 +26,10 @@ class QualityPlugin(Plugin):
{'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'median_size': 40000, 'label': 'BR-Disk', 'alternative': ['bd25', ('br', 'disk')], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
{'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'median_size': 10000, 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts'], 'tags': ['m2ts', 'x264', 'h264']},
{'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'median_size': 5500, 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
{'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip')], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
{'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'median_size': 2000, 'label': 'BR-Rip', 'alternative': ['bdrip', ('br', 'rip'), 'hdtv', 'hdrip'], 'allow': ['720p', '1080p'], 'ext':['mp4', 'avi'], 'tags': ['webdl', ('web', 'dl')]},
{'identifier': 'dvdr', 'size': (3000, 10000), 'median_size': 4500, 'label': 'DVD-R', 'alternative': ['br2dvd', ('dvd', 'r')], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
{'identifier': 'dvdrip', 'size': (600, 2400), 'median_size': 1500, 'label': 'DVD-Rip', 'width': 720, 'alternative': [('dvd', 'rip')], 'allow': [], 'ext':['avi'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
{'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
{'identifier': 'scr', 'size': (600, 1600), 'median_size': 700, 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr', 'webrip', ('web', 'rip')], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': []},
{'identifier': 'r5', 'size': (600, 1000), 'median_size': 700, 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr', '720p'], 'ext':[]},
{'identifier': 'tc', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': ['720p'], 'ext':[]},
{'identifier': 'ts', 'size': (600, 1000), 'median_size': 700, 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': ['720p'], 'ext':[]},
@@ -293,10 +293,6 @@ class QualityPlugin(Plugin):
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
score += points.get(tag_type)
if list(set(qualities) & set(words)):
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
score += points.get(tag_type)
# Check extention
for ext in quality.get('ext', []):
if ext == extension:
@@ -486,6 +482,9 @@ class QualityPlugin(Plugin):
'Movie Name (2014).mkv': {'size': 4500, 'quality': '720p', 'extra': {'titles': ['Movie Name 2014 720p Bluray']}},
'Movie Name (2015).mkv': {'size': 500, 'quality': '1080p', 'extra': {'resolution_width': 1920}},
'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
'Movie Name (2015).mp4': {'size': 6500, 'quality': 'brrip'},
'Movie Name.2014.720p Web-Dl Aac2.0 h264-ReleaseGroup': {'size': 3800, 'quality': 'brrip'},
'Movie Name.2014.720p.WEBRip.x264.AC3-ReleaseGroup': {'size': 3000, 'quality': 'scr'},
}
correct = 0

39
couchpotato/core/plugins/release/main.py
View File

@@ -70,38 +70,53 @@ class Release(Plugin):
db = get_db()
# Get (and remove) parentless releases
releases = db.all('release', with_doc = True)
releases = db.all('release', with_doc = False)
media_exist = []
reindex = 0
for release in releases:
if release.get('key') in media_exist:
continue
try:
try:
doc = db.get('id', release.get('_id'))
except RecordDeleted:
reindex += 1
continue
db.get('id', release.get('key'))
media_exist.append(release.get('key'))
try:
if release['doc'].get('status') == 'ignore':
release['doc']['status'] = 'ignored'
db.update(release['doc'])
if doc.get('status') == 'ignore':
doc['status'] = 'ignored'
db.update(doc)
except:
log.error('Failed fixing mis-status tag: %s', traceback.format_exc())
except ValueError:
fireEvent('database.delete_corrupted', release.get('key'), traceback_error = traceback.format_exc(0))
reindex += 1
except RecordDeleted:
db.delete(release['doc'])
log.debug('Deleted orphaned release: %s', release['doc'])
db.delete(doc)
log.debug('Deleted orphaned release: %s', doc)
reindex += 1
except:
log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc())
if reindex > 0:
db.reindex()
del media_exist
# get movies last_edit more than a week ago
medias = fireEvent('media.with_status', ['done','active'], single = True)
medias = fireEvent('media.with_status', ['done', 'active'], single = True)
for media in medias:
if media.get('last_edit', 0) > (now - week):
continue
for rel in fireEvent('release.for_media', media['_id'], single = True):
for rel in self.forMedia(media['_id']):
# Remove all available releases
if rel['status'] in ['available']:
@@ -528,11 +543,15 @@ class Release(Plugin):
def forMedia(self, media_id):
db = get_db()
raw_releases = list(db.get_many('release', media_id, with_doc = True))
raw_releases = db.get_many('release', media_id)
releases = []
for r in raw_releases:
releases.append(r['doc'])
try:
doc = db.get('id', r.get('_id'))
releases.append(doc)
except RecordDeleted:
pass
releases = sorted(releases, key = lambda k: k.get('info', {}).get('score', 0), reverse = True)

4
couchpotato/runner.py
View File

@@ -19,6 +19,7 @@ from couchpotato.core.event import fireEventAsync, fireEvent
from couchpotato.core.helpers.encoding import sp
from couchpotato.core.helpers.variable import getDataDir, tryInt, getFreeSpace
import requests
from requests.packages.urllib3 import disable_warnings
from tornado.httpserver import HTTPServer
from tornado.web import Application, StaticFileHandler, RedirectHandler
@@ -174,6 +175,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon