From 1dda7edf1cda966a8cfc98ce8145586402ec1215 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 19 Jan 2014 22:45:37 +1300 Subject: [PATCH 01/42] Fixed bug when parsing torrents page in IPT provider --- couchpotato/core/providers/torrent/iptorrents/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/providers/torrent/iptorrents/main.py b/couchpotato/core/providers/torrent/iptorrents/main.py index a02b97f2..6d41c090 100644 --- a/couchpotato/core/providers/torrent/iptorrents/main.py +++ b/couchpotato/core/providers/torrent/iptorrents/main.py @@ -99,7 +99,8 @@ class IPTorrents(TorrentProvider): result = {} for x, col in enumerate(entries[0].find_all('th')): - key = toSafeString(col.text).strip().lower() + name = col.text or col.find('img')['title'] + key = toSafeString(name).strip().lower() if not key: continue From 8217fecb33eade04b6d25603f9156b2f9ea55325 Mon Sep 17 00:00:00 2001 From: Sami Haahtinen Date: Sun, 19 Jan 2014 12:01:35 +0200 Subject: [PATCH 02/42] Make Log messages pasteable --- couchpotato/core/plugins/log/static/log.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index e9e4af05..159bfeaa 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -73,10 +73,10 @@ Page.Log = new Class({ .replace(/\u001b\[31m/gi, '') .replace(/\u001b\[36m/gi, '') .replace(/\u001b\[33m/gi, '') - .replace(/\u001b\[0m\n/gi, '') + .replace(/\u001b\[0m\n/gi, '
') .replace(/\u001b\[0m/gi, '') - return '' + text + ''; + return '
' + text + '
'; } -}) \ No newline at end of file +}) From 1f941a5105398856210421814900dbb85116f7f4 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 19 Jan 2014 21:58:13 +1300 Subject: [PATCH 03/42] Ensure files returned from rTorrent are absolute and inside the torrent directory. --- couchpotato/core/downloaders/rtorrent/main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 50276314..853899cb 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -173,9 +173,16 @@ class rTorrent(Downloader): for torrent in torrents: if torrent.info_hash in ids: + torrent_directory = os.path.normpath(torrent.directory) torrent_files = [] - for file_item in torrent.get_files(): - torrent_files.append(sp(os.path.join(torrent.directory, file_item.path))) + + for file in torrent.get_files(): + if not os.path.normpath(file.path).startswith(torrent_directory): + file_path = os.path.join(torrent_directory, file.path.lstrip('/')) + else: + file_path = file.path + + torrent_files.append(sp(file_path)) status = 'busy' if torrent.complete: From 799299c7ccc63e02bd958e0562a251814a094cef Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 19 Jan 2014 19:15:58 +0100 Subject: [PATCH 04/42] Code cleanup --- CouchPotato.py | 8 +-- couchpotato/__init__.py | 28 +++++----- couchpotato/api.py | 10 +++- couchpotato/core/_base/_core/__init__.py | 1 + couchpotato/core/_base/_core/main.py | 2 +- .../core/_base/clientscript/__init__.py | 1 + couchpotato/core/_base/clientscript/main.py | 6 +-- couchpotato/core/_base/desktop/__init__.py | 1 + couchpotato/core/_base/scheduler/__init__.py | 1 + couchpotato/core/_base/updater/__init__.py | 1 + couchpotato/core/_base/updater/main.py | 24 ++++----- .../core/downloaders/blackhole/__init__.py | 1 + .../core/downloaders/deluge/__init__.py | 1 + couchpotato/core/downloaders/deluge/main.py | 7 +-- .../core/downloaders/nzbget/__init__.py | 1 + couchpotato/core/downloaders/nzbget/main.py | 10 ++-- .../core/downloaders/nzbvortex/__init__.py | 1 + .../core/downloaders/nzbvortex/main.py | 10 ++-- .../core/downloaders/pneumatic/__init__.py | 1 + .../core/downloaders/pneumatic/main.py | 12 ++--- .../core/downloaders/rtorrent/__init__.py | 1 + couchpotato/core/downloaders/rtorrent/main.py | 1 - .../core/downloaders/sabnzbd/__init__.py | 1 + couchpotato/core/downloaders/sabnzbd/main.py | 8 +-- .../core/downloaders/synology/__init__.py | 1 + couchpotato/core/downloaders/synology/main.py | 1 + .../core/downloaders/transmission/__init__.py | 1 + .../core/downloaders/transmission/main.py | 2 +- .../core/downloaders/utorrent/__init__.py | 1 + couchpotato/core/downloaders/utorrent/main.py | 8 +-- couchpotato/core/event.py | 28 ++++++---- couchpotato/core/helpers/encoding.py | 14 +++-- couchpotato/core/helpers/request.py | 1 + couchpotato/core/helpers/rss.py | 3 +- couchpotato/core/helpers/variable.py | 44 ++++++++++++--- couchpotato/core/logger.py | 3 +- couchpotato/core/media/__init__.py | 4 +- .../core/media/_base/media/__init__.py | 1 + couchpotato/core/media/_base/media/main.py | 3 +- .../core/media/_base/search/__init__.py | 1 + .../core/media/_base/searcher/__init__.py | 1 + couchpotato/core/media/_base/searcher/base.py | 4 +- couchpotato/core/media/_base/searcher/main.py | 4 +- .../core/media/movie/_base/__init__.py | 1 + couchpotato/core/media/movie/_base/main.py | 1 - .../media/movie/library/movie/__init__.py | 1 + .../core/media/movie/library/movie/main.py | 2 +- .../core/media/movie/searcher/__init__.py | 1 + couchpotato/core/media/movie/searcher/main.py | 4 +- .../core/media/movie/suggestion/__init__.py | 1 + .../core/media/movie/suggestion/main.py | 2 +- .../migration/versions/002_Movie_category.py | 1 + .../core/notifications/boxcar/__init__.py | 1 + .../core/notifications/core/__init__.py | 1 + couchpotato/core/notifications/core/main.py | 1 - .../core/notifications/email/__init__.py | 3 +- couchpotato/core/notifications/email/main.py | 2 +- .../core/notifications/growl/__init__.py | 1 + couchpotato/core/notifications/growl/main.py | 2 +- .../core/notifications/nmj/__init__.py | 1 + couchpotato/core/notifications/nmj/main.py | 7 ++- .../notifications/notifymyandroid/__init__.py | 1 + .../core/notifications/notifymywp/__init__.py | 1 + .../core/notifications/notifymywp/main.py | 3 +- .../core/notifications/plex/__init__.py | 1 + couchpotato/core/notifications/plex/main.py | 7 +-- .../core/notifications/prowl/__init__.py | 1 + couchpotato/core/notifications/prowl/main.py | 2 +- .../core/notifications/pushalot/__init__.py | 1 + .../core/notifications/pushalot/main.py | 1 + .../core/notifications/pushbullet/__init__.py | 1 + .../core/notifications/pushover/__init__.py | 1 + .../core/notifications/pushover/main.py | 6 +-- .../core/notifications/synoindex/__init__.py | 1 + .../core/notifications/synoindex/main.py | 2 +- .../core/notifications/toasty/__init__.py | 1 + couchpotato/core/notifications/toasty/main.py | 1 + .../core/notifications/trakt/__init__.py | 1 + couchpotato/core/notifications/trakt/main.py | 1 + .../core/notifications/twitter/__init__.py | 1 + .../core/notifications/twitter/main.py | 2 +- .../core/notifications/xbmc/__init__.py | 1 + couchpotato/core/notifications/xbmc/main.py | 3 +- .../core/notifications/xmpp/__init__.py | 1 + .../core/plugins/automation/__init__.py | 1 + couchpotato/core/plugins/base.py | 9 ++-- couchpotato/core/plugins/browser/__init__.py | 1 + couchpotato/core/plugins/browser/main.py | 2 +- couchpotato/core/plugins/category/__init__.py | 1 + couchpotato/core/plugins/category/main.py | 2 +- couchpotato/core/plugins/custom/__init__.py | 1 + .../core/plugins/dashboard/__init__.py | 1 + couchpotato/core/plugins/dashboard/main.py | 2 +- couchpotato/core/plugins/file/__init__.py | 1 + couchpotato/core/plugins/file/main.py | 1 - couchpotato/core/plugins/log/__init__.py | 1 + couchpotato/core/plugins/log/main.py | 2 +- couchpotato/core/plugins/manage/__init__.py | 1 + couchpotato/core/plugins/manage/main.py | 1 + couchpotato/core/plugins/profile/__init__.py | 1 + couchpotato/core/plugins/profile/main.py | 2 +- couchpotato/core/plugins/quality/__init__.py | 1 + couchpotato/core/plugins/quality/main.py | 8 +-- couchpotato/core/plugins/release/__init__.py | 1 + couchpotato/core/plugins/release/main.py | 17 +++--- couchpotato/core/plugins/renamer/__init__.py | 4 +- couchpotato/core/plugins/renamer/main.py | 53 ++++++++++--------- couchpotato/core/plugins/scanner/__init__.py | 1 + couchpotato/core/plugins/scanner/main.py | 32 +++++------ couchpotato/core/plugins/score/__init__.py | 1 + couchpotato/core/plugins/score/scores.py | 1 + couchpotato/core/plugins/status/__init__.py | 1 + couchpotato/core/plugins/status/main.py | 2 +- couchpotato/core/plugins/subtitle/__init__.py | 1 + couchpotato/core/plugins/trailer/__init__.py | 1 + couchpotato/core/plugins/trailer/main.py | 2 +- .../core/plugins/userscript/__init__.py | 1 + couchpotato/core/plugins/userscript/main.py | 5 +- couchpotato/core/plugins/wizard/__init__.py | 1 + .../providers/automation/bluray/__init__.py | 1 + .../core/providers/automation/bluray/main.py | 4 +- .../core/providers/automation/cp/__init__.py | 1 + .../providers/automation/flixster/__init__.py | 1 + .../providers/automation/flixster/main.py | 5 +- .../automation/goodfilms/__init__.py | 3 +- .../providers/automation/goodfilms/main.py | 7 ++- .../providers/automation/imdb/__init__.py | 1 + .../providers/automation/itunes/__init__.py | 1 + .../core/providers/automation/itunes/main.py | 6 +-- .../automation/kinepolis/__init__.py | 1 + .../automation/letterboxd/__init__.py | 1 + .../providers/automation/letterboxd/main.py | 2 +- .../automation/moviemeter/__init__.py | 1 + .../automation/movies_io/__init__.py | 1 + .../automation/rottentomatoes/__init__.py | 1 + .../automation/rottentomatoes/main.py | 1 + .../providers/automation/trakt/__init__.py | 1 + couchpotato/core/providers/base.py | 29 +++++----- .../core/providers/info/_modifier/__init__.py | 2 +- .../core/providers/info/_modifier/main.py | 4 +- .../providers/info/couchpotatoapi/__init__.py | 1 + .../providers/info/couchpotatoapi/main.py | 2 +- .../core/providers/info/omdbapi/__init__.py | 1 + .../providers/info/themoviedb/__init__.py | 1 + .../core/providers/info/themoviedb/main.py | 2 +- .../core/providers/metadata/wmc/__init__.py | 1 + .../core/providers/metadata/wmc/main.py | 1 + .../core/providers/metadata/xbmc/__init__.py | 1 + .../core/providers/metadata/xbmc/main.py | 2 +- .../core/providers/nzb/binsearch/__init__.py | 1 + .../core/providers/nzb/binsearch/main.py | 2 +- .../core/providers/nzb/newznab/__init__.py | 1 + .../core/providers/nzb/newznab/main.py | 4 +- .../core/providers/nzb/nzbclub/__init__.py | 1 + .../core/providers/nzb/nzbclub/main.py | 2 +- .../core/providers/nzb/nzbindex/__init__.py | 1 + .../core/providers/nzb/nzbindex/main.py | 2 +- .../core/providers/nzb/omgwtfnzbs/__init__.py | 1 + .../core/providers/nzb/omgwtfnzbs/main.py | 2 +- .../providers/torrent/awesomehd/__init__.py | 1 + .../core/providers/torrent/awesomehd/main.py | 8 +-- couchpotato/core/providers/torrent/base.py | 19 +------ .../providers/torrent/bithdtv/__init__.py | 1 + .../core/providers/torrent/bithdtv/main.py | 9 ++-- .../providers/torrent/bitsoup/__init__.py | 1 + .../core/providers/torrent/bitsoup/main.py | 2 +- .../core/providers/torrent/hdbits/__init__.py | 1 + .../core/providers/torrent/hdbits/main.py | 10 ++-- .../torrent/ilovetorrents/__init__.py | 5 +- .../providers/torrent/ilovetorrents/main.py | 8 +-- .../providers/torrent/iptorrents/__init__.py | 1 + .../core/providers/torrent/iptorrents/main.py | 8 +-- .../torrent/kickasstorrents/__init__.py | 1 + .../providers/torrent/kickasstorrents/main.py | 5 +- .../torrent/passthepopcorn/__init__.py | 1 + .../providers/torrent/passthepopcorn/main.py | 12 ++--- .../providers/torrent/publichd/__init__.py | 1 + .../core/providers/torrent/publichd/main.py | 2 +- .../providers/torrent/sceneaccess/__init__.py | 1 + .../providers/torrent/sceneaccess/main.py | 2 +- .../torrent/thepiratebay/__init__.py | 1 + .../providers/torrent/thepiratebay/main.py | 14 ++--- .../torrent/torrentbytes/__init__.py | 1 + .../providers/torrent/torrentbytes/main.py | 12 ++--- .../providers/torrent/torrentday/__init__.py | 1 + .../core/providers/torrent/torrentday/main.py | 2 +- .../torrent/torrentleech/__init__.py | 1 + .../providers/torrent/torrentleech/main.py | 10 ++-- .../torrent/torrentpotato/__init__.py | 1 + .../providers/torrent/torrentpotato/main.py | 2 +- .../torrent/torrentshack/__init__.py | 1 + .../providers/torrent/torrentshack/main.py | 10 ++-- .../core/providers/torrent/yify/__init__.py | 1 + .../core/providers/torrent/yify/main.py | 10 ++-- .../providers/trailer/hdtrailers/__init__.py | 1 + .../core/providers/trailer/hdtrailers/main.py | 2 +- .../providers/userscript/allocine/__init__.py | 1 + .../userscript/appletrailers/__init__.py | 1 + .../userscript/criticker/__init__.py | 1 + .../providers/userscript/filmweb/__init__.py | 1 + .../userscript/flickchart/__init__.py | 1 + .../providers/userscript/imdb/__init__.py | 1 + .../userscript/letterboxd/__init__.py | 1 + .../userscript/moviemeter/__init__.py | 1 + .../providers/userscript/moviesio/__init__.py | 1 + .../userscript/rottentomatoes/__init__.py | 1 + .../providers/userscript/sharethe/__init__.py | 1 + .../providers/userscript/tmdb/__init__.py | 1 + .../providers/userscript/trakt/__init__.py | 1 + .../providers/userscript/whiwa/__init__.py | 1 + .../userscript/youteather/__init__.py | 1 + couchpotato/core/settings/model.py | 8 ++- couchpotato/environment.py | 1 + couchpotato/runner.py | 25 ++++----- 214 files changed, 497 insertions(+), 324 deletions(-) diff --git a/CouchPotato.py b/CouchPotato.py index e777f9bf..069fb65d 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -132,14 +132,14 @@ if __name__ == '__main__': pass except SystemExit: raise - except socket.error as (nr, msg): + except socket.error as e: # log when socket receives SIGINT, but continue. # previous code would have skipped over other types of IO errors too. if nr != 4: try: l.log.critical(traceback.format_exc()) except: - print traceback.format_exc() + print(traceback.format_exc()) raise except: try: @@ -148,7 +148,7 @@ if __name__ == '__main__': if l: l.log.critical(traceback.format_exc()) else: - print traceback.format_exc() + print(traceback.format_exc()) except: - print traceback.format_exc() + print(traceback.format_exc()) raise diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index b8aa3ab9..b61cd843 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -9,13 +9,12 @@ import os import time import traceback -log = CPLog(__name__) +log = CPLog(__name__) views = {} template_loader = template.Loader(os.path.join(os.path.dirname(__file__), 'templates')) - class BaseHandler(RequestHandler): def get_current_user(self): @@ -24,9 +23,10 @@ class BaseHandler(RequestHandler): if username and password: return self.get_secure_cookie('user') - else: # Login when no username or password are set + else: # Login when no username or password are set return True + # Main web handler class WebHandler(BaseHandler): @@ -43,9 +43,11 @@ class WebHandler(BaseHandler): log.error("Failed doing web request '%s': %s", (route, traceback.format_exc())) self.write({'success': False, 'error': 'Failed returning results'}) + def addView(route, func, static = False): views[route] = func + def get_session(engine = None): return Env.getSession(engine) @@ -55,6 +57,7 @@ def index(): return template_loader.load('index.html').generate(sep = os.sep, fireEvent = fireEvent, Env = Env) addView('', index) + # API docs def apiDocs(): routes = [] @@ -70,21 +73,22 @@ def apiDocs(): addView('docs', apiDocs) + # Make non basic auth option to get api key class KeyHandler(RequestHandler): def get(self, *args, **kwargs): - api = None + api_key = None try: username = Env.setting('username') password = Env.setting('password') if (self.get_argument('u') == md5(username) or not username) and (self.get_argument('p') == password or not password): - api = Env.setting('api_key') + api_key = Env.setting('api_key') self.write({ - 'success': api is not None, - 'api_key': api + 'success': api_key is not None, + 'api_key': api_key }) except: log.error('Failed doing key request: %s', (traceback.format_exc())) @@ -102,20 +106,21 @@ class LoginHandler(BaseHandler): def post(self, *args, **kwargs): - api = None + api_key = None username = Env.setting('username') password = Env.setting('password') if (self.get_argument('username') == username or not username) and (md5(self.get_argument('password')) == password or not password): - api = Env.setting('api_key') + api_key = Env.setting('api_key') - if api: + if api_key: remember_me = tryInt(self.get_argument('remember_me', default = 0)) - self.set_secure_cookie('user', api, expires_days = 30 if remember_me > 0 else None) + self.set_secure_cookie('user', api_key, expires_days = 30 if remember_me > 0 else None) self.redirect(Env.get('web_base')) + class LogoutHandler(BaseHandler): def get(self, *args, **kwargs): @@ -136,4 +141,3 @@ def page_not_found(rh): rh.set_status(404) rh.write('Wrong API key used') - diff --git a/couchpotato/api.py b/couchpotato/api.py index e86b127f..ba7f7b69 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -20,6 +20,7 @@ api_nonblock = {} api_docs = {} api_docs_missing = [] + def run_async(func): @wraps(func) def async_func(*args, **kwargs): @@ -29,6 +30,7 @@ def run_async(func): return async_func + # NonBlock API handler class NonBlockHandler(RequestHandler): @@ -61,6 +63,7 @@ class NonBlockHandler(RequestHandler): self.stopper = None + def addNonBlockApiView(route, func_tuple, docs = None, **kwargs): api_nonblock[route] = func_tuple @@ -69,6 +72,7 @@ def addNonBlockApiView(route, func_tuple, docs = None, **kwargs): else: api_docs_missing.append(route) + # Blocking API handler class ApiHandler(RequestHandler): @@ -98,11 +102,12 @@ class ApiHandler(RequestHandler): @run_async def run_handler(callback): try: - result = api[route](**kwargs) - callback(result) + res = api[route](**kwargs) + callback(res) except: log.error('Failed doing api request "%s": %s', (route, traceback.format_exc())) callback({'success': False, 'error': 'Failed returning results'}) + result = yield tornado.gen.Task(run_handler) # Check JSONP callback @@ -122,6 +127,7 @@ class ApiHandler(RequestHandler): api_locks[route].release() + def addApiView(route, func, static = False, docs = None, **kwargs): if static: func(route) diff --git a/couchpotato/core/_base/_core/__init__.py b/couchpotato/core/_base/_core/__init__.py index 4d1a6840..58965bbb 100644 --- a/couchpotato/core/_base/_core/__init__.py +++ b/couchpotato/core/_base/_core/__init__.py @@ -1,6 +1,7 @@ from .main import Core from uuid import uuid4 + def start(): return Core() diff --git a/couchpotato/core/_base/_core/main.py b/couchpotato/core/_base/_core/main.py index f2435eb8..02e21f2d 100644 --- a/couchpotato/core/_base/_core/main.py +++ b/couchpotato/core/_base/_core/main.py @@ -117,7 +117,7 @@ class Core(Plugin): if len(still_running) == 0: break - elif starttime < time.time() - 30: # Always force break after 30s wait + elif starttime < time.time() - 30: # Always force break after 30s wait break running = list(set(still_running) - set(self.ignore_restart)) diff --git a/couchpotato/core/_base/clientscript/__init__.py b/couchpotato/core/_base/clientscript/__init__.py index 8490eae7..8070044e 100644 --- a/couchpotato/core/_base/clientscript/__init__.py +++ b/couchpotato/core/_base/clientscript/__init__.py @@ -1,5 +1,6 @@ from .main import ClientScript + def start(): return ClientScript() diff --git a/couchpotato/core/_base/clientscript/main.py b/couchpotato/core/_base/clientscript/main.py index b80ddcc1..248d2bc5 100644 --- a/couchpotato/core/_base/clientscript/main.py +++ b/couchpotato/core/_base/clientscript/main.py @@ -53,9 +53,9 @@ class ClientScript(Plugin): } - urls = {'style': {}, 'script': {}, } - minified = {'style': {}, 'script': {}, } - paths = {'style': {}, 'script': {}, } + urls = {'style': {}, 'script': {}} + minified = {'style': {}, 'script': {}} + paths = {'style': {}, 'script': {}} comment = { 'style': '/*** %s:%d ***/\n', 'script': '// %s:%d\n' diff --git a/couchpotato/core/_base/desktop/__init__.py b/couchpotato/core/_base/desktop/__init__.py index 064492f2..e59ca523 100644 --- a/couchpotato/core/_base/desktop/__init__.py +++ b/couchpotato/core/_base/desktop/__init__.py @@ -1,5 +1,6 @@ from .main import Desktop + def start(): return Desktop() diff --git a/couchpotato/core/_base/scheduler/__init__.py b/couchpotato/core/_base/scheduler/__init__.py index aa1c5c90..abfc2305 100644 --- a/couchpotato/core/_base/scheduler/__init__.py +++ b/couchpotato/core/_base/scheduler/__init__.py @@ -1,5 +1,6 @@ from .main import Scheduler + def start(): return Scheduler() diff --git a/couchpotato/core/_base/updater/__init__.py b/couchpotato/core/_base/updater/__init__.py index a304f9e7..7ad30d27 100644 --- a/couchpotato/core/_base/updater/__init__.py +++ b/couchpotato/core/_base/updater/__init__.py @@ -2,6 +2,7 @@ from .main import Updater from couchpotato.environment import Env import os + def start(): return Updater() diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 8dc78939..22668d92 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -63,7 +63,7 @@ class Updater(Plugin): fireEvent('schedule.remove', 'updater.check', single = True) if self.isEnabled(): fireEvent('schedule.interval', 'updater.check', self.autoUpdate, hours = 6) - self.autoUpdate() # Check after enabling + self.autoUpdate() # Check after enabling def autoUpdate(self): if self.isEnabled() and self.check() and self.conf('automatic') and not self.updater.update_failed: @@ -151,6 +151,9 @@ class BaseUpdater(Plugin): 'branch': self.branch, } + def getVersion(self): + pass + def check(self): pass @@ -179,7 +182,6 @@ class BaseUpdater(Plugin): log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc())) - class GitUpdater(BaseUpdater): def __init__(self, git_command): @@ -206,7 +208,7 @@ class GitUpdater(BaseUpdater): if not self.version: try: - output = self.repo.getHead() # Yes, please + output = self.repo.getHead() # Yes, please log.debug('Git version output: %s', output.hash) self.version = { 'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.branch, output.hash[:8], datetime.fromtimestamp(output.getDate())), @@ -214,7 +216,7 @@ class GitUpdater(BaseUpdater): 'date': output.getDate(), 'type': 'git', } - except Exception, e: + except Exception as e: log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e) return 'No GIT' @@ -250,7 +252,6 @@ class GitUpdater(BaseUpdater): return False - class SourceUpdater(BaseUpdater): def __init__(self): @@ -276,9 +277,9 @@ class SourceUpdater(BaseUpdater): # Extract if download_data.get('type') == 'zip': - zip = zipfile.ZipFile(destination) - zip.extractall(extracted_path) - zip.close() + zip_file = zipfile.ZipFile(destination) + zip_file.extractall(extracted_path) + zip_file.close() else: tar = tarfile.open(destination) tar.extractall(path = extracted_path) @@ -345,7 +346,6 @@ class SourceUpdater(BaseUpdater): return True - def removeDir(self, path): try: if os.path.isdir(path): @@ -366,7 +366,7 @@ class SourceUpdater(BaseUpdater): self.version = output self.version['type'] = 'source' self.version['repr'] = 'source:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.branch, output.get('hash', '')[:8], datetime.fromtimestamp(output.get('date', 0))) - except Exception, e: + except Exception as e: log.error('Failed using source updater. %s', e) return {} @@ -396,7 +396,7 @@ class SourceUpdater(BaseUpdater): return { 'hash': commit['sha'], - 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), + 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), } except: log.error('Failed getting latest request from github: %s', traceback.format_exc()) @@ -441,7 +441,7 @@ class DesktopUpdater(BaseUpdater): if latest and latest != current_version.get('hash'): self.update_version = { 'hash': latest, - 'date': None, + 'date': None, 'changelog': self.desktop._changelogURL, } diff --git a/couchpotato/core/downloaders/blackhole/__init__.py b/couchpotato/core/downloaders/blackhole/__init__.py index 91164d66..92d18e7f 100644 --- a/couchpotato/core/downloaders/blackhole/__init__.py +++ b/couchpotato/core/downloaders/blackhole/__init__.py @@ -1,6 +1,7 @@ from .main import Blackhole from couchpotato.core.helpers.variable import getDownloadDir + def start(): return Blackhole() diff --git a/couchpotato/core/downloaders/deluge/__init__.py b/couchpotato/core/downloaders/deluge/__init__.py index c7aa26e6..09fae751 100644 --- a/couchpotato/core/downloaders/deluge/__init__.py +++ b/couchpotato/core/downloaders/deluge/__init__.py @@ -1,5 +1,6 @@ from .main import Deluge + def start(): return Deluge() diff --git a/couchpotato/core/downloaders/deluge/main.py b/couchpotato/core/downloaders/deluge/main.py index 2d9084bf..53b87d91 100644 --- a/couchpotato/core/downloaders/deluge/main.py +++ b/couchpotato/core/downloaders/deluge/main.py @@ -109,7 +109,7 @@ class Deluge(Downloader): continue log.debug('name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_on_completed'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) - + # Deluge has no easy way to work out if a torrent is stalled or failing. #status = 'failed' status = 'busy' @@ -125,11 +125,11 @@ class Deluge(Downloader): download_dir = sp(torrent['save_path']) if torrent['move_on_completed']: download_dir = torrent['move_completed_path'] - + torrent_files = [] for file_item in torrent['files']: torrent_files.append(sp(os.path.join(download_dir, file_item['path']))) - + release_downloads.append({ 'id': torrent['hash'], 'name': torrent['name'], @@ -157,6 +157,7 @@ class Deluge(Downloader): log.debug('Requesting Deluge to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) return self.drpc.remove_torrent(release_download['id'], remove_local_data = delete_files) + class DelugeRPC(object): host = 'localhost' diff --git a/couchpotato/core/downloaders/nzbget/__init__.py b/couchpotato/core/downloaders/nzbget/__init__.py index 1f21c056..551eb42c 100644 --- a/couchpotato/core/downloaders/nzbget/__init__.py +++ b/couchpotato/core/downloaders/nzbget/__init__.py @@ -1,5 +1,6 @@ from .main import NZBGet + def start(): return NZBGet() diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index a05fb118..a690572c 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -42,7 +42,7 @@ class NZBGet(Downloader): except socket.error: log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') return False - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: if e.errcode == 401: log.error('Password is incorrect.') else: @@ -56,7 +56,7 @@ class NZBGet(Downloader): if xml_response: log.info('NZB sent successfully to NZBGet') - nzb_id = md5(data['url']) # about as unique as they come ;) + nzb_id = md5(data['url']) # about as unique as they come ;) couchpotato_id = "couchpotato=" + nzb_id groups = rpc.listgroups() file_id = [item['LastID'] for item in groups if item['NZBFilename'] == nzb_name] @@ -83,7 +83,7 @@ class NZBGet(Downloader): except socket.error: log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') return [] - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: if e.errcode == 401: log.error('Password is incorrect.') else: @@ -116,7 +116,7 @@ class NZBGet(Downloader): timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) except: pass - + release_downloads.append({ 'id': nzb_id, 'name': nzb['NZBFilename'], @@ -169,7 +169,7 @@ class NZBGet(Downloader): except socket.error: log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') return False - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: if e.errcode == 401: log.error('Password is incorrect.') else: diff --git a/couchpotato/core/downloaders/nzbvortex/__init__.py b/couchpotato/core/downloaders/nzbvortex/__init__.py index 3087d75b..1c2d699e 100644 --- a/couchpotato/core/downloaders/nzbvortex/__init__.py +++ b/couchpotato/core/downloaders/nzbvortex/__init__.py @@ -1,5 +1,6 @@ from .main import NZBVortex + def start(): return NZBVortex() diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index d2615bfd..205ceb1b 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -56,13 +56,13 @@ class NZBVortex(Downloader): status = 'completed' elif nzb['state'] in [21, 22, 24]: status = 'failed' - + release_downloads.append({ 'id': nzb['id'], 'name': nzb['uiTitle'], 'status': status, 'original_status': nzb['state'], - 'timeleft':-1, + 'timeleft': -1, 'folder': sp(nzb['destinationPath']), }) @@ -102,7 +102,6 @@ class NZBVortex(Downloader): log.error('Login failed, please check you api-key') return False - def call(self, call, parameters = None, repeat = False, auth = True, *args, **kwargs): # Login first @@ -123,7 +122,7 @@ class NZBVortex(Downloader): if data: return json.loads(data) - except URLError, e: + except URLError as e: if hasattr(e, 'code') and e.code == 403: # Try login and do again if not repeat: @@ -145,7 +144,7 @@ class NZBVortex(Downloader): try: data = self.urlopen(url, show_error = False) self.api_level = float(json.loads(data).get('apilevel')) - except URLError, e: + except URLError as e: if hasattr(e, 'code') and e.code == 403: log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher') else: @@ -175,6 +174,7 @@ class HTTPSConnection(httplib.HTTPSConnection): self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1) + class HTTPSHandler(urllib2.HTTPSHandler): def https_open(self, req): return self.do_open(HTTPSConnection, req) diff --git a/couchpotato/core/downloaders/pneumatic/__init__.py b/couchpotato/core/downloaders/pneumatic/__init__.py index 96574a7a..698643fb 100644 --- a/couchpotato/core/downloaders/pneumatic/__init__.py +++ b/couchpotato/core/downloaders/pneumatic/__init__.py @@ -1,5 +1,6 @@ from .main import Pneumatic + def start(): return Pneumatic() diff --git a/couchpotato/core/downloaders/pneumatic/main.py b/couchpotato/core/downloaders/pneumatic/main.py index d3fdad83..6af22d2d 100644 --- a/couchpotato/core/downloaders/pneumatic/main.py +++ b/couchpotato/core/downloaders/pneumatic/main.py @@ -26,26 +26,26 @@ class Pneumatic(Downloader): log.error('No nzb available!') return False - fullPath = os.path.join(directory, self.createFileName(data, filedata, media)) + full_path = os.path.join(directory, self.createFileName(data, filedata, media)) try: - if not os.path.isfile(fullPath): - log.info('Downloading %s to %s.', (data.get('protocol'), fullPath)) - with open(fullPath, 'wb') as f: + if not os.path.isfile(full_path): + log.info('Downloading %s to %s.', (data.get('protocol'), full_path)) + with open(full_path, 'wb') as f: f.write(filedata) nzb_name = self.createNzbName(data, media) strm_path = os.path.join(directory, nzb_name) strm_file = open(strm_path + '.strm', 'wb') - strmContent = self.strm_syntax % (fullPath, nzb_name) + strmContent = self.strm_syntax % (full_path, nzb_name) strm_file.write(strmContent) strm_file.close() return self.downloadReturnId('') else: - log.info('File %s already exists.', fullPath) + log.info('File %s already exists.', full_path) return self.downloadReturnId('') except: diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index dbef6e6f..4a593fd2 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -1,5 +1,6 @@ from .main import rTorrent + def start(): return rTorrent() diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 50276314..e8794ec8 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -111,7 +111,6 @@ class rTorrent(Downloader): if self.conf('label'): torrent_params['label'] = self.conf('label') - if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 1edbbebf..2990078a 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -1,5 +1,6 @@ from .main import Sabnzbd + def start(): return Sabnzbd() diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 1d9073ff..72c23708 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -95,7 +95,7 @@ class Sabnzbd(Downloader): status = 'busy' if 'ENCRYPTED / ' in nzb['filename']: status = 'failed' - + release_downloads.append({ 'id': nzb['nzo_id'], 'name': nzb['filename'], @@ -112,7 +112,7 @@ class Sabnzbd(Downloader): status = 'failed' elif nzb['status'] == 'Completed': status = 'completed' - + release_downloads.append({ 'id': nzb['nzo_id'], 'name': nzb['name'], @@ -166,8 +166,8 @@ class Sabnzbd(Downloader): def call(self, request_params, use_json = True, **kwargs): url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api?' + tryUrlencode(mergeDicts(request_params, { - 'apikey': self.conf('api_key'), - 'output': 'json' + 'apikey': self.conf('api_key'), + 'output': 'json' })) data = self.urlopen(url, timeout = 60, show_error = False, headers = {'User-Agent': Env.getIdentifier()}, **kwargs) diff --git a/couchpotato/core/downloaders/synology/__init__.py b/couchpotato/core/downloaders/synology/__init__.py index 8be16f61..d0c57c2f 100644 --- a/couchpotato/core/downloaders/synology/__init__.py +++ b/couchpotato/core/downloaders/synology/__init__.py @@ -1,5 +1,6 @@ from .main import Synology + def start(): return Synology() diff --git a/couchpotato/core/downloaders/synology/main.py b/couchpotato/core/downloaders/synology/main.py index 7299fa81..26a4558d 100644 --- a/couchpotato/core/downloaders/synology/main.py +++ b/couchpotato/core/downloaders/synology/main.py @@ -65,6 +65,7 @@ class Synology(Downloader): return super(Synology, self).isEnabled(manual, data) and\ ((self.conf('use_for') in for_protocol)) + class SynologyRPC(object): """SynologyRPC lite library""" diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index f96e628e..4c9b4aad 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -1,5 +1,6 @@ from .main import Transmission + def start(): return Transmission() diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index d41337f0..d3f17598 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -105,7 +105,7 @@ class Transmission(Downloader): for torrent in queue['torrents']: if torrent['hashString'] in ids: log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / isStalled=%s / eta=%s / uploadRatio=%s / isFinished=%s / incomplete-dir-enabled=%s / incomplete-dir=%s', - (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent.get('isStalled', 'N/A'), torrent['eta'], torrent['uploadRatio'], torrent['isFinished'], session['incomplete-dir-enabled'], session['incomplete-dir'])) + (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent.get('isStalled', 'N/A'), torrent['eta'], torrent['uploadRatio'], torrent['isFinished'], session['incomplete-dir-enabled'], session['incomplete-dir'])) status = 'busy' if torrent.get('isStalled') and not torrent['percentDone'] == 1 and self.conf('stalled_as_failed'): diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index d45e2e6c..77bd1cd8 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -1,5 +1,6 @@ from .main import uTorrent + def start(): return uTorrent() diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 89f75ccf..e527230c 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -66,7 +66,7 @@ class uTorrent(Downloader): new_settings['seed_prio_limitul_flag'] = True log.info('Updated uTorrent settings to set a torrent to complete after it the seeding requirements are met.') - if settings.get('bt.read_only_on_complete'): #This doesn't work as this option seems to be not available through the api. Mitigated with removeReadOnly function + if settings.get('bt.read_only_on_complete'): #This doesn't work as this option seems to be not available through the api. Mitigated with removeReadOnly function new_settings['bt.read_only_on_complete'] = False log.info('Updated uTorrent settings to not set the files to read only after completing.') @@ -149,7 +149,7 @@ class uTorrent(Downloader): torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]] except: log.debug('Failed getting files from torrent: %s', torrent[2]) - + status = 'busy' if (torrent[1] & self.status_flags['STARTED'] or torrent[1] & self.status_flags['QUEUED']) and torrent[4] == 1000: status = 'seeding' @@ -157,10 +157,10 @@ class uTorrent(Downloader): status = 'failed' elif torrent[4] == 1000: status = 'completed' - + if not status == 'busy': self.removeReadOnly(torrent_files) - + release_downloads.append({ 'id': torrent[0], 'name': torrent[2], diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 7b01fbd8..88df65cf 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -7,6 +7,7 @@ import traceback log = CPLog(__name__) events = {} + def runHandler(name, handler, *args, **kwargs): try: return handler(*args, **kwargs) @@ -14,6 +15,7 @@ def runHandler(name, handler, *args, **kwargs): from couchpotato.environment import Env log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all() if not Env.get('dev') else '')) + def addEvent(name, handler, priority = 100): if not events.get(name): @@ -48,22 +50,24 @@ def addEvent(name, handler, priority = 100): 'priority': priority, }) + def removeEvent(name, handler): e = events[name] e -= handler + def fireEvent(name, *args, **kwargs): - if not events.has_key(name): return + if name not in events: return #log.debug('Firing event %s', name) try: options = { - 'is_after_event': False, # Fire after event - 'on_complete': False, # onComplete event - 'single': False, # Return single handler - 'merge': False, # Merge items - 'in_order': False, # Fire them in specific order, waits for the other to finish + 'is_after_event': False, # Fire after event + 'on_complete': False, # onComplete event + 'single': False, # Return single handler + 'merge': False, # Merge items + 'in_order': False, # Fire them in specific order, waits for the other to finish } # Do options @@ -101,11 +105,14 @@ def fireEvent(name, *args, **kwargs): # Fire result = e(*args, **kwargs) + result_keys = result.keys() + result_keys.sort(natcmp) + if options['single'] and not options['merge']: results = None # Loop over results, stop when first not None result is found. - for r_key in sorted(result.iterkeys(), cmp = natcmp): + for r_key in result_keys: r = result[r_key] if r[0] is True and r[1] is not None: results = r[1] @@ -117,7 +124,7 @@ def fireEvent(name, *args, **kwargs): else: results = [] - for r_key in sorted(result.iterkeys(), cmp = natcmp): + for r_key in result_keys: r = result[r_key] if r[0] == True and r[1]: results.append(r[1]) @@ -160,18 +167,21 @@ def fireEvent(name, *args, **kwargs): except Exception: log.error('%s: %s', (name, traceback.format_exc())) + def fireEventAsync(*args, **kwargs): try: t = threading.Thread(target = fireEvent, args = args, kwargs = kwargs) t.setDaemon(True) t.start() return True - except Exception, e: + except Exception as e: log.error('%s: %s', (args[0], e)) + def errorHandler(error): etype, value, tb = error log.error(''.join(traceback.format_exception(etype, value, tb))) + def getEvent(name): return events[name] diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index fc7c919c..1c418418 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -11,16 +11,18 @@ log = CPLog(__name__) def toSafeString(original): valid_chars = "-_.() %s%s" % (ascii_letters, digits) - cleanedFilename = unicodedata.normalize('NFKD', toUnicode(original)).encode('ASCII', 'ignore') - valid_string = ''.join(c for c in cleanedFilename if c in valid_chars) + cleaned_filename = unicodedata.normalize('NFKD', toUnicode(original)).encode('ASCII', 'ignore') + valid_string = ''.join(c for c in cleaned_filename if c in valid_chars) return ' '.join(valid_string.split()) + def simplifyString(original): string = stripAccents(original.lower()) string = toSafeString(' '.join(re.split('\W+', string))) split = re.split('\W+|_', string.lower()) return toUnicode(' '.join(split)) + def toUnicode(original, *args): try: if isinstance(original, unicode): @@ -38,16 +40,18 @@ def toUnicode(original, *args): ascii_text = str(original).encode('string_escape') return toUnicode(ascii_text) + def ss(original, *args): u_original = toUnicode(original, *args) try: from couchpotato.environment import Env return u_original.encode(Env.get('encoding')) - except Exception, e: + except Exception as e: log.debug('Failed ss encoding char, force UTF8: %s', e) return u_original.encode('UTF-8') + def sp(path, *args): # Standardise encoding, normalise case, path and strip trailing '/' or '\' @@ -73,6 +77,7 @@ def sp(path, *args): return path + def ek(original, *args): if isinstance(original, (str, unicode)): try: @@ -83,6 +88,7 @@ def ek(original, *args): return original + def isInt(value): try: int(value) @@ -90,9 +96,11 @@ def isInt(value): except ValueError: return False + def stripAccents(s): return ''.join((c for c in unicodedata.normalize('NFD', toUnicode(s)) if unicodedata.category(c) != 'Mn')) + def tryUrlencode(s): new = u'' if isinstance(s, dict): diff --git a/couchpotato/core/helpers/request.py b/couchpotato/core/helpers/request.py index 888e63fd..fb91f453 100644 --- a/couchpotato/core/helpers/request.py +++ b/couchpotato/core/helpers/request.py @@ -37,6 +37,7 @@ def getParams(params): return dictToList(temp) + def dictToList(params): if type(params) is dict: diff --git a/couchpotato/core/helpers/rss.py b/couchpotato/core/helpers/rss.py index b840d862..1a4d37c2 100644 --- a/couchpotato/core/helpers/rss.py +++ b/couchpotato/core/helpers/rss.py @@ -3,6 +3,7 @@ import xml.etree.ElementTree as XMLTree log = CPLog(__name__) + class RSS(object): def getTextElements(self, xml, path): @@ -46,6 +47,6 @@ class RSS(object): def getItems(self, data, path = 'channel/item'): try: return XMLTree.parse(data).findall(path) - except Exception, e: + except Exception as e: log.error('Error parsing RSS. %s', e) return [] diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 658ea8c7..b191d0a0 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -11,8 +11,10 @@ import sys log = CPLog(__name__) + def fnEscape(pattern): - return pattern.replace('[','[[').replace(']','[]]').replace('[[','[[]') + return pattern.replace('[', '[[').replace(']', '[]]').replace('[[', '[[]') + def link(src, dst): if os.name == 'nt': @@ -21,6 +23,7 @@ def link(src, dst): else: os.link(src, dst) + def symlink(src, dst): if os.name == 'nt': import ctypes @@ -28,6 +31,7 @@ def symlink(src, dst): else: os.symlink(src, dst) + def getUserDir(): try: import pwd @@ -37,6 +41,7 @@ def getUserDir(): return os.path.expanduser('~') + def getDownloadDir(): user_dir = getUserDir() @@ -49,6 +54,7 @@ def getDownloadDir(): return user_dir + def getDataDir(): # Windows @@ -68,8 +74,10 @@ def getDataDir(): # Linux return os.path.join(user_dir, '.couchpotato') -def isDict(object): - return isinstance(object, dict) + +def isDict(obj): + return isinstance(obj, dict) + def mergeDicts(a, b, prepend_list = False): assert isDict(a), isDict(b) @@ -91,6 +99,7 @@ def mergeDicts(a, b, prepend_list = False): current_dst[key] = current_src[key] return dst + def removeListDuplicates(seq): checked = [] for e in seq: @@ -98,26 +107,32 @@ def removeListDuplicates(seq): checked.append(e) return checked + def flattenList(l): if isinstance(l, list): return sum(map(flattenList, l)) else: return l + def md5(text): return hashlib.md5(ss(text)).hexdigest() + def sha1(text): return hashlib.sha1(text).hexdigest() + def isLocalIP(ip): ip = ip.lstrip('htps:/') regex = '/(^127\.)|(^192\.168\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^::1)$/' return re.search(regex, ip) is not None or 'localhost' in ip or ip[:4] == '127.' + def getExt(filename): return os.path.splitext(filename)[1][1:] + def cleanHost(host, protocol = True, ssl = False, username = None, password = None): if not '://' in host and protocol: @@ -129,7 +144,7 @@ def cleanHost(host, protocol = True, ssl = False, username = None, password = No if protocol and username and password: login = '%s:%s@' % (username, password) if not login in host: - host = host.replace('://', '://' + login, 1) + host = host.replace('://', '://' + login, 1) host = host.rstrip('/ ') if protocol: @@ -137,6 +152,7 @@ def cleanHost(host, protocol = True, ssl = False, username = None, password = No return host + def getImdb(txt, check_inside = False, multiple = False): if not check_inside: @@ -161,10 +177,12 @@ def getImdb(txt, check_inside = False, multiple = False): return False + def tryInt(s, default = 0): try: return int(s) except: return default + def tryFloat(s): try: if isinstance(s, str): @@ -173,17 +191,24 @@ def tryFloat(s): return float(s) except: return 0 + def natsortKey(s): return map(tryInt, re.findall(r'(\d+|\D+)', s)) + def natcmp(a, b): - return cmp(natsortKey(a), natsortKey(b)) + a2 = natsortKey(a) + b2 = natsortKey(b) + + return (a2 > b2) - (a2 < b2) + def toIterable(value): if isinstance(value, collections.Iterable): return value return [value] + def getTitle(library_dict): try: try: @@ -206,6 +231,7 @@ def getTitle(library_dict): log.error('Could not get title for library item: %s', library_dict) return None + def possibleTitles(raw_title): titles = [ @@ -220,16 +246,20 @@ def possibleTitles(raw_title): return list(set(titles)) + def randomString(size = 8, chars = string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for x in range(size)) + def splitString(str, split_on = ',', clean = True): - list = [x.strip() for x in str.split(split_on)] if str else [] - return filter(None, list) if clean else list + l = [x.strip() for x in str.split(split_on)] if str else [] + return filter(None, l) if clean else l + def dictIsSubset(a, b): return all([k in b and b[k] == v for k, v in a.items()]) + def isSubFolder(sub_folder, base_folder): # Returns True if sub_folder is the same as or inside base_folder return base_folder and sub_folder and os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep in os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index 69a031f1..c2a7a836 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -1,6 +1,7 @@ import logging import re + class CPLog(object): context = '' @@ -49,7 +50,7 @@ class CPLog(object): msg = msg % tuple([ss(x) for x in list(replace_tuple)]) else: msg = msg % ss(replace_tuple) - except Exception, e: + except Exception as e: self.logger.error(u'Failed encoding stuff to log "%s": %s' % (msg, e)) if not Env.get('dev'): diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 88094890..2635ca6c 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -10,8 +10,8 @@ class MediaBase(Plugin): default_dict = { 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}, 'files':{}, 'info': {}}, - 'library': {'titles': {}, 'files':{}}, + 'releases': {'status': {}, 'quality': {}, 'files': {}, 'info': {}}, + 'library': {'titles': {}, 'files': {}}, 'files': {}, 'status': {}, 'category': {}, diff --git a/couchpotato/core/media/_base/media/__init__.py b/couchpotato/core/media/_base/media/__init__.py index a9693a3d..e5f5a0ec 100644 --- a/couchpotato/core/media/_base/media/__init__.py +++ b/couchpotato/core/media/_base/media/__init__.py @@ -1,5 +1,6 @@ from .main import MediaPlugin + def start(): return MediaPlugin() diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 206187fb..b9bf2196 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -102,7 +102,6 @@ class MediaPlugin(MediaBase): def handler(): fireEvent('library.update.%s' % media.type, identifier = identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(id)) - return handler def addSingleRefreshView(self): @@ -254,7 +253,7 @@ class MediaPlugin(MediaBase): # Merge releases with movie dict movies.append(mergeDicts(movie_dict[media_id].to_dict({ - 'library': {'titles': {}, 'files':{}}, + 'library': {'titles': {}, 'files': {}}, 'files': {}, }), { 'releases': releases, diff --git a/couchpotato/core/media/_base/search/__init__.py b/couchpotato/core/media/_base/search/__init__.py index 4b2eae27..09bc84ef 100644 --- a/couchpotato/core/media/_base/search/__init__.py +++ b/couchpotato/core/media/_base/search/__init__.py @@ -1,5 +1,6 @@ from .main import Search + def start(): return Search() diff --git a/couchpotato/core/media/_base/searcher/__init__.py b/couchpotato/core/media/_base/searcher/__init__.py index 5e029a25..72c7d6ef 100644 --- a/couchpotato/core/media/_base/searcher/__init__.py +++ b/couchpotato/core/media/_base/searcher/__init__.py @@ -1,5 +1,6 @@ from .main import Searcher + def start(): return Searcher() diff --git a/couchpotato/core/media/_base/searcher/base.py b/couchpotato/core/media/_base/searcher/base.py index 368c6e2d..5322d850 100644 --- a/couchpotato/core/media/_base/searcher/base.py +++ b/couchpotato/core/media/_base/searcher/base.py @@ -12,7 +12,6 @@ class SearcherBase(Plugin): def __init__(self): super(SearcherBase, self).__init__() - addEvent('searcher.progress', self.getProgress) addEvent('%s.searcher.progress' % self.getType(), self.getProgress) @@ -26,9 +25,8 @@ class SearcherBase(Plugin): _type = self.getType() def setCrons(): - fireEvent('schedule.cron', '%s.searcher.all' % _type, self.searchAll, - day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) addEvent('app.load', setCrons) addEvent('setting.save.%s_searcher.cron_day.after' % _type, setCrons) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 3c73eb27..6ff09728 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -107,10 +107,10 @@ class Searcher(SearcherBase): # Hack for older movies that don't contain quality tag year_name = fireEvent('scanner.name_year', name, single = True) if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): - if size > 3000: # Assume dvdr + if size > 3000: # Assume dvdr log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', size) found['dvdr'] = True - else: # Assume dvdrip + else: # Assume dvdrip log.info('Quality was missing in name, assuming it\'s a DVD-Rip based on the size: %s', size) found['dvdrip'] = True diff --git a/couchpotato/core/media/movie/_base/__init__.py b/couchpotato/core/media/movie/_base/__init__.py index 4be3b127..22211332 100644 --- a/couchpotato/core/media/movie/_base/__init__.py +++ b/couchpotato/core/media/movie/_base/__init__.py @@ -1,5 +1,6 @@ from .main import MovieBase + def start(): return MovieBase() diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 03c22e3d..cd2158cb 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -61,7 +61,6 @@ class MovieBase(MovieTypeBase): except: pass - library = fireEvent('library.add.movie', single = True, attrs = params, update_after = update_library) # Status diff --git a/couchpotato/core/media/movie/library/movie/__init__.py b/couchpotato/core/media/movie/library/movie/__init__.py index 03494a11..98ed54c0 100644 --- a/couchpotato/core/media/movie/library/movie/__init__.py +++ b/couchpotato/core/media/movie/library/movie/__init__.py @@ -1,5 +1,6 @@ from .main import MovieLibraryPlugin + def start(): return MovieLibraryPlugin() diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index b0d05202..61d93ae9 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -13,7 +13,7 @@ log = CPLog(__name__) class MovieLibraryPlugin(LibraryBase): - default_dict = {'titles': {}, 'files':{}} + default_dict = {'titles': {}, 'files': {}} def __init__(self): addEvent('library.add.movie', self.add) diff --git a/couchpotato/core/media/movie/searcher/__init__.py b/couchpotato/core/media/movie/searcher/__init__.py index bae18902..4ae1ed32 100644 --- a/couchpotato/core/media/movie/searcher/__init__.py +++ b/couchpotato/core/media/movie/searcher/__init__.py @@ -1,6 +1,7 @@ from .main import MovieSearcher import random + def start(): return MovieSearcher() diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 1c4810ec..1775f672 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -91,7 +91,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): 'category': {}, 'profile': {'types': {'quality': {}}}, 'releases': {'status': {}, 'quality': {}}, - 'library': {'titles': {}, 'files':{}}, + 'library': {'titles': {}, 'files': {}}, 'files': {}, }) @@ -117,7 +117,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): def single(self, movie, search_protocols = None, manual = False): # movies don't contain 'type' yet, so just set to default here - if not movie.has_key('type'): + if 'type' not in movie: movie['type'] = 'movie' # Find out search type diff --git a/couchpotato/core/media/movie/suggestion/__init__.py b/couchpotato/core/media/movie/suggestion/__init__.py index b63b5b13..50083fe7 100644 --- a/couchpotato/core/media/movie/suggestion/__init__.py +++ b/couchpotato/core/media/movie/suggestion/__init__.py @@ -1,5 +1,6 @@ from .main import Suggestion + def start(): return Suggestion() diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index f29281ea..5240bb2b 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -40,7 +40,7 @@ class Suggestion(Plugin): movies.extend(splitString(Env.prop('suggest_seen', default = ''))) suggestions = fireEvent('movie.suggest', movies = movies, ignore = ignored, single = True) - self.setCache('suggestion_cached', suggestions, timeout = 6048000) # Cache for 10 weeks + self.setCache('suggestion_cached', suggestions, timeout = 6048000) # Cache for 10 weeks return { 'success': True, diff --git a/couchpotato/core/migration/versions/002_Movie_category.py b/couchpotato/core/migration/versions/002_Movie_category.py index 234e1136..023e47c6 100644 --- a/couchpotato/core/migration/versions/002_Movie_category.py +++ b/couchpotato/core/migration/versions/002_Movie_category.py @@ -13,5 +13,6 @@ def upgrade(migrate_engine): create_column(category_column, movie) Index('ix_movie_category_id', movie.c.category_id).create() + def downgrade(migrate_engine): pass diff --git a/couchpotato/core/notifications/boxcar/__init__.py b/couchpotato/core/notifications/boxcar/__init__.py index ab244c32..faab7a5c 100644 --- a/couchpotato/core/notifications/boxcar/__init__.py +++ b/couchpotato/core/notifications/boxcar/__init__.py @@ -1,5 +1,6 @@ from .main import Boxcar + def start(): return Boxcar() diff --git a/couchpotato/core/notifications/core/__init__.py b/couchpotato/core/notifications/core/__init__.py index 6e923dac..b68a915a 100644 --- a/couchpotato/core/notifications/core/__init__.py +++ b/couchpotato/core/notifications/core/__init__.py @@ -1,5 +1,6 @@ from .main import CoreNotifier + def start(): return CoreNotifier() diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index cd63c2cb..02545ea8 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -71,7 +71,6 @@ class CoreNotifier(Notification): db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete() db.commit() - def markAsRead(self, ids = None, **kwargs): ids = splitString(ids) if ids else None diff --git a/couchpotato/core/notifications/email/__init__.py b/couchpotato/core/notifications/email/__init__.py index 33c2f634..aaf087b9 100644 --- a/couchpotato/core/notifications/email/__init__.py +++ b/couchpotato/core/notifications/email/__init__.py @@ -1,5 +1,6 @@ from .main import Email + def start(): return Email() @@ -30,7 +31,7 @@ config = [{ }, { 'name': 'smtp_port', 'label': 'SMTP server port', - 'default': '25', + 'default': '25', 'type': 'int', }, { diff --git a/couchpotato/core/notifications/email/main.py b/couchpotato/core/notifications/email/main.py index 41a4323b..b8544016 100644 --- a/couchpotato/core/notifications/email/main.py +++ b/couchpotato/core/notifications/email/main.py @@ -40,7 +40,7 @@ class Email(Notification): log.debug("SMTP over SSL %s", ("enabled" if ssl == 1 else "disabled")) mailserver = smtplib.SMTP_SSL(smtp_server) if ssl == 1 else smtplib.SMTP(smtp_server) - if (starttls): + if starttls: log.debug("Using StartTLS to initiate the connection with the SMTP server") mailserver.starttls() diff --git a/couchpotato/core/notifications/growl/__init__.py b/couchpotato/core/notifications/growl/__init__.py index 8e462236..dd01cb91 100644 --- a/couchpotato/core/notifications/growl/__init__.py +++ b/couchpotato/core/notifications/growl/__init__.py @@ -1,5 +1,6 @@ from .main import Growl + def start(): return Growl() diff --git a/couchpotato/core/notifications/growl/main.py b/couchpotato/core/notifications/growl/main.py index dabeea01..a3927ed2 100644 --- a/couchpotato/core/notifications/growl/main.py +++ b/couchpotato/core/notifications/growl/main.py @@ -37,7 +37,7 @@ class Growl(Notification): ) self.growl.register() self.registered = True - except Exception, e: + except Exception as e: if 'timed out' in str(e): self.registered = True else: diff --git a/couchpotato/core/notifications/nmj/__init__.py b/couchpotato/core/notifications/nmj/__init__.py index 08a21a3e..461a450e 100644 --- a/couchpotato/core/notifications/nmj/__init__.py +++ b/couchpotato/core/notifications/nmj/__init__.py @@ -1,5 +1,6 @@ from .main import NMJ + def start(): return NMJ() diff --git a/couchpotato/core/notifications/nmj/main.py b/couchpotato/core/notifications/nmj/main.py index 1479fb1b..967b70e7 100644 --- a/couchpotato/core/notifications/nmj/main.py +++ b/couchpotato/core/notifications/nmj/main.py @@ -86,18 +86,17 @@ class NMJ(Notification): 'arg3': '', } params = tryUrlencode(params) - UPDATE_URL = 'http://%(host)s:8008/metadata_database?%(params)s' - updateUrl = UPDATE_URL % {'host': host, 'params': params} + update_url = 'http://%(host)s:8008/metadata_database?%(params)s' % {'host': host, 'params': params} try: - response = self.urlopen(updateUrl) + response = self.urlopen(update_url) except: return False try: et = etree.fromstring(response) result = et.findtext('returnValue') - except SyntaxError, e: + except SyntaxError as e: log.error('Unable to parse XML returned from the Popcorn Hour: %s', e) return False diff --git a/couchpotato/core/notifications/notifymyandroid/__init__.py b/couchpotato/core/notifications/notifymyandroid/__init__.py index 9ee5d90a..7d4f4aeb 100644 --- a/couchpotato/core/notifications/notifymyandroid/__init__.py +++ b/couchpotato/core/notifications/notifymyandroid/__init__.py @@ -1,5 +1,6 @@ from .main import NotifyMyAndroid + def start(): return NotifyMyAndroid() diff --git a/couchpotato/core/notifications/notifymywp/__init__.py b/couchpotato/core/notifications/notifymywp/__init__.py index 6e0bd06d..4fcf1a9a 100644 --- a/couchpotato/core/notifications/notifymywp/__init__.py +++ b/couchpotato/core/notifications/notifymywp/__init__.py @@ -1,5 +1,6 @@ from .main import NotifyMyWP + def start(): return NotifyMyWP() diff --git a/couchpotato/core/notifications/notifymywp/main.py b/couchpotato/core/notifications/notifymywp/main.py index 167b6eeb..1cdd980a 100644 --- a/couchpotato/core/notifications/notifymywp/main.py +++ b/couchpotato/core/notifications/notifymywp/main.py @@ -8,7 +8,8 @@ log = CPLog(__name__) class NotifyMyWP(Notification): - def notify(self, message = '', data = {}, listener = None): + def notify(self, message = '', data = None, listener = None): + if not data: data = {} keys = splitString(self.conf('api_key')) p = PyNMWP(keys, self.conf('dev_key')) diff --git a/couchpotato/core/notifications/plex/__init__.py b/couchpotato/core/notifications/plex/__init__.py index d68ddb19..0de92ca3 100755 --- a/couchpotato/core/notifications/plex/__init__.py +++ b/couchpotato/core/notifications/plex/__init__.py @@ -1,5 +1,6 @@ from .main import Plex + def start(): return Plex() diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index ce25c8f0..a6853b2f 100755 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -23,9 +23,9 @@ class Plex(Notification): addEvent('renamer.after', self.addToLibrary) - - def addToLibrary(self, message = None, group = {}): + def addToLibrary(self, message = None, group = None): if self.isDisabled(): return + if not group: group = {} return self.server.refresh() @@ -57,7 +57,8 @@ class Plex(Notification): return success - def notify(self, message = '', data = {}, listener = None): + def notify(self, message = '', data = None, listener = None): + if not data: data = {} return self.notifyClients(message, self.getClientNames()) def test(self, **kwargs): diff --git a/couchpotato/core/notifications/prowl/__init__.py b/couchpotato/core/notifications/prowl/__init__.py index e0564289..3721a0ad 100644 --- a/couchpotato/core/notifications/prowl/__init__.py +++ b/couchpotato/core/notifications/prowl/__init__.py @@ -1,5 +1,6 @@ from .main import Prowl + def start(): return Prowl() diff --git a/couchpotato/core/notifications/prowl/main.py b/couchpotato/core/notifications/prowl/main.py index 26e156a2..b3385863 100644 --- a/couchpotato/core/notifications/prowl/main.py +++ b/couchpotato/core/notifications/prowl/main.py @@ -22,7 +22,7 @@ class Prowl(Notification): 'priority': self.conf('priority'), } headers = { - 'Content-type': 'application/x-www-form-urlencoded' + 'Content-type': 'application/x-www-form-urlencoded' } try: diff --git a/couchpotato/core/notifications/pushalot/__init__.py b/couchpotato/core/notifications/pushalot/__init__.py index a2a297a3..ad0c853f 100644 --- a/couchpotato/core/notifications/pushalot/__init__.py +++ b/couchpotato/core/notifications/pushalot/__init__.py @@ -1,5 +1,6 @@ from .main import Pushalot + def start(): return Pushalot() diff --git a/couchpotato/core/notifications/pushalot/main.py b/couchpotato/core/notifications/pushalot/main.py index 0afb84b9..306ee1d1 100644 --- a/couchpotato/core/notifications/pushalot/main.py +++ b/couchpotato/core/notifications/pushalot/main.py @@ -5,6 +5,7 @@ import traceback log = CPLog(__name__) + class Pushalot(Notification): urls = { diff --git a/couchpotato/core/notifications/pushbullet/__init__.py b/couchpotato/core/notifications/pushbullet/__init__.py index e61a44e3..c52e7781 100644 --- a/couchpotato/core/notifications/pushbullet/__init__.py +++ b/couchpotato/core/notifications/pushbullet/__init__.py @@ -1,5 +1,6 @@ from .main import Pushbullet + def start(): return Pushbullet() diff --git a/couchpotato/core/notifications/pushover/__init__.py b/couchpotato/core/notifications/pushover/__init__.py index 1ea1d5c0..da764860 100644 --- a/couchpotato/core/notifications/pushover/__init__.py +++ b/couchpotato/core/notifications/pushover/__init__.py @@ -1,5 +1,6 @@ from .main import Pushover + def start(): return Pushover() diff --git a/couchpotato/core/notifications/pushover/main.py b/couchpotato/core/notifications/pushover/main.py index 76f730b6..ba954a54 100644 --- a/couchpotato/core/notifications/pushover/main.py +++ b/couchpotato/core/notifications/pushover/main.py @@ -30,9 +30,9 @@ class Pushover(Notification): }) http_handler.request('POST', - "/1/messages.json", - headers = {'Content-type': 'application/x-www-form-urlencoded'}, - body = tryUrlencode(api_data) + "/1/messages.json", + headers = {'Content-type': 'application/x-www-form-urlencoded'}, + body = tryUrlencode(api_data) ) response = http_handler.getresponse() diff --git a/couchpotato/core/notifications/synoindex/__init__.py b/couchpotato/core/notifications/synoindex/__init__.py index eb3a793f..89d07b06 100644 --- a/couchpotato/core/notifications/synoindex/__init__.py +++ b/couchpotato/core/notifications/synoindex/__init__.py @@ -1,5 +1,6 @@ from .main import Synoindex + def start(): return Synoindex() diff --git a/couchpotato/core/notifications/synoindex/main.py b/couchpotato/core/notifications/synoindex/main.py index 0f7775d6..ec7a64ef 100644 --- a/couchpotato/core/notifications/synoindex/main.py +++ b/couchpotato/core/notifications/synoindex/main.py @@ -26,7 +26,7 @@ class Synoindex(Notification): out = p.communicate() log.info('Result from synoindex: %s', str(out)) return True - except OSError, e: + except OSError as e: log.error('Unable to run synoindex: %s', e) return False diff --git a/couchpotato/core/notifications/toasty/__init__.py b/couchpotato/core/notifications/toasty/__init__.py index 8e2dae76..31e055a0 100644 --- a/couchpotato/core/notifications/toasty/__init__.py +++ b/couchpotato/core/notifications/toasty/__init__.py @@ -1,5 +1,6 @@ from .main import Toasty + def start(): return Toasty() diff --git a/couchpotato/core/notifications/toasty/main.py b/couchpotato/core/notifications/toasty/main.py index c65b6b42..ea1f2192 100644 --- a/couchpotato/core/notifications/toasty/main.py +++ b/couchpotato/core/notifications/toasty/main.py @@ -5,6 +5,7 @@ import traceback log = CPLog(__name__) + class Toasty(Notification): urls = { diff --git a/couchpotato/core/notifications/trakt/__init__.py b/couchpotato/core/notifications/trakt/__init__.py index b119736c..20e2e3f9 100644 --- a/couchpotato/core/notifications/trakt/__init__.py +++ b/couchpotato/core/notifications/trakt/__init__.py @@ -1,5 +1,6 @@ from .main import Trakt + def start(): return Trakt() diff --git a/couchpotato/core/notifications/trakt/main.py b/couchpotato/core/notifications/trakt/main.py index e67f5fa8..c759c6db 100644 --- a/couchpotato/core/notifications/trakt/main.py +++ b/couchpotato/core/notifications/trakt/main.py @@ -3,6 +3,7 @@ from couchpotato.core.notifications.base import Notification log = CPLog(__name__) + class Trakt(Notification): urls = { diff --git a/couchpotato/core/notifications/twitter/__init__.py b/couchpotato/core/notifications/twitter/__init__.py index 9db8dcb8..1b9c7699 100644 --- a/couchpotato/core/notifications/twitter/__init__.py +++ b/couchpotato/core/notifications/twitter/__init__.py @@ -1,5 +1,6 @@ from .main import Twitter + def start(): return Twitter() diff --git a/couchpotato/core/notifications/twitter/main.py b/couchpotato/core/notifications/twitter/main.py index ad4fc315..559c830f 100644 --- a/couchpotato/core/notifications/twitter/main.py +++ b/couchpotato/core/notifications/twitter/main.py @@ -64,7 +64,7 @@ class Twitter(Notification): api.PostUpdate(update_message[135:] + ' 2/2') else: api.PostUpdate(update_message) - except Exception, e: + except Exception as e: log.error('Error sending tweet: %s', e) return False diff --git a/couchpotato/core/notifications/xbmc/__init__.py b/couchpotato/core/notifications/xbmc/__init__.py index 04662e27..34fed632 100644 --- a/couchpotato/core/notifications/xbmc/__init__.py +++ b/couchpotato/core/notifications/xbmc/__init__.py @@ -1,5 +1,6 @@ from .main import XBMC + def start(): return XBMC() diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index b53485aa..bfda85e1 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -1,7 +1,6 @@ from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification -from urllib2 import URLError import base64 import json import socket @@ -46,7 +45,7 @@ class XBMC(Notification): max_successful += len(calls) response = self.request(host, calls) else: - response = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message}) + response = self.notifyXBMCnoJSON(host, {'title': self.default_title, 'message': message}) if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0): response += self.request(host, [('VideoLibrary.Scan', {})]) diff --git a/couchpotato/core/notifications/xmpp/__init__.py b/couchpotato/core/notifications/xmpp/__init__.py index a52242ff..0e3e14d9 100644 --- a/couchpotato/core/notifications/xmpp/__init__.py +++ b/couchpotato/core/notifications/xmpp/__init__.py @@ -1,5 +1,6 @@ from .main import Xmpp + def start(): return Xmpp() diff --git a/couchpotato/core/plugins/automation/__init__.py b/couchpotato/core/plugins/automation/__init__.py index a81719c4..482a0090 100644 --- a/couchpotato/core/plugins/automation/__init__.py +++ b/couchpotato/core/plugins/automation/__init__.py @@ -1,5 +1,6 @@ from .main import Automation + def start(): return Automation() diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index b4688e49..73e41c75 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -85,7 +85,7 @@ class Plugin(object): class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() # View path - path = 'static/plugin/%s/' % (class_name) + path = 'static/plugin/%s/' % class_name # Add handler to Tornado Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})]) @@ -110,7 +110,7 @@ class Plugin(object): f.write(content) f.close() os.chmod(path, Env.getPermission('file')) - except Exception, e: + except Exception as e: log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) if os.path.isfile(path): os.remove(path) @@ -121,7 +121,7 @@ class Plugin(object): if not os.path.isdir(path): os.makedirs(path, Env.getPermission('folder')) return True - except Exception, e: + except Exception as e: log.error('Unable to create folder "%s": %s', (path, e)) return False @@ -243,7 +243,6 @@ class Plugin(object): except: log.error("Something went wrong when finishing the plugin function. Could not find the 'is_running' key") - def getCache(self, cache_key, url = None, **kwargs): cache_key_md5 = md5(cache_key) cache = Env.get('cache').get(cache_key_md5) @@ -255,7 +254,7 @@ class Plugin(object): try: cache_timeout = 300 - if kwargs.has_key('cache_timeout'): + if 'cache_timeout' in kwargs: cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] diff --git a/couchpotato/core/plugins/browser/__init__.py b/couchpotato/core/plugins/browser/__init__.py index 976fcd10..fae50657 100644 --- a/couchpotato/core/plugins/browser/__init__.py +++ b/couchpotato/core/plugins/browser/__init__.py @@ -1,5 +1,6 @@ from .main import FileBrowser + def start(): return FileBrowser() diff --git a/couchpotato/core/plugins/browser/main.py b/couchpotato/core/plugins/browser/main.py index 380e6826..c8f7536b 100644 --- a/couchpotato/core/plugins/browser/main.py +++ b/couchpotato/core/plugins/browser/main.py @@ -14,7 +14,7 @@ if os.name == 'nt': raise ImportError("Missing the win32file module, which is a part of the prerequisite \ pywin32 package. You can get it from http://sourceforge.net/projects/pywin32/files/pywin32/") else: - import win32file #@UnresolvedImport + import win32file #@UnresolvedImport class FileBrowser(Plugin): diff --git a/couchpotato/core/plugins/category/__init__.py b/couchpotato/core/plugins/category/__init__.py index 6dc41df7..dcdae90b 100644 --- a/couchpotato/core/plugins/category/__init__.py +++ b/couchpotato/core/plugins/category/__init__.py @@ -1,5 +1,6 @@ from .main import CategoryPlugin + def start(): return CategoryPlugin() diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index 87cd0ea4..35597fdb 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -101,7 +101,7 @@ class CategoryPlugin(Plugin): self.removeFromMovie(id) success = True - except Exception, e: + except Exception as e: message = log.error('Failed deleting category: %s', e) db.expire_all() diff --git a/couchpotato/core/plugins/custom/__init__.py b/couchpotato/core/plugins/custom/__init__.py index 573cd99f..20a39351 100644 --- a/couchpotato/core/plugins/custom/__init__.py +++ b/couchpotato/core/plugins/custom/__init__.py @@ -1,5 +1,6 @@ from .main import Custom + def start(): return Custom() diff --git a/couchpotato/core/plugins/dashboard/__init__.py b/couchpotato/core/plugins/dashboard/__init__.py index 81279291..c43a44eb 100644 --- a/couchpotato/core/plugins/dashboard/__init__.py +++ b/couchpotato/core/plugins/dashboard/__init__.py @@ -1,5 +1,6 @@ from .main import Dashboard + def start(): return Dashboard() diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index 4f4d85ab..949d8f50 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -115,7 +115,7 @@ class Dashboard(Plugin): for movie_id in movie_ids: movies.append(movie_dict[movie_id].to_dict({ - 'library': {'titles': {}, 'files':{}}, + 'library': {'titles': {}, 'files': {}}, 'files': {}, })) diff --git a/couchpotato/core/plugins/file/__init__.py b/couchpotato/core/plugins/file/__init__.py index 54d9cbe5..3dced3d0 100644 --- a/couchpotato/core/plugins/file/__init__.py +++ b/couchpotato/core/plugins/file/__init__.py @@ -1,5 +1,6 @@ from .main import FileManager + def start(): return FileManager() diff --git a/couchpotato/core/plugins/file/main.py b/couchpotato/core/plugins/file/main.py index fc63aca8..806af167 100644 --- a/couchpotato/core/plugins/file/main.py +++ b/couchpotato/core/plugins/file/main.py @@ -82,7 +82,6 @@ class FileManager(Plugin): def showCacheFile(self, route, **kwargs): Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})]) - def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): if not urlopen_kwargs: urlopen_kwargs = {} diff --git a/couchpotato/core/plugins/log/__init__.py b/couchpotato/core/plugins/log/__init__.py index 33dcf338..f5d9d105 100644 --- a/couchpotato/core/plugins/log/__init__.py +++ b/couchpotato/core/plugins/log/__init__.py @@ -1,5 +1,6 @@ from .main import Logging + def start(): return Logging() diff --git a/couchpotato/core/plugins/log/main.py b/couchpotato/core/plugins/log/main.py index dc8f740f..2f471586 100644 --- a/couchpotato/core/plugins/log/main.py +++ b/couchpotato/core/plugins/log/main.py @@ -42,7 +42,7 @@ class Logging(Plugin): 'desc': 'Log errors', 'params': { 'type': {'desc': 'Type of logging, default "error"'}, - '**kwargs': {'type':'object', 'desc': 'All other params will be printed in the log string.'}, + '**kwargs': {'type': 'object', 'desc': 'All other params will be printed in the log string.'}, } }) diff --git a/couchpotato/core/plugins/manage/__init__.py b/couchpotato/core/plugins/manage/__init__.py index 912296b4..c992dee6 100644 --- a/couchpotato/core/plugins/manage/__init__.py +++ b/couchpotato/core/plugins/manage/__init__.py @@ -1,5 +1,6 @@ from .main import Manage + def start(): return Manage() diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index a764f317..3b684f9a 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -14,6 +14,7 @@ import traceback log = CPLog(__name__) + class Manage(Plugin): in_progress = False diff --git a/couchpotato/core/plugins/profile/__init__.py b/couchpotato/core/plugins/profile/__init__.py index ac19b018..c07bc7c5 100644 --- a/couchpotato/core/plugins/profile/__init__.py +++ b/couchpotato/core/plugins/profile/__init__.py @@ -1,5 +1,6 @@ from .main import ProfilePlugin + def start(): return ProfilePlugin() diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 9ff3ead2..374bc927 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -149,7 +149,7 @@ class ProfilePlugin(Plugin): self.forceDefaults() success = True - except Exception, e: + except Exception as e: message = log.error('Failed deleting Profile: %s', e) db.expire_all() diff --git a/couchpotato/core/plugins/quality/__init__.py b/couchpotato/core/plugins/quality/__init__.py index e1b97ad0..2630f1a3 100644 --- a/couchpotato/core/plugins/quality/__init__.py +++ b/couchpotato/core/plugins/quality/__init__.py @@ -1,5 +1,6 @@ from .main import QualityPlugin + def start(): return QualityPlugin() diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index ed12d011..a49c17e2 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -133,10 +133,10 @@ class QualityPlugin(Plugin): # Create single quality profile prof = db.query(Profile).filter( - Profile.core == True - ).filter( - Profile.types.any(quality = qual) - ).all() + Profile.core == True + ).filter( + Profile.types.any(quality = qual) + ).all() if not prof: log.info('Creating profile: %s', q.get('label')) diff --git a/couchpotato/core/plugins/release/__init__.py b/couchpotato/core/plugins/release/__init__.py index b6a667c2..08c6a57c 100644 --- a/couchpotato/core/plugins/release/__init__.py +++ b/couchpotato/core/plugins/release/__init__.py @@ -1,5 +1,6 @@ from .main import Release + def start(): return Release() diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index e5b6d92f..2d67d12c 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -96,7 +96,6 @@ class Release(Plugin): identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier']) - done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) # Add movie @@ -237,15 +236,15 @@ class Release(Plugin): success = self.download(data = item, media = rel.movie.to_dict({ 'profile': {'types': {'quality': {}}}, 'releases': {'status': {}, 'quality': {}}, - 'library': {'titles': {}, 'files':{}}, + 'library': {'titles': {}, 'files': {}}, 'files': {} }), manual = True) - if success == True: - db.expunge_all() - rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again?? + db.expunge_all() + if success: fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name']) + return { 'success': success == True } @@ -317,8 +316,8 @@ class Release(Plugin): # If renamer isn't used, mark media done if finished or release downloaded else: if media['status_id'] == active_status.get('id'): - finished = next((True for profile_type in media['profile']['types'] if \ - profile_type['quality_id'] == rls.quality.id and profile_type['finish']), False) + finished = next((True for profile_type in media['profile']['types'] + if profile_type['quality_id'] == rls.quality.id and profile_type['finish']), False) if finished: log.info('Renamer disabled, marking media as finished: %s', log_movie) @@ -423,7 +422,7 @@ class Release(Plugin): .filter(Relea.movie_id == id) \ .all() - releases = [r.to_dict({'info':{}, 'files':{}}) for r in releases_raw] + releases = [r.to_dict({'info': {}, 'files': {}}) for r in releases_raw] releases = sorted(releases, key = lambda k: k['info'].get('score', 0), reverse = True) return releases @@ -449,6 +448,7 @@ class Release(Plugin): for info in rel.info: item[info.identifier] = info.value + release_name = None if rel.files: for file_item in rel.files: if file_item.type.identifier == 'movie': @@ -456,6 +456,7 @@ class Release(Plugin): break else: release_name = item['name'] + #update status in Db log.debug('Marking release %s as %s', (release_name, status.get("label"))) rel.status_id = status.get('id') diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index 8b602cbd..e238f5eb 100755 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -1,6 +1,7 @@ from couchpotato.core.plugins.renamer.main import Renamer import os + def start(): return Renamer() @@ -136,7 +137,8 @@ config = [{ 'default': 'link', 'type': 'dropdown', 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], - 'description': ('Link, Copy or Move after download completed.', 'Link first tries hard link, then sym link and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'), + 'description': ('Link, Copy or Move after download completed.', + 'Link first tries hard link, then sym link and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'), 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 68a8fe45..9ba1c09b 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -20,6 +20,7 @@ import traceback log = CPLog(__name__) + class Renamer(Plugin): renaming_started = False @@ -33,7 +34,7 @@ class Renamer(Plugin): 'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'}, 'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'}, 'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'}, - 'downloader' : {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'}, + 'downloader': {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'}, 'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in media_folder. \'downloader\' is required with this option.'}, 'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''}, }, @@ -274,25 +275,25 @@ class Renamer(Plugin): name_the = movie_name[4:] + ', The' replacements = { - 'ext': 'mkv', - 'namethe': name_the.strip(), - 'thename': movie_name.strip(), - 'year': library['year'], - 'first': name_the[0].upper(), - 'quality': group['meta_data']['quality']['label'], - 'quality_type': group['meta_data']['quality_type'], - 'video': group['meta_data'].get('video'), - 'audio': group['meta_data'].get('audio'), - 'group': group['meta_data']['group'], - 'source': group['meta_data']['source'], - 'resolution_width': group['meta_data'].get('resolution_width'), - 'resolution_height': group['meta_data'].get('resolution_height'), - 'audio_channels': group['meta_data'].get('audio_channels'), - 'imdb_id': library['identifier'], - 'cd': '', - 'cd_nr': '', - 'mpaa': library['info'].get('mpaa', ''), - 'category': category_label, + 'ext': 'mkv', + 'namethe': name_the.strip(), + 'thename': movie_name.strip(), + 'year': library['year'], + 'first': name_the[0].upper(), + 'quality': group['meta_data']['quality']['label'], + 'quality_type': group['meta_data']['quality_type'], + 'video': group['meta_data'].get('video'), + 'audio': group['meta_data'].get('audio'), + 'group': group['meta_data']['group'], + 'source': group['meta_data']['source'], + 'resolution_width': group['meta_data'].get('resolution_width'), + 'resolution_height': group['meta_data'].get('resolution_height'), + 'audio_channels': group['meta_data'].get('audio_channels'), + 'imdb_id': library['identifier'], + 'cd': '', + 'cd_nr': '', + 'mpaa': library['info'].get('mpaa', ''), + 'category': category_label, } for file_type in group['files']: @@ -434,7 +435,7 @@ class Renamer(Plugin): movie.status_id = done_status.get('id') movie.last_edit = int(time.time()) db.commit() - except Exception, e: + except Exception as e: log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc())) # Go over current movie releases @@ -526,7 +527,7 @@ class Renamer(Plugin): for delete_folder in delete_folders: try: self.deleteEmptyFolder(delete_folder, show_error = False) - except Exception, e: + except Exception as e: log.error('Failed to delete folder: %s %s', (e, traceback.format_exc())) # Rename all files marked @@ -592,7 +593,7 @@ class Renamer(Plugin): # Break if CP wants to shut down if self.shuttingDown(): break - + self.renaming_started = False def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False): @@ -1160,7 +1161,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False) extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))) del rar_handle - except Exception, e: + except Exception as e: log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc())) continue @@ -1169,7 +1170,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if cleanup: try: os.remove(filename) - except Exception, e: + except Exception as e: log.error('Failed to remove %s: %s %s', (filename, e, traceback.format_exc())) continue files.remove(filename) @@ -1182,7 +1183,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) try: self.makeDir(os.path.dirname(move_to)) self.moveFile(leftoverfile, move_to, cleanup) - except Exception, e: + except Exception as e: log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc())) # As we probably tried to overwrite the nfo file, check if it exists and then remove the original if os.path.isfile(move_to): diff --git a/couchpotato/core/plugins/scanner/__init__.py b/couchpotato/core/plugins/scanner/__init__.py index 3d640465..66c6b39c 100644 --- a/couchpotato/core/plugins/scanner/__init__.py +++ b/couchpotato/core/plugins/scanner/__init__.py @@ -1,5 +1,6 @@ from .main import Scanner + def start(): return Scanner() diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 1cb66ce3..4f8f2aba 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -23,7 +23,7 @@ class Scanner(Plugin): ignored_in_path = [os.path.sep + 'extracted' + os.path.sep, 'extracting', '_unpack', '_failed_', '_unknown_', '_exists_', '_failed_remove_', '_failed_rename_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo', - 'thumbs.db', 'ehthumbs.db', 'desktop.ini'] #unpacking, smb-crap, hidden files + 'thumbs.db', 'ehthumbs.db', 'desktop.ini'] #unpacking, smb-crap, hidden files ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films', 'download', 'downloads', 'video_ts', 'audio_ts', 'bdmv', 'certificate'] extensions = { 'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img', 'mdf', 'ts', 'm4v'], @@ -48,7 +48,7 @@ class Scanner(Plugin): 'leftover': ('leftover', 'leftover'), } - file_sizes = { # in MB + file_sizes = { # in MB 'movie': {'min': 300}, 'trailer': {'min': 2, 'max': 250}, 'backdrop': {'min': 0, 'max': 5}, @@ -83,17 +83,17 @@ class Scanner(Plugin): clean = '[ _\,\.\(\)\[\]\-]?(extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \ '|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)' multipart_regex = [ - '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 - '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 - '[ _\.-]+part[ _\.-]*([0-9a-d]+)', #*part1 - '[ _\.-]+dis[ck][ _\.-]*([0-9a-d]+)', #*disk1 - 'cd[ _\.-]*([0-9a-d]+)$', #cd1.ext - 'dvd[ _\.-]*([0-9a-d]+)$', #dvd1.ext - 'part[ _\.-]*([0-9a-d]+)$', #part1.mkv - 'dis[ck][ _\.-]*([0-9a-d]+)$', #disk1.mkv + '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 + '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 + '[ _\.-]+part[ _\.-]*([0-9a-d]+)', #*part1 + '[ _\.-]+dis[ck][ _\.-]*([0-9a-d]+)', #*disk1 + 'cd[ _\.-]*([0-9a-d]+)$', #cd1.ext + 'dvd[ _\.-]*([0-9a-d]+)$', #dvd1.ext + 'part[ _\.-]*([0-9a-d]+)$', #part1.mkv + 'dis[ck][ _\.-]*([0-9a-d]+)$', #disk1.mkv '()[ _\.-]+([0-9]*[abcd]+)(\.....?)$', '([a-z])([0-9]+)(\.....?)$', - '()([ab])(\.....?)$' #*a.mkv + '()([ab])(\.....?)$' #*a.mkv ] cp_imdb = '(.cp.(?Ptt[0-9{7}]+).)' @@ -263,7 +263,7 @@ class Scanner(Plugin): delete_identifiers.append(identifier) # Remove the found files from the leftover stack - leftovers = leftovers - set([ff]) + leftovers -= leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): @@ -445,7 +445,7 @@ class Scanner(Plugin): files = list(group['files']['movie']) for cur_file in files: - if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files + if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files meta = self.getMeta(cur_file) @@ -727,7 +727,9 @@ class Scanner(Plugin): if is_sample: log.debug('Is sample file: %s', filename) return is_sample - def filesizeBetween(self, file, file_size = []): + def filesizeBetween(self, file, file_size = None): + if not file_size: file_size = [] + try: return (file_size.get('min', 0) * 1048576) < os.path.getsize(file) < (file_size.get('max', 100000) * 1048576) except: @@ -875,7 +877,7 @@ class Scanner(Plugin): except: pass - if not cp_guess: # Split name on multiple spaces + if not cp_guess: # Split name on multiple spaces try: movie_name = cleaned.split(' ').pop(0).strip() cp_guess = { diff --git a/couchpotato/core/plugins/score/__init__.py b/couchpotato/core/plugins/score/__init__.py index 2c367f89..a960081c 100644 --- a/couchpotato/core/plugins/score/__init__.py +++ b/couchpotato/core/plugins/score/__init__.py @@ -1,5 +1,6 @@ from .main import Score + def start(): return Score() diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index 895f5fc0..c1f5123a 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -51,6 +51,7 @@ def nameScore(name, year, preferred_words): return score + def nameRatioScore(nzb_name, movie_name): nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) movie_words = re.split('\W+', simplifyString(movie_name)) diff --git a/couchpotato/core/plugins/status/__init__.py b/couchpotato/core/plugins/status/__init__.py index fb5b4cc7..204fbee7 100644 --- a/couchpotato/core/plugins/status/__init__.py +++ b/couchpotato/core/plugins/status/__init__.py @@ -1,5 +1,6 @@ from .main import StatusPlugin + def start(): return StatusPlugin() diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py index b3b37bdc..ad25d3c8 100644 --- a/couchpotato/core/plugins/status/main.py +++ b/couchpotato/core/plugins/status/main.py @@ -33,7 +33,7 @@ class StatusPlugin(Plugin): addEvent('status.get_by_id', self.getById) addEvent('status.all', self.all) addEvent('app.initialize', self.fill) - addEvent('app.load', self.all) # Cache all statuses + addEvent('app.load', self.all) # Cache all statuses addApiView('status.list', self.list, docs = { 'desc': 'Check for available update', diff --git a/couchpotato/core/plugins/subtitle/__init__.py b/couchpotato/core/plugins/subtitle/__init__.py index fcff4cdf..59847aee 100644 --- a/couchpotato/core/plugins/subtitle/__init__.py +++ b/couchpotato/core/plugins/subtitle/__init__.py @@ -1,5 +1,6 @@ from .main import Subtitle + def start(): return Subtitle() diff --git a/couchpotato/core/plugins/trailer/__init__.py b/couchpotato/core/plugins/trailer/__init__.py index d8496b30..e7a6d26e 100644 --- a/couchpotato/core/plugins/trailer/__init__.py +++ b/couchpotato/core/plugins/trailer/__init__.py @@ -1,5 +1,6 @@ from .main import Trailer + def start(): return Trailer() diff --git a/couchpotato/core/plugins/trailer/main.py b/couchpotato/core/plugins/trailer/main.py index e27e3f9f..ba040058 100644 --- a/couchpotato/core/plugins/trailer/main.py +++ b/couchpotato/core/plugins/trailer/main.py @@ -28,7 +28,7 @@ class Trailer(Plugin): destination = os.path.join(group['destination_dir'], filename) if not os.path.isfile(destination): trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) - if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one + if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one os.unlink(trailer_file) continue else: diff --git a/couchpotato/core/plugins/userscript/__init__.py b/couchpotato/core/plugins/userscript/__init__.py index 5df5a801..184f5d79 100644 --- a/couchpotato/core/plugins/userscript/__init__.py +++ b/couchpotato/core/plugins/userscript/__init__.py @@ -1,5 +1,6 @@ from .main import Userscript + def start(): return Userscript() diff --git a/couchpotato/core/plugins/userscript/main.py b/couchpotato/core/plugins/userscript/main.py index 1e220d6f..113c0351 100644 --- a/couchpotato/core/plugins/userscript/main.py +++ b/couchpotato/core/plugins/userscript/main.py @@ -42,7 +42,7 @@ class Userscript(Plugin): 'excludes': fireEvent('userscript.get_excludes', merge = True), } - def getUserScript(self, route, **kwargs): + def getUserScript(self, script_route, **kwargs): klass = self @@ -63,8 +63,7 @@ class Userscript(Plugin): self.redirect(Env.get('api_base') + 'file.cache/couchpotato.user.js') - Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), UserscriptHandler)]) - + Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), script_route), UserscriptHandler)]) def getVersion(self): diff --git a/couchpotato/core/plugins/wizard/__init__.py b/couchpotato/core/plugins/wizard/__init__.py index 78876470..eda6f25a 100644 --- a/couchpotato/core/plugins/wizard/__init__.py +++ b/couchpotato/core/plugins/wizard/__init__.py @@ -1,5 +1,6 @@ from .main import Wizard + def start(): return Wizard() diff --git a/couchpotato/core/providers/automation/bluray/__init__.py b/couchpotato/core/providers/automation/bluray/__init__.py index ed270056..519a7119 100644 --- a/couchpotato/core/providers/automation/bluray/__init__.py +++ b/couchpotato/core/providers/automation/bluray/__init__.py @@ -1,5 +1,6 @@ from .main import Bluray + def start(): return Bluray() diff --git a/couchpotato/core/providers/automation/bluray/main.py b/couchpotato/core/providers/automation/bluray/main.py index d98557ec..ddd7b8ab 100644 --- a/couchpotato/core/providers/automation/bluray/main.py +++ b/couchpotato/core/providers/automation/bluray/main.py @@ -21,7 +21,7 @@ class Bluray(Automation, RSS): page = 0 while True: - page = page + 1 + page += 1 url = self.backlog_url % page data = self.getHTMLData(url) @@ -37,7 +37,7 @@ class Bluray(Automation, RSS): name = table.h3.get_text().lower().split('blu-ray')[0].strip() year = table.small.get_text().split('|')[1].strip() - if not name.find('/') == -1: # make sure it is not a double movie release + if not name.find('/') == -1: # make sure it is not a double movie release continue if tryInt(year) < self.getMinimal('year'): diff --git a/couchpotato/core/providers/automation/cp/__init__.py b/couchpotato/core/providers/automation/cp/__init__.py index a4b55a83..6b2163be 100644 --- a/couchpotato/core/providers/automation/cp/__init__.py +++ b/couchpotato/core/providers/automation/cp/__init__.py @@ -1,5 +1,6 @@ from .main import CP + def start(): return CP() diff --git a/couchpotato/core/providers/automation/flixster/__init__.py b/couchpotato/core/providers/automation/flixster/__init__.py index 1c6c4590..71bd83c0 100644 --- a/couchpotato/core/providers/automation/flixster/__init__.py +++ b/couchpotato/core/providers/automation/flixster/__init__.py @@ -1,5 +1,6 @@ from .main import Flixster + def start(): return Flixster() diff --git a/couchpotato/core/providers/automation/flixster/main.py b/couchpotato/core/providers/automation/flixster/main.py index 7fd2f717..f07ecd6b 100644 --- a/couchpotato/core/providers/automation/flixster/main.py +++ b/couchpotato/core/providers/automation/flixster/main.py @@ -42,6 +42,9 @@ class Flixster(Automation): data = self.getJsonData(self.url % user_id, decode_from = 'iso-8859-1') for movie in data: - movies.append({'title': movie['movie']['title'], 'year': movie['movie']['year'] }) + movies.append({ + 'title': movie['movie']['title'], + 'year': movie['movie']['year'] + }) return movies diff --git a/couchpotato/core/providers/automation/goodfilms/__init__.py b/couchpotato/core/providers/automation/goodfilms/__init__.py index 795e21da..e04ccd0d 100644 --- a/couchpotato/core/providers/automation/goodfilms/__init__.py +++ b/couchpotato/core/providers/automation/goodfilms/__init__.py @@ -1,5 +1,6 @@ from .main import Goodfilms + def start(): return Goodfilms() @@ -25,4 +26,4 @@ config = [{ ], }, ], -}] \ No newline at end of file +}] diff --git a/couchpotato/core/providers/automation/goodfilms/main.py b/couchpotato/core/providers/automation/goodfilms/main.py index e1125615..c4a7bd91 100644 --- a/couchpotato/core/providers/automation/goodfilms/main.py +++ b/couchpotato/core/providers/automation/goodfilms/main.py @@ -35,9 +35,12 @@ class Goodfilms(Automation): data = self.getHTMLData(url) soup = BeautifulSoup(data) - this_watch_list = soup.find_all('div', attrs = { 'class': 'movie', 'data-film-title': True }) + this_watch_list = soup.find_all('div', attrs = { + 'class': 'movie', + 'data-film-title': True + }) - if not this_watch_list: # No Movies + if not this_watch_list: # No Movies break for movie in this_watch_list: diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py index 39bbef0a..ce2ea5ea 100644 --- a/couchpotato/core/providers/automation/imdb/__init__.py +++ b/couchpotato/core/providers/automation/imdb/__init__.py @@ -1,5 +1,6 @@ from .main import IMDB + def start(): return IMDB() diff --git a/couchpotato/core/providers/automation/itunes/__init__.py b/couchpotato/core/providers/automation/itunes/__init__.py index cc5dddc7..13526f43 100644 --- a/couchpotato/core/providers/automation/itunes/__init__.py +++ b/couchpotato/core/providers/automation/itunes/__init__.py @@ -1,5 +1,6 @@ from .main import ITunes + def start(): return ITunes() diff --git a/couchpotato/core/providers/automation/itunes/main.py b/couchpotato/core/providers/automation/itunes/main.py index eb68e348..76763244 100644 --- a/couchpotato/core/providers/automation/itunes/main.py +++ b/couchpotato/core/providers/automation/itunes/main.py @@ -22,7 +22,7 @@ class ITunes(Automation, RSS): urls = splitString(self.conf('automation_urls')) namespace = 'http://www.w3.org/2005/Atom' - namespaceIM = 'http://itunes.apple.com/rss' + namespace_im = 'http://itunes.apple.com/rss' index = -1 for url in urls: @@ -42,10 +42,10 @@ class ITunes(Automation, RSS): rss_movies = self.getElements(data, entry_tag) for movie in rss_movies: - name_tag = str(QName(namespaceIM, 'name')) + name_tag = str(QName(namespace_im, 'name')) name = self.getTextElement(movie, name_tag) - releaseDate_tag = str(QName(namespaceIM, 'releaseDate')) + releaseDate_tag = str(QName(namespace_im, 'releaseDate')) releaseDateText = self.getTextElement(movie, releaseDate_tag) year = datetime.datetime.strptime(releaseDateText, '%Y-%m-%dT00:00:00-07:00').strftime("%Y") diff --git a/couchpotato/core/providers/automation/kinepolis/__init__.py b/couchpotato/core/providers/automation/kinepolis/__init__.py index 24bd4ebb..cc4c5706 100644 --- a/couchpotato/core/providers/automation/kinepolis/__init__.py +++ b/couchpotato/core/providers/automation/kinepolis/__init__.py @@ -1,5 +1,6 @@ from .main import Kinepolis + def start(): return Kinepolis() diff --git a/couchpotato/core/providers/automation/letterboxd/__init__.py b/couchpotato/core/providers/automation/letterboxd/__init__.py index f2b8486b..88bfe6a1 100644 --- a/couchpotato/core/providers/automation/letterboxd/__init__.py +++ b/couchpotato/core/providers/automation/letterboxd/__init__.py @@ -1,5 +1,6 @@ from .main import Letterboxd + def start(): return Letterboxd() diff --git a/couchpotato/core/providers/automation/letterboxd/main.py b/couchpotato/core/providers/automation/letterboxd/main.py index 1f106dd1..ca457a21 100644 --- a/couchpotato/core/providers/automation/letterboxd/main.py +++ b/couchpotato/core/providers/automation/letterboxd/main.py @@ -44,7 +44,7 @@ class Letterboxd(Automation): soup = BeautifulSoup(self.getHTMLData(self.url % username)) - for movie in soup.find_all('a', attrs = { 'class': 'frame' }): + for movie in soup.find_all('a', attrs = {'class': 'frame'}): match = filter(None, self.pattern.split(movie['title'])) movies.append({'title': match[0], 'year': match[1] }) diff --git a/couchpotato/core/providers/automation/moviemeter/__init__.py b/couchpotato/core/providers/automation/moviemeter/__init__.py index aff5d09d..0e9a4edc 100644 --- a/couchpotato/core/providers/automation/moviemeter/__init__.py +++ b/couchpotato/core/providers/automation/moviemeter/__init__.py @@ -1,5 +1,6 @@ from .main import Moviemeter + def start(): return Moviemeter() diff --git a/couchpotato/core/providers/automation/movies_io/__init__.py b/couchpotato/core/providers/automation/movies_io/__init__.py index 9b280930..0361223b 100644 --- a/couchpotato/core/providers/automation/movies_io/__init__.py +++ b/couchpotato/core/providers/automation/movies_io/__init__.py @@ -1,5 +1,6 @@ from .main import MoviesIO + def start(): return MoviesIO() diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 4675fac2..1d3026d3 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -1,5 +1,6 @@ from .main import Rottentomatoes + def start(): return Rottentomatoes() diff --git a/couchpotato/core/providers/automation/rottentomatoes/main.py b/couchpotato/core/providers/automation/rottentomatoes/main.py index 69611705..c873a8e1 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/main.py +++ b/couchpotato/core/providers/automation/rottentomatoes/main.py @@ -8,6 +8,7 @@ import re log = CPLog(__name__) + class Rottentomatoes(Automation, RSS): interval = 1800 diff --git a/couchpotato/core/providers/automation/trakt/__init__.py b/couchpotato/core/providers/automation/trakt/__init__.py index cbaaece3..6ae2806b 100644 --- a/couchpotato/core/providers/automation/trakt/__init__.py +++ b/couchpotato/core/providers/automation/trakt/__init__.py @@ -1,5 +1,6 @@ from .main import Trakt + def start(): return Trakt() diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 89967df1..1aceac40 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -14,6 +14,7 @@ import xml.etree.ElementTree as XMLTree log = CPLog(__name__) + class MultiProvider(Plugin): def __init__(self): @@ -36,8 +37,8 @@ class MultiProvider(Plugin): class Provider(Plugin): - type = None # movie, show, subtitle, trailer, ... - http_time_between_calls = 10 # Default timeout for url requests + type = None # movie, show, subtitle, trailer, ... + http_time_between_calls = 10 # Default timeout for url requests last_available_check = {} is_available = {} @@ -100,15 +101,15 @@ class Provider(Plugin): class YarrProvider(Provider): - protocol = None # nzb, torrent, torrent_magnet + protocol = None # nzb, torrent, torrent_magnet type = 'movie' cat_ids = {} cat_backup_id = None - sizeGb = ['gb', 'gib'] - sizeMb = ['mb', 'mib'] - sizeKb = ['kb', 'kib'] + size_gb = ['gb', 'gib'] + size_mb = ['mb', 'mib'] + size_kb = ['kb', 'kib'] last_login_check = None @@ -223,19 +224,19 @@ class YarrProvider(Provider): def parseSize(self, size): - sizeRaw = size.lower() + size_raw = size.lower() size = tryFloat(re.sub(r'[^0-9.]', '', size).strip()) - for s in self.sizeGb: - if s in sizeRaw: + for s in self.size_gb: + if s in size_raw: return size * 1024 - for s in self.sizeMb: - if s in sizeRaw: + for s in self.size_mb: + if s in size_raw: return size - for s in self.sizeKb: - if s in sizeRaw: + for s in self.size_kb: + if s in size_raw: return size / 1024 return 0 @@ -279,7 +280,7 @@ class ResultList(list): new_result = self.fillResult(result) is_correct = fireEvent('searcher.correct_release', new_result, self.media, self.quality, - imdb_results = self.kwargs.get('imdb_results', False), single = True) + imdb_results = self.kwargs.get('imdb_results', False), single = True) if is_correct and new_result['id'] not in self.result_ids: is_correct_weight = float(is_correct) diff --git a/couchpotato/core/providers/info/_modifier/__init__.py b/couchpotato/core/providers/info/_modifier/__init__.py index 3bdf5e0d..9dfab703 100644 --- a/couchpotato/core/providers/info/_modifier/__init__.py +++ b/couchpotato/core/providers/info/_modifier/__init__.py @@ -1,7 +1,7 @@ from .main import MovieResultModifier -def start(): +def start(): return MovieResultModifier() config = [] diff --git a/couchpotato/core/providers/info/_modifier/main.py b/couchpotato/core/providers/info/_modifier/main.py index c7e3a5ef..88d4381c 100644 --- a/couchpotato/core/providers/info/_modifier/main.py +++ b/couchpotato/core/providers/info/_modifier/main.py @@ -44,13 +44,13 @@ class MovieResultModifier(Plugin): new_results = {} for r in results: type_name = r.get('type', 'movie') + 's' - if not new_results.has_key(type_name): + if type_name not in new_results: new_results[type_name] = [] new_results[type_name].append(r) # Combine movies, needs a cleaner way.. - if new_results.has_key('movies'): + if 'movies' in new_results: new_results['movies'] = self.combineOnIMDB(new_results['movies']) return new_results diff --git a/couchpotato/core/providers/info/couchpotatoapi/__init__.py b/couchpotato/core/providers/info/couchpotatoapi/__init__.py index 37d9eca9..196dde6a 100644 --- a/couchpotato/core/providers/info/couchpotatoapi/__init__.py +++ b/couchpotato/core/providers/info/couchpotatoapi/__init__.py @@ -1,5 +1,6 @@ from .main import CouchPotatoApi + def start(): return CouchPotatoApi() diff --git a/couchpotato/core/providers/info/couchpotatoapi/main.py b/couchpotato/core/providers/info/couchpotatoapi/main.py index cf512816..e5efbcf6 100644 --- a/couchpotato/core/providers/info/couchpotatoapi/main.py +++ b/couchpotato/core/providers/info/couchpotatoapi/main.py @@ -110,5 +110,5 @@ class CouchPotatoApi(MovieProvider): 'X-CP-Version': fireEvent('app.version', single = True), 'X-CP-API': self.api_version, 'X-CP-Time': time.time(), - 'X-CP-Identifier': '+%s' % Env.setting('api_key', 'core')[:10], # Use first 10 as identifier, so we don't need to use IP address in api stats + 'X-CP-Identifier': '+%s' % Env.setting('api_key', 'core')[:10], # Use first 10 as identifier, so we don't need to use IP address in api stats } diff --git a/couchpotato/core/providers/info/omdbapi/__init__.py b/couchpotato/core/providers/info/omdbapi/__init__.py index 765662e9..b7ea3932 100644 --- a/couchpotato/core/providers/info/omdbapi/__init__.py +++ b/couchpotato/core/providers/info/omdbapi/__init__.py @@ -1,5 +1,6 @@ from .main import OMDBAPI + def start(): return OMDBAPI() diff --git a/couchpotato/core/providers/info/themoviedb/__init__.py b/couchpotato/core/providers/info/themoviedb/__init__.py index 66ac536a..b981950e 100644 --- a/couchpotato/core/providers/info/themoviedb/__init__.py +++ b/couchpotato/core/providers/info/themoviedb/__init__.py @@ -1,5 +1,6 @@ from .main import TheMovieDb + def start(): return TheMovieDb() diff --git a/couchpotato/core/providers/info/themoviedb/main.py b/couchpotato/core/providers/info/themoviedb/main.py index fa6896b2..d2f48934 100644 --- a/couchpotato/core/providers/info/themoviedb/main.py +++ b/couchpotato/core/providers/info/themoviedb/main.py @@ -56,7 +56,7 @@ class TheMovieDb(MovieProvider): self.setCache(cache_key, results) return results - except SyntaxError, e: + except SyntaxError as e: log.error('Failed to parse XML response: %s', e) return False diff --git a/couchpotato/core/providers/metadata/wmc/__init__.py b/couchpotato/core/providers/metadata/wmc/__init__.py index 290436c6..167a24d7 100644 --- a/couchpotato/core/providers/metadata/wmc/__init__.py +++ b/couchpotato/core/providers/metadata/wmc/__init__.py @@ -1,5 +1,6 @@ from .main import WindowsMediaCenter + def start(): return WindowsMediaCenter() diff --git a/couchpotato/core/providers/metadata/wmc/main.py b/couchpotato/core/providers/metadata/wmc/main.py index 89258918..f84897b4 100644 --- a/couchpotato/core/providers/metadata/wmc/main.py +++ b/couchpotato/core/providers/metadata/wmc/main.py @@ -1,6 +1,7 @@ from couchpotato.core.providers.metadata.base import MetaDataBase import os + class WindowsMediaCenter(MetaDataBase): def getThumbnailName(self, name, root): diff --git a/couchpotato/core/providers/metadata/xbmc/__init__.py b/couchpotato/core/providers/metadata/xbmc/__init__.py index ea426dba..deb5c908 100644 --- a/couchpotato/core/providers/metadata/xbmc/__init__.py +++ b/couchpotato/core/providers/metadata/xbmc/__init__.py @@ -1,5 +1,6 @@ from .main import XBMC + def start(): return XBMC() diff --git a/couchpotato/core/providers/metadata/xbmc/main.py b/couchpotato/core/providers/metadata/xbmc/main.py index 93e717f8..267b2822 100644 --- a/couchpotato/core/providers/metadata/xbmc/main.py +++ b/couchpotato/core/providers/metadata/xbmc/main.py @@ -132,7 +132,7 @@ class XBMC(MetaDataBase): # Add trailer if found trailer_found = False if data.get('renamed_files'): - for filename in data.get('renamed_files'): + for filename in data.get('renamed_files'): if 'trailer' in filename: trailer = SubElement(nfoxml, 'trailer') trailer.text = toUnicode(filename) diff --git a/couchpotato/core/providers/nzb/binsearch/__init__.py b/couchpotato/core/providers/nzb/binsearch/__init__.py index 1cfb0b73..c80ee6d9 100644 --- a/couchpotato/core/providers/nzb/binsearch/__init__.py +++ b/couchpotato/core/providers/nzb/binsearch/__init__.py @@ -1,5 +1,6 @@ from .main import BinSearch + def start(): return BinSearch() diff --git a/couchpotato/core/providers/nzb/binsearch/main.py b/couchpotato/core/providers/nzb/binsearch/main.py index 54cb1abc..c54dd435 100644 --- a/couchpotato/core/providers/nzb/binsearch/main.py +++ b/couchpotato/core/providers/nzb/binsearch/main.py @@ -18,7 +18,7 @@ class BinSearch(NZBProvider): 'search': 'https://www.binsearch.info/index.php?%s', } - http_time_between_calls = 4 # Seconds + http_time_between_calls = 4 # Seconds def _search(self, movie, quality, results): diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py index e0bf5704..97f1cfad 100644 --- a/couchpotato/core/providers/nzb/newznab/__init__.py +++ b/couchpotato/core/providers/nzb/newznab/__init__.py @@ -1,5 +1,6 @@ from .main import Newznab + def start(): return Newznab() diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index 49c3ae92..deadaa1c 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -25,7 +25,7 @@ class Newznab(NZBProvider, RSS): limits_reached = {} - http_time_between_calls = 1 # Seconds + http_time_between_calls = 1 # Seconds def search(self, movie, quality): hosts = self.getHosts() @@ -180,7 +180,7 @@ class Newznab(NZBProvider, RSS): data = self.urlopen(finalurl, show_error = False) self.limits_reached[host] = False return data - except HTTPError, e: + except HTTPError as e: if e.code == 503: response = e.read().lower() if 'maximum api' in response or 'download limit' in response: diff --git a/couchpotato/core/providers/nzb/nzbclub/__init__.py b/couchpotato/core/providers/nzb/nzbclub/__init__.py index 95eeea13..02a69404 100644 --- a/couchpotato/core/providers/nzb/nzbclub/__init__.py +++ b/couchpotato/core/providers/nzb/nzbclub/__init__.py @@ -1,5 +1,6 @@ from .main import NZBClub + def start(): return NZBClub() diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py index 59382dfd..778cdbcd 100644 --- a/couchpotato/core/providers/nzb/nzbclub/main.py +++ b/couchpotato/core/providers/nzb/nzbclub/main.py @@ -16,7 +16,7 @@ class NZBClub(NZBProvider, RSS): 'search': 'http://www.nzbclub.com/nzbfeed.aspx?%s', } - http_time_between_calls = 4 #seconds + http_time_between_calls = 4 #seconds def _searchOnTitle(self, title, movie, quality, results): diff --git a/couchpotato/core/providers/nzb/nzbindex/__init__.py b/couchpotato/core/providers/nzb/nzbindex/__init__.py index 47461e63..acb53e19 100644 --- a/couchpotato/core/providers/nzb/nzbindex/__init__.py +++ b/couchpotato/core/providers/nzb/nzbindex/__init__.py @@ -1,5 +1,6 @@ from .main import NzbIndex + def start(): return NzbIndex() diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index 17b87fac..a143c199 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -19,7 +19,7 @@ class NzbIndex(NZBProvider, RSS): 'search': 'https://www.nzbindex.com/rss/?%s', } - http_time_between_calls = 1 # Seconds + http_time_between_calls = 1 # Seconds def _searchOnTitle(self, title, movie, quality, results): diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py b/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py index 933aff3e..2f3990de 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py @@ -1,5 +1,6 @@ from .main import OMGWTFNZBs + def start(): return OMGWTFNZBs() diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py index 8cc4a3eb..93925752 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py @@ -18,7 +18,7 @@ class OMGWTFNZBs(NZBProvider, RSS): 'detail_url': 'https://omgwtfnzbs.org/details.php?id=%s', } - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds cat_ids = [ ([15], ['dvdrip']), diff --git a/couchpotato/core/providers/torrent/awesomehd/__init__.py b/couchpotato/core/providers/torrent/awesomehd/__init__.py index de6a2144..6f076703 100644 --- a/couchpotato/core/providers/torrent/awesomehd/__init__.py +++ b/couchpotato/core/providers/torrent/awesomehd/__init__.py @@ -1,5 +1,6 @@ from .main import AwesomeHD + def start(): return AwesomeHD() diff --git a/couchpotato/core/providers/torrent/awesomehd/main.py b/couchpotato/core/providers/torrent/awesomehd/main.py index 79482f2a..ca6a30df 100644 --- a/couchpotato/core/providers/torrent/awesomehd/main.py +++ b/couchpotato/core/providers/torrent/awesomehd/main.py @@ -11,10 +11,10 @@ log = CPLog(__name__) class AwesomeHD(TorrentProvider): urls = { - 'test' : 'https://awesome-hd.net/', - 'detail' : 'https://awesome-hd.net/torrents.php?torrentid=%s', - 'search' : 'https://awesome-hd.net/searchapi.php?action=imdbsearch&passkey=%s&imdb=%s&internal=%s', - 'download' : 'https://awesome-hd.net/torrents.php?action=download&id=%s&authkey=%s&torrent_pass=%s', + 'test': 'https://awesome-hd.net/', + 'detail': 'https://awesome-hd.net/torrents.php?torrentid=%s', + 'search': 'https://awesome-hd.net/searchapi.php?action=imdbsearch&passkey=%s&imdb=%s&internal=%s', + 'download': 'https://awesome-hd.net/torrents.php?action=download&id=%s&authkey=%s&torrent_pass=%s', } http_time_between_calls = 1 diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index c16e6c52..02cc549e 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -14,22 +14,6 @@ class TorrentProvider(YarrProvider): proxy_domain = None proxy_list = [] - def imdbMatch(self, url, imdbId): - if getImdb(url) == imdbId: - return True - - if url[:4] == 'http': - try: - cache_key = md5(url) - data = self.getCache(cache_key, url) - except IOError: - log.error('Failed to open %s.', url) - return False - - return getImdb(data) == imdbId - - return False - def getDomain(self, url = ''): forced_domain = self.conf('domain') @@ -63,9 +47,10 @@ class TorrentProvider(YarrProvider): return cleanHost(self.proxy_domain).rstrip('/') + url - def correctProxy(self): + def correctProxy(self, data): return True + class TorrentMagnetProvider(TorrentProvider): protocol = 'torrent_magnet' diff --git a/couchpotato/core/providers/torrent/bithdtv/__init__.py b/couchpotato/core/providers/torrent/bithdtv/__init__.py index 8c6f97a0..ffc5363f 100644 --- a/couchpotato/core/providers/torrent/bithdtv/__init__.py +++ b/couchpotato/core/providers/torrent/bithdtv/__init__.py @@ -1,5 +1,6 @@ from .main import BiTHDTV + def start(): return BiTHDTV() diff --git a/couchpotato/core/providers/torrent/bithdtv/main.py b/couchpotato/core/providers/torrent/bithdtv/main.py index 0045fb80..90117de4 100644 --- a/couchpotato/core/providers/torrent/bithdtv/main.py +++ b/couchpotato/core/providers/torrent/bithdtv/main.py @@ -7,14 +7,15 @@ import traceback log = CPLog(__name__) + class BiTHDTV(TorrentProvider): urls = { - 'test' : 'http://www.bit-hdtv.com/', - 'login' : 'http://www.bit-hdtv.com/takelogin.php', + 'test': 'http://www.bit-hdtv.com/', + 'login': 'http://www.bit-hdtv.com/takelogin.php', 'login_check': 'http://www.bit-hdtv.com/messages.php', - 'detail' : 'http://www.bit-hdtv.com/details.php?id=%s', - 'search' : 'http://www.bit-hdtv.com/torrents.php?', + 'detail': 'http://www.bit-hdtv.com/details.php?id=%s', + 'search': 'http://www.bit-hdtv.com/torrents.php?', } # Searches for movies only - BiT-HDTV's subcategory and resolution search filters appear to be broken diff --git a/couchpotato/core/providers/torrent/bitsoup/__init__.py b/couchpotato/core/providers/torrent/bitsoup/__init__.py index a36ab08f..da07cc3b 100644 --- a/couchpotato/core/providers/torrent/bitsoup/__init__.py +++ b/couchpotato/core/providers/torrent/bitsoup/__init__.py @@ -1,5 +1,6 @@ from .main import Bitsoup + def start(): return Bitsoup() diff --git a/couchpotato/core/providers/torrent/bitsoup/main.py b/couchpotato/core/providers/torrent/bitsoup/main.py index 0c6c9574..a709c5c1 100644 --- a/couchpotato/core/providers/torrent/bitsoup/main.py +++ b/couchpotato/core/providers/torrent/bitsoup/main.py @@ -12,7 +12,7 @@ class Bitsoup(TorrentProvider): urls = { 'test': 'https://www.bitsoup.me/', - 'login' : 'https://www.bitsoup.me/takelogin.php', + 'login': 'https://www.bitsoup.me/takelogin.php', 'login_check': 'https://www.bitsoup.me/my.php', 'search': 'https://www.bitsoup.me/browse.php?', 'baseurl': 'https://www.bitsoup.me/%s', diff --git a/couchpotato/core/providers/torrent/hdbits/__init__.py b/couchpotato/core/providers/torrent/hdbits/__init__.py index 07ea95d6..f0873798 100644 --- a/couchpotato/core/providers/torrent/hdbits/__init__.py +++ b/couchpotato/core/providers/torrent/hdbits/__init__.py @@ -1,5 +1,6 @@ from .main import HDBits + def start(): return HDBits() diff --git a/couchpotato/core/providers/torrent/hdbits/main.py b/couchpotato/core/providers/torrent/hdbits/main.py index 1d3516fe..1ecc97e4 100644 --- a/couchpotato/core/providers/torrent/hdbits/main.py +++ b/couchpotato/core/providers/torrent/hdbits/main.py @@ -10,11 +10,11 @@ log = CPLog(__name__) class HDBits(TorrentProvider): urls = { - 'test' : 'https://hdbits.org/', - 'login' : 'https://hdbits.org/login/doLogin/', - 'detail' : 'https://hdbits.org/details.php?id=%s&source=browse', - 'search' : 'https://hdbits.org/json_search.php?imdb=%s', - 'download' : 'https://hdbits.org/download.php/%s.torrent?id=%s&passkey=%s&source=details.browse', + 'test': 'https://hdbits.org/', + 'login': 'https://hdbits.org/login/doLogin/', + 'detail': 'https://hdbits.org/details.php?id=%s&source=browse', + 'search': 'https://hdbits.org/json_search.php?imdb=%s', + 'download': 'https://hdbits.org/download.php/%s.torrent?id=%s&passkey=%s&source=details.browse', 'login_check': 'http://hdbits.org/inbox.php', } diff --git a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py index c6702d7f..44fa353e 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py @@ -1,5 +1,6 @@ from main import ILoveTorrents + def start(): return ILoveTorrents() @@ -18,14 +19,14 @@ config = [{ 'type': 'enabler', 'default': False }, - { + { 'name': 'username', 'label': 'Username', 'type': 'string', 'default': '', 'description': 'The user name for your ILT account', }, - { + { 'name': 'password', 'label': 'Password', 'type': 'password', diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 7181016a..6d56ea48 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -14,10 +14,10 @@ class ILoveTorrents(TorrentProvider): urls = { 'download': 'http://www.ilovetorrents.me/%s', 'detail': 'http://www.ilovetorrents.me/%s', - 'search': 'http://www.ilovetorrents.me/browse.php?search=%s&page=%s&cat=%s', - 'test' : 'http://www.ilovetorrents.me/', - 'login' : 'http://www.ilovetorrents.me/takelogin.php', - 'login_check' : 'http://www.ilovetorrents.me' + 'search': 'http://www.ilovetorrents.me/browse.php?search=%s&page=%s&cat=%s', + 'test': 'http://www.ilovetorrents.me/', + 'login': 'http://www.ilovetorrents.me/takelogin.php', + 'login_check': 'http://www.ilovetorrents.me' } cat_ids = [ diff --git a/couchpotato/core/providers/torrent/iptorrents/__init__.py b/couchpotato/core/providers/torrent/iptorrents/__init__.py index 6cb2dead..579d7974 100644 --- a/couchpotato/core/providers/torrent/iptorrents/__init__.py +++ b/couchpotato/core/providers/torrent/iptorrents/__init__.py @@ -1,5 +1,6 @@ from .main import IPTorrents + def start(): return IPTorrents() diff --git a/couchpotato/core/providers/torrent/iptorrents/main.py b/couchpotato/core/providers/torrent/iptorrents/main.py index a02b97f2..e29aa415 100644 --- a/couchpotato/core/providers/torrent/iptorrents/main.py +++ b/couchpotato/core/providers/torrent/iptorrents/main.py @@ -11,11 +11,11 @@ log = CPLog(__name__) class IPTorrents(TorrentProvider): urls = { - 'test' : 'http://www.iptorrents.com/', - 'base_url' : 'http://www.iptorrents.com', - 'login' : 'http://www.iptorrents.com/torrents/', + 'test': 'http://www.iptorrents.com/', + 'base_url': 'http://www.iptorrents.com', + 'login': 'http://www.iptorrents.com/torrents/', 'login_check': 'http://www.iptorrents.com/inbox.php', - 'search' : 'http://www.iptorrents.com/torrents/?l%d=1%s&q=%s&qf=ti&p=%d', + 'search': 'http://www.iptorrents.com/torrents/?l%d=1%s&q=%s&qf=ti&p=%d', } cat_ids = [ diff --git a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py index 0b79c81a..ffe1040b 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py @@ -1,5 +1,6 @@ from .main import KickAssTorrents + def start(): return KickAssTorrents() diff --git a/couchpotato/core/providers/torrent/kickasstorrents/main.py b/couchpotato/core/providers/torrent/kickasstorrents/main.py index 50f14ce2..f96e9812 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/main.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/main.py @@ -24,7 +24,7 @@ class KickAssTorrents(TorrentMagnetProvider): (['dvd'], ['dvdr']), ] - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds cat_backup_id = None proxy_list = [ @@ -45,7 +45,7 @@ class KickAssTorrents(TorrentMagnetProvider): try: html = BeautifulSoup(data) - resultdiv = html.find('div', attrs = {'class':'tabs'}) + resultdiv = html.find('div', attrs = {'class': 'tabs'}) for result in resultdiv.find_all('div', recursive = False): if result.get('id').lower().strip('tab-') not in cat_ids: continue @@ -107,7 +107,6 @@ class KickAssTorrents(TorrentMagnetProvider): return tryInt(age) - def isEnabled(self): return super(KickAssTorrents, self).isEnabled() and self.getDomain() diff --git a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py index 66b3ea76..9593c44e 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py @@ -1,5 +1,6 @@ from main import PassThePopcorn + def start(): return PassThePopcorn() diff --git a/couchpotato/core/providers/torrent/passthepopcorn/main.py b/couchpotato/core/providers/torrent/passthepopcorn/main.py index 5cd3aed6..48a7e2a1 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/main.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/main.py @@ -15,12 +15,12 @@ log = CPLog(__name__) class PassThePopcorn(TorrentProvider): urls = { - 'domain': 'https://tls.passthepopcorn.me', - 'detail': 'https://tls.passthepopcorn.me/torrents.php?torrentid=%s', - 'torrent': 'https://tls.passthepopcorn.me/torrents.php', - 'login': 'https://tls.passthepopcorn.me/ajax.php?action=login', - 'login_check': 'https://tls.passthepopcorn.me/ajax.php?action=login', - 'search': 'https://tls.passthepopcorn.me/search/%s/0/7/%d' + 'domain': 'https://tls.passthepopcorn.me', + 'detail': 'https://tls.passthepopcorn.me/torrents.php?torrentid=%s', + 'torrent': 'https://tls.passthepopcorn.me/torrents.php', + 'login': 'https://tls.passthepopcorn.me/ajax.php?action=login', + 'login_check': 'https://tls.passthepopcorn.me/ajax.php?action=login', + 'search': 'https://tls.passthepopcorn.me/search/%s/0/7/%d' } http_time_between_calls = 2 diff --git a/couchpotato/core/providers/torrent/publichd/__init__.py b/couchpotato/core/providers/torrent/publichd/__init__.py index ace12880..3c20c51f 100644 --- a/couchpotato/core/providers/torrent/publichd/__init__.py +++ b/couchpotato/core/providers/torrent/publichd/__init__.py @@ -1,5 +1,6 @@ from .main import PublicHD + def start(): return PublicHD() diff --git a/couchpotato/core/providers/torrent/publichd/main.py b/couchpotato/core/providers/torrent/publichd/main.py index 7b497fd9..b7c32fba 100644 --- a/couchpotato/core/providers/torrent/publichd/main.py +++ b/couchpotato/core/providers/torrent/publichd/main.py @@ -76,7 +76,7 @@ class PublicHD(TorrentMagnetProvider): try: full_description = self.urlopen(item['detail_url']) html = BeautifulSoup(full_description) - nfo_pre = html.find('div', attrs = {'id':'torrmain'}) + nfo_pre = html.find('div', attrs = {'id': 'torrmain'}) description = toUnicode(nfo_pre.text) if nfo_pre else '' except: log.error('Failed getting more info for %s', item['name']) diff --git a/couchpotato/core/providers/torrent/sceneaccess/__init__.py b/couchpotato/core/providers/torrent/sceneaccess/__init__.py index 4b675573..3fa5d97f 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/__init__.py +++ b/couchpotato/core/providers/torrent/sceneaccess/__init__.py @@ -1,5 +1,6 @@ from .main import SceneAccess + def start(): return SceneAccess() diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 4113d389..a43c49bb 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -25,7 +25,7 @@ class SceneAccess(TorrentProvider): ([8], ['dvdr']), ] - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds def _search(self, movie, quality, results): diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index 8cf9f86c..3cb35d14 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -1,5 +1,6 @@ from main import ThePirateBay + def start(): return ThePirateBay() diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index b967d5f0..46508506 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -12,15 +12,15 @@ log = CPLog(__name__) class ThePirateBay(TorrentMagnetProvider): urls = { - 'detail': '%s/torrent/%s', - 'search': '%s/search/%s/%s/7/%s' + 'detail': '%s/torrent/%s', + 'search': '%s/search/%s/%s/7/%s' } cat_ids = [ - ([207], ['720p', '1080p']), - ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), - ([201, 207], ['brrip']), - ([202], ['dvdr']) + ([207], ['720p', '1080p']), + ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), + ([201, 207], ['brrip']), + ([202], ['dvdr']) ] cat_backup_id = 200 @@ -111,7 +111,7 @@ class ThePirateBay(TorrentMagnetProvider): def getMoreInfo(self, item): full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) html = BeautifulSoup(full_description) - nfo_pre = html.find('div', attrs = {'class':'nfo'}) + nfo_pre = html.find('div', attrs = {'class': 'nfo'}) description = toUnicode(nfo_pre.text) if nfo_pre else '' item['description'] = description diff --git a/couchpotato/core/providers/torrent/torrentbytes/__init__.py b/couchpotato/core/providers/torrent/torrentbytes/__init__.py index 712eac85..79dec932 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/__init__.py +++ b/couchpotato/core/providers/torrent/torrentbytes/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentBytes + def start(): return TorrentBytes() diff --git a/couchpotato/core/providers/torrent/torrentbytes/main.py b/couchpotato/core/providers/torrent/torrentbytes/main.py index 63f82768..91939e78 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/main.py +++ b/couchpotato/core/providers/torrent/torrentbytes/main.py @@ -11,12 +11,12 @@ log = CPLog(__name__) class TorrentBytes(TorrentProvider): urls = { - 'test' : 'https://www.torrentbytes.net/', - 'login' : 'https://www.torrentbytes.net/takelogin.php', - 'login_check' : 'https://www.torrentbytes.net/inbox.php', - 'detail' : 'https://www.torrentbytes.net/details.php?id=%s', - 'search' : 'https://www.torrentbytes.net/browse.php?search=%s&cat=%d', - 'download' : 'https://www.torrentbytes.net/download.php?id=%s&name=%s', + 'test': 'https://www.torrentbytes.net/', + 'login': 'https://www.torrentbytes.net/takelogin.php', + 'login_check': 'https://www.torrentbytes.net/inbox.php', + 'detail': 'https://www.torrentbytes.net/details.php?id=%s', + 'search': 'https://www.torrentbytes.net/browse.php?search=%s&cat=%d', + 'download': 'https://www.torrentbytes.net/download.php?id=%s&name=%s', } cat_ids = [ diff --git a/couchpotato/core/providers/torrent/torrentday/__init__.py b/couchpotato/core/providers/torrent/torrentday/__init__.py index d98bb917..133ec914 100644 --- a/couchpotato/core/providers/torrent/torrentday/__init__.py +++ b/couchpotato/core/providers/torrent/torrentday/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentDay + def start(): return TorrentDay() diff --git a/couchpotato/core/providers/torrent/torrentday/main.py b/couchpotato/core/providers/torrent/torrentday/main.py index ffd88f86..6d343234 100644 --- a/couchpotato/core/providers/torrent/torrentday/main.py +++ b/couchpotato/core/providers/torrent/torrentday/main.py @@ -23,7 +23,7 @@ class TorrentDay(TorrentProvider): ([5], ['bd50']), ] - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds def _searchOnTitle(self, title, movie, quality, results): diff --git a/couchpotato/core/providers/torrent/torrentleech/__init__.py b/couchpotato/core/providers/torrent/torrentleech/__init__.py index c788477f..e64d4baa 100644 --- a/couchpotato/core/providers/torrent/torrentleech/__init__.py +++ b/couchpotato/core/providers/torrent/torrentleech/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentLeech + def start(): return TorrentLeech() diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 017829bd..ea6158df 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -12,12 +12,12 @@ log = CPLog(__name__) class TorrentLeech(TorrentProvider): urls = { - 'test' : 'http://www.torrentleech.org/', - 'login' : 'http://www.torrentleech.org/user/account/login/', + 'test': 'http://www.torrentleech.org/', + 'login': 'http://www.torrentleech.org/user/account/login/', 'login_check': 'http://torrentleech.org/user/messages', - 'detail' : 'http://www.torrentleech.org/torrent/%s', - 'search' : 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', - 'download' : 'http://www.torrentleech.org%s', + 'detail': 'http://www.torrentleech.org/torrent/%s', + 'search': 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', + 'download': 'http://www.torrentleech.org%s', } cat_ids = [ diff --git a/couchpotato/core/providers/torrent/torrentpotato/__init__.py b/couchpotato/core/providers/torrent/torrentpotato/__init__.py index 5054f98c..03795ffb 100644 --- a/couchpotato/core/providers/torrent/torrentpotato/__init__.py +++ b/couchpotato/core/providers/torrent/torrentpotato/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentPotato + def start(): return TorrentPotato() diff --git a/couchpotato/core/providers/torrent/torrentpotato/main.py b/couchpotato/core/providers/torrent/torrentpotato/main.py index a76c0c8f..eaaf8d2c 100644 --- a/couchpotato/core/providers/torrent/torrentpotato/main.py +++ b/couchpotato/core/providers/torrent/torrentpotato/main.py @@ -15,7 +15,7 @@ class TorrentPotato(TorrentProvider): urls = {} limits_reached = {} - http_time_between_calls = 1 # Seconds + http_time_between_calls = 1 # Seconds def search(self, movie, quality): hosts = self.getHosts() diff --git a/couchpotato/core/providers/torrent/torrentshack/__init__.py b/couchpotato/core/providers/torrent/torrentshack/__init__.py index 4171fc49..0e552116 100644 --- a/couchpotato/core/providers/torrent/torrentshack/__init__.py +++ b/couchpotato/core/providers/torrent/torrentshack/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentShack + def start(): return TorrentShack() diff --git a/couchpotato/core/providers/torrent/torrentshack/main.py b/couchpotato/core/providers/torrent/torrentshack/main.py index 03a5f762..669c28fd 100644 --- a/couchpotato/core/providers/torrent/torrentshack/main.py +++ b/couchpotato/core/providers/torrent/torrentshack/main.py @@ -11,12 +11,12 @@ log = CPLog(__name__) class TorrentShack(TorrentProvider): urls = { - 'test' : 'https://torrentshack.net/', - 'login' : 'https://torrentshack.net/login.php', + 'test': 'https://torrentshack.net/', + 'login': 'https://torrentshack.net/login.php', 'login_check': 'https://torrentshack.net/inbox.php', - 'detail' : 'https://torrentshack.net/torrent/%s', - 'search' : 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', - 'download' : 'https://torrentshack.net/%s', + 'detail': 'https://torrentshack.net/torrent/%s', + 'search': 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', + 'download': 'https://torrentshack.net/%s', } cat_ids = [ diff --git a/couchpotato/core/providers/torrent/yify/__init__.py b/couchpotato/core/providers/torrent/yify/__init__.py index 99c1162e..8734af26 100644 --- a/couchpotato/core/providers/torrent/yify/__init__.py +++ b/couchpotato/core/providers/torrent/yify/__init__.py @@ -1,5 +1,6 @@ from main import Yify + def start(): return Yify() diff --git a/couchpotato/core/providers/torrent/yify/main.py b/couchpotato/core/providers/torrent/yify/main.py index 0b5e8b87..bb94016c 100644 --- a/couchpotato/core/providers/torrent/yify/main.py +++ b/couchpotato/core/providers/torrent/yify/main.py @@ -9,13 +9,13 @@ log = CPLog(__name__) class Yify(TorrentMagnetProvider): urls = { - 'test' : '%s/api', - 'search' : '%s/api/list.json?keywords=%s&quality=%s', + 'test': '%s/api', + 'search': '%s/api/list.json?keywords=%s&quality=%s', 'detail': '%s/api/movie.json?id=%s' } - http_time_between_calls = 1 #seconds - + http_time_between_calls = 1 #seconds + proxy_list = [ 'https://yify-torrents.im', 'http://yify.unlocktorrent.com', @@ -51,7 +51,7 @@ class Yify(TorrentMagnetProvider): 'id': result['MovieID'], 'name': title, 'url': result['TorrentMagnetUrl'], - 'detail_url': self.urls['detail'] % (self.getDomain(),result['MovieID']), + 'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']), 'size': self.parseSize(result['Size']), 'seeders': tryInt(result['TorrentSeeds']), 'leechers': tryInt(result['TorrentPeers']) diff --git a/couchpotato/core/providers/trailer/hdtrailers/__init__.py b/couchpotato/core/providers/trailer/hdtrailers/__init__.py index 016db7a2..83b93004 100644 --- a/couchpotato/core/providers/trailer/hdtrailers/__init__.py +++ b/couchpotato/core/providers/trailer/hdtrailers/__init__.py @@ -1,5 +1,6 @@ from .main import HDTrailers + def start(): return HDTrailers() diff --git a/couchpotato/core/providers/trailer/hdtrailers/main.py b/couchpotato/core/providers/trailer/hdtrailers/main.py index 14b85549..cba7609f 100644 --- a/couchpotato/core/providers/trailer/hdtrailers/main.py +++ b/couchpotato/core/providers/trailer/hdtrailers/main.py @@ -29,7 +29,7 @@ class HDTrailers(TrailerProvider): log.debug('No page found for: %s', movie_name) data = None - result_data = {'480p':[], '720p':[], '1080p':[]} + result_data = {'480p': [], '720p': [], '1080p': []} if not data: return result_data diff --git a/couchpotato/core/providers/userscript/allocine/__init__.py b/couchpotato/core/providers/userscript/allocine/__init__.py index e451996f..cb2ba992 100644 --- a/couchpotato/core/providers/userscript/allocine/__init__.py +++ b/couchpotato/core/providers/userscript/allocine/__init__.py @@ -1,5 +1,6 @@ from .main import AlloCine + def start(): return AlloCine() diff --git a/couchpotato/core/providers/userscript/appletrailers/__init__.py b/couchpotato/core/providers/userscript/appletrailers/__init__.py index e8078f47..075217a8 100644 --- a/couchpotato/core/providers/userscript/appletrailers/__init__.py +++ b/couchpotato/core/providers/userscript/appletrailers/__init__.py @@ -1,5 +1,6 @@ from .main import AppleTrailers + def start(): return AppleTrailers() diff --git a/couchpotato/core/providers/userscript/criticker/__init__.py b/couchpotato/core/providers/userscript/criticker/__init__.py index 129d878f..ae24aa1e 100644 --- a/couchpotato/core/providers/userscript/criticker/__init__.py +++ b/couchpotato/core/providers/userscript/criticker/__init__.py @@ -1,5 +1,6 @@ from .main import Criticker + def start(): return Criticker() diff --git a/couchpotato/core/providers/userscript/filmweb/__init__.py b/couchpotato/core/providers/userscript/filmweb/__init__.py index 8ead54d6..3098610c 100644 --- a/couchpotato/core/providers/userscript/filmweb/__init__.py +++ b/couchpotato/core/providers/userscript/filmweb/__init__.py @@ -1,5 +1,6 @@ from .main import Filmweb + def start(): return Filmweb() diff --git a/couchpotato/core/providers/userscript/flickchart/__init__.py b/couchpotato/core/providers/userscript/flickchart/__init__.py index 89d45d9c..18a88ffe 100644 --- a/couchpotato/core/providers/userscript/flickchart/__init__.py +++ b/couchpotato/core/providers/userscript/flickchart/__init__.py @@ -1,5 +1,6 @@ from .main import Flickchart + def start(): return Flickchart() diff --git a/couchpotato/core/providers/userscript/imdb/__init__.py b/couchpotato/core/providers/userscript/imdb/__init__.py index f10505da..c25319b7 100644 --- a/couchpotato/core/providers/userscript/imdb/__init__.py +++ b/couchpotato/core/providers/userscript/imdb/__init__.py @@ -1,5 +1,6 @@ from .main import IMDB + def start(): return IMDB() diff --git a/couchpotato/core/providers/userscript/letterboxd/__init__.py b/couchpotato/core/providers/userscript/letterboxd/__init__.py index c8c17977..2fd89000 100644 --- a/couchpotato/core/providers/userscript/letterboxd/__init__.py +++ b/couchpotato/core/providers/userscript/letterboxd/__init__.py @@ -1,5 +1,6 @@ from .main import Letterboxd + def start(): return Letterboxd() diff --git a/couchpotato/core/providers/userscript/moviemeter/__init__.py b/couchpotato/core/providers/userscript/moviemeter/__init__.py index 5e3813c4..7a05a75a 100644 --- a/couchpotato/core/providers/userscript/moviemeter/__init__.py +++ b/couchpotato/core/providers/userscript/moviemeter/__init__.py @@ -1,5 +1,6 @@ from .main import MovieMeter + def start(): return MovieMeter() diff --git a/couchpotato/core/providers/userscript/moviesio/__init__.py b/couchpotato/core/providers/userscript/moviesio/__init__.py index 473f847d..e29e8d08 100644 --- a/couchpotato/core/providers/userscript/moviesio/__init__.py +++ b/couchpotato/core/providers/userscript/moviesio/__init__.py @@ -1,5 +1,6 @@ from .main import MoviesIO + def start(): return MoviesIO() diff --git a/couchpotato/core/providers/userscript/rottentomatoes/__init__.py b/couchpotato/core/providers/userscript/rottentomatoes/__init__.py index ee8266eb..363f103e 100644 --- a/couchpotato/core/providers/userscript/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/userscript/rottentomatoes/__init__.py @@ -1,5 +1,6 @@ from .main import RottenTomatoes + def start(): return RottenTomatoes() diff --git a/couchpotato/core/providers/userscript/sharethe/__init__.py b/couchpotato/core/providers/userscript/sharethe/__init__.py index 7661f761..3cf393af 100644 --- a/couchpotato/core/providers/userscript/sharethe/__init__.py +++ b/couchpotato/core/providers/userscript/sharethe/__init__.py @@ -1,5 +1,6 @@ from .main import ShareThe + def start(): return ShareThe() diff --git a/couchpotato/core/providers/userscript/tmdb/__init__.py b/couchpotato/core/providers/userscript/tmdb/__init__.py index be33372c..c77330c3 100644 --- a/couchpotato/core/providers/userscript/tmdb/__init__.py +++ b/couchpotato/core/providers/userscript/tmdb/__init__.py @@ -1,5 +1,6 @@ from .main import TMDB + def start(): return TMDB() diff --git a/couchpotato/core/providers/userscript/trakt/__init__.py b/couchpotato/core/providers/userscript/trakt/__init__.py index ff67c1ec..39c17c32 100644 --- a/couchpotato/core/providers/userscript/trakt/__init__.py +++ b/couchpotato/core/providers/userscript/trakt/__init__.py @@ -1,5 +1,6 @@ from .main import Trakt + def start(): return Trakt() diff --git a/couchpotato/core/providers/userscript/whiwa/__init__.py b/couchpotato/core/providers/userscript/whiwa/__init__.py index 6577ae33..c8fd3c9d 100644 --- a/couchpotato/core/providers/userscript/whiwa/__init__.py +++ b/couchpotato/core/providers/userscript/whiwa/__init__.py @@ -1,5 +1,6 @@ from .main import WHiWA + def start(): return WHiWA() diff --git a/couchpotato/core/providers/userscript/youteather/__init__.py b/couchpotato/core/providers/userscript/youteather/__init__.py index a07bf56b..f31e911e 100644 --- a/couchpotato/core/providers/userscript/youteather/__init__.py +++ b/couchpotato/core/providers/userscript/youteather/__init__.py @@ -1,5 +1,6 @@ from .main import YouTheater + def start(): return YouTheater() diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index 8601c2b4..ef6e8a5c 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -18,6 +18,7 @@ options_defaults["shortnames"] = True # http://elixir.ematia.de/trac/wiki/Recipes/MultipleDatabasesOneMetadata __session__ = None + class SetEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, set): @@ -40,6 +41,7 @@ class JsonType(TypeDecorator): def process_result_value(self, value, dialect): return json.loads(value if value else '{}') + class MutableDict(Mutable, dict): @classmethod @@ -78,7 +80,7 @@ class Movie(Entity): such as trailers, nfo, thumbnails""" last_edit = Field(Integer, default = lambda: int(time.time()), index = True) - type = 'movie' # Compat tv branch + type = 'movie' # Compat tv branch library = ManyToOne('Library', cascade = 'delete, delete-orphan', single_parent = True) status = ManyToOne('Status') @@ -87,7 +89,8 @@ class Movie(Entity): releases = OneToMany('Release', cascade = 'all, delete-orphan') files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) -Media = Movie # Compat tv branch +Media = Movie # Compat tv branch + class Library(Entity): """""" @@ -215,6 +218,7 @@ class Profile(Entity): return orig_dict + class Category(Entity): """""" using_options(order_by = 'order') diff --git a/couchpotato/environment.py b/couchpotato/environment.py index b393ef94..5fb43299 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -6,6 +6,7 @@ from sqlalchemy.orm import scoped_session from sqlalchemy.orm.session import sessionmaker import os + class Env(object): _appname = 'CouchPotato' diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 36d43564..5c175201 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -18,6 +18,7 @@ import time import traceback import warnings + def getOptions(base_path, args): # Options @@ -52,6 +53,7 @@ def getOptions(base_path, args): return options + # Tornado monkey patch logging.. def _log(status_code, request): @@ -121,7 +123,6 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En os.rmdir(backup) total_backups -= 1 - # Register environment settings Env.set('app_dir', toUnicode(base_path)) Env.set('data_dir', toUnicode(data_dir)) @@ -234,10 +235,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En 'ssl_key': Env.setting('ssl_key', default = None), } - # Load the app application = Application([], - log_function = lambda x : None, + log_function = lambda x: None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, @@ -250,9 +250,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers - (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler - (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key - (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs + (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler + (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key + (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), @@ -267,37 +267,32 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ - ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) + ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) - # Load configs & plugins loader = Env.get('loader') loader.preload(root = toUnicode(base_path)) loader.run() - # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order = True) - # Go go go! from tornado.ioloop import IOLoop loop = IOLoop.current() - # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') - if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { - "certfile": config['ssl_cert'], - "keyfile": config['ssl_key'], + 'certfile': config['ssl_cert'], + 'keyfile': config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive = True) @@ -309,7 +304,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En try: server.listen(config['port'], config['host']) loop.start() - except Exception, e: + except Exception as e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e From 04539edb457add1dfe94e64c102ed9df16120110 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 16:47:49 +0100 Subject: [PATCH 05/42] Update library: APScheduler --- libs/apscheduler/__init__.py | 2 +- libs/apscheduler/jobstores/shelve_store.py | 14 ++++++++++---- libs/apscheduler/scheduler.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/libs/apscheduler/__init__.py b/libs/apscheduler/__init__.py index d93e1b3b..71cc53db 100644 --- a/libs/apscheduler/__init__.py +++ b/libs/apscheduler/__init__.py @@ -1,3 +1,3 @@ -version_info = (2, 1, 1) +version_info = (2, 1, 2) version = '.'.join(str(n) for n in version_info[:3]) release = '.'.join(str(n) for n in version_info) diff --git a/libs/apscheduler/jobstores/shelve_store.py b/libs/apscheduler/jobstores/shelve_store.py index bd68333f..d1be58f9 100644 --- a/libs/apscheduler/jobstores/shelve_store.py +++ b/libs/apscheduler/jobstores/shelve_store.py @@ -21,7 +21,10 @@ class ShelveJobStore(JobStore): self.jobs = [] self.path = path self.pickle_protocol = pickle_protocol - self.store = shelve.open(path, 'c', self.pickle_protocol) + self._open_store() + + def _open_store(self): + self.store = shelve.open(self.path, 'c', self.pickle_protocol) def _generate_id(self): id = None @@ -33,7 +36,8 @@ class ShelveJobStore(JobStore): def add_job(self, job): job.id = self._generate_id() self.store[job.id] = job.__getstate__() - self.store.sync() + self.store.close() + self._open_store() self.jobs.append(job) def update_job(self, job): @@ -41,11 +45,13 @@ class ShelveJobStore(JobStore): job_dict['next_run_time'] = job.next_run_time job_dict['runs'] = job.runs self.store[job.id] = job_dict - self.store.sync() + self.store.close() + self._open_store() def remove_job(self, job): del self.store[job.id] - self.store.sync() + self.store.close() + self._open_store() self.jobs.remove(job) def load_jobs(self): diff --git a/libs/apscheduler/scheduler.py b/libs/apscheduler/scheduler.py index d6afcad2..319037a9 100644 --- a/libs/apscheduler/scheduler.py +++ b/libs/apscheduler/scheduler.py @@ -586,7 +586,10 @@ class Scheduler(object): wait_seconds = time_difference(next_wakeup_time, now) logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time, wait_seconds) - self._wakeup.wait(wait_seconds) + try: + self._wakeup.wait(wait_seconds) + except IOError: # Catch errno 514 on some Linux kernels + pass self._wakeup.clear() elif self.standalone: logger.debug('No jobs left; shutting down scheduler') @@ -594,7 +597,10 @@ class Scheduler(object): break else: logger.debug('No jobs; waiting until a job is added') - self._wakeup.wait() + try: + self._wakeup.wait() + except IOError: # Catch errno 514 on some Linux kernels + pass self._wakeup.clear() logger.info('Scheduler has been shut down') From f318524070beb21a1f763982ed3f37a011631229 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 16:50:21 +0100 Subject: [PATCH 06/42] Update library: html5lib --- libs/html5lib/__init__.py | 2 +- libs/html5lib/inputstream.py | 7 ++++++- libs/html5lib/treebuilders/__init__.py | 0 libs/html5lib/treebuilders/_base.py | 0 libs/html5lib/treebuilders/etree.py | 0 libs/html5lib/treewalkers/lxmletree.py | 6 +----- 6 files changed, 8 insertions(+), 7 deletions(-) mode change 100755 => 100644 libs/html5lib/treebuilders/__init__.py mode change 100755 => 100644 libs/html5lib/treebuilders/_base.py mode change 100755 => 100644 libs/html5lib/treebuilders/etree.py diff --git a/libs/html5lib/__init__.py b/libs/html5lib/__init__.py index 66c1a8eb..19a4b7d6 100644 --- a/libs/html5lib/__init__.py +++ b/libs/html5lib/__init__.py @@ -20,4 +20,4 @@ from .serializer import serialize __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder", "getTreeWalker", "serialize"] -__version__ = "0.99" +__version__ = "0.999" diff --git a/libs/html5lib/inputstream.py b/libs/html5lib/inputstream.py index 004bdd4a..9e03b931 100644 --- a/libs/html5lib/inputstream.py +++ b/libs/html5lib/inputstream.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, unicode_literals from six import text_type +from six.moves import http_client import codecs import re @@ -118,7 +119,11 @@ class BufferedStream(object): def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True): - if hasattr(source, "read"): + if isinstance(source, http_client.HTTPResponse): + # Work around Python bug #20007: read(0) closes the connection. + # http://bugs.python.org/issue20007 + isUnicode = False + elif hasattr(source, "read"): isUnicode = isinstance(source.read(0), text_type) else: isUnicode = isinstance(source, text_type) diff --git a/libs/html5lib/treebuilders/__init__.py b/libs/html5lib/treebuilders/__init__.py old mode 100755 new mode 100644 diff --git a/libs/html5lib/treebuilders/_base.py b/libs/html5lib/treebuilders/_base.py old mode 100755 new mode 100644 diff --git a/libs/html5lib/treebuilders/etree.py b/libs/html5lib/treebuilders/etree.py old mode 100755 new mode 100644 diff --git a/libs/html5lib/treewalkers/lxmletree.py b/libs/html5lib/treewalkers/lxmletree.py index 375cc2e8..bc934ac0 100644 --- a/libs/html5lib/treewalkers/lxmletree.py +++ b/libs/html5lib/treewalkers/lxmletree.py @@ -87,10 +87,6 @@ class FragmentWrapper(object): self.tail = ensure_str(self.obj.tail) else: self.tail = None - self.isstring = isinstance(obj, str) or isinstance(obj, bytes) - # Support for bytes here is Py2 - if self.isstring: - self.obj = ensure_str(self.obj) def __getattr__(self, name): return getattr(self.obj, name) @@ -143,7 +139,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker): elif isinstance(node, Doctype): return _base.DOCTYPE, node.name, node.public_id, node.system_id - elif isinstance(node, FragmentWrapper) and node.isstring: + elif isinstance(node, FragmentWrapper) and not hasattr(node, "tag"): return _base.TEXT, node.obj elif node.tag == etree.Comment: From b47a94852a0f1608af1c5448eaeb59a27d7d03b0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 16:55:58 +0100 Subject: [PATCH 07/42] Update library: tornado --- libs/tornado/__init__.py | 4 +- libs/tornado/auth.py | 27 +++--- libs/tornado/escape.py | 11 ++- libs/tornado/gen.py | 1 - libs/tornado/ioloop.py | 26 ++++-- libs/tornado/log.py | 115 ++++++++++++++----------- libs/tornado/platform/asyncio.py | 8 +- libs/tornado/platform/caresresolver.py | 1 + libs/tornado/platform/twisted.py | 1 + libs/tornado/web.py | 9 ++ 10 files changed, 123 insertions(+), 80 deletions(-) diff --git a/libs/tornado/__init__.py b/libs/tornado/__init__.py index bec636f3..c41ec97b 100755 --- a/libs/tornado/__init__.py +++ b/libs/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "3.2b1" -version_info = (3, 2, 0, -98) +version = "3.2" +version_info = (3, 2, 0, 0) diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index f2080f1e..9baac9ba 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -36,7 +36,6 @@ Example usage for Google OpenID:: class GoogleLoginHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): if self.get_argument("openid.mode", None): @@ -607,7 +606,6 @@ class TwitterMixin(OAuthMixin): class TwitterLoginHandler(tornado.web.RequestHandler, tornado.auth.TwitterMixin): - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): if self.get_argument("oauth_token", None): @@ -669,7 +667,6 @@ class TwitterMixin(OAuthMixin): class MainHandler(tornado.web.RequestHandler, tornado.auth.TwitterMixin): @tornado.web.authenticated - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): new_entry = yield self.twitter_request( @@ -748,7 +745,6 @@ class FriendFeedMixin(OAuthMixin): class FriendFeedLoginHandler(tornado.web.RequestHandler, tornado.auth.FriendFeedMixin): - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): if self.get_argument("oauth_token", None): @@ -793,7 +789,6 @@ class FriendFeedMixin(OAuthMixin): class MainHandler(tornado.web.RequestHandler, tornado.auth.FriendFeedMixin): @tornado.web.authenticated - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): new_entry = yield self.friendfeed_request( @@ -877,7 +872,6 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): class GoogleLoginHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): if self.get_argument("openid.mode", None): @@ -949,7 +943,10 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): class GoogleOAuth2Mixin(OAuth2Mixin): - """Google authentication using OAuth2.""" + """Google authentication using OAuth2. + + .. versionadded:: 3.2 + """ _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth" _OAUTH_ACCESS_TOKEN_URL = "https://accounts.google.com/o/oauth2/token" _OAUTH_NO_CALLBACKS = False @@ -961,22 +958,22 @@ class GoogleOAuth2Mixin(OAuth2Mixin): Example usage:: - class GoogleOAuth2LoginHandler(LoginHandler, tornado.auth.GoogleOAuth2Mixin): - @tornado.web.asynchronous + class GoogleOAuth2LoginHandler(LoginHandler, + tornado.auth.GoogleOAuth2Mixin): @tornado.gen.coroutine def get(self): - if self.get_argument("code", False): + if self.get_argument('code', False): user = yield self.get_authenticated_user( redirect_uri='http://your.site.com/auth/google', - code=self.get_argument("code")) + code=self.get_argument('code')) # Save the user with e.g. set_secure_cookie else: yield self.authorize_redirect( redirect_uri='http://your.site.com/auth/google', - client_id=self.settings["google_consumer_key"], - scope=['openid', 'email'], + client_id=self.settings['google_oauth']['key'], + scope=['profile', 'email'], response_type='code', - extra_params={"approval_prompt": "auto"}) + extra_params={'approval_prompt': 'auto'}) """ http = self.get_auth_http_client() body = urllib_parse.urlencode({ @@ -1234,7 +1231,6 @@ class FacebookGraphMixin(OAuth2Mixin): Example usage:: class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin): - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): if self.get_argument("code", False): @@ -1321,7 +1317,6 @@ class FacebookGraphMixin(OAuth2Mixin): class MainHandler(tornado.web.RequestHandler, tornado.auth.FacebookGraphMixin): @tornado.web.authenticated - @tornado.web.asynchronous @tornado.gen.coroutine def get(self): new_entry = yield self.facebook_request( diff --git a/libs/tornado/escape.py b/libs/tornado/escape.py index 302e556f..95c0f24e 100755 --- a/libs/tornado/escape.py +++ b/libs/tornado/escape.py @@ -55,7 +55,16 @@ _XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"', def xhtml_escape(value): - """Escapes a string so it is valid within HTML or XML.""" + """Escapes a string so it is valid within HTML or XML. + + Escapes the characters ``<``, ``>``, ``"``, ``'``, and ``&``. + When used in attribute values the escaped strings must be enclosed + in quotes. + + .. versionchanged:: 3.2 + + Added the single quote to the list of escaped characters. + """ return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], to_basestring(value)) diff --git a/libs/tornado/gen.py b/libs/tornado/gen.py index 21f692ab..aa931b45 100755 --- a/libs/tornado/gen.py +++ b/libs/tornado/gen.py @@ -59,7 +59,6 @@ For more complicated interfaces, `Task` can be split into two parts: `Callback` and `Wait`:: class GenAsyncHandler2(RequestHandler): - @asynchronous @gen.coroutine def get(self): http_client = AsyncHTTPClient() diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 0477ade0..e7b84dd7 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -301,6 +301,22 @@ class IOLoop(Configurable): """ raise NotImplementedError() + def _setup_logging(self): + """The IOLoop catches and logs exceptions, so it's + important that log output be visible. However, python's + default behavior for non-root loggers (prior to python + 3.2) is to print an unhelpful "no handlers could be + found" message rather than the actual log entry, so we + must explicitly configure logging if we've made it this + far without anything. + + This method should be called from start() in subclasses. + """ + if not any([logging.getLogger().handlers, + logging.getLogger('tornado').handlers, + logging.getLogger('tornado.application').handlers]): + logging.basicConfig() + def stop(self): """Stop the I/O loop. @@ -550,15 +566,7 @@ class PollIOLoop(IOLoop): action if action is not None else signal.SIG_DFL) def start(self): - if not logging.getLogger().handlers: - # The IOLoop catches and logs exceptions, so it's - # important that log output be visible. However, python's - # default behavior for non-root loggers (prior to python - # 3.2) is to print an unhelpful "no handlers could be - # found" message rather than the actual log entry, so we - # must explicitly configure logging if we've made it this - # far without anything. - logging.basicConfig() + self._setup_logging() if self._stopped: self._stopped = False return diff --git a/libs/tornado/log.py b/libs/tornado/log.py index bc6898c8..36c3dd40 100755 --- a/libs/tornado/log.py +++ b/libs/tornado/log.py @@ -60,6 +60,13 @@ def _stderr_supports_color(): return color +def _safe_unicode(s): + try: + return _unicode(s) + except UnicodeDecodeError: + return repr(s) + + class LogFormatter(logging.Formatter): """Log formatter used in Tornado. @@ -73,23 +80,37 @@ class LogFormatter(logging.Formatter): `tornado.options.parse_command_line` (unless ``--logging=none`` is used). """ - DEFAULT_PREFIX_FORMAT = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' + DEFAULT_FORMAT = '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s' DEFAULT_DATE_FORMAT = '%y%m%d %H:%M:%S' + DEFAULT_COLORS = { + logging.DEBUG: 4, # Blue + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + } - def __init__(self, color=True, prefix_fmt=None, datefmt=None): + def __init__(self, color=True, fmt=DEFAULT_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, colors=DEFAULT_COLORS): r""" - :arg bool color: Enables color support - :arg string prefix_fmt: Log message prefix format. - Prefix is a part of the log message, directly preceding the actual - message text. + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code :arg string datefmt: Datetime format. Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + + .. versionchanged:: 3.2 + + Added ``fmt`` and ``datefmt`` arguments. """ - self.__prefix_fmt = prefix_fmt if prefix_fmt is not None else self.DEFAULT_PREFIX_FORMAT - datefmt = datefmt if datefmt is not None else self.DEFAULT_DATE_FORMAT logging.Formatter.__init__(self, datefmt=datefmt) - self._color = color and _stderr_supports_color() - if self._color: + self._fmt = fmt + + self._colors = {} + if color and _stderr_supports_color(): # The curses module has some str/bytes confusion in # python3. Until version 3.2.3, most methods return # bytes, but only accept strings. In addition, we want to @@ -101,62 +122,56 @@ class LogFormatter(logging.Formatter): curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): fg_color = unicode_type(fg_color, "ascii") - self._colors = { - logging.DEBUG: unicode_type(curses.tparm(fg_color, 4), # Blue - "ascii"), - logging.INFO: unicode_type(curses.tparm(fg_color, 2), # Green - "ascii"), - logging.WARNING: unicode_type(curses.tparm(fg_color, 3), # Yellow - "ascii"), - logging.ERROR: unicode_type(curses.tparm(fg_color, 1), # Red - "ascii"), - } + + for levelno, code in colors.items(): + self._colors[levelno] = unicode_type(curses.tparm(fg_color, code), "ascii") self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") + else: + self._normal = '' def format(self, record): try: - record.message = record.getMessage() + message = record.getMessage() + assert isinstance(message, basestring_type) # guaranteed by logging + # Encoding notes: The logging module prefers to work with character + # strings, but only enforces that log messages are instances of + # basestring. In python 2, non-ascii bytestrings will make + # their way through the logging framework until they blow up with + # an unhelpful decoding error (with this formatter it happens + # when we attach the prefix, but there are other opportunities for + # exceptions further along in the framework). + # + # If a byte string makes it this far, convert it to unicode to + # ensure it will make it out to the logs. Use repr() as a fallback + # to ensure that all byte strings can be converted successfully, + # but don't do it by default so we don't add extra quotes to ascii + # bytestrings. This is a bit of a hacky place to do this, but + # it's worth it since the encoding errors that would otherwise + # result are so useless (and tornado is fond of using utf8-encoded + # byte strings whereever possible). + record.message = _safe_unicode(message) except Exception as e: record.message = "Bad message (%r): %r" % (e, record.__dict__) - assert isinstance(record.message, basestring_type) # guaranteed by logging + record.asctime = self.formatTime(record, self.datefmt) - prefix = self.__prefix_fmt % record.__dict__ - if self._color: - prefix = (self._colors.get(record.levelno, self._normal) + - prefix + self._normal) - # Encoding notes: The logging module prefers to work with character - # strings, but only enforces that log messages are instances of - # basestring. In python 2, non-ascii bytestrings will make - # their way through the logging framework until they blow up with - # an unhelpful decoding error (with this formatter it happens - # when we attach the prefix, but there are other opportunities for - # exceptions further along in the framework). - # - # If a byte string makes it this far, convert it to unicode to - # ensure it will make it out to the logs. Use repr() as a fallback - # to ensure that all byte strings can be converted successfully, - # but don't do it by default so we don't add extra quotes to ascii - # bytestrings. This is a bit of a hacky place to do this, but - # it's worth it since the encoding errors that would otherwise - # result are so useless (and tornado is fond of using utf8-encoded - # byte strings whereever possible). - def safe_unicode(s): - try: - return _unicode(s) - except UnicodeDecodeError: - return repr(s) + if record.levelno in self._colors: + record.color = self._colors[record.levelno] + record.end_color = self._normal + else: + record.color = record.end_color = '' + + formatted = self._fmt % record.__dict__ - formatted = prefix + " " + safe_unicode(record.message) if record.exc_info: if not record.exc_text: record.exc_text = self.formatException(record.exc_info) if record.exc_text: - # exc_text contains multiple lines. We need to safe_unicode + # exc_text contains multiple lines. We need to _safe_unicode # each line separately so that non-utf8 bytes don't cause # all the newlines to turn into '\n'. lines = [formatted.rstrip()] - lines.extend(safe_unicode(ln) for ln in record.exc_text.split('\n')) + lines.extend(_safe_unicode(ln) for ln in record.exc_text.split('\n')) formatted = '\n'.join(lines) return formatted.replace("\n", "\n ") diff --git a/libs/tornado/platform/asyncio.py b/libs/tornado/platform/asyncio.py index 09b2bf3d..162b3673 100644 --- a/libs/tornado/platform/asyncio.py +++ b/libs/tornado/platform/asyncio.py @@ -8,6 +8,8 @@ python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOMain (the tests log a few warnings with AsyncIOMainLoop because they leave some unfinished callbacks on the event loop that fail when it resumes) """ + +from __future__ import absolute_import, division, print_function, with_statement import asyncio import datetime import functools @@ -34,7 +36,10 @@ class BaseAsyncIOLoop(IOLoop): for fd in list(self.handlers): self.remove_handler(fd) if all_fds: - os.close(fd) + try: + os.close(fd) + except OSError: + pass if self.close_loop: self.asyncio_loop.close() @@ -86,6 +91,7 @@ class BaseAsyncIOLoop(IOLoop): self.handlers[fd](fd, events) def start(self): + self._setup_logging() self.asyncio_loop.run_forever() def stop(self): diff --git a/libs/tornado/platform/caresresolver.py b/libs/tornado/platform/caresresolver.py index 7c16705d..c4648c22 100755 --- a/libs/tornado/platform/caresresolver.py +++ b/libs/tornado/platform/caresresolver.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, with_statement import pycares import socket diff --git a/libs/tornado/platform/twisted.py b/libs/tornado/platform/twisted.py index 86ef71b9..0c8a3105 100755 --- a/libs/tornado/platform/twisted.py +++ b/libs/tornado/platform/twisted.py @@ -456,6 +456,7 @@ class TwistedIOLoop(tornado.ioloop.IOLoop): del self.fds[fd] def start(self): + self._setup_logging() self.reactor.run() def stop(self): diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 65a76cd4..b22b11fe 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -516,6 +516,10 @@ class RequestHandler(object): See `clear_cookie` for more information on the path and domain parameters. + + .. versionchanged:: 3.2 + + Added the ``path`` and ``domain`` parameters. """ for name in self.request.cookies: self.clear_cookie(name, path=path, domain=domain) @@ -1850,6 +1854,11 @@ class StaticFileHandler(RequestHandler): class method. Instance methods may use the attributes ``self.path`` ``self.absolute_path``, and ``self.modified``. + Subclasses should only override methods discussed in this section; + overriding other methods is error-prone. Overriding + ``StaticFileHandler.get`` is particularly problematic due to the + tight coupling with ``compute_etag`` and other methods. + To change the way static urls are generated (e.g. to match the behavior of another server or CDN), override `make_static_url`, `parse_url_path`, `get_cache_time`, and/or `get_version`. From 2c43b9a926acb573773da15afc710c3033bffae4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 23:23:40 +0100 Subject: [PATCH 08/42] Update six --- libs/six.py | 336 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 301 insertions(+), 35 deletions(-) diff --git a/libs/six.py b/libs/six.py index a91961ec..7ec7f1be 100644 --- a/libs/six.py +++ b/libs/six.py @@ -1,14 +1,35 @@ """Utilities for writing code that runs on Python 2 and 3""" +# Copyright (c) 2010-2014 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + import operator import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.2.0" +__version__ = "1.5.2" -# True if we are running on Python 3. +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 if PY3: @@ -26,7 +47,7 @@ else: text_type = unicode binary_type = str - if sys.platform == "java": + if sys.platform.startswith("java"): # Jython always uses 32 bits. MAXSIZE = int((1 << 31) - 1) else: @@ -42,7 +63,7 @@ else: else: # 64-bit MAXSIZE = int((1 << 63) - 1) - del X + del X def _add_doc(func, doc): @@ -63,9 +84,9 @@ class _LazyDescr(object): def __get__(self, obj, tp): result = self._resolve() - setattr(obj, self.name, result) + setattr(obj, self.name, result) # Invokes __set__. # This is a bit ugly, but it avoids running this again. - delattr(tp, self.name) + delattr(obj.__class__, self.name) return result @@ -83,6 +104,35 @@ class MovedModule(_LazyDescr): def _resolve(self): return _import_module(self.mod) + def __getattr__(self, attr): + # Hack around the Django autoreloader. The reloader tries to get + # __file__ or __name__ of every module in sys.modules. This doesn't work + # well if this MovedModule is for an module that is unavailable on this + # machine (like winreg on Unix systems). Thus, we pretend __file__ and + # __name__ don't exist if the module hasn't been loaded yet. See issues + # #51 and #53. + if attr in ("__file__", "__name__") and self.mod not in sys.modules: + raise AttributeError + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + class MovedAttribute(_LazyDescr): @@ -110,29 +160,37 @@ class MovedAttribute(_LazyDescr): -class _MovedItems(types.ModuleType): +class _MovedItems(_LazyModule): """Lazy loading of moved objects""" _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("reload_module", "__builtin__", "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -140,12 +198,14 @@ _moved_attributes = [ MovedModule("queue", "Queue"), MovedModule("reprlib", "repr"), MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), MovedModule("tkinter", "Tkinter"), MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), MovedModule("tkinter_colorchooser", "tkColorChooser", @@ -157,14 +217,167 @@ _moved_attributes = [ MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), MovedModule("winreg", "_winreg"), ] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + sys.modules[__name__ + ".moves." + attr.name] = attr del attr -moves = sys.modules["six.moves"] = _MovedItems("moves") +_MovedItems._moved_attributes = _moved_attributes + +moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +sys.modules[__name__ + ".moves.urllib_parse"] = sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse") + + +class Module_six_moves_urllib_error(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +sys.modules[__name__ + ".moves.urllib_error"] = sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +sys.modules[__name__ + ".moves.urllib_request"] = sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +sys.modules[__name__ + ".moves.urllib_response"] = sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +sys.modules[__name__ + ".moves.urllib_robotparser"] = sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + parse = sys.modules[__name__ + ".moves.urllib_parse"] + error = sys.modules[__name__ + ".moves.urllib_error"] + request = sys.modules[__name__ + ".moves.urllib_request"] + response = sys.modules[__name__ + ".moves.urllib_response"] + robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"] + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + + +sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib") def add_move(move): @@ -187,22 +400,28 @@ if PY3: _meth_func = "__func__" _meth_self = "__self__" + _func_closure = "__closure__" _func_code = "__code__" _func_defaults = "__defaults__" + _func_globals = "__globals__" _iterkeys = "keys" _itervalues = "values" _iteritems = "items" + _iterlists = "lists" else: _meth_func = "im_func" _meth_self = "im_self" + _func_closure = "func_closure" _func_code = "func_code" _func_defaults = "func_defaults" + _func_globals = "func_globals" _iterkeys = "iterkeys" _itervalues = "itervalues" _iteritems = "iteritems" + _iterlists = "iterlists" try: @@ -213,18 +432,27 @@ except NameError: next = advance_iterator +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + if PY3: def get_unbound_function(unbound): return unbound - Iterator = object + create_bound_method = types.MethodType - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + Iterator = object else: def get_unbound_function(unbound): return unbound.im_func + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + class Iterator(object): def next(self): @@ -237,21 +465,27 @@ _add_doc(get_unbound_function, get_method_function = operator.attrgetter(_meth_func) get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) get_function_code = operator.attrgetter(_func_code) get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) -def iterkeys(d): +def iterkeys(d, **kw): """Return an iterator over the keys of a dictionary.""" - return iter(getattr(d, _iterkeys)()) + return iter(getattr(d, _iterkeys)(**kw)) -def itervalues(d): +def itervalues(d, **kw): """Return an iterator over the values of a dictionary.""" - return iter(getattr(d, _itervalues)()) + return iter(getattr(d, _itervalues)(**kw)) -def iteritems(d): +def iteritems(d, **kw): """Return an iterator over the (key, value) pairs of a dictionary.""" - return iter(getattr(d, _iteritems)()) + return iter(getattr(d, _iteritems)(**kw)) + +def iterlists(d, **kw): + """Return an iterator over the (key, [values]) pairs of a dictionary.""" + return iter(getattr(d, _iterlists)(**kw)) if PY3: @@ -259,21 +493,33 @@ if PY3: return s.encode("latin-1") def u(s): return s + unichr = chr if sys.version_info[1] <= 1: def int2byte(i): return bytes((i,)) else: # This is about 2x faster than the implementation above on 3.2+ int2byte = operator.methodcaller("to_bytes", 1, "big") + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter import io StringIO = io.StringIO BytesIO = io.BytesIO else: def b(s): return s + # Workaround for standalone backslash def u(s): - return unicode(s, "unicode_escape") + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr int2byte = chr + def byte2int(bs): + return ord(bs[0]) + def indexbytes(buf, i): + return ord(buf[i]) + def iterbytes(buf): + return (ord(byte) for byte in buf) import StringIO StringIO = BytesIO = StringIO.StringIO _add_doc(b, """Byte literal""") @@ -281,8 +527,7 @@ _add_doc(u, """Text literal""") if PY3: - import builtins - exec_ = getattr(builtins, "exec") + exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): @@ -290,22 +535,18 @@ if PY3: raise value.with_traceback(tb) raise value - - print_ = getattr(builtins, "print") - del builtins - else: - def exec_(code, globs=None, locs=None): + def exec_(_code_, _globs_=None, _locs_=None): """Execute code in a namespace.""" - if globs is None: + if _globs_ is None: frame = sys._getframe(1) - globs = frame.f_globals - if locs is None: - locs = frame.f_locals + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals del frame - elif locs is None: - locs = globs - exec("""exec code in globs, locs""") + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") exec_("""def reraise(tp, value, tb=None): @@ -313,14 +554,24 @@ else: """) +print_ = getattr(moves.builtins, "print", None) +if print_ is None: def print_(*args, **kwargs): - """The new-style print function.""" + """The new-style print function for Python 2.4 and 2.5.""" fp = kwargs.pop("file", sys.stdout) if fp is None: return def write(data): if not isinstance(data, basestring): data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) fp.write(data) want_unicode = False sep = kwargs.pop("sep", None) @@ -361,6 +612,21 @@ else: _add_doc(reraise, """Reraise an exception.""") -def with_metaclass(meta, base=object): +def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" - return meta("NewBase", (base,), {}) + return meta("NewBase", bases, {}) + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper From bb3faaf2cdc8b2677c7ee7350726283d04928e74 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 23:27:58 +0100 Subject: [PATCH 09/42] Exception cleanup --- couchpotato/core/downloaders/deluge/main.py | 12 ++++++------ couchpotato/core/downloaders/rtorrent/main.py | 6 +++--- couchpotato/core/downloaders/synology/main.py | 6 +++--- couchpotato/core/downloaders/transmission/main.py | 6 +++--- couchpotato/core/downloaders/utorrent/main.py | 8 ++++---- couchpotato/core/notifications/plex/client.py | 4 ++-- couchpotato/core/notifications/pushbullet/main.py | 2 +- couchpotato/core/plugins/renamer/main.py | 2 +- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/couchpotato/core/downloaders/deluge/main.py b/couchpotato/core/downloaders/deluge/main.py index 53b87d91..c5f80167 100644 --- a/couchpotato/core/downloaders/deluge/main.py +++ b/couchpotato/core/downloaders/deluge/main.py @@ -188,7 +188,7 @@ class DelugeRPC(object): if torrent_id and options['label']: self.client.label.set_torrent(torrent_id, options['label']).get() - except Exception, err: + except Exception as err: log.error('Failed to add torrent magnet %s: %s %s', (torrent, err, traceback.format_exc())) finally: if self.client: @@ -206,7 +206,7 @@ class DelugeRPC(object): if torrent_id and options['label']: self.client.label.set_torrent(torrent_id, options['label']).get() - except Exception, err: + except Exception as err: log.error('Failed to add torrent file %s: %s %s', (filename, err, traceback.format_exc())) finally: if self.client: @@ -219,7 +219,7 @@ class DelugeRPC(object): try: self.connect() ret = self.client.core.get_torrents_status({'id': ids}, ('name', 'hash', 'save_path', 'move_completed_path', 'progress', 'state', 'eta', 'ratio', 'stop_ratio', 'is_seed', 'is_finished', 'paused', 'move_on_completed', 'files')).get() - except Exception, err: + except Exception as err: log.error('Failed to get all torrents: %s %s', (err, traceback.format_exc())) finally: if self.client: @@ -230,7 +230,7 @@ class DelugeRPC(object): try: self.connect() self.client.core.pause_torrent(torrent_ids).get() - except Exception, err: + except Exception as err: log.error('Failed to pause torrent: %s %s', (err, traceback.format_exc())) finally: if self.client: @@ -240,7 +240,7 @@ class DelugeRPC(object): try: self.connect() self.client.core.resume_torrent(torrent_ids).get() - except Exception, err: + except Exception as err: log.error('Failed to resume torrent: %s %s', (err, traceback.format_exc())) finally: if self.client: @@ -251,7 +251,7 @@ class DelugeRPC(object): try: self.connect() ret = self.client.core.remove_torrent(torrent_id, remove_local_data).get() - except Exception, err: + except Exception as err: log.error('Failed to remove torrent: %s %s', (err, traceback.format_exc())) finally: if self.client: diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index f934de5f..ff801db7 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -87,7 +87,7 @@ class rTorrent(Downloader): # Reset group action and disable it group.set_command() group.disable() - except MethodError, err: + except MethodError as err: log.error('Unable to set group options: %s', err.msg) return False @@ -155,7 +155,7 @@ class rTorrent(Downloader): torrent.start() return self.downloadReturnId(torrent_hash) - except Exception, err: + except Exception as err: log.error('Failed to send torrent to rTorrent: %s', err) return False @@ -203,7 +203,7 @@ class rTorrent(Downloader): return release_downloads - except Exception, err: + except Exception as err: log.error('Failed to get status from rTorrent: %s', err) return [] diff --git a/couchpotato/core/downloaders/synology/main.py b/couchpotato/core/downloaders/synology/main.py index 26a4558d..f964f37f 100644 --- a/couchpotato/core/downloaders/synology/main.py +++ b/couchpotato/core/downloaders/synology/main.py @@ -108,11 +108,11 @@ class SynologyRPC(object): if response['success']: log.info('Synology action successfull') return response - except requests.ConnectionError, err: + except requests.ConnectionError as err: log.error('Synology connection error, check your config %s', err) - except requests.HTTPError, err: + except requests.HTTPError as err: log.error('SynologyRPC HTTPError: %s', err) - except Exception, err: + except Exception as err: log.error('Exception: %s', err) finally: return response diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index d3f17598..2daeab46 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -187,10 +187,10 @@ class TransmissionRPC(object): else: log.debug('Unknown failure sending command to Transmission. Return text is: %s', response['result']) return False - except httplib.InvalidURL, err: + except httplib.InvalidURL as err: log.error('Invalid Transmission host, check your config %s', err) return False - except urllib2.HTTPError, err: + except urllib2.HTTPError as err: if err.code == 401: log.error('Invalid Transmission Username or Password, check your config') return False @@ -208,7 +208,7 @@ class TransmissionRPC(object): log.error('Unable to get Transmission Session-Id %s', err) else: log.error('TransmissionRPC HTTPError: %s', err) - except urllib2.URLError, err: + except urllib2.URLError as err: log.error('Unable to connect to Transmission %s', err) def get_session(self): diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index e527230c..f8108492 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -231,14 +231,14 @@ class uTorrentAPI(object): return response else: log.debug('Unknown failure sending command to uTorrent. Return text is: %s', response) - except httplib.InvalidURL, err: + except httplib.InvalidURL as err: log.error('Invalid uTorrent host, check your config %s', err) - except urllib2.HTTPError, err: + except urllib2.HTTPError as err: if err.code == 401: log.error('Invalid uTorrent Username or Password, check your config') else: log.error('uTorrent HTTPError: %s', err) - except urllib2.URLError, err: + except urllib2.URLError as err: log.error('Unable to connect to uTorrent %s', err) return False @@ -304,7 +304,7 @@ class uTorrentAPI(object): #log.debug('uTorrent settings: %s', settings_dict) - except Exception, err: + except Exception as err: log.error('Failed to get settings from uTorrent: %s', err) return settings_dict diff --git a/couchpotato/core/notifications/plex/client.py b/couchpotato/core/notifications/plex/client.py index b873518e..8864230d 100644 --- a/couchpotato/core/notifications/plex/client.py +++ b/couchpotato/core/notifications/plex/client.py @@ -29,7 +29,7 @@ class PlexClientHTTP(PlexClientProtocol): try: self.plex.urlopen(url, headers = headers, timeout = 3, show_error = False) - except Exception, err: + except Exception as err: log.error("Couldn't sent command to Plex: %s", err) return False @@ -68,7 +68,7 @@ class PlexClientJSON(PlexClientProtocol): try: requests.post(url, headers = headers, timeout = 3, data = json.dumps(request)) - except Exception, err: + except Exception as err: log.error("Couldn't sent command to Plex: %s", err) return False diff --git a/couchpotato/core/notifications/pushbullet/main.py b/couchpotato/core/notifications/pushbullet/main.py index bc9fd64c..15120f0b 100644 --- a/couchpotato/core/notifications/pushbullet/main.py +++ b/couchpotato/core/notifications/pushbullet/main.py @@ -79,7 +79,7 @@ class Pushbullet(Notification): data = self.urlopen(self.url % method, headers = headers, data = kwargs) return json.loads(data) - except Exception, ex: + except Exception as ex: log.error('Pushbullet request failed') log.debug(ex) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 9ba1c09b..97e59bde 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -762,7 +762,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) except: log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1))) - except OSError, err: + except OSError as err: # Copying from a filesystem with octal permission to an NTFS file system causes a permission error. In this case ignore it. if not hasattr(os, 'chmod') or err.errno != errno.EPERM: raise From 2d37022525f46bc7c192bfb0181961a2f5b503ae Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 23:28:35 +0100 Subject: [PATCH 10/42] Relative import --- couchpotato/core/providers/torrent/ilovetorrents/__init__.py | 2 +- couchpotato/core/providers/torrent/passthepopcorn/__init__.py | 2 +- couchpotato/core/providers/torrent/thepiratebay/__init__.py | 2 +- couchpotato/core/providers/torrent/yify/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py index 44fa353e..f3f7b479 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py @@ -1,4 +1,4 @@ -from main import ILoveTorrents +from .main import ILoveTorrents def start(): diff --git a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py index 9593c44e..a3e57c79 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py @@ -1,4 +1,4 @@ -from main import PassThePopcorn +from .main import PassThePopcorn def start(): diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index 3cb35d14..8b3921cd 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -1,4 +1,4 @@ -from main import ThePirateBay +from .main import ThePirateBay def start(): diff --git a/couchpotato/core/providers/torrent/yify/__init__.py b/couchpotato/core/providers/torrent/yify/__init__.py index 8734af26..3a359608 100644 --- a/couchpotato/core/providers/torrent/yify/__init__.py +++ b/couchpotato/core/providers/torrent/yify/__init__.py @@ -1,4 +1,4 @@ -from main import Yify +from .main import Yify def start(): From 08e2a3a883d6c1885b6e8a14468815fefe112a8f Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 23:28:48 +0100 Subject: [PATCH 11/42] Import print function --- CouchPotato.py | 1 + 1 file changed, 1 insertion(+) diff --git a/CouchPotato.py b/CouchPotato.py index 069fb65d..252dfe1c 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -1,3 +1,4 @@ +from __future__ import print_function #!/usr/bin/env python from logging import handlers from os.path import dirname From 29a34fef8cce868c4ae9135a69b539d8e058bc13 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 20 Jan 2014 23:58:54 +0100 Subject: [PATCH 12/42] py3k port helpers --- couchpotato/__init__.py | 5 +---- couchpotato/core/_base/updater/main.py | 5 +++-- couchpotato/core/downloaders/utorrent/main.py | 2 +- couchpotato/core/event.py | 2 +- couchpotato/core/helpers/encoding.py | 9 +++++---- couchpotato/core/helpers/request.py | 6 +++--- couchpotato/core/helpers/variable.py | 8 +++++--- couchpotato/core/loader.py | 7 ++++--- couchpotato/core/logger.py | 2 +- couchpotato/core/media/_base/searcher/main.py | 1 + couchpotato/core/media/movie/library/movie/main.py | 3 ++- couchpotato/core/notifications/notifymyandroid/main.py | 3 ++- couchpotato/core/notifications/notifymywp/main.py | 3 ++- couchpotato/core/plugins/base.py | 2 +- couchpotato/core/plugins/browser/main.py | 3 ++- couchpotato/core/plugins/manage/main.py | 2 +- couchpotato/core/plugins/renamer/main.py | 10 ++++++---- couchpotato/core/plugins/scanner/main.py | 7 ++++--- couchpotato/core/plugins/status/main.py | 2 +- couchpotato/core/plugins/subtitle/main.py | 2 +- .../core/providers/automation/letterboxd/main.py | 1 + couchpotato/core/providers/info/couchpotatoapi/main.py | 2 +- couchpotato/core/providers/info/omdbapi/main.py | 2 +- couchpotato/core/providers/info/themoviedb/main.py | 2 +- .../core/providers/torrent/passthepopcorn/main.py | 3 ++- .../core/providers/torrent/thepiratebay/main.py | 3 ++- .../core/providers/torrent/torrentshack/main.py | 3 ++- couchpotato/core/settings/__init__.py | 2 +- 28 files changed, 58 insertions(+), 44 deletions(-) diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index b61cd843..966da0cf 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -60,10 +60,7 @@ addView('', index) # API docs def apiDocs(): - routes = [] - - for route in api.iterkeys(): - routes.append(route) + routes = list(api.keys()) if api_docs.get(''): del api_docs[''] diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 22668d92..ef595ad7 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -15,6 +15,7 @@ import time import traceback import version import zipfile +from six.moves import filter log = CPLog(__name__) @@ -350,8 +351,8 @@ class SourceUpdater(BaseUpdater): try: if os.path.isdir(path): shutil.rmtree(path) - except OSError, inst: - os.chmod(inst.filename, 0777) + except OSError as inst: + os.chmod(inst.filename, 0o777) self.removeDir(path) def getVersion(self): diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index f8108492..e0d6a921 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -261,7 +261,7 @@ class uTorrentAPI(object): def set_torrent(self, hash, params): action = 'action=setprops&hash=%s' % hash - for k, v in params.iteritems(): + for k, v in params.items(): action += '&s=%s&v=%s' % (k, v) return self._request(action) diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 88df65cf..a36c430d 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -29,7 +29,7 @@ def addEvent(name, handler, priority = 100): has_parent = hasattr(handler, 'im_self') parent = None if has_parent: - parent = handler.im_self + parent = handler.__self__ bc = hasattr(parent, 'beforeCall') if bc: parent.beforeCall(handler) diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index 1c418418..0b85f64b 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -5,6 +5,7 @@ import os import re import traceback import unicodedata +import six log = CPLog(__name__) @@ -29,7 +30,7 @@ def toUnicode(original, *args): return original else: try: - return unicode(original, *args) + return six.text_type(original, *args) except: try: return ek(original, *args) @@ -102,10 +103,10 @@ def stripAccents(s): def tryUrlencode(s): - new = u'' + new = six.u('') if isinstance(s, dict): - for key, value in s.iteritems(): - new += u'&%s=%s' % (key, tryUrlencode(value)) + for key, value in s.items(): + new += six.u('&%s=%s') % (key, tryUrlencode(value)) return new[1:] else: diff --git a/couchpotato/core/helpers/request.py b/couchpotato/core/helpers/request.py index fb91f453..8a10f078 100644 --- a/couchpotato/core/helpers/request.py +++ b/couchpotato/core/helpers/request.py @@ -9,7 +9,7 @@ def getParams(params): reg = re.compile('^[a-z0-9_\.]+$') temp = {} - for param, value in sorted(params.iteritems()): + for param, value in sorted(params.items()): nest = re.split("([\[\]]+)", param) if len(nest) > 1: @@ -42,9 +42,9 @@ def dictToList(params): if type(params) is dict: new = {} - for x, value in params.iteritems(): + for x, value in params.items(): try: - new_value = [dictToList(value[k]) for k in sorted(value.iterkeys(), cmp = natcmp)] + new_value = [dictToList(value[k]) for k in sorted(value.keys(), cmp = natcmp)] except: new_value = value diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index b191d0a0..0a47f1de 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -8,6 +8,8 @@ import random import re import string import sys +import six +from six.moves import map, zip log = CPLog(__name__) @@ -19,7 +21,7 @@ def fnEscape(pattern): def link(src, dst): if os.name == 'nt': import ctypes - if ctypes.windll.kernel32.CreateHardLinkW(unicode(dst), unicode(src), 0) == 0: raise ctypes.WinError() + if ctypes.windll.kernel32.CreateHardLinkW(six.text_type(dst), six.text_type(src), 0) == 0: raise ctypes.WinError() else: os.link(src, dst) @@ -27,7 +29,7 @@ def link(src, dst): def symlink(src, dst): if os.name == 'nt': import ctypes - if ctypes.windll.kernel32.CreateSymbolicLinkW(unicode(dst), unicode(src), 1 if os.path.isdir(src) else 0) in [0, 1280]: raise ctypes.WinError() + if ctypes.windll.kernel32.CreateSymbolicLinkW(six.text_type(dst), six.text_type(src), 1 if os.path.isdir(src) else 0) in [0, 1280]: raise ctypes.WinError() else: os.symlink(src, dst) @@ -253,7 +255,7 @@ def randomString(size = 8, chars = string.ascii_uppercase + string.digits): def splitString(str, split_on = ',', clean = True): l = [x.strip() for x in str.split(split_on)] if str else [] - return filter(None, l) if clean else l + return [x for x in l if x] if clean else l def dictIsSubset(a, b): diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index c14b55bd..ee94949f 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -4,6 +4,7 @@ from importlib import import_module import os import sys import traceback +import six log = CPLog(__name__) @@ -37,7 +38,7 @@ class Loader(object): self.paths['custom_plugins'] = (30, '', custom_plugin_dir) # Loop over all paths and add to module list - for plugin_type, plugin_tuple in self.paths.iteritems(): + for plugin_type, plugin_tuple in self.paths.items(): priority, module, dir_name = plugin_tuple self.addFromDir(plugin_type, priority, module, dir_name) @@ -45,7 +46,7 @@ class Loader(object): did_save = 0 for priority in sorted(self.modules): - for module_name, plugin in sorted(self.modules[priority].iteritems()): + for module_name, plugin in sorted(self.modules[priority].items()): # Load module try: @@ -81,7 +82,7 @@ class Loader(object): for filename in os.listdir(root_path): path = os.path.join(root_path, filename) if os.path.isdir(path) and filename[:2] != '__': - if u'__init__.py' in os.listdir(path): + if six.u('__init__.py') in os.listdir(path): new_base_path = ''.join(s + '.' for s in base_path) + filename self.paths[new_base_path.replace('.', '_')] = (priority, new_base_path, path) diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index c2a7a836..fc198cdd 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -51,7 +51,7 @@ class CPLog(object): else: msg = msg % ss(replace_tuple) except Exception as e: - self.logger.error(u'Failed encoding stuff to log "%s": %s' % (msg, e)) + self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e)) if not Env.get('dev'): diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 6ff09728..17dc9b57 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -6,6 +6,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase import datetime import re +from six.moves import filter log = CPLog(__name__) diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index 61d93ae9..3f07305c 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -7,6 +7,7 @@ from couchpotato.core.settings.model import Library, LibraryTitle, File from string import ascii_letters import time import traceback +import six log = CPLog(__name__) @@ -109,7 +110,7 @@ class MovieLibraryPlugin(LibraryBase): t = LibraryTitle( title = title, simple_title = self.simplifyTitle(title), - default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title) + default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title) ) library.titles.append(t) counter += 1 diff --git a/couchpotato/core/notifications/notifymyandroid/main.py b/couchpotato/core/notifications/notifymyandroid/main.py index 92e59562..16465101 100644 --- a/couchpotato/core/notifications/notifymyandroid/main.py +++ b/couchpotato/core/notifications/notifymyandroid/main.py @@ -2,6 +2,7 @@ from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification import pynma +import six log = CPLog(__name__) @@ -26,7 +27,7 @@ class NotifyMyAndroid(Notification): successful = 0 for key in keys: - if not response[str(key)]['code'] == u'200': + if not response[str(key)]['code'] == six.u('200'): log.error('Could not send notification to NotifyMyAndroid (%s). %s', (key, response[key]['message'])) else: successful += 1 diff --git a/couchpotato/core/notifications/notifymywp/main.py b/couchpotato/core/notifications/notifymywp/main.py index 1cdd980a..74010441 100644 --- a/couchpotato/core/notifications/notifymywp/main.py +++ b/couchpotato/core/notifications/notifymywp/main.py @@ -2,6 +2,7 @@ from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from pynmwp import PyNMWP +import six log = CPLog(__name__) @@ -17,7 +18,7 @@ class NotifyMyWP(Notification): response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1) for key in keys: - if not response[key]['Code'] == u'200': + if not response[key]['Code'] == six.u('200'): log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message'])) return False diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 73e41c75..f9280098 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -169,7 +169,7 @@ class Plugin(object): } method = 'post' if len(data) > 0 or files else 'get' - log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.iterkeys()] if isinstance(data, dict) else 'with data')) + log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data')) response = r.request(method, url, verify = False, **kwargs) data = response.content diff --git a/couchpotato/core/plugins/browser/main.py b/couchpotato/core/plugins/browser/main.py index c8f7536b..956a7680 100644 --- a/couchpotato/core/plugins/browser/main.py +++ b/couchpotato/core/plugins/browser/main.py @@ -4,6 +4,7 @@ from couchpotato.core.plugins.base import Plugin import ctypes import os import string +import six if os.name == 'nt': import imp @@ -96,7 +97,7 @@ class FileBrowser(Plugin): def has_hidden_attribute(self, filepath): try: - attrs = ctypes.windll.kernel32.GetFileAttributesW(unicode(filepath)) #@UndefinedVariable + attrs = ctypes.windll.kernel32.GetFileAttributesW(six.text_type(filepath)) #@UndefinedVariable assert attrs != -1 result = bool(attrs & 2) except (AttributeError, AssertionError): diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 3b684f9a..9c7e80c7 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -223,7 +223,7 @@ class Manage(Plugin): groups = fireEvent('scanner.scan', folder = folder, files = files, single = True) if groups: - for group in groups.itervalues(): + for group in groups.values(): if group['library'] and group['library'].get('identifier'): fireEvent('release.add', group = group) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 97e59bde..f9f44d85 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -17,6 +17,8 @@ import re import shutil import time import traceback +import six +from six.moves import filter log = CPLog(__name__) @@ -787,19 +789,19 @@ Remove it if you want it to be renamed (again, or at least let it try again) replacements['cd_nr'] = '' replaced = toUnicode(string) - for x, r in replacements.iteritems(): + for x, r in replacements.items(): if x in ['thename', 'namethe']: continue if r is not None: - replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r)) + replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r)) else: #If information is not available, we don't want the tag in the filename replaced = replaced.replace('<' + x + '>', '') replaced = self.replaceDoubles(replaced.lstrip('. ')) - for x, r in replacements.iteritems(): + for x, r in replacements.items(): if x in ['thename', 'namethe']: - replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r)) + replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r)) replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced) sep = self.conf('foldersep') if folder else self.conf('separator') diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 4f8f2aba..803c63c1 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -15,6 +15,7 @@ import re import threading import time import traceback +from six.moves import filter, map, zip log = CPLog(__name__) @@ -187,7 +188,7 @@ class Scanner(Plugin): # Group files minus extension ignored_identifiers = [] - for identifier, group in movie_files.iteritems(): + for identifier, group in movie_files.items(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) @@ -228,7 +229,7 @@ class Scanner(Plugin): # Group the files based on the identifier delete_identifiers = [] - for identifier, found_files in path_identifiers.iteritems(): + for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) @@ -251,7 +252,7 @@ class Scanner(Plugin): # Group based on folder delete_identifiers = [] - for identifier, found_files in path_identifiers.iteritems(): + for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py index ad25d3c8..3ec2b45a 100644 --- a/couchpotato/core/plugins/status/main.py +++ b/couchpotato/core/plugins/status/main.py @@ -108,7 +108,7 @@ class StatusPlugin(Plugin): db = get_session() - for identifier, label in self.statuses.iteritems(): + for identifier, label in self.statuses.items(): s = db.query(Status).filter_by(identifier = identifier).first() if not s: log.info('Creating status: %s', label) diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index 7504d6a9..56056c0a 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -45,7 +45,7 @@ class Subtitle(Plugin): if self.isDisabled(): return try: - available_languages = sum(group['subtitle_language'].itervalues(), []) + available_languages = sum(group['subtitle_language'].values(), []) downloaded = [] files = [toUnicode(x) for x in group['files']['movie']] log.debug('Searching for subtitles for: %s', files) diff --git a/couchpotato/core/providers/automation/letterboxd/main.py b/couchpotato/core/providers/automation/letterboxd/main.py index ca457a21..efd04991 100644 --- a/couchpotato/core/providers/automation/letterboxd/main.py +++ b/couchpotato/core/providers/automation/letterboxd/main.py @@ -3,6 +3,7 @@ from couchpotato.core.helpers.variable import tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation import re +from six.moves import filter log = CPLog(__name__) diff --git a/couchpotato/core/providers/info/couchpotatoapi/main.py b/couchpotato/core/providers/info/couchpotatoapi/main.py index e5efbcf6..848cbf05 100644 --- a/couchpotato/core/providers/info/couchpotatoapi/main.py +++ b/couchpotato/core/providers/info/couchpotatoapi/main.py @@ -81,7 +81,7 @@ class CouchPotatoApi(MovieProvider): result = self.getJsonData(self.urls['info'] % identifier, headers = self.getRequestHeaders()) if result: - return dict((k, v) for k, v in result.iteritems() if v) + return dict((k, v) for k, v in result.items() if v) return {} diff --git a/couchpotato/core/providers/info/omdbapi/main.py b/couchpotato/core/providers/info/omdbapi/main.py index 7c8be386..8f04d3b6 100755 --- a/couchpotato/core/providers/info/omdbapi/main.py +++ b/couchpotato/core/providers/info/omdbapi/main.py @@ -107,7 +107,7 @@ class OMDBAPI(MovieProvider): 'writers': splitString(movie.get('Writer', '')), 'actors': splitString(movie.get('Actors', '')), } - movie_data = dict((k, v) for k, v in movie_data.iteritems() if v) + movie_data = dict((k, v) for k, v in movie_data.items() if v) except: log.error('Failed parsing IMDB API json: %s', traceback.format_exc()) diff --git a/couchpotato/core/providers/info/themoviedb/main.py b/couchpotato/core/providers/info/themoviedb/main.py index d2f48934..5207eb11 100644 --- a/couchpotato/core/providers/info/themoviedb/main.py +++ b/couchpotato/core/providers/info/themoviedb/main.py @@ -145,7 +145,7 @@ class TheMovieDb(MovieProvider): 'actor_roles': actors } - movie_data = dict((k, v) for k, v in movie_data.iteritems() if v) + movie_data = dict((k, v) for k, v in movie_data.items() if v) # Add alternative names if extended: diff --git a/couchpotato/core/providers/torrent/passthepopcorn/main.py b/couchpotato/core/providers/torrent/passthepopcorn/main.py index 48a7e2a1..66cad33c 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/main.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/main.py @@ -8,6 +8,7 @@ import json import re import time import traceback +import six log = CPLog(__name__) @@ -178,7 +179,7 @@ class PassThePopcorn(TorrentProvider): except KeyError: pass return text # leave as is - return re.sub("&#?\w+;", fixup, u'%s' % text) + return re.sub("&#?\w+;", fixup, six.u('%s') % text) def unicodeToASCII(self, text): import unicodedata diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index 46508506..f8b77787 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -5,6 +5,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentMagnetProvider import re import traceback +import six log = CPLog(__name__) @@ -73,7 +74,7 @@ class ThePirateBay(TorrentMagnetProvider): download = result.find(href = re.compile('magnet:')) try: - size = re.search('Size (?P.+),', unicode(result.select('font.detDesc')[0])).group('size') + size = re.search('Size (?P.+),', six.text_type(result.select('font.detDesc')[0])).group('size') except: continue diff --git a/couchpotato/core/providers/torrent/torrentshack/main.py b/couchpotato/core/providers/torrent/torrentshack/main.py index 669c28fd..f0cd5997 100644 --- a/couchpotato/core/providers/torrent/torrentshack/main.py +++ b/couchpotato/core/providers/torrent/torrentshack/main.py @@ -4,6 +4,7 @@ from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider import traceback +import six log = CPLog(__name__) @@ -53,7 +54,7 @@ class TorrentShack(TorrentProvider): results.append({ 'id': link['href'].replace('torrents.php?torrentid=', ''), - 'name': unicode(link.span.string).translate({ord(u'\xad'): None}), + 'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}), 'url': self.urls['download'] % url['href'], 'detail_url': self.urls['download'] % link['href'], 'size': self.parseSize(result.find_all('td')[4].string), diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index 85dc7a8f..5e4952ae 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -77,7 +77,7 @@ class Settings(object): self.addSection(section_name) - for option_name, option in options.iteritems(): + for option_name, option in options.items(): self.setDefault(section_name, option_name, option.get('default', '')) # Migrate old settings from old location to the new location From bd9a4289d1c87fa591cdf6c6754e87d3b149be08 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 21 Jan 2014 00:19:26 +0100 Subject: [PATCH 13/42] Rename importlib --- couchpotato/core/loader.py | 2 +- libs/importhelper/__init__.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 libs/importhelper/__init__.py diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index ee94949f..6c3a719d 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -1,6 +1,6 @@ from couchpotato.core.event import fireEvent from couchpotato.core.logger import CPLog -from importlib import import_module +from importhelper import import_module import os import sys import traceback diff --git a/libs/importhelper/__init__.py b/libs/importhelper/__init__.py new file mode 100644 index 00000000..ad31a1ac --- /dev/null +++ b/libs/importhelper/__init__.py @@ -0,0 +1,38 @@ +"""Backport of importlib.import_module from 3.x.""" +# While not critical (and in no way guaranteed!), it would be nice to keep this +# code compatible with Python 2.3. +import sys + +def _resolve_name(name, package, level): + """Return the absolute name of the module to be imported.""" + if not hasattr(package, 'rindex'): + raise ValueError("'package' not set to a string") + dot = len(package) + for x in xrange(level, 1, -1): + try: + dot = package.rindex('.', 0, dot) + except ValueError: + raise ValueError("attempted relative import beyond top-level " + "package") + return "%s.%s" % (package[:dot], name) + + +def import_module(name, package=None): + """Import a module. + + The 'package' argument is required when performing a relative import. It + specifies the package to use as the anchor point from which to resolve the + relative import to an absolute import. + + """ + if name.startswith('.'): + if not package: + raise TypeError("relative imports require the 'package' argument") + level = 0 + for character in name: + if character != '.': + break + level += 1 + name = _resolve_name(name[level:], package, level) + __import__(name) + return sys.modules[name] From 63c72853f4d85bcce783292068d52901d11949d3 Mon Sep 17 00:00:00 2001 From: georgewhewell Date: Tue, 21 Jan 2014 12:07:38 +0000 Subject: [PATCH 14/42] Change HDBits provider to use API instead of scraping site --- .../core/providers/torrent/hdbits/__init__.py | 5 -- .../core/providers/torrent/hdbits/main.py | 77 +++++++++++-------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/couchpotato/core/providers/torrent/hdbits/__init__.py b/couchpotato/core/providers/torrent/hdbits/__init__.py index f0873798..1e9aa3ce 100644 --- a/couchpotato/core/providers/torrent/hdbits/__init__.py +++ b/couchpotato/core/providers/torrent/hdbits/__init__.py @@ -22,11 +22,6 @@ config = [{ 'name': 'username', 'default': '', }, - { - 'name': 'password', - 'default': '', - 'type': 'password', - }, { 'name': 'passkey', 'default': '', diff --git a/couchpotato/core/providers/torrent/hdbits/main.py b/couchpotato/core/providers/torrent/hdbits/main.py index 1ecc97e4..2a1b1828 100644 --- a/couchpotato/core/providers/torrent/hdbits/main.py +++ b/couchpotato/core/providers/torrent/hdbits/main.py @@ -1,7 +1,10 @@ -from bs4 import BeautifulSoup from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider + +import re +import json +import requests import traceback log = CPLog(__name__) @@ -10,50 +13,62 @@ log = CPLog(__name__) class HDBits(TorrentProvider): urls = { - 'test': 'https://hdbits.org/', - 'login': 'https://hdbits.org/login/doLogin/', - 'detail': 'https://hdbits.org/details.php?id=%s&source=browse', - 'search': 'https://hdbits.org/json_search.php?imdb=%s', - 'download': 'https://hdbits.org/download.php/%s.torrent?id=%s&passkey=%s&source=details.browse', - 'login_check': 'http://hdbits.org/inbox.php', + 'test' : 'https://hdbits.org/', + 'detail' : 'https://hdbits.org/details.php?id=%s', + 'download' : 'https://hdbits.org/download.php?id=%s&passkey=%s', + 'api': 'https://hdbits.org/api/torrents' + } + + status_codes = { + 1: 'Failure (something bad happened)', + 2: 'SSL Required - You should be on https://', + 3: 'JSON Malformed', + 4: 'Auth data missing', + 5: 'Auth failed' } http_time_between_calls = 1 #seconds + def _post_query(self, **params): + + post_data = { + 'username': self.conf('username'), + 'passkey': self.conf('passkey') + } + post_data.update(params) + + req = requests.post(self.urls['api'], + data=json.dumps(post_data)) + + response = json.loads(req.text) + + if response['status'] != 0: + log.error('Error searching hdbits: %s' + % self.status_codes[response['status']]) + else: + return response['data'] + def _search(self, movie, quality, results): - data = self.getJsonData(self.urls['search'] % movie['library']['identifier']) + match = re.match(r'tt(\d{7})', + movie['library']['identifier']) + + data = self._post_query(imdb={'id': match.group(1)}) if data: try: for result in data: results.append({ 'id': result['id'], - 'name': result['title'], - 'url': self.urls['download'] % (result['id'], result['id'], self.conf('passkey')), + 'name': result['name'], + 'url': self.urls['download'] + % (result['id'], self.conf('passkey')), 'detail_url': self.urls['detail'] % result['id'], 'size': self.parseSize(result['size']), - 'seeders': tryInt(result['seeder']), - 'leechers': tryInt(result['leecher']) + 'seeders': tryInt(result['seeders']), + 'leechers': tryInt(result['leechers']) }) - except: - log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + log.error('Failed getting results from %s: %s', + (self.getName(), traceback.format_exc())) - def getLoginParams(self): - data = self.getHTMLData('https://hdbits.org/login', cache_timeout = 0) - - bs = BeautifulSoup(data) - secret = bs.find('input', attrs = {'name': 'lol'})['value'] - - return { - 'uname': self.conf('username'), - 'password': self.conf('password'), - 'returnto': '/', - 'lol': secret - } - - def loginSuccess(self, output): - return '/logout.php' in output.lower() - - loginCheckSuccess = loginSuccess From f3dee504488c572277f6414e5f5be55089a23d0c Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 21 Jan 2014 21:29:54 +0100 Subject: [PATCH 15/42] Properly handle and trigger events --- couchpotato/core/_base/updater/static/updater.js | 2 +- couchpotato/core/notifications/core/static/notification.js | 2 +- couchpotato/static/scripts/couchpotato.js | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/_base/updater/static/updater.js b/couchpotato/core/_base/updater/static/updater.js index 860ad514..be436ed2 100644 --- a/couchpotato/core/_base/updater/static/updater.js +++ b/couchpotato/core/_base/updater/static/updater.js @@ -24,7 +24,7 @@ var UpdaterBase = new Class({ self.doUpdate(); else { App.unBlockPage(); - App.on('message', 'No updates available'); + App.trigger('message', ['No updates available']); } } }) diff --git a/couchpotato/core/notifications/core/static/notification.js b/couchpotato/core/notifications/core/static/notification.js index a0c3b15c..18d09e76 100644 --- a/couchpotato/core/notifications/core/static/notification.js +++ b/couchpotato/core/notifications/core/static/notification.js @@ -147,7 +147,7 @@ var NotificationBase = new Class({ // Process data if(json){ Array.each(json.result, function(result){ - App.trigger(result.type, result); + App.trigger(result.type, [result]); if(result.message && result.read === undefined) self.showMessage(result.message); }) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index eae865f4..03332281 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -364,7 +364,7 @@ if(!on_complete && typeOf(args) == 'function'){ on_complete = args; - args = {}; + args = []; } // Create parallel callback @@ -372,7 +372,7 @@ self.global_events[name].each(function(handle, nr){ callbacks.push(function(callback){ - var results = handle(args || {}); + var results = handle.apply(handle, args || []); callback(null, results || null); }); @@ -593,4 +593,4 @@ var createSpinner = function(target, options){ }, options); return new Spinner(opts).spin(target); -}; \ No newline at end of file +}; From 405b63acdda23e8aecf29fc7e43d208a97bdf71e Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 21 Jan 2014 21:46:03 +0100 Subject: [PATCH 16/42] Remove unused CP automation provider --- couchpotato/core/providers/automation/cp/__init__.py | 7 ------- couchpotato/core/providers/automation/cp/main.py | 11 ----------- 2 files changed, 18 deletions(-) delete mode 100644 couchpotato/core/providers/automation/cp/__init__.py delete mode 100644 couchpotato/core/providers/automation/cp/main.py diff --git a/couchpotato/core/providers/automation/cp/__init__.py b/couchpotato/core/providers/automation/cp/__init__.py deleted file mode 100644 index 6b2163be..00000000 --- a/couchpotato/core/providers/automation/cp/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .main import CP - - -def start(): - return CP() - -config = [] diff --git a/couchpotato/core/providers/automation/cp/main.py b/couchpotato/core/providers/automation/cp/main.py deleted file mode 100644 index 22b7942a..00000000 --- a/couchpotato/core/providers/automation/cp/main.py +++ /dev/null @@ -1,11 +0,0 @@ -from couchpotato.core.logger import CPLog -from couchpotato.core.providers.automation.base import Automation - -log = CPLog(__name__) - - -class CP(Automation): - - def getMovies(self): - - return [] From fc6839b441d59efe9375925452f2ce0916e53191 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 21 Jan 2014 22:38:36 +0100 Subject: [PATCH 17/42] Force remove duplicate in suggested movies --- couchpotato/core/media/movie/suggestion/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index 5240bb2b..79766842 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -79,8 +79,10 @@ class Suggestion(Plugin): seen = [] if not seen else seen if ignore_imdb: + suggested_imdbs = [] for cs in cached_suggestion: - if cs.get('imdb') != ignore_imdb: + if cs.get('imdb') != ignore_imdb and cs.get('imdb') not in suggested_imdbs: + suggested_imdbs.append(cs.get('imdb')) new_suggestions.append(cs) # Get new results and add them From 18c64e493be558a3ec755c91873e0b1c2494515a Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 21 Jan 2014 23:06:02 +0100 Subject: [PATCH 18/42] Don't cache post requests --- couchpotato/core/plugins/base.py | 16 ++++++++++------ couchpotato/core/providers/base.py | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index f9280098..0625535e 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -244,11 +244,15 @@ class Plugin(object): log.error("Something went wrong when finishing the plugin function. Could not find the 'is_running' key") def getCache(self, cache_key, url = None, **kwargs): - cache_key_md5 = md5(cache_key) - cache = Env.get('cache').get(cache_key_md5) - if cache: - if not Env.get('dev'): log.debug('Getting cache %s', cache_key) - return cache + + use_cache = not len(kwargs.get('data', {})) > 0 and not kwargs.get('files') + + if use_cache: + cache_key_md5 = md5(cache_key) + cache = Env.get('cache').get(cache_key_md5) + if cache: + if not Env.get('dev'): log.debug('Getting cache %s', cache_key) + return cache if url: try: @@ -259,7 +263,7 @@ class Plugin(object): del kwargs['cache_timeout'] data = self.urlopen(url, **kwargs) - if data and cache_timeout > 0: + if data and cache_timeout > 0 and use_cache: self.setCache(cache_key, data, timeout = cache_timeout) return data except: diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 1aceac40..7f755c3c 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -64,7 +64,7 @@ class Provider(Plugin): def getJsonData(self, url, decode_from = None, **kwargs): - cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {}))) + cache_key = md5(url) data = self.getCache(cache_key, url, **kwargs) if data: @@ -81,7 +81,7 @@ class Provider(Plugin): def getRSSData(self, url, item_path = 'channel/item', **kwargs): - cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {}))) + cache_key = md5(url) data = self.getCache(cache_key, url, **kwargs) if data and len(data) > 0: From 4b1151bda116cb83e4990f63fe93b65191d541d6 Mon Sep 17 00:00:00 2001 From: Ruud Burger Date: Wed, 22 Jan 2014 08:48:22 +0100 Subject: [PATCH 19/42] Put future import after shebang --- CouchPotato.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CouchPotato.py b/CouchPotato.py index 252dfe1c..e73ed9a5 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -1,5 +1,5 @@ -from __future__ import print_function #!/usr/bin/env python +from __future__ import print_function from logging import handlers from os.path import dirname import logging From d1dbf3745a892b0755016af7dd5f77ba4f2e76e5 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Wed, 22 Jan 2014 22:24:22 +1300 Subject: [PATCH 20/42] Updated rtorrent-python library --- libs/rtorrent/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/rtorrent/__init__.py b/libs/rtorrent/__init__.py index 683ef1c7..2c0f3fa9 100755 --- a/libs/rtorrent/__init__.py +++ b/libs/rtorrent/__init__.py @@ -199,7 +199,7 @@ class RTorrent: return(func_name) - def load_torrent(self, torrent, start=False, verbose=False, verify_load=True): + def load_torrent(self, torrent, start=False, verbose=False, verify_load=True, verify_retries=3): """ Loads torrent into rTorrent (with various enhancements) @@ -244,9 +244,8 @@ class RTorrent: getattr(p, func_name)(torrent) if verify_load: - MAX_RETRIES = 3 i = 0 - while i < MAX_RETRIES: + while i < verify_retries: self.get_torrents() if info_hash in [t.info_hash for t in self.torrents]: break From d860680823e0f42d156fc083b4e0035e2af7f08b Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Wed, 22 Jan 2014 22:27:19 +1300 Subject: [PATCH 21/42] Increased rTorrent load_torrent max waiting time to 10 retries/seconds --- couchpotato/core/downloaders/rtorrent/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index ff801db7..27177756 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -134,7 +134,7 @@ class rTorrent(Downloader): # Send request to rTorrent try: # Send torrent to rTorrent - torrent = self.rt.load_torrent(filedata) + torrent = self.rt.load_torrent(filedata, verify_retries=10) if not torrent: log.error('Unable to find the torrent, did it fail to load?') From ca94d48f8b357886a133ab4b3f4825e7bf824e27 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 22 Jan 2014 23:31:45 +0100 Subject: [PATCH 22/42] No need to try and cach htmldata --- couchpotato/core/providers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 7f755c3c..93e0900f 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -95,7 +95,7 @@ class Provider(Plugin): def getHTMLData(self, url, **kwargs): - cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('data', {}))) + cache_key = md5(url) return self.getCache(cache_key, url, **kwargs) From 52c64c1a6a5715ec360f0a90ede516b16b431074 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 22 Jan 2014 23:38:18 +0100 Subject: [PATCH 23/42] Get full imdb (watch)list without login. fix #2715 --- .../providers/automation/imdb/__init__.py | 2 +- .../core/providers/automation/imdb/main.py | 51 +++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py index ce2ea5ea..f9baabf2 100644 --- a/couchpotato/core/providers/automation/imdb/__init__.py +++ b/couchpotato/core/providers/automation/imdb/__init__.py @@ -12,7 +12,7 @@ config = [{ 'list': 'watchlist_providers', 'name': 'imdb_automation_watchlist', 'label': 'IMDB', - 'description': 'From any public IMDB watchlists. Url should be the CSV link.', + 'description': 'From any public IMDB watchlists.', 'options': [ { 'name': 'automation_enabled', diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index 25f2fee8..4f41bc14 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -1,4 +1,5 @@ import traceback +import re from bs4 import BeautifulSoup from couchpotato import fireEvent @@ -42,23 +43,55 @@ class IMDBWatchlist(IMDBBase): index = -1 for watchlist_url in watchlist_urls: + try: + # Get list ID + ids = re.findall('[list/|list_id=]([a-zA-Z0-9\-_]{11})', watchlist_url) + if len(ids) == 1: + watchlist_url = 'http://www.imdb.com/list/%s/?view=compact&sort=created:asc' % ids[0] + # Try find user id with watchlist + else: + userids = re.findall('(ur\d{7})', watchlist_url) + if len(userids) == 1: + watchlist_url = 'http://www.imdb.com/user/%s/watchlist?view=compact&sort=created:asc' % userids[0] + except: + log.error('Failed getting id from watchlist: %s', traceback.format_exc()) + index += 1 if not watchlist_enablers[index]: continue - try: - log.debug('Started IMDB watchlists: %s', watchlist_url) - rss_data = self.getHTMLData(watchlist_url) - imdbs = getImdb(rss_data, multiple = True) if rss_data else [] + start = 0 + while True: + try: - for imdb in imdbs: - movies.append(imdb) + w_url = '%s&start=%s' % (watchlist_url, start) + log.debug('Started IMDB watchlists: %s', w_url) + html = self.getHTMLData(w_url) - if self.shuttingDown(): + try: + split = splitString(html, split_on="
")[1] + html = splitString(split, split_on="
")[0] + except: + pass + + imdbs = getImdb(html, multiple = True) if html else [] + + for imdb in imdbs: + if imdb not in movies: + movies.append(imdb) + + if self.shuttingDown(): + break + + log.debug('Found %s movies on %s', (len(imdbs), w_url)) + + if len(imdbs) < 250: break - except: - log.error('Failed loading IMDB watchlist: %s %s', (watchlist_url, traceback.format_exc())) + start += 250 + + except: + log.error('Failed loading IMDB watchlist: %s %s', (watchlist_url, traceback.format_exc())) return movies From ca8bbdc293ce18f1193e82cc7cd4724405fab347 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 22 Jan 2014 23:59:25 +0100 Subject: [PATCH 24/42] Allow longer imdb user_id parse --- couchpotato/core/providers/automation/imdb/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index 4f41bc14..1eff172f 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -50,7 +50,7 @@ class IMDBWatchlist(IMDBBase): watchlist_url = 'http://www.imdb.com/list/%s/?view=compact&sort=created:asc' % ids[0] # Try find user id with watchlist else: - userids = re.findall('(ur\d{7})', watchlist_url) + userids = re.findall('(ur\d{7,9})', watchlist_url) if len(userids) == 1: watchlist_url = 'http://www.imdb.com/user/%s/watchlist?view=compact&sort=created:asc' % userids[0] except: From 76943b65293d7a67876eee6c69a5a9f88e729eea Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 23 Jan 2014 00:00:59 +0100 Subject: [PATCH 25/42] Make sure imdb list_id regex matches whole string. Thanks @basrieter --- couchpotato/core/providers/automation/imdb/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index 1eff172f..6ca81b70 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -45,7 +45,7 @@ class IMDBWatchlist(IMDBBase): try: # Get list ID - ids = re.findall('[list/|list_id=]([a-zA-Z0-9\-_]{11})', watchlist_url) + ids = re.findall('(?:list/|list_id=)([a-zA-Z0-9\-_]{11})', watchlist_url) if len(ids) == 1: watchlist_url = 'http://www.imdb.com/list/%s/?view=compact&sort=created:asc' % ids[0] # Try find user id with watchlist From cf867196073feaa707e25aa5ba7a64f86ff203b5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 23 Jan 2014 00:08:38 +0100 Subject: [PATCH 26/42] Encode before logging --- couchpotato/core/providers/info/themoviedb/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/info/themoviedb/main.py b/couchpotato/core/providers/info/themoviedb/main.py index 5207eb11..d301db2b 100644 --- a/couchpotato/core/providers/info/themoviedb/main.py +++ b/couchpotato/core/providers/info/themoviedb/main.py @@ -1,5 +1,5 @@ from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import simplifyString, toUnicode +from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.info.base import MovieProvider @@ -166,7 +166,7 @@ class TheMovieDb(MovieProvider): try: image_url = getattr(movie, type).geturl(size = size) except: - log.debug('Failed getting %s.%s for "%s"', (type, size, str(movie))) + log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie)))) return image_url From e20c7763642de91b2eed34c1b5648a6c88e3c641 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 23 Jan 2014 22:00:12 +0100 Subject: [PATCH 27/42] Use urlopen for HD Bits requests --- .../core/providers/torrent/hdbits/main.py | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/couchpotato/core/providers/torrent/hdbits/main.py b/couchpotato/core/providers/torrent/hdbits/main.py index 2a1b1828..ce17bbac 100644 --- a/couchpotato/core/providers/torrent/hdbits/main.py +++ b/couchpotato/core/providers/torrent/hdbits/main.py @@ -4,7 +4,6 @@ from couchpotato.core.providers.torrent.base import TorrentProvider import re import json -import requests import traceback log = CPLog(__name__) @@ -13,20 +12,12 @@ log = CPLog(__name__) class HDBits(TorrentProvider): urls = { - 'test' : 'https://hdbits.org/', - 'detail' : 'https://hdbits.org/details.php?id=%s', - 'download' : 'https://hdbits.org/download.php?id=%s&passkey=%s', + 'test': 'https://hdbits.org/', + 'detail': 'https://hdbits.org/details.php?id=%s', + 'download': 'https://hdbits.org/download.php?id=%s&passkey=%s', 'api': 'https://hdbits.org/api/torrents' } - status_codes = { - 1: 'Failure (something bad happened)', - 2: 'SSL Required - You should be on https://', - 3: 'JSON Malformed', - 4: 'Auth data missing', - 5: 'Auth failed' - } - http_time_between_calls = 1 #seconds def _post_query(self, **params): @@ -37,23 +28,24 @@ class HDBits(TorrentProvider): } post_data.update(params) - req = requests.post(self.urls['api'], - data=json.dumps(post_data)) + try: + result = self.getJsonData(self.urls['api'], data = json.dumps(post_data)) - response = json.loads(req.text) + if result: + if result['status'] != 0: + log.error('Error searching hdbits: %s' % result['message']) + else: + return result['data'] + except: + pass - if response['status'] != 0: - log.error('Error searching hdbits: %s' - % self.status_codes[response['status']]) - else: - return response['data'] + return None def _search(self, movie, quality, results): - match = re.match(r'tt(\d{7})', - movie['library']['identifier']) + match = re.match(r'tt(\d{7})', movie['library']['identifier']) - data = self._post_query(imdb={'id': match.group(1)}) + data = self._post_query(imdb = {'id': match.group(1)}) if data: try: @@ -61,14 +53,11 @@ class HDBits(TorrentProvider): results.append({ 'id': result['id'], 'name': result['name'], - 'url': self.urls['download'] - % (result['id'], self.conf('passkey')), + 'url': self.urls['download'] % (result['id'], self.conf('passkey')), 'detail_url': self.urls['detail'] % result['id'], 'size': self.parseSize(result['size']), 'seeders': tryInt(result['seeders']), 'leechers': tryInt(result['leechers']) }) except: - log.error('Failed getting results from %s: %s', - (self.getName(), traceback.format_exc())) - + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) From d65667ce160de772390034ad68b73539d59b440c Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 24 Jan 2014 14:50:54 +0100 Subject: [PATCH 28/42] Don't force add basic auth to url --- couchpotato/core/helpers/variable.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 0a47f1de..69067e8d 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -144,9 +144,14 @@ def cleanHost(host, protocol = True, ssl = False, username = None, password = No host = host.split('://', 1)[-1] if protocol and username and password: - login = '%s:%s@' % (username, password) - if not login in host: - host = host.replace('://', '://' + login, 1) + try: + auth = re.findall('^(?:.+?//)(.+?):(.+?)@(?:.+)$', host) + if auth: + log.error('Cleanhost error: auth already defined in url: %s, please remove BasicAuth from url.', host) + else: + host = host.replace('://', '://%s:%s@' % (username, password), 1) + except: + pass host = host.rstrip('/ ') if protocol: From c9e0910c550559c99842a7e6c680dc7d9cf05853 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 24 Jan 2014 15:29:24 +0100 Subject: [PATCH 29/42] Can't use len() on filter iterator. fix #2762 --- couchpotato/core/helpers/variable.py | 13 +++++++++++-- couchpotato/core/media/_base/searcher/main.py | 7 +++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 69067e8d..7e9d6ba6 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -9,7 +9,7 @@ import re import string import sys import six -from six.moves import map, zip +from six.moves import map, zip, filter log = CPLog(__name__) @@ -260,7 +260,16 @@ def randomString(size = 8, chars = string.ascii_uppercase + string.digits): def splitString(str, split_on = ',', clean = True): l = [x.strip() for x in str.split(split_on)] if str else [] - return [x for x in l if x] if clean else l + return removeEmpty(l) if clean else l + + +def removeEmpty(l): + return list(filter(None, l)) + + +def removeDuplicate(l): + seen = set() + return [x for x in l if x not in seen and not seen.add(x)] def dictIsSubset(a, b): diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 17dc9b57..d05ef09c 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -1,12 +1,11 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import simplifyString -from couchpotato.core.helpers.variable import splitString +from couchpotato.core.helpers.variable import splitString, removeEmpty from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase import datetime import re -from six.moves import filter log = CPLog(__name__) @@ -155,8 +154,8 @@ class Searcher(SearcherBase): check_movie = fireEvent('scanner.name_year', check_name, single = True) try: - check_words = filter(None, re.split('\W+', check_movie.get('name', ''))) - movie_words = filter(None, re.split('\W+', simplifyString(movie_name))) + check_words = removeEmpty(re.split('\W+', check_movie.get('name', ''))) + movie_words = removeEmpty(re.split('\W+', simplifyString(movie_name))) if len(check_words) > 0 and len(movie_words) > 0 and len(list(set(check_words) - set(movie_words))) == 0: return True From 3101926e9b0e5749781dd4c96f6a256a1dfaf80f Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 24 Jan 2014 15:42:29 +0100 Subject: [PATCH 30/42] removeDuplicate helper --- couchpotato/core/helpers/variable.py | 4 ++-- couchpotato/core/media/_base/searcher/main.py | 8 ++++---- couchpotato/core/media/movie/suggestion/main.py | 4 ++-- couchpotato/core/plugins/score/main.py | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 7e9d6ba6..a1939c20 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -176,7 +176,7 @@ def getImdb(txt, check_inside = False, multiple = False): ids = re.findall('(tt\d{4,7})', txt) if multiple: - return list(set(['tt%07d' % tryInt(x[2:]) for x in ids])) if len(ids) > 0 else [] + return removeDuplicate(['tt%07d' % tryInt(x[2:]) for x in ids]) if len(ids) > 0 else [] return 'tt%07d' % tryInt(ids[0][2:]) except IndexError: @@ -251,7 +251,7 @@ def possibleTitles(raw_title): new_title = raw_title.replace('&', 'and') titles.append(simplifyString(new_title)) - return list(set(titles)) + return removeDuplicate(titles) def randomString(size = 8, chars = string.ascii_uppercase + string.digits): diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index d05ef09c..e7209b60 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -1,7 +1,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import simplifyString -from couchpotato.core.helpers.variable import splitString, removeEmpty +from couchpotato.core.helpers.variable import splitString, removeEmpty, removeDuplicate from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase import datetime @@ -150,7 +150,7 @@ class Searcher(SearcherBase): try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', check_name), key = len).strip()) except: pass - for check_name in list(set(check_names)): + for check_name in removeDuplicate(check_names): check_movie = fireEvent('scanner.name_year', check_name, single = True) try: @@ -173,7 +173,7 @@ class Searcher(SearcherBase): # Make sure it has required words required_words = splitString(self.conf('required_words', section = 'searcher').lower()) - try: required_words = list(set(required_words + splitString(media['category']['required'].lower()))) + try: required_words = removeDuplicate(required_words + splitString(media['category']['required'].lower())) except: pass req_match = 0 @@ -187,7 +187,7 @@ class Searcher(SearcherBase): # Ignore releases ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) - try: ignored_words = list(set(ignored_words + splitString(media['category']['ignored'].lower()))) + try: ignored_words = removeDuplicate(ignored_words + splitString(media['category']['ignored'].lower())) except: pass ignored_match = 0 diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index 79766842..22e23fe2 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -1,7 +1,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import splitString +from couchpotato.core.helpers.variable import splitString, removeDuplicate from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Media, Library from couchpotato.environment import Env @@ -99,7 +99,7 @@ class Suggestion(Plugin): movies.extend(seen) ignored.extend([x.get('imdb') for x in cached_suggestion]) - suggestions = fireEvent('movie.suggest', movies = movies, ignore = list(set(ignored)), single = True) + suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True) if suggestions: new_suggestions.extend(suggestions) diff --git a/couchpotato/core/plugins/score/main.py b/couchpotato/core/plugins/score/main.py index 17448eab..30e7baca 100644 --- a/couchpotato/core/plugins/score/main.py +++ b/couchpotato/core/plugins/score/main.py @@ -1,6 +1,6 @@ -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import getTitle, splitString +from couchpotato.core.helpers.variable import getTitle, splitString, removeDuplicate from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \ @@ -21,7 +21,7 @@ class Score(Plugin): # Merge global and category preferred_words = splitString(Env.setting('preferred_words', section = 'searcher').lower()) - try: preferred_words = list(set(preferred_words + splitString(movie['category']['preferred'].lower()))) + try: preferred_words = removeDuplicate(preferred_words + splitString(movie['category']['preferred'].lower())) except: pass score = nameScore(toUnicode(nzb['name']), movie['library']['year'], preferred_words) @@ -48,7 +48,7 @@ class Score(Plugin): # Merge global and category ignored_words = splitString(Env.setting('ignored_words', section = 'searcher').lower()) - try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower()))) + try: ignored_words = removeDuplicate(ignored_words + splitString(movie['category']['ignored'].lower())) except: pass # Partial ignored words From cbd8981ee2d455f4938ad088ef85dbdd96c27355 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 24 Jan 2014 16:33:10 +0100 Subject: [PATCH 31/42] Use helper --- couchpotato/core/providers/automation/letterboxd/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/automation/letterboxd/main.py b/couchpotato/core/providers/automation/letterboxd/main.py index efd04991..dbbf53b1 100644 --- a/couchpotato/core/providers/automation/letterboxd/main.py +++ b/couchpotato/core/providers/automation/letterboxd/main.py @@ -1,9 +1,8 @@ from bs4 import BeautifulSoup -from couchpotato.core.helpers.variable import tryInt, splitString +from couchpotato.core.helpers.variable import tryInt, splitString, removeEmpty from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation import re -from six.moves import filter log = CPLog(__name__) @@ -46,7 +45,7 @@ class Letterboxd(Automation): soup = BeautifulSoup(self.getHTMLData(self.url % username)) for movie in soup.find_all('a', attrs = {'class': 'frame'}): - match = filter(None, self.pattern.split(movie['title'])) + match = removeEmpty(self.pattern.split(movie['title'])) movies.append({'title': match[0], 'year': match[1] }) return movies From 88c328af8e90be540d53775cb83c638d9668ad15 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 24 Jan 2014 22:33:22 +0100 Subject: [PATCH 32/42] Improved manage scanning Expire after db get --- couchpotato/core/helpers/variable.py | 2 +- couchpotato/core/logger.py | 4 +- couchpotato/core/media/_base/media/main.py | 10 +- .../core/media/movie/library/movie/main.py | 99 +++++++++---------- couchpotato/core/media/movie/searcher/main.py | 35 ++++--- .../core/media/movie/suggestion/main.py | 2 + couchpotato/core/plugins/category/main.py | 2 + couchpotato/core/plugins/dashboard/main.py | 1 + couchpotato/core/plugins/manage/main.py | 27 +++-- couchpotato/core/plugins/profile/main.py | 2 + couchpotato/core/plugins/release/main.py | 4 + couchpotato/core/plugins/scanner/main.py | 6 +- couchpotato/core/providers/metadata/base.py | 2 +- couchpotato/environment.py | 2 +- couchpotato/static/scripts/page/manage.js | 4 +- 15 files changed, 117 insertions(+), 85 deletions(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index a1939c20..47810a66 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -278,4 +278,4 @@ def dictIsSubset(a, b): def isSubFolder(sub_folder, base_folder): # Returns True if sub_folder is the same as or inside base_folder - return base_folder and sub_folder and os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep in os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep + return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep) diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index fc198cdd..8223f146 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -38,7 +38,7 @@ class CPLog(object): def safeMessage(self, msg, replace_tuple = ()): from couchpotato.environment import Env - from couchpotato.core.helpers.encoding import ss + from couchpotato.core.helpers.encoding import ss, toUnicode msg = ss(msg) @@ -67,4 +67,4 @@ class CPLog(object): except: pass - return msg + return toUnicode(msg) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index b9bf2196..0ae1b3a5 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -70,8 +70,6 @@ class MediaPlugin(MediaBase): addEvent('media.restatus', self.restatus) def refresh(self, id = '', **kwargs): - db = get_session() - handlers = [] ids = splitString(id) @@ -97,11 +95,14 @@ class MediaPlugin(MediaBase): default_title = getTitle(media.library) identifier = media.library.identifier - db.expire_all() + event = 'library.update.%s' % media.type def handler(): - fireEvent('library.update.%s' % media.type, identifier = identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(id)) + fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id)) + db.expire_all() + + if handler: return handler def addSingleRefreshView(self): @@ -467,6 +468,7 @@ class MediaPlugin(MediaBase): m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') db.commit() + db.expire_all() return True diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index 3f07305c..d305a163 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -60,7 +60,7 @@ class MovieLibraryPlugin(LibraryBase): db.expire_all() return library_dict - def update(self, identifier, default_title = '', force = False): + def update(self, identifier, default_title = '', extended = False): if self.shuttingDown(): return @@ -69,13 +69,7 @@ class MovieLibraryPlugin(LibraryBase): library = db.query(Library).filter_by(identifier = identifier).first() done_status = fireEvent('status.get', 'done', single = True) - library_dict = None - if library: - library_dict = library.to_dict(self.default_dict) - - do_update = True - - info = fireEvent('movie.info', merge = True, identifier = identifier) + info = fireEvent('movie.info', merge = True, extended = extended, identifier = identifier) # Don't need those here try: del info['in_wanted'] @@ -88,55 +82,54 @@ class MovieLibraryPlugin(LibraryBase): return False # Main info - if do_update: - library.plot = toUnicode(info.get('plot', '')) - library.tagline = toUnicode(info.get('tagline', '')) - library.year = info.get('year', 0) - library.status_id = done_status.get('id') - library.info.update(info) - db.commit() + library.plot = toUnicode(info.get('plot', '')) + library.tagline = toUnicode(info.get('tagline', '')) + library.year = info.get('year', 0) + library.status_id = done_status.get('id') + library.info.update(info) + db.commit() - # Titles - [db.delete(title) for title in library.titles] - db.commit() + # Titles + [db.delete(title) for title in library.titles] + db.commit() - titles = info.get('titles', []) - log.debug('Adding titles: %s', titles) - counter = 0 - for title in titles: - if not title: + titles = info.get('titles', []) + log.debug('Adding titles: %s', titles) + counter = 0 + for title in titles: + if not title: + continue + title = toUnicode(title) + t = LibraryTitle( + title = title, + simple_title = self.simplifyTitle(title), + default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title) + ) + library.titles.append(t) + counter += 1 + + db.commit() + + # Files + images = info.get('images', []) + for image_type in ['poster']: + for image in images.get(image_type, []): + if not isinstance(image, (str, unicode)): continue - title = toUnicode(title) - t = LibraryTitle( - title = title, - simple_title = self.simplifyTitle(title), - default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title) - ) - library.titles.append(t) - counter += 1 - db.commit() + file_path = fireEvent('file.download', url = image, single = True) + if file_path: + file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) + try: + file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() + library.files.append(file_obj) + db.commit() - # Files - images = info.get('images', []) - for image_type in ['poster']: - for image in images.get(image_type, []): - if not isinstance(image, (str, unicode)): - continue + break + except: + log.debug('Failed to attach to library: %s', traceback.format_exc()) - file_path = fireEvent('file.download', url = image, single = True) - if file_path: - file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) - try: - file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() - library.files.append(file_obj) - db.commit() - - break - except: - log.debug('Failed to attach to library: %s', traceback.format_exc()) - - library_dict = library.to_dict(self.default_dict) + library_dict = library.to_dict(self.default_dict) db.expire_all() return library_dict @@ -147,14 +140,14 @@ class MovieLibraryPlugin(LibraryBase): library = db.query(Library).filter_by(identifier = identifier).first() if not library.info: - library_dict = self.update(identifier, force = True) + library_dict = self.update(identifier) dates = library_dict.get('info', {}).get('release_date') else: dates = library.info.get('release_date') if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: dates = fireEvent('movie.release_date', identifier = identifier, merge = True) - library.info.update({'release_date': dates }) + library.info.update({'release_date': dates}) db.commit() db.expire_all() diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 1775f672..aa215606 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -73,11 +73,24 @@ class MovieSearcher(SearcherBase, MovieTypeBase): db = get_session() - movies = db.query(Media).filter( + movies_raw = db.query(Media).filter( Media.status.has(identifier = 'active') ).all() + random.shuffle(movies) + movies = [] + for m in movies_raw: + movies.append(m.to_dict({ + 'category': {}, + 'profile': {'types': {'quality': {}}}, + 'releases': {'status': {}, 'quality': {}}, + 'library': {'titles': {}, 'files': {}}, + 'files': {}, + })) + + db.expire_all() + self.in_progress = { 'total': len(movies), 'to_go': len(movies), @@ -87,21 +100,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): search_protocols = fireEvent('searcher.protocols', single = True) for movie in movies: - movie_dict = movie.to_dict({ - 'category': {}, - 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}}, - 'library': {'titles': {}, 'files': {}}, - 'files': {}, - }) try: - self.single(movie_dict, search_protocols) + self.single(movie, search_protocols) except IndexError: - log.error('Forcing library update for %s, if you see this often, please report: %s', (movie_dict['library']['identifier'], traceback.format_exc())) - fireEvent('library.update.movie', movie_dict['library']['identifier'], force = True) + log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['library']['identifier'], traceback.format_exc())) + fireEvent('library.update.movie', movie['library']['identifier']) except: - log.error('Search failed for %s: %s', (movie_dict['library']['identifier'], traceback.format_exc())) + log.error('Search failed for %s: %s', (movie['library']['identifier'], traceback.format_exc())) self.in_progress['to_go'] -= 1 @@ -133,8 +139,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.') return - db = get_session() - pre_releases = fireEvent('quality.pre_releases', single = True) release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True) available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True) @@ -150,6 +154,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title) + db = get_session() ret = False for quality_type in movie['profile']['types']: @@ -204,6 +209,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']}) + db.expire_all() return ret def correctRelease(self, nzb = None, media = None, quality = None, **kwargs): @@ -336,6 +342,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): for rel in rels: rel.status_id = ignored_status.get('id') db.commit() + db.expire_all() movie_dict = fireEvent('media.get', media_id = media_id, single = True) log.info('Trying next release for: %s', getTitle(movie_dict['library'])) diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index 22e23fe2..02fd27f7 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -33,6 +33,7 @@ class Suggestion(Plugin): .options(joinedload_all('library')) \ .filter(or_(*[Media.status.has(identifier = s) for s in ['active', 'done']])).all() movies = [x.library.identifier for x in active_movies] + db.expire_all() if not ignored or len(ignored) == 0: ignored = splitString(Env.prop('suggest_ignore', default = '')) @@ -97,6 +98,7 @@ class Suggestion(Plugin): .filter(Media.status_id.in_([active_status.get('id'), done_status.get('id')])).all() movies = [x[0] for x in active_movies] movies.extend(seen) + db.expire_all() ignored.extend([x.get('imdb') for x in cached_suggestion]) suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True) diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index 35597fdb..6ac91274 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -119,3 +119,5 @@ class CategoryPlugin(Plugin): for movie in movies: movie.category_id = None db.commit() + + db.expire_all() diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index 949d8f50..2c5939da 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -119,6 +119,7 @@ class Dashboard(Plugin): 'files': {}, })) + db.expire_all() return { 'success': True, 'empty': len(movies) == 0, diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 9c7e80c7..2f297491 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -1,7 +1,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent, fireEventAsync from couchpotato.core.helpers.encoding import sp -from couchpotato.core.helpers.variable import splitString, getTitle +from couchpotato.core.helpers.variable import splitString, getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -59,6 +59,7 @@ class Manage(Plugin): fireEventAsync('manage.update', full = True if full == '1' else False) return { + 'progress': self.in_progress, 'success': True } @@ -84,15 +85,17 @@ class Manage(Plugin): added_identifiers = [] # Add some progress - self.in_progress = {} for directory in directories: self.in_progress[os.path.normpath(directory)] = { + 'started': False, + 'eta': -1, 'total': None, 'to_go': None, } for directory in directories: folder = os.path.normpath(directory) + self.in_progress[os.path.normpath(directory)]['started'] = tryInt(time.time()) if not os.path.isdir(folder): if len(directory) > 0: @@ -102,6 +105,7 @@ class Manage(Plugin): log.info('Updating manage library: %s', folder) fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder) + onFound = self.createAddToLibrary(folder, added_identifiers) fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True) @@ -175,10 +179,10 @@ class Manage(Plugin): def addToLibrary(group, total_found, to_go): if self.in_progress[folder]['total'] is None: - self.in_progress[folder] = { + self.in_progress[folder].update({ 'total': total_found, 'to_go': total_found, - } + }) if group['library'] and group['library'].get('identifier'): identifier = group['library'].get('identifier') @@ -186,9 +190,9 @@ class Manage(Plugin): # Add it to release and update the info fireEvent('release.add', group = group) - fireEventAsync('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier)) + fireEvent('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier)) else: - self.in_progress[folder]['to_go'] -= 1 + self.updateProgress(folder) return addToLibrary @@ -199,7 +203,7 @@ class Manage(Plugin): if not self.in_progress or self.shuttingDown(): return - self.in_progress[folder]['to_go'] -= 1 + self.updateProgress(folder) total = self.in_progress[folder]['total'] movie_dict = fireEvent('media.get', identifier, single = True) @@ -207,6 +211,15 @@ class Manage(Plugin): return afterUpdate + def updateProgress(self, folder): + + pr = self.in_progress[folder] + pr['to_go'] -= 1 + + avg = (time.time() - pr['started'])/(pr['total'] - pr['to_go']) + pr['eta'] = tryInt(avg * pr['to_go']) + + def directories(self): try: if self.conf('library', default = '').strip(): diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 374bc927..31321852 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -46,6 +46,8 @@ class ProfilePlugin(Plugin): movie.profile_id = default_profile.get('id') db.commit() + db.expire_all() + def allView(self, **kwargs): return { diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 2d67d12c..4726dbfa 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -143,6 +143,7 @@ class Release(Plugin): fireEvent('media.restatus', media.id) + db.expire_all() return True def saveFile(self, filepath, type = 'unknown', include_media_info = False): @@ -338,6 +339,8 @@ class Release(Plugin): except: log.error('Failed storing download status: %s', traceback.format_exc()) return False + finally: + db.expire_all() return True @@ -466,4 +469,5 @@ class Release(Plugin): #Update all movie info as there is no release update function fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict()) + db.expire_all() return True diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 803c63c1..16c9e1db 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -134,6 +134,8 @@ class Scanner(Plugin): except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) + + log.debug('Found %s files to scan and group in %s', (len(files), folder)) else: check_file_date = False files = [sp(x) for x in files] @@ -421,6 +423,7 @@ class Scanner(Plugin): else: movie = db.query(Media).filter_by(library_id = group['library']['id']).first() group['movie_id'] = None if not movie else movie.id + db.expire_all() processed_movies[identifier] = group @@ -604,6 +607,7 @@ class Scanner(Plugin): break except: pass + db.expire_all() # Search based on identifiers if not imdb_id: @@ -637,7 +641,7 @@ class Scanner(Plugin): try: m = re.search(self.cp_imdb, string.lower()) id = m.group('id') - if id: return id + if id: return id except AttributeError: pass diff --git a/couchpotato/core/providers/metadata/base.py b/couchpotato/core/providers/metadata/base.py index f5610030..72d07609 100644 --- a/couchpotato/core/providers/metadata/base.py +++ b/couchpotato/core/providers/metadata/base.py @@ -25,7 +25,7 @@ class MetaDataBase(Plugin): # Update library to get latest info try: - updated_library = fireEvent('library.update.movie', group['library']['identifier'], force = True, single = True) + updated_library = fireEvent('library.update.movie', group['library']['identifier'], extended = True, single = True) group['library'] = mergeDicts(group['library'], updated_library) except: log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc()) diff --git a/couchpotato/environment.py b/couchpotato/environment.py index 5fb43299..c409da80 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -67,7 +67,7 @@ class Env(object): @staticmethod def getEngine(): - return create_engine(Env.get('db_path'), echo = False, pool_recycle = 30) + return create_engine(Env.get('db_path'), echo = False) @staticmethod def setting(attr, section = 'core', value = None, default = '', type = None): diff --git a/couchpotato/static/scripts/page/manage.js b/couchpotato/static/scripts/page/manage.js index eeeef628..ed9120fa 100644 --- a/couchpotato/static/scripts/page/manage.js +++ b/couchpotato/static/scripts/page/manage.js @@ -119,7 +119,9 @@ Page.Manage = new Class({ sorted_table.each(function(folder){ var folder_progress = progress[folder] new Element('div').adopt( - new Element('span.folder', {'text': folder}), + new Element('span.folder', {'text': folder + + (folder_progress.eta > 0 ? ', ' + new Date ().increment('second', folder_progress.eta).timeDiffInWords().replace('from now', 'to go') : '') + }), new Element('span.percentage', {'text': folder_progress.total ? (((folder_progress.total-folder_progress.to_go)/folder_progress.total)*100).round() + '%' : '0%'}) ).inject(self.progress_container) }); From fbae706b0f5f2c817461e472cc2bb5778e67b4a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 25 Jan 2014 15:26:00 +0100 Subject: [PATCH 33/42] Use correct var to shuffle --- couchpotato/core/media/movie/searcher/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index aa215606..0daf1b4e 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -77,7 +77,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): Media.status.has(identifier = 'active') ).all() - random.shuffle(movies) + random.shuffle(movies_raw) movies = [] for m in movies_raw: From 31c39650a9e46386549e4b31790b5c5fcef85b93 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 25 Jan 2014 15:26:35 +0100 Subject: [PATCH 34/42] Force default title when none match --- couchpotato/core/media/movie/library/movie/main.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index d305a163..6266ca4e 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -96,6 +96,17 @@ class MovieLibraryPlugin(LibraryBase): titles = info.get('titles', []) log.debug('Adding titles: %s', titles) counter = 0 + + def_title = None + for title in titles: + if (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title): + def_title = toUnicode(title) + break + counter += 1 + + #if not def_title: + # def_title = toUnicode(titles[0]) + for title in titles: if not title: continue @@ -103,10 +114,9 @@ class MovieLibraryPlugin(LibraryBase): t = LibraryTitle( title = title, simple_title = self.simplifyTitle(title), - default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title) + default = title == def_title ) library.titles.append(t) - counter += 1 db.commit() From 7001ed476d4597df8ee227c332cca8b0929d94b2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 26 Jan 2014 00:33:21 +0100 Subject: [PATCH 35/42] Wrap all commits with try/except --- couchpotato/__init__.py | 4 +- couchpotato/core/media/__init__.py | 33 +- couchpotato/core/media/_base/media/main.py | 126 ++++---- couchpotato/core/media/movie/_base/main.py | 195 ++++++------ .../core/media/movie/library/movie/main.py | 225 +++++++------- couchpotato/core/media/movie/searcher/main.py | 7 +- .../core/media/movie/suggestion/main.py | 2 - couchpotato/core/notifications/core/main.py | 69 +++-- couchpotato/core/plugins/category/main.py | 128 +++++--- couchpotato/core/plugins/dashboard/main.py | 1 - couchpotato/core/plugins/file/main.py | 70 +++-- couchpotato/core/plugins/profile/main.py | 228 ++++++++------ couchpotato/core/plugins/quality/main.py | 109 ++++--- couchpotato/core/plugins/release/main.py | 285 ++++++++++-------- couchpotato/core/plugins/renamer/main.py | 1 + couchpotato/core/plugins/scanner/main.py | 2 +- couchpotato/core/plugins/status/main.py | 75 +++-- couchpotato/core/settings/__init__.py | 23 +- couchpotato/environment.py | 25 +- 19 files changed, 919 insertions(+), 689 deletions(-) diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index 966da0cf..6b8cfd36 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -48,8 +48,8 @@ def addView(route, func, static = False): views[route] = func -def get_session(engine = None): - return Env.getSession(engine) +def get_session(): + return Env.getSession() # Web view diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 2635ca6c..512c52c2 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.core.event import addEvent, fireEventAsync, fireEvent from couchpotato.core.plugins.base import Plugin @@ -26,25 +27,33 @@ class MediaBase(Plugin): def createOnComplete(self, id): def onComplete(): - db = get_session() - media = db.query(Media).filter_by(id = id).first() - media_dict = media.to_dict(self.default_dict) - event_name = '%s.searcher.single' % media.type - db.expire_all() + try: + db = get_session() + media = db.query(Media).filter_by(id = id).first() + media_dict = media.to_dict(self.default_dict) + event_name = '%s.searcher.single' % media.type - fireEvent(event_name, media_dict, on_complete = self.createNotifyFront(id)) + fireEvent(event_name, media_dict, on_complete = self.createNotifyFront(id)) + except: + log.error('Failed creating onComplete: %s', traceback.format_exc()) + finally: + db.close() return onComplete def createNotifyFront(self, media_id): def notifyFront(): - db = get_session() - media = db.query(Media).filter_by(id = media_id).first() - media_dict = media.to_dict(self.default_dict) - event_name = '%s.update' % media.type - db.expire_all() + try: + db = get_session() + media = db.query(Media).filter_by(id = media_id).first() + media_dict = media.to_dict(self.default_dict) + event_name = '%s.update' % media.type - fireEvent('notify.frontend', type = event_name, data = media_dict) + fireEvent('notify.frontend', type = event_name, data = media_dict) + except: + log.error('Failed creating onComplete: %s', traceback.format_exc()) + finally: + db.close() return notifyFront diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 0ae1b3a5..cbcd4245 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session, tryInt from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent @@ -100,8 +101,6 @@ class MediaPlugin(MediaBase): def handler(): fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id)) - db.expire_all() - if handler: return handler @@ -125,7 +124,6 @@ class MediaPlugin(MediaBase): if m: results = m.to_dict(self.default_dict) - db.expire_all() return results def getView(self, id = None, **kwargs): @@ -261,7 +259,6 @@ class MediaPlugin(MediaBase): 'releases_count': releases_count.get(media_id), })) - db.expire_all() return total_count, movies def listView(self, **kwargs): @@ -355,7 +352,6 @@ class MediaPlugin(MediaBase): if len(chars) == 25: break - db.expire_all() return ''.join(sorted(chars)) def charView(self, **kwargs): @@ -380,50 +376,55 @@ class MediaPlugin(MediaBase): def delete(self, media_id, delete_from = None): - db = get_session() + try: + db = get_session() - media = db.query(Media).filter_by(id = media_id).first() - if media: - deleted = False - if delete_from == 'all': - db.delete(media) - db.commit() - deleted = True - else: - done_status = fireEvent('status.get', 'done', single = True) - - total_releases = len(media.releases) - total_deleted = 0 - new_movie_status = None - for release in media.releases: - if delete_from in ['wanted', 'snatched', 'late']: - if release.status_id != done_status.get('id'): - db.delete(release) - total_deleted += 1 - new_movie_status = 'done' - elif delete_from == 'manage': - if release.status_id == done_status.get('id'): - db.delete(release) - total_deleted += 1 - new_movie_status = 'active' - db.commit() - - if total_releases == total_deleted: + media = db.query(Media).filter_by(id = media_id).first() + if media: + deleted = False + if delete_from == 'all': db.delete(media) db.commit() deleted = True - elif new_movie_status: - new_status = fireEvent('status.get', new_movie_status, single = True) - media.profile_id = None - media.status_id = new_status.get('id') - db.commit() else: - fireEvent('media.restatus', media.id, single = True) + done_status = fireEvent('status.get', 'done', single = True) - if deleted: - fireEvent('notify.frontend', type = 'movie.deleted', data = media.to_dict()) + total_releases = len(media.releases) + total_deleted = 0 + new_movie_status = None + for release in media.releases: + if delete_from in ['wanted', 'snatched', 'late']: + if release.status_id != done_status.get('id'): + db.delete(release) + total_deleted += 1 + new_movie_status = 'done' + elif delete_from == 'manage': + if release.status_id == done_status.get('id'): + db.delete(release) + total_deleted += 1 + new_movie_status = 'active' + db.commit() + + if total_releases == total_deleted: + db.delete(media) + db.commit() + deleted = True + elif new_movie_status: + new_status = fireEvent('status.get', new_movie_status, single = True) + media.profile_id = None + media.status_id = new_status.get('id') + db.commit() + else: + fireEvent('media.restatus', media.id, single = True) + + if deleted: + fireEvent('notify.frontend', type = 'movie.deleted', data = media.to_dict()) + except: + log.error('Failed deleting media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() return True def deleteView(self, id = '', **kwargs): @@ -447,28 +448,33 @@ class MediaPlugin(MediaBase): active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True) - db = get_session() + try: + db = get_session() - m = db.query(Media).filter_by(id = media_id).first() - if not m or len(m.library.titles) == 0: - log.debug('Can\'t restatus movie, doesn\'t seem to exist.') - return False + m = db.query(Media).filter_by(id = media_id).first() + if not m or len(m.library.titles) == 0: + log.debug('Can\'t restatus movie, doesn\'t seem to exist.') + return False - log.debug('Changing status for %s', m.library.titles[0].title) - if not m.profile: - m.status_id = done_status.get('id') - else: - move_to_wanted = True + log.debug('Changing status for %s', m.library.titles[0].title) + if not m.profile: + m.status_id = done_status.get('id') + else: + move_to_wanted = True - for t in m.profile.types: - for release in m.releases: - if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): - move_to_wanted = False + for t in m.profile.types: + for release in m.releases: + if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): + move_to_wanted = False - m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') + m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') - db.commit() - db.expire_all() + db.commit() - return True + return True + except: + log.error('Failed restatus: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index cd2158cb..a7ecf2d2 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent @@ -70,76 +71,81 @@ class MovieBase(MovieTypeBase): default_profile = fireEvent('profile.default', single = True) cat_id = params.get('category_id') - db = get_session() - m = db.query(Media).filter_by(library_id = library.get('id')).first() - added = True - do_search = False - search_after = search_after and self.conf('search_on_add', section = 'moviesearcher') - if not m: - m = Media( - library_id = library.get('id'), - profile_id = params.get('profile_id', default_profile.get('id')), - status_id = status_id if status_id else status_active.get('id'), - category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, - ) - db.add(m) - db.commit() - - onComplete = None - if search_after: - onComplete = self.createOnComplete(m.id) - - fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) - search_after = False - elif force_readd: - - # Clean snatched history - for release in m.releases: - if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]: - if params.get('ignore_previous', False): - release.status_id = ignored_status.get('id') - else: - fireEvent('release.delete', release.id, single = True) - - m.profile_id = params.get('profile_id', default_profile.get('id')) - m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m.category_id or None) - else: - log.debug('Movie already exists, not updating: %s', params) - added = False - - if force_readd: - m.status_id = status_id if status_id else status_active.get('id') - m.last_edit = int(time.time()) - do_search = True - - db.commit() - - # Remove releases - available_status = fireEvent('status.get', 'available', single = True) - for rel in m.releases: - if rel.status_id is available_status.get('id'): - db.delete(rel) + try: + db = get_session() + m = db.query(Media).filter_by(library_id = library.get('id')).first() + added = True + do_search = False + search_after = search_after and self.conf('search_on_add', section = 'moviesearcher') + if not m: + m = Media( + library_id = library.get('id'), + profile_id = params.get('profile_id', default_profile.get('id')), + status_id = status_id if status_id else status_active.get('id'), + category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, + ) + db.add(m) db.commit() - movie_dict = m.to_dict(self.default_dict) + onComplete = None + if search_after: + onComplete = self.createOnComplete(m.id) - if do_search and search_after: - onComplete = self.createOnComplete(m.id) - onComplete() + fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) + search_after = False + elif force_readd: - if added: - if params.get('title'): - message = 'Successfully added "%s" to your wanted list.' % params.get('title', '') + # Clean snatched history + for release in m.releases: + if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]: + if params.get('ignore_previous', False): + release.status_id = ignored_status.get('id') + else: + fireEvent('release.delete', release.id, single = True) + + m.profile_id = params.get('profile_id', default_profile.get('id')) + m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m.category_id or None) else: - title = getTitle(m.library) - if title: - message = 'Successfully added "%s" to your wanted list.' % title - else: - message = 'Succesfully added to your wanted list.' - fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = message) + log.debug('Movie already exists, not updating: %s', params) + added = False - db.expire_all() - return movie_dict + if force_readd: + m.status_id = status_id if status_id else status_active.get('id') + m.last_edit = int(time.time()) + do_search = True + + db.commit() + + # Remove releases + available_status = fireEvent('status.get', 'available', single = True) + for rel in m.releases: + if rel.status_id is available_status.get('id'): + db.delete(rel) + db.commit() + + movie_dict = m.to_dict(self.default_dict) + + if do_search and search_after: + onComplete = self.createOnComplete(m.id) + onComplete() + + if added: + if params.get('title'): + message = 'Successfully added "%s" to your wanted list.' % params.get('title', '') + else: + title = getTitle(m.library) + if title: + message = 'Successfully added "%s" to your wanted list.' % title + else: + message = 'Succesfully added to your wanted list.' + fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = message) + + return movie_dict + except: + log.error('Failed deleting media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def addView(self, **kwargs): add_dict = self.add(params = kwargs) @@ -151,42 +157,51 @@ class MovieBase(MovieTypeBase): def edit(self, id = '', **kwargs): - db = get_session() + try: + db = get_session() - available_status = fireEvent('status.get', 'available', single = True) + available_status = fireEvent('status.get', 'available', single = True) - ids = splitString(id) - for media_id in ids: + ids = splitString(id) + for media_id in ids: - m = db.query(Media).filter_by(id = media_id).first() - if not m: - continue + m = db.query(Media).filter_by(id = media_id).first() + if not m: + continue - m.profile_id = kwargs.get('profile_id') + m.profile_id = kwargs.get('profile_id') - cat_id = kwargs.get('category_id') - if cat_id is not None: - m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None + cat_id = kwargs.get('category_id') + if cat_id is not None: + m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None - # Remove releases - for rel in m.releases: - if rel.status_id is available_status.get('id'): - db.delete(rel) - db.commit() + # Remove releases + for rel in m.releases: + if rel.status_id is available_status.get('id'): + db.delete(rel) + db.commit() - # Default title - if kwargs.get('default_title'): - for title in m.library.titles: - title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower() + # Default title + if kwargs.get('default_title'): + for title in m.library.titles: + title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower() - db.commit() + db.commit() - fireEvent('media.restatus', m.id) + fireEvent('media.restatus', m.id) - movie_dict = m.to_dict(self.default_dict) - fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id)) + movie_dict = m.to_dict(self.default_dict) + fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id)) + + return { + 'success': True, + } + except: + log.error('Failed deleting media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() return { - 'success': True, + 'success': False, } diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index 6266ca4e..034a8fb0 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -26,142 +26,163 @@ class MovieLibraryPlugin(LibraryBase): primary_provider = attrs.get('primary_provider', 'imdb') - db = get_session() + try: + db = get_session() - l = db.query(Library).filter_by(identifier = attrs.get('identifier')).first() - if not l: - status = fireEvent('status.get', 'needs_update', single = True) - l = Library( - year = attrs.get('year'), - identifier = attrs.get('identifier'), - plot = toUnicode(attrs.get('plot')), - tagline = toUnicode(attrs.get('tagline')), - status_id = status.get('id'), - info = {} - ) + l = db.query(Library).filter_by(identifier = attrs.get('identifier')).first() + if not l: + status = fireEvent('status.get', 'needs_update', single = True) + l = Library( + year = attrs.get('year'), + identifier = attrs.get('identifier'), + plot = toUnicode(attrs.get('plot')), + tagline = toUnicode(attrs.get('tagline')), + status_id = status.get('id'), + info = {} + ) - title = LibraryTitle( - title = toUnicode(attrs.get('title')), - simple_title = self.simplifyTitle(attrs.get('title')), - ) + title = LibraryTitle( + title = toUnicode(attrs.get('title')), + simple_title = self.simplifyTitle(attrs.get('title')), + ) - l.titles.append(title) + l.titles.append(title) - db.add(l) - db.commit() + db.add(l) + db.commit() - # Update library info - if update_after is not False: - handle = fireEventAsync if update_after is 'async' else fireEvent - handle('library.update.movie', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) + # Update library info + if update_after is not False: + handle = fireEventAsync if update_after is 'async' else fireEvent + handle('library.update.movie', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) - library_dict = l.to_dict(self.default_dict) + library_dict = l.to_dict(self.default_dict) + return library_dict + except: + log.error('Failed adding media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() - return library_dict + return {} def update(self, identifier, default_title = '', extended = False): if self.shuttingDown(): return - db = get_session() - library = db.query(Library).filter_by(identifier = identifier).first() - done_status = fireEvent('status.get', 'done', single = True) + try: + db = get_session() - info = fireEvent('movie.info', merge = True, extended = extended, identifier = identifier) + library = db.query(Library).filter_by(identifier = identifier).first() + done_status = fireEvent('status.get', 'done', single = True) - # Don't need those here - try: del info['in_wanted'] - except: pass - try: del info['in_library'] - except: pass + info = fireEvent('movie.info', merge = True, extended = extended, identifier = identifier) - if not info or len(info) == 0: - log.error('Could not update, no movie info to work with: %s', identifier) - return False + # Don't need those here + try: del info['in_wanted'] + except: pass + try: del info['in_library'] + except: pass - # Main info - library.plot = toUnicode(info.get('plot', '')) - library.tagline = toUnicode(info.get('tagline', '')) - library.year = info.get('year', 0) - library.status_id = done_status.get('id') - library.info.update(info) - db.commit() + if not info or len(info) == 0: + log.error('Could not update, no movie info to work with: %s', identifier) + return False - # Titles - [db.delete(title) for title in library.titles] - db.commit() + # Main info + library.plot = toUnicode(info.get('plot', '')) + library.tagline = toUnicode(info.get('tagline', '')) + library.year = info.get('year', 0) + library.status_id = done_status.get('id') + library.info.update(info) + db.commit() - titles = info.get('titles', []) - log.debug('Adding titles: %s', titles) - counter = 0 + # Titles + [db.delete(title) for title in library.titles] + db.commit() - def_title = None - for title in titles: - if (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title): - def_title = toUnicode(title) - break - counter += 1 + titles = info.get('titles', []) + log.debug('Adding titles: %s', titles) + counter = 0 - #if not def_title: - # def_title = toUnicode(titles[0]) + def_title = None + for title in titles: + if (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title): + def_title = toUnicode(title) + break + counter += 1 - for title in titles: - if not title: - continue - title = toUnicode(title) - t = LibraryTitle( - title = title, - simple_title = self.simplifyTitle(title), - default = title == def_title - ) - library.titles.append(t) + if not def_title: + def_title = toUnicode(titles[0]) - db.commit() - - # Files - images = info.get('images', []) - for image_type in ['poster']: - for image in images.get(image_type, []): - if not isinstance(image, (str, unicode)): + for title in titles: + if not title: continue + title = toUnicode(title) + t = LibraryTitle( + title = title, + simple_title = self.simplifyTitle(title), + default = title == def_title + ) + library.titles.append(t) - file_path = fireEvent('file.download', url = image, single = True) - if file_path: - file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) - try: - file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() - library.files.append(file_obj) - db.commit() + db.commit() - break - except: - log.debug('Failed to attach to library: %s', traceback.format_exc()) + # Files + images = info.get('images', []) + for image_type in ['poster']: + for image in images.get(image_type, []): + if not isinstance(image, (str, unicode)): + continue - library_dict = library.to_dict(self.default_dict) + file_path = fireEvent('file.download', url = image, single = True) + if file_path: + file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) + try: + file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() + library.files.append(file_obj) + db.commit() - db.expire_all() - return library_dict + break + except: + log.debug('Failed to attach to library: %s', traceback.format_exc()) + db.rollback() + + library_dict = library.to_dict(self.default_dict) + return library_dict + except: + log.error('Failed update media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return {} def updateReleaseDate(self, identifier): - db = get_session() - library = db.query(Library).filter_by(identifier = identifier).first() + try: + db = get_session() + library = db.query(Library).filter_by(identifier = identifier).first() - if not library.info: - library_dict = self.update(identifier) - dates = library_dict.get('info', {}).get('release_date') - else: - dates = library.info.get('release_date') + if not library.info: + library_dict = self.update(identifier) + dates = library_dict.get('info', {}).get('release_date') + else: + dates = library.info.get('release_date') - if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: - dates = fireEvent('movie.release_date', identifier = identifier, merge = True) - library.info.update({'release_date': dates}) - db.commit() + if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: + dates = fireEvent('movie.release_date', identifier = identifier, merge = True) + library.info.update({'release_date': dates}) + db.commit() - db.expire_all() - return dates + return dates + except: + log.error('Failed updating release dates: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return {} def simplifyTitle(self, title): diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 0daf1b4e..7ae76a4e 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -89,8 +89,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): 'files': {}, })) - db.expire_all() - self.in_progress = { 'total': len(movies), 'to_go': len(movies), @@ -209,7 +207,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']}) - db.expire_all() return ret def correctRelease(self, nzb = None, media = None, quality = None, **kwargs): @@ -342,7 +339,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): for rel in rels: rel.status_id = ignored_status.get('id') db.commit() - db.expire_all() movie_dict = fireEvent('media.get', media_id = media_id, single = True) log.info('Trying next release for: %s', getTitle(movie_dict['library'])) @@ -352,7 +348,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): except: log.error('Failed searching for next release: %s', traceback.format_exc()) + db.rollback() return False + finally: + db.close() def getSearchTitle(self, media): if media['type'] == 'movie': diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index 02fd27f7..22e23fe2 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -33,7 +33,6 @@ class Suggestion(Plugin): .options(joinedload_all('library')) \ .filter(or_(*[Media.status.has(identifier = s) for s in ['active', 'done']])).all() movies = [x.library.identifier for x in active_movies] - db.expire_all() if not ignored or len(ignored) == 0: ignored = splitString(Env.prop('suggest_ignore', default = '')) @@ -98,7 +97,6 @@ class Suggestion(Plugin): .filter(Media.status_id.in_([active_status.get('id'), done_status.get('id')])).all() movies = [x[0] for x in active_movies] movies.extend(seen) - db.expire_all() ignored.extend([x.get('imdb') for x in cached_suggestion]) suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index 02545ea8..93f94d6a 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -67,27 +67,42 @@ class CoreNotifier(Notification): def clean(self): - db = get_session() - db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete() - db.commit() + try: + db = get_session() + db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete() + db.commit() + except: + log.error('Failed cleaning notification: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def markAsRead(self, ids = None, **kwargs): ids = splitString(ids) if ids else None - db = get_session() + try: + db = get_session() - if ids: - q = db.query(Notif).filter(or_(*[Notif.id == tryInt(s) for s in ids])) - else: - q = db.query(Notif).filter_by(read = False) + if ids: + q = db.query(Notif).filter(or_(*[Notif.id == tryInt(s) for s in ids])) + else: + q = db.query(Notif).filter_by(read = False) - q.update({Notif.read: True}) + q.update({Notif.read: True}) + db.commit() - db.commit() + return { + 'success': True + } + except: + log.error('Failed mark as read: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def listView(self, limit_offset = None, **kwargs): @@ -139,24 +154,30 @@ class CoreNotifier(Notification): def notify(self, message = '', data = None, listener = None): if not data: data = {} - db = get_session() + try: + db = get_session() - data['notification_type'] = listener if listener else 'unknown' + data['notification_type'] = listener if listener else 'unknown' - n = Notif( - message = toUnicode(message), - data = data - ) - db.add(n) - db.commit() + n = Notif( + message = toUnicode(message), + data = data + ) + db.add(n) + db.commit() - ndict = n.to_dict() - ndict['type'] = 'notification' - ndict['time'] = time.time() + ndict = n.to_dict() + ndict['type'] = 'notification' + ndict['time'] = time.time() - self.frontend(type = listener, data = data) + self.frontend(type = listener, data = data) - return True + return True + except: + log.error('Failed notify: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def frontend(self, type = 'notification', data = None, message = None): if not data: data = {} diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index 6ac91274..c7abaee4 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent @@ -41,83 +42,116 @@ class CategoryPlugin(Plugin): for category in categories: temp.append(category.to_dict()) - db.expire_all() return temp def save(self, **kwargs): - db = get_session() + try: + db = get_session() - c = db.query(Category).filter_by(id = kwargs.get('id')).first() - if not c: - c = Category() - db.add(c) + c = db.query(Category).filter_by(id = kwargs.get('id')).first() + if not c: + c = Category() + db.add(c) - c.order = kwargs.get('order', c.order if c.order else 0) - c.label = toUnicode(kwargs.get('label', '')) - c.ignored = toUnicode(kwargs.get('ignored', '')) - c.preferred = toUnicode(kwargs.get('preferred', '')) - c.required = toUnicode(kwargs.get('required', '')) - c.destination = toUnicode(kwargs.get('destination', '')) + c.order = kwargs.get('order', c.order if c.order else 0) + c.label = toUnicode(kwargs.get('label', '')) + c.ignored = toUnicode(kwargs.get('ignored', '')) + c.preferred = toUnicode(kwargs.get('preferred', '')) + c.required = toUnicode(kwargs.get('required', '')) + c.destination = toUnicode(kwargs.get('destination', '')) - db.commit() + db.commit() - category_dict = c.to_dict() + category_dict = c.to_dict() + + return { + 'success': True, + 'category': category_dict + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True, - 'category': category_dict + 'success': False, + 'category': None } def saveOrder(self, **kwargs): - db = get_session() + try: + db = get_session() - order = 0 - for category_id in kwargs.get('ids', []): - c = db.query(Category).filter_by(id = category_id).first() - c.order = order + order = 0 + for category_id in kwargs.get('ids', []): + c = db.query(Category).filter_by(id = category_id).first() + c.order = order - order += 1 + order += 1 - db.commit() + db.commit() + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def delete(self, id = None, **kwargs): - db = get_session() - - success = False - message = '' try: - c = db.query(Category).filter_by(id = id).first() - db.delete(c) - db.commit() + db = get_session() - # Force defaults on all empty category movies - self.removeFromMovie(id) + success = False + message = '' + try: + c = db.query(Category).filter_by(id = id).first() + db.delete(c) + db.commit() - success = True - except Exception as e: - message = log.error('Failed deleting category: %s', e) + # Force defaults on all empty category movies + self.removeFromMovie(id) + + success = True + except Exception as e: + message = log.error('Failed deleting category: %s', e) + + return { + 'success': success, + 'message': message + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() return { - 'success': success, - 'message': message + 'success': False } def removeFromMovie(self, category_id): - db = get_session() - movies = db.query(Media).filter(Media.category_id == category_id).all() + try: + db = get_session() + movies = db.query(Media).filter(Media.category_id == category_id).all() - if len(movies) > 0: - for movie in movies: - movie.category_id = None - db.commit() - - db.expire_all() + if len(movies) > 0: + for movie in movies: + movie.category_id = None + db.commit() + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index 2c5939da..949d8f50 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -119,7 +119,6 @@ class Dashboard(Plugin): 'files': {}, })) - db.expire_all() return { 'success': True, 'empty': len(movies) == 0, diff --git a/couchpotato/core/plugins/file/main.py b/couchpotato/core/plugins/file/main.py index 806af167..c52a9801 100644 --- a/couchpotato/core/plugins/file/main.py +++ b/couchpotato/core/plugins/file/main.py @@ -66,7 +66,6 @@ class FileManager(Plugin): time.sleep(3) log.debug('Cleaning up unused files') - python_cache = Env.get('cache')._path try: db = get_session() for root, dirs, walk_files in os.walk(Env.get('cache_dir')): @@ -78,6 +77,9 @@ class FileManager(Plugin): os.remove(file_path) except: log.error('Failed removing unused file: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def showCacheFile(self, route, **kwargs): Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})]) @@ -103,42 +105,56 @@ class FileManager(Plugin): def add(self, path = '', part = 1, type_tuple = (), available = 1, properties = None): if not properties: properties = {} - type_id = self.getType(type_tuple).get('id') - db = get_session() + try: + db = get_session() + type_id = self.getType(type_tuple).get('id') - f = db.query(File).filter(File.path == toUnicode(path)).first() - if not f: - f = File() - db.add(f) + f = db.query(File).filter(File.path == toUnicode(path)).first() + if not f: + f = File() + db.add(f) - f.path = toUnicode(path) - f.part = part - f.available = available - f.type_id = type_id + f.path = toUnicode(path) + f.part = part + f.available = available + f.type_id = type_id - db.commit() + db.commit() - file_dict = f.to_dict() + file_dict = f.to_dict() - return file_dict + return file_dict + except: + log.error('Failed adding file: %s, %s', (path, traceback.format_exc())) + db.rollback() + finally: + db.close() def getType(self, type_tuple): - db = get_session() - type_type, type_identifier = type_tuple + try: + db = get_session() + type_type, type_identifier = type_tuple - ft = db.query(FileType).filter_by(identifier = type_identifier).first() - if not ft: - ft = FileType( - type = toUnicode(type_type), - identifier = type_identifier, - name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:]) - ) - db.add(ft) - db.commit() + ft = db.query(FileType).filter_by(identifier = type_identifier).first() + if not ft: + ft = FileType( + type = toUnicode(type_type), + identifier = type_identifier, + name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:]) + ) + db.add(ft) + db.commit() + + type_dict = ft.to_dict() + + return type_dict + except: + log.error('Failed getting type: %s, %s', (type_tuple, traceback.format_exc())) + db.rollback() + finally: + db.close() - type_dict = ft.to_dict() - return type_dict def getTypes(self): diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 31321852..914d46f3 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent @@ -37,16 +38,20 @@ class ProfilePlugin(Plugin): # Get all active movies without profile active_status = fireEvent('status.get', 'active', single = True) - db = get_session() - movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all() + try: + db = get_session() + movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all() - if len(movies) > 0: - default_profile = self.default() - for movie in movies: - movie.profile_id = default_profile.get('id') - db.commit() - - db.expire_all() + if len(movies) > 0: + default_profile = self.default() + for movie in movies: + movie.profile_id = default_profile.get('id') + db.commit() + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def allView(self, **kwargs): @@ -66,44 +71,53 @@ class ProfilePlugin(Plugin): for profile in profiles: temp.append(profile.to_dict(self.to_dict)) - db.expire_all() return temp def save(self, **kwargs): - db = get_session() + try: + db = get_session() - p = db.query(Profile).filter_by(id = kwargs.get('id')).first() - if not p: - p = Profile() - db.add(p) + p = db.query(Profile).filter_by(id = kwargs.get('id')).first() + if not p: + p = Profile() + db.add(p) - p.label = toUnicode(kwargs.get('label')) - p.order = kwargs.get('order', p.order if p.order else 0) - p.core = kwargs.get('core', False) + p.label = toUnicode(kwargs.get('label')) + p.order = kwargs.get('order', p.order if p.order else 0) + p.core = kwargs.get('core', False) - #delete old types - [db.delete(t) for t in p.types] + #delete old types + [db.delete(t) for t in p.types] - order = 0 - for type in kwargs.get('types', []): - t = ProfileType( - order = order, - finish = type.get('finish') if order > 0 else 1, - wait_for = kwargs.get('wait_for'), - quality_id = type.get('quality_id') - ) - p.types.append(t) + order = 0 + for type in kwargs.get('types', []): + t = ProfileType( + order = order, + finish = type.get('finish') if order > 0 else 1, + wait_for = kwargs.get('wait_for'), + quality_id = type.get('quality_id') + ) + p.types.append(t) - order += 1 + order += 1 - db.commit() + db.commit() - profile_dict = p.to_dict(self.to_dict) + profile_dict = p.to_dict(self.to_dict) + + return { + 'success': True, + 'profile': profile_dict + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True, - 'profile': profile_dict + 'success': False } def default(self): @@ -114,93 +128,119 @@ class ProfilePlugin(Plugin): .first() default_dict = default.to_dict(self.to_dict) - db.expire_all() return default_dict def saveOrder(self, **kwargs): - db = get_session() + try: + db = get_session() - order = 0 - for profile in kwargs.get('ids', []): - p = db.query(Profile).filter_by(id = profile).first() - p.hide = kwargs.get('hidden')[order] - p.order = order + order = 0 + for profile in kwargs.get('ids', []): + p = db.query(Profile).filter_by(id = profile).first() + p.hide = kwargs.get('hidden')[order] + p.order = order - order += 1 + order += 1 - db.commit() + db.commit() + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def delete(self, id = None, **kwargs): - db = get_session() - - success = False - message = '' try: - p = db.query(Profile).filter_by(id = id).first() + db = get_session() - db.delete(p) - db.commit() + success = False + message = '' + try: + p = db.query(Profile).filter_by(id = id).first() - # Force defaults on all empty profile movies - self.forceDefaults() + db.delete(p) + db.commit() - success = True - except Exception as e: - message = log.error('Failed deleting Profile: %s', e) + # Force defaults on all empty profile movies + self.forceDefaults() + + success = True + except Exception as e: + message = log.error('Failed deleting Profile: %s', e) + + return { + 'success': success, + 'message': message + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() return { - 'success': success, - 'message': message + 'success': False } def fill(self): - db = get_session() + try: + db = get_session() - profiles = [{ - 'label': 'Best', - 'qualities': ['720p', '1080p', 'brrip', 'dvdrip'] - }, { - 'label': 'HD', - 'qualities': ['720p', '1080p'] - }, { - 'label': 'SD', - 'qualities': ['dvdrip', 'dvdr'] - }] + profiles = [{ + 'label': 'Best', + 'qualities': ['720p', '1080p', 'brrip', 'dvdrip'] + }, { + 'label': 'HD', + 'qualities': ['720p', '1080p'] + }, { + 'label': 'SD', + 'qualities': ['dvdrip', 'dvdr'] + }] - # Create default quality profile - order = -2 - for profile in profiles: - log.info('Creating default profile: %s', profile.get('label')) - p = Profile( - label = toUnicode(profile.get('label')), - order = order - ) - db.add(p) - - quality_order = 0 - for quality in profile.get('qualities'): - quality = fireEvent('quality.single', identifier = quality, single = True) - profile_type = ProfileType( - quality_id = quality.get('id'), - profile = p, - finish = True, - wait_for = 0, - order = quality_order + # Create default quality profile + order = -2 + for profile in profiles: + log.info('Creating default profile: %s', profile.get('label')) + p = Profile( + label = toUnicode(profile.get('label')), + order = order ) - p.types.append(profile_type) + db.add(p) - quality_order += 1 + quality_order = 0 + for quality in profile.get('qualities'): + quality = fireEvent('quality.single', identifier = quality, single = True) + profile_type = ProfileType( + quality_id = quality.get('id'), + profile = p, + finish = True, + wait_for = 0, + order = quality_order + ) + p.types.append(profile_type) - order += 1 + quality_order += 1 - db.commit() + order += 1 - return True + db.commit() + + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return False diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index a49c17e2..d56ff89f 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent @@ -98,70 +99,88 @@ class QualityPlugin(Plugin): def saveSize(self, **kwargs): - db = get_session() - quality = db.query(Quality).filter_by(identifier = kwargs.get('identifier')).first() + try: + db = get_session() + quality = db.query(Quality).filter_by(identifier = kwargs.get('identifier')).first() - if quality: - setattr(quality, kwargs.get('value_type'), kwargs.get('value')) - db.commit() + if quality: + setattr(quality, kwargs.get('value_type'), kwargs.get('value')) + db.commit() - self.cached_qualities = None + self.cached_qualities = None + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def fill(self): - db = get_session() + try: + db = get_session() - order = 0 - for q in self.qualities: + order = 0 + for q in self.qualities: - # Create quality - qual = db.query(Quality).filter_by(identifier = q.get('identifier')).first() + # Create quality + qual = db.query(Quality).filter_by(identifier = q.get('identifier')).first() - if not qual: - log.info('Creating quality: %s', q.get('label')) - qual = Quality() - qual.order = order - qual.identifier = q.get('identifier') - qual.label = toUnicode(q.get('label')) - qual.size_min, qual.size_max = q.get('size') + if not qual: + log.info('Creating quality: %s', q.get('label')) + qual = Quality() + qual.order = order + qual.identifier = q.get('identifier') + qual.label = toUnicode(q.get('label')) + qual.size_min, qual.size_max = q.get('size') - db.add(qual) + db.add(qual) - # Create single quality profile - prof = db.query(Profile).filter( - Profile.core == True - ).filter( - Profile.types.any(quality = qual) - ).all() + # Create single quality profile + prof = db.query(Profile).filter( + Profile.core == True + ).filter( + Profile.types.any(quality = qual) + ).all() - if not prof: - log.info('Creating profile: %s', q.get('label')) - prof = Profile( - core = True, - label = toUnicode(qual.label), - order = order - ) - db.add(prof) + if not prof: + log.info('Creating profile: %s', q.get('label')) + prof = Profile( + core = True, + label = toUnicode(qual.label), + order = order + ) + db.add(prof) - profile_type = ProfileType( - quality = qual, - profile = prof, - finish = True, - order = 0 - ) - prof.types.append(profile_type) + profile_type = ProfileType( + quality = qual, + profile = prof, + finish = True, + order = 0 + ) + prof.types.append(profile_type) - order += 1 + order += 1 - db.commit() + db.commit() - time.sleep(0.3) # Wait a moment + time.sleep(0.3) # Wait a moment - return True + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return False def guess(self, files, extra = None): if not extra: extra = {} diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 4726dbfa..a478b64d 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -88,63 +88,69 @@ class Release(Plugin): elif rel.status_id in [snatched_status.get('id'), downloaded_status.get('id')]: self.updateStatus(id = rel.id, status = ignored_status) - db.expire_all() def add(self, group): - db = get_session() - - identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier']) - - done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) - - # Add movie - media = db.query(Media).filter_by(library_id = group['library'].get('id')).first() - if not media: - media = Media( - library_id = group['library'].get('id'), - profile_id = 0, - status_id = done_status.get('id') - ) - db.add(media) - db.commit() - - # Add Release - rel = db.query(Relea).filter( - or_( - Relea.identifier == identifier, - and_(Relea.identifier.startswith(group['library']['identifier']), Relea.status_id == snatched_status.get('id')) - ) - ).first() - if not rel: - rel = Relea( - identifier = identifier, - movie = media, - quality_id = group['meta_data']['quality'].get('id'), - status_id = done_status.get('id') - ) - db.add(rel) - db.commit() - - # Add each file type - added_files = [] - for type in group['files']: - for cur_file in group['files'][type]: - added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie') - added_files.append(added_file.get('id')) - - # Add the release files in batch try: - added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all() - rel.files.extend(added_files) - db.commit() + db = get_session() + + identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier']) + + done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) + + # Add movie + media = db.query(Media).filter_by(library_id = group['library'].get('id')).first() + if not media: + media = Media( + library_id = group['library'].get('id'), + profile_id = 0, + status_id = done_status.get('id') + ) + db.add(media) + db.commit() + + # Add Release + rel = db.query(Relea).filter( + or_( + Relea.identifier == identifier, + and_(Relea.identifier.startswith(group['library']['identifier']), Relea.status_id == snatched_status.get('id')) + ) + ).first() + if not rel: + rel = Relea( + identifier = identifier, + movie = media, + quality_id = group['meta_data']['quality'].get('id'), + status_id = done_status.get('id') + ) + db.add(rel) + db.commit() + + # Add each file type + added_files = [] + for type in group['files']: + for cur_file in group['files'][type]: + added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie') + added_files.append(added_file.get('id')) + + # Add the release files in batch + try: + added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all() + rel.files.extend(added_files) + db.commit() + except: + log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc())) + + fireEvent('media.restatus', media.id) + + return True except: - log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc())) + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - fireEvent('media.restatus', media.id) - - db.expire_all() - return True + return False def saveFile(self, filepath, type = 'unknown', include_media_info = False): @@ -165,31 +171,43 @@ class Release(Plugin): def delete(self, id): - db = get_session() + try: + db = get_session() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - rel.delete() - db.commit() - return True + rel = db.query(Relea).filter_by(id = id).first() + if rel: + rel.delete() + db.commit() + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return False def clean(self, id): - db = get_session() + try: + db = get_session() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - for release_file in rel.files: - if not os.path.isfile(ss(release_file.path)): - db.delete(release_file) - db.commit() + rel = db.query(Relea).filter_by(id = id).first() + if rel: + for release_file in rel.files: + if not os.path.isfile(ss(release_file.path)): + db.delete(release_file) + db.commit() - if len(rel.files) == 0: - self.delete(id) + if len(rel.files) == 0: + self.delete(id) - return True + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return False @@ -338,9 +356,10 @@ class Release(Plugin): except: log.error('Failed storing download status: %s', traceback.format_exc()) + db.rollback() return False finally: - db.expire_all() + db.close() return True @@ -371,49 +390,58 @@ class Release(Plugin): def createFromSearch(self, search_results, media, quality_type): available_status = fireEvent('status.get', ['available'], single = True) - db = get_session() - found_releases = [] + try: + db = get_session() - for rel in search_results: + found_releases = [] - rel_identifier = md5(rel['url']) - found_releases.append(rel_identifier) + for rel in search_results: - rls = db.query(Relea).filter_by(identifier = rel_identifier).first() - if not rls: - rls = Relea( - identifier = rel_identifier, - movie_id = media.get('id'), - #media_id = media.get('id'), - quality_id = quality_type.get('quality_id'), - status_id = available_status.get('id') - ) - db.add(rls) - else: - [db.delete(old_info) for old_info in rls.info] - rls.last_edit = int(time.time()) + rel_identifier = md5(rel['url']) + found_releases.append(rel_identifier) - db.commit() - - for info in rel: - try: - if not isinstance(rel[info], (str, unicode, int, long, float)): - continue - - rls_info = ReleaseInfo( - identifier = info, - value = toUnicode(rel[info]) + rls = db.query(Relea).filter_by(identifier = rel_identifier).first() + if not rls: + rls = Relea( + identifier = rel_identifier, + movie_id = media.get('id'), + #media_id = media.get('id'), + quality_id = quality_type.get('quality_id'), + status_id = available_status.get('id') ) - rls.info.append(rls_info) - except InterfaceError: - log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) + db.add(rls) + else: + [db.delete(old_info) for old_info in rls.info] + rls.last_edit = int(time.time()) - db.commit() + db.commit() - rel['status_id'] = rls.status_id + for info in rel: + try: + if not isinstance(rel[info], (str, unicode, int, long, float)): + continue - return found_releases + rls_info = ReleaseInfo( + identifier = info, + value = toUnicode(rel[info]) + ) + rls.info.append(rls_info) + except InterfaceError: + log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) + + db.commit() + + rel['status_id'] = rls.status_id + + return found_releases + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return [] def forMovie(self, id = None): @@ -442,32 +470,39 @@ class Release(Plugin): def updateStatus(self, id, status = None): if not status: return False - db = get_session() + try: + db = get_session() - rel = db.query(Relea).filter_by(id = id).first() - if rel and status and rel.status_id != status.get('id'): + rel = db.query(Relea).filter_by(id = id).first() + if rel and status and rel.status_id != status.get('id'): - item = {} - for info in rel.info: - item[info.identifier] = info.value + item = {} + for info in rel.info: + item[info.identifier] = info.value - release_name = None - if rel.files: - for file_item in rel.files: - if file_item.type.identifier == 'movie': - release_name = os.path.basename(file_item.path) - break - else: - release_name = item['name'] + release_name = None + if rel.files: + for file_item in rel.files: + if file_item.type.identifier == 'movie': + release_name = os.path.basename(file_item.path) + break + else: + release_name = item['name'] - #update status in Db - log.debug('Marking release %s as %s', (release_name, status.get("label"))) - rel.status_id = status.get('id') - rel.last_edit = int(time.time()) - db.commit() + #update status in Db + log.debug('Marking release %s as %s', (release_name, status.get("label"))) + rel.status_id = status.get('id') + rel.last_edit = int(time.time()) + db.commit() - #Update all movie info as there is no release update function - fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict()) + #Update all movie info as there is no release update function + fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict()) - db.expire_all() - return True + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return False diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index f9f44d85..f86bebad 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -439,6 +439,7 @@ class Renamer(Plugin): db.commit() except Exception as e: log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc())) + db.rollback() # Go over current movie releases for release in movie.releases: diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 16c9e1db..daff0919 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -597,6 +597,7 @@ class Scanner(Plugin): # Check if path is already in db if not imdb_id: + db = get_session() for cf in files['movie']: f = db.query(File).filter_by(path = toUnicode(cf)).first() @@ -607,7 +608,6 @@ class Scanner(Plugin): break except: pass - db.expire_all() # Search based on identifiers if not imdb_id: diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py index 3ec2b45a..08f46984 100644 --- a/couchpotato/core/plugins/status/main.py +++ b/couchpotato/core/plugins/status/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent @@ -79,47 +80,57 @@ class StatusPlugin(Plugin): if not isinstance(identifiers, list): identifiers = [identifiers] - db = get_session() - return_list = [] + try: + db = get_session() + return_list = [] - for identifier in identifiers: + for identifier in identifiers: - if self.status_cached.get(identifier): - return_list.append(self.status_cached.get(identifier)) - continue + if self.status_cached.get(identifier): + return_list.append(self.status_cached.get(identifier)) + continue - s = db.query(Status).filter_by(identifier = identifier).first() - if not s: - s = Status( - identifier = identifier, - label = toUnicode(identifier.capitalize()) - ) - db.add(s) - db.commit() + s = db.query(Status).filter_by(identifier = identifier).first() + if not s: + s = Status( + identifier = identifier, + label = toUnicode(identifier.capitalize()) + ) + db.add(s) + db.commit() - status_dict = s.to_dict() + status_dict = s.to_dict() - self.status_cached[identifier] = status_dict - return_list.append(status_dict) + self.status_cached[identifier] = status_dict + return_list.append(status_dict) - return return_list if len(identifiers) > 1 else return_list[0] + return return_list if len(identifiers) > 1 else return_list[0] + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def fill(self): - db = get_session() + try: + db = get_session() - for identifier, label in self.statuses.items(): - s = db.query(Status).filter_by(identifier = identifier).first() - if not s: - log.info('Creating status: %s', label) - s = Status( - identifier = identifier, - label = toUnicode(label) - ) - db.add(s) + for identifier, label in self.statuses.items(): + s = db.query(Status).filter_by(identifier = identifier).first() + if not s: + log.info('Creating status: %s', label) + s = Status( + identifier = identifier, + label = toUnicode(label) + ) + db.add(s) - s.label = toUnicode(label) - db.commit() - - #db.close() + s.label = toUnicode(label) + db.commit() + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index 5e4952ae..3b575176 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -1,4 +1,5 @@ from __future__ import with_statement +import traceback from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode @@ -220,14 +221,20 @@ class Settings(object): def setProperty(self, identifier, value = ''): from couchpotato import get_session - db = get_session() + try: + db = get_session() - p = db.query(Properties).filter_by(identifier = identifier).first() - if not p: - p = Properties() - db.add(p) + p = db.query(Properties).filter_by(identifier = identifier).first() + if not p: + p = Properties() + db.add(p) - p.identifier = identifier - p.value = toUnicode(value) + p.identifier = identifier + p.value = toUnicode(value) - db.commit() + db.commit() + except: + self.log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/environment.py b/couchpotato/environment.py index c409da80..1c5863d1 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -2,7 +2,6 @@ from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.loader import Loader from couchpotato.core.settings import Settings from sqlalchemy.engine import create_engine -from sqlalchemy.orm import scoped_session from sqlalchemy.orm.session import sessionmaker import os @@ -24,7 +23,7 @@ class Env(object): _quiet = False _daemonized = False _desktop = None - _session = None + _engine = None ''' Data paths and directories ''' _app_dir = "" @@ -54,20 +53,20 @@ class Env(object): return setattr(Env, '_' + attr, value) @staticmethod - def getSession(engine = None): - existing_session = Env.get('session') - if existing_session: - return existing_session - - engine = Env.getEngine() - session = scoped_session(sessionmaker(bind = engine)) - Env.set('session', session) - - return session + def getSession(): + session = sessionmaker(bind = Env.getEngine()) + return session() @staticmethod def getEngine(): - return create_engine(Env.get('db_path'), echo = False) + existing_engine = Env.get('engine') + if existing_engine: + return existing_engine + + engine = create_engine(Env.get('db_path'), echo = False) + Env.set('engine', engine) + + return engine @staticmethod def setting(attr, section = 'core', value = None, default = '', type = None): From afac06081cd388e786d7c0ce484c092e087f2d4a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 26 Jan 2014 10:05:05 +0100 Subject: [PATCH 36/42] Defer settings dom injection --- couchpotato/static/scripts/page/settings.js | 37 ++++++++++++--------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index ba97bcd0..9c9453e0 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -118,20 +118,18 @@ Page.Settings = new Class({ create: function(json){ var self = this; - self.el.adopt( - self.tabs_container = new Element('ul.tabs'), - self.containers = new Element('form.uniForm.containers').adopt( - new Element('label.advanced_toggle').adopt( - new Element('span', { - 'text': 'Show advanced settings' - }), - self.advanced_toggle = new Element('input[type=checkbox].inlay', { - 'checked': +Cookie.read('advanced_toggle_checked'), - 'events': { - 'change': self.showAdvanced.bind(self) - } - }) - ) + self.tabs_container = new Element('ul.tabs'); + self.containers = new Element('form.uniForm.containers').adopt( + new Element('label.advanced_toggle').adopt( + new Element('span', { + 'text': 'Show advanced settings' + }), + self.advanced_toggle = new Element('input[type=checkbox].inlay', { + 'checked': +Cookie.read('advanced_toggle_checked'), + 'events': { + 'change': self.showAdvanced.bind(self) + } + }) ) ); self.showAdvanced(); @@ -197,8 +195,15 @@ Page.Settings = new Class({ }); }); - self.fireEvent('create'); - self.openTab(); + setTimeout(function(){ + self.fireEvent('create'); + self.openTab(); + + self.el.adopt( + self.tabs_container, + self.containers + ); + }, 0); }, From 9e991e1595dda2af91bd67cf8edccee32500503b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 26 Jan 2014 10:40:15 +0100 Subject: [PATCH 37/42] Fix Yify proxy check --- couchpotato/core/providers/torrent/base.py | 3 ++- couchpotato/core/providers/torrent/yify/main.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index 02cc549e..e134c8f3 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -1,3 +1,4 @@ +import traceback from couchpotato.core.helpers.variable import getImdb, md5, cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import YarrProvider @@ -32,7 +33,7 @@ class TorrentProvider(YarrProvider): try: data = self.urlopen(proxy, timeout = 3, show_error = False) except: - log.debug('Failed %s proxy %s', (self.getName(), proxy)) + log.debug('Failed %s proxy %s: %s', (self.getName(), proxy, traceback.format_exc())) if self.correctProxy(data): log.debug('Using proxy for %s: %s', (self.getName(), proxy)) diff --git a/couchpotato/core/providers/torrent/yify/main.py b/couchpotato/core/providers/torrent/yify/main.py index bb94016c..6deaf7bb 100644 --- a/couchpotato/core/providers/torrent/yify/main.py +++ b/couchpotato/core/providers/torrent/yify/main.py @@ -17,10 +17,11 @@ class Yify(TorrentMagnetProvider): http_time_between_calls = 1 #seconds proxy_list = [ - 'https://yify-torrents.im', 'http://yify.unlocktorrent.com', 'http://yify.ftwnet.co.uk', 'http://yify-torrents.com.come.in', + 'http://yts.re', + 'https://yify-torrents.im', ] def search(self, movie, quality): @@ -61,4 +62,5 @@ class Yify(TorrentMagnetProvider): log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) def correctProxy(self, data): - return 'title="YIFY-Torrents RSS feed"' in data + data = data.lower() + return 'yify' in data and 'yts' in data From f91081e39cfb69c754d39cf7dbba0133b4dd854f Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 26 Jan 2014 10:52:50 +0100 Subject: [PATCH 38/42] uTorrent hostname hint --- couchpotato/core/downloaders/utorrent/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 77bd1cd8..da160956 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -24,7 +24,7 @@ config = [{ { 'name': 'host', 'default': 'localhost:8000', - 'description': 'Hostname with port. Usually localhost:8000', + 'description': 'Port can be found in settings when enabling WebUI.', }, { 'name': 'username', From 15a19949b89032b3078489722d3dda23a17e991c Mon Sep 17 00:00:00 2001 From: Sami Haahtinen Date: Sun, 26 Jan 2014 19:26:15 +0200 Subject: [PATCH 39/42] Fix rTorrent connectivity The combination of cleanHost and rTorrent.connect issues caused rTorrent connections to fail. This update fixes cleanHost() so that it can actually cope with SSL based hosts and finishes the migration to cleanHost() in connect() Conflicts: couchpotato/core/helpers/variable.py --- couchpotato/core/downloaders/rtorrent/main.py | 2 +- couchpotato/core/helpers/variable.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 27177756..cfd1dce0 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -42,7 +42,7 @@ class rTorrent(Downloader): if self.rt is not None: return self.rt - url = cleanHost(self.conf('host'), protocol = True, ssl = self.conf('ssl')) + '/' + self.conf('rpc_url').strip('/ ') + '/' + url = cleanHost(self.conf('host'), protocol = True, ssl = self.conf('ssl')) + self.conf('rpc_url') if self.conf('username') and self.conf('password'): self.rt = RTorrent( diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 47810a66..a586ceff 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -136,9 +136,25 @@ def getExt(filename): def cleanHost(host, protocol = True, ssl = False, username = None, password = None): + """Return a cleaned up host with given url options set + + Changes protocol to https if ssl is set to True and http if ssl is set to false. + >>> cleanHost("localhost:80", ssl=True) + 'https://localhost:80/' + >>> cleanHost("localhost:80", ssl=False) + 'http://localhost:80/' + + Username and password is managed with the username and password variables + >>> cleanHost("localhost:80", username="user", password="passwd") + 'http://user:passwd@localhost:80/' + + Output without scheme (protocol) can be forced with protocol=False + >>> cleanHost("localhost:80", protocol=False) + 'localhost:80' + """ if not '://' in host and protocol: - host = 'https://' if ssl else 'http://' + host + host = ('https://' if ssl else 'http://') + host if not protocol: host = host.split('://', 1)[-1] From 9d55ecffe9911d8d5f49812a0f304c80934d17a7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 27 Jan 2014 21:58:48 +0100 Subject: [PATCH 40/42] Add log var --- couchpotato/core/media/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 512c52c2..cf9302b1 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -1,9 +1,11 @@ import traceback -from couchpotato import get_session +from couchpotato import get_session, CPLog from couchpotato.core.event import addEvent, fireEventAsync, fireEvent from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Media +log = CPLog(__name__) + class MediaBase(Plugin): From 461e469f2853f5048d4743151209439ecbd746d9 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Thu, 6 Feb 2014 22:18:41 +1300 Subject: [PATCH 41/42] Updated rtorrent-python library - Fixed bencode encoding bug with long types --- libs/rtorrent/lib/bencode.py | 2 +- libs/rtorrent/lib/torrentparser.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/rtorrent/lib/bencode.py b/libs/rtorrent/lib/bencode.py index 97bd2f0e..ba99ef93 100755 --- a/libs/rtorrent/lib/bencode.py +++ b/libs/rtorrent/lib/bencode.py @@ -267,7 +267,7 @@ def _encode_dict(data): def encode(data): if isinstance(data, bool): return False - elif isinstance(data, int): + elif isinstance(data, (int, long)): return _encode_int(data) elif isinstance(data, bytes): return _encode_string(data) diff --git a/libs/rtorrent/lib/torrentparser.py b/libs/rtorrent/lib/torrentparser.py index 30170d32..b339bce4 100755 --- a/libs/rtorrent/lib/torrentparser.py +++ b/libs/rtorrent/lib/torrentparser.py @@ -90,10 +90,10 @@ class TorrentParser(): def _calc_info_hash(self): self.info_hash = None if "info" in self._torrent_decoded.keys(): - info_encoded = bencode.encode(self._torrent_decoded["info"]) + info_encoded = bencode.encode(self._torrent_decoded["info"]) - if info_encoded: - self.info_hash = hashlib.sha1(info_encoded).hexdigest().upper() + if info_encoded: + self.info_hash = hashlib.sha1(info_encoded).hexdigest().upper() return(self.info_hash) From 62524e01e158caac6a94eb5b6bf84fa0774151e2 Mon Sep 17 00:00:00 2001 From: Ramon van Dam Date: Fri, 7 Feb 2014 12:08:17 +0100 Subject: [PATCH 42/42] * Added category 'bd50' (BR-Disk) to Torrent provider TorrentBytes * Changed category identifier for category 'brrip' for Torrent provider TorrentBytes (see issue #2795) --- couchpotato/core/providers/torrent/torrentbytes/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/torrent/torrentbytes/main.py b/couchpotato/core/providers/torrent/torrentbytes/main.py index 91939e78..603da6e0 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/main.py +++ b/couchpotato/core/providers/torrent/torrentbytes/main.py @@ -20,12 +20,12 @@ class TorrentBytes(TorrentProvider): } cat_ids = [ - ([5], ['720p', '1080p']), + ([5], ['720p', '1080p', 'bd50']), ([19], ['cam']), ([19], ['ts', 'tc']), ([19], ['r5', 'scr']), ([19], ['dvdrip']), - ([5], ['brrip']), + ([19], ['brrip']), ([20], ['dvdr']), ]