From a6c32a7e30111524c9e3511cac548f6177a7350d Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 29 Jul 2013 21:07:35 +1200 Subject: [PATCH 01/57] Fixed Plex notifications Conflicts: couchpotato/core/notifications/plex/main.py --- .../core/notifications/plex/__init__.py | 10 +- couchpotato/core/notifications/plex/main.py | 202 +++++++++++++----- 2 files changed, 152 insertions(+), 60 deletions(-) mode change 100644 => 100755 couchpotato/core/notifications/plex/__init__.py mode change 100644 => 100755 couchpotato/core/notifications/plex/main.py diff --git a/couchpotato/core/notifications/plex/__init__.py b/couchpotato/core/notifications/plex/__init__.py old mode 100644 new mode 100755 index c00ea6d4..e2984ad9 --- a/couchpotato/core/notifications/plex/__init__.py +++ b/couchpotato/core/notifications/plex/__init__.py @@ -17,10 +17,14 @@ config = [{ 'type': 'enabler', }, { - 'name': 'host', + 'name': 'media_server', 'default': 'localhost', - 'description': 'Default should be on localhost', - 'advanced': True, + 'description': 'Media server hostname' + }, + { + 'name': 'clients', + 'default': '', + 'description': 'Comma separated list of client names\'s (computer names).' }, { 'name': 'on_snatch', diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py old mode 100644 new mode 100755 index 02c9b30a..f9746a09 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -1,77 +1,179 @@ +from datetime import datetime +import json from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.helpers.variable import cleanHost, splitString +from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from urllib2 import URLError -from urlparse import urlparse from xml.dom import minidom import traceback +import requests + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree log = CPLog(__name__) class Plex(Notification): + client_update_time = 5 * 60 def __init__(self): super(Plex, self).__init__() + self.clients = {} + self.clients_updated = None addEvent('renamer.after', self.addToLibrary) - def addToLibrary(self, message = None, group = {}): + def updateClients(self, force=False): + if not self.conf('media_server'): + log.warning("Plex media server hostname is required") + return + + since_update = ((datetime.now() - self.clients_updated).total_seconds())\ + if self.clients_updated is not None else None + + if force or self.clients_updated is None or since_update > self.client_update_time: + self.clients = {} + + client_result = etree.fromstring(self.urlopen('http://%s:32400/clients' % self.conf('media_server'))) + + hosts = [x.strip().lower() for x in self.conf('clients').split(',')] + + for server in client_result.findall('Server'): + if server.get('name').lower() in hosts: + hosts.remove(server.get('name').lower()) + protocol = server.get('protocol', 'xbmchttp') + + if protocol in ['xbmcjson', 'xbmchttp']: + self.clients[server.get('name')] = { + 'name': server.get('name'), + 'address': server.get('address'), + 'port': server.get('port'), + 'protocol': protocol + } + + if len(hosts) > 0: + log.warning('unable to find some plex hosts: %s', ', '.join(hosts)) + + log.info('found hosts: %s', ', '.join(self.clients.keys())) + + self.clients_updated = datetime.now() + + + def addToLibrary(self, message=None, group={}): if self.isDisabled(): return log.info('Sending notification to Plex') - hosts = self.getHosts(port = 32400) - for host in hosts: + source_type = ['movie'] + base_url = 'http://%s:32400/library/sections' % self.conf('media_server') + refresh_url = '%s/%%s/refresh' % base_url - source_type = ['movie'] - base_url = '%s/library/sections' % host - refresh_url = '%s/%%s/refresh' % base_url + try: + sections_xml = self.urlopen(base_url) + xml_sections = minidom.parseString(sections_xml) + sections = xml_sections.getElementsByTagName('Directory') - try: - sections_xml = self.urlopen(base_url) - xml_sections = minidom.parseString(sections_xml) - sections = xml_sections.getElementsByTagName('Directory') + for s in sections: + if s.getAttribute('type') in source_type: + url = refresh_url % s.getAttribute('key') + x = self.urlopen(url) - for s in sections: - if s.getAttribute('type') in source_type: - url = refresh_url % s.getAttribute('key') - x = self.urlopen(url) - - except: - log.error('Plex library update failed for %s, Media Server not running: %s', (host, traceback.format_exc(1))) - return False + except: + log.error('Plex library update failed for %s, Media Server not running: %s', + (self.conf('media_server'), traceback.format_exc(1))) + return False return True - def notify(self, message = '', data = {}, listener = None): + def send_http(self, command, client): + url = 'http://%s:%s/xbmcCmds/xbmcHttp/?%s' % ( + client['address'], + client['port'], + tryUrlencode(command) + ) - hosts = self.getHosts(port = 3000) - successful = 0 - for host in hosts: - if self.send({'command': 'ExecBuiltIn', 'parameter': 'Notification(CouchPotato, %s)' % message}, host): - successful += 1 - - return successful == len(hosts) - - def send(self, command, host): - - url = '%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command)) headers = {} try: - self.urlopen(url, headers = headers, show_error = False) - except URLError: - log.error("Couldn't sent command to Plex, probably just running Media Server") - return False - except: - log.error("Couldn't sent command to Plex: %s", traceback.format_exc()) + self.urlopen(url, headers=headers, timeout=3, show_error=False) + except Exception, err: + log.error("Couldn't sent command to Plex: %s", err) return False - log.info('Plex notification to %s successful.', host) return True + def notify_http(self, message='', data={}, listener=None): + total = 0 + successful = 0 + + data = { + 'command': 'ExecBuiltIn', + 'parameter': 'Notification(CouchPotato, %s)' % message + } + + for name, client in self.clients.items(): + if client['protocol'] == 'xbmchttp': + total += 1 + if self.send_http(data, client): + successful += 1 + + return successful == total + + def send_json(self, method, params, client): + log.debug('send_json("%s", %s, %s)', (method, params, client)) + url = 'http://%s:%s/jsonrpc' % ( + client['address'], + client['port'] + ) + + headers = { + 'Content-Type': 'application/json' + } + + request = { + 'id':1, + 'jsonrpc': '2.0', + 'method': method, + 'params': params + } + + try: + requests.post(url, headers=headers, timeout=3, data=json.dumps(request)) + except Exception, err: + log.error("Couldn't sent command to Plex: %s", err) + return False + + return True + + def notify_json(self, message='', data={}, listener=None): + total = 0 + successful = 0 + + params = { + 'title': 'CouchPotato', + 'message': message + } + + for name, client in self.clients.items(): + if client['protocol'] == 'xbmcjson': + total += 1 + if self.send_json('GUI.ShowNotification', params, client): + successful += 1 + + return successful == total + + def notify(self, message='', data={}, listener=None, forceUpdate=True): + self.updateClients(forceUpdate) + + http_result = self.notify_http(message, data, listener) + json_result = self.notify_json(message, data, listener) + + return http_result and json_result + def test(self, **kwargs): test_type = self.testNotifyName() @@ -79,27 +181,13 @@ class Plex(Notification): log.info('Sending test to %s', test_type) success = self.notify( - message = self.test_message, - data = {}, - listener = 'test' + message=self.test_message, + data={}, + listener='test', + forceUpdate=True ) success2 = self.addToLibrary() return { 'success': success or success2 } - - def getHosts(self, port = None): - - raw_hosts = splitString(self.conf('host')) - hosts = [] - - for h in raw_hosts: - h = cleanHost(h) - p = urlparse(h) - h = h.rstrip('/') - if port and not p.port: - h += ':%s' % port - hosts.append(h) - - return hosts From c92aa91aa7e2624f36171943220d271d3d4ab13f Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 29 Jul 2013 21:07:50 +1200 Subject: [PATCH 02/57] Corrected notify() force parameter default. --- couchpotato/core/notifications/plex/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index f9746a09..962e0b87 100755 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -166,8 +166,8 @@ class Plex(Notification): return successful == total - def notify(self, message='', data={}, listener=None, forceUpdate=True): - self.updateClients(forceUpdate) + def notify(self, message='', data={}, listener=None, force=False): + self.updateClients(force) http_result = self.notify_http(message, data, listener) json_result = self.notify_json(message, data, listener) From b824ef93bded98b09cfa41fa72374504c0753bfa Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 4 Aug 2013 15:39:02 +1200 Subject: [PATCH 03/57] Fix plex notifications test method. --- couchpotato/core/notifications/plex/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index 962e0b87..09fd6ff1 100755 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -184,7 +184,7 @@ class Plex(Notification): message=self.test_message, data={}, listener='test', - forceUpdate=True + force=True ) success2 = self.addToLibrary() From 9b5166826f804ee0218c273de29cacf13b06ad55 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 24 Sep 2013 22:37:40 +0200 Subject: [PATCH 04/57] Cleanup Plex notification --- .../core/notifications/plex/__init__.py | 7 +- couchpotato/core/notifications/plex/main.py | 75 +++++++++++-------- 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/couchpotato/core/notifications/plex/__init__.py b/couchpotato/core/notifications/plex/__init__.py index e2984ad9..70c0a3e5 100755 --- a/couchpotato/core/notifications/plex/__init__.py +++ b/couchpotato/core/notifications/plex/__init__.py @@ -17,14 +17,15 @@ config = [{ 'type': 'enabler', }, { - 'name': 'media_server', + 'name': 'host', + 'label': 'Media Server', 'default': 'localhost', - 'description': 'Media server hostname' + 'description': 'Hostname/IP, default localhost' }, { 'name': 'clients', 'default': '', - 'description': 'Comma separated list of client names\'s (computer names).' + 'description': 'Comma separated list of client names\'s (computer names). Top right when you start Plex' }, { 'name': 'on_snatch', diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index 09fd6ff1..5558289d 100755 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -1,14 +1,14 @@ -from datetime import datetime -import json from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification -from urllib2 import URLError +from datetime import datetime +from urlparse import urlparse from xml.dom import minidom -import traceback +import json import requests +import traceback try: import xml.etree.cElementTree as etree @@ -19,15 +19,19 @@ log = CPLog(__name__) class Plex(Notification): + client_update_time = 5 * 60 + http_time_between_calls = 0 def __init__(self): super(Plex, self).__init__() + self.clients = {} self.clients_updated = None + addEvent('renamer.after', self.addToLibrary) - def updateClients(self, force=False): + def updateClients(self, force = False): if not self.conf('media_server'): log.warning("Plex media server hostname is required") return @@ -38,13 +42,14 @@ class Plex(Notification): if force or self.clients_updated is None or since_update > self.client_update_time: self.clients = {} - client_result = etree.fromstring(self.urlopen('http://%s:32400/clients' % self.conf('media_server'))) + data = self.urlopen('%s/clients' % self.createHost(self.conf('media_server'), port = 32400)) + client_result = etree.fromstring(data) - hosts = [x.strip().lower() for x in self.conf('clients').split(',')] + clients = [x.strip().lower() for x in self.conf('clients').split(',')] for server in client_result.findall('Server'): - if server.get('name').lower() in hosts: - hosts.remove(server.get('name').lower()) + if server.get('name').lower() in clients: + clients.remove(server.get('name').lower()) protocol = server.get('protocol', 'xbmchttp') if protocol in ['xbmcjson', 'xbmchttp']: @@ -55,21 +60,21 @@ class Plex(Notification): 'protocol': protocol } - if len(hosts) > 0: - log.warning('unable to find some plex hosts: %s', ', '.join(hosts)) + if len(clients) > 0: + log.info2('Unable to find plex clients: %s', ', '.join(clients)) - log.info('found hosts: %s', ', '.join(self.clients.keys())) + log.info2('Found hosts: %s', ', '.join(self.clients.keys())) self.clients_updated = datetime.now() - def addToLibrary(self, message=None, group={}): + def addToLibrary(self, message = None, group = {}): if self.isDisabled(): return log.info('Sending notification to Plex') source_type = ['movie'] - base_url = 'http://%s:32400/library/sections' % self.conf('media_server') + base_url = '%s/library/sections' % self.createHost(self.conf('media_server'), port = 32400) refresh_url = '%s/%%s/refresh' % base_url try: @@ -89,7 +94,7 @@ class Plex(Notification): return True - def send_http(self, command, client): + def sendHTTP(self, command, client): url = 'http://%s:%s/xbmcCmds/xbmcHttp/?%s' % ( client['address'], client['port'], @@ -99,14 +104,14 @@ class Plex(Notification): headers = {} try: - self.urlopen(url, headers=headers, timeout=3, show_error=False) + self.urlopen(url, headers = headers, timeout = 3, show_error = False) except Exception, err: log.error("Couldn't sent command to Plex: %s", err) return False return True - def notify_http(self, message='', data={}, listener=None): + def notifyHTTP(self, message = '', data = {}, listener = None): total = 0 successful = 0 @@ -118,13 +123,13 @@ class Plex(Notification): for name, client in self.clients.items(): if client['protocol'] == 'xbmchttp': total += 1 - if self.send_http(data, client): + if self.sendHTTP(data, client): successful += 1 return successful == total - def send_json(self, method, params, client): - log.debug('send_json("%s", %s, %s)', (method, params, client)) + def sendJSON(self, method, params, client): + log.debug('sendJSON("%s", %s, %s)', (method, params, client)) url = 'http://%s:%s/jsonrpc' % ( client['address'], client['port'] @@ -142,14 +147,14 @@ class Plex(Notification): } try: - requests.post(url, headers=headers, timeout=3, data=json.dumps(request)) + requests.post(url, headers = headers, timeout = 3, data = json.dumps(request)) except Exception, err: log.error("Couldn't sent command to Plex: %s", err) return False return True - def notify_json(self, message='', data={}, listener=None): + def notifyJSON(self, message = '', data = {}, listener = None): total = 0 successful = 0 @@ -161,16 +166,16 @@ class Plex(Notification): for name, client in self.clients.items(): if client['protocol'] == 'xbmcjson': total += 1 - if self.send_json('GUI.ShowNotification', params, client): + if self.sendJSON('GUI.ShowNotification', params, client): successful += 1 return successful == total - def notify(self, message='', data={}, listener=None, force=False): + def notify(self, message = '', data = {}, listener = None, force = False): self.updateClients(force) - http_result = self.notify_http(message, data, listener) - json_result = self.notify_json(message, data, listener) + http_result = self.notifyHTTP(message, data, listener) + json_result = self.notifyJSON(message, data, listener) return http_result and json_result @@ -181,13 +186,23 @@ class Plex(Notification): log.info('Sending test to %s', test_type) success = self.notify( - message=self.test_message, - data={}, - listener='test', - force=True + message = self.test_message, + data = {}, + listener = 'test', + force = True ) success2 = self.addToLibrary() return { 'success': success or success2 } + + def createHost(self, host, port = None): + + h = cleanHost(host) + p = urlparse(h) + h = h.rstrip('/') + if port and not p.port: + h += ':%s' % port + + return h From 4a5c878c360ecb15fe85c4fbd746e763b914fef2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 24 Sep 2013 22:44:14 +0200 Subject: [PATCH 05/57] Wrong config name for plex host --- couchpotato/core/notifications/plex/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/plex/__init__.py b/couchpotato/core/notifications/plex/__init__.py index 70c0a3e5..d68ddb19 100755 --- a/couchpotato/core/notifications/plex/__init__.py +++ b/couchpotato/core/notifications/plex/__init__.py @@ -17,7 +17,7 @@ config = [{ 'type': 'enabler', }, { - 'name': 'host', + 'name': 'media_server', 'label': 'Media Server', 'default': 'localhost', 'description': 'Hostname/IP, default localhost' From 8474d0d95dd2f2ac58f150c844a118bf6ef58f37 Mon Sep 17 00:00:00 2001 From: Techmunk Date: Wed, 25 Sep 2013 21:44:05 +1000 Subject: [PATCH 06/57] Fix the way the client auth file is found and processed to match the defaults in the deluge clients. --- libs/synchronousdeluge/client.py | 48 ++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/libs/synchronousdeluge/client.py b/libs/synchronousdeluge/client.py index 98a80848..afd5971a 100644 --- a/libs/synchronousdeluge/client.py +++ b/libs/synchronousdeluge/client.py @@ -1,4 +1,5 @@ import os +import platform from collections import defaultdict from itertools import imap @@ -23,22 +24,47 @@ class DelugeClient(object): self._request_counter = 0 def _get_local_auth(self): - xdg_config = os.path.expanduser(os.environ.get("XDG_CONFIG_HOME", "~/.config")) - config_home = os.path.join(xdg_config, "deluge") - auth_file = os.path.join(config_home, "auth") - + auth_file = "" username = password = "" - with open(auth_file) as fd: - for line in fd: + if platform.system() in ('Windows', 'Microsoft'): + appDataPath = os.environ.get("APPDATA") + if not appDataPath: + import _winreg + hkey = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders") + appDataReg = _winreg.QueryValueEx(hkey, "AppData") + appDataPath = appDataReg[0] + _winreg.CloseKey(hkey) + auth_file = os.path.join(appDataPath, "deluge", "auth") + else: + from xdg.BaseDirectory import save_config_path + try: + auth_file = os.path.join(save_config_path("deluge"), "auth") + except OSError, e: + return username, password + + + if os.path.exists(auth_file): + for line in open(auth_file): if line.startswith("#"): + # This is a comment line + continue + line = line.strip() + try: + lsplit = line.split(":") + except Exception, e: continue - auth = line.split(":") - if len(auth) >= 2 and auth[0] == "localclient": - username, password = auth[0], auth[1] - break + if len(lsplit) == 2: + username, password = lsplit + elif len(lsplit) == 3: + username, password, level = lsplit + else: + continue - return username, password + if username == "localclient": + return (username, password) + + return ("", "") def _create_module_method(self, module, method): fullname = "{0}.{1}".format(module, method) From c7c64c60025e1331517e94d680f7867093e3bd3b Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 25 Sep 2013 14:05:16 +0200 Subject: [PATCH 07/57] Changed implementation of "scene_only" parameter to use filter criteria instead of parsing the information from query result. --- .../providers/torrent/torrentshack/main.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/couchpotato/core/providers/torrent/torrentshack/main.py b/couchpotato/core/providers/torrent/torrentshack/main.py index 353b606e..bc7077bb 100644 --- a/couchpotato/core/providers/torrent/torrentshack/main.py +++ b/couchpotato/core/providers/torrent/torrentshack/main.py @@ -15,7 +15,7 @@ class TorrentShack(TorrentProvider): 'login' : 'https://torrentshack.net/login.php', 'login_check': 'https://torrentshack.net/inbox.php', 'detail' : 'https://torrentshack.net/torrent/%s', - 'search' : 'https://torrentshack.net/torrents.php?searchstr=%s&filter_cat[%d]=1', + 'search' : 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', 'download' : 'https://torrentshack.net/%s', } @@ -31,7 +31,14 @@ class TorrentShack(TorrentProvider): def _searchOnTitle(self, title, movie, quality, results): - url = self.urls['search'] % (tryUrlencode('"%s" %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0]) + # scene only unset by default + scene_only = "" + if self.conf('scene_only'): + scene_only = "1" + + url = self.urls['search'] % (tryUrlencode('"%s" %s' % (title.replace(':', ''), movie['library']['year'])), + scene_only, + self.getCatId(quality['identifier'])[0]) data = self.getHTMLData(url, opener = self.login_opener) if data: @@ -49,22 +56,15 @@ class TorrentShack(TorrentProvider): link = result.find('span', attrs = {'class' : 'torrent_name_link'}).parent url = result.find('td', attrs = {'class' : 'torrent_td'}).find('a') - extra_info = '' - if result.find('span', attrs = {'class' : 'torrent_extra_info'}): - extra_info = result.find('span', attrs = {'class' : 'torrent_extra_info'}).text - - if not self.conf('scene_only') or extra_info != '[NotScene]': - results.append({ - 'id': link['href'].replace('torrents.php?torrentid=', ''), - 'name': unicode(link.span.string).translate({ord(u'\xad'): None}), - 'url': self.urls['download'] % url['href'], - 'detail_url': self.urls['download'] % link['href'], - 'size': self.parseSize(result.find_all('td')[4].string), - 'seeders': tryInt(result.find_all('td')[6].string), - 'leechers': tryInt(result.find_all('td')[7].string), - }) - else: - log.info('Not adding release %s [NotScene]' % unicode(link.span.string).translate({ord(u'\xad'): None})) + results.append({ + 'id': link['href'].replace('torrents.php?torrentid=', ''), + 'name': unicode(link.span.string).translate({ord(u'\xad'): None}), + 'url': self.urls['download'] % url['href'], + 'detail_url': self.urls['download'] % link['href'], + 'size': self.parseSize(result.find_all('td')[4].string), + 'seeders': tryInt(result.find_all('td')[6].string), + 'leechers': tryInt(result.find_all('td')[7].string), + }) except: log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) From 74a4e7d19df742e4d62781fcf39fab7fa924d7de Mon Sep 17 00:00:00 2001 From: Techmunk Date: Fri, 27 Sep 2013 14:59:03 +1000 Subject: [PATCH 08/57] Indenting on deluge auth fix was incorrect. --- libs/synchronousdeluge/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/synchronousdeluge/client.py b/libs/synchronousdeluge/client.py index afd5971a..22419e80 100644 --- a/libs/synchronousdeluge/client.py +++ b/libs/synchronousdeluge/client.py @@ -34,7 +34,8 @@ class DelugeClient(object): appDataReg = _winreg.QueryValueEx(hkey, "AppData") appDataPath = appDataReg[0] _winreg.CloseKey(hkey) - auth_file = os.path.join(appDataPath, "deluge", "auth") + + auth_file = os.path.join(appDataPath, "deluge", "auth") else: from xdg.BaseDirectory import save_config_path try: From f10d1824681be4eeda982c985d5148f9b87dd5ba Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 31 Aug 2013 17:20:22 +0200 Subject: [PATCH 09/57] Added Blu-ray.com backlog automation I missed a few movies, so I added backlog functionality to Blu-ray.com If you want to add all Blu-rays that ever came out to the wanted list, you can use this. Be careful with what you wish for :D --- .../providers/automation/bluray/__init__.py | 7 ++++ .../core/providers/automation/bluray/main.py | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/couchpotato/core/providers/automation/bluray/__init__.py b/couchpotato/core/providers/automation/bluray/__init__.py index e0675247..a47deee1 100644 --- a/couchpotato/core/providers/automation/bluray/__init__.py +++ b/couchpotato/core/providers/automation/bluray/__init__.py @@ -18,6 +18,13 @@ config = [{ 'default': False, 'type': 'enabler', }, + { + 'name': 'backlog', + 'advanced': True, + 'description': 'Parses the history tables until the minimum movie year is reached. Note: only do this once!', + 'default': False, + 'type': 'bool', + }, ], }, ], diff --git a/couchpotato/core/providers/automation/bluray/main.py b/couchpotato/core/providers/automation/bluray/main.py index 235a1e5f..50a47d87 100644 --- a/couchpotato/core/providers/automation/bluray/main.py +++ b/couchpotato/core/providers/automation/bluray/main.py @@ -1,3 +1,4 @@ +from bs4 import BeautifulSoup from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog @@ -10,11 +11,47 @@ class Bluray(Automation, RSS): interval = 1800 rss_url = 'http://www.blu-ray.com/rss/newreleasesfeed.xml' + backlog_url = 'http://www.blu-ray.com/movies/movies.php?show=newreleases&page=%s' def getIMDBids(self): movies = [] + if self.conf('backlog'): + + page = 0 + while True: + page = page + 1 + + url = self.backlog_url % page + data = self.getHTMLData(url) + soup = BeautifulSoup(data) + + try: + # Stop if the release year is before the minimal year + page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive=False)[3].h3.get_text().split(', ')[1] + if tryInt(page_year) < self.getMinimal('year'): + break + + for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive=False)[3].find_all('table')[1:20]: + name = table.h3.get_text().lower().split('blu-ray')[0].strip() + year = table.small.get_text().split('|')[1].strip() + + if not name.find('/') == -1: # make sure it is not a double movie release + continue + + if tryInt(year) < self.getMinimal('year'): + continue + + imdb = self.search(name, year) + + if imdb: + if self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + except: + log.debug('Error loading page: %s', page) + break + rss_movies = self.getRSSData(self.rss_url) for movie in rss_movies: From 00bb055474b0fb28336b29932e700fb5d0a299ea Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 31 Aug 2013 19:08:23 +0200 Subject: [PATCH 10/57] set backlog to False after backlog search --- couchpotato/core/providers/automation/bluray/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/providers/automation/bluray/main.py b/couchpotato/core/providers/automation/bluray/main.py index 50a47d87..335d7768 100644 --- a/couchpotato/core/providers/automation/bluray/main.py +++ b/couchpotato/core/providers/automation/bluray/main.py @@ -52,6 +52,8 @@ class Bluray(Automation, RSS): log.debug('Error loading page: %s', page) break + self.conf('backlog', value = False) + rss_movies = self.getRSSData(self.rss_url) for movie in rss_movies: From 871aecb689715a918e6911adeac64c616d538ae6 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 28 Sep 2013 13:35:26 +0200 Subject: [PATCH 11/57] Fix transmission #2168 --- couchpotato/core/downloaders/transmission/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 5ff33c05..1c359967 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -136,11 +136,11 @@ class Transmission(Downloader): def removeFailed(self, item): log.info('%s failed downloading, deleting...', item['name']) - return self.trpc.remove_torrent(item['hashString'], True) + return self.trpc.remove_torrent(item['id'], True) def processComplete(self, item, delete_files = False): log.debug('Requesting Transmission to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) - return self.trpc.remove_torrent(item['hashString'], delete_files) + return self.trpc.remove_torrent(item['id'], delete_files) class TransmissionRPC(object): From 3310bdf5517e7af276c2b95adb2e89e26c563468 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 14:30:20 +0200 Subject: [PATCH 12/57] Don't use quotes for torrentshack --- couchpotato/core/providers/torrent/torrentshack/main.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/providers/torrent/torrentshack/main.py b/couchpotato/core/providers/torrent/torrentshack/main.py index bc7077bb..6b3b5548 100644 --- a/couchpotato/core/providers/torrent/torrentshack/main.py +++ b/couchpotato/core/providers/torrent/torrentshack/main.py @@ -31,14 +31,9 @@ class TorrentShack(TorrentProvider): def _searchOnTitle(self, title, movie, quality, results): - # scene only unset by default - scene_only = "" - if self.conf('scene_only'): - scene_only = "1" + scene_only = '1' if self.conf('scene_only') else '' - url = self.urls['search'] % (tryUrlencode('"%s" %s' % (title.replace(':', ''), movie['library']['year'])), - scene_only, - self.getCatId(quality['identifier'])[0]) + url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), scene_only, self.getCatId(quality['identifier'])[0]) data = self.getHTMLData(url, opener = self.login_opener) if data: From 92a0af5ce3f8417c0a4ab78b26accbafa67bcef5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 15:23:45 +0200 Subject: [PATCH 13/57] Use label for quality guess also. closes #2237 --- couchpotato/core/plugins/quality/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 1149c036..3319d2f5 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -194,13 +194,16 @@ class QualityPlugin(Plugin): def containsTag(self, quality, words, cur_file = ''): # Check alt and tags - for tag_type in ['alternative', 'tags']: - for alt in quality.get(tag_type, []): - if isinstance(alt, tuple) and '.'.join(alt) in '.'.join(words): + for tag_type in ['alternative', 'tags', 'label']: + qualities = quality.get(tag_type, []) + qualities = [qualities] if isinstance(qualities, (str, unicode)) else qualities + + for alt in qualities: + if (isinstance(alt, tuple) and '.'.join(alt) in '.'.join(words)) or (isinstance(alt, (str, unicode)) and alt.lower() in cur_file.lower()): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) return True - if list(set(quality.get(tag_type, [])) & set(words)): + if list(set(qualities) & set(words)): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) return True From ededfcb82208d69fe6ce5eb6cbac1de79749460c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 16:28:46 +0200 Subject: [PATCH 14/57] Escape spaces for each request. fix #2256 --- couchpotato/core/plugins/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index ce7c1b49..c90c48c6 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -121,7 +121,7 @@ class Plugin(object): # http request def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True): - url = ss(url) + url = urllib2.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]") if not headers: headers = {} if not params: params = {} From 8011634b7a88f3f47eeec50cdc81f514149a203c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 16:39:31 +0200 Subject: [PATCH 15/57] Use correct encoding for emails. fix #2254 --- couchpotato/core/notifications/email/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/email/main.py b/couchpotato/core/notifications/email/main.py index f94688d5..508e0823 100644 --- a/couchpotato/core/notifications/email/main.py +++ b/couchpotato/core/notifications/email/main.py @@ -2,6 +2,7 @@ from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification +from couchpotato.environment import Env from email.mime.text import MIMEText import smtplib import traceback @@ -23,7 +24,7 @@ class Email(Notification): smtp_pass = self.conf('smtp_pass') # Make the basic message - message = MIMEText(toUnicode(message)) + message = MIMEText(toUnicode(message), _charset = Env.get('encoding')) message['Subject'] = self.default_title message['From'] = from_address message['To'] = to_address From 49015b7d64b70f73f2a61c1bb7ffca50bc9a5296 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 17:45:32 +0200 Subject: [PATCH 16/57] Be sure to ss quality alt in guess --- couchpotato/core/plugins/quality/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 3319d2f5..e12a712a 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -1,7 +1,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, ss from couchpotato.core.helpers.variable import mergeDicts, md5, getExt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -192,6 +192,7 @@ class QualityPlugin(Plugin): return None def containsTag(self, quality, words, cur_file = ''): + cur_file = ss(cur_file) # Check alt and tags for tag_type in ['alternative', 'tags', 'label']: @@ -199,7 +200,7 @@ class QualityPlugin(Plugin): qualities = [qualities] if isinstance(qualities, (str, unicode)) else qualities for alt in qualities: - if (isinstance(alt, tuple) and '.'.join(alt) in '.'.join(words)) or (isinstance(alt, (str, unicode)) and alt.lower() in cur_file.lower()): + if (isinstance(alt, tuple) and '.'.join(alt) in '.'.join(words)) or (isinstance(alt, (str, unicode)) and ss(alt.lower()) in cur_file.lower()): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) return True From 475ac1bb9c20095315bdf6c913470db83344894a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 18:06:45 +0200 Subject: [PATCH 17/57] Only use filename for identification when possible. fix #2233 & #954 --- couchpotato/core/plugins/scanner/main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 0662d008..9990bda7 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -1,7 +1,8 @@ from couchpotato import get_session from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss -from couchpotato.core.helpers.variable import getExt, getImdb, tryInt +from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \ + splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import File, Movie @@ -741,9 +742,14 @@ class Scanner(Plugin): def createStringIdentifier(self, file_path, folder = '', exclude_filename = False): - identifier = file_path.replace(folder, '') # root folder + year = self.findYear(file_path) + + identifier = file_path.replace(folder, '').lstrip(os.path.sep) # root folder identifier = os.path.splitext(identifier)[0] # ext + path_split = splitString(identifier, os.path.sep) + identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename + if exclude_filename: identifier = identifier[:len(identifier) - len(os.path.split(identifier)[-1])] @@ -757,7 +763,6 @@ class Scanner(Plugin): identifier = re.sub(self.clean, '::', simplifyString(identifier)).strip(':') # Year - year = self.findYear(identifier) if year and identifier[:4] != year: identifier = '%s %s' % (identifier.split(year)[0].strip(), year) else: From 2f4f140662a1ddd22b274d6ec97733d2c397c8f9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 18:19:17 +0200 Subject: [PATCH 18/57] Don't overwrite data variable in utorrent download. fix #2222 --- couchpotato/core/downloaders/utorrent/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index ce82c8c2..d5262e23 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -107,9 +107,9 @@ class uTorrent(Downloader): count += 1 # Check if torrent is saved in subfolder of torrent name - data = self.utorrent_api.get_files(torrent_hash) + getfiles_data = self.utorrent_api.get_files(torrent_hash) - torrent_files = json.loads(data) + torrent_files = json.loads(getfiles_data) if torrent_files.get('error'): log.error('Error getting data from uTorrent: %s', torrent_files.get('error')) return False @@ -200,7 +200,7 @@ class uTorrent(Downloader): if not self.connect(): return False return self.utorrent_api.remove_torrent(item['id'], remove_data = delete_files) - + def removeReadOnly(self, folder): #Removes all read-only flags in a folder if folder and os.path.isdir(folder): From 116bc839fc75ca1523db2dad558b804e1b7a9c62 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 19:12:05 +0200 Subject: [PATCH 19/57] Make description more clear --- couchpotato/core/providers/automation/bluray/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/bluray/__init__.py b/couchpotato/core/providers/automation/bluray/__init__.py index a47deee1..ed270056 100644 --- a/couchpotato/core/providers/automation/bluray/__init__.py +++ b/couchpotato/core/providers/automation/bluray/__init__.py @@ -21,7 +21,7 @@ config = [{ { 'name': 'backlog', 'advanced': True, - 'description': 'Parses the history tables until the minimum movie year is reached. Note: only do this once!', + 'description': 'Parses the history until the minimum movie year is reached. (Will be disabled once it has completed)', 'default': False, 'type': 'bool', }, From 7d4f9d60b1374f97c520fa7ef4eea676f9a0775c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 19:17:41 +0200 Subject: [PATCH 20/57] Code formating --- couchpotato/core/providers/automation/bluray/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/automation/bluray/main.py b/couchpotato/core/providers/automation/bluray/main.py index 335d7768..d98557ec 100644 --- a/couchpotato/core/providers/automation/bluray/main.py +++ b/couchpotato/core/providers/automation/bluray/main.py @@ -29,11 +29,11 @@ class Bluray(Automation, RSS): try: # Stop if the release year is before the minimal year - page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive=False)[3].h3.get_text().split(', ')[1] + page_year = soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].h3.get_text().split(', ')[1] if tryInt(page_year) < self.getMinimal('year'): break - for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive=False)[3].find_all('table')[1:20]: + for table in soup.body.find_all('center')[3].table.tr.find_all('td', recursive = False)[3].find_all('table')[1:20]: name = table.h3.get_text().lower().split('blu-ray')[0].strip() year = table.small.get_text().split('|')[1].strip() From 364e355114bdc429fde88c7d5c812df7fdab96ad Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 21:25:25 +0200 Subject: [PATCH 21/57] Also try to load the root module for each path --- couchpotato/core/downloaders/__init__.py | 4 ++-- couchpotato/core/loader.py | 14 +++----------- couchpotato/core/notifications/__init__.py | 4 ++-- couchpotato/core/providers/automation/__init__.py | 4 ++-- couchpotato/core/providers/nzb/__init__.py | 4 ++-- couchpotato/core/providers/torrent/__init__.py | 4 ++-- 6 files changed, 13 insertions(+), 21 deletions(-) diff --git a/couchpotato/core/downloaders/__init__.py b/couchpotato/core/downloaders/__init__.py index 5fb7125f..a81ce881 100644 --- a/couchpotato/core/downloaders/__init__.py +++ b/couchpotato/core/downloaders/__init__.py @@ -1,4 +1,4 @@ -config = { +config = [{ 'name': 'download_providers', 'groups': [ { @@ -10,4 +10,4 @@ config = { 'options': [], }, ], -} +}] diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index 2016d287..9362bb8c 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -1,6 +1,5 @@ from couchpotato.core.event import fireEvent from couchpotato.core.logger import CPLog -import glob import os import traceback @@ -81,17 +80,10 @@ class Loader(object): def addFromDir(self, plugin_type, priority, module, dir_name): # Load dir module - try: - m = __import__(module) - splitted = module.split('.') - for sub in splitted[1:]: - m = getattr(m, sub) - except: - raise + self.addModule(priority, plugin_type, module, os.path.basename(dir_name)) - for cur_file in glob.glob(os.path.join(dir_name, '*')): - name = os.path.basename(cur_file) - if os.path.isdir(os.path.join(dir_name, name)) and name != 'static' and os.path.isfile(os.path.join(cur_file, '__init__.py')): + for name in os.listdir(dir_name): + if os.path.isdir(os.path.join(dir_name, name)) and name != 'static' and os.path.isfile(os.path.join(dir_name, name, '__init__.py')): module_name = '%s.%s' % (module, name) self.addModule(priority, plugin_type, module_name, name) diff --git a/couchpotato/core/notifications/__init__.py b/couchpotato/core/notifications/__init__.py index 8ac24dfb..5958fe66 100644 --- a/couchpotato/core/notifications/__init__.py +++ b/couchpotato/core/notifications/__init__.py @@ -1,4 +1,4 @@ -config = { +config = [{ 'name': 'notification_providers', 'groups': [ { @@ -10,4 +10,4 @@ config = { 'options': [], }, ], -} +}] diff --git a/couchpotato/core/providers/automation/__init__.py b/couchpotato/core/providers/automation/__init__.py index a217948a..93f6c10a 100644 --- a/couchpotato/core/providers/automation/__init__.py +++ b/couchpotato/core/providers/automation/__init__.py @@ -1,4 +1,4 @@ -config = { +config = [{ 'name': 'automation_providers', 'groups': [ { @@ -18,4 +18,4 @@ config = { 'options': [], }, ], -} +}] diff --git a/couchpotato/core/providers/nzb/__init__.py b/couchpotato/core/providers/nzb/__init__.py index 36098bb3..88d9865d 100644 --- a/couchpotato/core/providers/nzb/__init__.py +++ b/couchpotato/core/providers/nzb/__init__.py @@ -1,4 +1,4 @@ -config = { +config = [{ 'name': 'nzb_providers', 'groups': [ { @@ -11,4 +11,4 @@ config = { 'options': [], }, ], -} +}] diff --git a/couchpotato/core/providers/torrent/__init__.py b/couchpotato/core/providers/torrent/__init__.py index 250bcead..12dda708 100644 --- a/couchpotato/core/providers/torrent/__init__.py +++ b/couchpotato/core/providers/torrent/__init__.py @@ -1,4 +1,4 @@ -config = { +config = [{ 'name': 'torrent_providers', 'groups': [ { @@ -11,4 +11,4 @@ config = { 'options': [], }, ], -} +}] From b4bccc9be218469c8fb787e574eed3c5c577f8d5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Sep 2013 23:41:15 +0200 Subject: [PATCH 22/57] Flixter automation support Thanks @mikedm139 --- .../providers/automation/flixster/__init__.py | 34 +++++++++++++ .../providers/automation/flixster/main.py | 48 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 couchpotato/core/providers/automation/flixster/__init__.py create mode 100644 couchpotato/core/providers/automation/flixster/main.py diff --git a/couchpotato/core/providers/automation/flixster/__init__.py b/couchpotato/core/providers/automation/flixster/__init__.py new file mode 100644 index 00000000..1c6c4590 --- /dev/null +++ b/couchpotato/core/providers/automation/flixster/__init__.py @@ -0,0 +1,34 @@ +from .main import Flixster + +def start(): + return Flixster() + +config = [{ + 'name': 'flixster', + 'groups': [ + { + 'tab': 'automation', + 'list': 'watchlist_providers', + 'name': 'flixster_automation', + 'label': 'Flixster', + 'description': 'Import movies from any public Flixster watchlist', + 'options': [ + { + 'name': 'automation_enabled', + 'default': False, + 'type': 'enabler', + }, + { + 'name': 'automation_ids_use', + 'label': 'Use', + }, + { + 'name': 'automation_ids', + 'label': 'User ID', + 'type': 'combined', + 'combine': ['automation_ids_use', 'automation_ids'], + }, + ], + }, + ], +}] diff --git a/couchpotato/core/providers/automation/flixster/main.py b/couchpotato/core/providers/automation/flixster/main.py new file mode 100644 index 00000000..46dcfba3 --- /dev/null +++ b/couchpotato/core/providers/automation/flixster/main.py @@ -0,0 +1,48 @@ +from couchpotato.core.helpers.variable import tryInt, splitString +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.automation.base import Automation +import json + +log = CPLog(__name__) + + +class Flixster(Automation): + + url = 'http://www.flixster.com/api/users/%s/movies/ratings?scoreTypes=wts' + + interval = 60 + + def getIMDBids(self): + + ids = splitString(self.conf('automation_ids')) + + if len(ids) == 0: + return [] + + movies = [] + + for movie in self.getWatchlist(): + imdb_id = self.search(movie.get('title'), movie.get('year'), imdb_only = True) + movies.append(imdb_id) + + return movies + + def getWatchlist(self): + + enablers = [tryInt(x) for x in splitString(self.conf('automation_ids_use'))] + ids = splitString(self.conf('automation_ids')) + + index = -1 + movies = [] + for user_id in ids: + + index += 1 + if not enablers[index]: + continue + + data = json.loads(self.getHTMLData(self.url % user_id)) + + for movie in data: + movies.append({'title': movie['movie']['title'], 'year': movie['movie']['year'] }) + + return movies From 96b4af1fea17f55b8fe63b58160c72a857ded386 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 00:08:26 +0200 Subject: [PATCH 23/57] Hide first item in combined table --- couchpotato/static/style/settings.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/static/style/settings.css b/couchpotato/static/style/settings.css index 61d5239f..744531a9 100644 --- a/couchpotato/static/style/settings.css +++ b/couchpotato/static/style/settings.css @@ -542,7 +542,7 @@ line-height: 140%; cursor: help; } - .page .combined_table .head abbr.use, .page .combined_table .head abbr.automation_urls_use { + .page .combined_table .head abbr:first-child { display: none; } .page .combined_table .head abbr.host { From 91332e06e5f4a3261ca2338539b806e0e8fbd8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sun, 29 Sep 2013 01:45:24 +0200 Subject: [PATCH 24/57] add option to create sub directory --- couchpotato/core/downloaders/blackhole/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/couchpotato/core/downloaders/blackhole/__init__.py b/couchpotato/core/downloaders/blackhole/__init__.py index 290e8d43..5de8ac76 100644 --- a/couchpotato/core/downloaders/blackhole/__init__.py +++ b/couchpotato/core/downloaders/blackhole/__init__.py @@ -35,6 +35,13 @@ config = [{ 'type': 'dropdown', 'values': [('usenet & torrents', 'both'), ('usenet', 'nzb'), ('torrent', 'torrent')], }, + { + 'name': 'create_subdir', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Create a sub directory when adding torrent to blackhole.', + }, { 'name': 'manual', 'default': 0, From e38d68c01934316d37311f4d000efaebe4a843d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sun, 29 Sep 2013 01:45:50 +0200 Subject: [PATCH 25/57] actual code --- couchpotato/core/downloaders/blackhole/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index 9a5a6217..e3e070cc 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -35,6 +35,15 @@ class Blackhole(Downloader): fullPath = os.path.join(directory, self.createFileName(data, filedata, movie)) + if self.conf('create_subdir'): + try: + new_path = os.path.splitext(fullPath)[0] + if not os.path.exists(new_path): + os.makedirs(new_path) + fullPath = os.path.join(new_path, self.createFileName(data, filedata, movie)) + except: + log.error('Couldnt create sub dir, reverting to old one: %s', fullPath) + try: if not os.path.isfile(fullPath): log.info('Downloading %s to %s.', (data.get('protocol'), fullPath)) From 201185f7e7e6833436be4db51b101944bb32d949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sun, 29 Sep 2013 01:49:51 +0200 Subject: [PATCH 26/57] better english damnit! --- couchpotato/core/downloaders/blackhole/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/blackhole/__init__.py b/couchpotato/core/downloaders/blackhole/__init__.py index 5de8ac76..6b5279a1 100644 --- a/couchpotato/core/downloaders/blackhole/__init__.py +++ b/couchpotato/core/downloaders/blackhole/__init__.py @@ -40,7 +40,7 @@ config = [{ 'default': 0, 'type': 'bool', 'advanced': True, - 'description': 'Create a sub directory when adding torrent to blackhole.', + 'description': 'Create a sub directory when saving the .nzb (or .torrent).', }, { 'name': 'manual', From 1f2c2269e69f754332454352a3e516c2bd9ebe0a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 09:54:37 +0200 Subject: [PATCH 27/57] Ignore thumbs.db files and don't fail on single path split. fix #2265 --- couchpotato/core/plugins/scanner/main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 9990bda7..58627093 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -25,7 +25,9 @@ class Scanner(Plugin): 'media': 314572800, # 300MB 'trailer': 1048576, # 1MB } - ignored_in_path = [os.path.sep + 'extracted' + os.path.sep, 'extracting', '_unpack', '_failed_', '_unknown_', '_exists_', '_failed_remove_', '_failed_rename_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo'] #unpacking, smb-crap, hidden files + ignored_in_path = [os.path.sep + 'extracted' + os.path.sep, 'extracting', '_unpack', '_failed_', '_unknown_', '_exists_', '_failed_remove_', + '_failed_rename_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo', + 'thumbs.db', 'ehthumbs.db', 'desktop.ini'] #unpacking, smb-crap, hidden files ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films', 'download', 'downloads', 'video_ts', 'audio_ts', 'bdmv', 'certificate'] extensions = { 'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img', 'mdf', 'ts', 'm4v'], @@ -747,8 +749,10 @@ class Scanner(Plugin): identifier = file_path.replace(folder, '').lstrip(os.path.sep) # root folder identifier = os.path.splitext(identifier)[0] # ext - path_split = splitString(identifier, os.path.sep) - identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename + try: + path_split = splitString(identifier, os.path.sep) + identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename + except: pass if exclude_filename: identifier = identifier[:len(identifier) - len(os.path.split(identifier)[-1])] From 99c899ea3a22dfa7cedbbbb3bb9629d2d4956c51 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 10:06:12 +0200 Subject: [PATCH 28/57] Proper variable naming --- .../core/downloaders/blackhole/main.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index e3e070cc..854860cd 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -33,26 +33,27 @@ class Blackhole(Downloader): log.error('No nzb/torrent available: %s', data.get('url')) return False - fullPath = os.path.join(directory, self.createFileName(data, filedata, movie)) + file_name = self.createFileName(data, filedata, movie) + full_path = os.path.join(directory, file_name) if self.conf('create_subdir'): try: - new_path = os.path.splitext(fullPath)[0] + new_path = os.path.splitext(full_path)[0] if not os.path.exists(new_path): os.makedirs(new_path) - fullPath = os.path.join(new_path, self.createFileName(data, filedata, movie)) + full_path = os.path.join(new_path, file_name) except: - log.error('Couldnt create sub dir, reverting to old one: %s', fullPath) + log.error('Couldnt create sub dir, reverting to old one: %s', full_path) try: - if not os.path.isfile(fullPath): - log.info('Downloading %s to %s.', (data.get('protocol'), fullPath)) - with open(fullPath, 'wb') as f: + if not os.path.isfile(full_path): + log.info('Downloading %s to %s.', (data.get('protocol'), full_path)) + with open(full_path, 'wb') as f: f.write(filedata) - os.chmod(fullPath, Env.getPermission('file')) + os.chmod(full_path, Env.getPermission('file')) return True else: - log.info('File %s already exists.', fullPath) + log.info('File %s already exists.', full_path) return True except: From ae4e15286a12ea2b4da469bd14c19abcefec5296 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 12:17:09 +0200 Subject: [PATCH 29/57] Don't try to loop over None. fix #2268 --- couchpotato/core/plugins/manage/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 702b1293..e8ccaf7e 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -222,9 +222,10 @@ class Manage(Plugin): groups = fireEvent('scanner.scan', folder = folder, files = files, single = True) - for group in groups.itervalues(): - if group['library'] and group['library'].get('identifier'): - fireEvent('release.add', group = group) + if groups: + for group in groups.itervalues(): + if group['library'] and group['library'].get('identifier'): + fireEvent('release.add', group = group) def getDiskSpace(self): From 48db4c8b8efe585eff2de249cb1370e28a6a8cc2 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 29 Sep 2013 23:21:53 +1300 Subject: [PATCH 30/57] Updated rtorrent-python library --- libs/rtorrent/__init__.py | 6 ++---- libs/rtorrent/lib/torrentparser.py | 7 ++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/libs/rtorrent/__init__.py b/libs/rtorrent/__init__.py index d19c78b4..b6ff73a0 100755 --- a/libs/rtorrent/__init__.py +++ b/libs/rtorrent/__init__.py @@ -71,12 +71,10 @@ class RTorrent: def _verify_conn(self): # check for rpc methods that should be available - assert {"system.client_version", - "system.library_version"}.issubset(set(self._get_rpc_methods())),\ - "Required RPC methods not available." + assert "system.client_version" in self._get_rpc_methods(), "Required RPC method not available." + assert "system.library_version" in self._get_rpc_methods(), "Required RPC method not available." # minimum rTorrent version check - assert self._meets_version_requirement() is True,\ "Error: Minimum rTorrent version required is {0}".format( MIN_RTORRENT_VERSION_STR) diff --git a/libs/rtorrent/lib/torrentparser.py b/libs/rtorrent/lib/torrentparser.py index 19dd12aa..30170d32 100755 --- a/libs/rtorrent/lib/torrentparser.py +++ b/libs/rtorrent/lib/torrentparser.py @@ -90,9 +90,10 @@ class TorrentParser(): def _calc_info_hash(self): self.info_hash = None if "info" in self._torrent_decoded.keys(): - info_dict = self._torrent_decoded["info"] - self.info_hash = hashlib.sha1(bencode.encode( - info_dict)).hexdigest().upper() + info_encoded = bencode.encode(self._torrent_decoded["info"]) + + if info_encoded: + self.info_hash = hashlib.sha1(info_encoded).hexdigest().upper() return(self.info_hash) From 226835e3d0a8db3e5cd22183897cf41a3a092745 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 29 Sep 2013 23:32:03 +1300 Subject: [PATCH 31/57] Added a check to ensure a torrent has been loaded (and found). --- couchpotato/core/downloaders/rtorrent/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 161c671a..d655f5a6 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -116,6 +116,10 @@ class rTorrent(Downloader): # Send torrent to rTorrent torrent = self.rt.load_torrent(filedata) + if not torrent: + log.error('Unable to find the torrent, did it fail to load?') + return False + # Set label if self.conf('label'): torrent.set_custom(1, self.conf('label')) From 333abd248663969cfdaab7486281e127f4712bcd Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 13:25:10 +0200 Subject: [PATCH 32/57] Custom plugin folder outside source. fix #2076 --- couchpotato/core/loader.py | 46 ++++++++++++--------- couchpotato/core/plugins/custom/__init__.py | 6 +++ couchpotato/core/plugins/custom/main.py | 21 ++++++++++ libs/importlib/__init__.py | 38 +++++++++++++++++ 4 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 couchpotato/core/plugins/custom/__init__.py create mode 100644 couchpotato/core/plugins/custom/main.py create mode 100644 libs/importlib/__init__.py diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index 9362bb8c..6ceee4ed 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -1,6 +1,8 @@ from couchpotato.core.event import fireEvent from couchpotato.core.logger import CPLog +from importlib import import_module import os +import sys import traceback log = CPLog(__name__) @@ -11,17 +13,6 @@ class Loader(object): providers = {} modules = {} - def addPath(self, root, base_path, priority, recursive = False): - for filename in os.listdir(os.path.join(root, *base_path)): - path = os.path.join(os.path.join(root, *base_path), filename) - if os.path.isdir(path) and filename[:2] != '__': - if u'__init__.py' in os.listdir(path): - new_base_path = ''.join(s + '.' for s in base_path) + filename - self.paths[new_base_path.replace('.', '_')] = (priority, new_base_path, path) - - if recursive: - self.addPath(root, base_path + [filename], priority, recursive = True) - def preload(self, root = ''): core = os.path.join(root, 'couchpotato', 'core') @@ -38,6 +29,13 @@ class Loader(object): # Add media to loader self.addPath(root, ['couchpotato', 'core', 'media'], 25, recursive = True) + # Add custom plugin folder + from couchpotato.environment import Env + custom_plugin_dir = os.path.join(Env.get('data_dir'), 'custom_plugins') + sys.path.insert(0, custom_plugin_dir) + self.paths['custom_plugins'] = (30, '', custom_plugin_dir) + + # Loop over all paths and add to module list for plugin_type, plugin_tuple in self.paths.iteritems(): priority, module, dir_name = plugin_tuple self.addFromDir(plugin_type, priority, module, dir_name) @@ -45,8 +43,9 @@ class Loader(object): def run(self): did_save = 0 - for priority in self.modules: + for priority in sorted(self.modules): for module_name, plugin in sorted(self.modules[priority].iteritems()): + # Load module try: if plugin.get('name')[:2] == '__': @@ -55,7 +54,6 @@ class Loader(object): m = self.loadModule(module_name) if m is None: continue - m = getattr(m, plugin.get('name')) log.info('Loading %s: %s', (plugin['type'], plugin['name'])) @@ -77,10 +75,23 @@ class Loader(object): if did_save: fireEvent('settings.save') + def addPath(self, root, base_path, priority, recursive = False): + root_path = os.path.join(root, *base_path) + for filename in os.listdir(root_path): + path = os.path.join(root_path, filename) + if os.path.isdir(path) and filename[:2] != '__': + if u'__init__.py' in os.listdir(path): + new_base_path = ''.join(s + '.' for s in base_path) + filename + self.paths[new_base_path.replace('.', '_')] = (priority, new_base_path, path) + + if recursive: + self.addPath(root, base_path + [filename], priority, recursive = True) + def addFromDir(self, plugin_type, priority, module, dir_name): # Load dir module - self.addModule(priority, plugin_type, module, os.path.basename(dir_name)) + if module and len(module) > 0: + self.addModule(priority, plugin_type, module, os.path.basename(dir_name)) for name in os.listdir(dir_name): if os.path.isdir(os.path.join(dir_name, name)) and name != 'static' and os.path.isfile(os.path.join(dir_name, name, '__init__.py')): @@ -123,6 +134,7 @@ class Loader(object): if not self.modules.get(priority): self.modules[priority] = {} + module = module.lstrip('.') self.modules[priority][module] = { 'priority': priority, 'module': module, @@ -132,11 +144,7 @@ class Loader(object): def loadModule(self, name): try: - m = __import__(name) - splitted = name.split('.') - for sub in splitted[1:-1]: - m = getattr(m, sub) - return m + return import_module(name) except ImportError: log.debug('Skip loading module plugin %s: %s', (name, traceback.format_exc())) return None diff --git a/couchpotato/core/plugins/custom/__init__.py b/couchpotato/core/plugins/custom/__init__.py new file mode 100644 index 00000000..573cd99f --- /dev/null +++ b/couchpotato/core/plugins/custom/__init__.py @@ -0,0 +1,6 @@ +from .main import Custom + +def start(): + return Custom() + +config = [] diff --git a/couchpotato/core/plugins/custom/main.py b/couchpotato/core/plugins/custom/main.py new file mode 100644 index 00000000..a15c915c --- /dev/null +++ b/couchpotato/core/plugins/custom/main.py @@ -0,0 +1,21 @@ +from couchpotato.core.event import addEvent +from couchpotato.core.logger import CPLog +from couchpotato.core.plugins.base import Plugin +from couchpotato.environment import Env +import os + +log = CPLog(__name__) + + +class Custom(Plugin): + + def __init__(self): + addEvent('app.load', self.createStructure) + + def createStructure(self): + + custom_dir = os.path.join(Env.get('data_dir'), 'custom_plugins') + + if not os.path.isdir(custom_dir): + self.makeDir(custom_dir) + self.createFile(os.path.join(custom_dir, '__init__.py'), '# Don\'t remove this file') diff --git a/libs/importlib/__init__.py b/libs/importlib/__init__.py new file mode 100644 index 00000000..ad31a1ac --- /dev/null +++ b/libs/importlib/__init__.py @@ -0,0 +1,38 @@ +"""Backport of importlib.import_module from 3.x.""" +# While not critical (and in no way guaranteed!), it would be nice to keep this +# code compatible with Python 2.3. +import sys + +def _resolve_name(name, package, level): + """Return the absolute name of the module to be imported.""" + if not hasattr(package, 'rindex'): + raise ValueError("'package' not set to a string") + dot = len(package) + for x in xrange(level, 1, -1): + try: + dot = package.rindex('.', 0, dot) + except ValueError: + raise ValueError("attempted relative import beyond top-level " + "package") + return "%s.%s" % (package[:dot], name) + + +def import_module(name, package=None): + """Import a module. + + The 'package' argument is required when performing a relative import. It + specifies the package to use as the anchor point from which to resolve the + relative import to an absolute import. + + """ + if name.startswith('.'): + if not package: + raise TypeError("relative imports require the 'package' argument") + level = 0 + for character in name: + if character != '.': + break + level += 1 + name = _resolve_name(name[level:], package, level) + __import__(name) + return sys.modules[name] From e7aa91b3e1f5cc28c5d678b7a7dfe6b038d44ea8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 13:44:52 +0200 Subject: [PATCH 33/57] Don't try to use custom_plugins when folder doesn't exist --- couchpotato/core/loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index 6ceee4ed..c14b55bd 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -32,8 +32,9 @@ class Loader(object): # Add custom plugin folder from couchpotato.environment import Env custom_plugin_dir = os.path.join(Env.get('data_dir'), 'custom_plugins') - sys.path.insert(0, custom_plugin_dir) - self.paths['custom_plugins'] = (30, '', custom_plugin_dir) + if os.path.isdir(custom_plugin_dir): + sys.path.insert(0, custom_plugin_dir) + self.paths['custom_plugins'] = (30, '', custom_plugin_dir) # Loop over all paths and add to module list for plugin_type, plugin_tuple in self.paths.iteritems(): From 0b00f2d9e600109713038d9975e3322ef9d7ead3 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 30 Sep 2013 00:44:36 +1300 Subject: [PATCH 34/57] Fixed Plex notifications on latest PHT (protocol renamed to 'plex') --- couchpotato/core/notifications/plex/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index 5558289d..19ca670d 100755 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -52,7 +52,7 @@ class Plex(Notification): clients.remove(server.get('name').lower()) protocol = server.get('protocol', 'xbmchttp') - if protocol in ['xbmcjson', 'xbmchttp']: + if protocol in ['plex', 'xbmcjson', 'xbmchttp']: self.clients[server.get('name')] = { 'name': server.get('name'), 'address': server.get('address'), @@ -164,7 +164,7 @@ class Plex(Notification): } for name, client in self.clients.items(): - if client['protocol'] == 'xbmcjson': + if client['protocol'] in ['xbmcjson', 'plex']: total += 1 if self.sendJSON('GUI.ShowNotification', params, client): successful += 1 From cc4350b0f98736ebcf946f87ed1339948513ddac Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 29 Sep 2013 14:05:28 +0200 Subject: [PATCH 35/57] NZBGet missing in wizard. fix #2262 --- couchpotato/core/downloaders/nzbget/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/downloaders/nzbget/__init__.py b/couchpotato/core/downloaders/nzbget/__init__.py index 19483713..00763cfb 100644 --- a/couchpotato/core/downloaders/nzbget/__init__.py +++ b/couchpotato/core/downloaders/nzbget/__init__.py @@ -12,6 +12,7 @@ config = [{ 'name': 'nzbget', 'label': 'NZBGet', 'description': 'Use NZBGet to download NZBs.', + 'wizard': True, 'options': [ { 'name': 'enabled', From b128ef17c9a78c12756f05885649d5c9a076bf4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sun, 29 Sep 2013 15:32:23 +0200 Subject: [PATCH 36/57] Added directory option and an option to append label to directory path --- couchpotato/core/downloaders/rtorrent/__init__.py | 13 +++++++++++++ couchpotato/core/downloaders/rtorrent/main.py | 8 +++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index efc2234b..b04e6898 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -35,6 +35,11 @@ config = [{ 'name': 'label', 'description': 'Label to apply on added torrents.', }, + { + 'name': 'directory', + 'type': 'directory', + 'description': 'Directory where rtorrent should download the files too.', + }, { 'name': 'remove_complete', 'label': 'Remove torrent', @@ -43,6 +48,14 @@ config = [{ 'type': 'bool', 'description': 'Remove the torrent after it finishes seeding.', }, + { + 'name': 'append_label', + 'label': 'Append Label', + 'default': False, + 'advanced': True, + 'type': 'bool', + 'description': 'Append label to download location. Requires you to set the download location above.', + }, { 'name': 'delete_files', 'label': 'Remove files', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 161c671a..c7cf03b9 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -7,7 +7,7 @@ from datetime import timedelta from hashlib import sha1 from rtorrent import RTorrent from rtorrent.err import MethodError -import shutil +import shutil, os log = CPLog(__name__) @@ -91,6 +91,7 @@ class rTorrent(Downloader): if self.conf('label'): torrent_params['label'] = self.conf('label') + if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False @@ -120,6 +121,11 @@ class rTorrent(Downloader): if self.conf('label'): torrent.set_custom(1, self.conf('label')) + if self.conf('directory') and self.conf('append_label'): + torrent.set_directory(os.path.join(self.conf('directory'), self.conf('label'))) + elif self.conf('directory') and not self.conf('append_label'): + torrent.set_directory(self.conf('directory')) + # Set Ratio Group torrent.set_visible(group_name) From 317a1f119b13ef47f3876e18314c6a78e6c149d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Sun, 29 Sep 2013 18:03:52 +0200 Subject: [PATCH 37/57] not needed --- couchpotato/core/downloaders/rtorrent/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index c7cf03b9..ac11c7b4 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -123,7 +123,7 @@ class rTorrent(Downloader): if self.conf('directory') and self.conf('append_label'): torrent.set_directory(os.path.join(self.conf('directory'), self.conf('label'))) - elif self.conf('directory') and not self.conf('append_label'): + elif self.conf('directory'): torrent.set_directory(self.conf('directory')) # Set Ratio Group From f0f843f746feb3e7363cad8bcf3e16cb85c05ffc Mon Sep 17 00:00:00 2001 From: mano3m Date: Fri, 30 Aug 2013 12:15:28 +0200 Subject: [PATCH 38/57] Add release.update event Proof of concept commit. It updates the database and calls movie.update.id to refresh the entire movie in the frontend. It would be better to crease a static js file in the release folder and add release functionality there including updating one release only. --- .../media/movie/_base/static/movie.actions.js | 11 ----- couchpotato/core/plugins/release/main.py | 41 ++++++++++++++++--- couchpotato/core/plugins/renamer/main.py | 35 +++++----------- 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index e9f6141f..542287fa 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -397,17 +397,6 @@ MA.Release = new Class({ 'data': { 'id': release.id }, - 'onComplete': function(){ - var el = release.el; - if(el && (el.hasClass('failed') || el.hasClass('ignored'))){ - el.removeClass('failed').removeClass('ignored'); - el.getElement('.release_status').set('text', 'available'); - } - else if(el) { - el.addClass('ignored'); - el.getElement('.release_status').set('text', 'ignored'); - } - } }) }, diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 46857adf..fe41d06e 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -10,12 +10,21 @@ from sqlalchemy.orm import joinedload_all from sqlalchemy.sql.expression import and_, or_ import os import traceback +import time log = CPLog(__name__) class Release(Plugin): + default_movie_dict = { + 'profile': {'types': {'quality': {}}}, + 'releases': {'status': {}, 'quality': {}, 'files':{}, 'info': {}}, + 'library': {'titles': {}, 'files':{}}, + 'files': {}, + 'status': {} + } + def __init__(self): addEvent('release.add', self.add) @@ -47,6 +56,7 @@ class Release(Plugin): addEvent('release.for_movie', self.forMovie) addEvent('release.delete', self.delete) addEvent('release.clean', self.clean) + addEvent('release.update', self.update_status) def add(self, group): @@ -159,8 +169,7 @@ class Release(Plugin): rel = db.query(Relea).filter_by(id = id).first() if rel: ignored_status, failed_status, available_status = fireEvent('status.get', ['ignored', 'failed', 'available'], single = True) - rel.status_id = available_status.get('id') if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status.get('id') - db.commit() + self.update_status(id, available_status if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status) return { 'success': True @@ -199,14 +208,12 @@ class Release(Plugin): if success: db.expunge_all() - rel = db.query(Relea).filter_by(id = id).first() # Get release again + rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again?? if rel.status_id != done_status.get('id'): - rel.status_id = snatched_status.get('id') - db.commit() + fireEvent('release.update', id = id, status = snatched_status, single = True) fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Successfully snatched "%s"' % item['name']) - return { 'success': success } @@ -241,3 +248,25 @@ class Release(Plugin): 'success': True } + def update_status(self, id = None, status = None): + + db = get_session() + + rel = db.query(Relea).filter_by(id = id).first() + if rel and status and rel.status_id != status.get('id'): + + item = {} + for info in rel.info: + item[info.identifier] = info.value + + #update status in Db + log.debug('Marking release %s as %s', (item['name'], status.get("label"))) + rel.status_id = status.get('id') + rel.last_edit = int(time.time()) + db.commit() + + #Notify frontend + fireEvent('notify.frontend', type = 'release.download', data = True, message = '"%s" updated to %s' % (item['name'], status.get("label"))) + + #Update all movie info as there is no release update function + fireEvent('notify.frontend', type = 'movie.update.%s' % rel.movie.id, data = rel.movie.to_dict(self.default_movie_dict)) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index ad7df1cf..6a070f01 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -395,14 +395,8 @@ class Renamer(Plugin): break elif release.status_id is snatched_status.get('id'): if release.quality.id is group['meta_data']['quality']['id']: - log.debug('Marking release as downloaded') - try: - release.status_id = downloaded_status.get('id') - release.last_edit = int(time.time()) - except Exception, e: - log.error('Failed marking release as finished: %s %s', (e, traceback.format_exc())) - - db.commit() + # Set the release to downloaded + fireEvent('release.update', id = release.id, status = downloaded_status, single = True) # Remove leftover files if not remove_leftovers: # Don't remove anything @@ -677,7 +671,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) try: for rel in rels: rel_dict = rel.to_dict({'info': {}}) - movie_dict = fireEvent('movie.get', rel.movie_id, single = True) # check status @@ -712,26 +705,22 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Remove the downloading tag self.untagDir(item['folder'], 'downloading') - rel.status_id = seeding_status.get('id') - rel.last_edit = int(time.time()) - db.commit() + # Set the release to seeding + fireEvent('release.update', id = rel.id, status = seeding_status, single = True) # Scan and set the torrent to paused if required item.update({'pause': True, 'scan': True, 'process_complete': False}) scan_items.append(item) else: - if rel.status_id != seeding_status.get('id'): - rel.status_id = seeding_status.get('id') - rel.last_edit = int(time.time()) - db.commit() + # Set the release to seeding + fireEvent('release.update', id = rel.id, status = seeding_status, single = True) #let it seed log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio'])) elif item['status'] == 'failed': fireEvent('download.remove_failed', item, single = True) - rel.status_id = failed_status.get('id') - rel.last_edit = int(time.time()) - db.commit() + # Set the release to failed + fireEvent('release.update', id = rel.id, status = failed_status, single = True) if self.conf('next_on_failed'): fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id) @@ -743,18 +732,14 @@ Remove it if you want it to be renamed (again, or at least let it try again) if rel.status_id == seeding_status.get('id'): if rel.movie.status_id == done_status.get('id'): # Set the release to done as the movie has already been renamed - rel.status_id = downloaded_status.get('id') - rel.last_edit = int(time.time()) - db.commit() + fireEvent('release.update', id = rel.id, status = downloaded_status, single = True) # Allow the downloader to clean-up item.update({'pause': False, 'scan': False, 'process_complete': True}) scan_items.append(item) else: # Set the release to snatched so that the renamer can process the release as if it was never seeding - rel.status_id = snatched_status.get('id') - rel.last_edit = int(time.time()) - db.commit() + fireEvent('release.update', id = rel.id, status = snatched_status, single = True) # Scan and Allow the downloader to clean-up item.update({'pause': False, 'scan': True, 'process_complete': True}) From 1bddadf3a4809ef72156517d6e645ce6cf0f09ea Mon Sep 17 00:00:00 2001 From: mano3m Date: Fri, 30 Aug 2013 13:16:15 +0200 Subject: [PATCH 39/57] clean-up searcher --- couchpotato/core/media/_base/searcher/main.py | 16 +++------------- couchpotato/core/plugins/release/main.py | 3 --- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index f09be64b..662ed80f 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -60,7 +60,7 @@ class Searcher(SearcherBase): if downloader_enabled: - snatched_status = fireEvent('status.get', 'snatched', single = True) + snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True) # Download movie to temp filedata = None @@ -79,9 +79,7 @@ class Searcher(SearcherBase): rls = db.query(Release).filter_by(identifier = md5(data['url'])).first() if rls: renamer_enabled = Env.setting('enabled', 'renamer') - - done_status = fireEvent('status.get', 'done', single = True) - rls.status_id = done_status.get('id') if not renamer_enabled else snatched_status.get('id') + fireEvent('release.update', id = rls.id, status = done_status if not renamer_enabled else snatched_status, single = True) # Save download-id info if returned if isinstance(download_result, dict): @@ -100,20 +98,12 @@ class Searcher(SearcherBase): # If renamer isn't used, mark movie done if not renamer_enabled: - active_status = fireEvent('status.get', 'active', single = True) - done_status = fireEvent('status.get', 'done', single = True) try: if movie['status_id'] == active_status.get('id'): for profile_type in movie['profile']['types']: if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: - log.info('Renamer disabled, marking movie as finished: %s', log_movie) - - # Mark release done - rls.status_id = done_status.get('id') - rls.last_edit = int(time.time()) - db.commit() - # Mark movie done + log.info('Renamer disabled, marking movie as finished: %s', log_movie) mvie = db.query(Movie).filter_by(id = movie['id']).first() mvie.status_id = done_status.get('id') mvie.last_edit = int(time.time()) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index fe41d06e..0466a19e 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -210,9 +210,6 @@ class Release(Plugin): db.expunge_all() rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again?? - if rel.status_id != done_status.get('id'): - fireEvent('release.update', id = id, status = snatched_status, single = True) - fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Successfully snatched "%s"' % item['name']) return { 'success': success From a2cb0ec8adbac5b7ae2802d51531c86d9064b960 Mon Sep 17 00:00:00 2001 From: mano3m Date: Sat, 31 Aug 2013 10:18:41 +0200 Subject: [PATCH 40/57] frontend release.update --- .../core/media/movie/_base/static/movie.actions.js | 10 +++++++++- couchpotato/core/plugins/release/main.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index 542287fa..6dca1141 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -241,7 +241,6 @@ MA.Release = new Class({ } }) ).inject(self.release_container); - release['el'] = item; if(status.identifier == 'ignored' || status.identifier == 'failed' || status.identifier == 'snatched'){ @@ -251,6 +250,15 @@ MA.Release = new Class({ else if(!self.next_release && status.identifier == 'available'){ self.next_release = release; } + + App.addEvent('release.update.'+release.id, function(notification){ + var new_status=Status.get(notification.data); + release.el.className='item '+new_status.identifier; + var status_el=release.el.getElement('.release_status'); + status_el.className='release_status '+new_status.identifier; + status_el.set('text', new_status.identifier); + }); + }); if(self.last_release) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 0466a19e..2c1e2f54 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -266,4 +266,4 @@ class Release(Plugin): fireEvent('notify.frontend', type = 'release.download', data = True, message = '"%s" updated to %s' % (item['name'], status.get("label"))) #Update all movie info as there is no release update function - fireEvent('notify.frontend', type = 'movie.update.%s' % rel.movie.id, data = rel.movie.to_dict(self.default_movie_dict)) + fireEvent('notify.frontend', type = 'release.update.%s' % rel.id, data = status.get('id')) From d11f9d26c007076443000162fc4235c14762123b Mon Sep 17 00:00:00 2001 From: mano3m Date: Sat, 31 Aug 2013 22:24:02 +0200 Subject: [PATCH 41/57] Add missing status --- .../core/media/movie/_base/static/movie.css | 1 + couchpotato/core/plugins/renamer/main.py | 26 ++++++++++++------- couchpotato/core/plugins/status/main.py | 1 + 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css index 0200417c..0b3162e3 100644 --- a/couchpotato/core/media/movie/_base/static/movie.css +++ b/couchpotato/core/media/movie/_base/static/movie.css @@ -426,6 +426,7 @@ .movies .data .quality .available { background-color: #578bc3; } .movies .data .quality .failed { background-color: #a43d34; } + .movies .data .quality .missing { background-color: #a43d34; } .movies .data .quality .snatched { background-color: #a2a232; } .movies .data .quality .seeding { background-color: #0a6819; } .movies .data .quality .done { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 6a070f01..344fe55f 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -650,12 +650,13 @@ Remove it if you want it to be renamed (again, or at least let it try again) self.checking_snatched = True - snatched_status, ignored_status, failed_status, done_status, seeding_status, downloaded_status = \ - fireEvent('status.get', ['snatched', 'ignored', 'failed', 'done', 'seeding', 'downloaded'], single = True) + snatched_status, ignored_status, failed_status, done_status, seeding_status, downloaded_status, missing_status = \ + fireEvent('status.get', ['snatched', 'ignored', 'failed', 'done', 'seeding', 'downloaded', 'missing'], single = True) db = get_session() rels = db.query(Release).filter_by(status_id = snatched_status.get('id')).all() rels.extend(db.query(Release).filter_by(status_id = seeding_status.get('id')).all()) + rels.extend(db.query(Release).filter_by(status_id = missing_status.get('id')).all()) scan_items = [] scan_required = False @@ -692,11 +693,16 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.debug('Found %s: %s, time to go: %s', (item['name'], item['status'].upper(), timeleft)) if item['status'] == 'busy': + # Set the release to snatched if it was missing before + fireEvent('release.update', id = rel.id, status = snatched_status, single = True) + # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading if item['folder'] and self.conf('from') in item['folder']: self.tagDir(item['folder'], 'downloading') elif item['status'] == 'seeding': + # Set the release to seeding + fireEvent('release.update', id = rel.id, status = seeding_status, single = True) #If linking setting is enabled, process release if self.conf('file_action') != 'move' and not rel.movie.status_id == done_status.get('id') and self.statusInfoComplete(item): @@ -705,23 +711,19 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Remove the downloading tag self.untagDir(item['folder'], 'downloading') - # Set the release to seeding - fireEvent('release.update', id = rel.id, status = seeding_status, single = True) - # Scan and set the torrent to paused if required item.update({'pause': True, 'scan': True, 'process_complete': False}) scan_items.append(item) else: - # Set the release to seeding - fireEvent('release.update', id = rel.id, status = seeding_status, single = True) - #let it seed log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio'])) + elif item['status'] == 'failed': - fireEvent('download.remove_failed', item, single = True) # Set the release to failed fireEvent('release.update', id = rel.id, status = failed_status, single = True) + fireEvent('download.remove_failed', item, single = True) + if self.conf('next_on_failed'): fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id) elif item['status'] == 'completed': @@ -746,6 +748,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) scan_items.append(item) else: + # Set the release to snatched if it was missing before + fireEvent('release.update', id = rel.id, status = snatched_status, single = True) + # Remove the downloading tag self.untagDir(item['folder'], 'downloading') @@ -761,6 +766,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) if not found: log.info('%s not found in downloaders', nzbname) + # Set the release to missing + fireEvent('release.update', id = rel.id, status = missing_status, single = True) + except: log.error('Failed checking for release in downloader: %s', traceback.format_exc()) diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py index 7546c651..b3b37bdc 100644 --- a/couchpotato/core/plugins/status/main.py +++ b/couchpotato/core/plugins/status/main.py @@ -24,6 +24,7 @@ class StatusPlugin(Plugin): 'available': 'Available', 'suggest': 'Suggest', 'seeding': 'Seeding', + 'missing': 'Missing', } status_cached = {} From 0c6c172d6af56d083a52dc10d3c701cecda347fb Mon Sep 17 00:00:00 2001 From: mano3m Date: Sat, 31 Aug 2013 23:48:38 +0200 Subject: [PATCH 42/57] Update movie quality status colour and text It isnt perfect this way. I think we need to add a sepperate function to do this and call that from both when CPS is loading the page and when it updates a release (e.g. just rebuild the icons) --- .../core/media/movie/_base/static/movie.actions.js | 13 +++++++++++++ couchpotato/core/media/movie/_base/static/movie.js | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index 6dca1141..2a979574 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -252,11 +252,24 @@ MA.Release = new Class({ } App.addEvent('release.update.'+release.id, function(notification){ + var q = self.movie.quality.getElement('.q_id'+ release.quality_id), + status = Status.get(release.status_id); + var new_status=Status.get(notification.data); + release.status_id = new_status.id release.el.className='item '+new_status.identifier; + var status_el=release.el.getElement('.release_status'); status_el.className='release_status '+new_status.identifier; status_el.set('text', new_status.identifier); + + if(!q && (new_status.identifier == 'snatched' || new_status.identifier == 'seeding' || new_status.identifier == 'done')) + var q = self.addQuality(release.quality_id); + + if (new_status && q && !q.hasClass(new_status.identifier)){ + q.removeClass(status.identifier).addClass(new_status.identifier); + q.set('title', q.get('title').replace(status.label, new_status.label)); + } }); }); diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index 6defc2ad..a865325b 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -185,7 +185,7 @@ var Movie = new Class({ var q = self.quality.getElement('.q_id'+ release.quality_id), status = Status.get(release.status_id); - if(!q && (status.identifier == 'snatched' || status.identifier == 'done')) + if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done')) var q = self.addQuality(release.quality_id) if (status && q && !q.hasClass(status.identifier)){ From 516447a1048470433e10efdb6ac657464f9d5f06 Mon Sep 17 00:00:00 2001 From: mano3m Date: Sun, 1 Sep 2013 00:30:11 +0200 Subject: [PATCH 43/57] Remove movie_dict --- couchpotato/core/plugins/release/main.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 2c1e2f54..97e350c5 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -17,14 +17,6 @@ log = CPLog(__name__) class Release(Plugin): - default_movie_dict = { - 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}, 'files':{}, 'info': {}}, - 'library': {'titles': {}, 'files':{}}, - 'files': {}, - 'status': {} - } - def __init__(self): addEvent('release.add', self.add) From 27fdbff619b6df5bee97544f8fed31bb389af1fd Mon Sep 17 00:00:00 2001 From: mano3m Date: Thu, 5 Sep 2013 18:49:19 +0200 Subject: [PATCH 44/57] Set missing to ignored after 1 week --- couchpotato/core/plugins/renamer/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 344fe55f..7e2fdbcc 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -766,8 +766,13 @@ Remove it if you want it to be renamed (again, or at least let it try again) if not found: log.info('%s not found in downloaders', nzbname) - # Set the release to missing - fireEvent('release.update', id = rel.id, status = missing_status, single = True) + #Check status if already missing and for how long, if > 1 week, set to ignored else to missing + if rel.status_id == missing_status.get('id'): + if rel.last_edit < int(time.time()) - 7*24*60*60: + fireEvent('release.update', id = rel.id, status = ignored_status, single = True) + else: + # Set the release to missing + fireEvent('release.update', id = rel.id, status = missing_status, single = True) except: log.error('Failed checking for release in downloader: %s', traceback.format_exc()) From 7c5616cc79ecb092a712eb8aa1d829013930b183 Mon Sep 17 00:00:00 2001 From: mano3m Date: Fri, 6 Sep 2013 23:03:12 +0200 Subject: [PATCH 45/57] fix colour order --- .../core/media/movie/_base/static/movie.css | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css index 0b3162e3..c72eb136 100644 --- a/couchpotato/core/media/movie/_base/static/movie.css +++ b/couchpotato/core/media/movie/_base/static/movie.css @@ -419,23 +419,25 @@ } .movies .data .quality .available, - .movies .data .quality .snatched { + .movies .data .quality .snatched, + .movies .data .quality .seeding { opacity: 1; cursor: pointer; } .movies .data .quality .available { background-color: #578bc3; } - .movies .data .quality .failed { background-color: #a43d34; } - .movies .data .quality .missing { background-color: #a43d34; } + .movies .data .quality .failed, + .movies .data .quality .missing, + .movies .data .quality .ignored { background-color: #a43d34; } .movies .data .quality .snatched { background-color: #a2a232; } - .movies .data .quality .seeding { background-color: #0a6819; } .movies .data .quality .done { background-color: #369545; opacity: 1; } + .movies .data .quality .seeding { background-color: #0a6819; } .movies .data .quality .finish { background-image: url('../images/sprite.png'); - background-repeat: no-repeat; + background-repeat: no-repeat; background-position: 0 2px; padding-left: 14px; background-size: 14px @@ -647,7 +649,7 @@ margin-top: 25px; } } - + .trailer_container.hide { height: 0 !important; } @@ -1030,7 +1032,7 @@ .movies .progress > div .folder { display: inline-block; padding: 5px 20px 5px 0; - white-space: nowrap; + white-space: nowrap; text-overflow: ellipsis; overflow: hidden; width: 85%; From 89daa836e75948f74e4725ea7e716dc243faaedd Mon Sep 17 00:00:00 2001 From: mano3m Date: Sun, 1 Sep 2013 01:05:53 +0200 Subject: [PATCH 46/57] Remove all empty folders Quite often there is a subfolder in the movie folder after extraction. This folder is deleted but the actual movie folder remains behind. This update fixes that in both cases: move_folder is known, or we work in the 'from' folder. --- couchpotato/core/plugins/renamer/main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index ad7df1cf..be886d42 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -477,8 +477,15 @@ class Renamer(Plugin): if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(download_info): try: - log.info('Deleting folder: %s', group['parentdir']) - self.deleteEmptyFolder(group['parentdir']) + if movie_folder: + # Delete the movie folder + group_folder = movie_folder + else: + # Delete the first empty subfolder in the tree relative to the 'from' folder + group_folder = os.path.join(self.conf('from'), os.path.relpath(group['parentdir'], self.conf('from')).split(os.path.sep)[0]) + + log.info('Deleting folder: %s', group_folder) + self.deleteEmptyFolder(group_folder) except: log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc())) From 6174f121c869a726fbaabd900aceb285b162f8bd Mon Sep 17 00:00:00 2001 From: mano3m Date: Mon, 30 Sep 2013 19:27:11 +0200 Subject: [PATCH 47/57] fix log message --- couchpotato/core/plugins/renamer/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index be886d42..ea7926eb 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -476,18 +476,18 @@ class Renamer(Plugin): log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(download_info): + if movie_folder: + # Delete the movie folder + group_folder = movie_folder + else: + # Delete the first empty subfolder in the tree relative to the 'from' folder + group_folder = os.path.join(self.conf('from'), os.path.relpath(group['parentdir'], self.conf('from')).split(os.path.sep)[0]) + try: - if movie_folder: - # Delete the movie folder - group_folder = movie_folder - else: - # Delete the first empty subfolder in the tree relative to the 'from' folder - group_folder = os.path.join(self.conf('from'), os.path.relpath(group['parentdir'], self.conf('from')).split(os.path.sep)[0]) - log.info('Deleting folder: %s', group_folder) self.deleteEmptyFolder(group_folder) except: - log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc())) + log.error('Failed removing %s: %s', (group_folder, traceback.format_exc())) # Notify on download, search for trailers etc download_message = 'Downloaded %s (%s)' % (movie_title, replacements['quality']) From 6bda5f5b0373fb699db072630fd38bfa66782160 Mon Sep 17 00:00:00 2001 From: mano3m Date: Mon, 30 Sep 2013 19:33:53 +0200 Subject: [PATCH 48/57] Don't use movie done status to check seeding Fixes #2278 --- couchpotato/core/plugins/renamer/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index ad7df1cf..202bf439 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -706,7 +706,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif item['status'] == 'seeding': #If linking setting is enabled, process release - if self.conf('file_action') != 'move' and not rel.movie.status_id == done_status.get('id') and self.statusInfoComplete(item): + if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(item): log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (item['name'], item['seed_ratio'])) # Remove the downloading tag From fd8e50b53383593c90a7be8b65ced038972a5019 Mon Sep 17 00:00:00 2001 From: mano3m Date: Sat, 28 Sep 2013 13:10:54 +0200 Subject: [PATCH 49/57] [SabNZBd] Consider encrypted as failed --- couchpotato/core/downloaders/sabnzbd/main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 08ee409c..41f9f709 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -90,9 +90,14 @@ class Sabnzbd(Downloader): # Get busy releases for item in queue.get('slots', []): + status = 'busy' + if 'ENCRYPTED / ' in item['filename']: + status = 'failed' + statuses.append({ 'id': item['nzo_id'], 'name': item['filename'], + 'status': status, 'original_status': item['status'], 'timeleft': item['timeleft'] if not queue['paused'] else -1, }) @@ -122,6 +127,12 @@ class Sabnzbd(Downloader): log.info('%s failed downloading, deleting...', item['name']) try: + self.call({ + 'mode': 'queue', + 'name': 'delete', + 'del_files': '1', + 'value': item['id'] + }, use_json = False) self.call({ 'mode': 'history', 'name': 'delete', From 0876d1ff8e6f88e5eb40f656ba4e7b863d58b186 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Sep 2013 20:52:04 +0200 Subject: [PATCH 50/57] Rename release.update to update_status --- couchpotato/core/media/_base/searcher/main.py | 2 +- .../media/movie/_base/static/movie.actions.js | 36 ++++++++++--------- couchpotato/core/plugins/release/main.py | 12 +++---- couchpotato/core/plugins/renamer/main.py | 20 +++++------ 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 662ed80f..934a1472 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -79,7 +79,7 @@ class Searcher(SearcherBase): rls = db.query(Release).filter_by(identifier = md5(data['url'])).first() if rls: renamer_enabled = Env.setting('enabled', 'renamer') - fireEvent('release.update', id = rls.id, status = done_status if not renamer_enabled else snatched_status, single = True) + fireEvent('release.update_status', rls.id, status = done_status if not renamer_enabled else snatched_status, single = True) # Save download-id info if returned if isinstance(download_result, dict): diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index 2a979574..9dd6bdfe 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -250,27 +250,29 @@ MA.Release = new Class({ else if(!self.next_release && status.identifier == 'available'){ self.next_release = release; } + + var update_handle = function(notification) { + var q = self.movie.quality.getElement('.q_id' + release.quality_id), + status = Status.get(release.status_id), + new_status = Status.get(notification.data); + + release.status_id = new_status.id + release.el.set('class', 'item ' + new_status.identifier); - App.addEvent('release.update.'+release.id, function(notification){ - var q = self.movie.quality.getElement('.q_id'+ release.quality_id), - status = Status.get(release.status_id); + var status_el = release.el.getElement('.release_status'); + status_el.set('class', 'release_status ' + new_status.identifier); + status_el.set('text', new_status.identifier); - var new_status=Status.get(notification.data); - release.status_id = new_status.id - release.el.className='item '+new_status.identifier; + if(!q && (new_status.identifier == 'snatched' || new_status.identifier == 'seeding' || new_status.identifier == 'done')) + var q = self.addQuality(release.quality_id); - var status_el=release.el.getElement('.release_status'); - status_el.className='release_status '+new_status.identifier; - status_el.set('text', new_status.identifier); + if(new_status && q && !q.hasClass(new_status.identifier)) { + q.removeClass(status.identifier).addClass(new_status.identifier); + q.set('title', q.get('title').replace(status.label, new_status.label)); + } + } - if(!q && (new_status.identifier == 'snatched' || new_status.identifier == 'seeding' || new_status.identifier == 'done')) - var q = self.addQuality(release.quality_id); - - if (new_status && q && !q.hasClass(new_status.identifier)){ - q.removeClass(status.identifier).addClass(new_status.identifier); - q.set('title', q.get('title').replace(status.label, new_status.label)); - } - }); + App.addEvent('release.update_status.' + release.id, update_handle); }); diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 97e350c5..e8d1815f 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -48,7 +48,7 @@ class Release(Plugin): addEvent('release.for_movie', self.forMovie) addEvent('release.delete', self.delete) addEvent('release.clean', self.clean) - addEvent('release.update', self.update_status) + addEvent('release.update_status', self.updateStatus) def add(self, group): @@ -161,7 +161,7 @@ class Release(Plugin): rel = db.query(Relea).filter_by(id = id).first() if rel: ignored_status, failed_status, available_status = fireEvent('status.get', ['ignored', 'failed', 'available'], single = True) - self.update_status(id, available_status if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status) + self.updateStatus(id, available_status if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status) return { 'success': True @@ -237,7 +237,8 @@ class Release(Plugin): 'success': True } - def update_status(self, id = None, status = None): + def updateStatus(self, id, status = None): + if not status: return db = get_session() @@ -254,8 +255,5 @@ class Release(Plugin): rel.last_edit = int(time.time()) db.commit() - #Notify frontend - fireEvent('notify.frontend', type = 'release.download', data = True, message = '"%s" updated to %s' % (item['name'], status.get("label"))) - #Update all movie info as there is no release update function - fireEvent('notify.frontend', type = 'release.update.%s' % rel.id, data = status.get('id')) + fireEvent('notify.frontend', type = 'release.update_status.%s' % rel.id, data = status.get('id')) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index b0df89dd..c2b71361 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -396,7 +396,7 @@ class Renamer(Plugin): elif release.status_id is snatched_status.get('id'): if release.quality.id is group['meta_data']['quality']['id']: # Set the release to downloaded - fireEvent('release.update', id = release.id, status = downloaded_status, single = True) + fireEvent('release.update_status', release.id, status = downloaded_status, single = True) # Remove leftover files if not remove_leftovers: # Don't remove anything @@ -701,7 +701,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if item['status'] == 'busy': # Set the release to snatched if it was missing before - fireEvent('release.update', id = rel.id, status = snatched_status, single = True) + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading if item['folder'] and self.conf('from') in item['folder']: @@ -709,7 +709,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif item['status'] == 'seeding': # Set the release to seeding - fireEvent('release.update', id = rel.id, status = seeding_status, single = True) + fireEvent('release.update_status', rel.id, status = seeding_status, single = True) #If linking setting is enabled, process release if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(item): @@ -727,7 +727,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif item['status'] == 'failed': # Set the release to failed - fireEvent('release.update', id = rel.id, status = failed_status, single = True) + fireEvent('release.update_status', rel.id, status = failed_status, single = True) fireEvent('download.remove_failed', item, single = True) @@ -741,14 +741,14 @@ Remove it if you want it to be renamed (again, or at least let it try again) if rel.status_id == seeding_status.get('id'): if rel.movie.status_id == done_status.get('id'): # Set the release to done as the movie has already been renamed - fireEvent('release.update', id = rel.id, status = downloaded_status, single = True) + fireEvent('release.update_status', rel.id, status = downloaded_status, single = True) # Allow the downloader to clean-up item.update({'pause': False, 'scan': False, 'process_complete': True}) scan_items.append(item) else: # Set the release to snatched so that the renamer can process the release as if it was never seeding - fireEvent('release.update', id = rel.id, status = snatched_status, single = True) + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) # Scan and Allow the downloader to clean-up item.update({'pause': False, 'scan': True, 'process_complete': True}) @@ -756,7 +756,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) else: # Set the release to snatched if it was missing before - fireEvent('release.update', id = rel.id, status = snatched_status, single = True) + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) # Remove the downloading tag self.untagDir(item['folder'], 'downloading') @@ -775,11 +775,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) #Check status if already missing and for how long, if > 1 week, set to ignored else to missing if rel.status_id == missing_status.get('id'): - if rel.last_edit < int(time.time()) - 7*24*60*60: - fireEvent('release.update', id = rel.id, status = ignored_status, single = True) + if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60: + fireEvent('release.update_status', rel.id, status = ignored_status, single = True) else: # Set the release to missing - fireEvent('release.update', id = rel.id, status = missing_status, single = True) + fireEvent('release.update_status', rel.id, status = missing_status, single = True) except: log.error('Failed checking for release in downloader: %s', traceback.format_exc()) From f121db059e4dfc6cc074af5b2a4045f038963283 Mon Sep 17 00:00:00 2001 From: salfab Date: Sat, 28 Sep 2013 22:22:41 +0200 Subject: [PATCH 51/57] add new provider for ILT. --- .../torrent/ilovetorrents/__init__.py | 52 +++++++ .../providers/torrent/ilovetorrents/main.py | 147 ++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 couchpotato/core/providers/torrent/ilovetorrents/__init__.py create mode 100644 couchpotato/core/providers/torrent/ilovetorrents/main.py diff --git a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py new file mode 100644 index 00000000..8cd86d35 --- /dev/null +++ b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py @@ -0,0 +1,52 @@ +from main import ILoveTorrents + +def start(): + return ILoveTorrents() + +config = [{ + 'name': 'ilovetorrents', + 'groups': [ + { + 'tab': 'searcher', + 'list': 'torrent_providers', + 'name': 'ILoveTorrents', + 'description': 'Where the Love of Torrents is Born', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': True + }, + { + 'name': 'domain', + 'advanced': True, + 'label': 'Proxy server', + 'description': 'Domain for requests, keep empty to let CouchPotato pick.', + }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', + }, + { + 'name': 'extra_score', + 'advanced': True, + 'label': 'Extra Score', + 'type': 'int', + 'default': 0, + 'description': 'Starting score for each release found via this provider.', + } + ], + } + ] +}] diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py new file mode 100644 index 00000000..5659b6a8 --- /dev/null +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -0,0 +1,147 @@ +from bs4 import BeautifulSoup +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode +from couchpotato.core.helpers.variable import tryInt, cleanHost +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.torrent.base import TorrentMagnetProvider +from couchpotato.environment import Env +import re +import time +import traceback + +log = CPLog(__name__) + + +class ILoveTorrents(TorrentMagnetProvider): + + urls = { + 'detail': '%s/torrent/%s', + 'search': '%s/browse.php?search=%s&page=%s&cat=%s' + } + + cat_ids = [ + (["41"], ['720p', '1080p', 'brrip']), + (["19"], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), + (["20"], ['dvdr']) + ] + + cat_backup_id = 200 + disable_provider = False + http_time_between_calls = 0 + + proxy_list = [ + 'http://www.ilovetorrents.me', + ] + + def __init__(self): + self.domain = self.conf('domain') + super(ILoveTorrents, self).__init__() + + def _searchOnTitle(self, title, movie, quality, results): + + page = 0 + total_pages = 1 + cats = self.getCatId(quality['identifier']) + + while page < total_pages: + + search_url = self.urls['search'] % (self.getDomain(), tryUrlencode('"%s" %s' % (title, movie['library']['year'])), page, cats[0])) + page += 1 + + data = self.getHTMLData(search_url) + + if data: + try: + soup = BeautifulSoup(data, "html5lib") + + results_table = soup.find('table', attrs = {'class': 'koptekst'}) + + if not results_table: + return + + try: + pagelinks = soup.findAll(href=re.compile("page")) + pageNumbers = [int(re.search('page=(?P.+'')', i["href"]).group('pageNumber')) for i in pagelinks] + total_pages = max(pageNumbers) + + except: + pass + + entries = results_table.find_all('tr') + + for result in entries[1:]: + link = result.find(href = re.compile('details.php'))['href'] + download = result.find('a', href = re.compile('download.php'))['href'] + + try: + matches = re.search('>(?P.+)
(?P.B)', unicode(result.select('td.rowhead')[5])) + size = matches.group('size') + " " + matches.group('unit') + + except: + continue + + if link and download: + + def extra_score(item): + trusted = (0, 10)[result.find('img', alt = re.compile('Trusted')) is not None] + vip = (0, 20)[result.find('img', alt = re.compile('VIP')) is not None] + confirmed = (0, 30)[result.find('img', alt = re.compile('Helpers')) is not None] + moderated = (0, 50)[result.find('img', alt = re.compile('Moderator')) is not None] + + return confirmed + trusted + vip + moderated + + results.append({ + 'id': re.search('/(?P\d+)/', link['href']).group('id'), + 'name': link.string, + 'url': download['href'], + 'detail_url': self.getDomain(link['href']), + 'size': self.parseSize(size), + 'seeders': tryInt(result.find_all('td')[2].string), + 'leechers': tryInt(result.find_all('td')[3].string), + 'extra_score': extra_score, + 'get_more_info': self.getMoreInfo + }) + + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + + + def isEnabled(self): + return super(ILoveTorrents, self).isEnabled() and self.getDomain() + + def getDomain(self, url = ''): + + if not self.domain: + for proxy in self.proxy_list: + + prop_name = 'tpb_proxy.%s' % proxy + last_check = float(Env.prop(prop_name, default = 0)) + if last_check > time.time() - 1209600: + continue + + data = '' + try: + data = self.urlopen(proxy, timeout = 3, show_error = False) + except: + log.debug('Failed tpb proxy %s', proxy) + + if 'title="Pirate Search"' in data: + log.debug('Using proxy: %s', proxy) + self.domain = proxy + break + + Env.prop(prop_name, time.time()) + + if not self.domain: + log.error('No ILT proxies left, please add one in settings, or let us know which one to add on the forum.') + return None + + return cleanHost(self.domain).rstrip('/') + url + + def getMoreInfo(self, item): + full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) + html = BeautifulSoup(full_description) + nfo_pre = html.find('div', attrs = {'class':'nfo'}) + description = toUnicode(nfo_pre.text) if nfo_pre else '' + + item['description'] = description + return item From 87754047fa13ad2c103f568a7b17cdb0b8a9b4cf Mon Sep 17 00:00:00 2001 From: salfab Date: Sun, 29 Sep 2013 12:44:53 +0200 Subject: [PATCH 52/57] torrents are found and appended to the results argument --- .../torrent/ilovetorrents/__init__.py | 18 +++- .../providers/torrent/ilovetorrents/main.py | 100 ++++++++---------- 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py index 8cd86d35..84776e72 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py @@ -18,11 +18,19 @@ config = [{ 'type': 'enabler', 'default': True }, - { - 'name': 'domain', - 'advanced': True, - 'label': 'Proxy server', - 'description': 'Domain for requests, keep empty to let CouchPotato pick.', + { + 'name': 'username', + 'label': 'Username', + 'type': 'string', + 'default': '', + 'description': 'The user name for your ILT account', + }, + { + 'name': 'password', + 'label': 'Password', + 'type': 'password', + 'default': '', + 'description': 'The password for your ILT account.', }, { 'name': 'seed_ratio', diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 5659b6a8..755af526 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -14,8 +14,13 @@ log = CPLog(__name__) class ILoveTorrents(TorrentMagnetProvider): urls = { - 'detail': '%s/torrent/%s', - 'search': '%s/browse.php?search=%s&page=%s&cat=%s' + 'domain': 'www.ilovetorrents.me', + 'download': 'http://www.ilovetorrents.me/%s', + 'detail': '%s/torrent/%s', + 'search': '%s/browse.php?search=%s&page=%s&cat=%s', + 'test' : 'http://www.ilovetorrents.me/', + 'login' : 'http://www.ilovetorrents.me/takelogin.php', + 'login_check' : 'http://www.ilovetorrents.me' } cat_ids = [ @@ -28,12 +33,8 @@ class ILoveTorrents(TorrentMagnetProvider): disable_provider = False http_time_between_calls = 0 - proxy_list = [ - 'http://www.ilovetorrents.me', - ] - def __init__(self): - self.domain = self.conf('domain') + self.domain = self.urls['domain'] super(ILoveTorrents, self).__init__() def _searchOnTitle(self, title, movie, quality, results): @@ -43,31 +44,30 @@ class ILoveTorrents(TorrentMagnetProvider): cats = self.getCatId(quality['identifier']) while page < total_pages: - - search_url = self.urls['search'] % (self.getDomain(), tryUrlencode('"%s" %s' % (title, movie['library']['year'])), page, cats[0])) + + movieTitle = tryUrlencode('"%s" %s' % (title, movie['library']['year'])) + search_url = self.urls['search'] % (self.getDomain(), movieTitle, page, cats[0]) page += 1 - - data = self.getHTMLData(search_url) - + + data = self.getHTMLData(search_url, opener = self.login_opener) if data: try: soup = BeautifulSoup(data, "html5lib") results_table = soup.find('table', attrs = {'class': 'koptekst'}) - if not results_table: return - + try: - pagelinks = soup.findAll(href=re.compile("page")) + pagelinks = soup.findAll(href=re.compile("page")) pageNumbers = [int(re.search('page=(?P.+'')', i["href"]).group('pageNumber')) for i in pagelinks] total_pages = max(pageNumbers) - + except: pass entries = results_table.find_all('tr') - + for result in entries[1:]: link = result.find(href = re.compile('details.php'))['href'] download = result.find('a', href = re.compile('download.php'))['href'] @@ -80,68 +80,58 @@ class ILoveTorrents(TorrentMagnetProvider): continue if link and download: - def extra_score(item): trusted = (0, 10)[result.find('img', alt = re.compile('Trusted')) is not None] vip = (0, 20)[result.find('img', alt = re.compile('VIP')) is not None] confirmed = (0, 30)[result.find('img', alt = re.compile('Helpers')) is not None] moderated = (0, 50)[result.find('img', alt = re.compile('Moderator')) is not None] - return confirmed + trusted + vip + moderated - + return confirmed + trusted + vip + moderated + id = re.search('id=(?P\d+)&', link).group('id') + url = self.urls['download'] % (download) + + detail_url = self.getDomain("/"+link) + fileSize = self.parseSize(size) results.append({ - 'id': re.search('/(?P\d+)/', link['href']).group('id'), - 'name': link.string, - 'url': download['href'], - 'detail_url': self.getDomain(link['href']), - 'size': self.parseSize(size), + 'id': id, + 'name': link, + 'url': url, + 'detail_url': detail_url, + 'size': fileSize, 'seeders': tryInt(result.find_all('td')[2].string), 'leechers': tryInt(result.find_all('td')[3].string), - 'extra_score': extra_score, - 'get_more_info': self.getMoreInfo + 'extra_score': extra_score, + 'get_more_info': self.getMoreInfo }) + log.info(results) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + def getLoginParams(self): + return tryUrlencode({ + 'username': self.conf('username'), + 'password': self.conf('password'), + 'submit': 'Welcome to ILT', + }) def isEnabled(self): return super(ILoveTorrents, self).isEnabled() and self.getDomain() - def getDomain(self, url = ''): - - if not self.domain: - for proxy in self.proxy_list: - - prop_name = 'tpb_proxy.%s' % proxy - last_check = float(Env.prop(prop_name, default = 0)) - if last_check > time.time() - 1209600: - continue - - data = '' - try: - data = self.urlopen(proxy, timeout = 3, show_error = False) - except: - log.debug('Failed tpb proxy %s', proxy) - - if 'title="Pirate Search"' in data: - log.debug('Using proxy: %s', proxy) - self.domain = proxy - break - - Env.prop(prop_name, time.time()) - - if not self.domain: - log.error('No ILT proxies left, please add one in settings, or let us know which one to add on the forum.') - return None - + def getDomain(self, url = ''): return cleanHost(self.domain).rstrip('/') + url def getMoreInfo(self, item): - full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) + log.info('Getting more info') + full_description = self.getCache('ilt.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) html = BeautifulSoup(full_description) nfo_pre = html.find('div', attrs = {'class':'nfo'}) description = toUnicode(nfo_pre.text) if nfo_pre else '' item['description'] = description return item + + def loginSuccess(self, output): + return 'logout.php' in output.lower() + + loginCheckSuccess = loginSuccess From 75360f734c85afbc0b9f0d18cc502482a8fd5ad3 Mon Sep 17 00:00:00 2001 From: salfab Date: Sun, 29 Sep 2013 14:12:52 +0200 Subject: [PATCH 53/57] use a proper name, instead of the link --- couchpotato/core/providers/torrent/ilovetorrents/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 755af526..558277b9 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -69,7 +69,10 @@ class ILoveTorrents(TorrentMagnetProvider): entries = results_table.find_all('tr') for result in entries[1:]: - link = result.find(href = re.compile('details.php'))['href'] + prelink = result.find(href = re.compile('details.php')) + contents = prelink.find('b').contents + name = str(contents[0]) + link = prelink['href'] download = result.find('a', href = re.compile('download.php'))['href'] try: @@ -94,7 +97,7 @@ class ILoveTorrents(TorrentMagnetProvider): fileSize = self.parseSize(size) results.append({ 'id': id, - 'name': link, + 'name': name, 'url': url, 'detail_url': detail_url, 'size': fileSize, From 83051b2576bdf38e310a63e656af6b3ec2fb4878 Mon Sep 17 00:00:00 2001 From: salfab Date: Sun, 29 Sep 2013 15:56:32 +0200 Subject: [PATCH 54/57] support getting more info. --- .../providers/torrent/ilovetorrents/main.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 558277b9..5f011e1e 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -124,12 +124,22 @@ class ILoveTorrents(TorrentMagnetProvider): def getDomain(self, url = ''): return cleanHost(self.domain).rstrip('/') + url - def getMoreInfo(self, item): - log.info('Getting more info') - full_description = self.getCache('ilt.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) - html = BeautifulSoup(full_description) - nfo_pre = html.find('div', attrs = {'class':'nfo'}) - description = toUnicode(nfo_pre.text) if nfo_pre else '' + def getMoreInfo(self, item): + cache_key = 'ilt.%s' % item['id'] + description = self.getCache(cache_key) + + if not description: + + try: + full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener) + html = BeautifulSoup(full_description, "html5lib") + nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1].findAll('td')[5] + description = toUnicode(nfo_pre.text) if nfo_pre else '' + except: + log.error('Failed getting more info for %s', item['name']) + description = '' + + self.setCache(cache_key, description, timeout = 25920000) item['description'] = description return item From c37bf12c8a5eaa5a98ba45c179e1dbb64c02c6cd Mon Sep 17 00:00:00 2001 From: salfab Date: Sun, 29 Sep 2013 17:25:30 +0200 Subject: [PATCH 55/57] improve resilience to retrieve description in get_more_info --- couchpotato/core/providers/torrent/ilovetorrents/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 5f011e1e..7ddce50f 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -133,7 +133,7 @@ class ILoveTorrents(TorrentMagnetProvider): try: full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener) html = BeautifulSoup(full_description, "html5lib") - nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1].findAll('td')[5] + nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1] description = toUnicode(nfo_pre.text) if nfo_pre else '' except: log.error('Failed getting more info for %s', item['name']) From 8df0ecc2231c5a087e0c92b2ca1cd5b3ab6d1cda Mon Sep 17 00:00:00 2001 From: salfab Date: Sun, 29 Sep 2013 17:40:35 +0200 Subject: [PATCH 56/57] disabled by default --- couchpotato/core/providers/torrent/thepiratebay/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index 83de7a94..8cf9f86c 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -16,7 +16,7 @@ config = [{ { 'name': 'enabled', 'type': 'enabler', - 'default': True + 'default': False }, { 'name': 'domain', From bbf42da87514ba9e973564e2f19f47d4aa9a4f8f Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 30 Sep 2013 22:18:36 +0200 Subject: [PATCH 57/57] ILoveTorrents cleanup --- .../torrent/ilovetorrents/__init__.py | 2 +- .../providers/torrent/ilovetorrents/main.py | 96 +++++++------------ 2 files changed, 38 insertions(+), 60 deletions(-) diff --git a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py index 84776e72..c6702d7f 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py @@ -16,7 +16,7 @@ config = [{ { 'name': 'enabled', 'type': 'enabler', - 'default': True + 'default': False }, { 'name': 'username', diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 7ddce50f..8c060ec3 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -1,41 +1,34 @@ from bs4 import BeautifulSoup from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode -from couchpotato.core.helpers.variable import tryInt, cleanHost +from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog -from couchpotato.core.providers.torrent.base import TorrentMagnetProvider -from couchpotato.environment import Env +from couchpotato.core.providers.torrent.base import TorrentProvider import re -import time import traceback log = CPLog(__name__) -class ILoveTorrents(TorrentMagnetProvider): +class ILoveTorrents(TorrentProvider): urls = { - 'domain': 'www.ilovetorrents.me', 'download': 'http://www.ilovetorrents.me/%s', - 'detail': '%s/torrent/%s', - 'search': '%s/browse.php?search=%s&page=%s&cat=%s', + 'detail': 'http://www.ilovetorrents.me/%s', + 'search': 'http://www.ilovetorrents.me/browse.php?search=%s&page=%s&cat=%s', 'test' : 'http://www.ilovetorrents.me/', 'login' : 'http://www.ilovetorrents.me/takelogin.php', 'login_check' : 'http://www.ilovetorrents.me' } cat_ids = [ - (["41"], ['720p', '1080p', 'brrip']), - (["19"], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), - (["20"], ['dvdr']) + (['41'], ['720p', '1080p', 'brrip']), + (['19'], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), + (['20'], ['dvdr']) ] cat_backup_id = 200 disable_provider = False - http_time_between_calls = 0 - - def __init__(self): - self.domain = self.urls['domain'] - super(ILoveTorrents, self).__init__() + http_time_between_calls = 1 def _searchOnTitle(self, title, movie, quality, results): @@ -44,69 +37,60 @@ class ILoveTorrents(TorrentMagnetProvider): cats = self.getCatId(quality['identifier']) while page < total_pages: - + movieTitle = tryUrlencode('"%s" %s' % (title, movie['library']['year'])) - search_url = self.urls['search'] % (self.getDomain(), movieTitle, page, cats[0]) + search_url = self.urls['search'] % (movieTitle, page, cats[0]) page += 1 - - data = self.getHTMLData(search_url, opener = self.login_opener) + + data = self.getHTMLData(search_url, opener = self.login_opener) if data: try: - soup = BeautifulSoup(data, "html5lib") - + soup = BeautifulSoup(data) + results_table = soup.find('table', attrs = {'class': 'koptekst'}) if not results_table: return - + try: - pagelinks = soup.findAll(href=re.compile("page")) - pageNumbers = [int(re.search('page=(?P.+'')', i["href"]).group('pageNumber')) for i in pagelinks] + pagelinks = soup.findAll(href = re.compile('page')) + pageNumbers = [int(re.search('page=(?P.+'')', i['href']).group('pageNumber')) for i in pagelinks] total_pages = max(pageNumbers) - + except: pass entries = results_table.find_all('tr') - + for result in entries[1:]: prelink = result.find(href = re.compile('details.php')) - contents = prelink.find('b').contents - name = str(contents[0]) link = prelink['href'] - download = result.find('a', href = re.compile('download.php'))['href'] - - try: - matches = re.search('>(?P.+)
(?P.B)', unicode(result.select('td.rowhead')[5])) - size = matches.group('size') + " " + matches.group('unit') - - except: - continue + download = result.find('a', href = re.compile('download.php'))['href'] if link and download: + def extra_score(item): trusted = (0, 10)[result.find('img', alt = re.compile('Trusted')) is not None] vip = (0, 20)[result.find('img', alt = re.compile('VIP')) is not None] confirmed = (0, 30)[result.find('img', alt = re.compile('Helpers')) is not None] moderated = (0, 50)[result.find('img', alt = re.compile('Moderator')) is not None] - return confirmed + trusted + vip + moderated + return confirmed + trusted + vip + moderated + id = re.search('id=(?P\d+)&', link).group('id') url = self.urls['download'] % (download) - - detail_url = self.getDomain("/"+link) - fileSize = self.parseSize(size) + + fileSize = self.parseSize(result.select('td.rowhead')[5].text) results.append({ 'id': id, - 'name': name, + 'name': toUnicode(prelink.find('b').text), 'url': url, - 'detail_url': detail_url, + 'detail_url': self.urls['detail'] % link, 'size': fileSize, 'seeders': tryInt(result.find_all('td')[2].string), 'leechers': tryInt(result.find_all('td')[3].string), - 'extra_score': extra_score, - 'get_more_info': self.getMoreInfo + 'extra_score': extra_score, + 'get_more_info': self.getMoreInfo }) - log.info(results) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) @@ -118,21 +102,15 @@ class ILoveTorrents(TorrentMagnetProvider): 'submit': 'Welcome to ILT', }) - def isEnabled(self): - return super(ILoveTorrents, self).isEnabled() and self.getDomain() - - def getDomain(self, url = ''): - return cleanHost(self.domain).rstrip('/') + url - - def getMoreInfo(self, item): + def getMoreInfo(self, item): cache_key = 'ilt.%s' % item['id'] description = self.getCache(cache_key) if not description: try: - full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener) - html = BeautifulSoup(full_description, "html5lib") + full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener) + html = BeautifulSoup(full_description) nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1] description = toUnicode(nfo_pre.text) if nfo_pre else '' except: @@ -143,8 +121,8 @@ class ILoveTorrents(TorrentMagnetProvider): item['description'] = description return item - - def loginSuccess(self, output): - return 'logout.php' in output.lower() - loginCheckSuccess = loginSuccess + def loginSuccess(self, output): + return 'logout.php' in output.lower() + + loginCheckSuccess = loginSuccess