From 843ff0eabc537f6bf465772545caa9de2a9e29c9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 26 Mar 2013 22:02:43 +0100 Subject: [PATCH 01/25] Add some default Newznab providers --- couchpotato/core/providers/nzb/newznab/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py index 2349590f..625da1ac 100644 --- a/couchpotato/core/providers/nzb/newznab/__init__.py +++ b/couchpotato/core/providers/nzb/newznab/__init__.py @@ -12,9 +12,10 @@ config = [{ 'list': 'nzb_providers', 'name': 'newznab', 'order': 10, - 'description': 'Enable NewzNab providers such as NZB.su, \ + 'description': 'Enable NewzNab such as NZB.su, \ NZBs.org, DOGnzb.cr, \ - Spotweb or NZBGeek', + Spotweb, NZBGeek, \ + SmackDown, NZBFinder', 'wizard': True, 'options': [ { @@ -23,18 +24,18 @@ config = [{ }, { 'name': 'use', - 'default': '0,0,0,0' + 'default': '0,0,0,0,0,0' }, { 'name': 'host', - 'default': 'nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info', + 'default': 'nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', 'description': 'The hostname of your newznab provider', }, { 'name': 'extra_score', 'advanced': True, 'label': 'Extra Score', - 'default': '0,0,0,0', + 'default': '0,0,0,0,0,0', 'description': 'Starting score for each release found via this provider.', }, { From 1df05cf34471de4897a9cc81cba6d2fb50e73dee Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 28 Mar 2013 21:51:42 +0100 Subject: [PATCH 02/25] Don't use directory when it's empty. fix #1448 --- .../core/downloaders/transmission/__init__.py | 2 +- .../core/downloaders/transmission/main.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 210a0d9e..702e61aa 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -41,7 +41,7 @@ config = [{ { 'name': 'directory', 'type': 'directory', - 'description': 'Where should Transmission saved the downloaded files?', + 'description': 'Download to this directory. Keep empty for default Transmission download directory.', }, { 'name': 'ratio', diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 6c4607fb..9b71d6b2 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -27,17 +27,19 @@ class Transmission(Downloader): return False # Set parameters for Transmission - folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] - folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep) - - # Create the empty folder to download too - self.makeDir(folder_path) - params = { 'paused': self.conf('paused', default = 0), - 'download-dir': folder_path } + if len(self.conf('directory', default = '')) > 0: + folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] + folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep) + + # Create the empty folder to download too + self.makeDir(folder_path) + + params['download-dir'] = folder_path + torrent_params = {} if self.conf('ratio'): torrent_params = { From eab9a735a9e3be5a146f2f5667f7886c4061e87c Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 29 Mar 2013 12:39:06 +0100 Subject: [PATCH 03/25] Show real transmission error. --- couchpotato/core/downloaders/transmission/main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 9b71d6b2..2383685c 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -6,6 +6,7 @@ import httplib import json import os.path import re +import traceback import urllib2 log = CPLog(__name__) @@ -60,13 +61,16 @@ class Transmission(Downloader): else: remote_torrent = trpc.add_torrent_file(b64encode(filedata), arguments = params) + if not remote_torrent: + return False + # Change settings of added torrents - if torrent_params: + elif torrent_params: trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) - except Exception, err: - log.error('Failed to change settings for transfer: %s', err) + except: + log.error('Failed to change settings for transfer: %s', traceback.format_exc()) return False From 4cedccb178e5aae35ff06b6b15a89ad8d5fc8201 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 29 Mar 2013 12:39:15 +0100 Subject: [PATCH 04/25] Move over html template --- couchpotato/templates/_desktop.html | 78 ---------------------------- couchpotato/templates/_mobile.html | 0 couchpotato/templates/index.html | 79 ++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 79 deletions(-) delete mode 100644 couchpotato/templates/_desktop.html delete mode 100644 couchpotato/templates/_mobile.html diff --git a/couchpotato/templates/_desktop.html b/couchpotato/templates/_desktop.html deleted file mode 100644 index 1d618066..00000000 --- a/couchpotato/templates/_desktop.html +++ /dev/null @@ -1,78 +0,0 @@ - - - - {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} - {% endfor %} - {% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'front', single = True) %} - {% endfor %} - - {% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'head', single = True) %} - {% endfor %} - {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'head', single = True) %} - {% endfor %} - - - - - - - - CouchPotato - - - \ No newline at end of file diff --git a/couchpotato/templates/_mobile.html b/couchpotato/templates/_mobile.html deleted file mode 100644 index e69de29b..00000000 diff --git a/couchpotato/templates/index.html b/couchpotato/templates/index.html index 217d6bfc..1d618066 100644 --- a/couchpotato/templates/index.html +++ b/couchpotato/templates/index.html @@ -1 +1,78 @@ -{% extends "_desktop.html" %} + + + + {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} + {% endfor %} + {% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'front', single = True) %} + {% endfor %} + + {% for url in fireEvent('clientscript.get_scripts', as_html = True, location = 'head', single = True) %} + {% endfor %} + {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'head', single = True) %} + {% endfor %} + + + + + + + + CouchPotato + + + \ No newline at end of file From 6a18e546ca1232b3880612aeed583a4d7b4d5701 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Mon, 25 Mar 2013 23:35:23 +0100 Subject: [PATCH 05/25] Add and make use of renamer.scanfolder in downloaders This is the next step in closing the loop between the downloaders and CPS. The download_id and folder from the downloader are used to find the downloaded files and start the renamer. This is done by adding an additional API call: renamer.scanfolder. I tested this for SabNZBd only (!) and everything works as expected. I also added transmission with thanks @manusfreedom for setting this up in f1cf0d91da. @manusfreedom, please check if this works as expected. Note that transmission now has a feature which is not in the other torrent providers: it waits until the seed ratio is met and then removes the torrent. I opened a topic in the forum to discuss how we want to deal with torrents: https://couchpota.to/forum/thread-1704.html --- couchpotato/core/downloaders/base.py | 1 + couchpotato/core/downloaders/nzbget/main.py | 1 + .../core/downloaders/nzbvortex/main.py | 3 +- couchpotato/core/downloaders/sabnzbd/main.py | 12 +- .../core/downloaders/transmission/__init__.py | 11 +- .../core/downloaders/transmission/main.py | 115 +++++++++++++++++- couchpotato/core/downloaders/utorrent/main.py | 4 +- couchpotato/core/plugins/renamer/main.py | 69 ++++++++++- couchpotato/core/plugins/scanner/main.py | 55 ++++++--- 9 files changed, 235 insertions(+), 36 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 69449249..8ccc1369 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -150,6 +150,7 @@ class StatusList(list): 'id': 0, 'status': 'busy', 'downloader': self.provider.getName(), + 'folder': '', } return mergeDicts(defaults, result) diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index 1bc54fd5..d0533d5c 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -120,6 +120,7 @@ class NZBGet(Downloader): 'status': 'completed' if item['ParStatus'] == 'SUCCESS' and item['ScriptStatus'] == 'SUCCESS' else 'failed', 'original_status': item['ParStatus'] + ', ' + item['ScriptStatus'], 'timeleft': str(timedelta(seconds = 0)), + 'folder': item['DestDir'] }) return statuses diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index e6cbd027..f1f8acc6 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -55,7 +55,8 @@ class NZBVortex(Downloader): 'name': item['uiTitle'], 'status': status, 'original_status': item['state'], - 'timeleft':-1, + 'timeleft': -1, + 'folder': item['destinationPath'], }) return statuses diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 8b374698..f2f217a1 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.helpers.variable import cleanHost, mergeDicts from couchpotato.core.logger import CPLog from couchpotato.environment import Env +from datetime import timedelta from urllib2 import URLError import json import traceback @@ -46,19 +47,15 @@ class Sabnzbd(Downloader): log.error('Failed sending release, use API key, NOT the NZB key: %s', traceback.format_exc(0)) return False - if sab_data.get('error'): - log.error('Error getting data from SABNZBd: %s', sab_data.get('error')) - return False - log.debug('Result from SAB: %s', sab_data) - if sab_data.get('status'): + if sab_data.get('status') and not sab_data.get('error'): log.info('NZB sent to SAB successfully.') if filedata: return self.downloadReturnId(sab_data.get('nzo_ids')[0]) else: return True else: - log.error(sab_data) + log.error('Error getting data from SABNZBd: %s', sab_data) return False def getAllDownloadStatus(self): @@ -109,7 +106,8 @@ class Sabnzbd(Downloader): 'name': item['name'], 'status': status, 'original_status': item['status'], - 'timeleft': 0, + 'timeleft': str(timedelta(seconds = 0)), + 'folder': item['storage'], }) return statuses diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 210a0d9e..0fd783b8 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -41,15 +41,22 @@ config = [{ { 'name': 'directory', 'type': 'directory', - 'description': 'Where should Transmission saved the downloaded files?', + 'description': 'Where should Transmission save the downloaded files?', }, { 'name': 'ratio', 'default': 10, - 'type': 'int', + 'type': 'float', 'advanced': True, 'description': 'Stop transfer when reaching ratio', }, + { + 'name': 'ratiomode', + 'default': 0, + 'type': 'int', + 'advanced': True, + 'description': '0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.', + }, { 'name': 'manual', 'default': 0, diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 6c4607fb..24f7649d 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -1,12 +1,15 @@ from base64 import b64encode -from couchpotato.core.downloaders.base import Downloader +from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog +from couchpotato.environment import Env +from datetime import timedelta import httplib import json import os.path import re import urllib2 +import shutil log = CPLog(__name__) @@ -18,7 +21,7 @@ class Transmission(Downloader): def download(self, data, movie, filedata = None): - log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) + log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') @@ -30,7 +33,7 @@ class Transmission(Downloader): folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep) - # Create the empty folder to download too + # Create the empty folder to download to self.makeDir(folder_path) params = { @@ -42,7 +45,7 @@ class Transmission(Downloader): if self.conf('ratio'): torrent_params = { 'seedRatioLimit': self.conf('ratio'), - 'seedRatioMode': self.conf('ratio') + 'seedRatioMode': self.conf('ratiomode') } if not filedata and data.get('type') == 'torrent': @@ -62,11 +65,99 @@ class Transmission(Downloader): if torrent_params: trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) + log.info('Torrent sent to Transmission successfully.') return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) except Exception, err: log.error('Failed to change settings for transfer: %s', err) return False + def getAllDownloadStatus(self): + + log.debug('Checking Transmission download status.') + + # Load host from config and split out port. + host = self.conf('host').split(':') + if not isInt(host[1]): + log.error('Config properties are not filled in correctly, port is missing.') + return False + + # Go through Queue + try: + trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + return_params = { + 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isFinished', 'downloadDir', 'uploadRatio'] + } + queue = trpc.get_alltorrents(return_params) + + except Exception, err: + log.error('Failed getting queue: %s', err) + return False + + statuses = StatusList(self) + + # Get torrents status + # CouchPotato Status + #status = 'busy' + #status = 'failed' + #status = 'completed' + # Transmission Status + #status = 0 => "Torrent is stopped" + #status = 1 => "Queued to check files" + #status = 2 => "Checking files" + #status = 3 => "Queued to download" + #status = 4 => "Downloading" + #status = 4 => "Queued to seed" + #status = 6 => "Seeding" + #To do : + # add checking file + # manage no peer in a range time => fail + + for item in queue['torrents']: + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'] )) + + if not os.path.isdir(Env.setting('from', 'renamer')): + log.debug('Renamer folder has to exist.') + return + + if (item['percentDone'] * 100) >= 100 and (item['status'] == 6 or item['status'] == 0) and item['uploadRatio'] > self.conf('ratio'): + try: + trpc.stop_torrent(item['hashString'], {}) + + if not os.path.isdir(item['downloadDir']): + raise Exception('Missing folder: %s' % item['downloadDir']) + + else: + log.info('Moving folder from "%s" to "%s"', (item['downloadDir'], Env.setting('from', 'renamer'))) + shutil.move(item['downloadDir'], Env.setting('from', 'renamer')) + + statuses.append({ + 'id': item['hashString'], + 'name': item['downloadDir'], + 'status': 'completed', + 'original_status': item['status'], + 'timeleft': str(timedelta(seconds = 0)), + 'folder': os.path.join(Env.setting('from', 'renamer'), os.path.basename(item['downloadDir'].rstrip(os.path.sep))), + }) + trpc.remove_torrent(item['hashString'], True, {}) + except Exception, err: + log.error('Failed to stop and remove torrent "%s" with error: %s', (item['name'], err)) + statuses.append({ + 'id': item['hashString'], + 'name': item['downloadDir'], + 'status': 'failed', + 'original_status': item['status'], + 'timeleft': str(timedelta(seconds = 0)), + }) + else: + statuses.append({ + 'id': item['hashString'], + 'name': item['downloadDir'], + 'status': 'busy', + 'original_status': item['status'], + 'timeleft': str(timedelta(seconds = item['eta'])), # Is ETA in seconds?? + }) + + return statuses class TransmissionRPC(object): @@ -97,6 +188,7 @@ class TransmissionRPC(object): try: open_request = urllib2.urlopen(request) response = json.loads(open_request.read()) + log.debug('request: %s', json.dumps(ojson)) log.debug('response: %s', json.dumps(response)) if response['result'] == 'success': log.debug('Transmission action successfull') @@ -146,3 +238,18 @@ class TransmissionRPC(object): arguments['ids'] = torrent_id post_data = {'arguments': arguments, 'method': 'torrent-set', 'tag': self.tag} return self._request(post_data) + + def get_alltorrents(self, arguments): + post_data = {'arguments': arguments, 'method': 'torrent-get', 'tag': self.tag} + return self._request(post_data) + + def stop_torrent(self, torrent_id, arguments): + arguments['ids'] = torrent_id + post_data = {'arguments': arguments, 'method': 'torrent-stop', 'tag': self.tag} + return self._request(post_data) + + def remove_torrent(self, torrent_id, remove_local_data, arguments): + arguments['ids'] = torrent_id + arguments['delete-local-data'] = remove_local_data + post_data = {'arguments': arguments, 'method': 'torrent-remove', 'tag': self.tag} + return self._request(post_data) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index c64db135..ca0a0dae 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -5,6 +5,7 @@ from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.logger import CPLog from hashlib import sha1 from multipartpost import MultipartPostHandler +from datetime import timedelta import cookielib import httplib import json @@ -118,7 +119,8 @@ class uTorrent(Downloader): 'name': item[2], 'status': status, 'original_status': item[1], - 'timeleft': item[10], + 'timeleft': str(timedelta(seconds = item[10])), + 'folder': '', #no fucntion to get folder, but can be deduced with getSettings function. }) return statuses diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index aba20724..ad961571 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -2,7 +2,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss -from couchpotato.core.helpers.request import jsonified +from couchpotato.core.helpers.request import getParams, jsonified, getParam from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ getImdb from couchpotato.core.logger import CPLog @@ -31,6 +31,17 @@ class Renamer(Plugin): }) addEvent('renamer.scan', self.scan) + + addApiView('renamer.scanfolder', self.scanfolderView, docs = { + 'desc': 'For the renamer to check for new files to rename in a specified folder', + 'params': { + 'movie_folder': {'desc': 'The folder of the movie to scan'}, + 'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'}, + 'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'}, + }, + }) + + addEvent('renamer.scanfolder', self.scanfolder) addEvent('renamer.check_snatched', self.checkSnatched) addEvent('app.load', self.scan) @@ -51,6 +62,26 @@ class Renamer(Plugin): }) def scan(self): + self.scanfolder() + + def scanfolderView(self): + + params = getParams() + movie_folder = params.get('movie_folder', None) + downloader = params.get('downloader', None) + download_id = params.get('download_id', None) + + fireEventAsync('renamer.scanfolder', + movie_folder = movie_folder, + downloader = downloader, + download_id = download_id + ) + + return jsonified({ + 'success': True + }) + + def scanfolder(self, movie_folder = None, downloader = None, download_id = None): if self.isDisabled(): return @@ -59,18 +90,43 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return + self.renaming_started = True + # Check to see if the "to" folder is inside the "from" folder. - if not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): + if movie_folder and not os.path.isdir(movie_folder): # or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): log.debug('"To" and "From" have to exist.') return elif self.conf('from') in self.conf('to'): log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.') return + elif (movie_folder and movie_folder in [self.conf('to'), self.conf('from')]): + log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.') + return - groups = fireEvent('scanner.scan', folder = self.conf('from'), single = True) + # make sure the movie folder name is included in the search + folder = None + movie_files = [] + if movie_folder: + log.info('Scanning movie folder %s...', movie_folder) + movie_folder = movie_folder.rstrip(os.path.sep) + folder = os.path.dirname(movie_folder) - self.renaming_started = True + # Get all files from the specified folder + try: + for root, folders, names in os.walk(movie_folder): + movie_files.extend([os.path.join(root, name) for name in names]) + except: + log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, downloader = downloader, download_id = download_id, single = True) + + # Make sure only one movie was found if a download ID is provided + if downloader and download_id and not len(groups) == 1: + log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_id, len(groups))) + downloader = None + download_id = None + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, single = True) + destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') @@ -597,7 +653,10 @@ class Renamer(Plugin): db.commit() elif item['status'] == 'completed': log.info('Download of %s completed!', item['name']) - scan_required = True + if item['id'] and item['downloader'] and item['folder']: + fireEventAsync('renamer.scanfolder', movie_folder = item['folder'], downloader = item['downloader'], download_id = item['id']) + else: + scan_required = True found = True break diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index b2ac938c..6e9a9b6f 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -4,7 +4,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss from couchpotato.core.helpers.variable import getExt, getImdb, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import File, Movie +from couchpotato.core.settings.model import File, Movie, Release, ReleaseInfo from enzyme.exceptions import NoParserError, ParseError from guessit import guess_movie_info from subliminal.videos import Video @@ -101,7 +101,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, simple = False, newer_than = 0, on_found = None): + def scan(self, folder = None, files = None, downloader = None, download_id = None, simple = False, newer_than = 0, on_found = None): folder = ss(os.path.normpath(folder)) @@ -119,8 +119,7 @@ class Scanner(Plugin): try: files = [] for root, dirs, walk_files in os.walk(folder): - for filename in walk_files: - files.append(os.path.join(root, filename)) + files.extend(os.path.join(root, filename) for filename in walk_files) except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) else: @@ -129,6 +128,24 @@ class Scanner(Plugin): db = get_session() + # Get the release with the downloader ID that was downloded by the downloader + download_quality = None + download_imdb_id = None + if downloader and download_id: + # NOTE TO RUUD: Don't really know how to do this better... but there must be a way...? + rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader) + rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id) + for rlsnfo_dwnld in rlsnfo_dwnlds: + for rlsnfo_id in rlsnfo_ids: + if rlsnfo_id.release == rlsnfo_dwnld.release: + rls = rlsnfo_id.release + + if rls: + download_imdb_id = rls.movie.library.identifier + download_quality = rls.quality.identifier + else: + log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) + for file_path in files: if not os.path.exists(file_path): @@ -346,7 +363,7 @@ class Scanner(Plugin): continue log.debug('Getting metadata for %s', identifier) - group['meta_data'] = self.getMetaData(group, folder = folder) + group['meta_data'] = self.getMetaData(group, folder = folder, download_quality = download_quality) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {} @@ -376,7 +393,7 @@ class Scanner(Plugin): del group['unsorted_files'] # Determine movie - group['library'] = self.determineMovie(group) + group['library'] = self.determineMovie(group, download_imdb_id = download_imdb_id) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: @@ -401,7 +418,7 @@ class Scanner(Plugin): return processed_movies - def getMetaData(self, group, folder = ''): + def getMetaData(self, group, folder = '', download_quality = None): data = {} files = list(group['files']['movie']) @@ -423,10 +440,14 @@ class Scanner(Plugin): if data.get('audio'): break + # Use the quality guess first, if that failes use the quality we wanted to download data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True) if not data['quality']: - data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) - + if download_quality: + data['quality'] = fireEvent('quality.single', download_quality, single = True) + else: + data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) + data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD' filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) @@ -501,17 +522,19 @@ class Scanner(Plugin): return detected_languages - def determineMovie(self, group): - imdb_id = None + def determineMovie(self, group, download_imdb_id = None): + # Get imdb id from downloader + imdb_id = download_imdb_id files = group['files'] # Check for CP(imdb_id) string in the file paths - for cur_file in files['movie']: - imdb_id = self.getCPImdb(cur_file) - if imdb_id: - log.debug('Found movie via CP tag: %s', cur_file) - break + if not imdb_id: + for cur_file in files['movie']: + imdb_id = self.getCPImdb(cur_file) + if imdb_id: + log.debug('Found movie via CP tag: %s', cur_file) + break # Check and see if nfo contains the imdb-id if not imdb_id: From 0c44c486280c0024dbeeb433eb835a89ebd1d269 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Mar 2013 11:12:37 +0200 Subject: [PATCH 06/25] Notification test failed. closes #1561 Thanks @FredrikWendt --- couchpotato/core/notifications/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/base.py b/couchpotato/core/notifications/base.py index dda3dade..9e80aea8 100644 --- a/couchpotato/core/notifications/base.py +++ b/couchpotato/core/notifications/base.py @@ -44,7 +44,7 @@ class Notification(Provider): def _notify(self, *args, **kwargs): if self.isEnabled(): - self.notify(*args, **kwargs) + return self.notify(*args, **kwargs) def notify(self, message = '', data = {}, listener = None): pass From 8fe60a893c70eef48ab2fd565f558e042d319c2f Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sun, 31 Mar 2013 17:17:36 +0200 Subject: [PATCH 07/25] Update of uTorrent --- couchpotato/core/downloaders/utorrent/main.py | 41 ++++++++++++++++++- couchpotato/core/plugins/renamer/main.py | 2 +- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index ca0a0dae..bca57e71 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -6,6 +6,7 @@ from couchpotato.core.logger import CPLog from hashlib import sha1 from multipartpost import MultipartPostHandler from datetime import timedelta +import os import cookielib import httplib import json @@ -105,6 +106,35 @@ class uTorrent(Downloader): return False statuses = StatusList(self) + download_folder = '' + settings_dict = {} + + try: + data = self.utorrent_api.get_settings() + utorrent_settings = json.loads(data) + + # Create settings dict + for item in utorrent_settings['settings']: + if item[1] == 0: # int + settings_dict[item[0]] = int(item[2] if not item[2].strip() == '' else '0') + elif item[1] == 1: # bool + settings_dict[item[0]] = True if item[2] == 'true' else False + elif item[1] == 2: # string + settings_dict[item[0]] = item[2] + log.debug('uTorrent settings: %s', settings_dict) + + # Get the download path from the uTorrent settings + if settings_dict['dir_completed_download_flag']: + download_folder = settings_dict['dir_completed_download'] + elif settings_dict['dir_active_download_flag']: + download_folder = settings_dict['dir_active_download'] + else: + log.info('No download folder set in uTorrent. Please set a download folder') + return False + + except Exception, err: + log.error('Failed to get settings from uTorrent: %s', err) + return False # Get torrents for item in queue.get('torrents', []): @@ -114,13 +144,18 @@ class uTorrent(Downloader): if item[21] == 'Finished' or item[21] == 'Seeding': status = 'completed' + if settings_dict['dir_add_label']: + release_folder = os.path.join(download_folder, item[11], item[2]) + else: + release_folder = os.path.join(download_folder, item[2]) + statuses.append({ 'id': item[0], 'name': item[2], 'status': status, 'original_status': item[1], 'timeleft': str(timedelta(seconds = item[10])), - 'folder': '', #no fucntion to get folder, but can be deduced with getSettings function. + 'folder': release_folder, }) return statuses @@ -197,3 +232,7 @@ class uTorrentAPI(object): def get_status(self): action = "list=1" return self._request(action) + + def get_settings(self): + action = "action=getsettings" + return self._request(action) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index ad961571..4bd96d79 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -93,7 +93,7 @@ class Renamer(Plugin): self.renaming_started = True # Check to see if the "to" folder is inside the "from" folder. - if movie_folder and not os.path.isdir(movie_folder): # or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): + if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): log.debug('"To" and "From" have to exist.') return elif self.conf('from') in self.conf('to'): From 4cdb99a3839c511afebc40dfb52b81071660d4ca Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Mar 2013 22:42:27 +0200 Subject: [PATCH 08/25] Add dvdscreen to screener quality. fix #1555 --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index c32ad81d..cf0b046c 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -24,7 +24,7 @@ class QualityPlugin(Plugin): {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi']}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip'], 'allow': ['dvdr', 'dvd'], 'ext':['avi', 'mpg', 'mpeg']}, + {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener'], 'allow': ['dvdr', 'dvd'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': [], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, From a83c276aa2861636ce57e323d40e6de5532ec694 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 31 Mar 2013 23:50:47 +0200 Subject: [PATCH 09/25] Schedule start normal --- couchpotato/core/_base/scheduler/main.py | 52 +++++------------------- 1 file changed, 11 insertions(+), 41 deletions(-) diff --git a/couchpotato/core/_base/scheduler/main.py b/couchpotato/core/_base/scheduler/main.py index 4102552e..2c97e1b4 100644 --- a/couchpotato/core/_base/scheduler/main.py +++ b/couchpotato/core/_base/scheduler/main.py @@ -16,51 +16,19 @@ class Scheduler(Plugin): addEvent('schedule.cron', self.cron) addEvent('schedule.interval', self.interval) - addEvent('schedule.start', self.start) - addEvent('schedule.restart', self.start) - - addEvent('app.load', self.start) + addEvent('schedule.remove', self.remove) self.sched = Sched(misfire_grace_time = 60) - - def remove(self, identifier): - for type in ['interval', 'cron']: - try: - self.sched.unschedule_job(getattr(self, type)[identifier]['job']) - log.debug('%s unscheduled %s', (type.capitalize(), identifier)) - except: - pass - - def start(self): - - # Stop all running - self.stop() - - # Crons - for identifier in self.crons: - try: - self.remove(identifier) - cron = self.crons[identifier] - job = self.sched.add_cron_job(cron['handle'], day = cron['day'], hour = cron['hour'], minute = cron['minute']) - cron['job'] = job - except ValueError, e: - log.error('Failed adding cronjob: %s', e) - - # Intervals - for identifier in self.intervals: - try: - self.remove(identifier) - interval = self.intervals[identifier] - job = self.sched.add_interval_job(interval['handle'], hours = interval['hours'], minutes = interval['minutes'], seconds = interval['seconds']) - interval['job'] = job - except ValueError, e: - log.error('Failed adding interval cronjob: %s', e) - - # Start it - log.debug('Starting scheduler') self.sched.start() self.started = True - log.debug('Scheduler started') + + def remove(self, identifier): + for cron_type in ['intervals', 'crons']: + try: + self.sched.unschedule_job(getattr(self, cron_type)[identifier]['job']) + log.debug('%s unscheduled %s', (cron_type.capitalize(), identifier)) + except: + pass def doShutdown(self): super(Scheduler, self).doShutdown() @@ -82,6 +50,7 @@ class Scheduler(Plugin): 'day': day, 'hour': hour, 'minute': minute, + 'job': self.sched.add_cron_job(handle, day = day, hour = hour, minute = minute) } def interval(self, identifier = '', handle = None, hours = 0, minutes = 0, seconds = 0): @@ -93,4 +62,5 @@ class Scheduler(Plugin): 'hours': hours, 'minutes': minutes, 'seconds': seconds, + 'job': self.sched.add_interval_job(handle, hours = hours, minutes = minutes, seconds = seconds) } From 207e846ae6fe2147a32e96e02848e781b270695b Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Apr 2013 00:06:15 +0200 Subject: [PATCH 10/25] Check crons after saving settings. fix #1556 & #1557 --- couchpotato/core/_base/updater/main.py | 10 +++++++++- couchpotato/core/plugins/automation/main.py | 7 ++++++- couchpotato/core/plugins/renamer/main.py | 20 ++++++++++++++++---- couchpotato/core/plugins/searcher/main.py | 7 ++++++- couchpotato/core/settings/__init__.py | 3 +++ 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 18d2c303..07031556 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -32,7 +32,6 @@ class Updater(Plugin): else: self.updater = SourceUpdater() - fireEvent('schedule.interval', 'updater.check', self.autoUpdate, hours = 6) addEvent('app.load', self.autoUpdate) addEvent('updater.info', self.info) @@ -52,6 +51,15 @@ class Updater(Plugin): 'return': {'type': 'see updater.info'} }) + addEvent('setting.save.updater.enabled.after', self.setCrons) + + def setCrons(self): + + fireEvent('schedule.remove', 'updater.check', single = True) + if self.isEnabled(): + fireEvent('schedule.interval', 'updater.check', self.autoUpdate, hours = 6) + self.autoUpdate() # Check after enabling + def autoUpdate(self): if self.check() and self.conf('automatic') and not self.updater.update_failed: if self.updater.doUpdate(): diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index f4ede40d..67bae1d1 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -10,11 +10,16 @@ class Automation(Plugin): def __init__(self): - fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12)) + addEvent('app.load', self.setCrons) if not Env.get('dev'): addEvent('app.load', self.addMovies) + addEvent('setting.save.automation.hour.after', self.setCrons) + + def setCrons(self): + fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12)) + def addMovies(self): movies = fireEvent('automation.get_movies', merge = True) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index aba20724..06487aae 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -35,12 +35,24 @@ class Renamer(Plugin): addEvent('app.load', self.scan) addEvent('app.load', self.checkSnatched) + addEvent('app.load', self.setCrons) - if self.conf('run_every') > 0: - fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every')) + # Enable / disable interval + addEvent('setting.save.renamer.enabled.after', self.setCrons) + addEvent('setting.save.renamer.run_every.after', self.setCrons) + addEvent('setting.save.renamer.force_every.after', self.setCrons) - if self.conf('force_every') > 0: - fireEvent('schedule.interval', 'renamer.check_snatched_forced', self.scan, hours = self.conf('force_every')) + def setCrons(self): + + fireEvent('schedule.remove', 'renamer.check_snatched') + if self.isEnabled() and self.conf('run_every') > 0: + fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every'), single = True) + + fireEvent('schedule.remove', 'renamer.check_snatched_forced') + if self.isEnabled() and self.conf('force_every') > 0: + fireEvent('schedule.interval', 'renamer.check_snatched_forced', self.scan, hours = self.conf('force_every'), single = True) + + return True def scanView(self): diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index a34b0b47..8a23ca4b 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -50,7 +50,12 @@ class Searcher(Plugin): }"""}, }) - # Schedule cronjob + addEvent('app.load', self.setCrons) + addEvent('setting.save.searcher.cron_day.after', self.setCrons) + addEvent('setting.save.searcher.cron_hour.after', self.setCrons) + addEvent('setting.save.searcher.cron_minute.after', self.setCrons) + + def setCrons(self): fireEvent('schedule.cron', 'searcher.all', self.allMovies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) def allMoviesView(self): diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index 00f77a64..45443fcb 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -189,6 +189,9 @@ class Settings(object): self.set(section, option, (new_value if new_value else value).encode('unicode_escape')) self.save() + # After save (for re-interval etc) + fireEvent('setting.save.%s.%s.after' % (section, option), single = True) + return jsonified({ 'success': True, }) From 45b9919f67992b26b6f3398c2695864d389b9866 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Mon, 1 Apr 2013 10:13:24 +0200 Subject: [PATCH 11/25] Added some debugging info --- couchpotato/core/plugins/scanner/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 6e9a9b6f..c35d0418 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -525,6 +525,8 @@ class Scanner(Plugin): def determineMovie(self, group, download_imdb_id = None): # Get imdb id from downloader imdb_id = download_imdb_id + if imdb_id: + log.debug('Found movie via imdb id from it\'s download id: %s', download_imdb_id) files = group['files'] From 2851781a72e7e0708e5b48f3f6f5c15f654469b3 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Mon, 1 Apr 2013 12:56:08 +0200 Subject: [PATCH 12/25] Move around items between scanner and renamer --- couchpotato/core/plugins/renamer/main.py | 33 ++++++++++++++++-------- couchpotato/core/plugins/scanner/main.py | 28 ++++++-------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 4bd96d79..96d84b83 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -2,12 +2,12 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss -from couchpotato.core.helpers.request import getParams, jsonified, getParam +from couchpotato.core.helpers.request import getParams, jsonified from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ getImdb from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import Library, File, Profile, Release +from couchpotato.core.settings.model import Library, File, Profile, Release, ReleaseInfo from couchpotato.environment import Env import errno import os @@ -118,15 +118,28 @@ class Renamer(Plugin): except: log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) - groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, downloader = downloader, download_id = download_id, single = True) + db = get_session() - # Make sure only one movie was found if a download ID is provided - if downloader and download_id and not len(groups) == 1: - log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_id, len(groups))) - downloader = None - download_id = None - groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, single = True) + # Get the release with the downloader ID that was downloded by the downloader + download_quality = None + download_imdb_id = None + if downloader and download_id: + # NOTE TO RUUD: Don't really know how to do this better... but there must be a way...? + rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader) + rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id) + for rlsnfo_dwnld in rlsnfo_dwnlds: + for rlsnfo_id in rlsnfo_ids: + if rlsnfo_id.release == rlsnfo_dwnld.release: + rls = rlsnfo_id.release + + if rls: + download_imdb_id = rls.movie.library.identifier + download_quality = rls.quality.identifier + else: + log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, download_quality = download_quality, download_imdb_id = download_imdb_id, single = True) + destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') @@ -140,8 +153,6 @@ class Renamer(Plugin): downloaded_status = fireEvent('status.get', 'downloaded', single = True) snatched_status = fireEvent('status.get', 'snatched', single = True) - db = get_session() - for group_identifier in groups: group = groups[group_identifier] diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index c35d0418..d714481e 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -4,7 +4,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss from couchpotato.core.helpers.variable import getExt, getImdb, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import File, Movie, Release, ReleaseInfo +from couchpotato.core.settings.model import File, Movie from enzyme.exceptions import NoParserError, ParseError from guessit import guess_movie_info from subliminal.videos import Video @@ -101,7 +101,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, downloader = None, download_id = None, simple = False, newer_than = 0, on_found = None): + def scan(self, folder = None, files = None, download_imdb_id = None, download_quality = None, simple = False, newer_than = 0, on_found = None): folder = ss(os.path.normpath(folder)) @@ -128,24 +128,6 @@ class Scanner(Plugin): db = get_session() - # Get the release with the downloader ID that was downloded by the downloader - download_quality = None - download_imdb_id = None - if downloader and download_id: - # NOTE TO RUUD: Don't really know how to do this better... but there must be a way...? - rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader) - rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id) - for rlsnfo_dwnld in rlsnfo_dwnlds: - for rlsnfo_id in rlsnfo_ids: - if rlsnfo_id.release == rlsnfo_dwnld.release: - rls = rlsnfo_id.release - - if rls: - download_imdb_id = rls.movie.library.identifier - download_quality = rls.quality.identifier - else: - log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) - for file_path in files: if not os.path.exists(file_path): @@ -330,6 +312,12 @@ class Scanner(Plugin): valid_files[identifier] = group del movie_files + + # Make sure only one movie was found if a download ID is provided + if download_imdb_id and download_quality and not len(valid_files) == 1: + log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_imdb_id, len(valid_files))) + download_imdb_id = None + download_quality = None # Determine file types processed_movies = {} From 33a6a7d3a08b6d3502b89d838b06ab370aad0291 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Apr 2013 17:31:04 +0200 Subject: [PATCH 13/25] CP Provider API Identifier --- couchpotato/core/providers/movie/couchpotatoapi/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/providers/movie/couchpotatoapi/main.py b/couchpotato/core/providers/movie/couchpotatoapi/main.py index fc16e8ab..8415c2d0 100644 --- a/couchpotato/core/providers/movie/couchpotatoapi/main.py +++ b/couchpotato/core/providers/movie/couchpotatoapi/main.py @@ -5,6 +5,7 @@ from couchpotato.core.helpers.request import jsonified, getParams from couchpotato.core.logger import CPLog from couchpotato.core.providers.movie.base import MovieProvider from couchpotato.core.settings.model import Movie +from couchpotato.environment import Env import time log = CPLog(__name__) @@ -96,4 +97,5 @@ class CouchPotatoApi(MovieProvider): 'X-CP-Version': fireEvent('app.version', single = True), 'X-CP-API': self.api_version, 'X-CP-Time': time.time(), + 'X-CP-Identifier': '+%s' % Env.setting('api_key', 'core')[:10], # Use first 10 as identifier, so we don't need to use IP address in api stats } From c7ee8a063560529fd7c52d5bf5c98cd77fe23c28 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Apr 2013 20:11:49 +0200 Subject: [PATCH 14/25] Don't use object in correctmovie event --- couchpotato/core/plugins/searcher/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 8a23ca4b..ad33c6ff 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -371,7 +371,7 @@ class Searcher(Plugin): return search_types - def correctMovie(self, nzb = {}, movie = {}, quality = {}, **kwargs): + def correctMovie(self, nzb = None, movie = None, quality = None, **kwargs): imdb_results = kwargs.get('imdb_results', False) retention = Env.setting('retention', section = 'nzb') From 3fe7d2ea154bba95c9e26384df877692fa5c0ff2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Apr 2013 21:18:29 +0200 Subject: [PATCH 15/25] Download id cleanup --- couchpotato/core/plugins/renamer/main.py | 65 ++++++++---------------- couchpotato/core/plugins/scanner/main.py | 30 +++++------ 2 files changed, 37 insertions(+), 58 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index de35943e..e3291f64 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -7,7 +7,8 @@ from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ getImdb from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import Library, File, Profile, Release, ReleaseInfo +from couchpotato.core.settings.model import Library, File, Profile, Release, \ + ReleaseInfo from couchpotato.environment import Env import errno import os @@ -27,21 +28,15 @@ class Renamer(Plugin): def __init__(self): addApiView('renamer.scan', self.scanView, docs = { - 'desc': 'For the renamer to check for new files to rename', - }) - - addEvent('renamer.scan', self.scan) - - addApiView('renamer.scanfolder', self.scanfolderView, docs = { - 'desc': 'For the renamer to check for new files to rename in a specified folder', + 'desc': 'For the renamer to check for new files to rename in a folder', 'params': { - 'movie_folder': {'desc': 'The folder of the movie to scan'}, + 'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'}, 'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'}, 'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'}, }, }) - addEvent('renamer.scanfolder', self.scanfolder) + addEvent('renamer.scan', self.scan) addEvent('renamer.check_snatched', self.checkSnatched) addEvent('app.load', self.scan) @@ -67,33 +62,22 @@ class Renamer(Plugin): def scanView(self): - fireEventAsync('renamer.scan') - - return jsonified({ - 'success': True - }) - - def scan(self): - self.scanfolder() - - def scanfolderView(self): - params = getParams() movie_folder = params.get('movie_folder', None) downloader = params.get('downloader', None) download_id = params.get('download_id', None) - fireEventAsync('renamer.scanfolder', - movie_folder = movie_folder, - downloader = downloader, + fireEventAsync('renamer.scan', + movie_folder = movie_folder, + downloader = downloader, download_id = download_id ) return jsonified({ 'success': True }) - - def scanfolder(self, movie_folder = None, downloader = None, download_id = None): + + def scan(self, movie_folder = None, downloader = None, download_id = None): if self.isDisabled(): return @@ -116,7 +100,7 @@ class Renamer(Plugin): return # make sure the movie folder name is included in the search - folder = None + folder = None movie_files = [] if movie_folder: log.info('Scanning movie folder %s...', movie_folder) @@ -133,24 +117,19 @@ class Renamer(Plugin): db = get_session() # Get the release with the downloader ID that was downloded by the downloader - download_quality = None - download_imdb_id = None - if downloader and download_id: - # NOTE TO RUUD: Don't really know how to do this better... but there must be a way...? - rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader) - rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id) - for rlsnfo_dwnld in rlsnfo_dwnlds: - for rlsnfo_id in rlsnfo_ids: - if rlsnfo_id.release == rlsnfo_dwnld.release: - rls = rlsnfo_id.release + download_info = None + if download_id: + rls_info = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id).first() - if rls: - download_imdb_id = rls.movie.library.identifier - download_quality = rls.quality.identifier + if rls_info: + download_info = { + 'imdb_id': rls_info.release.movie.library.identifier, + 'quality': rls_info.release.quality.identifier, + } else: log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) - - groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, download_quality = download_quality, download_imdb_id = download_imdb_id, single = True) + + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, download_info = download_info, single = True) destination = self.conf('to') folder_name = self.conf('folder_name') @@ -677,7 +656,7 @@ class Renamer(Plugin): elif item['status'] == 'completed': log.info('Download of %s completed!', item['name']) if item['id'] and item['downloader'] and item['folder']: - fireEventAsync('renamer.scanfolder', movie_folder = item['folder'], downloader = item['downloader'], download_id = item['id']) + fireEventAsync('renamer.scan', movie_folder = item['folder'], downloader = item['downloader'], download_id = item['id']) else: scan_required = True diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index d714481e..e7a94598 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -101,7 +101,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, download_imdb_id = None, download_quality = None, simple = False, newer_than = 0, on_found = None): + def scan(self, folder = None, files = None, download_info = None, simple = False, newer_than = 0, on_found = None): folder = ss(os.path.normpath(folder)) @@ -312,12 +312,11 @@ class Scanner(Plugin): valid_files[identifier] = group del movie_files - + # Make sure only one movie was found if a download ID is provided - if download_imdb_id and download_quality and not len(valid_files) == 1: - log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_imdb_id, len(valid_files))) - download_imdb_id = None - download_quality = None + if download_info and not len(valid_files) == 1: + log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_info.get('imdb_id'), len(valid_files))) + download_info = None # Determine file types processed_movies = {} @@ -351,7 +350,7 @@ class Scanner(Plugin): continue log.debug('Getting metadata for %s', identifier) - group['meta_data'] = self.getMetaData(group, folder = folder, download_quality = download_quality) + group['meta_data'] = self.getMetaData(group, folder = folder, download_info = download_info) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {} @@ -381,7 +380,7 @@ class Scanner(Plugin): del group['unsorted_files'] # Determine movie - group['library'] = self.determineMovie(group, download_imdb_id = download_imdb_id) + group['library'] = self.determineMovie(group, download_info = download_info) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: @@ -406,7 +405,7 @@ class Scanner(Plugin): return processed_movies - def getMetaData(self, group, folder = '', download_quality = None): + def getMetaData(self, group, folder = '', download_info = None): data = {} files = list(group['files']['movie']) @@ -431,11 +430,11 @@ class Scanner(Plugin): # Use the quality guess first, if that failes use the quality we wanted to download data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True) if not data['quality']: - if download_quality: - data['quality'] = fireEvent('quality.single', download_quality, single = True) + if download_info and download_info.get('quality'): + data['quality'] = fireEvent('quality.single', download_info.get('quality'), single = True) else: data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) - + data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD' filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) @@ -510,11 +509,12 @@ class Scanner(Plugin): return detected_languages - def determineMovie(self, group, download_imdb_id = None): + def determineMovie(self, group, download_info = None): + # Get imdb id from downloader - imdb_id = download_imdb_id + imdb_id = download_info and download_info.get('imdb_id') if imdb_id: - log.debug('Found movie via imdb id from it\'s download id: %s', download_imdb_id) + log.debug('Found movie via imdb id from it\'s download id: %s', download_info.get('imdb_id')) files = group['files'] From 72cc3576d39decf370fa7408ab1cf5bce46fcf10 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Apr 2013 22:51:34 +0200 Subject: [PATCH 16/25] Use @mano3m code to check download_info --- couchpotato/core/plugins/renamer/main.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index e3291f64..a6260d7b 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -118,13 +118,23 @@ class Renamer(Plugin): # Get the release with the downloader ID that was downloded by the downloader download_info = None - if download_id: - rls_info = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id).first() + if download_id and downloader: + rls = None - if rls_info: + rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader).all() + rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id).all() + + for rlsnfo_dwnld in rlsnfo_dwnlds: + for rlsnfo_id in rlsnfo_ids: + if rlsnfo_id.release == rlsnfo_dwnld.release: + rls = rlsnfo_id.release + break + if rls: break + + if rls: download_info = { - 'imdb_id': rls_info.release.movie.library.identifier, - 'quality': rls_info.release.quality.identifier, + 'imdb_id': rls.movie.library.identifier, + 'quality': rls.quality.identifier, } else: log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) @@ -582,6 +592,7 @@ class Renamer(Plugin): loge('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc())) def checkSnatched(self): + if self.checking_snatched: log.debug('Already checking snatched') From 63609bb52cb4a670a45a168eccfa8ad247734c2f Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Fri, 5 Apr 2013 00:05:09 +0200 Subject: [PATCH 17/25] Fix Transmission --- .../core/downloaders/transmission/main.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 31c2f61e..0249bdac 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -128,27 +128,28 @@ class Transmission(Downloader): try: trpc.stop_torrent(item['hashString'], {}) - if not os.path.isdir(item['downloadDir']): - raise Exception('Missing folder: %s' % item['downloadDir']) - + if not os.path.isdir(os.path.join(item['downloadDir'], item['name'])): + raise Exception('Missing folder: %s' % os.path.join(item['downloadDir'], item['name'])) + elif item['downloadDir'].rstrip(os.path.sep) in Env.setting('from', 'renamer').rstrip(os.path.sep): + raise Exception('Transmission download folder should not be the same or in the renamer "from" folder!') else: - log.info('Moving folder from "%s" to "%s"', (item['downloadDir'], Env.setting('from', 'renamer'))) - shutil.move(item['downloadDir'], Env.setting('from', 'renamer')) - + log.info('Moving folder from "%s" to "%s"', (os.path.join(item['downloadDir'], item['name']), Env.setting('from', 'renamer'))) + shutil.move(os.path.join(item['downloadDir'], item['name']), Env.setting('from', 'renamer')) + statuses.append({ 'id': item['hashString'], - 'name': item['downloadDir'], + 'name': item['name'], 'status': 'completed', 'original_status': item['status'], 'timeleft': str(timedelta(seconds = 0)), - 'folder': os.path.join(Env.setting('from', 'renamer'), os.path.basename(item['downloadDir'].rstrip(os.path.sep))), + 'folder': os.path.join(Env.setting('from', 'renamer'), item['name']), }) trpc.remove_torrent(item['hashString'], True, {}) except Exception, err: log.error('Failed to stop and remove torrent "%s" with error: %s', (item['name'], err)) statuses.append({ 'id': item['hashString'], - 'name': item['downloadDir'], + 'name': item['name'], 'status': 'failed', 'original_status': item['status'], 'timeleft': str(timedelta(seconds = 0)), @@ -156,7 +157,7 @@ class Transmission(Downloader): else: statuses.append({ 'id': item['hashString'], - 'name': item['downloadDir'], + 'name': item['name'], 'status': 'busy', 'original_status': item['status'], 'timeleft': str(timedelta(seconds = item['eta'])), # Is ETA in seconds?? From 7f4373e0002450ac3a6709449869c024b264184d Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 5 Apr 2013 12:08:45 +0200 Subject: [PATCH 18/25] Traceback import missing --- couchpotato/core/downloaders/transmission/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 0249bdac..6a5aad10 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -8,8 +8,9 @@ import httplib import json import os.path import re -import urllib2 import shutil +import traceback +import urllib2 log = CPLog(__name__) @@ -118,7 +119,7 @@ class Transmission(Downloader): # manage no peer in a range time => fail for item in queue['torrents']: - log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'] )) + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'])) if not os.path.isdir(Env.setting('from', 'renamer')): log.debug('Renamer folder has to exist.') @@ -129,13 +130,13 @@ class Transmission(Downloader): trpc.stop_torrent(item['hashString'], {}) if not os.path.isdir(os.path.join(item['downloadDir'], item['name'])): - raise Exception('Missing folder: %s' % os.path.join(item['downloadDir'], item['name'])) + raise Exception('Missing folder: %s' % os.path.join(item['downloadDir'], item['name'])) elif item['downloadDir'].rstrip(os.path.sep) in Env.setting('from', 'renamer').rstrip(os.path.sep): raise Exception('Transmission download folder should not be the same or in the renamer "from" folder!') else: log.info('Moving folder from "%s" to "%s"', (os.path.join(item['downloadDir'], item['name']), Env.setting('from', 'renamer'))) shutil.move(os.path.join(item['downloadDir'], item['name']), Env.setting('from', 'renamer')) - + statuses.append({ 'id': item['hashString'], 'name': item['name'], From 57ae06e139ebf501b450e238395ce0ff346eaada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Fri, 5 Apr 2013 13:48:09 +0200 Subject: [PATCH 19/25] initial link support --- couchpotato/core/plugins/renamer/__init__.py | 14 ++++ couchpotato/core/plugins/renamer/main.py | 8 ++- libs/linktastic/README.txt | 19 +++++ libs/linktastic/__init__.py | 0 libs/linktastic/linktastic.py | 76 ++++++++++++++++++++ libs/linktastic/setup.py | 13 ++++ 6 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 libs/linktastic/README.txt create mode 100644 libs/linktastic/__init__.py create mode 100644 libs/linktastic/linktastic.py create mode 100644 libs/linktastic/setup.py diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index 22277ba5..0e46396c 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -112,6 +112,20 @@ config = [{ 'label': 'Separator', 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', }, + { + 'name': 'hardlink', + 'type': 'bool', + 'description': 'Use hard links instead of moving files, ideal for Torrent users. NOTE: From location and to location needs to be on the same drive/partition.', + 'default': False, + 'advanced': True, + }, + { + 'name': 'symlink', + 'type': 'bool', + 'description': 'Use sym links instead of moving files.', + 'default': False, + 'advanced': True, + }, { 'advanced': True, 'name': 'ntfs_permission', diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index a6260d7b..cd3447d2 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -10,6 +10,7 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ ReleaseInfo from couchpotato.environment import Env +import linktastic.linktastic as linktastic import errno import os import re @@ -523,7 +524,12 @@ class Renamer(Plugin): def moveFile(self, old, dest): dest = ss(dest) try: - shutil.move(old, dest) + if self.conf('hardlink') and not self.conf('symlink'): + linktastic.link(old, dest) + elif self.conf('symlink') and not self.conf('hardlink'): + linktastic.symlink(old, dest) + else: + shutil.move(old, dest) try: os.chmod(dest, Env.getPermission('file')) diff --git a/libs/linktastic/README.txt b/libs/linktastic/README.txt new file mode 100644 index 00000000..7fad24db --- /dev/null +++ b/libs/linktastic/README.txt @@ -0,0 +1,19 @@ +Linktastic + +Linktastic is an extension of the os.link and os.symlink functionality provided +by the python language since version 2. Python only supports file linking on +*NIX-based systems, even though it is relatively simple to engineer a solution +to utilize NTFS's built-in linking functionality. Linktastic attempts to unify +linking on the windows platform with linking on *NIX-based systems. + +Usage + +Linktastic is a single python module and can be imported as such. Examples: + +# Hard linking src to dest +import linktastic +linktastic.link(src, dest) + +# Symlinking src to dest +import linktastic +linktastic.symlink(src, dest) diff --git a/libs/linktastic/__init__.py b/libs/linktastic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/linktastic/linktastic.py b/libs/linktastic/linktastic.py new file mode 100644 index 00000000..76687666 --- /dev/null +++ b/libs/linktastic/linktastic.py @@ -0,0 +1,76 @@ +# Linktastic Module +# - A python2/3 compatible module that can create hardlinks/symlinks on windows-based systems +# +# Linktastic is distributed under the MIT License. The follow are the terms and conditions of using Linktastic. +# +# The MIT License (MIT) +# Copyright (c) 2012 Solipsis Development +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import subprocess +from subprocess import CalledProcessError +import os + + +# Prevent spaces from messing with us! +def _escape_param(param): + return '"%s"' % param + + +# Private function to create link on nt-based systems +def _link_windows(src, dest): + try: + subprocess.check_output( + 'cmd /C mklink /H %s %s' % (_escape_param(dest), _escape_param(src)), + stderr=subprocess.STDOUT) + except CalledProcessError as err: + + raise IOError(err.output.decode('utf-8')) + + # TODO, find out what kind of messages Windows sends us from mklink + # print(stdout) + # assume if they ret-coded 0 we're good + + +def _symlink_windows(src, dest): + try: + subprocess.check_output( + 'cmd /C mklink %s %s' % (_escape_param(dest), _escape_param(src)), + stderr=subprocess.STDOUT) + except CalledProcessError as err: + raise IOError(err.output.decode('utf-8')) + + # TODO, find out what kind of messages Windows sends us from mklink + # print(stdout) + # assume if they ret-coded 0 we're good + + +# Create a hard link to src named as dest +# This version of link, unlike os.link, supports nt systems as well +def link(src, dest): + if os.name == 'nt': + _link_windows(src, dest) + else: + os.link(src, dest) + + +# Create a symlink to src named as dest, but don't fail if you're on nt +def symlink(src, dest): + if os.name == 'nt': + _symlink_windows(src, dest) + else: + os.symlink(src, dest) diff --git a/libs/linktastic/setup.py b/libs/linktastic/setup.py new file mode 100644 index 00000000..e15cc2b7 --- /dev/null +++ b/libs/linktastic/setup.py @@ -0,0 +1,13 @@ +from distutils.core import setup + +setup( + name='Linktastic', + version='0.1.0', + author='Jon "Berkona" Monroe', + author_email='solipsis.dev@gmail.com', + py_modules=['linktastic'], + url="http://github.com/berkona/linktastic", + license='MIT License - See http://opensource.org/licenses/MIT for details', + description='Truly platform-independent file linking', + long_description=open('README.txt').read(), +) From 47ddf31f7605724e7071d7b9317c8d170931847a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Fri, 5 Apr 2013 14:07:10 +0200 Subject: [PATCH 20/25] ruud was bulling me ;) --- couchpotato/core/plugins/renamer/__init__.py | 16 +++++----------- couchpotato/core/plugins/renamer/main.py | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index 0e46396c..d80618bd 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -113,17 +113,11 @@ config = [{ 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', }, { - 'name': 'hardlink', - 'type': 'bool', - 'description': 'Use hard links instead of moving files, ideal for Torrent users. NOTE: From location and to location needs to be on the same drive/partition.', - 'default': False, - 'advanced': True, - }, - { - 'name': 'symlink', - 'type': 'bool', - 'description': 'Use sym links instead of moving files.', - 'default': False, + 'name': 'links', + 'default': '0', + 'type': 'dropdown', + 'values': [('Disable', 0), ('Hard link', 'hard'), ('Sym link', 'sym')], + 'description': 'Use hard links or sym links instead of moving files, ideal for Torrent users.', 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index cd3447d2..e4becf52 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -524,9 +524,9 @@ class Renamer(Plugin): def moveFile(self, old, dest): dest = ss(dest) try: - if self.conf('hardlink') and not self.conf('symlink'): + if self.conf('link') == 'hard': linktastic.link(old, dest) - elif self.conf('symlink') and not self.conf('hardlink'): + elif self.conf('link') == 'sym': linktastic.symlink(old, dest) else: shutil.move(old, dest) From 3c2a00b17b589c15005029a678a0b2e79ee0e846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20K=C3=A5berg?= Date: Fri, 5 Apr 2013 14:21:15 +0200 Subject: [PATCH 21/25] add copy and move to file actions --- couchpotato/core/plugins/renamer/__init__.py | 9 +++++---- couchpotato/core/plugins/renamer/main.py | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index d80618bd..f4ea8a75 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -113,11 +113,12 @@ config = [{ 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', }, { - 'name': 'links', - 'default': '0', + 'name': 'file_action', + 'label': 'File Action', + 'default': 'move', 'type': 'dropdown', - 'values': [('Disable', 0), ('Hard link', 'hard'), ('Sym link', 'sym')], - 'description': 'Use hard links or sym links instead of moving files, ideal for Torrent users.', + 'values': [('Move', 'move'), ('Copy', 'copy'), ('Hard link', 'hardlink'), ('Sym link', 'symlink')], + 'description': 'Define which kind of file operation you want to use. Before you start using hard links or sym links, read about thire possible drawbacks.', 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index e4becf52..d5509b0d 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -524,10 +524,12 @@ class Renamer(Plugin): def moveFile(self, old, dest): dest = ss(dest) try: - if self.conf('link') == 'hard': + if self.conf('file_action') == 'hardlink': linktastic.link(old, dest) - elif self.conf('link') == 'sym': + elif self.conf('file_action') == 'symlink': linktastic.symlink(old, dest) + elif self.conf('file_action') == 'copy': + shutil.copy(old, dest) else: shutil.move(old, dest) From 5b0fa9054b78a9a01cd174805fb03ab353c57e78 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 5 Apr 2013 17:51:19 +0200 Subject: [PATCH 22/25] Put renaming started lower --- couchpotato/core/plugins/renamer/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index a6260d7b..a880abdb 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -86,11 +86,9 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return - self.renaming_started = True - # Check to see if the "to" folder is inside the "from" folder. if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): - log.debug('"To" and "From" have to exist.') + log.error('"To" and "From" have to exist.') return elif self.conf('from') in self.conf('to'): log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.') @@ -99,6 +97,8 @@ class Renamer(Plugin): log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.') return + self.renaming_started = True + # make sure the movie folder name is included in the search folder = None movie_files = [] From ac045539d165456c4c37e2f3b8b7fcc1123483df Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 5 Apr 2013 17:51:45 +0200 Subject: [PATCH 23/25] Don't move or delete anything in status check --- couchpotato/core/downloaders/transmission/main.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 6a5aad10..8cf47380 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -122,30 +122,20 @@ class Transmission(Downloader): log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'])) if not os.path.isdir(Env.setting('from', 'renamer')): - log.debug('Renamer folder has to exist.') + log.error('Renamer "from" folder doesn\'t to exist.') return if (item['percentDone'] * 100) >= 100 and (item['status'] == 6 or item['status'] == 0) and item['uploadRatio'] > self.conf('ratio'): try: trpc.stop_torrent(item['hashString'], {}) - - if not os.path.isdir(os.path.join(item['downloadDir'], item['name'])): - raise Exception('Missing folder: %s' % os.path.join(item['downloadDir'], item['name'])) - elif item['downloadDir'].rstrip(os.path.sep) in Env.setting('from', 'renamer').rstrip(os.path.sep): - raise Exception('Transmission download folder should not be the same or in the renamer "from" folder!') - else: - log.info('Moving folder from "%s" to "%s"', (os.path.join(item['downloadDir'], item['name']), Env.setting('from', 'renamer'))) - shutil.move(os.path.join(item['downloadDir'], item['name']), Env.setting('from', 'renamer')) - statuses.append({ 'id': item['hashString'], 'name': item['name'], 'status': 'completed', 'original_status': item['status'], 'timeleft': str(timedelta(seconds = 0)), - 'folder': os.path.join(Env.setting('from', 'renamer'), item['name']), + 'folder': os.path.join(item['downloadDir'], item['name']), }) - trpc.remove_torrent(item['hashString'], True, {}) except Exception, err: log.error('Failed to stop and remove torrent "%s" with error: %s', (item['name'], err)) statuses.append({ From a600430be44a5b798b212353e913a478379b0c21 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 5 Apr 2013 21:56:33 +0200 Subject: [PATCH 24/25] file_action cleanup tag ignored/failed/renamed with custom file --- couchpotato/core/plugins/renamer/__init__.py | 2 +- couchpotato/core/plugins/renamer/main.py | 40 +++++++++----------- couchpotato/core/plugins/scanner/main.py | 18 +++++++-- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index f4ea8a75..f90828bc 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -118,7 +118,7 @@ config = [{ 'default': 'move', 'type': 'dropdown', 'values': [('Move', 'move'), ('Copy', 'copy'), ('Hard link', 'hardlink'), ('Sym link', 'symlink')], - 'description': 'Define which kind of file operation you want to use. Before you start using hard links or sym links, read about thire possible drawbacks.', + 'description': 'Define which kind of file operation you want to use. Before you start using hard links or sym links, PLEASE read about their possible drawbacks.', 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 0dbda0da..26aa9dea 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -10,8 +10,8 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ ReleaseInfo from couchpotato.environment import Env -import linktastic.linktastic as linktastic import errno +import linktastic.linktastic as linktastic import os import re import shutil @@ -140,7 +140,8 @@ class Renamer(Plugin): else: log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader)) - groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, download_info = download_info, single = True) + groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), + files = movie_files, download_info = download_info, return_ignored = False, single = True) destination = self.conf('to') folder_name = self.conf('folder_name') @@ -450,6 +451,9 @@ class Renamer(Plugin): log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) self.tagDir(group, 'failed_rename') + if self.conf('file_action') != 'move': + self.tagDir(group, 'renamed already') + # Remove matching releases for release in remove_releases: log.debug('Removing release %s', release.identifier) @@ -495,31 +499,23 @@ class Renamer(Plugin): return rename_files + # This adds a file to ignore / tag a release so it is ignored later def tagDir(self, group, tag): - rename_files = {} + ignore_file = None + for movie_file in sorted(list(group['files']['movie'])): + ignore_file = '%s.ignore' % os.path.splitext(movie_file)[0] + break - if group['dirname']: - rename_files[group['parentdir']] = group['parentdir'].replace(group['dirname'], '_%s_%s' % (tag.upper(), group['dirname'])) - else: # Add it to filename - for file_type in group['files']: - for rename_me in group['files'][file_type]: - filename = os.path.basename(rename_me) - rename_files[rename_me] = rename_me.replace(filename, '_%s_%s' % (tag.upper(), filename)) + text = """This file is from CouchPotato +It has marked this release as "%s" +This file hides the release from the renamer +Remove it if you want it to be renamed (again, or at least let it try again) +""" % tag - for src in rename_files: - if rename_files[src]: - dst = rename_files[src] - log.info('Renaming "%s" to "%s"', (src, dst)) + if ignore_file: + self.createFile(ignore_file, text) - # Create dir - self.makeDir(os.path.dirname(dst)) - - try: - self.moveFile(src, dst) - except: - log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) - raise def moveFile(self, old, dest): dest = ss(dest) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index e7a94598..79fe0018 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -101,7 +101,7 @@ class Scanner(Plugin): addEvent('scanner.name_year', self.getReleaseNameYear) addEvent('scanner.partnumber', self.getPartNumber) - def scan(self, folder = None, files = None, download_info = None, simple = False, newer_than = 0, on_found = None): + def scan(self, folder = None, files = None, download_info = None, simple = False, newer_than = 0, return_ignored = True, on_found = None): folder = ss(os.path.normpath(folder)) @@ -177,17 +177,25 @@ class Scanner(Plugin): # Group files minus extension + ignored_identifiers = [] for identifier, group in movie_files.iteritems(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) + has_ignored = 0 for file_path in group['unsorted_files']: - wo_ext = file_path[:-(len(getExt(file_path)) + 1)] + ext = getExt(file_path) + wo_ext = file_path[:-(len(ext) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group['unsorted_files'].extend(found_files) leftovers = leftovers - found_files + has_ignored += 1 if ext == 'ignore' else 0 + + if has_ignored > 0: + ignored_identifiers.append(identifier) + # Break if CP wants to shut down if self.shuttingDown(): break @@ -327,15 +335,17 @@ class Scanner(Plugin): except: break + if return_ignored is False and identifier in ignored_identifiers: + log.debug('Ignore file found, ignoring release: %s' % identifier) + continue + # Group extra (and easy) files first - # images = self.getImages(group['unsorted_files']) group['files'] = { 'movie_extra': self.getMovieExtras(group['unsorted_files']), 'subtitle': self.getSubtitles(group['unsorted_files']), 'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']), 'nfo': self.getNfo(group['unsorted_files']), 'trailer': self.getTrailers(group['unsorted_files']), - #'backdrop': images['backdrop'], 'leftover': set(group['unsorted_files']), } From 5fd4312ff807279d79756817d035099eff971eca Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 5 Apr 2013 23:50:28 +0200 Subject: [PATCH 25/25] Use simpler file.cache --- couchpotato/core/plugins/file/main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/file/main.py b/couchpotato/core/plugins/file/main.py index 0dc01783..87898f52 100644 --- a/couchpotato/core/plugins/file/main.py +++ b/couchpotato/core/plugins/file/main.py @@ -9,6 +9,8 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.scanner.main import Scanner from couchpotato.core.settings.model import FileType, File from couchpotato.environment import Env +from flask.helpers import send_file +from werkzeug.exceptions import NotFound import os.path import time import traceback @@ -81,11 +83,13 @@ class FileManager(Plugin): def showCacheFile(self, filename = ''): - cache_dir = Env.get('cache_dir') - filename = os.path.basename(filename) + file_path = os.path.join(Env.get('cache_dir'), os.path.basename(filename)) - from flask.helpers import send_from_directory - return send_from_directory(cache_dir, filename) + if not os.path.isfile(file_path): + log.error('File "%s" not found', file_path) + raise NotFound() + + return send_file(file_path, conditional = True) def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = {}):