From 2f0e197320e8c3f8dcbf3bfeb5340d74b250a762 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 00:14:52 +0200 Subject: [PATCH 01/26] Mark faulty movies done --- couchpotato/core/media/_base/media/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index e0a7886c..aa1ee382 100755 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -90,9 +90,15 @@ class MediaPlugin(MediaBase): # Wrongly tagged media files def cleanupFaults(self): - medias = fireEvent('media.with_status', 'ignored', with_doc = False, single = True) + medias = fireEvent('media.with_status', 'ignored', single = True) or [] + + db = get_db() for media in medias: - self.restatus(media.get('_id'), tag_recent = False) + try: + media['status'] = 'done' + db.update(media) + except: + pass def refresh(self, id = '', **kwargs): handlers = [] From 98540f2fcd8a91b14ff4abe4624f39e21b7b5540 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 18 Sep 2014 22:00:59 +0200 Subject: [PATCH 02/26] Make sure original_folder isn't empty fix #3747 --- couchpotato/core/plugins/renamer.py | 3 +++ couchpotato/core/plugins/scanner.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 9decf08c..4c7aa3bf 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -344,6 +344,9 @@ class Renamer(Plugin): replacements['original'] = os.path.splitext(os.path.basename(current_file))[0] replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True) + if not replacements['original_folder'] or len(replacements['original_folder']) == 0: + replacements['original_folder'] = replacements['original'] + # Extension replacements['ext'] = getExt(current_file) diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 8e835b58..a1b5cf88 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -694,7 +694,7 @@ class Scanner(Plugin): def removeCPTag(self, name): try: - return re.sub(self.cp_imdb, '', name) + return re.sub(self.cp_imdb, '', name).strip() except: pass return name From 7c674b3aab59bdf0c04b3d27f98c08f8fc47dc35 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:03:02 +0200 Subject: [PATCH 03/26] Re-add movie giving rev conflict fix #3939 --- couchpotato/core/media/movie/_base/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 75e10c7d..c5ed462c 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -150,7 +150,7 @@ class MovieBase(MovieTypeBase): for release in fireEvent('release.for_media', m['_id'], single = True): if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if params.get('ignore_previous', False): - fireEvent('release.update_status', m['_id'], status = 'ignored') + fireEvent('release.update_status', release['_id'], status = 'ignored') else: fireEvent('release.delete', release['_id'], single = True) From d70da1edcee260429f156ccc1365831fa71b40d8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:20:02 +0200 Subject: [PATCH 04/26] TorrentShack use correct columns fix #3940 --- .../core/media/_base/providers/torrent/torrentshack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentshack.py b/couchpotato/core/media/_base/providers/torrent/torrentshack.py index 226993cc..f56017f5 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentshack.py @@ -42,6 +42,7 @@ class Base(TorrentProvider): link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent url = result.find('td', attrs = {'class': 'torrent_td'}).find('a') + tds = result.find_all('td') results.append({ 'id': link['href'].replace('torrents.php?torrentid=', ''), @@ -49,8 +50,8 @@ class Base(TorrentProvider): 'url': self.urls['download'] % url['href'], 'detail_url': self.urls['download'] % link['href'], 'size': self.parseSize(result.find_all('td')[5].string), - 'seeders': tryInt(result.find_all('td')[7].string), - 'leechers': tryInt(result.find_all('td')[8].string), + 'seeders': tryInt(tds[len(tds)-2].string), + 'leechers': tryInt(tds[len(tds)-1].string), }) except: From a0b3ee818641675f7d048aa196314c7157de2196 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:39:54 +0200 Subject: [PATCH 05/26] Safe encode path names in renamer fix #3425 --- couchpotato/core/plugins/renamer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 4c7aa3bf..ad2199d4 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -266,7 +266,7 @@ class Renamer(Plugin): category_label = category['label'] if category['destination'] and len(category['destination']) > 0 and category['destination'] != 'None': - destination = category['destination'] + destination = sp(category['destination']) log.debug('Setting category destination for "%s": %s' % (media_title, destination)) else: log.debug('No category destination found for "%s"' % media_title) @@ -369,6 +369,9 @@ class Renamer(Plugin): if separator: final_file_name = final_file_name.replace(' ', separator) + final_folder_name = ss(final_folder_name) + final_file_name = ss(final_file_name) + # Move DVD files (no structure renaming) if group['is_dvd'] and file_type is 'movie': found = False From 9b62e32da82d5e0d5c9532fbc95e89f0e8e2aaf5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Sep 2014 10:42:48 +0200 Subject: [PATCH 06/26] Symlink failing on encode fix #3371 --- couchpotato/core/plugins/renamer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index ad2199d4..34ac555e 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -802,9 +802,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.debug('Couldn\'t hardlink file "%s" to "%s". Symlinking instead. Error: %s.', (old, dest, traceback.format_exc())) shutil.copy(old, dest) try: - symlink(dest, old + '.link') + old_link = '%s.link' % sp(old) + symlink(dest, old_link) os.unlink(old) - os.rename(old + '.link', old) + os.rename(old_link, old) except: log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc())) From 3093b21555b3141986019352eefc217468ab0159 Mon Sep 17 00:00:00 2001 From: softcat Date: Sat, 20 Sep 2014 17:57:51 +0200 Subject: [PATCH 07/26] Fixed filmstarts.de provider --- .../core/media/movie/providers/userscript/filmstarts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/providers/userscript/filmstarts.py b/couchpotato/core/media/movie/providers/userscript/filmstarts.py index 59027e03..4e61f299 100644 --- a/couchpotato/core/media/movie/providers/userscript/filmstarts.py +++ b/couchpotato/core/media/movie/providers/userscript/filmstarts.py @@ -25,6 +25,6 @@ class Filmstarts(UserscriptBase): name = html.find("meta", {"property":"og:title"})['content'] # Year of production is not available in the meta data, so get it from the table - year = table.find("tr", text="Produktionsjahr").parent.parent.parent.td.text + year = table.find(text="Produktionsjahr").parent.parent.next_sibling.text - return self.search(name, year) \ No newline at end of file + return self.search(name, year) From db367a80d121be54f014adce07b4aa0799b92653 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 20 Sep 2014 23:58:54 +0200 Subject: [PATCH 08/26] Do proper cleanup after rename --- couchpotato/core/plugins/base.py | 18 +++++++----------- couchpotato/core/plugins/renamer.py | 10 ++++++---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 8a4cada7..39b17a89 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -146,21 +146,17 @@ class Plugin(object): folder = sp(folder) for item in os.listdir(folder): - full_folder = os.path.join(folder, item) + full_folder = sp(os.path.join(folder, item)) if not only_clean or (item in only_clean and os.path.isdir(full_folder)): - for root, dirs, files in os.walk(full_folder): + for subfolder, dirs, files in os.walk(full_folder, topdown = False): - for dir_name in dirs: - full_path = os.path.join(root, dir_name) - - if len(os.listdir(full_path)) == 0: - try: - os.rmdir(full_path) - except: - if show_error: - log.info2('Couldn\'t remove directory %s: %s', (full_path, traceback.format_exc())) + try: + os.rmdir(subfolder) + except: + if show_error: + log.info2('Couldn\'t remove directory %s: %s', (subfolder, traceback.format_exc())) try: os.rmdir(folder) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 34ac555e..3c506a48 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -555,9 +555,9 @@ class Renamer(Plugin): os.remove(src) parent_dir = os.path.dirname(src) - if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and \ + if parent_dir not in delete_folders and os.path.isdir(parent_dir) and \ not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \ - not isSubFolder(parent_dir, base_folder): + isSubFolder(parent_dir, base_folder): delete_folders.append(parent_dir) @@ -566,6 +566,7 @@ class Renamer(Plugin): self.tagRelease(group = group, tag = 'failed_remove') # Delete leftover folder from older releases + delete_folders = sorted(delete_folders, key = len, reverse = True) for delete_folder in delete_folders: try: self.deleteEmptyFolder(delete_folder, show_error = False) @@ -620,8 +621,9 @@ class Renamer(Plugin): group_folder = sp(os.path.join(base_folder, os.path.relpath(group['parentdir'], base_folder).split(os.path.sep)[0])) try: - log.info('Deleting folder: %s', group_folder) - self.deleteEmptyFolder(group_folder) + if self.conf('cleanup') or self.conf('move_leftover'): + log.info('Deleting folder: %s', group_folder) + self.deleteEmptyFolder(group_folder) except: log.error('Failed removing %s: %s', (group_folder, traceback.format_exc())) From c4db4ace132f8878e764f8f381eeca27f6942fe6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 00:09:56 +0200 Subject: [PATCH 09/26] Log move, copy, link --- couchpotato/core/plugins/renamer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 3c506a48..af39aa32 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -579,7 +579,6 @@ class Renamer(Plugin): for src in rename_files: if rename_files[src]: dst = rename_files[src] - log.info('Renaming "%s" to "%s"', (src, dst)) # Create dir self.makeDir(os.path.dirname(dst)) @@ -785,6 +784,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if move_type not in ['copy', 'link']: try: + log.info('Moving "%s" to "%s"', (old, dest)) shutil.move(old, dest) except: if os.path.exists(dest): @@ -793,8 +793,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) else: raise elif move_type == 'copy': + log.info('Copying "%s" to "%s"', (old, dest)) shutil.copy(old, dest) else: + log.info('Linking "%s" to "%s"', (old, dest)) # First try to hardlink try: log.debug('Hardlinking file "%s" to "%s"...', (old, dest)) From 2639c5e9ad23004d5f1506d65d83884d093b9af4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 10:40:29 +0200 Subject: [PATCH 10/26] Force add if not set fix #1811 --- couchpotato/core/plugins/renamer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index af39aa32..a2d993ed 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -219,6 +219,12 @@ class Renamer(Plugin): nfo_name = self.conf('nfo_name') separator = self.conf('separator') + cd_keys = ['',''] + if not any(x in folder_name for x in cd_keys) and not any(x in file_name for x in cd_keys): + log.error('Missing `cd` or `cd_nr` in the renamer. This will cause multi-file releases of being renamed to the same file.' + 'Force adding it') + file_name = '%s %s' % ('', file_name) + # Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader. if not groups and self.statusInfoComplete(release_download): self.tagRelease(release_download = release_download, tag = 'failed_rename') From 7861416dc535324af22290a883d098a92d8a5e13 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 10:43:53 +0200 Subject: [PATCH 11/26] Don't write over files already renamed --- couchpotato/core/plugins/renamer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index a2d993ed..77840531 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -10,7 +10,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier + getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier, randomString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -586,6 +586,10 @@ class Renamer(Plugin): if rename_files[src]: dst = rename_files[src] + if dst in group['renamed_files']: + log.error('File "%s" already exists, adding random string at the end to prevent data loss', dst) + dst = '%s.random-%s' % (dst, randomString()) + # Create dir self.makeDir(os.path.dirname(dst)) From 0d166025d003f2d82e822481b2a59bbd5ef162dc Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 15:13:50 +0200 Subject: [PATCH 12/26] Safestring before base encode --- couchpotato/core/media/movie/providers/info/couchpotatoapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/media/movie/providers/info/couchpotatoapi.py b/couchpotato/core/media/movie/providers/info/couchpotatoapi.py index 4c65bf8c..51afbaef 100644 --- a/couchpotato/core/media/movie/providers/info/couchpotatoapi.py +++ b/couchpotato/core/media/movie/providers/info/couchpotatoapi.py @@ -2,7 +2,7 @@ import base64 import time from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.logger import CPLog from couchpotato.core.media.movie.providers.base import MovieProvider from couchpotato.environment import Env @@ -66,7 +66,7 @@ class CouchPotatoApi(MovieProvider): if not name: return - name_enc = base64.b64encode(name) + name_enc = base64.b64encode(ss(name)) return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders()) def isMovie(self, identifier = None): From 2deb6ee6a79368f07c9b0b96fe81656582ec14da Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 15:57:29 +0200 Subject: [PATCH 13/26] Trakt not moving movie to collection fix #3018 --- couchpotato/core/notifications/trakt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/notifications/trakt.py b/couchpotato/core/notifications/trakt.py index 8f35deab..fe170be8 100644 --- a/couchpotato/core/notifications/trakt.py +++ b/couchpotato/core/notifications/trakt.py @@ -1,4 +1,4 @@ -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.variable import getTitle, getIdentifier from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification @@ -16,7 +16,7 @@ class Trakt(Notification): 'test': 'account/test/%s', } - listen_to = ['movie.downloaded'] + listen_to = ['movie.snatched'] def notify(self, message = '', data = None, listener = None): if not data: data = {} @@ -38,7 +38,7 @@ class Trakt(Notification): 'username': self.conf('automation_username'), 'password': self.conf('automation_password'), 'movies': [{ - 'imdb_id': data['identifier'], + 'imdb_id': getIdentifier(data), 'title': getTitle(data), 'year': data['info']['year'] }] if data else [] From 2fa7834e6ef0cab77abd829508862f284ec9fa07 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 22:17:09 +0200 Subject: [PATCH 14/26] Allow typing in directory setting closes #479 --- couchpotato/static/scripts/page/settings.js | 122 ++++++++++++++++---- couchpotato/static/style/settings.css | 13 ++- 2 files changed, 112 insertions(+), 23 deletions(-) diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index b9f72aba..c8e0a5e4 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -634,6 +634,7 @@ Option.Directory = new Class({ browser: null, save_on_change: false, use_cache: false, + current_dir: '', create: function(){ var self = this; @@ -645,8 +646,17 @@ Option.Directory = new Class({ 'click': self.showBrowser.bind(self) } }).adopt( - self.input = new Element('span', { - 'text': self.getSettingValue() + self.input = new Element('input', { + 'value': self.getSettingValue(), + 'events': { + 'change': self.filterDirectory.bind(self), + 'keydown': function(e){ + if(e.key == 'enter' || e.key == 'tab') + (e).stop(); + }, + 'keyup': self.filterDirectory.bind(self), + 'paste': self.filterDirectory.bind(self) + } }) ) ); @@ -654,10 +664,55 @@ Option.Directory = new Class({ self.cached = {}; }, + filterDirectory: function(e){ + var self = this, + value = self.getValue(), + path_sep = Api.getOption('path_sep'), + active_selector = 'li:not(.blur):not(.empty)'; + + if(e.key == 'enter' || e.key == 'tab'){ + (e).stop(); + + var first = self.dir_list.getElement(active_selector); + if(first){ + self.selectDirectory(first.get('data-value')); + } + } + else { + + // New folder + if(value.substr(-1) == path_sep){ + if(self.current_dir != value) + self.selectDirectory(value) + } + else { + var pd = self.getParentDir(value); + if(self.current_dir != pd) + self.getDirs(pd); + + var folder_filter = value.split(path_sep).getLast() + self.dir_list.getElements('li').each(function(li){ + var valid = li.get('text').substr(0, folder_filter.length).toLowerCase() != folder_filter.toLowerCase() + li[valid ? 'addClass' : 'removeClass']('blur') + }); + + var first = self.dir_list.getElement(active_selector); + if(first){ + if(!self.dir_list_scroll) + self.dir_list_scroll = new Fx.Scroll(self.dir_list, { + 'transition': 'quint:in:out' + }); + + self.dir_list_scroll.toElement(first); + } + } + } + }, + selectDirectory: function(dir){ var self = this; - self.input.set('text', dir); + self.input.set('value', dir); self.getDirs() }, @@ -668,9 +723,28 @@ Option.Directory = new Class({ self.selectDirectory(self.getParentDir()) }, + caretAtEnd: function(){ + var self = this; + + self.input.focus(); + + if (typeof self.input.selectionStart == "number") { + self.input.selectionStart = self.input.selectionEnd = self.input.get('value').length; + } else if (typeof el.createTextRange != "undefined") { + self.input.focus(); + var range = self.input.createTextRange(); + range.collapse(false); + range.select(); + } + }, + showBrowser: function(){ var self = this; + // Move caret to back of the input + if(!self.browser || self.browser && !self.browser.isVisible()) + self.caretAtEnd() + if(!self.browser){ self.browser = new Element('div.directory_list').adopt( new Element('div.pointer'), @@ -686,7 +760,9 @@ Option.Directory = new Class({ }).adopt( self.show_hidden = new Element('input[type=checkbox].inlay', { 'events': { - 'change': self.getDirs.bind(self) + 'change': function(){ + self.getDirs() + } } }) ) @@ -707,7 +783,7 @@ Option.Directory = new Class({ 'text': 'Clear', 'events': { 'click': function(e){ - self.input.set('text', ''); + self.input.set('value', ''); self.hideBrowser(e, true); } } @@ -735,7 +811,7 @@ Option.Directory = new Class({ new Form.Check(self.show_hidden); } - self.initial_directory = self.input.get('text'); + self.initial_directory = self.input.get('value'); self.getDirs(); self.browser.show(); @@ -749,7 +825,7 @@ Option.Directory = new Class({ if(save) self.save(); else - self.input.set('text', self.initial_directory); + self.input.set('value', self.initial_directory); self.browser.hide(); self.el.removeEvents('outerClick') @@ -757,21 +833,21 @@ Option.Directory = new Class({ }, fillBrowser: function(json){ - var self = this; + var self = this, + v = self.getValue(); self.data = json; - var v = self.getValue(); - var previous_dir = self.getParentDir(); + var previous_dir = json.parent; if(v == '') - self.input.set('text', json.home); + self.input.set('value', json.home); - if(previous_dir != v && previous_dir.length >= 1 && !json.is_root){ + if(previous_dir.length >= 1 && !json.is_root){ var prev_dirname = self.getCurrentDirname(previous_dir); if(previous_dir == json.home) - prev_dirname = 'Home'; + prev_dirname = 'Home Folder'; else if(previous_dir == '/' && json.platform == 'nt') prev_dirname = 'Computer'; @@ -801,12 +877,13 @@ Option.Directory = new Class({ new Element('li.empty', { 'text': 'Selected folder is empty' }).inject(self.dir_list) + + self.caretAtEnd(); }, - getDirs: function(){ - var self = this; - - var c = self.getValue(); + getDirs: function(dir){ + var self = this, + c = dir || self.getValue(); if(self.cached[c] && self.use_cache){ self.fillBrowser() @@ -817,7 +894,10 @@ Option.Directory = new Class({ 'path': c, 'show_hidden': +self.show_hidden.checked }, - 'onComplete': self.fillBrowser.bind(self) + 'onComplete': function(json){ + self.current_dir = c; + self.fillBrowser(json); + } }) } }, @@ -831,8 +911,8 @@ Option.Directory = new Class({ var v = dir || self.getValue(); var sep = Api.getOption('path_sep'); var dirs = v.split(sep); - if(dirs.pop() == '') - dirs.pop(); + if(dirs.pop() == '') + dirs.pop(); return dirs.join(sep) + sep }, @@ -845,7 +925,7 @@ Option.Directory = new Class({ getValue: function(){ var self = this; - return self.input.get('text'); + return self.input.get('value'); } }); diff --git a/couchpotato/static/style/settings.css b/couchpotato/static/style/settings.css index 7fb1df29..e84d3814 100644 --- a/couchpotato/static/style/settings.css +++ b/couchpotato/static/style/settings.css @@ -302,15 +302,19 @@ font-family: 'Elusive-Icons'; color: #f5e39c; } - .page form .directory > span { + .page form .directory > input { height: 25px; display: inline-block; float: right; text-align: right; white-space: nowrap; cursor: pointer; + background: none; + border: 0; + color: #FFF; + width: 100%; } - .page form .directory span:empty:before { + .page form .directory input:empty:before { content: 'No folder selected'; font-style: italic; opacity: .3; @@ -353,6 +357,11 @@ white-space: nowrap; text-overflow: ellipsis; } + + .page .directory_list li.blur { + opacity: .3; + } + .page .directory_list li:last-child { border-bottom: 1px solid rgba(255,255,255,0.1); } From 0bae50931154172a6ae0915aec29cfcceace5d74 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Sep 2014 22:29:52 +0200 Subject: [PATCH 15/26] Use github img url for Growl notification fix #1363 --- couchpotato/core/notifications/growl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/growl.py b/couchpotato/core/notifications/growl.py index e60e7ef7..a0081a23 100644 --- a/couchpotato/core/notifications/growl.py +++ b/couchpotato/core/notifications/growl.py @@ -34,9 +34,9 @@ class Growl(Notification): self.growl = notifier.GrowlNotifier( applicationName = Env.get('appname'), - notifications = ["Updates"], - defaultNotifications = ["Updates"], - applicationIcon = '%s/static/images/couch.png' % fireEvent('app.api_url', single = True), + notifications = ['Updates'], + defaultNotifications = ['Updates'], + applicationIcon = self.getNotificationImage('medium'), hostname = hostname if hostname else 'localhost', password = password if password else None, port = port if port else 23053 @@ -56,7 +56,7 @@ class Growl(Notification): try: self.growl.notify( - noteType = "Updates", + noteType = 'Updates', title = self.default_title, description = message, sticky = False, From da9d2b5ed8ca45de5a5e19fced5528abd8fc7bbf Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:03:13 +0200 Subject: [PATCH 16/26] Check free diskspace before starting moving files fix #3893 --- couchpotato/core/helpers/variable.py | 22 ++++++++++++++++++++++ couchpotato/core/plugins/renamer.py | 11 ++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index db68da2f..9e064b7b 100755 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -382,6 +382,28 @@ def getFreeSpace(directories): return free_space +def getSize(paths): + + single = not isinstance(paths, (tuple, list)) + if single: + paths = [paths] + + total_size = 0 + for path in paths: + path = sp(path) + + if os.path.isdir(path): + total_size = 0 + for dirpath, _, filenames in os.walk(path): + for f in filenames: + total_size += os.path.getsize(sp(os.path.join(dirpath, f))) + + elif os.path.isfile(path): + total_size += os.path.getsize(path) + + return total_size / 1048576 # MB + + def find(func, iterable): for item in iterable: if func(item): diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 77840531..01e78f09 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -10,7 +10,8 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, getIdentifier, randomString + getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder, \ + getIdentifier, randomString, getFreeSpace, getSize from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -279,6 +280,7 @@ class Renamer(Plugin): except: log.error('Failed getting category label: %s', traceback.format_exc()) + # Find subtitle for renaming group['before_rename'] = [] fireEvent('renamer.before', group) @@ -546,6 +548,13 @@ class Renamer(Plugin): (not keep_original or self.fileIsAdded(current_file, group)): remove_files.append(current_file) + total_space, available_space = getFreeSpace(destination) + renaming_size = getSize(rename_files.keys()) + if renaming_size > available_space: + log.error('Not enough space left, need %s MB but only %s MB available', (renaming_size, available_space)) + self.tagRelease(group = group, tag = 'not_enough_space') + continue + # Remove files delete_folders = [] for src in remove_files: From 11e7fb23cab01d909e658a29b5ecddb2eeb30c0c Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:38:53 +0200 Subject: [PATCH 17/26] Stream larger file download fix #2488 --- couchpotato/core/plugins/base.py | 39 +++++++++++++++++++++++--------- couchpotato/core/plugins/file.py | 5 +++- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 39b17a89..29b34326 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -121,15 +121,31 @@ class Plugin(object): if os.path.exists(path): log.debug('%s already exists, overwriting file with new version', path) - try: - f = open(path, 'w+' if not binary else 'w+b') - f.write(content) - f.close() - os.chmod(path, Env.getPermission('file')) - except: - log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) - if os.path.isfile(path): - os.remove(path) + write_type = 'w+' if not binary else 'w+b' + + # Stream file using response object + if isinstance(content, requests.models.Response): + + # Write file to temp + with open('%s.tmp' % path, write_type) as f: + for chunk in content.iter_content(chunk_size = 1048576): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + + # Rename to destination + os.rename('%s.tmp' % path, path) + + else: + try: + f = open(path, write_type) + f.write(content) + f.close() + os.chmod(path, Env.getPermission('file')) + except: + log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) + if os.path.isfile(path): + os.remove(path) def makeDir(self, path): path = sp(path) @@ -165,7 +181,7 @@ class Plugin(object): log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc())) # http request - def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True): + def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, stream = False): url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]") if not headers: headers = {} @@ -206,6 +222,7 @@ class Plugin(object): 'timeout': timeout, 'files': files, 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. + 'stream': stream } method = 'post' if len(data) > 0 or files else 'get' @@ -214,7 +231,7 @@ class Plugin(object): status_code = response.status_code if response.status_code == requests.codes.ok: - data = response.content + data = response if stream else response.content else: response.raise_for_status() diff --git a/couchpotato/core/plugins/file.py b/couchpotato/core/plugins/file.py index 80c073fc..db6787df 100644 --- a/couchpotato/core/plugins/file.py +++ b/couchpotato/core/plugins/file.py @@ -64,6 +64,9 @@ class FileManager(Plugin): def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): if not urlopen_kwargs: urlopen_kwargs = {} + # Return response object to stream download + urlopen_kwargs['stream'] = True + if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) @@ -107,4 +110,4 @@ class FileManager(Plugin): else: log.info('Subfolder test succeeded') - return failed == 0 \ No newline at end of file + return failed == 0 From 6fa6d530ec1f45aeb73614b528433de84c31b462 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:45:43 +0200 Subject: [PATCH 18/26] Remove older backup folders --- couchpotato/runner.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 1c6f5779..bc46518a 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -9,6 +9,7 @@ import traceback import warnings import re import tarfile +import shutil from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase from argparse import ArgumentParser @@ -108,14 +109,19 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in os.walk(backup_path): - for backup_file in sorted(files): - ints = re.findall('\d+', backup_file) + # Only consider files being a direct child of the backup_path + if root == backup_path: + for backup_file in sorted(files): + ints = re.findall('\d+', backup_file) - # Delete non zip files - if len(ints) != 1: - os.remove(os.path.join(backup_path, backup_file)) - else: - existing_backups.append((int(ints[0]), backup_file)) + # Delete non zip files + if len(ints) != 1: + os.remove(os.path.join(root, backup_file)) + else: + existing_backups.append((int(ints[0]), backup_file)) + else: + # Delete stray directories. + shutil.rmtree(root) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: From b1f88c1c48266cae5f41dfd3908b1ff9faeb63a7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 21:53:27 +0200 Subject: [PATCH 19/26] Allow https for Transmission close #3880 --- couchpotato/core/downloaders/transmission.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index 409efaa0..d6112a91 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -25,7 +25,7 @@ class Transmission(DownloaderBase): def connect(self): # Load host from config and split out port. - host = cleanHost(self.conf('host'), protocol = False).split(':') + host = cleanHost(self.conf('host')).rstrip('/').rsplit(':', 1) if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False @@ -162,11 +162,11 @@ class Transmission(DownloaderBase): class TransmissionRPC(object): """TransmissionRPC lite library""" - def __init__(self, host = 'localhost', port = 9091, rpc_url = 'transmission', username = None, password = None): + def __init__(self, host = 'http://localhost', port = 9091, rpc_url = 'transmission', username = None, password = None): super(TransmissionRPC, self).__init__() - self.url = 'http://' + host + ':' + str(port) + '/' + rpc_url + '/rpc' + self.url = host + ':' + str(port) + '/' + rpc_url + '/rpc' self.tag = 0 self.session_id = 0 self.session = {} @@ -274,8 +274,8 @@ config = [{ }, { 'name': 'host', - 'default': 'localhost:9091', - 'description': 'Hostname with port. Usually localhost:9091', + 'default': 'http://localhost:9091', + 'description': 'Hostname with port. Usually http://localhost:9091', }, { 'name': 'rpc_url', From d7f43c2cf8fb528d86929847a0761cad61ff37a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Sep 2014 22:15:13 +0200 Subject: [PATCH 20/26] Make minimum seeders configurable fix #3202 --- .../core/media/_base/searcher/__init__.py | 20 +++++++++++++++++++ couchpotato/core/plugins/release/main.py | 7 ++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/_base/searcher/__init__.py b/couchpotato/core/media/_base/searcher/__init__.py index bf69b950..0e3655e8 100644 --- a/couchpotato/core/media/_base/searcher/__init__.py +++ b/couchpotato/core/media/_base/searcher/__init__.py @@ -73,4 +73,24 @@ config = [{ ], }, ], +}, { + 'name': 'torrent', + 'groups': [ + { + 'tab': 'searcher', + 'name': 'searcher', + 'wizard': True, + 'options': [ + { + 'name': 'minimum_seeders', + 'advanced': True, + 'label': 'Minimum seeders', + 'description': 'Ignore torrents with seeders below this number', + 'default': 1, + 'type': 'int', + 'unit': 'seeders' + }, + ], + }, + ], }] diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 0385e226..815cb404 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -8,7 +8,7 @@ from couchpotato import md5, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.helpers.encoding import toUnicode, sp -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.variable import getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from .index import ReleaseIndex, ReleaseStatusIndex, ReleaseIDIndex, ReleaseDownloadIndex @@ -380,6 +380,7 @@ class Release(Plugin): wait_for = False let_through = False filtered_results = [] + minimum_seeders = tryInt(Env.setting('minimum_seeders', section = 'torrent', default = 1)) # Filter out ignored and other releases we don't want for rel in results: @@ -396,8 +397,8 @@ class Release(Plugin): log.info('Ignored, size "%sMB" to low: %s', (rel['size'], rel['name'])) continue - if 'seeders' in rel and rel.get('seeders') <= 0: - log.info('Ignored, no seeders: %s', (rel['name'])) + if 'seeders' in rel and rel.get('seeders') < minimum_seeders: + log.info('Ignored, not enough seeders, has %s needs %s: %s', (rel.get('seeders'), minimum_seeders, rel['name'])) continue # If a single release comes through the "wait for", let through all From 70ca31a265ed6d26fb96fca301081a9e67b18d35 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 00:29:48 +0200 Subject: [PATCH 21/26] Only allow single redirect for now fix #3931 --- couchpotato/core/plugins/base.py | 2 +- couchpotato/runner.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 29b34326..5054b325 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -222,7 +222,7 @@ class Plugin(object): 'timeout': timeout, 'files': files, 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. - 'stream': stream + 'stream': stream, } method = 'post' if len(data) > 0 or files else 'get' diff --git a/couchpotato/runner.py b/couchpotato/runner.py index bc46518a..8718355e 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -151,12 +151,15 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En if not os.path.exists(python_cache): os.mkdir(python_cache) + session = requests.Session() + session.max_redirects = 1 + # Register environment settings Env.set('app_dir', sp(base_path)) Env.set('data_dir', sp(data_dir)) Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) - Env.set('http_opener', requests.Session()) + Env.set('http_opener', session) Env.set('cache_dir', cache_dir) Env.set('cache', FileSystemCache(python_cache)) Env.set('console_log', options.console_log) From 3338b72d1f6845f6a44e6407d62d4ba27deee266 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 00:38:36 +0200 Subject: [PATCH 22/26] Stop endless redirect loop fix #3931 --- libs/requests/sessions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index 508b0ef2..02c9fb2a 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -532,7 +532,11 @@ class Session(SessionRedirectMixin): if not isinstance(request, PreparedRequest): raise ValueError('You can only send PreparedRequests.') + redirect_count = 0 while request.url in self.redirect_cache: + redirect_count += 1 + if redirect_count > self.max_redirects: + raise TooManyRedirects request.url = self.redirect_cache.get(request.url) # Set up variables needed for resolve_redirects and dispatching of hooks From 17b940a271318cd94107f77cc8f1181eae63280b Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 00:38:44 +0200 Subject: [PATCH 23/26] Allow 5 redirects --- couchpotato/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 8718355e..36fc366f 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -152,7 +152,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En os.mkdir(python_cache) session = requests.Session() - session.max_redirects = 1 + session.max_redirects = 5 # Register environment settings Env.set('app_dir', sp(base_path)) From b3d75cb48538743b73098cc14e12f735c5ed3e9c Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 10:02:36 +0200 Subject: [PATCH 24/26] Check if file got moved successful on move/copy close #3893 --- couchpotato/core/plugins/renamer.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 01e78f09..aeeb3542 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -596,7 +596,7 @@ class Renamer(Plugin): dst = rename_files[src] if dst in group['renamed_files']: - log.error('File "%s" already exists, adding random string at the end to prevent data loss', dst) + log.error('File "%s" already renamed once, adding random string at the end to prevent data loss', dst) dst = '%s.random-%s' % (dst, randomString()) # Create dir @@ -797,6 +797,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) dest = sp(dest) try: + if os.path.exists(dest): + raise Exception('Destination "%s" already exists' % dest) + move_type = self.conf('file_action') if use_default: move_type = self.conf('default_file_action') @@ -806,10 +809,14 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.info('Moving "%s" to "%s"', (old, dest)) shutil.move(old, dest) except: - if os.path.exists(dest): + exists = os.path.exists(dest) + if exists and os.path.getsize(old) == os.path.getsize(dest): log.error('Successfully moved file "%s", but something went wrong: %s', (dest, traceback.format_exc())) os.unlink(old) else: + # remove faultly copied file + if exists: + os.unlink(dest) raise elif move_type == 'copy': log.info('Copying "%s" to "%s"', (old, dest)) @@ -1219,7 +1226,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) except Exception as e: log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc())) # As we probably tried to overwrite the nfo file, check if it exists and then remove the original - if os.path.isfile(move_to): + if os.path.isfile(move_to) and os.path.getsize(leftoverfile) == os.path.getsize(move_to): if cleanup: log.info('Deleting left over file %s instead...', leftoverfile) os.unlink(leftoverfile) From 39d0f91de251c75a97cee1f0f4591407175cc0f7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 12:27:09 +0200 Subject: [PATCH 25/26] Add permission calculator link #3953 --- couchpotato/core/_base/_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index 852c42c2..6bfb441d 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -286,13 +286,13 @@ config = [{ 'name': 'permission_folder', 'default': '0755', 'label': 'Folder CHMOD', - 'description': 'Can be either decimal (493) or octal (leading zero: 0755)', + 'description': 'Can be either decimal (493) or octal (leading zero: 0755). Calculate the correct value', }, { 'name': 'permission_file', 'default': '0755', 'label': 'File CHMOD', - 'description': 'Same as Folder CHMOD but for files', + 'description': 'See Folder CHMOD description, but for files', }, ], }, From 8f02b0eea0cd3a2c98650cbef2e7c643fef96ac3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 23 Sep 2014 12:36:46 +0200 Subject: [PATCH 26/26] Api documention updates close #3955 --- couchpotato/core/media/movie/_base/main.py | 4 ++++ couchpotato/core/plugins/category/main.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index c5ed462c..1b0881b5 100755 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -27,6 +27,10 @@ class MovieBase(MovieTypeBase): addApiView('movie.add', self.addView, docs = { 'desc': 'Add new movie to the wanted list', + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'movie': object +}"""}, 'params': { 'identifier': {'desc': 'IMDB id of the movie your want to add.'}, 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'}, diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index a0852cc1..4abc94c0 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -27,7 +27,7 @@ class CategoryPlugin(Plugin): 'desc': 'List all available categories', 'return': {'type': 'object', 'example': """{ 'success': True, - 'list': array, categories + 'categories': array, categories }"""} })