From 007597239fa8d791a811cb23f4646ddc769fe0cf Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 14 Jun 2013 15:06:59 +0930 Subject: [PATCH 001/209] add categories --- couchpotato/core/plugins/renamer/main.py | 4 ++++ couchpotato/core/plugins/searcher/main.py | 14 ++++++++++---- couchpotato/core/settings/model.py | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 7ab9f450..cf79b34f 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -143,6 +143,10 @@ class Renamer(Plugin): remove_releases = [] movie_title = getTitle(group['library']) + try: + destination = group['category']['path'] + except: + destination = self.conf('to') # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 23e8ea58..ab309845 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -385,24 +385,30 @@ class Searcher(Plugin): nzb_words = re.split('\W+', nzb_name) # Make sure it has required words - required_words = splitString(self.conf('required_words').lower()) + try: + required_words = splitString(movie['category']['required'].lower()) + except: + required_words = splitString(self.conf('required_words').lower()) req_match = 0 for req_set in required_words: req = splitString(req_set, '&') req_match += len(list(set(nzb_words) & set(req))) == len(req) - if self.conf('required_words') and req_match == 0: + if len(required_words) > 0 and req_match == 0: log.info2('Wrong: Required word missing: %s', nzb['name']) return False # Ignore releases - ignored_words = splitString(self.conf('ignored_words').lower()) + try: + ignored_words = splitString(movie['category']['ignored'].lower()) + except: + ignored_words = splitString(self.conf('ignored_words').lower()) ignored_match = 0 for ignored_set in ignored_words: ignored = splitString(ignored_set, '&') ignored_match += len(list(set(nzb_words) & set(ignored))) == len(ignored) - if self.conf('ignored_words') and ignored_match: + if len(ignored_words) > 0 and ignored_match: log.info2("Wrong: '%s' contains 'ignored words'", (nzb['name'])) return False diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index 6c8deb1e..3ec78f44 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -50,6 +50,7 @@ class Movie(Entity): library = ManyToOne('Library', cascade = 'delete, delete-orphan', single_parent = True) status = ManyToOne('Status') profile = ManyToOne('Profile') + category = ManyToOne('Category') releases = OneToMany('Release', cascade = 'all, delete-orphan') files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) @@ -174,6 +175,22 @@ class Profile(Entity): return orig_dict +class Category(Entity): + """""" + using_options(order_by = 'order') + + label = Field(Unicode(50)) + order = Field(Integer, default = 0, index = True) + core = Field(Boolean, default = False) + hide = Field(Boolean, default = False) + + movie = OneToMany('Movie') + path = Field(Unicode(255)) + required = Field(Unicode(255)) + preferred = Field(Unicode(255)) + ignored = Field(Unicode(255)) + + class ProfileType(Entity): """""" using_options(order_by = 'order') From 60034f2c963d4e72cf0d94b39165631f0ac94a08 Mon Sep 17 00:00:00 2001 From: clinton-hall Date: Fri, 14 Jun 2013 21:56:26 +0930 Subject: [PATCH 002/209] add category preffered words and partial ignore. --- couchpotato/core/plugins/score/main.py | 14 ++++++--- couchpotato/core/plugins/score/scores.py | 40 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/score/main.py b/couchpotato/core/plugins/score/main.py index f853be95..28baae7e 100644 --- a/couchpotato/core/plugins/score/main.py +++ b/couchpotato/core/plugins/score/main.py @@ -3,8 +3,8 @@ from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import getTitle from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \ - sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \ +from couchpotato.core.plugins.score.scores import nameScore, CatnameScore, nameRatioScore, \ + sizeScore, providerScore, duplicateScore, partialIgnoredScore, CatpartialIgnoredScore, namePositionScore, \ halfMultipartScore log = CPLog(__name__) @@ -18,7 +18,10 @@ class Score(Plugin): def calculate(self, nzb, movie): ''' Calculate the score of a NZB, used for sorting later ''' - score = nameScore(toUnicode(nzb['name']), movie['library']['year']) + if movie and movie['category'] and movie['category']['preferred']: + score = CatnameScore(toUnicode(nzb['name']), movie['library']['year'], movie['category']['preferred']) + else: + score = nameScore(toUnicode(nzb['name']), movie['library']['year']) for movie_title in movie['library']['titles']: score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title'])) @@ -41,7 +44,10 @@ class Score(Plugin): score += duplicateScore(nzb['name'], getTitle(movie['library'])) # Partial ignored words - score += partialIgnoredScore(nzb['name'], getTitle(movie['library'])) + if movie and movie['category'] and movie['category']['ignored']: + score = CatpartialIgnoredScore(nzb['name'], getTitle(movie['library']), movie['category']['ignored']) + else: + score += partialIgnoredScore(nzb['name'], getTitle(movie['library'])) # Ignore single downloads from multipart score += halfMultipartScore(nzb['name']) diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index c9345fc0..5feeff39 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -49,6 +49,32 @@ def nameScore(name, year): return score +def CatnameScore(name, year, preferred): + ''' Calculate score for words in the NZB name ''' + + score = 0 + name = name.lower() + + # give points for the cool stuff + for value in name_scores: + v = value.split(':') + add = int(v.pop()) + if v.pop() in name: + score = score + add + + # points if the year is correct + if str(year) in name: + score = score + 5 + + # Contains preferred word + nzb_words = re.split('\W+', simplifyString(name)) + preferred_words = [x.strip() for x in preferred.split(',')] + for word in preferred_words: + if word.strip() and word.strip().lower() in nzb_words: + score = score + 100 + + return score + def nameRatioScore(nzb_name, movie_name): nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) @@ -150,6 +176,20 @@ def partialIgnoredScore(nzb_name, movie_name): return score +def CatpartialIgnoredScore(nzb_name, movie_name, ignored): + + nzb_name = nzb_name.lower() + movie_name = movie_name.lower() + + ignored_words = [x.strip().lower() for x in ignored.split(',')] + + score = 0 + for ignored_word in ignored_words: + if ignored_word in nzb_name and ignored_word not in movie_name: + score -= 5 + + return score + def halfMultipartScore(nzb_name): wrong_found = 0 From 461a0b364592577466246c979425774921c811c0 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Wed, 17 Apr 2013 23:19:30 +0200 Subject: [PATCH 003/209] Seeding support Design intent: - Option to turn seeding support on or off - After torrent downloading is complete the seeding phase starts, seeding parameters can be set per torrent provide (0 disables them) - When the seeding phase starts the checkSnatched function renames all files if (sym)linking/copying is used. The movie is set to done (!), the release to seeding status. - Note that Direct symlink functionality is removed as the original file needs to end up in the movies store and not the downloader store (if the downloader cleans up his files, the original is deleted and the symlinks are useless) - checkSnatched waits until downloader sets the download to completed (met the seeding parameters) - When completed, checkSnatched intiates the renamer if move is used, or if linking is used asks the downloader to remove the torrent and clean-up it's files and sets the release to downloaded - Updated some of the .ignore file behavior to allow the downloader to remove its files Known items/issues: - only implemented for uTorrent and Transmission - text in downloader settings is too long and messes up the layout... To do (after this PR): - implement for other torrent downloaders - complete download removal for NZBs (remove from history in sabNZBd) - failed download management for torrents (no seeders, takes too long, etc.) - unrar support Updates: - Added transmission support - Simplified uTorrent - Added checkSnatched to renamer to make sure the poller is always first - Updated default values and removed advanced option tag for providers - Updated the tagger to allow removing of ignore tags and tagging when the group is not known - Added tagging of downloading torrents - fixed subtitles being leftover after seeding --- couchpotato/core/_base/_core/main.py | 2 +- couchpotato/core/downloaders/base.py | 35 ++- .../core/downloaders/transmission/__init__.py | 32 ++- .../core/downloaders/transmission/main.py | 199 ++++++++---------- .../core/downloaders/utorrent/__init__.py | 22 ++ couchpotato/core/downloaders/utorrent/main.py | 141 ++++++++----- couchpotato/core/plugins/renamer/__init__.py | 2 +- couchpotato/core/plugins/renamer/main.py | 156 +++++++++++--- couchpotato/core/plugins/scanner/main.py | 8 + couchpotato/core/plugins/status/main.py | 1 + couchpotato/core/plugins/subtitle/main.py | 1 + couchpotato/core/providers/base.py | 2 + .../core/providers/torrent/hdbits/__init__.py | 14 ++ .../providers/torrent/iptorrents/__init__.py | 14 ++ .../torrent/kickasstorrents/__init__.py | 14 ++ .../torrent/passthepopcorn/__init__.py | 16 +- .../providers/torrent/publichd/__init__.py | 14 ++ .../providers/torrent/sceneaccess/__init__.py | 14 ++ .../providers/torrent/scenehd/__init__.py | 14 ++ .../torrent/thepiratebay/__init__.py | 14 ++ .../torrent/torrentbytes/__init__.py | 14 ++ .../providers/torrent/torrentday/__init__.py | 14 ++ .../torrent/torrentleech/__init__.py | 14 ++ .../torrent/torrentshack/__init__.py | 14 ++ couchpotato/static/scripts/couchpotato.js | 4 +- 25 files changed, 571 insertions(+), 204 deletions(-) diff --git a/couchpotato/core/_base/_core/main.py b/couchpotato/core/_base/_core/main.py index 4ad37d6c..9647d959 100644 --- a/couchpotato/core/_base/_core/main.py +++ b/couchpotato/core/_base/_core/main.py @@ -124,7 +124,7 @@ class Core(Plugin): time.sleep(1) - log.debug('Save to shutdown/restart') + log.debug('Safe to shutdown/restart') try: IOLoop.current().stop() diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 900fd8c0..5ffad3da 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -39,6 +39,8 @@ class Downloader(Provider): addEvent('download.enabled_types', self.getEnabledDownloadType) addEvent('download.status', self._getAllDownloadStatus) addEvent('download.remove_failed', self._removeFailed) + addEvent('download.pause', self._pause) + addEvent('download.process_complete', self._processComplete) def getEnabledDownloadType(self): for download_type in self.type: @@ -65,14 +67,30 @@ class Downloader(Provider): if self.isDisabled(manual = True, data = {}): return - if self.conf('delete_failed', default = True): - return self.removeFailed(item) + if item and item.get('downloader') == self.getName(): + if self.conf('delete_failed'): + return self.removeFailed(item) - return False + return False + return def removeFailed(self, item): return + def _processComplete(self, item): + if self.isDisabled(manual = True, data = {}): + return + + if item and item.get('downloader') == self.getName(): + if self.conf('remove_complete'): + return self.processComplete(item = item, delete_files = self.conf('delete_files')) + + return False + return + + def processComplete(self, item, delete_files): + return + def isCorrectType(self, item_type): is_correct = item_type in self.type @@ -124,6 +142,17 @@ class Downloader(Provider): ((d_manual and manual) or (d_manual is False)) and \ (not data or self.isCorrectType(data.get('type'))) + def _pause(self, item, pause = True): + if self.isDisabled(manual = True, data = {}): + return + + if item and item.get('downloader') == self.getName(): + self.pause(item, pause) + return True + return + + def pause(self, item, pause): + return class StatusList(list): diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 6dfbd3f9..8f19e34f 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -44,18 +44,32 @@ config = [{ 'description': 'Download to this directory. Keep empty for default Transmission download directory.', }, { - 'name': 'ratio', - 'default': 10, - 'type': 'float', + 'name': 'seeding', + 'label': 'Seeding support', + 'default': True, + 'type': 'bool', + 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving. Set the seeding goal in the torrent providers.', + }, + { + 'name': 'remove_complete', + 'label': 'Remove torrent', + 'default': True, + 'type': 'bool', + 'description': 'Remove the torrent from Transmission after it finished seeding.', + }, + { + 'name': 'delete_files', + 'label': 'Remove files', + 'default': True, + 'type': 'bool', 'advanced': True, - 'description': 'Stop transfer when reaching ratio', + 'description': 'Also remove the leftover files.', }, { - 'name': 'ratiomode', - 'default': 0, - 'type': 'int', - 'advanced': True, - 'description': '0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.', + 'name': 'paused', + 'type': 'bool', + 'default': False, + 'description': 'Add the torrent paused.', }, { 'name': 'manual', diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 6094e89b..dc798d2b 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -1,6 +1,7 @@ from base64 import b64encode from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import isInt +from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog from couchpotato.environment import Env from datetime import timedelta @@ -8,7 +9,6 @@ import httplib import json import os.path import re -import traceback import urllib2 log = CPLog(__name__) @@ -18,144 +18,123 @@ class Transmission(Downloader): type = ['torrent', 'torrent_magnet'] log = CPLog(__name__) + trpc = None + + def connect(self): + # Load host from config and split out port. + host = self.conf('host').split(':') + if not isInt(host[1]): + log.error('Config properties are not filled in correctly, port is missing.') + return False + if not self.trpc: + self.trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + return self.trpc def download(self, data, movie, filedata = None): log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) - # Load host from config and split out port. - host = self.conf('host').split(':') - if not isInt(host[1]): - log.error('Config properties are not filled in correctly, port is missing.') + if not self.connect(): return False - # Set parameters for Transmission - params = { - 'paused': self.conf('paused', default = 0), - } - - if len(self.conf('directory', default = '')) > 0: - folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] - params['download-dir'] = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep) - - torrent_params = {} - if self.conf('ratio'): - torrent_params = { - 'seedRatioLimit': self.conf('ratio'), - 'seedRatioMode': self.conf('ratiomode') - } - if not filedata and data.get('type') == 'torrent': log.error('Failed sending torrent, no data') return False - # Send request to Transmission - try: - trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) - if data.get('type') == 'torrent_magnet': - remote_torrent = trpc.add_torrent_uri(data.get('url'), arguments = params) - torrent_params['trackerAdd'] = self.torrent_trackers + # Set parameters for adding torrent + params = {} + params['paused'] = self.conf('paused', default = False) + + if self.conf('directory'): + if os.path.isdir(self.conf('directory')): + params['download-dir'] = self.conf('directory') else: - remote_torrent = trpc.add_torrent_file(b64encode(filedata), arguments = params) + log.error('Download directory from Transmission settings: %s doesn\'t exist', self.conf('directory')) - if not remote_torrent: - return False + # Change parameters of torrent + torrent_params = {} + if data.get('seed_ratio') and self.conf('seeding'): + torrent_params['seedRatioLimit'] = tryFloat(data.get('seed_ratio')) + torrent_params['seedRatioMode'] = 1 - # Change settings of added torrents - elif torrent_params: - trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) + if data.get('seed_time') and self.conf('seeding'): + torrent_params['seedIdleLimit'] = tryInt(data.get('seed_time'))*60 + torrent_params['seedIdleMode'] = 1 - log.info('Torrent sent to Transmission successfully.') - return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) - except: - log.error('Failed to change settings for transfer: %s', traceback.format_exc()) + # Send request to Transmission + if data.get('type') == 'torrent_magnet': + remote_torrent = self.trpc.add_torrent_uri(data.get('url'), arguments = params) + torrent_params['trackerAdd'] = self.torrent_trackers + else: + remote_torrent = self.trpc.add_torrent_file(b64encode(filedata), arguments = params) + + if not remote_torrent: + log.error('Failed sending torrent to Transmission') return False + # Change settings of added torrents + if torrent_params: + self.trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) + + log.info('Torrent sent to Transmission successfully.') + return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) + def getAllDownloadStatus(self): log.debug('Checking Transmission download status.') - # Load host from config and split out port. - host = self.conf('host').split(':') - if not isInt(host[1]): - log.error('Config properties are not filled in correctly, port is missing.') + if not self.connect(): return False - # Go through Queue - try: - trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) - return_params = { - 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isFinished', 'downloadDir', 'uploadRatio'] - } - queue = trpc.get_alltorrents(return_params) - except Exception, err: - log.error('Failed getting queue: %s', err) - return False - - if not queue: - return [] - statuses = StatusList(self) - # Get torrents status - # CouchPotato Status - #status = 'busy' - #status = 'failed' - #status = 'completed' - # Transmission Status - #status = 0 => "Torrent is stopped" - #status = 1 => "Queued to check files" - #status = 2 => "Checking files" - #status = 3 => "Queued to download" - #status = 4 => "Downloading" - #status = 4 => "Queued to seed" - #status = 6 => "Seeding" - #To do : - # add checking file - # manage no peer in a range time => fail + return_params = { + 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit'] + } + + queue = self.trpc.get_alltorrents(return_params) + if not (queue and queue.get('torrents')): + log.debug('Nothing in queue or error') + return False for item in queue['torrents']: - log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'])) + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], item['isFinished'])) if not os.path.isdir(Env.setting('from', 'renamer')): log.error('Renamer "from" folder doesn\'t to exist.') return - if (item['percentDone'] * 100) >= 100 and (item['status'] == 6 or item['status'] == 0) and item['uploadRatio'] > self.conf('ratio'): - try: - trpc.stop_torrent(item['hashString'], {}) - statuses.append({ - 'id': item['hashString'], - 'name': item['name'], - 'status': 'completed', - 'original_status': item['status'], - 'timeleft': str(timedelta(seconds = 0)), - 'folder': os.path.join(item['downloadDir'], item['name']), - }) - except Exception, err: - log.error('Failed to stop and remove torrent "%s" with error: %s', (item['name'], err)) - statuses.append({ - 'id': item['hashString'], - 'name': item['name'], - 'status': 'failed', - 'original_status': item['status'], - 'timeleft': str(timedelta(seconds = 0)), - }) - else: - statuses.append({ - 'id': item['hashString'], - 'name': item['name'], - 'status': 'busy', - 'original_status': item['status'], - 'timeleft': str(timedelta(seconds = item['eta'])), # Is ETA in seconds?? - }) + status = 'busy' + if item['status'] == 0 and item['percentDone'] == 1: + status = 'completed' + elif item['status'] in [5, 6]: + status = 'seeding' + + statuses.append({ + 'id': item['hashString'], + 'name': item['name'], + 'status': status, + 'original_status': item['status'], + 'seed_ratio': item['uploadRatio'], + 'timeleft': str(timedelta(seconds = item['eta'])), + 'folder': os.path.join(item['downloadDir'], item['name']), + }) return statuses + def pause(self, item, pause = True): + if pause: + return self.trpc.stop_torrent(item['hashString']) + else: + return self.trpc.start_torrent(item['hashString']) + + def processComplete(self, item, delete_files = False): + log.debug('Requesting Transmission to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) + return self.trpc.remove_torrent(self, item['hashString'], delete_files) + class TransmissionRPC(object): """TransmissionRPC lite library""" - def __init__(self, host = 'localhost', port = 9091, username = None, password = None): super(TransmissionRPC, self).__init__() @@ -184,7 +163,7 @@ class TransmissionRPC(object): log.debug('request: %s', json.dumps(ojson)) log.debug('response: %s', json.dumps(response)) if response['result'] == 'success': - log.debug('Transmission action successfull') + log.debug('Transmission action successful') return response['arguments'] else: log.debug('Unknown failure sending command to Transmission. Return text is: %s', response['result']) @@ -236,13 +215,15 @@ class TransmissionRPC(object): post_data = {'arguments': arguments, 'method': 'torrent-get', 'tag': self.tag} return self._request(post_data) - def stop_torrent(self, torrent_id, arguments): - arguments['ids'] = torrent_id - post_data = {'arguments': arguments, 'method': 'torrent-stop', 'tag': self.tag} + def stop_torrent(self, torrent_id): + post_data = {'arguments': {'ids': torrent_id}, 'method': 'torrent-stop', 'tag': self.tag} return self._request(post_data) - def remove_torrent(self, torrent_id, remove_local_data, arguments): - arguments['ids'] = torrent_id - arguments['delete-local-data'] = remove_local_data - post_data = {'arguments': arguments, 'method': 'torrent-remove', 'tag': self.tag} + def start_torrent(self, torrent_id): + post_data = {'arguments': {'ids': torrent_id}, 'method': 'torrent-start', 'tag': self.tag} return self._request(post_data) + + def remove_torrent(self, torrent_id, delete_local_data): + post_data = {'arguments': {'ids': torrent_id, 'delete-local-data': delete_local_data}, 'method': 'torrent-remove', 'tag': self.tag} + return self._request(post_data) + diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 2c494eb2..674a52a4 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -36,6 +36,28 @@ config = [{ 'name': 'label', 'description': 'Label to add torrent as.', }, + { + 'name': 'seeding', + 'label': 'Seeding support', + 'default': True, + 'type': 'bool', + 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving. Stop seeding manually in uTorrent, or check the option Queueing->When uTorrent reaches the seeding goal->Limit the upload rate and set it to 0 to stop seeding after the seeding goal set in the torrent providers is met.', + }, + { + 'name': 'remove_complete', + 'label': 'Remove torrent', + 'default': True, + 'type': 'bool', + 'description': 'Remove the torrent from uTorrent after it finished seeding.', + }, + { + 'name': 'delete_files', + 'label': 'Remove files', + 'default': True, + 'type': 'bool', + 'advanced': True, + 'description': 'Also remove the leftover files.', + }, { 'name': 'paused', 'type': 'bool', diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index f49859ab..cc02b803 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -2,6 +2,7 @@ from base64 import b16encode, b32decode from bencode import bencode, bdecode from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import isInt, ss +from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 @@ -23,16 +24,28 @@ class uTorrent(Downloader): type = ['torrent', 'torrent_magnet'] utorrent_api = None - def download(self, data, movie, filedata = None): - - log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('type'))) - + def connect(self): # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False + if not self.utorrent_api: + self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + return self.utorrent_api + + def download(self, data, movie, filedata = None): + + log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('type'))) + + if not self.connect(): + return False + + settings = self.utorrent_api.get_settings() + if not settings: + return False + torrent_params = {} if self.conf('label'): torrent_params['label'] = self.conf('label') @@ -49,75 +62,82 @@ class uTorrent(Downloader): torrent_hash = sha1(bencode(info)).hexdigest().upper() torrent_filename = self.createFileName(data, filedata, movie) + if data.get('seed_ratio') and self.conf('seeding'): + torrent_params['seed_override'] = 1 + torrent_params['seed_ratio'] = tryInt(tryFloat(data['seed_ratio'])*1000) + + # Check if uTorrent completes the torrent if seeding goal is met. + # Note that CPS can also check if the goal has been met but for now it should be done by uTorrent + if not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): + log.info('With the current settings uTorrent does not set torrents that completed the seed ratio and time to complete. Please stop them manually in uTorrent or check the option Queueing->When uTorrent reaches the seeding goal->Limit the upload rate and set it to 0') + + if data.get('seed_time') and self.conf('seeding'): + torrent_params['seed_override'] = 1 + torrent_params['seed_time'] = tryInt(data['seed_time'])*3600 + + # Check if uTorrent completes the torrent if seeding goal is met. + # Note that CPS can also check if the goal has been met but for now it should be done by uTorrent + if not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): + log.info('With the current settings uTorrent does not set torrents that completed the seed ratio and time to complete. Please stop them manually in uTorrent or check the option Queueing->When uTorrent reaches the seeding goal->Limit the upload rate and set it to 0') + # Convert base 32 to hex if len(torrent_hash) == 32: torrent_hash = b16encode(b32decode(torrent_hash)) # Send request to uTorrent - try: - if not self.utorrent_api: - self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + if data.get('type') == 'torrent_magnet': + self.utorrent_api.add_torrent_uri(data.get('url')) + else: + self.utorrent_api.add_torrent_file(torrent_filename, filedata) - if data.get('type') == 'torrent_magnet': - self.utorrent_api.add_torrent_uri(data.get('url')) - else: - self.utorrent_api.add_torrent_file(torrent_filename, filedata) + # Change settings of added torrents + self.utorrent_api.set_torrent(torrent_hash, torrent_params) + if self.conf('paused', default = 0): + self.utorrent_api.pause_torrent(torrent_hash) - # Change settings of added torrents - self.utorrent_api.set_torrent(torrent_hash, torrent_params) - if self.conf('paused', default = 0): - self.utorrent_api.pause_torrent(torrent_hash) - return self.downloadReturnId(torrent_hash) - except Exception, err: - log.error('Failed to send torrent to uTorrent: %s', err) - return False + return self.downloadReturnId(torrent_hash) def getAllDownloadStatus(self): log.debug('Checking uTorrent download status.') - # Load host from config and split out port. - host = self.conf('host').split(':') - if not isInt(host[1]): - log.error('Config properties are not filled in correctly, port is missing.') - return False - - try: - self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) - except Exception, err: - log.error('Failed to get uTorrent object: %s', err) - return False - - data = '' - try: - data = self.utorrent_api.get_status() - queue = json.loads(data) - if queue.get('error'): - log.error('Error getting data from uTorrent: %s', queue.get('error')) - return False - - except Exception, err: - log.error('Failed to get status from uTorrent: %s', err) - return False - - if queue.get('torrents', []) == []: - log.debug('Nothing in queue') + if not self.connect(): return False statuses = StatusList(self) + data = self.utorrent_api.get_status() + if not data: + log.error('Error getting data from uTorrent') + return False + + queue = json.loads(data) + if queue.get('error'): + log.error('Error getting data from uTorrent: %s', queue.get('error')) + return False + + if not queue.get('torrents'): + log.debug('Nothing in queue') + return False + # Get torrents - for item in queue.get('torrents', []): + for item in queue['torrents']: # item[21] = Paused | Downloading | Seeding | Finished status = 'busy' - if item[21] == 'Finished' or item[21] == 'Seeding': + if 'Finished' in item[21]: status = 'completed' + elif 'Seeding' in item[21]: + if self.conf('seeding'): + status = 'seeding' + else: + status = 'completed' statuses.append({ 'id': item[0], 'name': item[2], 'status': status, + 'seed_ratio': float(item[7])/1000, 'original_status': item[1], 'timeleft': str(timedelta(seconds = item[10])), 'folder': item[26], @@ -125,7 +145,16 @@ class uTorrent(Downloader): return statuses + def pause(self, download_info, pause = True): + if not self.connect(): + return False + return self.utorrent_api.pause_torrent(download_info['id'], pause) + def processComplete(self, item, delete_files = False): + log.debug('Requesting uTorrent to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) + if not self.connect(): + return False + return self.utorrent_api.remove_torrent(item['id'], remove_data = delete_files) class uTorrentAPI(object): @@ -190,10 +219,24 @@ class uTorrentAPI(object): action += "&s=%s&v=%s" % (k, v) return self._request(action) - def pause_torrent(self, hash): - action = "action=pause&hash=%s" % hash + def pause_torrent(self, hash, pause = True): + if pause: + action = "action=pause&hash=%s" % hash + else: + action = "action=unpause&hash=%s" % hash return self._request(action) + def stop_torrent(self, hash): + action = "action=stop&hash=%s" % hash + return self._request(action) + + def remove_torrent(self, hash, remove_data = False): + if remove_data: + action = "action=removedata&hash=%s" % hash + else: + action = "action=remove&hash=%s" % hash + return self._request(action) + def get_status(self): action = "list=1" return self._request(action) diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index 155a939b..e2bb4b93 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -121,7 +121,7 @@ config = [{ 'label': 'Torrent File Action', 'default': 'move', 'type': 'dropdown', - 'values': [('Move', 'move'), ('Copy', 'copy'), ('Hard link', 'hardlink'), ('Sym link', 'symlink'), ('Move & Sym link', 'move_symlink')], + 'values': [('Move', 'move'), ('Copy', 'copy'), ('Hard link', 'hardlink'), ('Move & Sym link', 'move_symlink')], 'description': 'Define which kind of file operation you want to use for torrents. Before you start using hard links or sym links, PLEASE read about their possible drawbacks.', 'advanced': True, }, diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index c219f17b..76f6c675 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -10,6 +10,7 @@ from couchpotato.core.settings.model import Library, File, Profile, Release, \ ReleaseInfo from couchpotato.environment import Env import errno +import fnmatch import os import re import shutil @@ -38,7 +39,6 @@ class Renamer(Plugin): addEvent('renamer.check_snatched', self.checkSnatched) addEvent('app.load', self.scan) - addEvent('app.load', self.checkSnatched) addEvent('app.load', self.setCrons) # Enable / disable interval @@ -65,18 +65,19 @@ class Renamer(Plugin): downloader = kwargs.get('downloader', None) download_id = kwargs.get('download_id', None) + download_info = {'folder': movie_folder} if movie_folder else None + if download_info: + download_info.update({'id': download_id, 'downloader': downloader} if download_id else {}) + fire_handle = fireEvent if not async else fireEventAsync - fire_handle('renamer.scan', - movie_folder = movie_folder, - download_info = {'id': download_id, 'downloader': downloader} if download_id else None - ) + fire_handle('renamer.scan', download_info) return { 'success': True } - def scan(self, movie_folder = None, download_info = None): + def scan(self, download_info = None): if self.isDisabled(): return @@ -85,6 +86,8 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return + movie_folder = download_info and download_info.get('folder') + # Check to see if the "to" folder is inside the "from" folder. if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')): l = log.debug if movie_folder else log.error @@ -97,6 +100,10 @@ class Renamer(Plugin): log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.') return + # Make sure a checkSnatched marked all downloads/seeds as such + if not download_info and self.conf('run_every') > 0: + fireEvent('renamer.check_snatched') + self.renaming_started = True # make sure the movie folder name is included in the search @@ -144,7 +151,7 @@ class Renamer(Plugin): # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: - self.tagDir(group, 'unknown') + self.tagDir(group['parentdir'], 'unknown') continue # Rename the files using the library data else: @@ -192,7 +199,7 @@ class Renamer(Plugin): # Move nfo depending on settings if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled', file_type) - if self.conf('cleanup'): + if self.conf('cleanup') and not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)): for current_file in group['files'][file_type]: remove_files.append(current_file) continue @@ -354,7 +361,7 @@ class Renamer(Plugin): log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label)) # Add exists tag to the .ignore file - self.tagDir(group, 'exists') + self.tagDir(group['parentdir'], 'exists') # Notify on rename fail download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) @@ -405,7 +412,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (src, traceback.format_exc())) - self.tagDir(group, 'failed_remove') + self.tagDir(group['parentdir'], 'failed_remove') # Delete leftover folder from older releases for delete_folder in delete_folders: @@ -425,14 +432,16 @@ class Renamer(Plugin): self.makeDir(os.path.dirname(dst)) try: - self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(download_info)) + self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(download_info) or self.fileIsAdded(src, group)) group['renamed_files'].append(dst) except: log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) - self.tagDir(group, 'failed_rename') + self.tagDir(group['parentdir'], 'failed_rename') - if self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info): - self.tagDir(group, 'renamed already') + # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent + if (movie_folder and self.conf('from') in movie_folder or not movie_folder) and \ + self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info): + self.tagDir(group['parentdir'], 'renamed_already') # Remove matching releases for release in remove_releases: @@ -480,12 +489,9 @@ class Renamer(Plugin): return rename_files # This adds a file to ignore / tag a release so it is ignored later - def tagDir(self, group, tag): - - ignore_file = None - for movie_file in sorted(list(group['files']['movie'])): - ignore_file = '%s.ignore' % os.path.splitext(movie_file)[0] - break + def tagDir(self, folder, tag): + if not os.path.isdir(folder) or not tag: + return text = """This file is from CouchPotato It has marked this release as "%s" @@ -493,9 +499,27 @@ This file hides the release from the renamer Remove it if you want it to be renamed (again, or at least let it try again) """ % tag - if ignore_file: - self.createFile(ignore_file, text) + self.createFile(os.path.join(folder, '%s.ignore' % tag), text) + def untagDir(self, folder, tag = None): + if not os.path.isdir(folder): + return + + # Remove any .ignore files + for root, dirnames, filenames in os.walk(folder): + for filename in fnmatch.filter(filenames, '%s.ignore' % tag if tag else '*'): + os.remove((os.path.join(root, filename))) + + def hastagDir(self, folder, tag = None): + if not os.path.isdir(folder): + return False + + # Find any .ignore files + for root, dirnames, filenames in os.walk(folder): + if fnmatch.filter(filenames, '%s.ignore' % tag if tag else '*'): + return True + + return False def moveFile(self, old, dest, forcemove = False): dest = ss(dest) @@ -504,8 +528,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) shutil.move(old, dest) elif self.conf('file_action') == 'hardlink': link(old, dest) - elif self.conf('file_action') == 'symlink': - symlink(old, dest) elif self.conf('file_action') == 'copy': shutil.copy(old, dest) elif self.conf('file_action') == 'move_symlink': @@ -584,19 +606,21 @@ Remove it if you want it to be renamed (again, or at least let it try again) if self.checking_snatched: log.debug('Already checking snatched') + return False self.checking_snatched = True - snatched_status, ignored_status, failed_status, done_status = \ - fireEvent('status.get', ['snatched', 'ignored', 'failed', 'done'], single = True) + snatched_status, ignored_status, failed_status, done_status, seeding_status, downloaded_status = \ + fireEvent('status.get', ['snatched', 'ignored', 'failed', 'done', 'seeding', 'downloaded'], single = True) db = get_session() rels = db.query(Release).filter_by(status_id = snatched_status.get('id')).all() + rels.extend(db.query(Release).filter_by(status_id = seeding_status.get('id')).all()) + scan_items = [] scan_required = False if rels: - self.checking_snatched = True log.debug('Checking status snatched releases...') statuses = fireEvent('download.status', merge = True) @@ -612,7 +636,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) default_title = getTitle(rel.movie.library) # Check if movie has already completed and is manage tab (legacy db correction) - if rel.movie.status_id == done_status.get('id'): + if rel.movie.status_id == done_status.get('id') and rel.status_id == snatched_status.get('id'): log.debug('Found a completed movie with a snatched release : %s. Setting release status to ignored...' , default_title) rel.status_id = ignored_status.get('id') rel.last_edit = int(time.time()) @@ -640,7 +664,30 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.debug('Found %s: %s, time to go: %s', (item['name'], item['status'].upper(), timeleft)) if item['status'] == 'busy': + # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading + if item['folder'] and self.conf('from') in item['folder']: + self.tagDir(item['folder'], 'downloading') + pass + elif item['status'] == 'seeding': + #If linking setting is enabled, process release + if self.conf('file_action') != 'move' and not rel.movie.status_id == done_status.get('id') and item['id'] and item['downloader'] and item['folder']: + log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (item['name'], item['seed_ratio'])) + + # Remove the downloading tag + self.untagDir(item['folder'], 'downloading') + + rel.status_id = seeding_status.get('id') + rel.last_edit = int(time.time()) + db.commit() + + # Scan and set the torrent to paused if required + item.update({'pause': True, 'scan': True, 'process_complete': False}) + scan_items.append(item) + else: + #let it seed + log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio'])) + pass elif item['status'] == 'failed': fireEvent('download.remove_failed', item, single = True) rel.status_id = failed_status.get('id') @@ -652,7 +699,35 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif item['status'] == 'completed': log.info('Download of %s completed!', item['name']) if item['id'] and item['downloader'] and item['folder']: - fireEventAsync('renamer.scan', movie_folder = item['folder'], download_info = item) + + # If the release has been seeding, process now the seeding is done + if rel.status_id == seeding_status.get('id'): + if rel.movie.status_id == done_status.get('id'): # and self.conf('file_action') != 'move': + # Set the release to done as the movie has already been renamed + rel.status_id = downloaded_status.get('id') + rel.last_edit = int(time.time()) + db.commit() + + # Allow the downloader to clean-up + item.update({'pause': False, 'scan': False, 'process_complete': True}) + scan_items.append(item) + else: + # Set the release to snatched so that the renamer can process the release as if it was never seeding + rel.status_id = snatched_status.get('id') + rel.last_edit = int(time.time()) + db.commit() + + # Scan and Allow the downloader to clean-up + item.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_items.append(item) + + else: + # Remove the downloading tag + self.untagDir(item['folder'], 'downloading') + + # Scan and Allow the downloader to clean-up + item.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_items.append(item) else: scan_required = True @@ -665,6 +740,23 @@ Remove it if you want it to be renamed (again, or at least let it try again) except: log.error('Failed checking for release in downloader: %s', traceback.format_exc()) + # The following can either be done here, or inside the scanner if we pass it scan_items in one go + for item in scan_items: + # Ask the renamer to scan the item + if item['scan']: + if item['pause'] and self.conf('file_action') == 'move_symlink': + fireEvent('download.pause', item = item, pause = True, single = True) + fireEvent('renamer.scan', download_info = item) + if item['pause'] and self.conf('file_action') == 'move_symlink': + fireEvent('download.pause', item = item, pause = False, single = True) + if item['process_complete']: + #First make sure the files were succesfully processed + if not self.hastagDir(item['folder'], 'failed_rename'): + # Remove the seeding tag if it exists + self.untagDir(item['folder'], 'renamed_already') + # Ask the downloader to process the item + fireEvent('download.process_complete', item = item, single = True) + if scan_required: fireEvent('renamer.scan') @@ -706,3 +798,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) def downloadIsTorrent(self, download_info): return download_info and download_info.get('type') in ['torrent', 'torrent_magnet'] + + def fileIsAdded(self, src, group): + if not group['files'].get('added'): + return False + return src in group['files']['added'] + \ No newline at end of file diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index a400b159..e48d2747 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -225,6 +225,10 @@ class Scanner(Plugin): # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) + exts = [getExt(ff) for ff in found_files] + if 'ignore' in exts: + ignored_identifiers.append(identifier) + # Break if CP wants to shut down if self.shuttingDown(): break @@ -251,6 +255,10 @@ class Scanner(Plugin): # Remove the found files from the leftover stack leftovers = leftovers - set([ff]) + ext = getExt(ff) + if ext == 'ignore': + ignored_identifiers.append(new_identifier) + # Break if CP wants to shut down if self.shuttingDown(): break diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py index c8c8f666..8db2bf77 100644 --- a/couchpotato/core/plugins/status/main.py +++ b/couchpotato/core/plugins/status/main.py @@ -23,6 +23,7 @@ class StatusPlugin(Plugin): 'ignored': 'Ignored', 'available': 'Available', 'suggest': 'Suggest', + 'seeding': 'Seeding', } status_cached = {} diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index c6bef6ae..0ea1de30 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -60,6 +60,7 @@ class Subtitle(Plugin): for d_sub in downloaded: log.info('Found subtitle (%s): %s', (d_sub.language.alpha2, files)) group['files']['subtitle'].append(d_sub.path) + group['files']['added'].append(d_sub.path) group['subtitle_language'][d_sub.path] = [d_sub.language.alpha2] return True diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 182a0310..cb7b16da 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -276,6 +276,8 @@ class ResultList(list): 'type': self.provider.type, 'provider': self.provider.getName(), 'download': self.provider.loginDownload if self.provider.urls.get('login') else self.provider.download, + 'seed_ratio': Env.setting('seed_ratio', section = self.provider.getName().lower(), default = ''), + 'seed_time': Env.setting('seed_time', section = self.provider.getName().lower(), default = ''), 'url': '', 'name': '', 'age': 0, diff --git a/couchpotato/core/providers/torrent/hdbits/__init__.py b/couchpotato/core/providers/torrent/hdbits/__init__.py index 8a9fc80e..0b370e19 100644 --- a/couchpotato/core/providers/torrent/hdbits/__init__.py +++ b/couchpotato/core/providers/torrent/hdbits/__init__.py @@ -31,6 +31,20 @@ config = [{ 'name': 'passkey', 'default': '', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/iptorrents/__init__.py b/couchpotato/core/providers/torrent/iptorrents/__init__.py index 24f9772b..c1eea5cd 100644 --- a/couchpotato/core/providers/torrent/iptorrents/__init__.py +++ b/couchpotato/core/providers/torrent/iptorrents/__init__.py @@ -34,6 +34,20 @@ config = [{ 'type': 'bool', 'description': 'Only search for [FreeLeech] torrents.', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py index 999dbb1b..90f9eeac 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py @@ -19,6 +19,20 @@ config = [{ 'type': 'enabler', 'default': True, }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py index a535034a..f0cb966e 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py @@ -62,6 +62,20 @@ config = [{ 'default': 0, 'description': 'Require staff-approval for releases to be accepted.' }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, @@ -71,6 +85,6 @@ config = [{ 'description': 'Starting score for each release found via this provider.', } ], -} + } ] }] diff --git a/couchpotato/core/providers/torrent/publichd/__init__.py b/couchpotato/core/providers/torrent/publichd/__init__.py index 3c27cf48..22e2dbb8 100644 --- a/couchpotato/core/providers/torrent/publichd/__init__.py +++ b/couchpotato/core/providers/torrent/publichd/__init__.py @@ -19,6 +19,20 @@ config = [{ 'type': 'enabler', 'default': True, }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/sceneaccess/__init__.py b/couchpotato/core/providers/torrent/sceneaccess/__init__.py index baad57f6..786f28a6 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/__init__.py +++ b/couchpotato/core/providers/torrent/sceneaccess/__init__.py @@ -28,6 +28,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/scenehd/__init__.py b/couchpotato/core/providers/torrent/scenehd/__init__.py index 3cd2132e..9b967f4b 100644 --- a/couchpotato/core/providers/torrent/scenehd/__init__.py +++ b/couchpotato/core/providers/torrent/scenehd/__init__.py @@ -28,6 +28,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index f2394dd6..2c902439 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -25,6 +25,20 @@ config = [{ 'label': 'Proxy server', 'description': 'Domain for requests, keep empty to let CouchPotato pick.', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/torrentbytes/__init__.py b/couchpotato/core/providers/torrent/torrentbytes/__init__.py index 10e581a6..c7f4437d 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/__init__.py +++ b/couchpotato/core/providers/torrent/torrentbytes/__init__.py @@ -28,6 +28,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/torrentday/__init__.py b/couchpotato/core/providers/torrent/torrentday/__init__.py index de715b53..d3bbaa14 100644 --- a/couchpotato/core/providers/torrent/torrentday/__init__.py +++ b/couchpotato/core/providers/torrent/torrentday/__init__.py @@ -28,6 +28,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/torrentleech/__init__.py b/couchpotato/core/providers/torrent/torrentleech/__init__.py index fa048d50..d3ee7616 100644 --- a/couchpotato/core/providers/torrent/torrentleech/__init__.py +++ b/couchpotato/core/providers/torrent/torrentleech/__init__.py @@ -28,6 +28,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/torrentshack/__init__.py b/couchpotato/core/providers/torrent/torrentshack/__init__.py index 203e0996..69ad176d 100644 --- a/couchpotato/core/providers/torrent/torrentshack/__init__.py +++ b/couchpotato/core/providers/torrent/torrentshack/__init__.py @@ -27,6 +27,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'scene_only', 'type': 'bool', diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 8e6ccede..db90c305 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -1,4 +1,4 @@ -var CouchPotato = new Class({ +var CouchPotato = new Class({ Implements: [Events, Options], @@ -179,7 +179,7 @@ var CouchPotato = new Class({ shutdown: function(){ var self = this; - self.blockPage('You have shutdown. This is what suppose to happen ;)'); + self.blockPage('You have shutdown. This is what is supposed to happen ;)'); Api.request('app.shutdown', { 'onComplete': self.blockPage.bind(self) }); From 7ed43da42530a5a6c29d1a7b0bd7edb90756a551 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 22 Jun 2013 16:22:33 +0200 Subject: [PATCH 004/209] Also set seeding status in case nothing is done --- couchpotato/core/plugins/renamer/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 76f6c675..92d7f905 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -685,6 +685,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) item.update({'pause': True, 'scan': True, 'process_complete': False}) scan_items.append(item) else: + if rel.status_id != seeding_status.get('id'): + rel.status_id = seeding_status.get('id') + rel.last_edit = int(time.time()) + db.commit() + #let it seed log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio'])) pass From cdee08bd367e623fa0eb218f2534fda9fb24da99 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 22 Jun 2013 16:24:59 +0200 Subject: [PATCH 005/209] Add status colours in dashboard --- couchpotato/core/plugins/movie/static/movie.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/static/movie.css b/couchpotato/core/plugins/movie/static/movie.css index 04600475..dccd6cd6 100644 --- a/couchpotato/core/plugins/movie/static/movie.css +++ b/couchpotato/core/plugins/movie/static/movie.css @@ -425,7 +425,9 @@ } .movies .data .quality .available { background-color: #578bc3; } - .movies .data .quality .snatched { background-color: #369545; } + .movies .data .quality .failed { background-color: #a43d34; } + .movies .data .quality .snatched { background-color: #a2a232; } + .movies .data .quality .seeding { background-color: #0a6819; } .movies .data .quality .done { background-color: #369545; opacity: 1; From 628c0e5dcc42fa75e377e3a209a9a99093beccff Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Wed, 26 Jun 2013 19:52:39 +0200 Subject: [PATCH 006/209] Add yify torrent provider --- .../core/providers/torrent/yify/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/couchpotato/core/providers/torrent/yify/__init__.py b/couchpotato/core/providers/torrent/yify/__init__.py index f7477519..23f9e34a 100644 --- a/couchpotato/core/providers/torrent/yify/__init__.py +++ b/couchpotato/core/providers/torrent/yify/__init__.py @@ -19,6 +19,20 @@ config = [{ 'type': 'enabler', 'default': True }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, From 84e9f9794d8f750b8de0afa5609d956f0bfc2c51 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Wed, 26 Jun 2013 19:53:28 +0200 Subject: [PATCH 007/209] Add awesomehd torrent provider --- .../core/providers/torrent/awesomehd/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/couchpotato/core/providers/torrent/awesomehd/__init__.py b/couchpotato/core/providers/torrent/awesomehd/__init__.py index 5c8c9794..e2587a52 100644 --- a/couchpotato/core/providers/torrent/awesomehd/__init__.py +++ b/couchpotato/core/providers/torrent/awesomehd/__init__.py @@ -23,6 +23,20 @@ config = [{ 'name': 'passkey', 'default': '', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'only_internal', 'advanced': True, From 18a88eab510d04863aa75c5c9fb19c5bf6819072 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Wed, 26 Jun 2013 20:02:25 +0200 Subject: [PATCH 008/209] Textual change --- couchpotato/core/downloaders/transmission/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 8f19e34f..7805dd01 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -48,7 +48,7 @@ config = [{ 'label': 'Seeding support', 'default': True, 'type': 'bool', - 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving. Set the seeding goal in the torrent providers.', + 'description': '(Hard)link/copy after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving and set the seeding goal from the torrent providers.', }, { 'name': 'remove_complete', From cfd23c395a8fdbc585efd6548b5f142e07d9df57 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 29 Jun 2013 10:23:08 +0200 Subject: [PATCH 009/209] Add failed download handling to Transmission --- .../core/downloaders/transmission/__init__.py | 12 ++++++++++++ couchpotato/core/downloaders/transmission/main.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 7805dd01..bca7eae5 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -78,6 +78,18 @@ config = [{ 'advanced': True, 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', }, + { + 'name': 'stalled_as_failed', + 'default': True, + 'type': 'bool', + 'description': 'Consider a stalled torrent as failed', + }, + { + 'name': 'delete_failed', + 'default': True, + 'type': 'bool', + 'description': 'Delete a release after the download has failed.', + }, ], } ], diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index dc798d2b..46b9541f 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -89,7 +89,7 @@ class Transmission(Downloader): statuses = StatusList(self) return_params = { - 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit'] + 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit'] } queue = self.trpc.get_alltorrents(return_params) @@ -105,7 +105,9 @@ class Transmission(Downloader): return status = 'busy' - if item['status'] == 0 and item['percentDone'] == 1: + if item['isStalled'] and self.conf('stalled_as_failed'): + status = 'failed' + elif item['status'] == 0 and item['percentDone'] == 1: status = 'completed' elif item['status'] in [5, 6]: status = 'seeding' @@ -128,6 +130,10 @@ class Transmission(Downloader): else: return self.trpc.start_torrent(item['hashString']) + def removeFailed(self, item): + log.info('%s failed downloading, deleting...', item['name']) + return self.trpc.remove_torrent(self, item['hashString'], True) + def processComplete(self, item, delete_files = False): log.debug('Requesting Transmission to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) return self.trpc.remove_torrent(self, item['hashString'], delete_files) From 7411670e2279827e494ef4195c4ea76527b8c27d Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 29 Jun 2013 10:36:02 +0200 Subject: [PATCH 010/209] Added complete download removal to SabNZBd --- couchpotato/core/downloaders/base.py | 2 +- couchpotato/core/downloaders/sabnzbd/__init__.py | 7 +++++++ couchpotato/core/downloaders/sabnzbd/main.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 5ffad3da..adbfb7ec 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -83,7 +83,7 @@ class Downloader(Provider): if item and item.get('downloader') == self.getName(): if self.conf('remove_complete'): - return self.processComplete(item = item, delete_files = self.conf('delete_files')) + return self.processComplete(item = item, delete_files = self.conf('delete_files', default = False)) return False return diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index f17db9c1..7fcec9e1 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -41,6 +41,13 @@ config = [{ 'advanced': True, 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', }, + { + 'name': 'remove_complete', + 'label': 'Remove torrent', + 'default': True, + 'type': 'bool', + 'description': 'Remove the NZB from SabNZBd history after it completed.', + }, { 'name': 'delete_failed', 'default': True, diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index f2f217a1..56e9a212 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -129,6 +129,22 @@ class Sabnzbd(Downloader): return True + def processComplete(self, item, delete_files = False): + log.debug('Requesting SabNZBd to remove the NZB %s%s.', (item['name'])) + + try: + self.call({ + 'mode': 'history', + 'name': 'delete', + 'del_files': '0', + 'value': item['id'] + }, use_json = False) + except: + log.error('Failed removing: %s', traceback.format_exc(0)) + return False + + return True + def call(self, request_params, use_json = True, **kwargs): url = cleanHost(self.conf('host')) + 'api?' + tryUrlencode(mergeDicts(request_params, { From 7d9920691fedc7e5c83bd00c68d4430d63f10b11 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 29 Jun 2013 22:50:25 +0200 Subject: [PATCH 011/209] Fix uTorrent settings automatically Note that this might not be the way we want to go? --- .../core/downloaders/utorrent/__init__.py | 2 +- couchpotato/core/downloaders/utorrent/main.py | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 674a52a4..f2fcc13e 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -41,7 +41,7 @@ config = [{ 'label': 'Seeding support', 'default': True, 'type': 'bool', - 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving. Stop seeding manually in uTorrent, or check the option Queueing->When uTorrent reaches the seeding goal->Limit the upload rate and set it to 0 to stop seeding after the seeding goal set in the torrent providers is met.', + 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving.', }, { 'name': 'remove_complete', diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index cc02b803..738fbadf 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -46,6 +46,18 @@ class uTorrent(Downloader): if not settings: return False + #Fix settings in case they are not set for CPS compatibility + new_settings = {} + if self.conf('seeding') and not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): + new_settings['seed_prio_limitul'] = 0 + new_settings['seed_prio_limitul_flag'] = True + log.info('Updated uTorrent settings to set a torrent to complete after it the seeding requirements are met.') + if settings.get('bt.read_only_on_complete'): #This doesnt work as this option seems to be not available through the api + new_settings['bt.read_only_on_complete'] = False + log.info('Updated uTorrent settings to not set the files to read only after completing.') + if new_settings: + self.utorrent_api.set_settings(new_settings) + torrent_params = {} if self.conf('label'): torrent_params['label'] = self.conf('label') @@ -65,21 +77,11 @@ class uTorrent(Downloader): if data.get('seed_ratio') and self.conf('seeding'): torrent_params['seed_override'] = 1 torrent_params['seed_ratio'] = tryInt(tryFloat(data['seed_ratio'])*1000) - - # Check if uTorrent completes the torrent if seeding goal is met. - # Note that CPS can also check if the goal has been met but for now it should be done by uTorrent - if not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): - log.info('With the current settings uTorrent does not set torrents that completed the seed ratio and time to complete. Please stop them manually in uTorrent or check the option Queueing->When uTorrent reaches the seeding goal->Limit the upload rate and set it to 0') if data.get('seed_time') and self.conf('seeding'): torrent_params['seed_override'] = 1 torrent_params['seed_time'] = tryInt(data['seed_time'])*3600 - # Check if uTorrent completes the torrent if seeding goal is met. - # Note that CPS can also check if the goal has been met but for now it should be done by uTorrent - if not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): - log.info('With the current settings uTorrent does not set torrents that completed the seed ratio and time to complete. Please stop them manually in uTorrent or check the option Queueing->When uTorrent reaches the seeding goal->Limit the upload rate and set it to 0') - # Convert base 32 to hex if len(torrent_hash) == 32: torrent_hash = b16encode(b32decode(torrent_hash)) @@ -262,3 +264,11 @@ class uTorrentAPI(object): log.error('Failed to get settings from uTorrent: %s', err) return settings_dict + + def set_settings(self, settings_dict = {}): + for key in settings_dict: + if isinstance(settings_dict[key], bool): + settings_dict[key] = 1 if settings_dict[key] else 0 + + action = 'action=setsetting' + ''.join(['&s=%s&v=%s' % (key, value) for (key, value) in settings_dict.items()]) + return self._request(action) From 998e487fe8e1db068f33103c48e21f4709b2a3b7 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sun, 30 Jun 2013 10:14:08 +0200 Subject: [PATCH 012/209] NZBs are not torrents :) --- couchpotato/core/downloaders/sabnzbd/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 7fcec9e1..8e132b72 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -43,10 +43,10 @@ config = [{ }, { 'name': 'remove_complete', - 'label': 'Remove torrent', + 'label': 'Remove NZB', 'default': True, 'type': 'bool', - 'description': 'Remove the NZB from SabNZBd history after it completed.', + 'description': 'Remove the NZB from history after it completed.', }, { 'name': 'delete_failed', From 1c3e6ba930064212052921238caf213afa66ee36 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 6 Jul 2013 00:24:57 +0200 Subject: [PATCH 013/209] Ignore current suggested results --- couchpotato/core/plugins/suggestion/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/suggestion/main.py b/couchpotato/core/plugins/suggestion/main.py index e913bb6c..0e7d701a 100644 --- a/couchpotato/core/plugins/suggestion/main.py +++ b/couchpotato/core/plugins/suggestion/main.py @@ -65,6 +65,7 @@ class Suggestion(Plugin): # Combine with previous suggestion_cache cached_suggestion = self.getCache('suggestion_cached') new_suggestions = [] + ignored = [] if not ignored else ignored if ignore_imdb: for cs in cached_suggestion: @@ -79,9 +80,7 @@ class Suggestion(Plugin): .filter(or_(*[Movie.status.has(identifier = s) for s in ['active', 'done']])).all() movies = [x.library.identifier for x in active_movies] - if ignored: - ignored.extend([x.get('imdb') for x in new_suggestions]) - + ignored.extend([x.get('imdb') for x in cached_suggestion]) suggestions = fireEvent('movie.suggest', movies = movies, ignore = list(set(ignored)), single = True) if suggestions: From 989d6c55c41752b5ec906e1b92326224ef764568 Mon Sep 17 00:00:00 2001 From: Garret Date: Sat, 6 Jul 2013 10:28:32 -0700 Subject: [PATCH 014/209] Added priority setting for SABnzbd Includes ability to add nzb to queue paused. --- couchpotato/core/downloaders/sabnzbd/__init__.py | 9 +++++++++ couchpotato/core/downloaders/sabnzbd/main.py | 1 + 2 files changed, 10 insertions(+) diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index f17db9c1..e36c090b 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -34,6 +34,15 @@ config = [{ 'label': 'Category', 'description': 'The category CP places the nzb in. Like movies or couchpotato', }, + { + 'name': 'priority', + 'label': 'Priority', + 'type': 'dropdown', + 'default': '0', + 'advanced': True, + 'values': [('Paused', -2), ('Low', -1), ('Normal', 0), ('High', 1), ('Forced', 2)], + 'description': 'Priority in the SABnzbd queue. Option to add paused.', + }, { 'name': 'manual', 'default': False, diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index f2f217a1..d916db82 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -22,6 +22,7 @@ class Sabnzbd(Downloader): 'cat': self.conf('category'), 'mode': 'addurl', 'nzbname': self.createNzbName(data, movie), + 'priority': self.conf('priority'), } if filedata: From a4a14cae9610179cde0118ab24c0d83739bbe7a6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 6 Jul 2013 23:26:46 +0200 Subject: [PATCH 015/209] Use forwarded host when provided. fix #1922 --- couchpotato/core/plugins/userscript/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/userscript/main.py b/couchpotato/core/plugins/userscript/main.py index a76cf58c..a940bee3 100644 --- a/couchpotato/core/plugins/userscript/main.py +++ b/couchpotato/core/plugins/userscript/main.py @@ -55,7 +55,7 @@ class Userscript(Plugin): 'excludes': fireEvent('userscript.get_excludes', merge = True), 'version': klass.getVersion(), 'api': '%suserscript/' % Env.get('api_base'), - 'host': '%s://%s' % (self.request.protocol, self.request.host), + 'host': '%s://%s' % (self.request.protocol, self.request.headers.get('X-Forwarded-Host') or self.request.headers.get('host')), } script = klass.renderTemplate(__file__, 'template.js', **params) From da9dda2c2b157fc5c77464973f8de8294b9405c0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 6 Jul 2013 23:39:34 +0200 Subject: [PATCH 016/209] Make minimal movie automation clearer. fix #1923 --- couchpotato/core/providers/automation/bluray/__init__.py | 2 +- couchpotato/core/providers/automation/itunes/__init__.py | 2 +- couchpotato/core/providers/automation/kinepolis/__init__.py | 2 +- couchpotato/core/providers/automation/moviemeter/__init__.py | 2 +- .../core/providers/automation/rottentomatoes/__init__.py | 3 ++- couchpotato/core/providers/automation/rottentomatoes/main.py | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/providers/automation/bluray/__init__.py b/couchpotato/core/providers/automation/bluray/__init__.py index b916b0af..e0675247 100644 --- a/couchpotato/core/providers/automation/bluray/__init__.py +++ b/couchpotato/core/providers/automation/bluray/__init__.py @@ -11,7 +11,7 @@ config = [{ 'list': 'automation_providers', 'name': 'bluray_automation', 'label': 'Blu-ray.com', - 'description': 'Imports movies from blu-ray.com. (uses minimal requirements)', + 'description': 'Imports movies from blu-ray.com.', 'options': [ { 'name': 'automation_enabled', diff --git a/couchpotato/core/providers/automation/itunes/__init__.py b/couchpotato/core/providers/automation/itunes/__init__.py index b5c565f6..cc5dddc7 100644 --- a/couchpotato/core/providers/automation/itunes/__init__.py +++ b/couchpotato/core/providers/automation/itunes/__init__.py @@ -11,7 +11,7 @@ config = [{ 'list': 'automation_providers', 'name': 'itunes_automation', 'label': 'iTunes', - 'description': 'From any iTunes Store feed. Url should be the RSS link. (uses minimal requirements)', + 'description': 'From any iTunes Store feed. Url should be the RSS link.', 'options': [ { 'name': 'automation_enabled', diff --git a/couchpotato/core/providers/automation/kinepolis/__init__.py b/couchpotato/core/providers/automation/kinepolis/__init__.py index d3b8e898..24bd4ebb 100644 --- a/couchpotato/core/providers/automation/kinepolis/__init__.py +++ b/couchpotato/core/providers/automation/kinepolis/__init__.py @@ -11,7 +11,7 @@ config = [{ 'list': 'automation_providers', 'name': 'kinepolis_automation', 'label': 'Kinepolis', - 'description': 'Imports movies from the current top 10 of kinepolis. (uses minimal requirements)', + 'description': 'Imports movies from the current top 10 of kinepolis.', 'options': [ { 'name': 'automation_enabled', diff --git a/couchpotato/core/providers/automation/moviemeter/__init__.py b/couchpotato/core/providers/automation/moviemeter/__init__.py index 773bed45..aff5d09d 100644 --- a/couchpotato/core/providers/automation/moviemeter/__init__.py +++ b/couchpotato/core/providers/automation/moviemeter/__init__.py @@ -11,7 +11,7 @@ config = [{ 'list': 'automation_providers', 'name': 'moviemeter_automation', 'label': 'Moviemeter', - 'description': 'Imports movies from the current top 10 of moviemeter.nl. (uses minimal requirements)', + 'description': 'Imports movies from the current top 10 of moviemeter.nl.', 'options': [ { 'name': 'automation_enabled', diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index dd96fe45..83a545b6 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -21,7 +21,8 @@ config = [{ { 'name': 'tomatometer_percent', 'default': '80', - 'label': 'Tomatometer' + 'label': 'Tomatometer', + 'description': 'Use as extra scoring requirement', } ], }, diff --git a/couchpotato/core/providers/automation/rottentomatoes/main.py b/couchpotato/core/providers/automation/rottentomatoes/main.py index b4482023..9842d4c9 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/main.py +++ b/couchpotato/core/providers/automation/rottentomatoes/main.py @@ -42,7 +42,7 @@ class Rottentomatoes(Automation, RSS): year = datetime.datetime.now().strftime("%Y") imdb = self.search(name, year) - if imdb: + if imdb and self.isMinimalMovie(imdb): movies.append(imdb['imdb']) return movies From 52163428e9de1e5339d2664390126ed9e2be717f Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Jul 2013 00:09:22 +0200 Subject: [PATCH 017/209] Break if media headers are corrupt. fix #1828 --- libs/enzyme/mp4.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/enzyme/mp4.py b/libs/enzyme/mp4.py index c53f30d3..a66d30ad 100644 --- a/libs/enzyme/mp4.py +++ b/libs/enzyme/mp4.py @@ -284,6 +284,10 @@ class MPEG4(core.AVContainer): while datasize: mdia = struct.unpack('>I4s', atomdata[pos:pos + 8]) + + if mdia[0] == 0: + break + if mdia[1] == 'mdhd': # Parse based on version of mdhd header. See # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd From 1ebb09226dfa18e3d74d9ff879d93467479a749a Mon Sep 17 00:00:00 2001 From: dkboy Date: Sun, 7 Jul 2013 14:23:15 +1200 Subject: [PATCH 018/209] Add Bitsoup provider --- .../providers/torrent/bitsoup/__init__.py | 42 +++++++++ .../core/providers/torrent/bitsoup/main.py | 85 +++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 couchpotato/core/providers/torrent/bitsoup/__init__.py create mode 100644 couchpotato/core/providers/torrent/bitsoup/main.py diff --git a/couchpotato/core/providers/torrent/bitsoup/__init__.py b/couchpotato/core/providers/torrent/bitsoup/__init__.py new file mode 100644 index 00000000..097f3782 --- /dev/null +++ b/couchpotato/core/providers/torrent/bitsoup/__init__.py @@ -0,0 +1,42 @@ +from .main import Bitsoup + +def start(): + return Bitsoup() + +config = [{ + 'name': 'bitsoup', + 'groups': [ + { + 'tab': 'searcher', + 'subtab': 'providers', + 'list': 'torrent_providers', + 'name': 'Bitsoup', + 'description': 'See Bitsoup', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': False, + }, + { + 'name': 'username', + 'default': '', + }, + { + 'name': 'password', + 'default': '', + 'type': 'password', + }, + { + 'name': 'extra_score', + 'advanced': True, + 'label': 'Extra Score', + 'type': 'int', + 'default': 20, + 'description': 'Starting score for each release found via this provider.', + } + ], + }, + ], +}] diff --git a/couchpotato/core/providers/torrent/bitsoup/main.py b/couchpotato/core/providers/torrent/bitsoup/main.py new file mode 100644 index 00000000..904b5469 --- /dev/null +++ b/couchpotato/core/providers/torrent/bitsoup/main.py @@ -0,0 +1,85 @@ +from bs4 import BeautifulSoup +from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode +from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.torrent.base import TorrentProvider +import traceback + +log = CPLog(__name__) + + +class Bitsoup(TorrentProvider): + + urls = { + 'test': 'https://www.bitsoup.me/', + 'login' : 'https://www.bitsoup.me/takelogin.php', + 'login_check': 'https://www.bitsoup.me/my.php', + 'detail': 'https://www.bitsoup.me/details.php?id=%s', + 'search': 'https://www.bitsoup.me/browse.php?', + 'download': 'https://www.bitsoup.me/%s', + } + + http_time_between_calls = 1 #seconds + + def _searchOnTitle(self, title, movie, quality, results): + + q = '"%s %s"' % (simplifyString(title), movie['library']['year']) + arguments = tryUrlencode({ + 'search': q, + }) + url = "%s&%s" % (self.urls['search'], arguments) + + data = self.getHTMLData(url, opener = self.login_opener) + + if data: + html = BeautifulSoup(data) + + try: + resultsTable = html.find_all('table')[8] + entries = resultsTable.find_all('tr') + for result in entries[1:]: + + all_cells = result.find_all('td') + + torrent = all_cells[1].find('a') + download = all_cells[3].find('a') + + torrent_id = torrent['href'] + torrent_id = torrent_id.replace('details.php?id=', '') + torrent_id = torrent_id.replace('&hit=1', '') + + torrent_name = torrent.getText() + + torrent_size = self.parseSize(all_cells[7].getText()) + torrent_seeders = tryInt(all_cells[9].getText()) + torrent_leechers = tryInt(all_cells[10].getText()) + torrent_url = self.urls['download'] % download['href'] + torrent_description = torrent['href'] + + results.append({ + 'id': torrent_id, + 'name': torrent_name, + 'size': torrent_size, + 'seeders': torrent_seeders, + 'leechers': torrent_leechers, + 'url': torrent_url, + 'description': torrent_description, + }) + + except: + log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + + + def getLoginParams(self): + return tryUrlencode({ + 'username': self.conf('username'), + 'password': self.conf('password'), + 'ssl': 'yes', + }) + + + def loginSuccess(self, output): + return 'logout.php' in output.lower() + + loginCheckSuccess = loginSuccess + From 8a252bff64b2bb2d376673a0b00e9624d44aaf4c Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Jul 2013 13:00:38 +0200 Subject: [PATCH 019/209] Don't use parentdir for tagging --- couchpotato/core/downloaders/base.py | 6 ++-- .../core/downloaders/sabnzbd/__init__.py | 2 +- .../core/downloaders/transmission/__init__.py | 4 +-- .../core/downloaders/transmission/main.py | 13 ++++---- .../core/downloaders/utorrent/__init__.py | 4 +-- couchpotato/core/downloaders/utorrent/main.py | 20 ++++++------- couchpotato/core/plugins/renamer/main.py | 30 ++++++++++++------- 7 files changed, 47 insertions(+), 32 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index adbfb7ec..5e6db0e2 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -86,7 +86,8 @@ class Downloader(Provider): return self.processComplete(item = item, delete_files = self.conf('delete_files', default = False)) return False - return + + return False def processComplete(self, item, delete_files): return @@ -149,7 +150,8 @@ class Downloader(Provider): if item and item.get('downloader') == self.getName(): self.pause(item, pause) return True - return + + return False def pause(self, item, pause): return diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 8e132b72..781141e2 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -47,7 +47,7 @@ config = [{ 'default': True, 'type': 'bool', 'description': 'Remove the NZB from history after it completed.', - }, + }, { 'name': 'delete_failed', 'default': True, diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index bca7eae5..11528f3c 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -49,14 +49,14 @@ config = [{ 'default': True, 'type': 'bool', 'description': '(Hard)link/copy after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving and set the seeding goal from the torrent providers.', - }, + }, { 'name': 'remove_complete', 'label': 'Remove torrent', 'default': True, 'type': 'bool', 'description': 'Remove the torrent from Transmission after it finished seeding.', - }, + }, { 'name': 'delete_files', 'label': 'Remove files', diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 46b9541f..653343b3 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -19,15 +19,17 @@ class Transmission(Downloader): type = ['torrent', 'torrent_magnet'] log = CPLog(__name__) trpc = None - + def connect(self): # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False + if not self.trpc: self.trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + return self.trpc def download(self, data, movie, filedata = None): @@ -58,7 +60,7 @@ class Transmission(Downloader): torrent_params['seedRatioMode'] = 1 if data.get('seed_time') and self.conf('seeding'): - torrent_params['seedIdleLimit'] = tryInt(data.get('seed_time'))*60 + torrent_params['seedIdleLimit'] = tryInt(data.get('seed_time')) * 60 torrent_params['seedIdleMode'] = 1 # Send request to Transmission @@ -89,8 +91,8 @@ class Transmission(Downloader): statuses = StatusList(self) return_params = { - 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit'] - } + 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit'] + } queue = self.trpc.get_alltorrents(return_params) if not (queue and queue.get('torrents')): @@ -98,7 +100,8 @@ class Transmission(Downloader): return False for item in queue['torrents']: - log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], item['isFinished'])) + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', + (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], item['isFinished'])) if not os.path.isdir(Env.setting('from', 'renamer')): log.error('Renamer "from" folder doesn\'t to exist.') diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index f2fcc13e..025eb9a2 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -42,14 +42,14 @@ config = [{ 'default': True, 'type': 'bool', 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving.', - }, + }, { 'name': 'remove_complete', 'label': 'Remove torrent', 'default': True, 'type': 'bool', 'description': 'Remove the torrent from uTorrent after it finished seeding.', - }, + }, { 'name': 'delete_files', 'label': 'Remove files', diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 738fbadf..5a76654e 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -76,11 +76,11 @@ class uTorrent(Downloader): if data.get('seed_ratio') and self.conf('seeding'): torrent_params['seed_override'] = 1 - torrent_params['seed_ratio'] = tryInt(tryFloat(data['seed_ratio'])*1000) + torrent_params['seed_ratio'] = tryInt(tryFloat(data['seed_ratio']) * 1000) if data.get('seed_time') and self.conf('seeding'): torrent_params['seed_override'] = 1 - torrent_params['seed_time'] = tryInt(data['seed_time'])*3600 + torrent_params['seed_time'] = tryInt(data['seed_time']) * 3600 # Convert base 32 to hex if len(torrent_hash) == 32: @@ -139,7 +139,7 @@ class uTorrent(Downloader): 'id': item[0], 'name': item[2], 'status': status, - 'seed_ratio': float(item[7])/1000, + 'seed_ratio': float(item[7]) / 1000, 'original_status': item[1], 'timeleft': str(timedelta(seconds = item[10])), 'folder': item[26], @@ -228,16 +228,16 @@ class uTorrentAPI(object): action = "action=unpause&hash=%s" % hash return self._request(action) - def stop_torrent(self, hash): - action = "action=stop&hash=%s" % hash - return self._request(action) + def stop_torrent(self, hash): + action = "action=stop&hash=%s" % hash + return self._request(action) - def remove_torrent(self, hash, remove_data = False): + def remove_torrent(self, hash, remove_data = False): if remove_data: action = "action=removedata&hash=%s" % hash else: - action = "action=remove&hash=%s" % hash - return self._request(action) + action = "action=remove&hash=%s" % hash + return self._request(action) def get_status(self): action = "list=1" @@ -270,5 +270,5 @@ class uTorrentAPI(object): if isinstance(settings_dict[key], bool): settings_dict[key] = 1 if settings_dict[key] else 0 - action = 'action=setsetting' + ''.join(['&s=%s&v=%s' % (key, value) for (key, value) in settings_dict.items()]) + action = 'action=setsetting' + ''.join(['&s=%s&v=%s' % (key, value) for (key, value) in settings_dict.items()]) return self._request(action) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 1070e4d4..119e5d45 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -151,7 +151,7 @@ class Renamer(Plugin): # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: - self.tagDir(group['parentdir'], 'unknown') + self.tagDir(group, 'unknown') continue # Rename the files using the library data else: @@ -361,7 +361,7 @@ class Renamer(Plugin): log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label)) # Add exists tag to the .ignore file - self.tagDir(group['parentdir'], 'exists') + self.tagDir(group, 'exists') # Notify on rename fail download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) @@ -412,7 +412,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (src, traceback.format_exc())) - self.tagDir(group['parentdir'], 'failed_remove') + self.tagDir(group, 'failed_remove') # Delete leftover folder from older releases for delete_folder in delete_folders: @@ -436,12 +436,12 @@ class Renamer(Plugin): group['renamed_files'].append(dst) except: log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) - self.tagDir(group['parentdir'], 'failed_rename') + self.tagDir(group, 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent if (movie_folder and self.conf('from') in movie_folder or not movie_folder) and \ self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info): - self.tagDir(group['parentdir'], 'renamed_already') + self.tagDir(group, 'renamed_already') # Remove matching releases for release in remove_releases: @@ -489,9 +489,18 @@ class Renamer(Plugin): return rename_files # This adds a file to ignore / tag a release so it is ignored later - def tagDir(self, folder, tag): - if not os.path.isdir(folder) or not tag: - return + def tagDir(self, group, tag): + + ignore_file = None + if isinstance(group, (dict)): + for movie_file in sorted(list(group['files']['movie'])): + ignore_file = '%s.%s.ignore' % (os.path.splitext(movie_file)[0], tag) + break + else: + if not os.path.isdir(group) or not tag: + return + ignore_file = os.path.join(group, '%s.ignore' % tag) + text = """This file is from CouchPotato It has marked this release as "%s" @@ -499,7 +508,8 @@ This file hides the release from the renamer Remove it if you want it to be renamed (again, or at least let it try again) """ % tag - self.createFile(os.path.join(folder, '%s.ignore' % tag), text) + if ignore_file: + self.createFile(ignore_file, text) def untagDir(self, folder, tag = None): if not os.path.isdir(folder): @@ -808,4 +818,4 @@ Remove it if you want it to be renamed (again, or at least let it try again) if not group['files'].get('added'): return False return src in group['files']['added'] - + From c0b3c9a330cd7dbde70f6a09f0f0f9c9b6b90dab Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 7 Jul 2013 13:44:49 +0200 Subject: [PATCH 020/209] Make description a bit shorter --- couchpotato/core/providers/torrent/awesomehd/__init__.py | 4 ++-- couchpotato/core/providers/torrent/hdbits/__init__.py | 4 ++-- couchpotato/core/providers/torrent/iptorrents/__init__.py | 4 ++-- .../core/providers/torrent/kickasstorrents/__init__.py | 4 ++-- couchpotato/core/providers/torrent/passthepopcorn/__init__.py | 4 ++-- couchpotato/core/providers/torrent/publichd/__init__.py | 4 ++-- couchpotato/core/providers/torrent/sceneaccess/__init__.py | 4 ++-- couchpotato/core/providers/torrent/scenehd/__init__.py | 4 ++-- couchpotato/core/providers/torrent/thepiratebay/__init__.py | 4 ++-- couchpotato/core/providers/torrent/torrentbytes/__init__.py | 4 ++-- couchpotato/core/providers/torrent/torrentday/__init__.py | 4 ++-- couchpotato/core/providers/torrent/torrentleech/__init__.py | 4 ++-- couchpotato/core/providers/torrent/torrentshack/__init__.py | 4 ++-- couchpotato/core/providers/torrent/yify/__init__.py | 4 ++-- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/couchpotato/core/providers/torrent/awesomehd/__init__.py b/couchpotato/core/providers/torrent/awesomehd/__init__.py index e2587a52..1dc5a2ea 100644 --- a/couchpotato/core/providers/torrent/awesomehd/__init__.py +++ b/couchpotato/core/providers/torrent/awesomehd/__init__.py @@ -28,14 +28,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'only_internal', diff --git a/couchpotato/core/providers/torrent/hdbits/__init__.py b/couchpotato/core/providers/torrent/hdbits/__init__.py index 0b370e19..f1613d36 100644 --- a/couchpotato/core/providers/torrent/hdbits/__init__.py +++ b/couchpotato/core/providers/torrent/hdbits/__init__.py @@ -36,14 +36,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/iptorrents/__init__.py b/couchpotato/core/providers/torrent/iptorrents/__init__.py index c1eea5cd..4cb90a18 100644 --- a/couchpotato/core/providers/torrent/iptorrents/__init__.py +++ b/couchpotato/core/providers/torrent/iptorrents/__init__.py @@ -39,14 +39,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py index 90f9eeac..eebca28a 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py @@ -24,14 +24,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py index f0cb966e..cc7736a7 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py @@ -67,14 +67,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/publichd/__init__.py b/couchpotato/core/providers/torrent/publichd/__init__.py index 22e2dbb8..d1cb1079 100644 --- a/couchpotato/core/providers/torrent/publichd/__init__.py +++ b/couchpotato/core/providers/torrent/publichd/__init__.py @@ -24,14 +24,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/sceneaccess/__init__.py b/couchpotato/core/providers/torrent/sceneaccess/__init__.py index 786f28a6..eaee026f 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/__init__.py +++ b/couchpotato/core/providers/torrent/sceneaccess/__init__.py @@ -33,14 +33,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/scenehd/__init__.py b/couchpotato/core/providers/torrent/scenehd/__init__.py index 9b967f4b..79d2550f 100644 --- a/couchpotato/core/providers/torrent/scenehd/__init__.py +++ b/couchpotato/core/providers/torrent/scenehd/__init__.py @@ -33,14 +33,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index 2c902439..6e469a6c 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -30,14 +30,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/torrentbytes/__init__.py b/couchpotato/core/providers/torrent/torrentbytes/__init__.py index c7f4437d..65c90c30 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/__init__.py +++ b/couchpotato/core/providers/torrent/torrentbytes/__init__.py @@ -33,14 +33,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/torrentday/__init__.py b/couchpotato/core/providers/torrent/torrentday/__init__.py index d3bbaa14..9eaa4990 100644 --- a/couchpotato/core/providers/torrent/torrentday/__init__.py +++ b/couchpotato/core/providers/torrent/torrentday/__init__.py @@ -33,14 +33,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/torrentleech/__init__.py b/couchpotato/core/providers/torrent/torrentleech/__init__.py index d3ee7616..d5b8b241 100644 --- a/couchpotato/core/providers/torrent/torrentleech/__init__.py +++ b/couchpotato/core/providers/torrent/torrentleech/__init__.py @@ -33,14 +33,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', diff --git a/couchpotato/core/providers/torrent/torrentshack/__init__.py b/couchpotato/core/providers/torrent/torrentshack/__init__.py index 69ad176d..f6ed401f 100644 --- a/couchpotato/core/providers/torrent/torrentshack/__init__.py +++ b/couchpotato/core/providers/torrent/torrentshack/__init__.py @@ -32,14 +32,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'scene_only', diff --git a/couchpotato/core/providers/torrent/yify/__init__.py b/couchpotato/core/providers/torrent/yify/__init__.py index f953e801..30d7ef02 100644 --- a/couchpotato/core/providers/torrent/yify/__init__.py +++ b/couchpotato/core/providers/torrent/yify/__init__.py @@ -24,14 +24,14 @@ config = [{ 'label': 'Seed ratio', 'type': 'float', 'default': 1, - 'description': 'Torrent will not be (re)moved until this seed ratio is met.', + 'description': 'Will not be (re)moved until this seed ratio is met.', }, { 'name': 'seed_time', 'label': 'Seed time', 'type': 'int', 'default': 40, - 'description': 'Torrent will not be (re)moved until this seed time (in hours) is met.', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', }, { 'name': 'extra_score', From ed8108a9d84c33263d42d40195ac04ebdf34d9fc Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 8 Jul 2013 11:30:55 +0200 Subject: [PATCH 021/209] Remove NZBsRus --- .../core/providers/nzb/nzbsrus/__init__.py | 49 --------------- .../core/providers/nzb/nzbsrus/main.py | 62 ------------------- 2 files changed, 111 deletions(-) delete mode 100644 couchpotato/core/providers/nzb/nzbsrus/__init__.py delete mode 100644 couchpotato/core/providers/nzb/nzbsrus/main.py diff --git a/couchpotato/core/providers/nzb/nzbsrus/__init__.py b/couchpotato/core/providers/nzb/nzbsrus/__init__.py deleted file mode 100644 index 863e6a37..00000000 --- a/couchpotato/core/providers/nzb/nzbsrus/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -from .main import Nzbsrus - -def start(): - return Nzbsrus() - -config = [{ - 'name': 'nzbsrus', - 'groups': [ - { - 'tab': 'searcher', - 'subtab': 'providers', - 'list': 'nzb_providers', - 'name': 'nzbsrus', - 'label': 'Nzbsrus', - 'description': 'See NZBsRus. You need a VIP account!', - 'wizard': True, - 'options': [ - { - 'name': 'enabled', - 'type': 'enabler', - }, - { - 'name': 'userid', - 'label': 'User ID', - }, - { - 'name': 'api_key', - 'default': '', - 'label': 'Api Key', - }, - { - 'name': 'english_only', - 'default': 1, - 'type': 'bool', - 'label': 'English only', - 'description': 'Only search for English spoken movies on Nzbsrus', - }, - { - 'name': 'extra_score', - 'advanced': True, - 'label': 'Extra Score', - 'type': 'int', - 'default': 0, - 'description': 'Starting score for each release found via this provider.', - } - ], - }, - ], -}] diff --git a/couchpotato/core/providers/nzb/nzbsrus/main.py b/couchpotato/core/providers/nzb/nzbsrus/main.py deleted file mode 100644 index d52212a7..00000000 --- a/couchpotato/core/providers/nzb/nzbsrus/main.py +++ /dev/null @@ -1,62 +0,0 @@ -from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.helpers.rss import RSS -from couchpotato.core.logger import CPLog -from couchpotato.core.providers.nzb.base import NZBProvider -from couchpotato.environment import Env -import time - -log = CPLog(__name__) - -class Nzbsrus(NZBProvider, RSS): - - urls = { - 'download': 'https://www.nzbsrus.com/nzbdownload_rss.php/%s', - 'detail': 'https://www.nzbsrus.com/nzbdetails.php?id=%s', - 'search': 'https://www.nzbsrus.com/api.php?extended=1&xml=1&listname={date,grabs}', - } - - cat_ids = [ - ([90, 45, 51], ['720p', '1080p', 'brrip', 'bd50', 'dvdr']), - ([48, 51], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), - ] - cat_backup_id = 240 - - def _search(self, movie, quality, results): - - cat_id_string = '&'.join(['c%s=1' % x for x in self.getCatId(quality.get('identifier'))]) - arguments = tryUrlencode({ - 'searchtext': 'imdb:' + movie['library']['identifier'][2:], - 'uid': self.conf('userid'), - 'key': self.conf('api_key'), - 'age': Env.setting('retention', section = 'nzb'), - - }) - - # check for english_only - if self.conf('english_only'): - arguments += '&lang0=1&lang3=1&lang1=1' - - url = '%s&%s&%s' % (self.urls['search'], arguments , cat_id_string) - nzbs = self.getRSSData(url, item_path = 'results/result', cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) - - for nzb in nzbs: - - title = self.getTextElement(nzb, 'name') - if 'error' in title.lower(): continue - - nzb_id = self.getTextElement(nzb, 'id') - size = int(round(int(self.getTextElement(nzb, 'size')) / 1048576)) - age = int(round((time.time() - int(self.getTextElement(nzb, 'postdate'))) / 86400)) - - results.append({ - 'id': nzb_id, - 'name': title, - 'age': age, - 'size': size, - 'url': self.urls['download'] % nzb_id + self.getApiExt() + self.getTextElement(nzb, 'key'), - 'detail_url': self.urls['detail'] % nzb_id, - 'description': self.getTextElement(nzb, 'addtext'), - }) - - def getApiExt(self): - return '/%s/' % (self.conf('userid')) From e20bb1364998fe045961306934d5e8d4abee34e7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 8 Jul 2013 11:31:13 +0200 Subject: [PATCH 022/209] Delete NZBx --- .../core/providers/nzb/nzbx/__init__.py | 33 ---------------- couchpotato/core/providers/nzb/nzbx/main.py | 38 ------------------- 2 files changed, 71 deletions(-) delete mode 100644 couchpotato/core/providers/nzb/nzbx/__init__.py delete mode 100644 couchpotato/core/providers/nzb/nzbx/main.py diff --git a/couchpotato/core/providers/nzb/nzbx/__init__.py b/couchpotato/core/providers/nzb/nzbx/__init__.py deleted file mode 100644 index 9cf80638..00000000 --- a/couchpotato/core/providers/nzb/nzbx/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from .main import Nzbx - -def start(): - return Nzbx() - -config = [{ - 'name': 'nzbx', - 'groups': [ - { - 'tab': 'searcher', - 'subtab': 'providers', - 'list': 'nzb_providers', - 'name': 'nzbX', - 'description': 'Free provider. See nzbX', - 'wizard': True, - 'options': [ - { - 'name': 'enabled', - 'type': 'enabler', - 'default': True, - }, - { - 'name': 'extra_score', - 'advanced': True, - 'label': 'Extra Score', - 'type': 'int', - 'default': 0, - 'description': 'Starting score for each release found via this provider.', - } - ], - }, - ], -}] diff --git a/couchpotato/core/providers/nzb/nzbx/main.py b/couchpotato/core/providers/nzb/nzbx/main.py deleted file mode 100644 index ec7fbfe2..00000000 --- a/couchpotato/core/providers/nzb/nzbx/main.py +++ /dev/null @@ -1,38 +0,0 @@ -from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.helpers.variable import tryInt -from couchpotato.core.logger import CPLog -from couchpotato.core.providers.nzb.base import NZBProvider -from couchpotato.environment import Env - -log = CPLog(__name__) - - -class Nzbx(NZBProvider): - - urls = { - 'search': 'https://nzbx.co/api/search?%s', - 'details': 'https://nzbx.co/api/details?guid=%s', - } - - http_time_between_calls = 1 # Seconds - - def _search(self, movie, quality, results): - - # Get nbzs - arguments = tryUrlencode({ - 'q': movie['library']['identifier'].replace('tt', ''), - 'sf': quality.get('size_min'), - }) - nzbs = self.getJsonData(self.urls['search'] % arguments, headers = {'User-Agent': Env.getIdentifier()}) - - for nzb in nzbs: - - results.append({ - 'id': nzb['guid'], - 'url': nzb['nzb'], - 'detail_url': self.urls['details'] % nzb['guid'], - 'name': nzb['name'], - 'age': self.calculateAge(int(nzb['postdate'])), - 'size': tryInt(nzb['size']) / 1024 / 1024, - 'score': 5 if nzb['votes']['upvotes'] > nzb['votes']['downvotes'] else 0 - }) From 71e280238dabc2d635bf361433b49178c6192b2e Mon Sep 17 00:00:00 2001 From: dkboy Date: Wed, 10 Jul 2013 01:48:11 +1200 Subject: [PATCH 023/209] Fixed missing detail_url --- couchpotato/core/providers/torrent/bitsoup/main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/providers/torrent/bitsoup/main.py b/couchpotato/core/providers/torrent/bitsoup/main.py index 904b5469..3239656b 100644 --- a/couchpotato/core/providers/torrent/bitsoup/main.py +++ b/couchpotato/core/providers/torrent/bitsoup/main.py @@ -14,9 +14,8 @@ class Bitsoup(TorrentProvider): 'test': 'https://www.bitsoup.me/', 'login' : 'https://www.bitsoup.me/takelogin.php', 'login_check': 'https://www.bitsoup.me/my.php', - 'detail': 'https://www.bitsoup.me/details.php?id=%s', 'search': 'https://www.bitsoup.me/browse.php?', - 'download': 'https://www.bitsoup.me/%s', + 'baseurl': 'https://www.bitsoup.me/%s', } http_time_between_calls = 1 #seconds @@ -53,8 +52,8 @@ class Bitsoup(TorrentProvider): torrent_size = self.parseSize(all_cells[7].getText()) torrent_seeders = tryInt(all_cells[9].getText()) torrent_leechers = tryInt(all_cells[10].getText()) - torrent_url = self.urls['download'] % download['href'] - torrent_description = torrent['href'] + torrent_url = self.urls['baseurl'] % download['href'] + torrent_detail_url = self.urls['baseurl'] % torrent['href'] results.append({ 'id': torrent_id, @@ -63,7 +62,7 @@ class Bitsoup(TorrentProvider): 'seeders': torrent_seeders, 'leechers': torrent_leechers, 'url': torrent_url, - 'description': torrent_description, + 'detail_url': torrent_detail_url, }) except: From a09fc1462526364202bd9d811e74b8126e79b959 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 9 Jul 2013 20:32:29 +0200 Subject: [PATCH 024/209] Twitter DM didn't work --- couchpotato/core/notifications/twitter/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/twitter/main.py b/couchpotato/core/notifications/twitter/main.py index 59fbb3a1..facc36b9 100644 --- a/couchpotato/core/notifications/twitter/main.py +++ b/couchpotato/core/notifications/twitter/main.py @@ -50,7 +50,7 @@ class Twitter(Notification): try: if direct_message: for user in direct_message_users.split(): - api.PostDirectMessage(user, '[%s] %s' % (self.default_title, message)) + api.PostDirectMessage('[%s] %s' % (self.default_title, message), screen_name = user) else: update_message = '[%s] %s' % (self.default_title, message) if len(update_message) > 140: From 36f63bdf991745b3d41788e44fc94b4ef941daac Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 9 Jul 2013 22:52:32 +0200 Subject: [PATCH 025/209] Seeding cleanup and better defaults --- couchpotato/core/downloaders/base.py | 2 +- .../core/downloaders/blackhole/main.py | 1 + .../core/downloaders/nzbget/__init__.py | 2 ++ couchpotato/core/downloaders/nzbget/main.py | 1 + .../core/downloaders/nzbvortex/__init__.py | 1 + .../core/downloaders/nzbvortex/main.py | 3 ++- .../core/downloaders/pneumatic/main.py | 1 + .../core/downloaders/sabnzbd/__init__.py | 4 ++- couchpotato/core/downloaders/sabnzbd/main.py | 1 + .../core/downloaders/transmission/__init__.py | 19 ++++---------- .../core/downloaders/transmission/main.py | 4 +-- .../core/downloaders/utorrent/__init__.py | 18 +++++++------ couchpotato/core/downloaders/utorrent/main.py | 26 ++++++++++++------- couchpotato/core/plugins/renamer/main.py | 21 ++++----------- couchpotato/core/plugins/subtitle/main.py | 2 +- 15 files changed, 52 insertions(+), 54 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 5e6db0e2..a7668205 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -82,7 +82,7 @@ class Downloader(Provider): return if item and item.get('downloader') == self.getName(): - if self.conf('remove_complete'): + if self.conf('remove_complete', default = False): return self.processComplete(item = item, delete_files = self.conf('delete_files', default = False)) return False diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index aad9ea7f..82b07276 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -7,6 +7,7 @@ import traceback log = CPLog(__name__) + class Blackhole(Downloader): type = ['nzb', 'torrent', 'torrent_magnet'] diff --git a/couchpotato/core/downloaders/nzbget/__init__.py b/couchpotato/core/downloaders/nzbget/__init__.py index a17b2dc5..19483713 100644 --- a/couchpotato/core/downloaders/nzbget/__init__.py +++ b/couchpotato/core/downloaders/nzbget/__init__.py @@ -42,6 +42,7 @@ config = [{ }, { 'name': 'priority', + 'advanced': True, 'default': '0', 'type': 'dropdown', 'values': [('Very Low', -100), ('Low', -50), ('Normal', 0), ('High', 50), ('Very High', 100)], @@ -57,6 +58,7 @@ config = [{ { 'name': 'delete_failed', 'default': True, + 'advanced': True, 'type': 'bool', 'description': 'Delete a release after the download has failed.', }, diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index 43061e17..c90cd2db 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -12,6 +12,7 @@ import xmlrpclib log = CPLog(__name__) + class NZBGet(Downloader): type = ['nzb'] diff --git a/couchpotato/core/downloaders/nzbvortex/__init__.py b/couchpotato/core/downloaders/nzbvortex/__init__.py index f1604ea8..3b95698e 100644 --- a/couchpotato/core/downloaders/nzbvortex/__init__.py +++ b/couchpotato/core/downloaders/nzbvortex/__init__.py @@ -38,6 +38,7 @@ config = [{ { 'name': 'delete_failed', 'default': True, + 'advanced': True, 'type': 'bool', 'description': 'Delete a release after the download has failed.', }, diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index f1f8acc6..805c4598 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -16,6 +16,7 @@ import urllib2 log = CPLog(__name__) + class NZBVortex(Downloader): type = ['nzb'] @@ -55,7 +56,7 @@ class NZBVortex(Downloader): 'name': item['uiTitle'], 'status': status, 'original_status': item['state'], - 'timeleft': -1, + 'timeleft':-1, 'folder': item['destinationPath'], }) diff --git a/couchpotato/core/downloaders/pneumatic/main.py b/couchpotato/core/downloaders/pneumatic/main.py index 5e2b7854..5564dca7 100644 --- a/couchpotato/core/downloaders/pneumatic/main.py +++ b/couchpotato/core/downloaders/pneumatic/main.py @@ -6,6 +6,7 @@ import traceback log = CPLog(__name__) + class Pneumatic(Downloader): type = ['nzb'] diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 781141e2..f0dde2c7 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -43,14 +43,16 @@ config = [{ }, { 'name': 'remove_complete', + 'advanced': True, 'label': 'Remove NZB', - 'default': True, + 'default': False, 'type': 'bool', 'description': 'Remove the NZB from history after it completed.', }, { 'name': 'delete_failed', 'default': True, + 'advanced': True, 'type': 'bool', 'description': 'Delete a release after the download has failed.', }, diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 56e9a212..bdf58dc0 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -10,6 +10,7 @@ import traceback log = CPLog(__name__) + class Sabnzbd(Downloader): type = ['nzb'] diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 11528f3c..2cbd8b14 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -32,28 +32,16 @@ config = [{ 'name': 'password', 'type': 'password', }, - { - 'name': 'paused', - 'type': 'bool', - 'default': False, - 'description': 'Add the torrent paused.', - }, { 'name': 'directory', 'type': 'directory', 'description': 'Download to this directory. Keep empty for default Transmission download directory.', }, - { - 'name': 'seeding', - 'label': 'Seeding support', - 'default': True, - 'type': 'bool', - 'description': '(Hard)link/copy after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving and set the seeding goal from the torrent providers.', - }, { 'name': 'remove_complete', 'label': 'Remove torrent', - 'default': True, + 'default': False, + 'advanced': True, 'type': 'bool', 'description': 'Remove the torrent from Transmission after it finished seeding.', }, @@ -68,6 +56,7 @@ config = [{ { 'name': 'paused', 'type': 'bool', + 'advanced': True, 'default': False, 'description': 'Add the torrent paused.', }, @@ -81,12 +70,14 @@ config = [{ { 'name': 'stalled_as_failed', 'default': True, + 'advanced': True, 'type': 'bool', 'description': 'Consider a stalled torrent as failed', }, { 'name': 'delete_failed', 'default': True, + 'advanced': True, 'type': 'bool', 'description': 'Delete a release after the download has failed.', }, diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 653343b3..af79cb96 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -55,11 +55,11 @@ class Transmission(Downloader): # Change parameters of torrent torrent_params = {} - if data.get('seed_ratio') and self.conf('seeding'): + if data.get('seed_ratio'): torrent_params['seedRatioLimit'] = tryFloat(data.get('seed_ratio')) torrent_params['seedRatioMode'] = 1 - if data.get('seed_time') and self.conf('seeding'): + if data.get('seed_time'): torrent_params['seedIdleLimit'] = tryInt(data.get('seed_time')) * 60 torrent_params['seedIdleMode'] = 1 diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 025eb9a2..8da2277c 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -36,17 +36,11 @@ config = [{ 'name': 'label', 'description': 'Label to add torrent as.', }, - { - 'name': 'seeding', - 'label': 'Seeding support', - 'default': True, - 'type': 'bool', - 'description': '(Hard)links/copies after download is complete (if enabled in renamer), wait for seeding to finish before (re)moving.', - }, { 'name': 'remove_complete', 'label': 'Remove torrent', - 'default': True, + 'default': False, + 'advanced': True, 'type': 'bool', 'description': 'Remove the torrent from uTorrent after it finished seeding.', }, @@ -61,6 +55,7 @@ config = [{ { 'name': 'paused', 'type': 'bool', + 'advanced': True, 'default': False, 'description': 'Add the torrent paused.', }, @@ -71,6 +66,13 @@ config = [{ 'advanced': True, 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', }, + { + 'name': 'delete_failed', + 'default': True, + 'advanced': True, + 'type': 'bool', + 'description': 'Delete a release after the download has failed.', + }, ], } ], diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 5a76654e..915c1c37 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -15,7 +15,6 @@ import time import urllib import urllib2 - log = CPLog(__name__) @@ -31,8 +30,8 @@ class uTorrent(Downloader): log.error('Config properties are not filled in correctly, port is missing.') return False - if not self.utorrent_api: - self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + return self.utorrent_api def download(self, data, movie, filedata = None): @@ -42,19 +41,23 @@ class uTorrent(Downloader): if not self.connect(): return False + print 'test' + settings = self.utorrent_api.get_settings() if not settings: return False #Fix settings in case they are not set for CPS compatibility new_settings = {} - if self.conf('seeding') and not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): + if not (settings.get('seed_prio_limitul') == 0 and settings['seed_prio_limitul_flag']): new_settings['seed_prio_limitul'] = 0 new_settings['seed_prio_limitul_flag'] = True log.info('Updated uTorrent settings to set a torrent to complete after it the seeding requirements are met.') + if settings.get('bt.read_only_on_complete'): #This doesnt work as this option seems to be not available through the api new_settings['bt.read_only_on_complete'] = False log.info('Updated uTorrent settings to not set the files to read only after completing.') + if new_settings: self.utorrent_api.set_settings(new_settings) @@ -74,11 +77,11 @@ class uTorrent(Downloader): torrent_hash = sha1(bencode(info)).hexdigest().upper() torrent_filename = self.createFileName(data, filedata, movie) - if data.get('seed_ratio') and self.conf('seeding'): + if data.get('seed_ratio'): torrent_params['seed_override'] = 1 torrent_params['seed_ratio'] = tryInt(tryFloat(data['seed_ratio']) * 1000) - if data.get('seed_time') and self.conf('seeding'): + if data.get('seed_time'): torrent_params['seed_override'] = 1 torrent_params['seed_time'] = tryInt(data['seed_time']) * 3600 @@ -130,10 +133,7 @@ class uTorrent(Downloader): if 'Finished' in item[21]: status = 'completed' elif 'Seeding' in item[21]: - if self.conf('seeding'): - status = 'seeding' - else: - status = 'completed' + status = 'seeding' statuses.append({ 'id': item[0], @@ -152,6 +152,12 @@ class uTorrent(Downloader): return False return self.utorrent_api.pause_torrent(download_info['id'], pause) + def removeFailed(self, item): + log.info('%s failed downloading, deleting...', item['name']) + if not self.connect(): + return False + return self.utorrent_api.remove_torrent(item['id'], remove_data = True) + def processComplete(self, item, delete_files = False): log.debug('Requesting uTorrent to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) if not self.connect(): diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 119e5d45..bdd8f8ec 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -164,6 +164,7 @@ class Renamer(Plugin): movie_title = getTitle(library) # Find subtitle for renaming + group['before_rename'] = [] fireEvent('renamer.before', group) # Remove weird chars from moviename @@ -451,7 +452,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) - if group['dirname'] and group['parentdir']: + if group['dirname'] and group['parentdir'] and self.conf('file_action') == 'move': try: log.info('Deleting folder: %s', group['parentdir']) self.deleteEmptyFolder(group['parentdir']) @@ -642,17 +643,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) for rel in rels: rel_dict = rel.to_dict({'info': {}}) - # Get current selected title - default_title = getTitle(rel.movie.library) - - # Check if movie has already completed and is manage tab (legacy db correction) - if rel.movie.status_id == done_status.get('id') and rel.status_id == snatched_status.get('id'): - log.debug('Found a completed movie with a snatched release : %s. Setting release status to ignored...' , default_title) - rel.status_id = ignored_status.get('id') - rel.last_edit = int(time.time()) - db.commit() - continue - movie_dict = fireEvent('movie.get', rel.movie_id, single = True) # check status @@ -678,8 +668,8 @@ Remove it if you want it to be renamed (again, or at least let it try again) if item['folder'] and self.conf('from') in item['folder']: self.tagDir(item['folder'], 'downloading') - pass elif item['status'] == 'seeding': + #If linking setting is enabled, process release if self.conf('file_action') != 'move' and not rel.movie.status_id == done_status.get('id') and item['id'] and item['downloader'] and item['folder']: log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (item['name'], item['seed_ratio'])) @@ -702,7 +692,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) #let it seed log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio'])) - pass elif item['status'] == 'failed': fireEvent('download.remove_failed', item, single = True) rel.status_id = failed_status.get('id') @@ -815,7 +804,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) return download_info and download_info.get('type') in ['torrent', 'torrent_magnet'] def fileIsAdded(self, src, group): - if not group['files'].get('added'): + if not group or not group.get('before_rename'): return False - return src in group['files']['added'] + return src in group['before_rename'] diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index 0ea1de30..ea836f08 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -60,7 +60,7 @@ class Subtitle(Plugin): for d_sub in downloaded: log.info('Found subtitle (%s): %s', (d_sub.language.alpha2, files)) group['files']['subtitle'].append(d_sub.path) - group['files']['added'].append(d_sub.path) + group['before_rename'].append(d_sub.path) group['subtitle_language'][d_sub.path] = [d_sub.language.alpha2] return True From 5ff8c7302f8e35dc17d848728e319a88c0084ecd Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 9 Jul 2013 23:08:33 +0200 Subject: [PATCH 026/209] Sabnzbd prio description --- couchpotato/core/downloaders/sabnzbd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 392bb193..48692dae 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -41,7 +41,7 @@ config = [{ 'default': '0', 'advanced': True, 'values': [('Paused', -2), ('Low', -1), ('Normal', 0), ('High', 1), ('Forced', 2)], - 'description': 'Priority in the SABnzbd queue. Option to add paused.', + 'description': 'Add to the queue with this priority.', }, { 'name': 'manual', From 318daaf0831bc3d4410626cf2b5945b6871b641f Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 9 Jul 2013 23:31:43 +0200 Subject: [PATCH 027/209] Cleanup BitSoup --- .../core/providers/torrent/bitsoup/__init__.py | 14 ++++++++++++++ .../core/providers/torrent/bitsoup/main.py | 16 ++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/providers/torrent/bitsoup/__init__.py b/couchpotato/core/providers/torrent/bitsoup/__init__.py index 097f3782..ac24e131 100644 --- a/couchpotato/core/providers/torrent/bitsoup/__init__.py +++ b/couchpotato/core/providers/torrent/bitsoup/__init__.py @@ -28,6 +28,20 @@ config = [{ 'default': '', 'type': 'password', }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', + }, { 'name': 'extra_score', 'advanced': True, diff --git a/couchpotato/core/providers/torrent/bitsoup/main.py b/couchpotato/core/providers/torrent/bitsoup/main.py index 3239656b..539ba43d 100644 --- a/couchpotato/core/providers/torrent/bitsoup/main.py +++ b/couchpotato/core/providers/torrent/bitsoup/main.py @@ -22,7 +22,7 @@ class Bitsoup(TorrentProvider): def _searchOnTitle(self, title, movie, quality, results): - q = '"%s %s"' % (simplifyString(title), movie['library']['year']) + q = '"%s" %s' % (simplifyString(title), movie['library']['year']) arguments = tryUrlencode({ 'search': q, }) @@ -34,19 +34,19 @@ class Bitsoup(TorrentProvider): html = BeautifulSoup(data) try: - resultsTable = html.find_all('table')[8] - entries = resultsTable.find_all('tr') + result_table = html.find('table', attrs = {'class': 'koptekst'}) + entries = result_table.find_all('tr') for result in entries[1:]: - + all_cells = result.find_all('td') torrent = all_cells[1].find('a') download = all_cells[3].find('a') - + torrent_id = torrent['href'] torrent_id = torrent_id.replace('details.php?id=', '') torrent_id = torrent_id.replace('&hit=1', '') - + torrent_name = torrent.getText() torrent_size = self.parseSize(all_cells[7].getText()) @@ -54,7 +54,7 @@ class Bitsoup(TorrentProvider): torrent_leechers = tryInt(all_cells[10].getText()) torrent_url = self.urls['baseurl'] % download['href'] torrent_detail_url = self.urls['baseurl'] % torrent['href'] - + results.append({ 'id': torrent_id, 'name': torrent_name, @@ -79,6 +79,6 @@ class Bitsoup(TorrentProvider): def loginSuccess(self, output): return 'logout.php' in output.lower() - + loginCheckSuccess = loginSuccess From ed60b4670e289cc00f76b6099c2a231b572dcedc Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 11 Jul 2013 15:04:39 +0200 Subject: [PATCH 028/209] Move root creation to metadata base --- couchpotato/core/providers/metadata/base.py | 6 +++--- couchpotato/core/providers/metadata/xbmc/main.py | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/providers/metadata/base.py b/couchpotato/core/providers/metadata/base.py index b41960a0..d7de8988 100644 --- a/couchpotato/core/providers/metadata/base.py +++ b/couchpotato/core/providers/metadata/base.py @@ -40,7 +40,7 @@ class MetaDataBase(Plugin): # Get file path name = getattr(self, 'get' + file_type.capitalize() + 'Name')(meta_name, root) - if name and self.conf('meta_' + file_type): + if name and (self.conf('meta_' + file_type) or self.conf('meta_' + file_type) is None): # Get file content content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = group) @@ -60,8 +60,8 @@ class MetaDataBase(Plugin): except: log.error('Unable to create %s file: %s', (file_type, traceback.format_exc())) - def getRootName(self, data): - return + def getRootName(self, data = {}): + return os.path.join(data['destination_dir'], data['filename']) def getFanartName(self, name, root): return diff --git a/couchpotato/core/providers/metadata/xbmc/main.py b/couchpotato/core/providers/metadata/xbmc/main.py index 1fd95846..820df15b 100644 --- a/couchpotato/core/providers/metadata/xbmc/main.py +++ b/couchpotato/core/providers/metadata/xbmc/main.py @@ -12,9 +12,6 @@ log = CPLog(__name__) class XBMC(MetaDataBase): - def getRootName(self, data = {}): - return os.path.join(data['destination_dir'], data['filename']) - def getFanartName(self, name, root): return self.createMetaName(self.conf('meta_fanart_name'), name, root) From 60e0ad1f5d4d5dd982fddfff20324306517d34e0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 11 Jul 2013 15:05:08 +0200 Subject: [PATCH 029/209] Add Windows Media Center / Explorer folder.jpg creation. closes #1932 --- .../core/providers/metadata/wmc/__init__.py | 24 +++++++++++++++++++ .../core/providers/metadata/wmc/main.py | 7 ++++++ 2 files changed, 31 insertions(+) create mode 100644 couchpotato/core/providers/metadata/wmc/__init__.py create mode 100644 couchpotato/core/providers/metadata/wmc/main.py diff --git a/couchpotato/core/providers/metadata/wmc/__init__.py b/couchpotato/core/providers/metadata/wmc/__init__.py new file mode 100644 index 00000000..290436c6 --- /dev/null +++ b/couchpotato/core/providers/metadata/wmc/__init__.py @@ -0,0 +1,24 @@ +from .main import WindowsMediaCenter + +def start(): + return WindowsMediaCenter() + +config = [{ + 'name': 'windowsmediacenter', + 'groups': [ + { + 'tab': 'renamer', + 'subtab': 'metadata', + 'name': 'windowsmediacenter_metadata', + 'label': 'Windows Explorer / Media Center', + 'description': 'Generate folder.jpg', + 'options': [ + { + 'name': 'meta_enabled', + 'default': False, + 'type': 'enabler', + }, + ], + }, + ], +}] diff --git a/couchpotato/core/providers/metadata/wmc/main.py b/couchpotato/core/providers/metadata/wmc/main.py new file mode 100644 index 00000000..89258918 --- /dev/null +++ b/couchpotato/core/providers/metadata/wmc/main.py @@ -0,0 +1,7 @@ +from couchpotato.core.providers.metadata.base import MetaDataBase +import os + +class WindowsMediaCenter(MetaDataBase): + + def getThumbnailName(self, name, root): + return os.path.join(root, 'folder.jpg') From 30f5a664872afee4e0669a3d0a2712cce8904da7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 11 Jul 2013 15:24:20 +0200 Subject: [PATCH 030/209] AwesomeHD: Log wrong passkey. fix #1912 --- couchpotato/core/providers/torrent/awesomehd/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchpotato/core/providers/torrent/awesomehd/main.py b/couchpotato/core/providers/torrent/awesomehd/main.py index fed12519..79482f2a 100644 --- a/couchpotato/core/providers/torrent/awesomehd/main.py +++ b/couchpotato/core/providers/torrent/awesomehd/main.py @@ -25,6 +25,11 @@ class AwesomeHD(TorrentProvider): if data: try: soup = BeautifulSoup(data) + + if soup.find('error'): + log.error(soup.find('error').get_text()) + return + authkey = soup.find('authkey').get_text() entries = soup.find_all('torrent') From 9fcf36a2ff4703e851a016860f2a2c5f94fb9f83 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 11 Jul 2013 17:34:55 +0200 Subject: [PATCH 031/209] Add WEB-DL and WEB-Rip. fix #1913 --- couchpotato/core/plugins/quality/main.py | 49 ++++++++++++++++-------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 6ac1c6b6..d7c1a556 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -19,10 +19,10 @@ class QualityPlugin(Plugin): {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, {'identifier': '1080p', 'hd': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts']}, {'identifier': '720p', 'hd': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts']}, - {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi']}, + {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']}, - {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener'], 'allow': ['dvdr', 'dvd'], 'ext':['avi', 'mpg', 'mpeg']}, + {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, + {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener'], 'allow': ['dvdr', 'dvd'], 'ext':['avi', 'mpg', 'mpeg'], 'tags': ['webrip', ('web', 'rip')]}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, @@ -162,24 +162,26 @@ class QualityPlugin(Plugin): for cur_file in files: words = re.split('\W+', cur_file.lower()) + found = {} + for quality in self.all(): + contains = self.containsTag(quality, words, cur_file) + if contains: + found[quality['identifier']] = True + + if 'web' in words and 'dl' in words: + print found + for quality in self.all(): - # Check tags + # Check identifier if quality['identifier'] in words: - log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file)) - return self.setCache(hash, quality) - - if list(set(quality.get('alternative', [])) & set(words)): - log.debug('Found %s via alt %s in %s', (quality['identifier'], quality.get('alternative'), cur_file)) - return self.setCache(hash, quality) - - for tag in quality.get('tags', []): - if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): - log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file)) + if len(found) == 0 or len(found) == 1 and found.get(quality['identifier']): + log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file)) return self.setCache(hash, quality) - if list(set(quality.get('tags', [])) & set(words)): - log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file)) + # Check alt and tags + contains = self.containsTag(quality, words, cur_file) + if contains: return self.setCache(hash, quality) # Try again with loose testing @@ -190,6 +192,21 @@ class QualityPlugin(Plugin): log.debug('Could not identify quality for: %s', files) return None + def containsTag(self, quality, words, cur_file = ''): + + # Check alt and tags + for tag_type in ['alternative', 'tags']: + for alt in quality.get(tag_type, []): + if isinstance(alt, tuple) and '.'.join(alt) in '.'.join(words): + log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) + return True + + if list(set(quality.get(tag_type, [])) & set(words)): + log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) + return True + + return + def guessLoose(self, hash, files = None, extra = None): if extra: From 1f35d0ec2f10b692eda584b79c8d081e09ba9c03 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 11 Jul 2013 17:36:27 +0200 Subject: [PATCH 032/209] Remove debug print --- couchpotato/core/plugins/quality/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index d7c1a556..b1034a53 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -168,9 +168,6 @@ class QualityPlugin(Plugin): if contains: found[quality['identifier']] = True - if 'web' in words and 'dl' in words: - print found - for quality in self.all(): # Check identifier From 9be10f7b79d1454d83217489fa709c8e56e13de0 Mon Sep 17 00:00:00 2001 From: dkboy Date: Fri, 12 Jul 2013 21:49:24 +1200 Subject: [PATCH 033/209] Add Rating / Genre to Dashboard Suggestions Add Rating and up to 3 Genres to movie suggestions, to avoid constantly jumping through to IMDB site. --- .../core/plugins/movie/static/search.js | 19 +++++++++++++++++++ .../plugins/suggestion/static/suggest.css | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 5530505f..67f9755e 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -223,6 +223,14 @@ Block.Search.Item = new Class({ 'text': info.year }) : null ) + ).adopt( + self.rating = info.rating && info.rating.imdb.length > 0 && info.rating.imdb[0] != '0' ? new Element('span.rating', { + 'text': info.rating.imdb[0] + '/10' + }) : null + ).adopt( + self.genre = info.genres && info.genres.length > 0 ? new Element('span.genres', { + 'text': self.getGenres(info.genres) + }) : null ) ) ) @@ -241,6 +249,17 @@ Block.Search.Item = new Class({ self.alternative_titles.include(alternative); }, + getGenres: function(data){ + var self = this; + var genres = []; + + for (var i=0;i Date: Fri, 12 Jul 2013 14:36:04 +0200 Subject: [PATCH 034/209] Style rating and genres --- .../core/plugins/movie/static/search.js | 28 ++++------ .../plugins/suggestion/static/suggest.css | 54 ++++++++++++------- 2 files changed, 45 insertions(+), 37 deletions(-) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 67f9755e..f013aa1a 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -223,14 +223,15 @@ Block.Search.Item = new Class({ 'text': info.year }) : null ) - ).adopt( - self.rating = info.rating && info.rating.imdb.length > 0 && info.rating.imdb[0] != '0' ? new Element('span.rating', { - 'text': info.rating.imdb[0] + '/10' - }) : null - ).adopt( - self.genre = info.genres && info.genres.length > 0 ? new Element('span.genres', { - 'text': self.getGenres(info.genres) - }) : null + ).grab( + self.rating = info.rating && info.rating.imdb.length == 2 && parseFloat(info.rating.imdb[0]) > 0 ? new Element('span.rating', { + 'text': parseFloat(info.rating.imdb[0]), + 'title': parseInt(info.rating.imdb[1]) + ' votes' + }) : null + ).grab( + self.genre = info.genres && info.genres.length > 0 ? new Element('span.genres', { + 'text': info.genres.slice(0, 3).join(', ') + }) : null ) ) ) @@ -249,17 +250,6 @@ Block.Search.Item = new Class({ self.alternative_titles.include(alternative); }, - getGenres: function(data){ - var self = this; - var genres = []; - - for (var i=0;i Date: Fri, 12 Jul 2013 14:42:59 +0200 Subject: [PATCH 035/209] Combine adopt --- couchpotato/core/plugins/movie/static/search.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index f013aa1a..86990f35 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -223,12 +223,11 @@ Block.Search.Item = new Class({ 'text': info.year }) : null ) - ).grab( + ).adopt( self.rating = info.rating && info.rating.imdb.length == 2 && parseFloat(info.rating.imdb[0]) > 0 ? new Element('span.rating', { 'text': parseFloat(info.rating.imdb[0]), 'title': parseInt(info.rating.imdb[1]) + ' votes' - }) : null - ).grab( + }) : null, self.genre = info.genres && info.genres.length > 0 ? new Element('span.genres', { 'text': info.genres.slice(0, 3).join(', ') }) : null From ebf37f7310d5963f964de3ffff5392550ca9fa86 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 12 Jul 2013 20:52:41 +0200 Subject: [PATCH 036/209] Cleanup plex urls --- couchpotato/core/notifications/plex/main.py | 27 ++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index 86da9cd5..02c9b30a 100644 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -1,9 +1,10 @@ from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import tryUrlencode -from couchpotato.core.helpers.variable import cleanHost +from couchpotato.core.helpers.variable import cleanHost, splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from urllib2 import URLError +from urlparse import urlparse from xml.dom import minidom import traceback @@ -20,12 +21,12 @@ class Plex(Notification): if self.isDisabled(): return log.info('Sending notification to Plex') - hosts = [cleanHost(x.strip() + ':32400') for x in self.conf('host').split(",")] + hosts = self.getHosts(port = 32400) for host in hosts: source_type = ['movie'] - base_url = '%slibrary/sections' % host + base_url = '%s/library/sections' % host refresh_url = '%s/%%s/refresh' % base_url try: @@ -46,7 +47,7 @@ class Plex(Notification): def notify(self, message = '', data = {}, listener = None): - hosts = [x.strip() + ':3000' for x in self.conf('host').split(",")] + hosts = self.getHosts(port = 3000) successful = 0 for host in hosts: if self.send({'command': 'ExecBuiltIn', 'parameter': 'Notification(CouchPotato, %s)' % message}, host): @@ -56,8 +57,7 @@ class Plex(Notification): def send(self, command, host): - url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command)) - + url = '%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command)) headers = {} try: @@ -88,3 +88,18 @@ class Plex(Notification): return { 'success': success or success2 } + + def getHosts(self, port = None): + + raw_hosts = splitString(self.conf('host')) + hosts = [] + + for h in raw_hosts: + h = cleanHost(h) + p = urlparse(h) + h = h.rstrip('/') + if port and not p.port: + h += ':%s' % port + hosts.append(h) + + return hosts From 954018fea2ca62dec0ef033e93776d287830dbe8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 12 Jul 2013 21:03:03 +0200 Subject: [PATCH 037/209] Youtube trailer search in https --- couchpotato/core/plugins/movie/static/movie.actions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/static/movie.actions.js b/couchpotato/core/plugins/movie/static/movie.actions.js index a0f7bad5..da47705c 100644 --- a/couchpotato/core/plugins/movie/static/movie.actions.js +++ b/couchpotato/core/plugins/movie/static/movie.actions.js @@ -408,7 +408,7 @@ MA.Trailer = new Class({ watch: function(offset){ var self = this; - var data_url = 'http://gdata.youtube.com/feeds/videos?vq="{title}" {year} trailer&max-results=1&alt=json-in-script&orderby=relevance&sortorder=descending&format=5&fmt=18' + var data_url = 'https://gdata.youtube.com/feeds/videos?vq="{title}" {year} trailer&max-results=1&alt=json-in-script&orderby=relevance&sortorder=descending&format=5&fmt=18' var url = data_url.substitute({ 'title': encodeURI(self.getTitle()), 'year': self.get('year'), From 7692322fbad526eeb6c37912301399bc22ccd8f1 Mon Sep 17 00:00:00 2001 From: dkboy Date: Sat, 13 Jul 2013 16:45:39 +1200 Subject: [PATCH 038/209] Expand IMDB automation provider to include charts Expand IMDB automation provider to include certain top charts, this includes the 'in theaters' list, as well as the top 250 list. They both respect the minimum requirement settings. --- .../providers/automation/imdb/__init__.py | 28 ++++++- .../core/providers/automation/imdb/main.py | 83 +++++++++++++++++-- 2 files changed, 105 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py index a0013c4a..ee804af1 100644 --- a/couchpotato/core/providers/automation/imdb/__init__.py +++ b/couchpotato/core/providers/automation/imdb/__init__.py @@ -9,7 +9,7 @@ config = [{ { 'tab': 'automation', 'list': 'watchlist_providers', - 'name': 'imdb_automation', + 'name': 'imdb_automation_watchlist', 'label': 'IMDB', 'description': 'From any public IMDB watchlists. Url should be the CSV link.', 'options': [ @@ -30,5 +30,31 @@ config = [{ }, ], }, + { + 'tab': 'automation', + 'list': 'automation_providers', + 'name': 'imdb_automation_charts', + 'label': 'IMDB', + 'description': 'Import movies from IMDB Charts', + 'options': [ + { + 'name': 'automation_enabled', + 'default': False, + 'type': 'enabler', + }, + { + 'name': 'automation_charts_theaters_use', + 'type': 'checkbox', + 'label': 'In Theaters', + 'description': 'New Movies In-Theaters chart', + }, + { + 'name': 'automation_charts_top250_use', + 'type': 'checkbox', + 'label': 'TOP 250', + 'description': 'IMDB TOP 250 chart', + }, + ], + }, ], }] diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index 75a2d75c..0d494949 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -1,7 +1,9 @@ +from bs4 import BeautifulSoup from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import getImdb, splitString, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation +import re import traceback log = CPLog(__name__) @@ -11,22 +13,91 @@ class IMDB(Automation, RSS): interval = 1800 + chart_urls = { + 'theater': 'http://www.imdb.com/movies-in-theaters/', + 'top250': 'http://www.imdb.com/chart/top', + } + + def getIMDBids(self): movies = [] - enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] - urls = splitString(self.conf('automation_urls')) + # Handle Chart URLs + if self.conf('automation_charts_theaters_use'): + log.debug('Started IMDB chart: %s', self.chart_urls['theater']) + data = self.getHTMLData(self.chart_urls['theater']) + if data: + html = BeautifulSoup(data) + + try: + result_div = html.find('div', attrs = {'id': 'main'}) + + entries = result_div.find_all('div', attrs = {'itemtype': 'http://schema.org/Movie'}) + + for entry in entries: + title = entry.find('h4', attrs = {'itemprop': 'name'}).getText() + + log.debug('Identified title: %s', title) + result = re.search('(.*) \((.*)\)', title) + + if result: + name = result.group(1) + year = result.group(2) + + imdb = self.search(name, year) + + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + + except: + log.error('Failed loading IMDB chart results from %s: %s', (self.chart_urls['theater'], traceback.format_exc())) + + if self.conf('automation_charts_top250_use'): + log.debug('Started IMDB chart: %s', self.chart_urls['top250']) + data = self.getHTMLData(self.chart_urls['top250']) + if data: + html = BeautifulSoup(data) + + try: + result_div = html.find('div', attrs = {'id': 'main'}) + + result_table = result_div.find_all('table')[1] + entries = result_table.find_all('tr') + + for entry in entries[1:]: + title = entry.find_all('td')[2].getText() + + log.debug('Identified title: %s', title) + result = re.search('(.*) \((.*)\)', title) + + if result: + name = result.group(1) + year = result.group(2) + + imdb = self.search(name, year) + + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) + + except: + log.error('Failed loading IMDB chart results from %s: %s', (self.chart_urls['theater'], traceback.format_exc())) + + + # Handle Watchlists + watchlist_enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] + watchlist_urls = splitString(self.conf('automation_urls')) index = -1 - for url in urls: + for watchlist_url in watchlist_urls: index += 1 - if not enablers[index]: + if not watchlist_enablers[index]: continue try: - rss_data = self.getHTMLData(url) + log.debug('Started IMDB watchlists: %s', watchlist_url) + rss_data = self.getHTMLData(watchlist_url) imdbs = getImdb(rss_data, multiple = True) if rss_data else [] for imdb in imdbs: @@ -35,4 +106,6 @@ class IMDB(Automation, RSS): except: log.error('Failed loading IMDB watchlist: %s %s', (url, traceback.format_exc())) + + # Return the combined resultset return movies From 2584abda0eafc225ca3ee41510382129b1a4eca2 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Wed, 10 Jul 2013 22:38:53 +0200 Subject: [PATCH 039/209] Several fixes and increased readability --- couchpotato/core/downloaders/base.py | 3 +-- couchpotato/core/downloaders/sabnzbd/main.py | 2 +- .../core/downloaders/utorrent/__init__.py | 2 +- couchpotato/core/downloaders/utorrent/main.py | 2 -- couchpotato/core/plugins/renamer/main.py | 26 +++++++++++++------ 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index a7668205..b820b9ff 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -86,8 +86,7 @@ class Downloader(Provider): return self.processComplete(item = item, delete_files = self.conf('delete_files', default = False)) return False - - return False + return def processComplete(self, item, delete_files): return diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index d72dc05b..776749be 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -132,7 +132,7 @@ class Sabnzbd(Downloader): return True def processComplete(self, item, delete_files = False): - log.debug('Requesting SabNZBd to remove the NZB %s%s.', (item['name'])) + log.debug('Requesting SabNZBd to remove the NZB %s.', item['name']) try: self.call({ diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 8da2277c..6a1da36b 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -11,7 +11,7 @@ config = [{ 'list': 'download_providers', 'name': 'utorrent', 'label': 'uTorrent', - 'description': 'Use uTorrent to download torrents.', + 'description': 'Use uTorrent (3.0+) to download torrents.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 915c1c37..be6ff107 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -41,8 +41,6 @@ class uTorrent(Downloader): if not self.connect(): return False - print 'test' - settings = self.utorrent_api.get_settings() if not settings: return False diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index bdd8f8ec..6757bb82 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -327,7 +327,7 @@ class Renamer(Plugin): for movie in library.movies: - # Mark movie "done" onces it found the quality with the finish check + # Mark movie "done" once it's found the quality with the finish check try: if movie.status_id == active_status.get('id') and movie.profile: for profile_type in movie.profile.types: @@ -365,7 +365,7 @@ class Renamer(Plugin): self.tagDir(group, 'exists') # Notify on rename fail - download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) + download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label) fireEvent('movie.renaming.canceled', message = download_message, data = group) remove_leftovers = False @@ -440,7 +440,7 @@ class Renamer(Plugin): self.tagDir(group, 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent - if (movie_folder and self.conf('from') in movie_folder or not movie_folder) and \ + if self.movieInFromFolder(movie_folder) and \ self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info): self.tagDir(group, 'renamed_already') @@ -452,7 +452,8 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) - if group['dirname'] and group['parentdir'] and self.conf('file_action') == 'move': + if group['dirname'] and group['parentdir'] and \ + not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)): try: log.info('Deleting folder: %s', group['parentdir']) self.deleteEmptyFolder(group['parentdir']) @@ -538,7 +539,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) if forcemove: shutil.move(old, dest) elif self.conf('file_action') == 'hardlink': - link(old, dest) + try: + link(old, dest) + except: + log.error('Couldn\'t hardlink file "%s" to "%s". Copying instead. Error: %s. ', (old, dest, traceback.format_exc())) + shutil.copy(old, dest) elif self.conf('file_action') == 'copy': shutil.copy(old, dest) elif self.conf('file_action') == 'move_symlink': @@ -671,7 +676,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif item['status'] == 'seeding': #If linking setting is enabled, process release - if self.conf('file_action') != 'move' and not rel.movie.status_id == done_status.get('id') and item['id'] and item['downloader'] and item['folder']: + if self.conf('file_action') != 'move' and not rel.movie.status_id == done_status.get('id') and self.statusInfoComplete(item): log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (item['name'], item['seed_ratio'])) # Remove the downloading tag @@ -702,11 +707,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) fireEvent('searcher.try_next_release', movie_id = rel.movie_id) elif item['status'] == 'completed': log.info('Download of %s completed!', item['name']) - if item['id'] and item['downloader'] and item['folder']: + if self.statusInfoComplete(item): # If the release has been seeding, process now the seeding is done if rel.status_id == seeding_status.get('id'): - if rel.movie.status_id == done_status.get('id'): # and self.conf('file_action') != 'move': + if rel.movie.status_id == done_status.get('id'): # Set the release to done as the movie has already been renamed rel.status_id = downloaded_status.get('id') rel.last_edit = int(time.time()) @@ -808,3 +813,8 @@ Remove it if you want it to be renamed (again, or at least let it try again) return False return src in group['before_rename'] + def statusInfoComplete(self, item): + return item['id'] and item['downloader'] and item['folder'] + + def movieInFromFolder(self, movie_folder): + return movie_folder and self.conf('from') in movie_folder or not movie_folder \ No newline at end of file From 412627aab06dbd6c23c3b8c9f18620fca10eb291 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 13 Jul 2013 17:52:40 +0200 Subject: [PATCH 040/209] Move rating and genres to suggestions only --- couchpotato/core/plugins/movie/static/search.js | 10 +--------- .../core/plugins/suggestion/static/suggest.js | 12 ++++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 86990f35..5dcef07b 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -215,7 +215,7 @@ Block.Search.Item = new Class({ 'click': self.showOptions.bind(self) } }).adopt( - new Element('div.info').adopt( + self.info_container = new Element('div.info').adopt( self.title = new Element('h2', { 'text': info.titles && info.titles.length > 0 ? info.titles[0] : 'Unknown' }).adopt( @@ -223,14 +223,6 @@ Block.Search.Item = new Class({ 'text': info.year }) : null ) - ).adopt( - self.rating = info.rating && info.rating.imdb.length == 2 && parseFloat(info.rating.imdb[0]) > 0 ? new Element('span.rating', { - 'text': parseFloat(info.rating.imdb[0]), - 'title': parseInt(info.rating.imdb[1]) + ' votes' - }) : null, - self.genre = info.genres && info.genres.length > 0 ? new Element('span.genres', { - 'text': info.genres.slice(0, 3).join(', ') - }) : null ) ) ) diff --git a/couchpotato/core/plugins/suggestion/static/suggest.js b/couchpotato/core/plugins/suggestion/static/suggest.js index 5be7d139..f287588a 100644 --- a/couchpotato/core/plugins/suggestion/static/suggest.js +++ b/couchpotato/core/plugins/suggestion/static/suggest.js @@ -71,6 +71,18 @@ var SuggestList = new Class({ ) ); m.data_container.removeEvents('click'); + + // Add rating + m.info_container.adopt( + m.rating = m.info.rating && m.info.rating.imdb.length == 2 && parseFloat(m.info.rating.imdb[0]) > 0 ? new Element('span.rating', { + 'text': parseFloat(m.info.rating.imdb[0]), + 'title': parseInt(m.info.rating.imdb[1]) + ' votes' + }) : null, + m.genre = m.info.genres && m.info.genres.length > 0 ? new Element('span.genres', { + 'text': m.info.genres.slice(0, 3).join(', ') + }) : null + ) + $(m).inject(self.el); }); From 4ebbc1a01d8bda9f5b4435afbc936ec9ff6c992d Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sun, 14 Jul 2013 02:19:35 +0200 Subject: [PATCH 041/209] XBMC: Only scan the new movie folder --- couchpotato/core/notifications/xbmc/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index ad6fa605..b1ad57a1 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -34,7 +34,7 @@ class XBMC(Notification): ] if not self.conf('only_first') or hosts.index(host) == 0: - calls.append(('VideoLibrary.Scan', {})) + calls.append(('VideoLibrary.Scan', {'directory': data.get('destination_dir', None)})) max_successful += len(calls) response = self.request(host, calls) From 564a27461d6ab020c4bf83ea5101958d74febda2 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sun, 14 Jul 2013 23:30:37 +0200 Subject: [PATCH 042/209] XBMC: Only add directory if XBMC is on localhost --- couchpotato/core/notifications/xbmc/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index b1ad57a1..7266b25c 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -34,7 +34,7 @@ class XBMC(Notification): ] if not self.conf('only_first') or hosts.index(host) == 0: - calls.append(('VideoLibrary.Scan', {'directory': data.get('destination_dir', None)})) + calls.append(('VideoLibrary.Scan', {'directory': data.get('destination_dir', None)} if 'localhost' in host else {})) max_successful += len(calls) response = self.request(host, calls) From 046c7e732fd7f68698c13b38b3579a0a66d6e29d Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sun, 14 Jul 2013 23:43:07 +0200 Subject: [PATCH 043/209] Add rpc_url to Transmission options Fixes #1832 --- couchpotato/core/downloaders/transmission/__init__.py | 7 +++++++ couchpotato/core/downloaders/transmission/main.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 2cbd8b14..d0e8279e 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -25,6 +25,13 @@ config = [{ 'default': 'localhost:9091', 'description': 'Hostname with port. Usually localhost:9091', }, + { + 'name': 'rpc_url', + 'type': 'string', + 'default': 'transmission', + 'advanced': True, + 'description': 'Change if you don\'t run Transmission RPC at the default url.', + }, { 'name': 'username', }, diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index af79cb96..a619d411 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -28,7 +28,7 @@ class Transmission(Downloader): return False if not self.trpc: - self.trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url'), username = self.conf('username'), password = self.conf('password')) return self.trpc @@ -144,11 +144,11 @@ class Transmission(Downloader): class TransmissionRPC(object): """TransmissionRPC lite library""" - def __init__(self, host = 'localhost', port = 9091, username = None, password = None): + def __init__(self, host = 'localhost', port = 9091, rpc_url = 'transmission', username = None, password = None): super(TransmissionRPC, self).__init__() - self.url = 'http://' + host + ':' + str(port) + '/transmission/rpc' + self.url = 'http://' + host + ':' + str(port) + '/' + rpc_url + '/rpc' self.tag = 0 self.session_id = 0 self.session = {} From 3650624e4b8e694b5343380219cd1453491a7fc7 Mon Sep 17 00:00:00 2001 From: iguyking Date: Sun, 14 Jul 2013 11:49:48 -0500 Subject: [PATCH 044/209] Update contributing.md Fixed to say what was intended --- contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing.md b/contributing.md index 572dd332..ef8546f0 100644 --- a/contributing.md +++ b/contributing.md @@ -12,4 +12,4 @@ * What hardware / OS are you using and what are the limits? NAS can be slow and maybe have a different python installed then when you use CP on OSX or Windows for example. * I will mark issues with the "can't reproduce" tag. Don't go asking me "why closed" if it clearly says the issue in the tag ;) -**If I don't get enough info, the change of the issue getting closed is a lot bigger ;)** \ No newline at end of file +**If I don't get enough info, the chance of the issue getting closed is a lot bigger ;)** From 9e8a3bc7010fe9223216624ff645af686de94ab0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 15 Jul 2013 22:51:53 +0200 Subject: [PATCH 045/209] Movie category migrate --- .../migration/versions/002_Movie_category.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 couchpotato/core/migration/versions/002_Movie_category.py diff --git a/couchpotato/core/migration/versions/002_Movie_category.py b/couchpotato/core/migration/versions/002_Movie_category.py new file mode 100644 index 00000000..234e1136 --- /dev/null +++ b/couchpotato/core/migration/versions/002_Movie_category.py @@ -0,0 +1,17 @@ +from migrate.changeset.schema import create_column +from sqlalchemy.schema import MetaData, Column, Table, Index +from sqlalchemy.types import Integer + +meta = MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + + category_column = Column('category_id', Integer) + movie = Table('movie', meta, category_column) + create_column(category_column, movie) + Index('ix_movie_category_id', movie.c.category_id).create() + +def downgrade(migrate_engine): + pass From 8b952d4be6da970a89126bc38608713e1cad1e40 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Jul 2013 16:58:49 +0200 Subject: [PATCH 046/209] Combine global and category words --- couchpotato/core/plugins/score/main.py | 27 +++++++------ couchpotato/core/plugins/score/scores.py | 47 +---------------------- couchpotato/core/plugins/searcher/main.py | 17 ++++---- 3 files changed, 27 insertions(+), 64 deletions(-) diff --git a/couchpotato/core/plugins/score/main.py b/couchpotato/core/plugins/score/main.py index 28baae7e..cc87c9ab 100644 --- a/couchpotato/core/plugins/score/main.py +++ b/couchpotato/core/plugins/score/main.py @@ -1,11 +1,12 @@ from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import getTitle +from couchpotato.core.helpers.variable import getTitle, splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.plugins.score.scores import nameScore, CatnameScore, nameRatioScore, \ - sizeScore, providerScore, duplicateScore, partialIgnoredScore, CatpartialIgnoredScore, namePositionScore, \ +from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \ + sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \ halfMultipartScore +from couchpotato.environment import Env log = CPLog(__name__) @@ -18,10 +19,12 @@ class Score(Plugin): def calculate(self, nzb, movie): ''' Calculate the score of a NZB, used for sorting later ''' - if movie and movie['category'] and movie['category']['preferred']: - score = CatnameScore(toUnicode(nzb['name']), movie['library']['year'], movie['category']['preferred']) - else: - score = nameScore(toUnicode(nzb['name']), movie['library']['year']) + # Merge global and category + preferred_words = splitString(Env.setting('preferred_words', section = 'searcher').lower()) + try: preferred_words = list(set(preferred_words + splitString(movie['category']['preferred'].lower()))) + except: pass + + score = nameScore(toUnicode(nzb['name']), movie['library']['year'], preferred_words) for movie_title in movie['library']['titles']: score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title'])) @@ -43,11 +46,13 @@ class Score(Plugin): # Duplicates in name score += duplicateScore(nzb['name'], getTitle(movie['library'])) + # Merge global and category + ignored_words = splitString(Env.setting('ignored_words', section = 'searcher').lower()) + try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower()))) + except: pass + # Partial ignored words - if movie and movie['category'] and movie['category']['ignored']: - score = CatpartialIgnoredScore(nzb['name'], getTitle(movie['library']), movie['category']['ignored']) - else: - score += partialIgnoredScore(nzb['name'], getTitle(movie['library'])) + score += partialIgnoredScore(nzb['name'], getTitle(movie['library']), ignored_words) # Ignore single downloads from multipart score += halfMultipartScore(nzb['name']) diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index 5af07fa4..4d966eb2 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -23,7 +23,7 @@ name_scores = [ ] -def nameScore(name, year): +def nameScore(name, year, preferred_words): ''' Calculate score for words in the NZB name ''' score = 0 @@ -42,38 +42,10 @@ def nameScore(name, year): # Contains preferred word nzb_words = re.split('\W+', simplifyString(name)) - preferred_words = splitString(Env.setting('preferred_words', section = 'searcher')) score += 100 * len(list(set(nzb_words) & set(preferred_words))) return score -def CatnameScore(name, year, preferred): - ''' Calculate score for words in the NZB name ''' - - score = 0 - name = name.lower() - - # give points for the cool stuff - for value in name_scores: - v = value.split(':') - add = int(v.pop()) - if v.pop() in name: - score = score + add - - # points if the year is correct - if str(year) in name: - score = score + 5 - - # Contains preferred word - nzb_words = re.split('\W+', simplifyString(name)) - preferred_words = [x.strip() for x in preferred.split(',')] - for word in preferred_words: - if word.strip() and word.strip().lower() in nzb_words: - score = score + 100 - - return score - - def nameRatioScore(nzb_name, movie_name): nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) movie_words = re.split('\W+', simplifyString(movie_name)) @@ -160,13 +132,11 @@ def duplicateScore(nzb_name, movie_name): return len(list(set(duplicates) - set(movie_words))) * -4 -def partialIgnoredScore(nzb_name, movie_name): +def partialIgnoredScore(nzb_name, movie_name, ignored_words): nzb_name = nzb_name.lower() movie_name = movie_name.lower() - ignored_words = [x.strip().lower() for x in Env.setting('ignored_words', section = 'searcher').split(',')] - score = 0 for ignored_word in ignored_words: if ignored_word in nzb_name and ignored_word not in movie_name: @@ -174,19 +144,6 @@ def partialIgnoredScore(nzb_name, movie_name): return score -def CatpartialIgnoredScore(nzb_name, movie_name, ignored): - - nzb_name = nzb_name.lower() - movie_name = movie_name.lower() - - ignored_words = [x.strip().lower() for x in ignored.split(',')] - - score = 0 - for ignored_word in ignored_words: - if ignored_word in nzb_name and ignored_word not in movie_name: - score -= 5 - - return score def halfMultipartScore(nzb_name): diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 759ac388..dfb6ccba 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -105,6 +105,7 @@ class Searcher(Plugin): for movie in movies: movie_dict = movie.to_dict({ + 'category': {}, 'profile': {'types': {'quality': {}}}, 'releases': {'status': {}, 'quality': {}}, 'library': {'titles': {}, 'files':{}}, @@ -392,10 +393,10 @@ class Searcher(Plugin): nzb_words = re.split('\W+', nzb_name) # Make sure it has required words - try: - required_words = splitString(movie['category']['required'].lower()) - except: - required_words = splitString(self.conf('required_words').lower()) + required_words = splitString(self.conf('required_words').lower()) + try: required_words = list(set(required_words + splitString(movie['category']['required'].lower()))) + except: pass + req_match = 0 for req_set in required_words: req = splitString(req_set, '&') @@ -406,10 +407,10 @@ class Searcher(Plugin): return False # Ignore releases - try: - ignored_words = splitString(movie['category']['ignored'].lower()) - except: - ignored_words = splitString(self.conf('ignored_words').lower()) + ignored_words = splitString(self.conf('ignored_words').lower()) + try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower()))) + except: pass + ignored_match = 0 for ignored_set in ignored_words: ignored = splitString(ignored_set, '&') From 1ea0d3bd8b8194f9a6d477f880aff1b1e8aa7898 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Jul 2013 19:12:32 +0200 Subject: [PATCH 047/209] Move providers to main searcher tab in settings --- couchpotato/core/providers/nzb/__init__.py | 3 +-- couchpotato/core/providers/nzb/binsearch/__init__.py | 1 - couchpotato/core/providers/nzb/ftdworld/__init__.py | 1 - couchpotato/core/providers/nzb/newznab/__init__.py | 1 - couchpotato/core/providers/nzb/nzbclub/__init__.py | 1 - couchpotato/core/providers/nzb/nzbindex/__init__.py | 1 - couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py | 1 - couchpotato/core/providers/torrent/__init__.py | 3 +-- couchpotato/core/providers/torrent/awesomehd/__init__.py | 1 - couchpotato/core/providers/torrent/bitsoup/__init__.py | 1 - couchpotato/core/providers/torrent/hdbits/__init__.py | 1 - couchpotato/core/providers/torrent/iptorrents/__init__.py | 1 - couchpotato/core/providers/torrent/kickasstorrents/__init__.py | 1 - couchpotato/core/providers/torrent/passthepopcorn/__init__.py | 1 - couchpotato/core/providers/torrent/publichd/__init__.py | 1 - couchpotato/core/providers/torrent/sceneaccess/__init__.py | 1 - couchpotato/core/providers/torrent/scenehd/__init__.py | 1 - couchpotato/core/providers/torrent/thepiratebay/__init__.py | 1 - couchpotato/core/providers/torrent/torrentbytes/__init__.py | 1 - couchpotato/core/providers/torrent/torrentday/__init__.py | 1 - couchpotato/core/providers/torrent/torrentleech/__init__.py | 1 - couchpotato/core/providers/torrent/torrentshack/__init__.py | 1 - couchpotato/core/providers/torrent/yify/__init__.py | 1 - 23 files changed, 2 insertions(+), 25 deletions(-) diff --git a/couchpotato/core/providers/nzb/__init__.py b/couchpotato/core/providers/nzb/__init__.py index 651ae8b9..36098bb3 100644 --- a/couchpotato/core/providers/nzb/__init__.py +++ b/couchpotato/core/providers/nzb/__init__.py @@ -2,13 +2,12 @@ config = { 'name': 'nzb_providers', 'groups': [ { - 'label': 'Usenet', + 'label': 'Usenet Providers', 'description': 'Providers searching usenet for new releases', 'wizard': True, 'type': 'list', 'name': 'nzb_providers', 'tab': 'searcher', - 'subtab': 'providers', 'options': [], }, ], diff --git a/couchpotato/core/providers/nzb/binsearch/__init__.py b/couchpotato/core/providers/nzb/binsearch/__init__.py index d3288604..1cfb0b73 100644 --- a/couchpotato/core/providers/nzb/binsearch/__init__.py +++ b/couchpotato/core/providers/nzb/binsearch/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'nzb_providers', 'name': 'binsearch', 'description': 'Free provider, less accurate. See BinSearch', diff --git a/couchpotato/core/providers/nzb/ftdworld/__init__.py b/couchpotato/core/providers/nzb/ftdworld/__init__.py index 37aedb2a..5a004a70 100644 --- a/couchpotato/core/providers/nzb/ftdworld/__init__.py +++ b/couchpotato/core/providers/nzb/ftdworld/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'nzb_providers', 'name': 'FTDWorld', 'description': 'Free provider, less accurate. See FTDWorld', diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py index 90f81cfc..3902ab13 100644 --- a/couchpotato/core/providers/nzb/newznab/__init__.py +++ b/couchpotato/core/providers/nzb/newznab/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'nzb_providers', 'name': 'newznab', 'order': 10, diff --git a/couchpotato/core/providers/nzb/nzbclub/__init__.py b/couchpotato/core/providers/nzb/nzbclub/__init__.py index 9955462c..95eeea13 100644 --- a/couchpotato/core/providers/nzb/nzbclub/__init__.py +++ b/couchpotato/core/providers/nzb/nzbclub/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'nzb_providers', 'name': 'NZBClub', 'description': 'Free provider, less accurate. See NZBClub', diff --git a/couchpotato/core/providers/nzb/nzbindex/__init__.py b/couchpotato/core/providers/nzb/nzbindex/__init__.py index aa8de4dd..47461e63 100644 --- a/couchpotato/core/providers/nzb/nzbindex/__init__.py +++ b/couchpotato/core/providers/nzb/nzbindex/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'nzb_providers', 'name': 'nzbindex', 'description': 'Free provider, less accurate. See NZBIndex', diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py b/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py index fe80518c..933aff3e 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'nzb_providers', 'name': 'OMGWTFNZBs', 'description': 'See OMGWTFNZBs', diff --git a/couchpotato/core/providers/torrent/__init__.py b/couchpotato/core/providers/torrent/__init__.py index 191e132e..250bcead 100644 --- a/couchpotato/core/providers/torrent/__init__.py +++ b/couchpotato/core/providers/torrent/__init__.py @@ -2,13 +2,12 @@ config = { 'name': 'torrent_providers', 'groups': [ { - 'label': 'Torrent', + 'label': 'Torrent Providers', 'description': 'Providers searching torrent sites for new releases', 'wizard': True, 'type': 'list', 'name': 'torrent_providers', 'tab': 'searcher', - 'subtab': 'providers', 'options': [], }, ], diff --git a/couchpotato/core/providers/torrent/awesomehd/__init__.py b/couchpotato/core/providers/torrent/awesomehd/__init__.py index 1dc5a2ea..de6a2144 100644 --- a/couchpotato/core/providers/torrent/awesomehd/__init__.py +++ b/couchpotato/core/providers/torrent/awesomehd/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'Awesome-HD', 'description': 'See AHD', diff --git a/couchpotato/core/providers/torrent/bitsoup/__init__.py b/couchpotato/core/providers/torrent/bitsoup/__init__.py index ac24e131..a36ab08f 100644 --- a/couchpotato/core/providers/torrent/bitsoup/__init__.py +++ b/couchpotato/core/providers/torrent/bitsoup/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'Bitsoup', 'description': 'See Bitsoup', diff --git a/couchpotato/core/providers/torrent/hdbits/__init__.py b/couchpotato/core/providers/torrent/hdbits/__init__.py index f1613d36..07ea95d6 100644 --- a/couchpotato/core/providers/torrent/hdbits/__init__.py +++ b/couchpotato/core/providers/torrent/hdbits/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'HDBits', 'description': 'See HDBits', diff --git a/couchpotato/core/providers/torrent/iptorrents/__init__.py b/couchpotato/core/providers/torrent/iptorrents/__init__.py index 4cb90a18..6cb2dead 100644 --- a/couchpotato/core/providers/torrent/iptorrents/__init__.py +++ b/couchpotato/core/providers/torrent/iptorrents/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'IPTorrents', 'description': 'See IPTorrents', diff --git a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py index eebca28a..b095a97d 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'KickAssTorrents', 'description': 'See KickAssTorrents', diff --git a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py index cc7736a7..66b3ea76 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'PassThePopcorn', 'description': 'See PassThePopcorn.me', diff --git a/couchpotato/core/providers/torrent/publichd/__init__.py b/couchpotato/core/providers/torrent/publichd/__init__.py index d1cb1079..ace12880 100644 --- a/couchpotato/core/providers/torrent/publichd/__init__.py +++ b/couchpotato/core/providers/torrent/publichd/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'PublicHD', 'description': 'Public Torrent site with only HD content. See PublicHD', diff --git a/couchpotato/core/providers/torrent/sceneaccess/__init__.py b/couchpotato/core/providers/torrent/sceneaccess/__init__.py index eaee026f..4b675573 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/__init__.py +++ b/couchpotato/core/providers/torrent/sceneaccess/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'SceneAccess', 'description': 'See SceneAccess', diff --git a/couchpotato/core/providers/torrent/scenehd/__init__.py b/couchpotato/core/providers/torrent/scenehd/__init__.py index 79d2550f..c0a82ae7 100644 --- a/couchpotato/core/providers/torrent/scenehd/__init__.py +++ b/couchpotato/core/providers/torrent/scenehd/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'SceneHD', 'description': 'See SceneHD', diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index 6e469a6c..83de7a94 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'ThePirateBay', 'description': 'The world\'s largest bittorrent tracker. See ThePirateBay', diff --git a/couchpotato/core/providers/torrent/torrentbytes/__init__.py b/couchpotato/core/providers/torrent/torrentbytes/__init__.py index 65c90c30..712eac85 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/__init__.py +++ b/couchpotato/core/providers/torrent/torrentbytes/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'TorrentBytes', 'description': 'See TorrentBytes', diff --git a/couchpotato/core/providers/torrent/torrentday/__init__.py b/couchpotato/core/providers/torrent/torrentday/__init__.py index 9eaa4990..d98bb917 100644 --- a/couchpotato/core/providers/torrent/torrentday/__init__.py +++ b/couchpotato/core/providers/torrent/torrentday/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'TorrentDay', 'description': 'See TorrentDay', diff --git a/couchpotato/core/providers/torrent/torrentleech/__init__.py b/couchpotato/core/providers/torrent/torrentleech/__init__.py index d5b8b241..c788477f 100644 --- a/couchpotato/core/providers/torrent/torrentleech/__init__.py +++ b/couchpotato/core/providers/torrent/torrentleech/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'TorrentLeech', 'description': 'See TorrentLeech', diff --git a/couchpotato/core/providers/torrent/torrentshack/__init__.py b/couchpotato/core/providers/torrent/torrentshack/__init__.py index f6ed401f..4171fc49 100644 --- a/couchpotato/core/providers/torrent/torrentshack/__init__.py +++ b/couchpotato/core/providers/torrent/torrentshack/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'TorrentShack', 'description': 'See TorrentShack', diff --git a/couchpotato/core/providers/torrent/yify/__init__.py b/couchpotato/core/providers/torrent/yify/__init__.py index 30d7ef02..775ecdbe 100644 --- a/couchpotato/core/providers/torrent/yify/__init__.py +++ b/couchpotato/core/providers/torrent/yify/__init__.py @@ -8,7 +8,6 @@ config = [{ 'groups': [ { 'tab': 'searcher', - 'subtab': 'providers', 'list': 'torrent_providers', 'name': 'Yify', 'description': 'Free provider, less accurate. Small HD movies, encoded by Yify.', From dd67239b6e2ded080f70bc5cd4e4c3366c28feb8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 21 Jul 2013 19:12:53 +0200 Subject: [PATCH 048/209] Add categories to settings --- couchpotato/core/plugins/category/__init__.py | 6 + couchpotato/core/plugins/category/main.py | 123 ++++++++ .../core/plugins/category/static/category.css | 84 +++++ .../core/plugins/category/static/category.js | 295 ++++++++++++++++++ .../core/plugins/category/static/handle.png | Bin 0 -> 160 bytes .../core/plugins/quality/static/quality.js | 3 +- couchpotato/core/plugins/searcher/__init__.py | 52 +-- couchpotato/core/settings/model.py | 25 +- couchpotato/static/scripts/page/settings.js | 4 +- couchpotato/templates/index.html | 2 + 10 files changed, 557 insertions(+), 37 deletions(-) create mode 100644 couchpotato/core/plugins/category/__init__.py create mode 100644 couchpotato/core/plugins/category/main.py create mode 100644 couchpotato/core/plugins/category/static/category.css create mode 100644 couchpotato/core/plugins/category/static/category.js create mode 100644 couchpotato/core/plugins/category/static/handle.png diff --git a/couchpotato/core/plugins/category/__init__.py b/couchpotato/core/plugins/category/__init__.py new file mode 100644 index 00000000..6dc41df7 --- /dev/null +++ b/couchpotato/core/plugins/category/__init__.py @@ -0,0 +1,6 @@ +from .main import CategoryPlugin + +def start(): + return CategoryPlugin() + +config = [] diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py new file mode 100644 index 00000000..a91930da --- /dev/null +++ b/couchpotato/core/plugins/category/main.py @@ -0,0 +1,123 @@ +from couchpotato import get_session +from couchpotato.api import addApiView +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.logger import CPLog +from couchpotato.core.plugins.base import Plugin +from couchpotato.core.settings.model import Movie, Category + +log = CPLog(__name__) + + +class CategoryPlugin(Plugin): + + to_dict = {'destination': {}} + + def __init__(self): + addEvent('category.all', self.all) + + addApiView('category.save', self.save) + addApiView('category.save_order', self.saveOrder) + addApiView('category.delete', self.delete) + addApiView('category.list', self.allView, docs = { + 'desc': 'List all available categories', + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'list': array, categories +}"""} + }) + + def allView(self, **kwargs): + + return { + 'success': True, + 'list': self.all() + } + + def all(self): + + db = get_session() + categories = db.query(Category).all() + + temp = [] + for category in categories: + temp.append(category.to_dict(self.to_dict)) + + db.expire_all() + return temp + + def save(self, **kwargs): + + db = get_session() + + c = db.query(Category).filter_by(id = kwargs.get('id')).first() + if not c: + c = Category() + db.add(c) + + c.order = kwargs.get('order', c.order if c.order else 0) + c.label = toUnicode(kwargs.get('label')) + c.path = toUnicode(kwargs.get('path')) + c.ignored = toUnicode(kwargs.get('ignored')) + c.preferred = toUnicode(kwargs.get('preferred')) + c.required = toUnicode(kwargs.get('required')) + + db.commit() + + category_dict = c.to_dict(self.to_dict) + + return { + 'success': True, + 'category': category_dict + } + + def saveOrder(self, **kwargs): + + db = get_session() + + order = 0 + for category_id in kwargs.get('ids', []): + c = db.query(Category).filter_by(id = category_id).first() + c.order = order + + order += 1 + + db.commit() + + return { + 'success': True + } + + def delete(self, id = None, **kwargs): + + db = get_session() + + success = False + message = '' + try: + c = db.query(Category).filter_by(id = id).first() + db.delete(c) + db.commit() + + # Force defaults on all empty category movies + self.removeFromMovie(id) + + success = True + except Exception, e: + message = log.error('Failed deleting category: %s', e) + + db.expire_all() + return { + 'success': success, + 'message': message + } + + def removeFromMovie(self, category_id): + + db = get_session() + movies = db.query(Movie).filter(Movie.category_id == category_id).all() + + if len(movies) > 0: + for movie in movies: + movie.category_id = None + db.commit() diff --git a/couchpotato/core/plugins/category/static/category.css b/couchpotato/core/plugins/category/static/category.css new file mode 100644 index 00000000..fda8ca6f --- /dev/null +++ b/couchpotato/core/plugins/category/static/category.css @@ -0,0 +1,84 @@ +.add_new_category { + padding: 20px; + display: block; + text-align: center; + font-size: 20px; + border-bottom: 1px solid rgba(255,255,255,0.2); +} + +.category { + border-bottom: 1px solid rgba(255,255,255,0.2); + position: relative; +} + + .category > .delete { + position: absolute; + padding: 16px; + right: 0; + cursor: pointer; + opacity: 0.6; + color: #fd5353; + } + .category > .delete:hover { + opacity: 1; + } + + .category .ctrlHolder:hover { + background: none; + } + + .category .formHint { + width: 250px !important; + vertical-align: top !important; + margin: 0 !important; + padding-left: 3px !important; + opacity: 0.1; + } + .category:hover .formHint { + opacity: 1; + } + +#category_ordering { + +} + + #category_ordering ul { + float: left; + margin: 0; + width: 275px; + padding: 0; + } + + #category_ordering li { + cursor: -webkit-grab; + cursor: -moz-grab; + cursor: grab; + border-bottom: 1px solid rgba(255,255,255,0.2); + padding: 0 5px; + } + #category_ordering li:last-child { border: 0; } + + #category_ordering li .check { + margin: 2px 10px 0 0; + vertical-align: top; + } + + #category_ordering li > span { + display: inline-block; + height: 20px; + vertical-align: top; + line-height: 20px; + } + + #category_ordering li .handle { + background: url('../../static/profile_plugin/handle.png') center; + width: 20px; + float: right; + } + + #category_ordering .formHint { + clear: none; + float: right; + width: 250px; + margin: 0; + } \ No newline at end of file diff --git a/couchpotato/core/plugins/category/static/category.js b/couchpotato/core/plugins/category/static/category.js new file mode 100644 index 00000000..092a74cd --- /dev/null +++ b/couchpotato/core/plugins/category/static/category.js @@ -0,0 +1,295 @@ +var CategoryListBase = new Class({ + + initialize: function(){ + var self = this; + + App.addEvent('load', self.addSettings.bind(self)); + }, + + setup: function(categories){ + var self = this; + + self.categories = [] + Array.each(categories, self.createCategory.bind(self)); + + }, + + addSettings: function(){ + var self = this; + + self.settings = App.getPage('Settings') + self.settings.addEvent('create', function(){ + var tab = self.settings.createSubTab('category', { + 'label': 'Categories', + 'name': 'category', + 'subtab_label': 'Category & filtering' + }, self.settings.tabs.searcher ,'searcher'); + + self.tab = tab.tab; + self.content = tab.content; + + self.createList(); + self.createOrdering(); + + }) + + }, + + createList: function(){ + var self = this; + + var count = self.categories.length; + + self.settings.createGroup({ + 'label': 'Categories', + 'description': 'Create your own categories.' + }).inject(self.content).adopt( + self.category_container = new Element('div.container'), + new Element('a.add_new_category', { + 'text': count > 0 ? 'Create another category' : 'Click here to create a category.', + 'events': { + 'click': function(){ + var category = self.createCategory(); + $(category).inject(self.category_container) + } + } + }) + ); + + // Add categories, that aren't part of the core (for editing) + Array.each(self.categories, function(category){ + $(category).inject(self.category_container) + }); + + }, + + createCategory: function(data){ + var self = this; + + var data = data || {'id': randomString()} + var category = new Category(data) + self.categories.include(category) + + return category; + }, + + createOrdering: function(){ + var self = this; + + var category_list; + var group = self.settings.createGroup({ + 'label': 'Category order' + }).adopt( + new Element('.ctrlHolder#category_ordering').adopt( + new Element('label[text=Order]'), + category_list = new Element('ul'), + new Element('p.formHint', { + 'html': 'Change the order the categories are in the dropdown list.
First one will be default.' + }) + ) + ).inject(self.content) + + Array.each(self.categories, function(category){ + new Element('li', {'data-id': category.data.id}).adopt( + new Element('span.category_label', { + 'text': category.data.label + }), + new Element('span.handle') + ).inject(category_list); + + }); + + // Sortable + self.category_sortable = new Sortables(category_list, { + 'revert': true, + 'handle': '', + 'opacity': 0.5, + 'onComplete': self.saveOrdering.bind(self) + }); + + }, + + saveOrdering: function(){ + var self = this; + + var ids = []; + + self.category_sortable.list.getElements('li').each(function(el, nr){ + ids.include(el.get('data-id')); + }); + + Api.request('category.save_order', { + 'data': { + 'ids': ids + } + }); + + } + +}) + +window.CategoryList = new CategoryListBase(); + +var Category = new Class({ + + data: {}, + + initialize: function(data){ + var self = this; + + self.data = data; + self.types = []; + + self.create(); + + self.el.addEvents({ + 'change:relay(select)': self.save.bind(self, 0), + 'keyup:relay(input[type=text])': self.save.bind(self, [300]) + }); + + }, + + create: function(){ + var self = this; + + var data = self.data; + + self.el = new Element('div.category').adopt( + self.delete_button = new Element('span.delete.icon2', { + 'events': { + 'click': self.del.bind(self) + } + }), + new Element('.category_label.ctrlHolder').adopt( + new Element('label', {'text':'Name'}), + new Element('input.inlay', { + 'type':'text', + 'value': data.label, + 'placeholder': 'Label' + }) + ), + new Element('.category_preferred.ctrlHolder').adopt( + new Element('label', {'text':'Preferred'}), + new Element('input.inlay', { + 'type':'text', + 'value': data.preferred, + 'placeholder': 'Ignored' + }) + ), + new Element('.category_required.ctrlHolder').adopt( + new Element('label', {'text':'Required'}), + new Element('input.inlay', { + 'type':'text', + 'value': data.required, + 'placeholder': 'Required' + }) + ), + new Element('.category_ignored.ctrlHolder').adopt( + new Element('label', {'text':'Ignored'}), + new Element('input.inlay', { + 'type':'text', + 'value': data.ignored, + 'placeholder': 'Ignored' + }) + ) + ); + + self.makeSortable() + + }, + + save: function(delay){ + var self = this; + + if(self.save_timer) clearTimeout(self.save_timer); + self.save_timer = (function(){ + + var data = self.getData(); + + Api.request('category.save', { + 'data': self.getData(), + 'useSpinner': true, + 'spinnerOptions': { + 'target': self.el + }, + 'onComplete': function(json){ + if(json.success){ + self.data = json.category; + } + } + }); + + }).delay(delay, self) + + }, + + getData: function(){ + var self = this; + + var data = { + 'id' : self.data.id, + 'label' : self.el.getElement('.category_label input').get('value'), + 'required' : self.el.getElement('.category_required input').get('value'), + 'preferred' : self.el.getElement('.category_preferred input').get('value'), + 'ignored' : self.el.getElement('.category_ignored input').get('value') + } + + return data + }, + + del: function(){ + var self = this; + + var label = self.el.getElement('.category_label input').get('value'); + var qObj = new Question('Are you sure you want to delete "'+label+'"?', '', [{ + 'text': 'Delete "'+label+'"', + 'class': 'delete', + 'events': { + 'click': function(e){ + (e).preventDefault(); + Api.request('category.delete', { + 'data': { + 'id': self.data.id + }, + 'useSpinner': true, + 'spinnerOptions': { + 'target': self.el + }, + 'onComplete': function(json){ + if(json.success) { + qObj.close(); + self.el.destroy(); + } else { + alert(json.message); + } + } + }); + } + } + }, { + 'text': 'Cancel', + 'cancel': true + }]); + + }, + + makeSortable: function(){ + var self = this; + + self.sortable = new Sortables(self.category_container, { + 'revert': true, + 'handle': '.handle', + 'opacity': 0.5, + 'onComplete': self.save.bind(self, 300) + }); + }, + + get: function(attr){ + return this.data[attr] + }, + + toElement: function(){ + return this.el + } + +}); \ No newline at end of file diff --git a/couchpotato/core/plugins/category/static/handle.png b/couchpotato/core/plugins/category/static/handle.png new file mode 100644 index 0000000000000000000000000000000000000000..adff5b2975d044c5f3285813f82fc467ddf240d5 GIT binary patch literal 160 zcmeAS@N?(olHy`uVBq!ia0vp^;y^6M!3HF?%h*|glw^r(L`iUdT1k0gQ7VIDN`6wR zf@f}GdTLN=VoGJ<$y6JlA}3E5$B>F!Nq_$Tw`X3 Date: Mon, 22 Jul 2013 21:56:22 +0200 Subject: [PATCH 049/209] Proper meta tag --- couchpotato/templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/templates/index.html b/couchpotato/templates/index.html index 500dc784..f9bc4634 100644 --- a/couchpotato/templates/index.html +++ b/couchpotato/templates/index.html @@ -2,7 +2,7 @@ - + {% for url in fireEvent('clientscript.get_styles', as_html = True, location = 'front', single = True) %} From e8993932c1928c63dd1a501c2ea69cfcf337b920 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Jul 2013 21:56:33 +0200 Subject: [PATCH 050/209] Check isMac function --- couchpotato/static/scripts/couchpotato.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index db90c305..fdc9bd10 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -56,6 +56,10 @@ History.push(url); } }, + + isMac: function(){ + return Browser.Platform.mac + }, createLayout: function(){ var self = this; From f12d878c0b6f43087d3004b11046578e77b95f80 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 22 Jul 2013 21:57:13 +0200 Subject: [PATCH 051/209] Select category for search, suggest & edit --- .../core/plugins/category/static/category.css | 2 - .../core/plugins/category/static/category.js | 26 ++++++---- couchpotato/core/plugins/movie/main.py | 6 ++- .../plugins/movie/static/movie.actions.js | 40 ++++++++++++++-- .../core/plugins/movie/static/movie.js | 2 + .../core/plugins/movie/static/search.css | 46 ++++++++++++++---- .../core/plugins/movie/static/search.js | 48 ++++++++++++++++--- .../core/plugins/quality/static/quality.js | 3 +- .../plugins/suggestion/static/suggest.css | 12 +++++ 9 files changed, 153 insertions(+), 32 deletions(-) diff --git a/couchpotato/core/plugins/category/static/category.css b/couchpotato/core/plugins/category/static/category.css index fda8ca6f..0987c197 100644 --- a/couchpotato/core/plugins/category/static/category.css +++ b/couchpotato/core/plugins/category/static/category.css @@ -29,9 +29,7 @@ .category .formHint { width: 250px !important; - vertical-align: top !important; margin: 0 !important; - padding-left: 3px !important; opacity: 0.1; } .category:hover .formHint { diff --git a/couchpotato/core/plugins/category/static/category.js b/couchpotato/core/plugins/category/static/category.js index 092a74cd..22bbc38e 100644 --- a/couchpotato/core/plugins/category/static/category.js +++ b/couchpotato/core/plugins/category/static/category.js @@ -42,7 +42,7 @@ var CategoryListBase = new Class({ self.settings.createGroup({ 'label': 'Categories', - 'description': 'Create your own categories.' + 'description': 'Create categories, each one extending global filters. (Needs refresh \'' +(App.isMac() ? 'CMD+R' : 'F5')+ '\' after editing)' }).inject(self.content).adopt( self.category_container = new Element('div.container'), new Element('a.add_new_category', { @@ -63,6 +63,16 @@ var CategoryListBase = new Class({ }, + getCategory: function(id){ + return this.categories.filter(function(category){ + return category.data.id == id + }).pick() + }, + + getAll: function(){ + return this.categories; + }, + createCategory: function(data){ var self = this; @@ -78,7 +88,7 @@ var CategoryListBase = new Class({ var category_list; var group = self.settings.createGroup({ - 'label': 'Category order' + 'label': 'Category ordering' }).adopt( new Element('.ctrlHolder#category_ordering').adopt( new Element('label[text=Order]'), @@ -138,7 +148,6 @@ var Category = new Class({ var self = this; self.data = data; - self.types = []; self.create(); @@ -165,15 +174,16 @@ var Category = new Class({ new Element('input.inlay', { 'type':'text', 'value': data.label, - 'placeholder': 'Label' - }) + 'placeholder': 'Example: Kids, Horror or His' + }), + new Element('p.formHint', {'text': 'See global filters for explanation.'}) ), new Element('.category_preferred.ctrlHolder').adopt( new Element('label', {'text':'Preferred'}), new Element('input.inlay', { 'type':'text', 'value': data.preferred, - 'placeholder': 'Ignored' + 'placeholder': 'Blu-ray, DTS' }) ), new Element('.category_required.ctrlHolder').adopt( @@ -181,7 +191,7 @@ var Category = new Class({ new Element('input.inlay', { 'type':'text', 'value': data.required, - 'placeholder': 'Required' + 'placeholder': 'Example: DTS, AC3 & English' }) ), new Element('.category_ignored.ctrlHolder').adopt( @@ -189,7 +199,7 @@ var Category = new Class({ new Element('input.inlay', { 'type':'text', 'value': data.ignored, - 'placeholder': 'Ignored' + 'placeholder': 'Example: dubbed, swesub, french' }) ) ); diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 0cc98fd3..0e6ef133 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -2,7 +2,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode, simplifyString -from couchpotato.core.helpers.variable import getImdb, splitString +from couchpotato.core.helpers.variable import getImdb, splitString, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, LibraryTitle, Movie, \ @@ -452,6 +452,10 @@ class MoviePlugin(Plugin): m.profile_id = kwargs.get('profile_id') + cat_id = kwargs.get('category_id', None) + if cat_id is not None: + m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None + # Remove releases for rel in m.releases: if rel.status_id is available_status.get('id'): diff --git a/couchpotato/core/plugins/movie/static/movie.actions.js b/couchpotato/core/plugins/movie/static/movie.actions.js index da47705c..ad02e80b 100644 --- a/couchpotato/core/plugins/movie/static/movie.actions.js +++ b/couchpotato/core/plugins/movie/static/movie.actions.js @@ -1,5 +1,5 @@ var MovieAction = new Class({ - + Implements: [Options], class_name: 'action icon2', @@ -521,6 +521,11 @@ MA.Edit = new Class({ self.profile_select = new Element('select', { 'name': 'profile' }), + self.category_select = new Element('select', { + 'name': 'category' + }).grab( + new Element('option', {'value': -1, 'text': 'None'}) + ), new Element('a.button.edit', { 'text': 'Save & Search', 'events': { @@ -540,7 +545,34 @@ MA.Edit = new Class({ }); - Quality.getActiveProfiles().each(function(profile){ + // Fill categories + var categories = CategoryList.getAll(); + + if(categories.length == 0) + self.category_select.hide(); + else { + self.category_select.show(); + categories.each(function(category){ + + var category_id = category.data.id; + + new Element('option', { + 'value': category_id, + 'text': category.data.label + }).inject(self.category_select); + + if(self.movie.category && self.movie.category.data && self.movie.category.data.id == category_id) + self.category_select.set('value', category_id); + + }); + } + + // Fill profiles + var profiles = Quality.getActiveProfiles(); + if(profiles.length == 1) + self.profile_select.hide(); + + profiles.each(function(profile){ var profile_id = profile.id ? profile.id : profile.data.id; @@ -551,6 +583,7 @@ MA.Edit = new Class({ if(self.movie.profile && self.movie.profile.data && self.movie.profile.data.id == profile_id) self.profile_select.set('value', profile_id); + }); } @@ -566,7 +599,8 @@ MA.Edit = new Class({ 'data': { 'id': self.movie.get('id'), 'default_title': self.title_select.get('value'), - 'profile_id': self.profile_select.get('value') + 'profile_id': self.profile_select.get('value'), + 'category_id': self.category_select.get('value') }, 'useSpinner': true, 'spinnerTarget': $(self.movie), diff --git a/couchpotato/core/plugins/movie/static/movie.js b/couchpotato/core/plugins/movie/static/movie.js index 5ca36c9d..f5b5a2d5 100644 --- a/couchpotato/core/plugins/movie/static/movie.js +++ b/couchpotato/core/plugins/movie/static/movie.js @@ -14,6 +14,7 @@ var Movie = new Class({ self.el = new Element('div.movie'); self.profile = Quality.getProfile(data.profile_id) || {}; + self.category = CategoryList.getCategory(data.category_id) || {}; self.parent(self, options); self.addEvents(); @@ -111,6 +112,7 @@ var Movie = new Class({ self.removeView(); self.profile = Quality.getProfile(self.data.profile_id) || {}; + self.category = CategoryList.getCategory(self.data.category_id) || {}; self.create(); self.busy(false); diff --git a/couchpotato/core/plugins/movie/static/search.css b/couchpotato/core/plugins/movie/static/search.css index e2aa0a47..73d867bb 100644 --- a/couchpotato/core/plugins/movie/static/search.css +++ b/couchpotato/core/plugins/movie/static/search.css @@ -165,7 +165,8 @@ @media all and (max-width: 480px) { .movie_result .options select[name=title] { width: 90px; } - .movie_result .options select[name=profile] { width: 60px; } + .movie_result .options select[name=profile] { width: 50px; } + .movie_result .options select[name=category] { width: 50px; } } @@ -217,26 +218,51 @@ position: absolute; top: 20%; left: 15px; - right: 60px; + right: 7px; vertical-align: middle; } - + .movie_result .info h2 { + margin: 0; font-weight: normal; font-size: 20px; + padding: 0; + } + + .search_form .info h2 { + position: absolute; + width: 100%; + } + + .movie_result .info h2 .title { display: block; margin: 0; text-overflow: ellipsis; overflow: hidden; white-space: nowrap; - width: 100%; - } - - .movie_result .info h2 span { - padding: 0 5px; - position: absolute; - right: -60px; } + + .search_form .info h2 .title { + position: absolute; + width: 88%; + } + + .movie_result .info h2 .year { + padding: 0 5px; + text-align: center; + position: absolute; + width: 12%; + right: 0; + } + + @media all and (max-width: 480px) { + + .search_form .info h2 .year { + font-size: 12px; + margin-top: 7px; + } + + } .search_form .mask, .movie_result .mask { diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 5dcef07b..2b318c10 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -216,9 +216,10 @@ Block.Search.Item = new Class({ } }).adopt( self.info_container = new Element('div.info').adopt( - self.title = new Element('h2', { - 'text': info.titles && info.titles.length > 0 ? info.titles[0] : 'Unknown' - }).adopt( + new Element('h2').adopt( + self.title = new Element('span.title', { + 'text': info.titles && info.titles.length > 0 ? info.titles[0] : 'Unknown' + }), self.year = info.year ? new Element('span.year', { 'text': info.year }) : null @@ -274,7 +275,9 @@ Block.Search.Item = new Class({ add: function(e){ var self = this; - (e).preventDefault(); + + if(e) + (e).preventDefault(); self.loadingMask(); @@ -282,7 +285,8 @@ Block.Search.Item = new Class({ 'data': { 'identifier': self.info.imdb, 'title': self.title_select.get('value'), - 'profile_id': self.profile_select.get('value') + 'profile_id': self.profile_select.get('value'), + 'category_id': self.category_select.get('value') }, 'onComplete': function(json){ self.options_el.empty(); @@ -335,7 +339,12 @@ Block.Search.Item = new Class({ self.profile_select = new Element('select', { 'name': 'profile' }), - new Element('a.button', { + self.category_select = new Element('select', { + 'name': 'category' + }).grab( + new Element('option', {'value': -1, 'text': 'None'}) + ), + self.add_button = new Element('a.button', { 'text': 'Add', 'events': { 'click': self.add.bind(self) @@ -350,7 +359,28 @@ Block.Search.Item = new Class({ }).inject(self.title_select) }) - Quality.getActiveProfiles().each(function(profile){ + + // Fill categories + var categories = CategoryList.getAll(); + + if(categories.length == 0) + self.category_select.hide(); + else { + self.category_select.show(); + categories.each(function(category){ + new Element('option', { + 'value': category.data.id, + 'text': category.data.label + }).inject(self.category_select); + }); + } + + // Fill profiles + var profiles = Quality.getActiveProfiles(); + if(profiles.length == 1) + self.profile_select.hide(); + + profiles.each(function(profile){ new Element('option', { 'value': profile.id ? profile.id : profile.data.id, 'text': profile.label ? profile.label : profile.data.label @@ -358,6 +388,10 @@ Block.Search.Item = new Class({ }); self.options_el.addClass('set'); + + if(categories.length == 0 && self.title_select.getElements('option').length == 1 && profiles.length == 1) + self.add(); + } }, diff --git a/couchpotato/core/plugins/quality/static/quality.js b/couchpotato/core/plugins/quality/static/quality.js index 84e80f83..ead9a904 100644 --- a/couchpotato/core/plugins/quality/static/quality.js +++ b/couchpotato/core/plugins/quality/static/quality.js @@ -103,7 +103,8 @@ var QualityBase = new Class({ var profile_list; var group = self.settings.createGroup({ - 'label': 'Profile Defaults' + 'label': 'Profile Defaults', + 'description': '(Needs refresh \'' +(App.isMac() ? 'CMD+R' : 'F5')+ '\' after editing)' }).adopt( new Element('.ctrlHolder#profile_ordering').adopt( new Element('label[text=Order]'), diff --git a/couchpotato/core/plugins/suggestion/static/suggest.css b/couchpotato/core/plugins/suggestion/static/suggest.css index 9eac0203..2b05abf9 100644 --- a/couchpotato/core/plugins/suggestion/static/suggest.css +++ b/couchpotato/core/plugins/suggestion/static/suggest.css @@ -34,6 +34,7 @@ left: 15px; right: 15px; bottom: 15px; + overflow: hidden; } .suggestions .movie_result .data .info h2 { @@ -83,6 +84,17 @@ .suggestions .movie_result .options { left: 100px; } + .suggestions .movie_result .options select[name=title] { width: 100%; } + .suggestions .movie_result .options select[name=profile] { width: 100%; } + .suggestions .movie_result .options select[name=category] { width: 100%; } + + .suggestions .movie_result .button { + position: absolute; + margin: 2px 0 0 0; + right: 15px; + bottom: 15px; + } + .suggestions .movie_result .thumbnail { width: 100px; From 470fde08902e6c91e2ef2a05286205c2289e53a6 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 20 Jul 2013 13:49:12 +0200 Subject: [PATCH 052/209] Unset the uTorrent read only flags Fix for #1871 Note that this is a fix for Windows only. I am unaware if this issue arises on Linux/Mac and what happens with this fix on those systems. --- couchpotato/core/downloaders/utorrent/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index be6ff107..04595546 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -10,7 +10,9 @@ from multipartpost import MultipartPostHandler import cookielib import httplib import json +import os import re +import stat import time import urllib import urllib2 @@ -52,7 +54,7 @@ class uTorrent(Downloader): new_settings['seed_prio_limitul_flag'] = True log.info('Updated uTorrent settings to set a torrent to complete after it the seeding requirements are met.') - if settings.get('bt.read_only_on_complete'): #This doesnt work as this option seems to be not available through the api + if settings.get('bt.read_only_on_complete'): #This doesn't work as this option seems to be not available through the api. Mitigated with removeReadOnly function new_settings['bt.read_only_on_complete'] = False log.info('Updated uTorrent settings to not set the files to read only after completing.') @@ -130,8 +132,10 @@ class uTorrent(Downloader): status = 'busy' if 'Finished' in item[21]: status = 'completed' + self.removeReadOnly(item[26]) elif 'Seeding' in item[21]: status = 'seeding' + self.removeReadOnly(item[26]) statuses.append({ 'id': item[0], @@ -161,6 +165,13 @@ class uTorrent(Downloader): if not self.connect(): return False return self.utorrent_api.remove_torrent(item['id'], remove_data = delete_files) + + def removeReadOnly(self, folder): + #Removes all read-only flags in a folder + if folder and os.path.isdir(folder): + for root, folders, filenames in os.walk(folder): + for filename in filenames: + os.chmod(os.path.join(root, filename), stat.S_IWRITE) class uTorrentAPI(object): From fd95364d5ffd835c7b4cb4e62da8e2e9d38127ed Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 20 Jul 2013 13:53:19 +0200 Subject: [PATCH 053/209] uTorrent ratio issue fixed The tryFloat function returns 0 if it is fed with a float(!). This resulted in the seed_ratio being set to 0 on first/automatic download. When manually downloading, it did work as the ratio is stored as a string. --- couchpotato/core/downloaders/utorrent/main.py | 2 +- couchpotato/core/helpers/variable.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 04595546..79f9f5b9 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -95,7 +95,7 @@ class uTorrent(Downloader): else: self.utorrent_api.add_torrent_file(torrent_filename, filedata) - # Change settings of added torrents + # Change settings of added torrent self.utorrent_api.set_torrent(torrent_hash, torrent_params) if self.conf('paused', default = 0): self.utorrent_api.pause_torrent(torrent_hash) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index fa8a8b51..48daa289 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -140,7 +140,11 @@ def tryInt(s): except: return 0 def tryFloat(s): - try: return float(s) if '.' in s else tryInt(s) + try: + if isinstance(s, str): + return float(s) if '.' in s else tryInt(s) + else: + return float(s) except: return 0 def natsortKey(s): From 56a788286c83f49581b455e4d7832f9b7b1f7458 Mon Sep 17 00:00:00 2001 From: Micah James Date: Wed, 31 Jul 2013 22:41:49 -0400 Subject: [PATCH 054/209] Adding code for custom urls UI --- .../providers/automation/rottentomatoes/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 83a545b6..579fe1f2 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -18,6 +18,16 @@ config = [{ 'default': False, 'type': 'enabler', }, + { + 'name': 'automation_urls_use', + 'label': 'Use', + }, + { + 'name': 'automation_urls', + 'label': 'url', + 'type': 'combined', + 'combine': ['automation_urls_use', 'automation_urls'], + }, { 'name': 'tomatometer_percent', 'default': '80', From 3a8f891c7d168d570b1c02ac51ebe00ebef34604 Mon Sep 17 00:00:00 2001 From: Micah James Date: Wed, 31 Jul 2013 22:45:48 -0400 Subject: [PATCH 055/209] Adding more code. --- .../core/providers/automation/rottentomatoes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 579fe1f2..19406144 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -8,7 +8,7 @@ config = [{ 'groups': [ { 'tab': 'automation', - 'list': 'automation_providers', + 'list': 'watchlist_providers', 'name': 'rottentomatoes_automation', 'label': 'Rottentomatoes', 'description': 'Imports movies from the rottentomatoes "in theaters"-feed.', From 797018fb8aabbc8caa39a9f91cdeb78a82c77168 Mon Sep 17 00:00:00 2001 From: Micah James Date: Wed, 31 Jul 2013 22:47:52 -0400 Subject: [PATCH 056/209] Revert "Adding more code." This reverts commit 3a8f891c7d168d570b1c02ac51ebe00ebef34604. --- .../core/providers/automation/rottentomatoes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 19406144..579fe1f2 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -8,7 +8,7 @@ config = [{ 'groups': [ { 'tab': 'automation', - 'list': 'watchlist_providers', + 'list': 'automation_providers', 'name': 'rottentomatoes_automation', 'label': 'Rottentomatoes', 'description': 'Imports movies from the rottentomatoes "in theaters"-feed.', From da50b19b6b08d456deef5b977eb2de40c05eafbb Mon Sep 17 00:00:00 2001 From: Micah James Date: Wed, 31 Jul 2013 23:06:12 -0400 Subject: [PATCH 057/209] Added custom url code handling --- .../automation/rottentomatoes/main.py | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/couchpotato/core/providers/automation/rottentomatoes/main.py b/couchpotato/core/providers/automation/rottentomatoes/main.py index 9842d4c9..47b9395d 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/main.py +++ b/couchpotato/core/providers/automation/rottentomatoes/main.py @@ -1,5 +1,5 @@ from couchpotato.core.helpers.rss import RSS -from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.helpers.variable import tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation from xml.etree.ElementTree import QName @@ -11,38 +11,48 @@ log = CPLog(__name__) class Rottentomatoes(Automation, RSS): interval = 1800 - urls = { - 'namespace': 'http://www.rottentomatoes.com/xmlns/rtmovie/', - 'theater': 'http://www.rottentomatoes.com/syndication/rss/in_theaters.xml', - } + + def getIMDBids(self): movies = [] - rss_movies = self.getRSSData(self.urls['theater']) - rating_tag = str(QName(self.urls['namespace'], 'tomatometer_percent')) + rotten_tomatoes_namespace = 'http://www.rottentomatoes.com/xmlns/rtmovie/' + enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] + urls = splitString(self.conf('automation_urls')) - for movie in rss_movies: + index = -1 - value = self.getTextElement(movie, "title") - result = re.search('(?<=%\s).*', value) + for url in urls: - if result: + index += 1 + if not enablers[index]: + continue - log.info2('Something smells...') - rating = tryInt(self.getTextElement(movie, rating_tag)) - name = result.group(0) + rss_movies = self.getRSSData(url) + rating_tag = str(QName(rotten_tomatoes_namespace, 'tomatometer_percent')) - if rating < tryInt(self.conf('tomatometer_percent')): - log.info2('%s seems to be rotten...', name) - else: + for movie in rss_movies: - log.info2('Found %s fresh enough movies, enqueuing: %s', (rating, name)) - year = datetime.datetime.now().strftime("%Y") - imdb = self.search(name, year) + value = self.getTextElement(movie, "title") + result = re.search('(?<=%\s).*', value) - if imdb and self.isMinimalMovie(imdb): - movies.append(imdb['imdb']) + if result: + + log.info2('Something smells...') + rating = tryInt(self.getTextElement(movie, rating_tag)) + name = result.group(0) + + if rating < tryInt(self.conf('tomatometer_percent')): + log.info2('%s seems to be rotten...', name) + else: + + log.info2('Found %s fresh enough movies, enqueuing: %s', (rating, name)) + year = datetime.datetime.now().strftime("%Y") + imdb = self.search(name, year) + + if imdb and self.isMinimalMovie(imdb): + movies.append(imdb['imdb']) return movies From 4330dc39bf37d6e1332d2a2471454c6667b86aa7 Mon Sep 17 00:00:00 2001 From: Micah James Date: Wed, 31 Jul 2013 23:14:58 -0400 Subject: [PATCH 058/209] Changed description to be better suited for this. --- .../core/providers/automation/rottentomatoes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 579fe1f2..52b1c882 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -11,7 +11,7 @@ config = [{ 'list': 'automation_providers', 'name': 'rottentomatoes_automation', 'label': 'Rottentomatoes', - 'description': 'Imports movies from the rottentomatoes "in theaters"-feed.', + 'description': 'Imports movies from rottentomatoes rss feeds specified below.', 'options': [ { 'name': 'automation_enabled', From b32d4fc42da353e1763c277a28de7fabebbe4d95 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Thu, 1 Aug 2013 23:24:25 +0200 Subject: [PATCH 059/209] Fix NZBGet url issue --- couchpotato/core/downloaders/nzbget/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index c90cd2db..ef9d4efa 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -33,7 +33,7 @@ class NZBGet(Downloader): rpc = xmlrpclib.ServerProxy(url) try: if rpc.writelog('INFO', 'CouchPotato connected to drop off %s.' % nzb_name): - log.info('Successfully connected to NZBGet') + log.debug('Successfully connected to NZBGet') else: log.info('Successfully connected to NZBGet, but unable to send a message') except socket.error: @@ -74,7 +74,7 @@ class NZBGet(Downloader): rpc = xmlrpclib.ServerProxy(url) try: if rpc.writelog('INFO', 'CouchPotato connected to check status'): - log.info('Successfully connected to NZBGet') + log.debug('Successfully connected to NZBGet') else: log.info('Successfully connected to NZBGet, but unable to send a message') except socket.error: @@ -152,12 +152,12 @@ class NZBGet(Downloader): log.info('%s failed downloading, deleting...', item['name']) - url = self.url % {'host': self.conf('host'), 'password': self.conf('password')} + url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} rpc = xmlrpclib.ServerProxy(url) try: if rpc.writelog('INFO', 'CouchPotato connected to delete some history'): - log.info('Successfully connected to NZBGet') + log.debug('Successfully connected to NZBGet') else: log.info('Successfully connected to NZBGet, but unable to send a message') except socket.error: From 4ffda9f705c32e903ea8cda2323b4272a290a653 Mon Sep 17 00:00:00 2001 From: Micah James Date: Thu, 1 Aug 2013 23:15:36 -0400 Subject: [PATCH 060/209] Made code more python-y per mano3ms recommendation. --- .../core/providers/automation/rottentomatoes/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/providers/automation/rottentomatoes/main.py b/couchpotato/core/providers/automation/rottentomatoes/main.py index 47b9395d..40f72a5c 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/main.py +++ b/couchpotato/core/providers/automation/rottentomatoes/main.py @@ -19,15 +19,11 @@ class Rottentomatoes(Automation, RSS): movies = [] rotten_tomatoes_namespace = 'http://www.rottentomatoes.com/xmlns/rtmovie/' - enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] - urls = splitString(self.conf('automation_urls')) - - index = -1 + urls = dict(zip(splitString(self.conf('automation_urls')), [tryInt(x) for x in splitString(self.conf('automation_urls_use'))])) for url in urls: - index += 1 - if not enablers[index]: + if not urls[url]: continue rss_movies = self.getRSSData(url) From 0492e90d6fdc76b97f505dcb851740d75187901d Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Tue, 16 Jul 2013 22:35:32 +0200 Subject: [PATCH 061/209] XBMC: properly check if host is local And added option to scan if remote --- couchpotato/core/notifications/xbmc/__init__.py | 8 ++++++++ couchpotato/core/notifications/xbmc/main.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/notifications/xbmc/__init__.py b/couchpotato/core/notifications/xbmc/__init__.py index e3c467ce..f0167ce8 100644 --- a/couchpotato/core/notifications/xbmc/__init__.py +++ b/couchpotato/core/notifications/xbmc/__init__.py @@ -38,6 +38,14 @@ config = [{ 'advanced': True, 'description': 'Only update the first host when movie snatched, useful for synced XBMC', }, + { + 'name': 'remote_dir_scan', + 'label': 'Remote Folder Scan', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Scan new movie folder at remote XBMC servers, only works if movie location is the same.', + }, { 'name': 'on_snatch', 'default': 0, diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index 7266b25c..34a9c1da 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -13,7 +13,7 @@ log = CPLog(__name__) class XBMC(Notification): - listen_to = ['renamer.after'] + listen_to = ['renamer.after', 'movie.snatched'] use_json_notifications = {} http_time_between_calls = 0 @@ -33,15 +33,19 @@ class XBMC(Notification): ('GUI.ShowNotification', {'title': self.default_title, 'message': message, 'image': self.getNotificationImage('small')}), ] - if not self.conf('only_first') or hosts.index(host) == 0: - calls.append(('VideoLibrary.Scan', {'directory': data.get('destination_dir', None)} if 'localhost' in host else {})) + if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0): + param = {} + if self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0]): + param = {'directory': data['destination_dir']} + + calls.append(('VideoLibrary.Scan', param)) max_successful += len(calls) response = self.request(host, calls) else: response = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message}) - if not self.conf('only_first') or hosts.index(host) == 0: + if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0): response += self.request(host, [('VideoLibrary.Scan', {})]) max_successful += 1 From c99a5cb5356b6b62b297d58d8f53afc2ec8be316 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 7 Aug 2013 20:06:30 +0200 Subject: [PATCH 062/209] Don't autoadd when already in wanted --- couchpotato/core/plugins/movie/static/search.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 2b318c10..23b9a3f4 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -389,7 +389,9 @@ Block.Search.Item = new Class({ self.options_el.addClass('set'); - if(categories.length == 0 && self.title_select.getElements('option').length == 1 && profiles.length == 1) + p(self.info.in_wanted, self.info.in_wanted.profile, in_library); + if(categories.length == 0 && self.title_select.getElements('option').length == 1 && profiles.length == 1 && + !(self.info.in_wanted && self.info.in_wanted.profile || in_library)) self.add(); } From 448c1d69a71c1bf7eca45df1e7444bac1e7aca3b Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sun, 11 Aug 2013 00:04:40 +0200 Subject: [PATCH 063/209] Regard torrents and torrent_magnet the same When sorting the torrents and torrent_magnets were sorted, by taking only the three first characters (as 'nzb; is three chars), the score prevails. Fixes #2004 --- couchpotato/core/plugins/searcher/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index dfb6ccba..b55e7201 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -195,7 +195,7 @@ class Searcher(Plugin): download_preference = self.conf('preferred_method') if download_preference != 'both': - sorted_results = sorted(sorted_results, key = lambda k: k['type'], reverse = (download_preference == 'torrent')) + sorted_results = sorted(sorted_results, key = lambda k: k['type'][:3], reverse = (download_preference == 'torrent')) # Check if movie isn't deleted while searching if not db.query(Movie).filter_by(id = movie.get('id')).first(): From 3bd18753211956e9e05b345deefdb7db330638ae Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sat, 27 Jul 2013 21:01:23 +1200 Subject: [PATCH 064/209] Added initial rtorrent downloader, currently testing, possibly has some bugs. --- .../core/downloaders/rtorrent/__init__.py | 54 ++ couchpotato/core/downloaders/rtorrent/main.py | 97 +++ libs/rtorrent/__init__.py | 567 ++++++++++++++++++ libs/rtorrent/common.py | 86 +++ libs/rtorrent/compat.py | 30 + libs/rtorrent/err.py | 40 ++ libs/rtorrent/file.py | 91 +++ libs/rtorrent/lib/__init__.py | 0 libs/rtorrent/lib/bencode.py | 281 +++++++++ libs/rtorrent/lib/torrentparser.py | 159 +++++ libs/rtorrent/lib/xmlrpc/__init__.py | 0 libs/rtorrent/lib/xmlrpc/http.py | 23 + libs/rtorrent/peer.py | 98 +++ libs/rtorrent/rpc/__init__.py | 354 +++++++++++ libs/rtorrent/torrent.py | 484 +++++++++++++++ libs/rtorrent/tracker.py | 138 +++++ 16 files changed, 2502 insertions(+) create mode 100755 couchpotato/core/downloaders/rtorrent/__init__.py create mode 100755 couchpotato/core/downloaders/rtorrent/main.py create mode 100755 libs/rtorrent/__init__.py create mode 100755 libs/rtorrent/common.py create mode 100755 libs/rtorrent/compat.py create mode 100755 libs/rtorrent/err.py create mode 100755 libs/rtorrent/file.py create mode 100755 libs/rtorrent/lib/__init__.py create mode 100755 libs/rtorrent/lib/bencode.py create mode 100755 libs/rtorrent/lib/torrentparser.py create mode 100755 libs/rtorrent/lib/xmlrpc/__init__.py create mode 100755 libs/rtorrent/lib/xmlrpc/http.py create mode 100755 libs/rtorrent/peer.py create mode 100755 libs/rtorrent/rpc/__init__.py create mode 100755 libs/rtorrent/torrent.py create mode 100755 libs/rtorrent/tracker.py diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py new file mode 100755 index 00000000..d0047893 --- /dev/null +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -0,0 +1,54 @@ +from .main import rTorrent + +def start(): + return rTorrent() + +config = [{ + 'name': 'rtorrent', + 'groups': [ + { + 'tab': 'downloaders', + 'list': 'download_providers', + 'name': 'rtorrent', + 'label': 'rTorrent', + 'description': '', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + 'radio_group': 'torrent', + }, + { + 'name': 'url', + 'default': 'http://localhost:80/RPC2', + }, + { + 'name': 'username', + }, + { + 'name': 'password', + 'type': 'password', + }, + { + 'name': 'label', + 'description': 'Label to add torrent as.', + }, + { + 'name': 'paused', + 'type': 'bool', + 'default': False, + 'description': 'Add the torrent paused.', + }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py new file mode 100755 index 00000000..5da64cb7 --- /dev/null +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -0,0 +1,97 @@ +from base64 import b16encode, b32decode +from bencode import bencode, bdecode +from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.helpers.encoding import isInt, ss +from couchpotato.core.logger import CPLog +from datetime import timedelta +from hashlib import sha1 +from multipartpost import MultipartPostHandler +import cookielib +import httplib +import json +import re +import time +import urllib +import urllib2 +from rtorrent import RTorrent + + +log = CPLog(__name__) + + +class rTorrent(Downloader): + + type = ['torrent', 'torrent_magnet'] + rtorrent_api = None + + def get_conn(self): + return RTorrent( + self.conf('url'), + self.conf('username'), + self.conf('password') + ) + + def download(self, data, movie, filedata=None): + log.debug('Sending "%s" (%s) to rTorrent.', (data.get('name'), data.get('type'))) + + torrent_params = {} + if self.conf('label'): + torrent_params['label'] = self.conf('label') + + if not filedata and data.get('type') == 'torrent': + log.error('Failed sending torrent, no data') + return False + + if data.get('type') == 'torrent_magnet': + log.info('magnet torrents are not supported') + return False + + info = bdecode(filedata)["info"] + torrent_hash = sha1(bencode(info)).hexdigest().upper() + torrent_filename = self.createFileName(data, filedata, movie) + + # Convert base 32 to hex + if len(torrent_hash) == 32: + torrent_hash = b16encode(b32decode(torrent_hash)) + + # Send request to rTorrent + try: + if not self.rtorrent_api: + self.rtorrent_api = self.get_conn() + + torrent = self.rtorrent_api.load_torrent(filedata, not self.conf('paused', default=0)) + + return self.downloadReturnId(torrent_hash) + except Exception, err: + log.error('Failed to send torrent to rTorrent: %s', err) + return False + + + def getAllDownloadStatus(self): + + log.debug('Checking rTorrent download status.') + + try: + if not self.rtorrent_api: + self.rtorrent_api = self.get_conn() + + torrents = self.rtorrent_api.get_torrents() + + statuses = StatusList(self) + + for item in torrents: + statuses.append({ + 'id': item.info_hash, + 'name': item.name, + 'status': 'completed' if item.complete else 'busy', + 'original_status': item.state, + 'timeleft': str(timedelta(seconds=float(item.left_bytes) / item.down_rate)) + if item.down_rate > 0 else -1, + 'folder': '' + }) + + return statuses + + except Exception, err: + log.error('Failed to send torrent to rTorrent: %s', err) + return False diff --git a/libs/rtorrent/__init__.py b/libs/rtorrent/__init__.py new file mode 100755 index 00000000..e427b65e --- /dev/null +++ b/libs/rtorrent/__init__.py @@ -0,0 +1,567 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from rtorrent.common import find_torrent, \ + is_valid_port, convert_version_tuple_to_str +from rtorrent.lib.torrentparser import TorrentParser +from rtorrent.lib.xmlrpc.http import HTTPServerProxy +from rtorrent.rpc import Method, BasicAuthTransport +from rtorrent.torrent import Torrent +import os.path +import rtorrent.rpc # @UnresolvedImport +import time +import xmlrpclib + +__version__ = "0.2.9" +__author__ = "Chris Lucas" +__contact__ = "chris@chrisjlucas.com" +__license__ = "MIT" + +MIN_RTORRENT_VERSION = (0, 8, 1) +MIN_RTORRENT_VERSION_STR = convert_version_tuple_to_str(MIN_RTORRENT_VERSION) + + +class RTorrent: + """ Create a new rTorrent connection """ + rpc_prefix = None + + def __init__(self, url, username=None, password=None, + verify=False, sp=HTTPServerProxy, sp_kwargs={}): + self.url = url # : From X{__init__(self, url)} + self.username = username + self.password = password + self.sp = sp + self.sp_kwargs = sp_kwargs + + self.torrents = [] # : List of L{Torrent} instances + self._rpc_methods = [] # : List of rTorrent RPC methods + self._torrent_cache = [] + self._client_version_tuple = () + + if verify is True: + self._verify_conn() + + def _get_conn(self): + """Get ServerProxy instance""" + if self.username is not None and self.password is not None: + return self.sp( + self.url, + transport=BasicAuthTransport(self.username, self.password), + **self.sp_kwargs + ) + return self.sp(self.url, **self.sp_kwargs) + + def _verify_conn(self): + # check for rpc methods that should be available + assert {"system.client_version", + "system.library_version"}.issubset(set(self._get_rpc_methods())),\ + "Required RPC methods not available." + + # minimum rTorrent version check + + assert self._meets_version_requirement() is True,\ + "Error: Minimum rTorrent version required is {0}".format( + MIN_RTORRENT_VERSION_STR) + + def _meets_version_requirement(self): + return self._get_client_version_tuple() >= MIN_RTORRENT_VERSION + + def _get_client_version_tuple(self): + conn = self._get_conn() + + if not self._client_version_tuple: + if not hasattr(self, "client_version"): + setattr(self, "client_version", + conn.system.client_version()) + + rtver = getattr(self, "client_version") + self._client_version_tuple = tuple([int(i) for i in + rtver.split(".")]) + + return self._client_version_tuple + + def _get_rpc_methods(self): + """ Get list of raw RPC commands + + @return: raw RPC commands + @rtype: list + """ + + if self._rpc_methods == []: + self._rpc_methods = self._get_conn().system.listMethods() + + return(self._rpc_methods) + + def get_torrents(self, view="main"): + """Get list of all torrents in specified view + + @return: list of L{Torrent} instances + + @rtype: list + + @todo: add validity check for specified view + """ + self.torrents = [] + methods = rtorrent.torrent.methods + retriever_methods = [m for m in methods + if m.is_retriever() and m.is_available(self)] + + m = rtorrent.rpc.Multicall(self) + m.add("d.multicall", view, "d.get_hash=", + *[method.rpc_call + "=" for method in retriever_methods]) + + results = m.call()[0] # only sent one call, only need first result + + for result in results: + results_dict = {} + # build results_dict + for m, r in zip(retriever_methods, result[1:]): # result[0] is the info_hash + results_dict[m.varname] = rtorrent.rpc.process_result(m, r) + + self.torrents.append( + Torrent(self, info_hash=result[0], **results_dict) + ) + + self._manage_torrent_cache() + return(self.torrents) + + def _manage_torrent_cache(self): + """Carry tracker/peer/file lists over to new torrent list""" + for torrent in self._torrent_cache: + new_torrent = rtorrent.common.find_torrent(torrent.info_hash, + self.torrents) + if new_torrent is not None: + new_torrent.files = torrent.files + new_torrent.peers = torrent.peers + new_torrent.trackers = torrent.trackers + + self._torrent_cache = self.torrents + + def _get_load_function(self, file_type, start, verbose): + """Determine correct "load torrent" RPC method""" + func_name = None + if file_type == "url": + # url strings can be input directly + if start and verbose: + func_name = "load_start_verbose" + elif start: + func_name = "load_start" + elif verbose: + func_name = "load_verbose" + else: + func_name = "load" + elif file_type in ["file", "raw"]: + if start and verbose: + func_name = "load_raw_start_verbose" + elif start: + func_name = "load_raw_start" + elif verbose: + func_name = "load_raw_verbose" + else: + func_name = "load_raw" + + return(func_name) + + def load_torrent(self, torrent, start=False, verbose=False, verify_load=True): + """ + Loads torrent into rTorrent (with various enhancements) + + @param torrent: can be a url, a path to a local file, or the raw data + of a torrent file + @type torrent: str + + @param start: start torrent when loaded + @type start: bool + + @param verbose: print error messages to rTorrent log + @type verbose: bool + + @param verify_load: verify that torrent was added to rTorrent successfully + @type verify_load: bool + + @return: Depends on verify_load: + - if verify_load is True, (and the torrent was + loaded successfully), it'll return a L{Torrent} instance + - if verify_load is False, it'll return None + + @rtype: L{Torrent} instance or None + + @raise AssertionError: If the torrent wasn't successfully added to rTorrent + - Check L{TorrentParser} for the AssertionError's + it raises + + + @note: Because this function includes url verification (if a url was input) + as well as verification as to whether the torrent was successfully added, + this function doesn't execute instantaneously. If that's what you're + looking for, use load_torrent_simple() instead. + """ + p = self._get_conn() + tp = TorrentParser(torrent) + torrent = xmlrpclib.Binary(tp._raw_torrent) + info_hash = tp.info_hash + + func_name = self._get_load_function("raw", start, verbose) + + # load torrent + getattr(p, func_name)(torrent) + + if verify_load: + MAX_RETRIES = 3 + i = 0 + while i < MAX_RETRIES: + self.get_torrents() + if info_hash in [t.info_hash for t in self.torrents]: + break + + # was still getting AssertionErrors, delay should help + time.sleep(1) + i += 1 + + assert info_hash in [t.info_hash for t in self.torrents],\ + "Adding torrent was unsuccessful." + + return(find_torrent(info_hash, self.torrents)) + + def load_torrent_simple(self, torrent, file_type, + start=False, verbose=False): + """Loads torrent into rTorrent + + @param torrent: can be a url, a path to a local file, or the raw data + of a torrent file + @type torrent: str + + @param file_type: valid options: "url", "file", or "raw" + @type file_type: str + + @param start: start torrent when loaded + @type start: bool + + @param verbose: print error messages to rTorrent log + @type verbose: bool + + @return: None + + @raise AssertionError: if incorrect file_type is specified + + @note: This function was written for speed, it includes no enhancements. + If you input a url, it won't check if it's valid. You also can't get + verification that the torrent was successfully added to rTorrent. + Use load_torrent() if you would like these features. + """ + p = self._get_conn() + + assert file_type in ["raw", "file", "url"], \ + "Invalid file_type, options are: 'url', 'file', 'raw'." + func_name = self._get_load_function(file_type, start, verbose) + + if file_type == "file": + # since we have to assume we're connected to a remote rTorrent + # client, we have to read the file and send it to rT as raw + assert os.path.isfile(torrent), \ + "Invalid path: \"{0}\"".format(torrent) + torrent = open(torrent, "rb").read() + + if file_type in ["raw", "file"]: + finput = xmlrpclib.Binary(torrent) + elif file_type == "url": + finput = torrent + + getattr(p, func_name)(finput) + + def set_dht_port(self, port): + """Set DHT port + + @param port: port + @type port: int + + @raise AssertionError: if invalid port is given + """ + assert is_valid_port(port), "Valid port range is 0-65535" + self.dht_port = self._p.set_dht_port(port) + + def enable_check_hash(self): + """Alias for set_check_hash(True)""" + self.set_check_hash(True) + + def disable_check_hash(self): + """Alias for set_check_hash(False)""" + self.set_check_hash(False) + + def find_torrent(self, info_hash): + """Frontend for rtorrent.common.find_torrent""" + return(rtorrent.common.find_torrent(info_hash, self.get_torrents())) + + def poll(self): + """ poll rTorrent to get latest torrent/peer/tracker/file information + + @note: This essentially refreshes every aspect of the rTorrent + connection, so it can be very slow if working with a remote + connection that has a lot of torrents loaded. + + @return: None + """ + self.update() + torrents = self.get_torrents() + for t in torrents: + t.poll() + + def update(self): + """Refresh rTorrent client info + + @note: All fields are stored as attributes to self. + + @return: None + """ + multicall = rtorrent.rpc.Multicall(self) + retriever_methods = [m for m in methods + if m.is_retriever() and m.is_available(self)] + for method in retriever_methods: + multicall.add(method) + + multicall.call() + + +def _build_class_methods(class_obj): + # multicall add class + caller = lambda self, multicall, method, *args:\ + multicall.add(method, self.rpc_id, *args) + + caller.__doc__ = """Same as Multicall.add(), but with automatic inclusion + of the rpc_id + + @param multicall: A L{Multicall} instance + @type: multicall: Multicall + + @param method: L{Method} instance or raw rpc method + @type: Method or str + + @param args: optional arguments to pass + """ + setattr(class_obj, "multicall_add", caller) + + +def __compare_rpc_methods(rt_new, rt_old): + from pprint import pprint + rt_new_methods = set(rt_new._get_rpc_methods()) + rt_old_methods = set(rt_old._get_rpc_methods()) + print("New Methods:") + pprint(rt_new_methods - rt_old_methods) + print("Methods not in new rTorrent:") + pprint(rt_old_methods - rt_new_methods) + + +def __check_supported_methods(rt): + from pprint import pprint + supported_methods = set([m.rpc_call for m in + methods + + rtorrent.file.methods + + rtorrent.torrent.methods + + rtorrent.tracker.methods + + rtorrent.peer.methods]) + all_methods = set(rt._get_rpc_methods()) + + print("Methods NOT in supported methods") + pprint(all_methods - supported_methods) + print("Supported methods NOT in all methods") + pprint(supported_methods - all_methods) + +methods = [ + # RETRIEVERS + Method(RTorrent, 'get_xmlrpc_size_limit', 'get_xmlrpc_size_limit'), + Method(RTorrent, 'get_proxy_address', 'get_proxy_address'), + Method(RTorrent, 'get_split_suffix', 'get_split_suffix'), + Method(RTorrent, 'get_up_limit', 'get_upload_rate'), + Method(RTorrent, 'get_max_memory_usage', 'get_max_memory_usage'), + Method(RTorrent, 'get_max_open_files', 'get_max_open_files'), + Method(RTorrent, 'get_min_peers_seed', 'get_min_peers_seed'), + Method(RTorrent, 'get_use_udp_trackers', 'get_use_udp_trackers'), + Method(RTorrent, 'get_preload_min_size', 'get_preload_min_size'), + Method(RTorrent, 'get_max_uploads', 'get_max_uploads'), + Method(RTorrent, 'get_max_peers', 'get_max_peers'), + Method(RTorrent, 'get_timeout_sync', 'get_timeout_sync'), + Method(RTorrent, 'get_receive_buffer_size', 'get_receive_buffer_size'), + Method(RTorrent, 'get_split_file_size', 'get_split_file_size'), + Method(RTorrent, 'get_dht_throttle', 'get_dht_throttle'), + Method(RTorrent, 'get_max_peers_seed', 'get_max_peers_seed'), + Method(RTorrent, 'get_min_peers', 'get_min_peers'), + Method(RTorrent, 'get_tracker_numwant', 'get_tracker_numwant'), + Method(RTorrent, 'get_max_open_sockets', 'get_max_open_sockets'), + Method(RTorrent, 'get_session', 'get_session'), + Method(RTorrent, 'get_ip', 'get_ip'), + Method(RTorrent, 'get_scgi_dont_route', 'get_scgi_dont_route'), + Method(RTorrent, 'get_hash_read_ahead', 'get_hash_read_ahead'), + Method(RTorrent, 'get_http_cacert', 'get_http_cacert'), + Method(RTorrent, 'get_dht_port', 'get_dht_port'), + Method(RTorrent, 'get_handshake_log', 'get_handshake_log'), + Method(RTorrent, 'get_preload_type', 'get_preload_type'), + Method(RTorrent, 'get_max_open_http', 'get_max_open_http'), + Method(RTorrent, 'get_http_capath', 'get_http_capath'), + Method(RTorrent, 'get_max_downloads_global', 'get_max_downloads_global'), + Method(RTorrent, 'get_name', 'get_name'), + Method(RTorrent, 'get_session_on_completion', 'get_session_on_completion'), + Method(RTorrent, 'get_down_limit', 'get_download_rate'), + Method(RTorrent, 'get_down_total', 'get_down_total'), + Method(RTorrent, 'get_up_rate', 'get_up_rate'), + Method(RTorrent, 'get_hash_max_tries', 'get_hash_max_tries'), + Method(RTorrent, 'get_peer_exchange', 'get_peer_exchange'), + Method(RTorrent, 'get_down_rate', 'get_down_rate'), + Method(RTorrent, 'get_connection_seed', 'get_connection_seed'), + Method(RTorrent, 'get_http_proxy', 'get_http_proxy'), + Method(RTorrent, 'get_stats_preloaded', 'get_stats_preloaded'), + Method(RTorrent, 'get_timeout_safe_sync', 'get_timeout_safe_sync'), + Method(RTorrent, 'get_hash_interval', 'get_hash_interval'), + Method(RTorrent, 'get_port_random', 'get_port_random'), + Method(RTorrent, 'get_directory', 'get_directory'), + Method(RTorrent, 'get_port_open', 'get_port_open'), + Method(RTorrent, 'get_max_file_size', 'get_max_file_size'), + Method(RTorrent, 'get_stats_not_preloaded', 'get_stats_not_preloaded'), + Method(RTorrent, 'get_memory_usage', 'get_memory_usage'), + Method(RTorrent, 'get_connection_leech', 'get_connection_leech'), + Method(RTorrent, 'get_check_hash', 'get_check_hash', + boolean=True, + ), + Method(RTorrent, 'get_session_lock', 'get_session_lock'), + Method(RTorrent, 'get_preload_required_rate', 'get_preload_required_rate'), + Method(RTorrent, 'get_max_uploads_global', 'get_max_uploads_global'), + Method(RTorrent, 'get_send_buffer_size', 'get_send_buffer_size'), + Method(RTorrent, 'get_port_range', 'get_port_range'), + Method(RTorrent, 'get_max_downloads_div', 'get_max_downloads_div'), + Method(RTorrent, 'get_max_uploads_div', 'get_max_uploads_div'), + Method(RTorrent, 'get_safe_sync', 'get_safe_sync'), + Method(RTorrent, 'get_bind', 'get_bind'), + Method(RTorrent, 'get_up_total', 'get_up_total'), + Method(RTorrent, 'get_client_version', 'system.client_version'), + Method(RTorrent, 'get_library_version', 'system.library_version'), + Method(RTorrent, 'get_api_version', 'system.api_version', + min_version=(0, 9, 1) + ), + Method(RTorrent, "get_system_time", "system.time", + docstring="""Get the current time of the system rTorrent is running on + + @return: time (posix) + @rtype: int""", + ), + + # MODIFIERS + Method(RTorrent, 'set_http_proxy', 'set_http_proxy'), + Method(RTorrent, 'set_max_memory_usage', 'set_max_memory_usage'), + Method(RTorrent, 'set_max_file_size', 'set_max_file_size'), + Method(RTorrent, 'set_bind', 'set_bind', + docstring="""Set address bind + + @param arg: ip address + @type arg: str + """, + ), + Method(RTorrent, 'set_up_limit', 'set_upload_rate', + docstring="""Set global upload limit (in bytes) + + @param arg: speed limit + @type arg: int + """, + ), + Method(RTorrent, 'set_port_random', 'set_port_random'), + Method(RTorrent, 'set_connection_leech', 'set_connection_leech'), + Method(RTorrent, 'set_tracker_numwant', 'set_tracker_numwant'), + Method(RTorrent, 'set_max_peers', 'set_max_peers'), + Method(RTorrent, 'set_min_peers', 'set_min_peers'), + Method(RTorrent, 'set_max_uploads_div', 'set_max_uploads_div'), + Method(RTorrent, 'set_max_open_files', 'set_max_open_files'), + Method(RTorrent, 'set_max_downloads_global', 'set_max_downloads_global'), + Method(RTorrent, 'set_session_lock', 'set_session_lock'), + Method(RTorrent, 'set_session', 'set_session'), + Method(RTorrent, 'set_split_suffix', 'set_split_suffix'), + Method(RTorrent, 'set_hash_interval', 'set_hash_interval'), + Method(RTorrent, 'set_handshake_log', 'set_handshake_log'), + Method(RTorrent, 'set_port_range', 'set_port_range'), + Method(RTorrent, 'set_min_peers_seed', 'set_min_peers_seed'), + Method(RTorrent, 'set_scgi_dont_route', 'set_scgi_dont_route'), + Method(RTorrent, 'set_preload_min_size', 'set_preload_min_size'), + Method(RTorrent, 'set_log.tracker', 'set_log.tracker'), + Method(RTorrent, 'set_max_uploads_global', 'set_max_uploads_global'), + Method(RTorrent, 'set_down_limit', 'set_download_rate', + docstring="""Set global download limit (in bytes) + + @param arg: speed limit + @type arg: int + """, + ), + Method(RTorrent, 'set_preload_required_rate', 'set_preload_required_rate'), + Method(RTorrent, 'set_hash_read_ahead', 'set_hash_read_ahead'), + Method(RTorrent, 'set_max_peers_seed', 'set_max_peers_seed'), + Method(RTorrent, 'set_max_uploads', 'set_max_uploads'), + Method(RTorrent, 'set_session_on_completion', 'set_session_on_completion'), + Method(RTorrent, 'set_max_open_http', 'set_max_open_http'), + Method(RTorrent, 'set_directory', 'set_directory'), + Method(RTorrent, 'set_http_cacert', 'set_http_cacert'), + Method(RTorrent, 'set_dht_throttle', 'set_dht_throttle'), + Method(RTorrent, 'set_hash_max_tries', 'set_hash_max_tries'), + Method(RTorrent, 'set_proxy_address', 'set_proxy_address'), + Method(RTorrent, 'set_split_file_size', 'set_split_file_size'), + Method(RTorrent, 'set_receive_buffer_size', 'set_receive_buffer_size'), + Method(RTorrent, 'set_use_udp_trackers', 'set_use_udp_trackers'), + Method(RTorrent, 'set_connection_seed', 'set_connection_seed'), + Method(RTorrent, 'set_xmlrpc_size_limit', 'set_xmlrpc_size_limit'), + Method(RTorrent, 'set_xmlrpc_dialect', 'set_xmlrpc_dialect'), + Method(RTorrent, 'set_safe_sync', 'set_safe_sync'), + Method(RTorrent, 'set_http_capath', 'set_http_capath'), + Method(RTorrent, 'set_send_buffer_size', 'set_send_buffer_size'), + Method(RTorrent, 'set_max_downloads_div', 'set_max_downloads_div'), + Method(RTorrent, 'set_name', 'set_name'), + Method(RTorrent, 'set_port_open', 'set_port_open'), + Method(RTorrent, 'set_timeout_sync', 'set_timeout_sync'), + Method(RTorrent, 'set_peer_exchange', 'set_peer_exchange'), + Method(RTorrent, 'set_ip', 'set_ip', + docstring="""Set IP + + @param arg: ip address + @type arg: str + """, + ), + Method(RTorrent, 'set_timeout_safe_sync', 'set_timeout_safe_sync'), + Method(RTorrent, 'set_preload_type', 'set_preload_type'), + Method(RTorrent, 'set_check_hash', 'set_check_hash', + docstring="""Enable/Disable hash checking on finished torrents + + @param arg: True to enable, False to disable + @type arg: bool + """, + boolean=True, + ), +] + +_all_methods_list = [methods, + rtorrent.file.methods, + rtorrent.torrent.methods, + rtorrent.tracker.methods, + rtorrent.peer.methods, + ] + +class_methods_pair = { + RTorrent: methods, + rtorrent.file.File: rtorrent.file.methods, + rtorrent.torrent.Torrent: rtorrent.torrent.methods, + rtorrent.tracker.Tracker: rtorrent.tracker.methods, + rtorrent.peer.Peer: rtorrent.peer.methods, +} +for c in class_methods_pair.keys(): + rtorrent.rpc._build_rpc_methods(c, class_methods_pair[c]) + _build_class_methods(c) diff --git a/libs/rtorrent/common.py b/libs/rtorrent/common.py new file mode 100755 index 00000000..371c71c3 --- /dev/null +++ b/libs/rtorrent/common.py @@ -0,0 +1,86 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +from rtorrent.compat import is_py3 + + +def bool_to_int(value): + """Translates python booleans to RPC-safe integers""" + if value is True: + return("1") + elif value is False: + return("0") + else: + return(value) + + +def cmd_exists(cmds_list, cmd): + """Check if given command is in list of available commands + + @param cmds_list: see L{RTorrent._rpc_methods} + @type cmds_list: list + + @param cmd: name of command to be checked + @type cmd: str + + @return: bool + """ + + return(cmd in cmds_list) + + +def find_torrent(info_hash, torrent_list): + """Find torrent file in given list of Torrent classes + + @param info_hash: info hash of torrent + @type info_hash: str + + @param torrent_list: list of L{Torrent} instances (see L{RTorrent.get_torrents}) + @type torrent_list: list + + @return: L{Torrent} instance, or -1 if not found + """ + for t in torrent_list: + if t.info_hash == info_hash: + return t + + return None + + +def is_valid_port(port): + """Check if given port is valid""" + return(0 <= int(port) <= 65535) + + +def convert_version_tuple_to_str(t): + return(".".join([str(n) for n in t])) + + +def safe_repr(fmt, *args, **kwargs): + """ Formatter that handles unicode arguments """ + + if not is_py3(): + # unicode fmt can take str args, str fmt cannot take unicode args + fmt = fmt.decode("utf-8") + out = fmt.format(*args, **kwargs) + return out.encode("utf-8") + else: + return fmt.format(*args, **kwargs) diff --git a/libs/rtorrent/compat.py b/libs/rtorrent/compat.py new file mode 100755 index 00000000..1778818b --- /dev/null +++ b/libs/rtorrent/compat.py @@ -0,0 +1,30 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import sys + + +def is_py3(): + return sys.version_info[0] == 3 + +if is_py3(): + import xmlrpc.client as xmlrpclib +else: + import xmlrpclib diff --git a/libs/rtorrent/err.py b/libs/rtorrent/err.py new file mode 100755 index 00000000..920b8385 --- /dev/null +++ b/libs/rtorrent/err.py @@ -0,0 +1,40 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from rtorrent.common import convert_version_tuple_to_str + + +class RTorrentVersionError(Exception): + def __init__(self, min_version, cur_version): + self.min_version = min_version + self.cur_version = cur_version + self.msg = "Minimum version required: {0}".format( + convert_version_tuple_to_str(min_version)) + + def __str__(self): + return(self.msg) + + +class MethodError(Exception): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return(self.msg) diff --git a/libs/rtorrent/file.py b/libs/rtorrent/file.py new file mode 100755 index 00000000..a3db35cf --- /dev/null +++ b/libs/rtorrent/file.py @@ -0,0 +1,91 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# from rtorrent.rpc import Method +import rtorrent.rpc + +from rtorrent.common import safe_repr + +Method = rtorrent.rpc.Method + + +class File: + """Represents an individual file within a L{Torrent} instance.""" + + def __init__(self, _rt_obj, info_hash, index, **kwargs): + self._rt_obj = _rt_obj + self.info_hash = info_hash # : info hash for the torrent the file is associated with + self.index = index # : The position of the file within the file list + for k in kwargs.keys(): + setattr(self, k, kwargs.get(k, None)) + + self.rpc_id = "{0}:f{1}".format( + self.info_hash, self.index) # : unique id to pass to rTorrent + + def update(self): + """Refresh file data + + @note: All fields are stored as attributes to self. + + @return: None + """ + multicall = rtorrent.rpc.Multicall(self) + retriever_methods = [m for m in methods + if m.is_retriever() and m.is_available(self._rt_obj)] + for method in retriever_methods: + multicall.add(method, self.rpc_id) + + multicall.call() + + def __repr__(self): + return safe_repr("File(index={0} path=\"{1}\")", self.index, self.path) + +methods = [ + # RETRIEVERS + Method(File, 'get_last_touched', 'f.get_last_touched'), + Method(File, 'get_range_second', 'f.get_range_second'), + Method(File, 'get_size_bytes', 'f.get_size_bytes'), + Method(File, 'get_priority', 'f.get_priority'), + Method(File, 'get_match_depth_next', 'f.get_match_depth_next'), + Method(File, 'is_resize_queued', 'f.is_resize_queued', + boolean=True, + ), + Method(File, 'get_range_first', 'f.get_range_first'), + Method(File, 'get_match_depth_prev', 'f.get_match_depth_prev'), + Method(File, 'get_path', 'f.get_path'), + Method(File, 'get_completed_chunks', 'f.get_completed_chunks'), + Method(File, 'get_path_components', 'f.get_path_components'), + Method(File, 'is_created', 'f.is_created', + boolean=True, + ), + Method(File, 'is_open', 'f.is_open', + boolean=True, + ), + Method(File, 'get_size_chunks', 'f.get_size_chunks'), + Method(File, 'get_offset', 'f.get_offset'), + Method(File, 'get_frozen_path', 'f.get_frozen_path'), + Method(File, 'get_path_depth', 'f.get_path_depth'), + Method(File, 'is_create_queued', 'f.is_create_queued', + boolean=True, + ), + + + # MODIFIERS +] diff --git a/libs/rtorrent/lib/__init__.py b/libs/rtorrent/lib/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/libs/rtorrent/lib/bencode.py b/libs/rtorrent/lib/bencode.py new file mode 100755 index 00000000..97bd2f0e --- /dev/null +++ b/libs/rtorrent/lib/bencode.py @@ -0,0 +1,281 @@ +# Copyright (C) 2011 by clueless +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# Version: 20111107 +# +# Changelog +# --------- +# 2011-11-07 - Added support for Python2 (tested on 2.6) +# 2011-10-03 - Fixed: moved check for end of list at the top of the while loop +# in _decode_list (in case the list is empty) (Chris Lucas) +# - Converted dictionary keys to str +# 2011-04-24 - Changed date format to YYYY-MM-DD for versioning, bigger +# integer denotes a newer version +# - Fixed a bug that would treat False as an integral type but +# encode it using the 'False' string, attempting to encode a +# boolean now results in an error +# - Fixed a bug where an integer value of 0 in a list or +# dictionary resulted in a parse error while decoding +# +# 2011-04-03 - Original release + +import sys + +_py3 = sys.version_info[0] == 3 + +if _py3: + _VALID_STRING_TYPES = (str,) +else: + _VALID_STRING_TYPES = (str, unicode) # @UndefinedVariable + +_TYPE_INT = 1 +_TYPE_STRING = 2 +_TYPE_LIST = 3 +_TYPE_DICTIONARY = 4 +_TYPE_END = 5 +_TYPE_INVALID = 6 + +# Function to determine the type of he next value/item +# Arguments: +# char First character of the string that is to be decoded +# Return value: +# Returns an integer that describes what type the next value/item is + + +def _gettype(char): + if not isinstance(char, int): + char = ord(char) + if char == 0x6C: # 'l' + return _TYPE_LIST + elif char == 0x64: # 'd' + return _TYPE_DICTIONARY + elif char == 0x69: # 'i' + return _TYPE_INT + elif char == 0x65: # 'e' + return _TYPE_END + elif char >= 0x30 and char <= 0x39: # '0' '9' + return _TYPE_STRING + else: + return _TYPE_INVALID + +# Function to parse a string from the bendcoded data +# Arguments: +# data bencoded data, must be guaranteed to be a string +# Return Value: +# Returns a tuple, the first member of the tuple is the parsed string +# The second member is whatever remains of the bencoded data so it can +# be used to parse the next part of the data + + +def _decode_string(data): + end = 1 + # if py3, data[end] is going to be an int + # if py2, data[end] will be a string + if _py3: + char = 0x3A + else: + char = chr(0x3A) + + while data[end] != char: # ':' + end = end + 1 + strlen = int(data[:end]) + return (data[end + 1:strlen + end + 1], data[strlen + end + 1:]) + +# Function to parse an integer from the bencoded data +# Arguments: +# data bencoded data, must be guaranteed to be an integer +# Return Value: +# Returns a tuple, the first member of the tuple is the parsed string +# The second member is whatever remains of the bencoded data so it can +# be used to parse the next part of the data + + +def _decode_int(data): + end = 1 + # if py3, data[end] is going to be an int + # if py2, data[end] will be a string + if _py3: + char = 0x65 + else: + char = chr(0x65) + + while data[end] != char: # 'e' + end = end + 1 + return (int(data[1:end]), data[end + 1:]) + +# Function to parse a bencoded list +# Arguments: +# data bencoded data, must be guaranted to be the start of a list +# Return Value: +# Returns a tuple, the first member of the tuple is the parsed list +# The second member is whatever remains of the bencoded data so it can +# be used to parse the next part of the data + + +def _decode_list(data): + x = [] + overflow = data[1:] + while True: # Loop over the data + if _gettype(overflow[0]) == _TYPE_END: # - Break if we reach the end of the list + return (x, overflow[1:]) # and return the list and overflow + + value, overflow = _decode(overflow) # + if isinstance(value, bool) or overflow == '': # - if we have a parse error + return (False, False) # Die with error + else: # - Otherwise + x.append(value) # add the value to the list + + +# Function to parse a bencoded list +# Arguments: +# data bencoded data, must be guaranted to be the start of a list +# Return Value: +# Returns a tuple, the first member of the tuple is the parsed dictionary +# The second member is whatever remains of the bencoded data so it can +# be used to parse the next part of the data +def _decode_dict(data): + x = {} + overflow = data[1:] + while True: # Loop over the data + if _gettype(overflow[0]) != _TYPE_STRING: # - If the key is not a string + return (False, False) # Die with error + key, overflow = _decode(overflow) # + if key == False or overflow == '': # - If parse error + return (False, False) # Die with error + value, overflow = _decode(overflow) # + if isinstance(value, bool) or overflow == '': # - If parse error + print("Error parsing value") + print(value) + print(overflow) + return (False, False) # Die with error + else: + # don't use bytes for the key + key = key.decode() + x[key] = value + if _gettype(overflow[0]) == _TYPE_END: + return (x, overflow[1:]) + +# Arguments: +# data bencoded data in bytes format +# Return Values: +# Returns a tuple, the first member is the parsed data, could be a string, +# an integer, a list or a dictionary, or a combination of those +# The second member is the leftover of parsing, if everything parses correctly this +# should be an empty byte string + + +def _decode(data): + btype = _gettype(data[0]) + if btype == _TYPE_INT: + return _decode_int(data) + elif btype == _TYPE_STRING: + return _decode_string(data) + elif btype == _TYPE_LIST: + return _decode_list(data) + elif btype == _TYPE_DICTIONARY: + return _decode_dict(data) + else: + return (False, False) + +# Function to decode bencoded data +# Arguments: +# data bencoded data, can be str or bytes +# Return Values: +# Returns the decoded data on success, this coud be bytes, int, dict or list +# or a combinatin of those +# If an error occurs the return value is False + + +def decode(data): + # if isinstance(data, str): + # data = data.encode() + decoded, overflow = _decode(data) + return decoded + +# Args: data as integer +# return: encoded byte string + + +def _encode_int(data): + return b'i' + str(data).encode() + b'e' + +# Args: data as string or bytes +# Return: encoded byte string + + +def _encode_string(data): + return str(len(data)).encode() + b':' + data + +# Args: data as list +# Return: Encoded byte string, false on error + + +def _encode_list(data): + elist = b'l' + for item in data: + eitem = encode(item) + if eitem == False: + return False + elist += eitem + return elist + b'e' + +# Args: data as dict +# Return: encoded byte string, false on error + + +def _encode_dict(data): + edict = b'd' + keys = [] + for key in data: + if not isinstance(key, _VALID_STRING_TYPES) and not isinstance(key, bytes): + return False + keys.append(key) + keys.sort() + for key in keys: + ekey = encode(key) + eitem = encode(data[key]) + if ekey == False or eitem == False: + return False + edict += ekey + eitem + return edict + b'e' + +# Function to encode a variable in bencoding +# Arguments: +# data Variable to be encoded, can be a list, dict, str, bytes, int or a combination of those +# Return Values: +# Returns the encoded data as a byte string when successful +# If an error occurs the return value is False + + +def encode(data): + if isinstance(data, bool): + return False + elif isinstance(data, int): + return _encode_int(data) + elif isinstance(data, bytes): + return _encode_string(data) + elif isinstance(data, _VALID_STRING_TYPES): + return _encode_string(data.encode()) + elif isinstance(data, list): + return _encode_list(data) + elif isinstance(data, dict): + return _encode_dict(data) + else: + return False diff --git a/libs/rtorrent/lib/torrentparser.py b/libs/rtorrent/lib/torrentparser.py new file mode 100755 index 00000000..19dd12aa --- /dev/null +++ b/libs/rtorrent/lib/torrentparser.py @@ -0,0 +1,159 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from rtorrent.compat import is_py3 +import os.path +import re +import rtorrent.lib.bencode as bencode +import hashlib + +if is_py3(): + from urllib.request import urlopen # @UnresolvedImport @UnusedImport +else: + from urllib2 import urlopen # @UnresolvedImport @Reimport + + +class TorrentParser(): + def __init__(self, torrent): + """Decode and parse given torrent + + @param torrent: handles: urls, file paths, string of torrent data + @type torrent: str + + @raise AssertionError: Can be raised for a couple reasons: + - If _get_raw_torrent() couldn't figure out + what X{torrent} is + - if X{torrent} isn't a valid bencoded torrent file + """ + self.torrent = torrent + self._raw_torrent = None # : testing yo + self._torrent_decoded = None # : what up + self.file_type = None + + self._get_raw_torrent() + assert self._raw_torrent is not None, "Couldn't get raw_torrent." + if self._torrent_decoded is None: + self._decode_torrent() + assert isinstance(self._torrent_decoded, dict), "Invalid torrent file." + self._parse_torrent() + + def _is_raw(self): + raw = False + if isinstance(self.torrent, (str, bytes)): + if isinstance(self._decode_torrent(self.torrent), dict): + raw = True + else: + # reset self._torrent_decoded (currently equals False) + self._torrent_decoded = None + + return(raw) + + def _get_raw_torrent(self): + """Get raw torrent data by determining what self.torrent is""" + # already raw? + if self._is_raw(): + self.file_type = "raw" + self._raw_torrent = self.torrent + return + # local file? + if os.path.isfile(self.torrent): + self.file_type = "file" + self._raw_torrent = open(self.torrent, "rb").read() + # url? + elif re.search("^(http|ftp):\/\/", self.torrent, re.I): + self.file_type = "url" + self._raw_torrent = urlopen(self.torrent).read() + + def _decode_torrent(self, raw_torrent=None): + if raw_torrent is None: + raw_torrent = self._raw_torrent + self._torrent_decoded = bencode.decode(raw_torrent) + return(self._torrent_decoded) + + def _calc_info_hash(self): + self.info_hash = None + if "info" in self._torrent_decoded.keys(): + info_dict = self._torrent_decoded["info"] + self.info_hash = hashlib.sha1(bencode.encode( + info_dict)).hexdigest().upper() + + return(self.info_hash) + + def _parse_torrent(self): + for k in self._torrent_decoded: + key = k.replace(" ", "_").lower() + setattr(self, key, self._torrent_decoded[k]) + + self._calc_info_hash() + + +class NewTorrentParser(object): + @staticmethod + def _read_file(fp): + return fp.read() + + @staticmethod + def _write_file(fp): + fp.write() + return fp + + @staticmethod + def _decode_torrent(data): + return bencode.decode(data) + + def __init__(self, input): + self.input = input + self._raw_torrent = None + self._decoded_torrent = None + self._hash_outdated = False + + if isinstance(self.input, (str, bytes)): + # path to file? + if os.path.isfile(self.input): + self._raw_torrent = self._read_file(open(self.input, "rb")) + else: + # assume input was the raw torrent data (do we really want + # this?) + self._raw_torrent = self.input + + # file-like object? + elif self.input.hasattr("read"): + self._raw_torrent = self._read_file(self.input) + + assert self._raw_torrent is not None, "Invalid input: input must be a path or a file-like object" + + self._decoded_torrent = self._decode_torrent(self._raw_torrent) + + assert isinstance( + self._decoded_torrent, dict), "File could not be decoded" + + def _calc_info_hash(self): + self.info_hash = None + info_dict = self._torrent_decoded["info"] + self.info_hash = hashlib.sha1(bencode.encode( + info_dict)).hexdigest().upper() + + return(self.info_hash) + + def set_tracker(self, tracker): + self._decoded_torrent["announce"] = tracker + + def get_tracker(self): + return self._decoded_torrent.get("announce") diff --git a/libs/rtorrent/lib/xmlrpc/__init__.py b/libs/rtorrent/lib/xmlrpc/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/libs/rtorrent/lib/xmlrpc/http.py b/libs/rtorrent/lib/xmlrpc/http.py new file mode 100755 index 00000000..3eb85210 --- /dev/null +++ b/libs/rtorrent/lib/xmlrpc/http.py @@ -0,0 +1,23 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from rtorrent.compat import xmlrpclib + +HTTPServerProxy = xmlrpclib.ServerProxy diff --git a/libs/rtorrent/peer.py b/libs/rtorrent/peer.py new file mode 100755 index 00000000..61ca0941 --- /dev/null +++ b/libs/rtorrent/peer.py @@ -0,0 +1,98 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# from rtorrent.rpc import Method +import rtorrent.rpc + +from rtorrent.common import safe_repr + +Method = rtorrent.rpc.Method + + +class Peer: + """Represents an individual peer within a L{Torrent} instance.""" + def __init__(self, _rt_obj, info_hash, **kwargs): + self._rt_obj = _rt_obj + self.info_hash = info_hash # : info hash for the torrent the peer is associated with + for k in kwargs.keys(): + setattr(self, k, kwargs.get(k, None)) + + self.rpc_id = "{0}:p{1}".format( + self.info_hash, self.id) # : unique id to pass to rTorrent + + def __repr__(self): + return safe_repr("Peer(id={0})", self.id) + + def update(self): + """Refresh peer data + + @note: All fields are stored as attributes to self. + + @return: None + """ + multicall = rtorrent.rpc.Multicall(self) + retriever_methods = [m for m in methods + if m.is_retriever() and m.is_available(self._rt_obj)] + for method in retriever_methods: + multicall.add(method, self.rpc_id) + + multicall.call() + +methods = [ + # RETRIEVERS + Method(Peer, 'is_preferred', 'p.is_preferred', + boolean=True, + ), + Method(Peer, 'get_down_rate', 'p.get_down_rate'), + Method(Peer, 'is_unwanted', 'p.is_unwanted', + boolean=True, + ), + Method(Peer, 'get_peer_total', 'p.get_peer_total'), + Method(Peer, 'get_peer_rate', 'p.get_peer_rate'), + Method(Peer, 'get_port', 'p.get_port'), + Method(Peer, 'is_snubbed', 'p.is_snubbed', + boolean=True, + ), + Method(Peer, 'get_id_html', 'p.get_id_html'), + Method(Peer, 'get_up_rate', 'p.get_up_rate'), + Method(Peer, 'is_banned', 'p.banned', + boolean=True, + ), + Method(Peer, 'get_completed_percent', 'p.get_completed_percent'), + Method(Peer, 'completed_percent', 'p.completed_percent'), + Method(Peer, 'get_id', 'p.get_id'), + Method(Peer, 'is_obfuscated', 'p.is_obfuscated', + boolean=True, + ), + Method(Peer, 'get_down_total', 'p.get_down_total'), + Method(Peer, 'get_client_version', 'p.get_client_version'), + Method(Peer, 'get_address', 'p.get_address'), + Method(Peer, 'is_incoming', 'p.is_incoming', + boolean=True, + ), + Method(Peer, 'is_encrypted', 'p.is_encrypted', + boolean=True, + ), + Method(Peer, 'get_options_str', 'p.get_options_str'), + Method(Peer, 'get_client_version', 'p.client_version'), + Method(Peer, 'get_up_total', 'p.get_up_total'), + + # MODIFIERS +] diff --git a/libs/rtorrent/rpc/__init__.py b/libs/rtorrent/rpc/__init__.py new file mode 100755 index 00000000..8190de46 --- /dev/null +++ b/libs/rtorrent/rpc/__init__.py @@ -0,0 +1,354 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +from base64 import encodestring +import httplib +import string + +import rtorrent +import re +from rtorrent.common import bool_to_int, convert_version_tuple_to_str,\ + safe_repr +from rtorrent.err import RTorrentVersionError, MethodError +from rtorrent.compat import xmlrpclib + + +class BasicAuthTransport(xmlrpclib.Transport): + def __init__(self, username=None, password=None): + xmlrpclib.Transport.__init__(self) + self.username = username + self.password = password + + def send_auth(self, h): + if self.username is not None and self.password is not None: + h.putheader('AUTHORIZATION', "Basic %s" % string.replace( + encodestring("%s:%s" % (self.username, self.password)), + "\012", "" + )) + + def single_request(self, host, handler, request_body, verbose=0): + # issue XML-RPC request + + h = self.make_connection(host) + if verbose: + h.set_debuglevel(1) + + try: + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + self.send_auth(h) + self.send_content(h, request_body) + + response = h.getresponse(buffering=True) + if response.status == 200: + self.verbose = verbose + return self.parse_response(response) + except xmlrpclib.Fault: + raise + except Exception: + self.close() + raise + + #discard any response data and raise exception + #if (response.getheader("content-length", 0)): + # response.read() + raise xmlrpclib.ProtocolError( + host + handler, + response.status, response.reason, + response.msg, + ) + + +def get_varname(rpc_call): + """Transform rpc method into variable name. + + @newfield example: Example + @example: if the name of the rpc method is 'p.get_down_rate', the variable + name will be 'down_rate' + """ + # extract variable name from xmlrpc func name + r = re.search( + "([ptdf]\.|system\.|get\_|is\_|set\_)+([^=]*)", rpc_call, re.I) + if r: + return(r.groups()[-1]) + else: + return(None) + + +def _handle_unavailable_rpc_method(method, rt_obj): + msg = "Method isn't available." + if rt_obj._get_client_version_tuple() < method.min_version: + msg = "This method is only available in " \ + "RTorrent version v{0} or later".format( + convert_version_tuple_to_str(method.min_version)) + + raise MethodError(msg) + + +class DummyClass: + def __init__(self): + pass + + +class Method: + """Represents an individual RPC method""" + + def __init__(self, _class, method_name, + rpc_call, docstring=None, varname=None, **kwargs): + self._class = _class # : Class this method is associated with + self.class_name = _class.__name__ + self.method_name = method_name # : name of public-facing method + self.rpc_call = rpc_call # : name of rpc method + self.docstring = docstring # : docstring for rpc method (optional) + self.varname = varname # : variable for the result of the method call, usually set to self.varname + self.min_version = kwargs.get("min_version", ( + 0, 0, 0)) # : Minimum version of rTorrent required + self.boolean = kwargs.get("boolean", False) # : returns boolean value? + self.post_process_func = kwargs.get( + "post_process_func", None) # : custom post process function + self.aliases = kwargs.get( + "aliases", []) # : aliases for method (optional) + self.required_args = [] + #: Arguments required when calling the method (not utilized) + + self.method_type = self._get_method_type() + + if self.varname is None: + self.varname = get_varname(self.rpc_call) + assert self.varname is not None, "Couldn't get variable name." + + def __repr__(self): + return safe_repr("Method(method_name='{0}', rpc_call='{1}')", + self.method_name, self.rpc_call) + + def _get_method_type(self): + """Determine whether method is a modifier or a retriever""" + if self.method_name[:4] == "set_": return('m') # modifier + else: + return('r') # retriever + + def is_modifier(self): + if self.method_type == 'm': + return(True) + else: + return(False) + + def is_retriever(self): + if self.method_type == 'r': + return(True) + else: + return(False) + + def is_available(self, rt_obj): + if rt_obj._get_client_version_tuple() < self.min_version or \ + self.rpc_call not in rt_obj._get_rpc_methods(): + return(False) + else: + return(True) + + +class Multicall: + def __init__(self, class_obj, **kwargs): + self.class_obj = class_obj + if class_obj.__class__.__name__ == "RTorrent": + self.rt_obj = class_obj + else: + self.rt_obj = class_obj._rt_obj + self.calls = [] + + def add(self, method, *args): + """Add call to multicall + + @param method: L{Method} instance or name of raw RPC method + @type method: Method or str + + @param args: call arguments + """ + # if a raw rpc method was given instead of a Method instance, + # try and find the instance for it. And if all else fails, create a + # dummy Method instance + if isinstance(method, str): + result = find_method(method) + # if result not found + if result == -1: + method = Method(DummyClass, method, method) + else: + method = result + + # ensure method is available before adding + if not method.is_available(self.rt_obj): + _handle_unavailable_rpc_method(method, self.rt_obj) + + self.calls.append((method, args)) + + def list_calls(self): + for c in self.calls: + print(c) + + def call(self): + """Execute added multicall calls + + @return: the results (post-processed), in the order they were added + @rtype: tuple + """ + m = xmlrpclib.MultiCall(self.rt_obj._get_conn()) + for call in self.calls: + method, args = call + rpc_call = getattr(method, "rpc_call") + getattr(m, rpc_call)(*args) + + results = m() + results = tuple(results) + results_processed = [] + + for r, c in zip(results, self.calls): + method = c[0] # Method instance + result = process_result(method, r) + results_processed.append(result) + # assign result to class_obj + setattr(self.class_obj, method.varname, result) + + return(tuple(results_processed)) + + +def call_method(class_obj, method, *args): + """Handles single RPC calls + + @param class_obj: Peer/File/Torrent/Tracker/RTorrent instance + @type class_obj: object + + @param method: L{Method} instance or name of raw RPC method + @type method: Method or str + """ + if method.is_retriever(): + args = args[:-1] + else: + assert args[-1] is not None, "No argument given." + + if class_obj.__class__.__name__ == "RTorrent": + rt_obj = class_obj + else: + rt_obj = class_obj._rt_obj + + # check if rpc method is even available + if not method.is_available(rt_obj): + _handle_unavailable_rpc_method(method, rt_obj) + + m = Multicall(class_obj) + m.add(method, *args) + # only added one method, only getting one result back + ret_value = m.call()[0] + + ####### OBSOLETE ########################################################## + # if method.is_retriever(): + # #value = process_result(method, ret_value) + # value = ret_value #MultiCall already processed the result + # else: + # # we're setting the user's input to method.varname + # # but we'll return the value that xmlrpc gives us + # value = process_result(method, args[-1]) + ########################################################################## + + return(ret_value) + + +def find_method(rpc_call): + """Return L{Method} instance associated with given RPC call""" + method_lists = [ + rtorrent.methods, + rtorrent.file.methods, + rtorrent.tracker.methods, + rtorrent.peer.methods, + rtorrent.torrent.methods, + ] + + for l in method_lists: + for m in l: + if m.rpc_call.lower() == rpc_call.lower(): + return(m) + + return(-1) + + +def process_result(method, result): + """Process given C{B{result}} based on flags set in C{B{method}} + + @param method: L{Method} instance + @type method: Method + + @param result: result to be processed (the result of given L{Method} instance) + + @note: Supported Processing: + - boolean - convert ones and zeros returned by rTorrent and + convert to python boolean values + """ + # handle custom post processing function + if method.post_process_func is not None: + result = method.post_process_func(result) + + # is boolean? + if method.boolean: + if result in [1, '1']: + result = True + elif result in [0, '0']: + result = False + + return(result) + + +def _build_rpc_methods(class_, method_list): + """Build glorified aliases to raw RPC methods""" + for m in method_list: + class_name = m.class_name + if class_name != class_.__name__: + continue + + if class_name == "RTorrent": + caller = lambda self, arg = None, method = m:\ + call_method(self, method, bool_to_int(arg)) + elif class_name == "Torrent": + caller = lambda self, arg = None, method = m:\ + call_method(self, method, self.rpc_id, + bool_to_int(arg)) + elif class_name in ["Tracker", "File"]: + caller = lambda self, arg = None, method = m:\ + call_method(self, method, self.rpc_id, + bool_to_int(arg)) + + elif class_name == "Peer": + caller = lambda self, arg = None, method = m:\ + call_method(self, method, self.rpc_id, + bool_to_int(arg)) + + if m.docstring is None: + m.docstring = "" + + # print(m) + docstring = """{0} + + @note: Variable where the result for this method is stored: {1}.{2}""".format( + m.docstring, + class_name, + m.varname) + + caller.__doc__ = docstring + + for method_name in [m.method_name] + list(m.aliases): + setattr(class_, method_name, caller) diff --git a/libs/rtorrent/torrent.py b/libs/rtorrent/torrent.py new file mode 100755 index 00000000..1e06e1c2 --- /dev/null +++ b/libs/rtorrent/torrent.py @@ -0,0 +1,484 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import rtorrent.rpc +# from rtorrent.rpc import Method +import rtorrent.peer +import rtorrent.tracker +import rtorrent.file +import rtorrent.compat + +from rtorrent.common import safe_repr + +Peer = rtorrent.peer.Peer +Tracker = rtorrent.tracker.Tracker +File = rtorrent.file.File +Method = rtorrent.rpc.Method + + +class Torrent: + """Represents an individual torrent within a L{RTorrent} instance.""" + + def __init__(self, _rt_obj, info_hash, **kwargs): + self._rt_obj = _rt_obj + self.info_hash = info_hash # : info hash for the torrent + self.rpc_id = self.info_hash # : unique id to pass to rTorrent + for k in kwargs.keys(): + setattr(self, k, kwargs.get(k, None)) + + self.peers = [] + self.trackers = [] + self.files = [] + + self._call_custom_methods() + + def __repr__(self): + return safe_repr("Torrent(info_hash=\"{0}\" name=\"{1}\")", + self.info_hash, self.name) + + def _call_custom_methods(self): + """only calls methods that check instance variables.""" + self._is_hash_checking_queued() + self._is_started() + self._is_paused() + + def get_peers(self): + """Get list of Peer instances for given torrent. + + @return: L{Peer} instances + @rtype: list + + @note: also assigns return value to self.peers + """ + self.peers = [] + retriever_methods = [m for m in rtorrent.peer.methods + if m.is_retriever() and m.is_available(self._rt_obj)] + # need to leave 2nd arg empty (dunno why) + m = rtorrent.rpc.Multicall(self) + m.add("p.multicall", self.info_hash, "", + *[method.rpc_call + "=" for method in retriever_methods]) + + results = m.call()[0] # only sent one call, only need first result + + for result in results: + results_dict = {} + # build results_dict + for m, r in zip(retriever_methods, result): + results_dict[m.varname] = rtorrent.rpc.process_result(m, r) + + self.peers.append(Peer( + self._rt_obj, self.info_hash, **results_dict)) + + return(self.peers) + + def get_trackers(self): + """Get list of Tracker instances for given torrent. + + @return: L{Tracker} instances + @rtype: list + + @note: also assigns return value to self.trackers + """ + self.trackers = [] + retriever_methods = [m for m in rtorrent.tracker.methods + if m.is_retriever() and m.is_available(self._rt_obj)] + + # need to leave 2nd arg empty (dunno why) + m = rtorrent.rpc.Multicall(self) + m.add("t.multicall", self.info_hash, "", + *[method.rpc_call + "=" for method in retriever_methods]) + + results = m.call()[0] # only sent one call, only need first result + + for result in results: + results_dict = {} + # build results_dict + for m, r in zip(retriever_methods, result): + results_dict[m.varname] = rtorrent.rpc.process_result(m, r) + + self.trackers.append(Tracker( + self._rt_obj, self.info_hash, **results_dict)) + + return(self.trackers) + + def get_files(self): + """Get list of File instances for given torrent. + + @return: L{File} instances + @rtype: list + + @note: also assigns return value to self.files + """ + + self.files = [] + retriever_methods = [m for m in rtorrent.file.methods + if m.is_retriever() and m.is_available(self._rt_obj)] + # 2nd arg can be anything, but it'll return all files in torrent + # regardless + m = rtorrent.rpc.Multicall(self) + m.add("f.multicall", self.info_hash, "", + *[method.rpc_call + "=" for method in retriever_methods]) + + results = m.call()[0] # only sent one call, only need first result + + offset_method_index = retriever_methods.index( + rtorrent.rpc.find_method("f.get_offset")) + + # make a list of the offsets of all the files, sort appropriately + offset_list = sorted([r[offset_method_index] for r in results]) + + for result in results: + results_dict = {} + # build results_dict + for m, r in zip(retriever_methods, result): + results_dict[m.varname] = rtorrent.rpc.process_result(m, r) + + # get proper index positions for each file (based on the file + # offset) + f_index = offset_list.index(results_dict["offset"]) + + self.files.append(File(self._rt_obj, self.info_hash, + f_index, **results_dict)) + + return(self.files) + + def set_directory(self, d): + """Modify download directory + + @note: Needs to stop torrent in order to change the directory. + Also doesn't restart after directory is set, that must be called + separately. + """ + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.try_stop") + self.multicall_add(m, "d.set_directory", d) + + self.directory = m.call()[-1] + + def start(self): + """Start the torrent""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.try_start") + self.multicall_add(m, "d.is_active") + + self.active = m.call()[-1] + return(self.active) + + def stop(self): + """"Stop the torrent""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.try_stop") + self.multicall_add(m, "d.is_active") + + self.active = m.call()[-1] + return(self.active) + + def close(self): + """Close the torrent and it's files""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.close") + + return(m.call()[-1]) + + def erase(self): + """Delete the torrent + + @note: doesn't delete the downloaded files""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.erase") + + return(m.call()[-1]) + + def check_hash(self): + """(Re)hash check the torrent""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.check_hash") + + return(m.call()[-1]) + + def poll(self): + """poll rTorrent to get latest peer/tracker/file information""" + self.get_peers() + self.get_trackers() + self.get_files() + + def update(self): + """Refresh torrent data + + @note: All fields are stored as attributes to self. + + @return: None + """ + multicall = rtorrent.rpc.Multicall(self) + retriever_methods = [m for m in methods + if m.is_retriever() and m.is_available(self._rt_obj)] + for method in retriever_methods: + multicall.add(method, self.rpc_id) + + multicall.call() + + # custom functions (only call private methods, since they only check + # local variables and are therefore faster) + self._call_custom_methods() + + def accept_seeders(self, accept_seeds): + """Enable/disable whether the torrent connects to seeders + + @param accept_seeds: enable/disable accepting seeders + @type accept_seeds: bool""" + if accept_seeds: + call = "d.accepting_seeders.enable" + else: + call = "d.accepting_seeders.disable" + + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, call) + + return(m.call()[-1]) + + def announce(self): + """Announce torrent info to tracker(s)""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.tracker_announce") + + return(m.call()[-1]) + + @staticmethod + def _assert_custom_key_valid(key): + assert type(key) == int and key > 0 and key < 6, \ + "key must be an integer between 1-5" + + def get_custom(self, key): + """ + Get custom value + + @param key: the index for the custom field (between 1-5) + @type key: int + + @rtype: str + """ + + self._assert_custom_key_valid(key) + m = rtorrent.rpc.Multicall(self) + + field = "custom{0}".format(key) + self.multicall_add(m, "d.get_{0}".format(field)) + setattr(self, field, m.call()[-1]) + + return (getattr(self, field)) + + def set_custom(self, key, value): + """ + Set custom value + + @param key: the index for the custom field (between 1-5) + @type key: int + + @param value: the value to be stored + @type value: str + + @return: if successful, value will be returned + @rtype: str + """ + + self._assert_custom_key_valid(key) + m = rtorrent.rpc.Multicall(self) + + self.multicall_add(m, "d.set_custom{0}".format(key), value) + + return(m.call()[-1]) + + ############################################################################ + # CUSTOM METHODS (Not part of the official rTorrent API) + ########################################################################## + def _is_hash_checking_queued(self): + """Only checks instance variables, shouldn't be called directly""" + # if hashing == 3, then torrent is marked for hash checking + # if hash_checking == False, then torrent is waiting to be checked + self.hash_checking_queued = (self.hashing == 3 and + self.hash_checking is False) + + return(self.hash_checking_queued) + + def is_hash_checking_queued(self): + """Check if torrent is waiting to be hash checked + + @note: Variable where the result for this method is stored Torrent.hash_checking_queued""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.get_hashing") + self.multicall_add(m, "d.is_hash_checking") + results = m.call() + + setattr(self, "hashing", results[0]) + setattr(self, "hash_checking", results[1]) + + return(self._is_hash_checking_queued()) + + def _is_paused(self): + """Only checks instance variables, shouldn't be called directly""" + self.paused = (self.state == 0) + return(self.paused) + + def is_paused(self): + """Check if torrent is paused + + @note: Variable where the result for this method is stored: Torrent.paused""" + self.get_state() + return(self._is_paused()) + + def _is_started(self): + """Only checks instance variables, shouldn't be called directly""" + self.started = (self.state == 1) + return(self.started) + + def is_started(self): + """Check if torrent is started + + @note: Variable where the result for this method is stored: Torrent.started""" + self.get_state() + return(self._is_started()) + + +methods = [ + # RETRIEVERS + Method(Torrent, 'is_hash_checked', 'd.is_hash_checked', + boolean=True, + ), + Method(Torrent, 'is_hash_checking', 'd.is_hash_checking', + boolean=True, + ), + Method(Torrent, 'get_peers_max', 'd.get_peers_max'), + Method(Torrent, 'get_tracker_focus', 'd.get_tracker_focus'), + Method(Torrent, 'get_skip_total', 'd.get_skip_total'), + Method(Torrent, 'get_state', 'd.get_state'), + Method(Torrent, 'get_peer_exchange', 'd.get_peer_exchange'), + Method(Torrent, 'get_down_rate', 'd.get_down_rate'), + Method(Torrent, 'get_connection_seed', 'd.get_connection_seed'), + Method(Torrent, 'get_uploads_max', 'd.get_uploads_max'), + Method(Torrent, 'get_priority_str', 'd.get_priority_str'), + Method(Torrent, 'is_open', 'd.is_open', + boolean=True, + ), + Method(Torrent, 'get_peers_min', 'd.get_peers_min'), + Method(Torrent, 'get_peers_complete', 'd.get_peers_complete'), + Method(Torrent, 'get_tracker_numwant', 'd.get_tracker_numwant'), + Method(Torrent, 'get_connection_current', 'd.get_connection_current'), + Method(Torrent, 'is_complete', 'd.get_complete', + boolean=True, + ), + Method(Torrent, 'get_peers_connected', 'd.get_peers_connected'), + Method(Torrent, 'get_chunk_size', 'd.get_chunk_size'), + Method(Torrent, 'get_state_counter', 'd.get_state_counter'), + Method(Torrent, 'get_base_filename', 'd.get_base_filename'), + Method(Torrent, 'get_state_changed', 'd.get_state_changed'), + Method(Torrent, 'get_peers_not_connected', 'd.get_peers_not_connected'), + Method(Torrent, 'get_directory', 'd.get_directory'), + Method(Torrent, 'is_incomplete', 'd.incomplete', + boolean=True, + ), + Method(Torrent, 'get_tracker_size', 'd.get_tracker_size'), + Method(Torrent, 'is_multi_file', 'd.is_multi_file', + boolean=True, + ), + Method(Torrent, 'get_local_id', 'd.get_local_id'), + Method(Torrent, 'get_ratio', 'd.get_ratio', + post_process_func=lambda x: x / 1000.0, + ), + Method(Torrent, 'get_loaded_file', 'd.get_loaded_file'), + Method(Torrent, 'get_max_file_size', 'd.get_max_file_size'), + Method(Torrent, 'get_size_chunks', 'd.get_size_chunks'), + Method(Torrent, 'is_pex_active', 'd.is_pex_active', + boolean=True, + ), + Method(Torrent, 'get_hashing', 'd.get_hashing'), + Method(Torrent, 'get_bitfield', 'd.get_bitfield'), + Method(Torrent, 'get_local_id_html', 'd.get_local_id_html'), + Method(Torrent, 'get_connection_leech', 'd.get_connection_leech'), + Method(Torrent, 'get_peers_accounted', 'd.get_peers_accounted'), + Method(Torrent, 'get_message', 'd.get_message'), + Method(Torrent, 'is_active', 'd.is_active', + boolean=True, + ), + Method(Torrent, 'get_size_bytes', 'd.get_size_bytes'), + Method(Torrent, 'get_ignore_commands', 'd.get_ignore_commands'), + Method(Torrent, 'get_creation_date', 'd.get_creation_date'), + Method(Torrent, 'get_base_path', 'd.get_base_path'), + Method(Torrent, 'get_left_bytes', 'd.get_left_bytes'), + Method(Torrent, 'get_size_files', 'd.get_size_files'), + Method(Torrent, 'get_size_pex', 'd.get_size_pex'), + Method(Torrent, 'is_private', 'd.is_private', + boolean=True, + ), + Method(Torrent, 'get_max_size_pex', 'd.get_max_size_pex'), + Method(Torrent, 'get_num_chunks_hashed', 'd.get_chunks_hashed', + aliases=("get_chunks_hashed",)), + Method(Torrent, 'get_num_chunks_wanted', 'd.wanted_chunks'), + Method(Torrent, 'get_priority', 'd.get_priority'), + Method(Torrent, 'get_skip_rate', 'd.get_skip_rate'), + Method(Torrent, 'get_completed_bytes', 'd.get_completed_bytes'), + Method(Torrent, 'get_name', 'd.get_name'), + Method(Torrent, 'get_completed_chunks', 'd.get_completed_chunks'), + Method(Torrent, 'get_throttle_name', 'd.get_throttle_name'), + Method(Torrent, 'get_free_diskspace', 'd.get_free_diskspace'), + Method(Torrent, 'get_directory_base', 'd.get_directory_base'), + Method(Torrent, 'get_hashing_failed', 'd.get_hashing_failed'), + Method(Torrent, 'get_tied_to_file', 'd.get_tied_to_file'), + Method(Torrent, 'get_down_total', 'd.get_down_total'), + Method(Torrent, 'get_bytes_done', 'd.get_bytes_done'), + Method(Torrent, 'get_up_rate', 'd.get_up_rate'), + Method(Torrent, 'get_up_total', 'd.get_up_total'), + Method(Torrent, 'is_accepting_seeders', 'd.accepting_seeders', + boolean=True, + ), + Method(Torrent, "get_chunks_seen", "d.chunks_seen", + min_version=(0, 9, 1), + ), + Method(Torrent, "is_partially_done", "d.is_partially_done", + boolean=True, + ), + Method(Torrent, "is_not_partially_done", "d.is_not_partially_done", + boolean=True, + ), + Method(Torrent, "get_time_started", "d.timestamp.started"), + Method(Torrent, "get_custom1", "d.get_custom1"), + Method(Torrent, "get_custom2", "d.get_custom2"), + Method(Torrent, "get_custom3", "d.get_custom3"), + Method(Torrent, "get_custom4", "d.get_custom4"), + Method(Torrent, "get_custom5", "d.get_custom5"), + + # MODIFIERS + Method(Torrent, 'set_uploads_max', 'd.set_uploads_max'), + Method(Torrent, 'set_tied_to_file', 'd.set_tied_to_file'), + Method(Torrent, 'set_tracker_numwant', 'd.set_tracker_numwant'), + Method(Torrent, 'set_priority', 'd.set_priority'), + Method(Torrent, 'set_peers_max', 'd.set_peers_max'), + Method(Torrent, 'set_hashing_failed', 'd.set_hashing_failed'), + Method(Torrent, 'set_message', 'd.set_message'), + Method(Torrent, 'set_throttle_name', 'd.set_throttle_name'), + Method(Torrent, 'set_peers_min', 'd.set_peers_min'), + Method(Torrent, 'set_ignore_commands', 'd.set_ignore_commands'), + Method(Torrent, 'set_max_file_size', 'd.set_max_file_size'), + Method(Torrent, 'set_custom5', 'd.set_custom5'), + Method(Torrent, 'set_custom4', 'd.set_custom4'), + Method(Torrent, 'set_custom2', 'd.set_custom2'), + Method(Torrent, 'set_custom1', 'd.set_custom1'), + Method(Torrent, 'set_custom3', 'd.set_custom3'), + Method(Torrent, 'set_connection_current', 'd.set_connection_current'), +] diff --git a/libs/rtorrent/tracker.py b/libs/rtorrent/tracker.py new file mode 100755 index 00000000..81af2e49 --- /dev/null +++ b/libs/rtorrent/tracker.py @@ -0,0 +1,138 @@ +# Copyright (c) 2013 Chris Lucas, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# from rtorrent.rpc import Method +import rtorrent.rpc + +from rtorrent.common import safe_repr + +Method = rtorrent.rpc.Method + + +class Tracker: + """Represents an individual tracker within a L{Torrent} instance.""" + + def __init__(self, _rt_obj, info_hash, **kwargs): + self._rt_obj = _rt_obj + self.info_hash = info_hash # : info hash for the torrent using this tracker + for k in kwargs.keys(): + setattr(self, k, kwargs.get(k, None)) + + # for clarity's sake... + self.index = self.group # : position of tracker within the torrent's tracker list + self.rpc_id = "{0}:t{1}".format( + self.info_hash, self.index) # : unique id to pass to rTorrent + + def __repr__(self): + return safe_repr("Tracker(index={0}, url=\"{1}\")", + self.index, self.url) + + def enable(self): + """Alias for set_enabled("yes")""" + self.set_enabled("yes") + + def disable(self): + """Alias for set_enabled("no")""" + self.set_enabled("no") + + def update(self): + """Refresh tracker data + + @note: All fields are stored as attributes to self. + + @return: None + """ + multicall = rtorrent.rpc.Multicall(self) + retriever_methods = [m for m in methods + if m.is_retriever() and m.is_available(self._rt_obj)] + for method in retriever_methods: + multicall.add(method, self.rpc_id) + + multicall.call() + +methods = [ + # RETRIEVERS + Method(Tracker, 'is_enabled', 't.is_enabled', boolean=True), + Method(Tracker, 'get_id', 't.get_id'), + Method(Tracker, 'get_scrape_incomplete', 't.get_scrape_incomplete'), + Method(Tracker, 'is_open', 't.is_open', boolean=True), + Method(Tracker, 'get_min_interval', 't.get_min_interval'), + Method(Tracker, 'get_scrape_downloaded', 't.get_scrape_downloaded'), + Method(Tracker, 'get_group', 't.get_group'), + Method(Tracker, 'get_scrape_time_last', 't.get_scrape_time_last'), + Method(Tracker, 'get_type', 't.get_type'), + Method(Tracker, 'get_normal_interval', 't.get_normal_interval'), + Method(Tracker, 'get_url', 't.get_url'), + Method(Tracker, 'get_scrape_complete', 't.get_scrape_complete', + min_version=(0, 8, 9), + ), + Method(Tracker, 'get_activity_time_last', 't.activity_time_last', + min_version=(0, 8, 9), + ), + Method(Tracker, 'get_activity_time_next', 't.activity_time_next', + min_version=(0, 8, 9), + ), + Method(Tracker, 'get_failed_time_last', 't.failed_time_last', + min_version=(0, 8, 9), + ), + Method(Tracker, 'get_failed_time_next', 't.failed_time_next', + min_version=(0, 8, 9), + ), + Method(Tracker, 'get_success_time_last', 't.success_time_last', + min_version=(0, 8, 9), + ), + Method(Tracker, 'get_success_time_next', 't.success_time_next', + min_version=(0, 8, 9), + ), + Method(Tracker, 'can_scrape', 't.can_scrape', + min_version=(0, 9, 1), + boolean=True + ), + Method(Tracker, 'get_failed_counter', 't.failed_counter', + min_version=(0, 8, 9) + ), + Method(Tracker, 'get_scrape_counter', 't.scrape_counter', + min_version=(0, 8, 9) + ), + Method(Tracker, 'get_success_counter', 't.success_counter', + min_version=(0, 8, 9) + ), + Method(Tracker, 'is_usable', 't.is_usable', + min_version=(0, 9, 1), + boolean=True + ), + Method(Tracker, 'is_busy', 't.is_busy', + min_version=(0, 9, 1), + boolean=True + ), + Method(Tracker, 'is_extra_tracker', 't.is_extra_tracker', + min_version=(0, 9, 1), + boolean=True, + ), + Method(Tracker, "get_latest_sum_peers", "t.latest_sum_peers", + min_version=(0, 9, 0) + ), + Method(Tracker, "get_latest_new_peers", "t.latest_new_peers", + min_version=(0, 9, 0) + ), + + # MODIFIERS + Method(Tracker, 'set_enabled', 't.set_enabled'), +] From d851be41d3fb6065e499a6ad3ddfd5e88a4a5275 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sat, 27 Jul 2013 21:06:58 +1200 Subject: [PATCH 065/209] Updated rtorrent-python library. --- libs/rtorrent/rpc/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/rtorrent/rpc/__init__.py b/libs/rtorrent/rpc/__init__.py index 8190de46..f83446ca 100755 --- a/libs/rtorrent/rpc/__init__.py +++ b/libs/rtorrent/rpc/__init__.py @@ -67,8 +67,8 @@ class BasicAuthTransport(xmlrpclib.Transport): raise #discard any response data and raise exception - #if (response.getheader("content-length", 0)): - # response.read() + if (response.getheader("content-length", 0)): + response.read() raise xmlrpclib.ProtocolError( host + handler, response.status, response.reason, From bf6265353146fa36468ded856b5e8c11b67305a0 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 28 Jul 2013 00:36:10 +1200 Subject: [PATCH 066/209] Added missing 'folder' parameter on the rtorrent downloader to fix moving/linking issues. --- couchpotato/core/downloaders/rtorrent/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 5da64cb7..a06ebe5e 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -87,11 +87,11 @@ class rTorrent(Downloader): 'original_status': item.state, 'timeleft': str(timedelta(seconds=float(item.left_bytes) / item.down_rate)) if item.down_rate > 0 else -1, - 'folder': '' + 'folder': item.directory }) return statuses except Exception, err: - log.error('Failed to send torrent to rTorrent: %s', err) + log.error('Failed to get status from rTorrent: %s', err) return False From 38e204dfe837457303e84e7831e0e6861d5901d7 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 28 Jul 2013 01:27:39 +1200 Subject: [PATCH 067/209] Added support for labels on the rtorrent downloader. --- .../core/downloaders/rtorrent/__init__.py | 2 +- couchpotato/core/downloaders/rtorrent/main.py | 24 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index d0047893..f3944c7e 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -33,7 +33,7 @@ config = [{ }, { 'name': 'label', - 'description': 'Label to add torrent as.', + 'description': 'Label to apply on added torrents.', }, { 'name': 'paused', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index a06ebe5e..8bd86f9e 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -25,11 +25,14 @@ class rTorrent(Downloader): rtorrent_api = None def get_conn(self): - return RTorrent( - self.conf('url'), - self.conf('username'), - self.conf('password') - ) + if self.conf('username') and self.conf('password'): + return RTorrent( + self.conf('url'), + self.conf('username'), + self.conf('password') + ) + + return RTorrent(self.conf('url')) def download(self, data, movie, filedata=None): log.debug('Sending "%s" (%s) to rTorrent.', (data.get('name'), data.get('type'))) @@ -59,7 +62,16 @@ class rTorrent(Downloader): if not self.rtorrent_api: self.rtorrent_api = self.get_conn() - torrent = self.rtorrent_api.load_torrent(filedata, not self.conf('paused', default=0)) + # Send torrent to rTorrent + torrent = self.rtorrent_api.load_torrent(filedata) + + # Set label + if self.conf('label'): + torrent.set_custom(1, self.conf('label')) + + # Start torrent + if not self.conf('paused', default=0): + torrent.start() return self.downloadReturnId(torrent_hash) except Exception, err: From 0fadbd52a31216e2e5b69677a64e993f4e32f2d3 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Mon, 29 Jul 2013 20:49:34 +1200 Subject: [PATCH 068/209] Cleaned up imports and added support for downloading magnet torrents via sources. --- couchpotato/core/downloaders/rtorrent/main.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 8bd86f9e..7544af89 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -1,18 +1,11 @@ from base64 import b16encode, b32decode -from bencode import bencode, bdecode -from couchpotato.core.downloaders.base import Downloader, StatusList -from couchpotato.core.helpers.encoding import isInt, ss -from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 -from multipartpost import MultipartPostHandler -import cookielib -import httplib -import json -import re -import time -import urllib -import urllib2 +import traceback + +from bencode import bencode, bdecode +from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.logger import CPLog from rtorrent import RTorrent @@ -45,13 +38,17 @@ class rTorrent(Downloader): log.error('Failed sending torrent, no data') return False + # Try download magnet torrents if data.get('type') == 'torrent_magnet': - log.info('magnet torrents are not supported') - return False + filedata = self.magnetToTorrent(data.get('url')) + + if filedata is False: + return False + + data['type'] = 'torrent' info = bdecode(filedata)["info"] torrent_hash = sha1(bencode(info)).hexdigest().upper() - torrent_filename = self.createFileName(data, filedata, movie) # Convert base 32 to hex if len(torrent_hash) == 32: From 7c680cac10b98861427407bee7af7c612113afe8 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Wed, 31 Jul 2013 22:41:20 +1200 Subject: [PATCH 069/209] Updated rTorrent downloader to set ratio stop action, added new seeding methods and updated the rTorrent library --- .../core/downloaders/rtorrent/__init__.py | 24 ++++ couchpotato/core/downloaders/rtorrent/main.py | 109 +++++++++++++++--- libs/rtorrent/__init__.py | 21 ++++ libs/rtorrent/group.py | 88 ++++++++++++++ libs/rtorrent/rpc/__init__.py | 19 ++- libs/rtorrent/torrent.py | 22 ++++ 6 files changed, 266 insertions(+), 17 deletions(-) create mode 100755 libs/rtorrent/group.py diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index f3944c7e..b28f5808 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -35,6 +35,30 @@ config = [{ 'name': 'label', 'description': 'Label to apply on added torrents.', }, + { + 'name': 'stop_complete', + 'label': 'Stop torrent', + 'default': False, + 'advanced': True, + 'type': 'bool', + 'description': 'Stop the torrent after it finishes seeding' + }, + { + 'name': 'remove_complete', + 'label': 'Remove torrent', + 'default': False, + 'advanced': True, + 'type': 'bool', + 'description': 'Remove the torrent after it finishes seeding.', + }, + { + 'name': 'delete_files', + 'label': 'Remove files', + 'default': True, + 'type': 'bool', + 'advanced': True, + 'description': 'Also remove the leftover files.', + }, { 'name': 'paused', 'type': 'bool', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 7544af89..33243fea 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -15,21 +15,64 @@ log = CPLog(__name__) class rTorrent(Downloader): type = ['torrent', 'torrent_magnet'] - rtorrent_api = None + rt = None + + def connect(self): + # Already connected? + if self.rt is not None: + return self.rt + + # Ensure url is set + if not self.conf('url'): + log.error('Config properties are not filled in correctly, url is missing.') + return False - def get_conn(self): if self.conf('username') and self.conf('password'): - return RTorrent( + self.rt = RTorrent( self.conf('url'), self.conf('username'), self.conf('password') ) + else: + self.rt = RTorrent(self.conf('url')) + + return self.rt + + def _update_provider_group(self, name, data): + if data.get('seed_time') is not None: + log.info('seeding time ignored, not supported') + + if name is None or data.get('seed_ratio') is None: + return False + + if not self.connect(): + return False + + views = self.rt.get_views() + + if name not in views: + self.rt.create_group(name) + + log.debug('Updating provider ratio to %s, group name: %s', (data.get('seed_ratio'), name)) + + group = self.rt.get_group(name) + group.get_min(data.get('seed_ratio') * 100) + + if self.conf('stop_complete'): + group.set_command('d.stop') + else: + group.set_command() - return RTorrent(self.conf('url')) def download(self, data, movie, filedata=None): log.debug('Sending "%s" (%s) to rTorrent.', (data.get('name'), data.get('type'))) + if not self.connect(): + return False + + group_name = 'cp_' + data.get('provider').lower() + self._update_provider_group(group_name, data) + torrent_params = {} if self.conf('label'): torrent_params['label'] = self.conf('label') @@ -56,16 +99,16 @@ class rTorrent(Downloader): # Send request to rTorrent try: - if not self.rtorrent_api: - self.rtorrent_api = self.get_conn() - # Send torrent to rTorrent - torrent = self.rtorrent_api.load_torrent(filedata) + torrent = self.rt.load_torrent(filedata) # Set label if self.conf('label'): torrent.set_custom(1, self.conf('label')) + # Set Ratio Group + torrent.set_visible(group_name) + # Start torrent if not self.conf('paused', default=0): torrent.start() @@ -75,24 +118,30 @@ class rTorrent(Downloader): log.error('Failed to send torrent to rTorrent: %s', err) return False - def getAllDownloadStatus(self): - log.debug('Checking rTorrent download status.') - try: - if not self.rtorrent_api: - self.rtorrent_api = self.get_conn() + if not self.connect(): + return False - torrents = self.rtorrent_api.get_torrents() + try: + torrents = self.rt.get_torrents() statuses = StatusList(self) for item in torrents: + status = 'busy' + if item.complete: + if item.active: + status = 'seeding' + else: + status = 'completed' + statuses.append({ 'id': item.info_hash, 'name': item.name, - 'status': 'completed' if item.complete else 'busy', + 'status': status, + 'seed_ratio': item.ratio, 'original_status': item.state, 'timeleft': str(timedelta(seconds=float(item.left_bytes) / item.down_rate)) if item.down_rate > 0 else -1, @@ -104,3 +153,33 @@ class rTorrent(Downloader): except Exception, err: log.error('Failed to get status from rTorrent: %s', err) return False + + def pause(self, download_info, pause = True): + if not self.connect(): + return False + + torrent = self.rt.find_torrent(download_info['id']) + if torrent is None: + return False + + if pause: + return torrent.pause() + return torrent.resume() + + def removeFailed(self, item): + log.info('%s failed downloading, deleting...', item['name']) + return self.processComplete(item, delete_files=True) + + def processComplete(self, item, delete_files): + log.debug('Requesting rTorrent to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) + if not self.connect(): + return False + + torrent = self.rt.find_torrent(item['id']) + if torrent is None: + return False + + if delete_files: + log.info('not deleting files, not supported') + + return torrent.erase() # just removes the torrent, doesn't delete data diff --git a/libs/rtorrent/__init__.py b/libs/rtorrent/__init__.py index e427b65e..d19c78b4 100755 --- a/libs/rtorrent/__init__.py +++ b/libs/rtorrent/__init__.py @@ -24,6 +24,7 @@ from rtorrent.lib.torrentparser import TorrentParser from rtorrent.lib.xmlrpc.http import HTTPServerProxy from rtorrent.rpc import Method, BasicAuthTransport from rtorrent.torrent import Torrent +from rtorrent.group import Group import os.path import rtorrent.rpc # @UnresolvedImport import time @@ -286,6 +287,26 @@ class RTorrent: getattr(p, func_name)(finput) + def get_views(self): + p = self._get_conn() + return p.view_list() + + def create_group(self, name, persistent=True, view=None): + p = self._get_conn() + + if persistent is True: + p.group.insert_persistent_view('', name) + else: + assert view is not None, "view parameter required on non-persistent groups" + p.group.insert('', name, view) + + def get_group(self, name): + assert name is not None, "group name required" + + group = Group(self, name) + group.update() + return group + def set_dht_port(self, port): """Set DHT port diff --git a/libs/rtorrent/group.py b/libs/rtorrent/group.py new file mode 100755 index 00000000..01f6bb3c --- /dev/null +++ b/libs/rtorrent/group.py @@ -0,0 +1,88 @@ +# Copyright (c) 2013 Dean Gardiner, +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import rtorrent.rpc + +Method = rtorrent.rpc.Method + + +class Group: + __name__ = 'Group' + + def __init__(self, _rt_obj, name): + self._rt_obj = _rt_obj + self.name = name + + self.methods = [ + # RETRIEVERS + Method(Group, 'get_max', 'group.' + self.name + '.ratio.max', varname='max'), + Method(Group, 'get_min', 'group.' + self.name + '.ratio.min', varname='min'), + Method(Group, 'get_upload', 'group.' + self.name + '.ratio.upload', varname='upload'), + + # MODIFIERS + Method(Group, 'set_max', 'group.' + self.name + '.ratio.max.set', varname='max'), + Method(Group, 'set_min', 'group.' + self.name + '.ratio.min.set', varname='min'), + Method(Group, 'set_upload', 'group.' + self.name + '.ratio.upload.set', varname='upload') + ] + + rtorrent.rpc._build_rpc_methods(self, self.methods) + + # Setup multicall_add method + caller = lambda multicall, method, *args: \ + multicall.add(method, *args) + setattr(self, "multicall_add", caller) + + def _get_prefix(self): + return 'group.' + self.name + '.ratio.' + + def update(self): + multicall = rtorrent.rpc.Multicall(self) + + retriever_methods = [m for m in self.methods + if m.is_retriever() and m.is_available(self._rt_obj)] + + for method in retriever_methods: + multicall.add(method) + + multicall.call() + + def enable(self): + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, self._get_prefix() + 'enable') + + return(m.call()[-1]) + + def disable(self): + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, self._get_prefix() + 'disable') + + return(m.call()[-1]) + + def set_command(self, *methods): + methods = [m + '=' for m in methods] + + m = rtorrent.rpc.Multicall(self) + self.multicall_add( + m, 'system.method.set', + self._get_prefix() + 'command', + *methods + ) + + return(m.call()[-1]) diff --git a/libs/rtorrent/rpc/__init__.py b/libs/rtorrent/rpc/__init__.py index f83446ca..034f4eef 100755 --- a/libs/rtorrent/rpc/__init__.py +++ b/libs/rtorrent/rpc/__init__.py @@ -19,6 +19,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from base64 import encodestring import httplib +import inspect import string import rtorrent @@ -223,7 +224,9 @@ class Multicall: result = process_result(method, r) results_processed.append(result) # assign result to class_obj - setattr(self.class_obj, method.varname, result) + exists = hasattr(self.class_obj, method.varname) + if not exists or not inspect.ismethod(getattr(self.class_obj, method.varname)): + setattr(self.class_obj, method.varname, result) return(tuple(results_processed)) @@ -315,6 +318,11 @@ def process_result(method, result): def _build_rpc_methods(class_, method_list): """Build glorified aliases to raw RPC methods""" + instance = None + if not inspect.isclass(class_): + instance = class_ + class_ = instance.__class__ + for m in method_list: class_name = m.class_name if class_name != class_.__name__: @@ -337,6 +345,10 @@ def _build_rpc_methods(class_, method_list): call_method(self, method, self.rpc_id, bool_to_int(arg)) + elif class_name == "Group": + caller = lambda arg = None, method = m: \ + call_method(instance, method, bool_to_int(arg)) + if m.docstring is None: m.docstring = "" @@ -351,4 +363,7 @@ def _build_rpc_methods(class_, method_list): caller.__doc__ = docstring for method_name in [m.method_name] + list(m.aliases): - setattr(class_, method_name, caller) + if instance is None: + setattr(class_, method_name, caller) + else: + setattr(instance, method_name, caller) diff --git a/libs/rtorrent/torrent.py b/libs/rtorrent/torrent.py index 1e06e1c2..c610e368 100755 --- a/libs/rtorrent/torrent.py +++ b/libs/rtorrent/torrent.py @@ -190,6 +190,20 @@ class Torrent: self.active = m.call()[-1] return(self.active) + def pause(self): + """Pause the torrent""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.pause") + + return(m.call()[-1]) + + def resume(self): + """Resume the torrent""" + m = rtorrent.rpc.Multicall(self) + self.multicall_add(m, "d.resume") + + return(m.call()[-1]) + def close(self): """Close the torrent and it's files""" m = rtorrent.rpc.Multicall(self) @@ -305,6 +319,14 @@ class Torrent: return(m.call()[-1]) + def set_visible(self, view, visible=True): + p = self._rt_obj._get_conn() + + if visible: + return p.view.set_visible(self.info_hash, view) + else: + return p.view.set_not_visible(self.info_hash, view) + ############################################################################ # CUSTOM METHODS (Not part of the official rTorrent API) ########################################################################## From 577baeca5938fe8eca0bc0f783f1ab8e4aefc6d1 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Thu, 1 Aug 2013 00:03:56 +1200 Subject: [PATCH 070/209] Hiding remove files in the rTorrent downloader until it's implemented. --- couchpotato/core/downloaders/rtorrent/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index b28f5808..877ff2d4 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -54,7 +54,8 @@ config = [{ { 'name': 'delete_files', 'label': 'Remove files', - 'default': True, + 'default': False, + 'hidden': True, 'type': 'bool', 'advanced': True, 'description': 'Also remove the leftover files.', From 317c3afb7a52026f3f96312d59b0f4c8d531d4f0 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Thu, 1 Aug 2013 17:02:58 +1200 Subject: [PATCH 071/209] Few minor fixes and implemented delete_files option via shutil.rmtree --- couchpotato/core/downloaders/rtorrent/__init__.py | 3 +-- couchpotato/core/downloaders/rtorrent/main.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index 877ff2d4..b28f5808 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -54,8 +54,7 @@ config = [{ { 'name': 'delete_files', 'label': 'Remove files', - 'default': False, - 'hidden': True, + 'default': True, 'type': 'bool', 'advanced': True, 'description': 'Also remove the leftover files.', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index 33243fea..e885b20f 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -1,6 +1,7 @@ from base64 import b16encode, b32decode from datetime import timedelta from hashlib import sha1 +import shutil import traceback from bencode import bencode, bdecode @@ -39,10 +40,10 @@ class rTorrent(Downloader): return self.rt def _update_provider_group(self, name, data): - if data.get('seed_time') is not None: + if data.get('seed_time'): log.info('seeding time ignored, not supported') - if name is None or data.get('seed_ratio') is None: + if not name or not data.get('seed_ratio'): return False if not self.connect(): @@ -179,7 +180,9 @@ class rTorrent(Downloader): if torrent is None: return False - if delete_files: - log.info('not deleting files, not supported') + torrent.erase() # just removes the torrent, doesn't delete data - return torrent.erase() # just removes the torrent, doesn't delete data + if delete_files: + shutil.rmtree(item['folder'], True) + + return True From 7202fbf084e1b73fb0b8fe9380121b74c1b47cc1 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Thu, 1 Aug 2013 18:08:49 +1200 Subject: [PATCH 072/209] Removed stop_complete option, Can instead be disabled by setting seed_ratio to zero on the provider. --- couchpotato/core/downloaders/rtorrent/__init__.py | 8 -------- couchpotato/core/downloaders/rtorrent/main.py | 9 ++++++--- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index b28f5808..db50eba3 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -35,14 +35,6 @@ config = [{ 'name': 'label', 'description': 'Label to apply on added torrents.', }, - { - 'name': 'stop_complete', - 'label': 'Stop torrent', - 'default': False, - 'advanced': True, - 'type': 'bool', - 'description': 'Stop the torrent after it finishes seeding' - }, { 'name': 'remove_complete', 'label': 'Remove torrent', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index e885b20f..b6d2fbe9 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -43,7 +43,7 @@ class rTorrent(Downloader): if data.get('seed_time'): log.info('seeding time ignored, not supported') - if not name or not data.get('seed_ratio'): + if not name: return False if not self.connect(): @@ -57,13 +57,16 @@ class rTorrent(Downloader): log.debug('Updating provider ratio to %s, group name: %s', (data.get('seed_ratio'), name)) group = self.rt.get_group(name) - group.get_min(data.get('seed_ratio') * 100) - if self.conf('stop_complete'): + if data.get('seed_ratio'): + group.set_min(int(data.get('seed_ratio') * 100)) group.set_command('d.stop') else: + # Reset group action group.set_command() + return True + def download(self, data, movie, filedata=None): log.debug('Sending "%s" (%s) to rTorrent.', (data.get('name'), data.get('type'))) From 0bdffc5036c4dd4fa19dae3c8b936fe255ce7457 Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Fri, 2 Aug 2013 01:40:31 +1200 Subject: [PATCH 073/209] Change to ratio group setup to ensure everything is set correctly. --- couchpotato/core/downloaders/rtorrent/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index b6d2fbe9..d27a3a03 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -59,11 +59,16 @@ class rTorrent(Downloader): group = self.rt.get_group(name) if data.get('seed_ratio'): + # Explicitly set all group options to ensure it is setup correctly + group.set_upload('1M') group.set_min(int(data.get('seed_ratio') * 100)) + group.set_max(int(data.get('seed_ratio') * 100)) group.set_command('d.stop') + group.enable() else: - # Reset group action + # Reset group action and disable it group.set_command() + group.disable() return True From 2bb2e28f91534cde5cc537146f069da50986572d Mon Sep 17 00:00:00 2001 From: Dean Gardiner Date: Sun, 4 Aug 2013 15:26:00 +1200 Subject: [PATCH 074/209] Updated rTorrent library and fixed some issues with ratio setup. --- couchpotato/core/downloaders/rtorrent/main.py | 36 +++++++++++-------- libs/rtorrent/group.py | 12 +++---- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index d27a3a03..97ef3e17 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -2,7 +2,7 @@ from base64 import b16encode, b32decode from datetime import timedelta from hashlib import sha1 import shutil -import traceback +from rtorrent.err import MethodError from bencode import bencode, bdecode from couchpotato.core.downloaders.base import Downloader, StatusList @@ -54,21 +54,26 @@ class rTorrent(Downloader): if name not in views: self.rt.create_group(name) - log.debug('Updating provider ratio to %s, group name: %s', (data.get('seed_ratio'), name)) - group = self.rt.get_group(name) - if data.get('seed_ratio'): - # Explicitly set all group options to ensure it is setup correctly - group.set_upload('1M') - group.set_min(int(data.get('seed_ratio') * 100)) - group.set_max(int(data.get('seed_ratio') * 100)) - group.set_command('d.stop') - group.enable() - else: - # Reset group action and disable it - group.set_command() - group.disable() + try: + if data.get('seed_ratio'): + ratio = int(float(data.get('seed_ratio')) * 100) + log.debug('Updating provider ratio to %s, group name: %s', (ratio, name)) + + # Explicitly set all group options to ensure it is setup correctly + group.set_upload('1M') + group.set_min(ratio) + group.set_max(ratio) + group.set_command('d.stop') + group.enable() + else: + # Reset group action and disable it + group.set_command() + group.disable() + except MethodError, err: + log.error('Unable to set group options: %s', err.message) + return False return True @@ -80,7 +85,8 @@ class rTorrent(Downloader): return False group_name = 'cp_' + data.get('provider').lower() - self._update_provider_group(group_name, data) + if not self._update_provider_group(group_name, data): + return False torrent_params = {} if self.conf('label'): diff --git a/libs/rtorrent/group.py b/libs/rtorrent/group.py index 01f6bb3c..e8246aa8 100755 --- a/libs/rtorrent/group.py +++ b/libs/rtorrent/group.py @@ -64,16 +64,12 @@ class Group: multicall.call() def enable(self): - m = rtorrent.rpc.Multicall(self) - self.multicall_add(m, self._get_prefix() + 'enable') - - return(m.call()[-1]) + p = self._rt_obj._get_conn() + return getattr(p, self._get_prefix() + 'enable')() def disable(self): - m = rtorrent.rpc.Multicall(self) - self.multicall_add(m, self._get_prefix() + 'disable') - - return(m.call()[-1]) + p = self._rt_obj._get_conn() + return getattr(p, self._get_prefix() + 'disable')() def set_command(self, *methods): methods = [m + '=' for m in methods] From 6eff724f97abe152df1333322d679a59e32159f6 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 15:36:11 +0200 Subject: [PATCH 075/209] Clean nonblocking requestshandler --- couchpotato/api.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/couchpotato/api.py b/couchpotato/api.py index 029ebce2..7020fd93 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -12,27 +12,28 @@ api_docs_missing = [] # NonBlock API handler class NonBlockHandler(RequestHandler): - stoppers = [] + stopper = None @asynchronous def get(self, route, *args, **kwargs): route = route.strip('/') start, stop = api_nonblock[route] - self.stoppers.append(stop) + self.stopper = stop - start(self.onNewMessage, last_id = self.get_argument("last_id", None)) + start(self.onNewMessage, last_id = self.get_argument('last_id', None)) def onNewMessage(self, response): if self.request.connection.stream.closed(): return + self.finish(response) def on_connection_close(self): - for stop in self.stoppers: - stop(self.onNewMessage) + if self.stopper: + self.stopper(self.onNewMessage) - self.stoppers = [] + self.stopper = None def addNonBlockApiView(route, func_tuple, docs = None, **kwargs): api_nonblock[route] = func_tuple From 57e92ff8d35d35e408063b328a39a069e68308d7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 15:40:56 +0200 Subject: [PATCH 076/209] Optimized frontend notifications --- couchpotato/core/notifications/core/main.py | 32 ++++++++++++--------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index b6c07f58..a6ce2a63 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -7,6 +7,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from couchpotato.core.settings.model import Notification as Notif from couchpotato.environment import Env +from operator import itemgetter from sqlalchemy.sql.expression import or_ import threading import time @@ -19,8 +20,6 @@ log = CPLog(__name__) class CoreNotifier(Notification): m_lock = threading.Lock() - messages = [] - listeners = [] def __init__(self): super(CoreNotifier, self).__init__() @@ -51,10 +50,14 @@ class CoreNotifier(Notification): addApiView('notification.listener', self.listener) fireEvent('schedule.interval', 'core.check_messages', self.checkMessages, hours = 12, single = True) + fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True) addEvent('app.load', self.clean) addEvent('app.load', self.checkMessages) + self.messages = [] + self.listeners = [] + def clean(self): db = get_session() @@ -169,8 +172,8 @@ class CoreNotifier(Notification): except: log.debug('Failed sending to listener: %s', traceback.format_exc()) + self.listeners = [] self.m_lock.release() - self.cleanMessages() log.debug('Done notifying frontend') @@ -199,12 +202,14 @@ class CoreNotifier(Notification): def cleanMessages(self): + if len(self.messages) == 0: + return + log.debug('Cleaning messages') self.m_lock.acquire() - for message in self.messages: - if message['time'] < (time.time() - 15): - self.messages.remove(message) + time_ago = (time.time() - 15) + self.messages[:] = [m for m in self.messages if (m['time'] > time_ago)] self.m_lock.release() log.debug('Done cleaning messages') @@ -215,16 +220,16 @@ class CoreNotifier(Notification): self.m_lock.acquire() recent = [] - index = 0 - for i in xrange(len(self.messages)): - index = len(self.messages) - i - 1 - if self.messages[index]["message_id"] == last_id: break - recent = self.messages[index:] + try: + index = map(itemgetter('message_id'), self.messages).index(last_id) + recent = self.messages[index+1:] + except: + pass self.m_lock.release() - log.debug('Returning for %s %s messages', (last_id, len(recent or []))) + log.debug('Returning for %s %s messages', (last_id, len(recent))) - return recent or [] + return recent def listener(self, init = False, **kwargs): @@ -237,6 +242,7 @@ class CoreNotifier(Notification): notifications = db.query(Notif) \ .filter(or_(Notif.read == False, Notif.added > (time.time() - 259200))) \ .all() + for n in notifications: ndict = n.to_dict() ndict['type'] = 'notification' From 8d058d9dc8f71be956fc744d681d0fe3ff6b467d Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 15:45:05 +0200 Subject: [PATCH 077/209] Add hdscr to screener quality --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index b1034a53..67c7f00d 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -22,7 +22,7 @@ class QualityPlugin(Plugin): {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener'], 'allow': ['dvdr', 'dvd'], 'ext':['avi', 'mpg', 'mpeg'], 'tags': ['webrip', ('web', 'rip')]}, + {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvd'], 'ext':['avi', 'mpg', 'mpeg'], 'tags': ['webrip', ('web', 'rip')]}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, From b8bed627a8199210aebd85ba049d9e725d7c84f7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 16:08:21 +0200 Subject: [PATCH 078/209] Add possible title with some char replacements --- couchpotato/core/helpers/variable.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index fa8a8b51..381889c0 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -176,6 +176,10 @@ def possibleTitles(raw_title): titles.append(raw_title.lower()) titles.append(simplifyString(raw_title)) + # replace some chars + new_title = raw_title.replace('&', 'and') + titles.append(simplifyString(new_title)) + return list(set(titles)) def randomString(size = 8, chars = string.ascii_uppercase + string.digits): From a0ccff23a3d91fccc959d7a5ea32d2e969a8af0f Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 16:08:34 +0200 Subject: [PATCH 079/209] Remove duplicate spaces --- couchpotato/core/helpers/encoding.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index a11dd88b..9b753db6 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -11,7 +11,8 @@ log = CPLog(__name__) def toSafeString(original): valid_chars = "-_.() %s%s" % (ascii_letters, digits) cleanedFilename = unicodedata.normalize('NFKD', toUnicode(original)).encode('ASCII', 'ignore') - return ''.join(c for c in cleanedFilename if c in valid_chars) + valid_string = ''.join(c for c in cleanedFilename if c in valid_chars) + return ' '.join(valid_string.split()) def simplifyString(original): string = stripAccents(original.lower()) From 52f1df98bb04d97896e9e9e76ab010fbc7eacab8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 16:51:46 +0200 Subject: [PATCH 080/209] Don't try to split on empty string --- couchpotato/core/plugins/score/scores.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index 4d966eb2..a95b0a42 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -68,9 +68,12 @@ def namePositionScore(nzb_name, movie_name): name_year = fireEvent('scanner.name_year', nzb_name, single = True) # Give points for movies beginning with the correct name - name_split = simplifyString(nzb_name).split(simplifyString(movie_name)) - if name_split[0].strip() == '': - score += 10 + split_by = simplifyString(movie_name) + name_split = [] + if len(split_by) > 0: + name_split = simplifyString(nzb_name).split(split_by) + if name_split[0].strip() == '': + score += 10 # If year is second in line, give more points if len(name_split) > 1 and name_year: From 16eeeda7874e1d86df4c7bbb3eac9af33db6680e Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 17:25:24 +0200 Subject: [PATCH 081/209] Ignore folder include with __ at beginning --- couchpotato/core/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index a97437a2..ceaa9efa 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -28,7 +28,7 @@ class Loader(object): provider_dir = os.path.join(root, 'couchpotato', 'core', 'providers') for provider in os.listdir(provider_dir): path = os.path.join(provider_dir, provider) - if os.path.isdir(path): + if os.path.isdir(path) and provider[:2] != '__': self.paths[provider + '_provider'] = (25, 'couchpotato.core.providers.' + provider, path) From 0f925a466aa2c5092d5187412bd715d7311e4e19 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 17:31:12 +0200 Subject: [PATCH 082/209] Also ignore __ when importing folders --- couchpotato/core/loader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index ceaa9efa..0f49700a 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -43,6 +43,9 @@ class Loader(object): for module_name, plugin in sorted(self.modules[priority].iteritems()): # Load module try: + if plugin.get('name')[:2] == '__': + continue + m = getattr(self.loadModule(module_name), plugin.get('name')) log.info('Loading %s: %s', (plugin['type'], plugin['name'])) From 2e93687bb44374339d62b5093819678c1080e0e5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 13 Aug 2013 17:46:41 +0200 Subject: [PATCH 083/209] Don't try to loop over empty enablers --- couchpotato/core/providers/automation/itunes/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/automation/itunes/main.py b/couchpotato/core/providers/automation/itunes/main.py index 14ca2a82..8e352370 100644 --- a/couchpotato/core/providers/automation/itunes/main.py +++ b/couchpotato/core/providers/automation/itunes/main.py @@ -31,7 +31,7 @@ class ITunes(Automation, RSS): for url in urls: index += 1 - if not enablers[index]: + if len(enablers) == 0 or len(enablers) < index or not enablers[index]: continue try: From dc36e154485ad95eabf20bcb784da7189953a5de Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 11:56:08 +0200 Subject: [PATCH 084/209] Don't run multiple manage.progress requests --- couchpotato/static/scripts/page/manage.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/static/scripts/page/manage.js b/couchpotato/static/scripts/page/manage.js index aef1f3c7..faef34af 100644 --- a/couchpotato/static/scripts/page/manage.js +++ b/couchpotato/static/scripts/page/manage.js @@ -86,9 +86,12 @@ Page.Manage = new Class({ self.progress_interval = setInterval(function(){ - Api.request('manage.progress', { + if(self.progress_request && self.progress_request.running) + return; + + self.update_in_progress = true; + self.progress_request = Api.request('manage.progress', { 'onComplete': function(json){ - self.update_in_progress = true; if(!json || !json.progress){ clearInterval(self.progress_interval); From 4b15563ba3a192c93b00bc38bc60918a8ccfe1c3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 12:13:52 +0200 Subject: [PATCH 085/209] Don't use in_progress when it isn't set --- couchpotato/core/plugins/manage/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 454e765c..03123a11 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -192,6 +192,9 @@ class Manage(Plugin): # Notify frontend def afterUpdate(): + if not self.in_progress: + return + self.in_progress[folder]['to_go'] = self.in_progress[folder]['to_go'] - 1 total = self.in_progress[folder]['total'] movie_dict = fireEvent('movie.get', identifier, single = True) From cf6f83a44b441126793280e2b16f54f99af98fb9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 12:14:52 +0200 Subject: [PATCH 086/209] Option to disable manage scan at startup. fix #1951 --- couchpotato/core/plugins/manage/__init__.py | 8 ++++++++ couchpotato/core/plugins/manage/main.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/manage/__init__.py b/couchpotato/core/plugins/manage/__init__.py index 46eee210..912296b4 100644 --- a/couchpotato/core/plugins/manage/__init__.py +++ b/couchpotato/core/plugins/manage/__init__.py @@ -28,6 +28,14 @@ config = [{ 'description': 'Remove movie from db if it can\'t be found after re-scan.', 'default': True, }, + { + 'label': 'Scan at startup', + 'name': 'startup_scan', + 'type': 'bool', + 'default': True, + 'advanced': True, + 'description': 'Do a quick scan on startup. On slow systems better disable this.', + }, ], }, ], diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 03123a11..25831f94 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -44,7 +44,7 @@ class Manage(Plugin): }"""}, }) - if not Env.get('dev'): + if not Env.get('dev') and self.conf('startup_scan'): addEvent('app.load', self.updateLibraryQuick) def getProgress(self, **kwargs): From 67bc3903d4d648a0f83f450787c7852f4774ebde Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 12:20:38 +0200 Subject: [PATCH 087/209] Don't show loader for scanner if page isn't loaded yet --- couchpotato/static/scripts/page/manage.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchpotato/static/scripts/page/manage.js b/couchpotato/static/scripts/page/manage.js index faef34af..36b1ef83 100644 --- a/couchpotato/static/scripts/page/manage.js +++ b/couchpotato/static/scripts/page/manage.js @@ -102,6 +102,11 @@ Page.Manage = new Class({ } } else { + + // Don't add loader when page is loading still + if(!self.list.navigation) + return; + if(!self.progress_container) self.progress_container = new Element('div.progress').inject(self.list.navigation, 'after') From d759280c18ff7a7ff8871e83d434d3fe16227d20 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 12:31:41 +0200 Subject: [PATCH 088/209] Don't update library items on shutdown --- couchpotato/core/plugins/library/main.py | 4 ++++ couchpotato/core/plugins/manage/main.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/library/main.py b/couchpotato/core/plugins/library/main.py index b463abfd..1842ed70 100644 --- a/couchpotato/core/plugins/library/main.py +++ b/couchpotato/core/plugins/library/main.py @@ -10,6 +10,7 @@ import traceback log = CPLog(__name__) + class LibraryPlugin(Plugin): default_dict = {'titles': {}, 'files':{}} @@ -57,6 +58,9 @@ class LibraryPlugin(Plugin): def update(self, identifier, default_title = '', force = False): + if self.shuttingDown(): + return + db = get_session() library = db.query(Library).filter_by(identifier = identifier).first() done_status = fireEvent('status.get', 'done', single = True) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 25831f94..63584636 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -192,7 +192,7 @@ class Manage(Plugin): # Notify frontend def afterUpdate(): - if not self.in_progress: + if not self.in_progress or self.shuttingDown(): return self.in_progress[folder]['to_go'] = self.in_progress[folder]['to_go'] - 1 From 8917d7c16c13f288d6645a48cd124dae1d92b4ee Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 16:47:59 +0200 Subject: [PATCH 089/209] Optimize movie.list query --- couchpotato/core/plugins/movie/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 0e6ef133..824633f3 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -200,7 +200,8 @@ class MoviePlugin(Plugin): q = q.subquery() q2 = db.query(Movie).join((q, q.c.id == Movie.id)) \ - .options(joinedload_all('releases')) \ + .options(joinedload_all('releases.files')) \ + .options(joinedload_all('releases.info')) \ .options(joinedload_all('profile.types')) \ .options(joinedload_all('library.titles')) \ .options(joinedload_all('library.files')) \ From 250f07ffa7084f0f6c4afec0b2fa4ada5e0f0036 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 14 Aug 2013 16:55:57 +0200 Subject: [PATCH 090/209] Optimize dashboard query --- couchpotato/core/plugins/dashboard/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index df21fefa..939b4154 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -46,7 +46,7 @@ class Dashboard(Plugin): q = db.query(Movie).join((subq, subq.c.id == Movie.id)) \ .options(joinedload_all('releases')) \ - .options(joinedload_all('profile.types')) \ + .options(joinedload_all('profile')) \ .options(joinedload_all('library.titles')) \ .options(joinedload_all('library.files')) \ .options(joinedload_all('status')) \ From 623571acbb9a26f72771595ab82a264cefccd602 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 15 Aug 2013 18:31:06 +0200 Subject: [PATCH 091/209] Make category destination editable --- couchpotato/core/plugins/category/main.py | 9 +++--- .../core/plugins/category/static/category.js | 29 +++++++++++++++++-- couchpotato/core/plugins/movie/main.py | 3 ++ .../core/plugins/movie/static/search.css | 3 +- .../core/plugins/movie/static/search.js | 1 - couchpotato/core/plugins/renamer/__init__.py | 2 +- couchpotato/core/plugins/renamer/main.py | 25 ++++++++++------ couchpotato/core/settings/model.py | 10 +------ couchpotato/static/scripts/page/settings.js | 4 +-- 9 files changed, 55 insertions(+), 31 deletions(-) diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index a91930da..6a60ae4e 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -1,6 +1,6 @@ from couchpotato import get_session from couchpotato.api import addApiView -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -11,8 +11,6 @@ log = CPLog(__name__) class CategoryPlugin(Plugin): - to_dict = {'destination': {}} - def __init__(self): addEvent('category.all', self.all) @@ -41,7 +39,7 @@ class CategoryPlugin(Plugin): temp = [] for category in categories: - temp.append(category.to_dict(self.to_dict)) + temp.append(category.to_dict()) db.expire_all() return temp @@ -61,10 +59,11 @@ class CategoryPlugin(Plugin): c.ignored = toUnicode(kwargs.get('ignored')) c.preferred = toUnicode(kwargs.get('preferred')) c.required = toUnicode(kwargs.get('required')) + c.destination = toUnicode(kwargs.get('destination')) db.commit() - category_dict = c.to_dict(self.to_dict) + category_dict = c.to_dict() return { 'success': True, diff --git a/couchpotato/core/plugins/category/static/category.js b/couchpotato/core/plugins/category/static/category.js index 22bbc38e..38e433c1 100644 --- a/couchpotato/core/plugins/category/static/category.js +++ b/couchpotato/core/plugins/category/static/category.js @@ -33,6 +33,28 @@ var CategoryListBase = new Class({ }) + // Add categories in renamer + self.settings.addEvent('create', function(){ + var renamer_group = self.settings.tabs.renamer.groups.renamer; + + p(renamer_group.getElement('.renamer_to')) + self.categories.each(function(category){ + + var input = new Option.Directory('section_name', 'option.name', category.get('destination'), { + 'name': category.get('label') + }); + input.inject(renamer_group.getElement('.renamer_to')); + input.fireEvent('injected'); + + input.save = function(){ + category.data.destination = input.getValue(); + category.save(); + }; + + }); + + }) + }, createList: function(){ @@ -68,7 +90,7 @@ var CategoryListBase = new Class({ return category.data.id == id }).pick() }, - + getAll: function(){ return this.categories; }, @@ -229,7 +251,7 @@ var Category = new Class({ } }); - }).delay(delay, self) + }).delay(delay || 0, self) }, @@ -241,7 +263,8 @@ var Category = new Class({ 'label' : self.el.getElement('.category_label input').get('value'), 'required' : self.el.getElement('.category_required input').get('value'), 'preferred' : self.el.getElement('.category_preferred input').get('value'), - 'ignored' : self.el.getElement('.category_ignored input').get('value') + 'ignored' : self.el.getElement('.category_ignored input').get('value'), + 'destination': self.data.destination } return data diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 824633f3..516ce85f 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -366,6 +366,7 @@ class MoviePlugin(Plugin): fireEvent('status.get', ['active', 'snatched', 'ignored', 'done', 'downloaded'], single = True) default_profile = fireEvent('profile.default', single = True) + cat_id = params.get('category_id', None) db = get_session() m = db.query(Movie).filter_by(library_id = library.get('id')).first() @@ -376,6 +377,7 @@ class MoviePlugin(Plugin): library_id = library.get('id'), profile_id = params.get('profile_id', default_profile.get('id')), status_id = status_id if status_id else status_active.get('id'), + category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, ) db.add(m) db.commit() @@ -397,6 +399,7 @@ class MoviePlugin(Plugin): fireEvent('release.delete', release.id, single = True) m.profile_id = params.get('profile_id', default_profile.get('id')) + m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None else: log.debug('Movie already exists, not updating: %s', params) added = False diff --git a/couchpotato/core/plugins/movie/static/search.css b/couchpotato/core/plugins/movie/static/search.css index 73d867bb..dc747346 100644 --- a/couchpotato/core/plugins/movie/static/search.css +++ b/couchpotato/core/plugins/movie/static/search.css @@ -159,8 +159,9 @@ display: inline-block; margin-right: 10px; } - .movie_result .options select[name=title] { width: 180px; } + .movie_result .options select[name=title] { width: 170px; } .movie_result .options select[name=profile] { width: 90px; } + .movie_result .options select[name=category] { width: 80px; } @media all and (max-width: 480px) { diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 23b9a3f4..376e61c9 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -389,7 +389,6 @@ Block.Search.Item = new Class({ self.options_el.addClass('set'); - p(self.info.in_wanted, self.info.in_wanted.profile, in_library); if(categories.length == 0 && self.title_select.getElements('option').length == 1 && profiles.length == 1 && !(self.info.in_wanted && self.info.in_wanted.profile || in_library)) self.add(); diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index e2bb4b93..04cd970d 100644 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -54,7 +54,7 @@ config = [{ { 'name': 'to', 'type': 'directory', - 'description': 'Folder where the movies should be moved to.', + 'description': 'Default folder where the movies are moved to.', }, { 'name': 'folder_name', diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index d1daf183..30ce4088 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -129,7 +129,6 @@ class Renamer(Plugin): groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = files, download_info = download_info, return_ignored = False, single = True) - destination = self.conf('to') folder_name = self.conf('folder_name') file_name = self.conf('file_name') trailer_name = self.conf('trailer_name') @@ -148,10 +147,6 @@ class Renamer(Plugin): remove_releases = [] movie_title = getTitle(group['library']) - try: - destination = group['category']['path'] - except: - destination = self.conf('to') # Add _UNKNOWN_ if no library item is connected if not group['library'] or not movie_title: @@ -165,8 +160,21 @@ class Renamer(Plugin): continue library = group['library'] + library_ent = db.query(Library).filter_by(identifier = group['library']['identifier']).first() + movie_title = getTitle(library) + # Overwrite destination when set in category + destination = self.conf('to') + for movie in library_ent.movies: + if movie.category and movie.category.destination and len(movie.category.destination) > 0: + destination = movie.category.destination + log.debug('Setting category destination for "%s": %s' % (movie_title, destination)) + else: + log.debug('No category destination found for "%s"' % movie_title) + + break + # Find subtitle for renaming group['before_rename'] = [] fireEvent('renamer.before', group) @@ -319,17 +327,16 @@ class Renamer(Plugin): cd += 1 # Before renaming, remove the lower quality files - library = db.query(Library).filter_by(identifier = group['library']['identifier']).first() remove_leftovers = True # Add it to the wanted list before we continue - if len(library.movies) == 0: + if len(library_ent.movies) == 0: profile = db.query(Profile).filter_by(core = True, label = group['meta_data']['quality']['label']).first() fireEvent('movie.add', params = {'identifier': group['library']['identifier'], 'profile_id': profile.id}, search_after = False) db.expire_all() library = db.query(Library).filter_by(identifier = group['library']['identifier']).first() - for movie in library.movies: + for movie in library_ent.movies: # Mark movie "done" once it's found the quality with the finish check try: @@ -819,6 +826,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) def statusInfoComplete(self, item): return item['id'] and item['downloader'] and item['folder'] - + def movieInFromFolder(self, movie_folder): return movie_folder and self.conf('from') in movie_folder or not movie_folder diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index 860c2e14..27373928 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -216,17 +216,9 @@ class Category(Entity): required = Field(Unicode(255)) preferred = Field(Unicode(255)) ignored = Field(Unicode(255)) + destination = Field(Unicode(255)) movie = OneToMany('Movie') - destination = ManyToOne('Destination') - - -class Destination(Entity): - """""" - - path = Field(Unicode(255)) - - category = OneToMany('Category') class ProfileType(Entity): diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index ef97506b..f7607547 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -329,8 +329,8 @@ var OptionBase = new Class({ * Create the element */ createBase: function(){ - var self = this - self.el = new Element('div.ctrlHolder') + var self = this; + self.el = new Element('div.ctrlHolder.'+self.section + '_' + self.name) }, create: function(){}, From 251d9cdb8abbf684a23c7ab7ef7f83bf6c3b1eda Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 15 Aug 2013 18:47:57 +0200 Subject: [PATCH 092/209] Placeholder for preferred words --- couchpotato/core/plugins/searcher/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/plugins/searcher/__init__.py b/couchpotato/core/plugins/searcher/__init__.py index 7e68a07b..d925ae7d 100644 --- a/couchpotato/core/plugins/searcher/__init__.py +++ b/couchpotato/core/plugins/searcher/__init__.py @@ -43,6 +43,7 @@ config = [{ 'name': 'preferred_words', 'label': 'Preferred', 'default': '', + 'placeholder': 'Example: CtrlHD, Amiable, Wiki', 'description': 'Words that give the releases a higher score.' }, { From 6395e5dbbb866c80df0efc4e5e087ae26f7b5ea2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 15 Aug 2013 23:51:42 +0200 Subject: [PATCH 093/209] Cleanup console log --- couchpotato/core/plugins/category/static/category.js | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/core/plugins/category/static/category.js b/couchpotato/core/plugins/category/static/category.js index 38e433c1..29a5a458 100644 --- a/couchpotato/core/plugins/category/static/category.js +++ b/couchpotato/core/plugins/category/static/category.js @@ -37,7 +37,6 @@ var CategoryListBase = new Class({ self.settings.addEvent('create', function(){ var renamer_group = self.settings.tabs.renamer.groups.renamer; - p(renamer_group.getElement('.renamer_to')) self.categories.each(function(category){ var input = new Option.Directory('section_name', 'option.name', category.get('destination'), { From 1620acedb1e5e4ef7b9a5443ca575dcf0f7c348d Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 15 Aug 2013 22:22:45 +0200 Subject: [PATCH 094/209] Move movie to new media type folder --- couchpotato/core/loader.py | 7 +++++++ couchpotato/core/media/__init__.py | 17 +++++++++++++++++ couchpotato/core/media/movie/__init__.py | 0 couchpotato/core/media/movie/_base/__init__.py | 6 ++++++ .../movie => media/movie/_base}/main.py | 12 ++++++++---- .../movie => media/movie/_base}/static/list.js | 0 .../movie/_base}/static/movie.actions.js | 0 .../movie/_base}/static/movie.css | 0 .../movie => media/movie/_base}/static/movie.js | 0 .../movie/_base}/static/search.css | 0 .../movie/_base}/static/search.js | 0 couchpotato/core/plugins/movie/__init__.py | 6 ------ 12 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 couchpotato/core/media/__init__.py create mode 100644 couchpotato/core/media/movie/__init__.py create mode 100644 couchpotato/core/media/movie/_base/__init__.py rename couchpotato/core/{plugins/movie => media/movie/_base}/main.py (98%) rename couchpotato/core/{plugins/movie => media/movie/_base}/static/list.js (100%) rename couchpotato/core/{plugins/movie => media/movie/_base}/static/movie.actions.js (100%) rename couchpotato/core/{plugins/movie => media/movie/_base}/static/movie.css (100%) rename couchpotato/core/{plugins/movie => media/movie/_base}/static/movie.js (100%) rename couchpotato/core/{plugins/movie => media/movie/_base}/static/search.css (100%) rename couchpotato/core/{plugins/movie => media/movie/_base}/static/search.js (100%) delete mode 100644 couchpotato/core/plugins/movie/__init__.py diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index 0f49700a..8aef89a4 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -31,6 +31,13 @@ class Loader(object): if os.path.isdir(path) and provider[:2] != '__': self.paths[provider + '_provider'] = (25, 'couchpotato.core.providers.' + provider, path) + # Add media to loader + media_dir = os.path.join(root, 'couchpotato', 'core', 'media') + for media in os.listdir(media_dir): + path = os.path.join(media_dir, media) + if os.path.isdir(path) and media[:2] != '__': + self.paths[media + '_media'] = (25, 'couchpotato.core.media.' + media, path) + for plugin_type, plugin_tuple in self.paths.iteritems(): priority, module, dir_name = plugin_tuple diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py new file mode 100644 index 00000000..2d339e5a --- /dev/null +++ b/couchpotato/core/media/__init__.py @@ -0,0 +1,17 @@ +from couchpotato.core.event import addEvent +from couchpotato.core.logger import CPLog +from couchpotato.core.plugins.base import Plugin + +log = CPLog(__name__) + + +class MediaBase(Plugin): + + identifier = None + + def __init__(self): + + addEvent('media.types', self.getType) + + def getType(self): + return self.identifier diff --git a/couchpotato/core/media/movie/__init__.py b/couchpotato/core/media/movie/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/couchpotato/core/media/movie/_base/__init__.py b/couchpotato/core/media/movie/_base/__init__.py new file mode 100644 index 00000000..4be3b127 --- /dev/null +++ b/couchpotato/core/media/movie/_base/__init__.py @@ -0,0 +1,6 @@ +from .main import MovieBase + +def start(): + return MovieBase() + +config = [] diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/media/movie/_base/main.py similarity index 98% rename from couchpotato/core/plugins/movie/main.py rename to couchpotato/core/media/movie/_base/main.py index 516ce85f..fc537ef0 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -4,7 +4,7 @@ from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode, simplifyString from couchpotato.core.helpers.variable import getImdb, splitString, tryInt from couchpotato.core.logger import CPLog -from couchpotato.core.plugins.base import Plugin +from couchpotato.core.media import MediaBase from couchpotato.core.settings.model import Library, LibraryTitle, Movie, \ Release from couchpotato.environment import Env @@ -16,7 +16,9 @@ import time log = CPLog(__name__) -class MoviePlugin(Plugin): +class MovieBase(MediaBase): + + identifier = 'movie' default_dict = { 'profile': {'types': {'quality': {}}}, @@ -27,6 +29,8 @@ class MoviePlugin(Plugin): } def __init__(self): + super(MovieBase, self).__init__() + addApiView('movie.search', self.search, docs = { 'desc': 'Search the movie providers for a movie', 'params': { @@ -476,7 +480,7 @@ class MoviePlugin(Plugin): fireEvent('movie.restatus', m.id) movie_dict = m.to_dict(self.default_dict) - fireEventAsync('searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id)) + fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id)) db.expire_all() return { @@ -574,7 +578,7 @@ class MoviePlugin(Plugin): def onComplete(): db = get_session() movie = db.query(Movie).filter_by(id = movie_id).first() - fireEventAsync('searcher.single', movie.to_dict(self.default_dict), on_complete = self.createNotifyFront(movie_id)) + fireEventAsync('movie.searcher.single', movie.to_dict(self.default_dict), on_complete = self.createNotifyFront(movie_id)) db.expire_all() return onComplete diff --git a/couchpotato/core/plugins/movie/static/list.js b/couchpotato/core/media/movie/_base/static/list.js similarity index 100% rename from couchpotato/core/plugins/movie/static/list.js rename to couchpotato/core/media/movie/_base/static/list.js diff --git a/couchpotato/core/plugins/movie/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js similarity index 100% rename from couchpotato/core/plugins/movie/static/movie.actions.js rename to couchpotato/core/media/movie/_base/static/movie.actions.js diff --git a/couchpotato/core/plugins/movie/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css similarity index 100% rename from couchpotato/core/plugins/movie/static/movie.css rename to couchpotato/core/media/movie/_base/static/movie.css diff --git a/couchpotato/core/plugins/movie/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js similarity index 100% rename from couchpotato/core/plugins/movie/static/movie.js rename to couchpotato/core/media/movie/_base/static/movie.js diff --git a/couchpotato/core/plugins/movie/static/search.css b/couchpotato/core/media/movie/_base/static/search.css similarity index 100% rename from couchpotato/core/plugins/movie/static/search.css rename to couchpotato/core/media/movie/_base/static/search.css diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/media/movie/_base/static/search.js similarity index 100% rename from couchpotato/core/plugins/movie/static/search.js rename to couchpotato/core/media/movie/_base/static/search.js diff --git a/couchpotato/core/plugins/movie/__init__.py b/couchpotato/core/plugins/movie/__init__.py deleted file mode 100644 index 4df29ad8..00000000 --- a/couchpotato/core/plugins/movie/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .main import MoviePlugin - -def start(): - return MoviePlugin() - -config = [] From 874655846c5814e3ba32f8e01e180a84d93dad1c Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 15 Aug 2013 23:38:14 +0200 Subject: [PATCH 095/209] Move movie plugin to media folder --- couchpotato/core/media/_base/__init__.py | 0 .../_base}/searcher/__init__.py | 52 +---- couchpotato/core/media/_base/searcher/main.py | 207 ++++++++++++++++++ .../core/media/movie/_base/static/movie.js | 6 +- .../core/media/movie/searcher/__init__.py | 60 +++++ .../{plugins => media/movie}/searcher/main.py | 198 +++-------------- couchpotato/core/plugins/automation/main.py | 2 +- couchpotato/core/plugins/dashboard/main.py | 4 +- couchpotato/core/plugins/renamer/main.py | 2 +- couchpotato/core/providers/base.py | 2 +- couchpotato/static/scripts/page/wanted.js | 4 +- 11 files changed, 306 insertions(+), 231 deletions(-) create mode 100644 couchpotato/core/media/_base/__init__.py rename couchpotato/core/{plugins => media/_base}/searcher/__init__.py (52%) create mode 100644 couchpotato/core/media/_base/searcher/main.py create mode 100644 couchpotato/core/media/movie/searcher/__init__.py rename couchpotato/core/{plugins => media/movie}/searcher/main.py (67%) diff --git a/couchpotato/core/media/_base/__init__.py b/couchpotato/core/media/_base/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/couchpotato/core/plugins/searcher/__init__.py b/couchpotato/core/media/_base/searcher/__init__.py similarity index 52% rename from couchpotato/core/plugins/searcher/__init__.py rename to couchpotato/core/media/_base/searcher/__init__.py index d925ae7d..f3d764d8 100644 --- a/couchpotato/core/plugins/searcher/__init__.py +++ b/couchpotato/core/media/_base/searcher/__init__.py @@ -11,8 +11,8 @@ config = [{ { 'tab': 'searcher', 'name': 'searcher', - 'label': 'Search', - 'description': 'Options for the searchers', + 'label': 'Basics', + 'description': 'General search options', 'options': [ { 'name': 'preferred_method', @@ -22,14 +22,6 @@ config = [{ 'type': 'dropdown', 'values': [('usenet & torrents', 'both'), ('usenet', 'nzb'), ('torrents', 'torrent')], }, - { - 'name': 'always_search', - 'default': False, - 'advanced': True, - 'type': 'bool', - 'label': 'Always search', - 'description': 'Search for movies even before there is a ETA. Enabling this will probably get you a lot of fakes.', - }, ], }, { 'tab': 'searcher', @@ -60,46 +52,6 @@ config = [{ 'description': 'Ignores releases that match any of these sets. (Works like explained above)' }, ], - }, { - 'tab': 'searcher', - 'name': 'cronjob', - 'label': 'Cronjob', - 'advanced': True, - 'description': 'Cron settings for the searcher see: APScheduler for details.', - 'options': [ - { - 'name': 'run_on_launch', - 'label': 'Run on launch', - 'advanced': True, - 'default': 0, - 'type': 'bool', - 'description': 'Force run the searcher after (re)start.', - }, - { - 'name': 'cron_day', - 'label': 'Day', - 'advanced': True, - 'default': '*', - 'type': 'string', - 'description': '*: Every day, */2: Every 2 days, 1: Every first of the month.', - }, - { - 'name': 'cron_hour', - 'label': 'Hour', - 'advanced': True, - 'default': random.randint(0, 23), - 'type': 'string', - 'description': '*: Every hour, */8: Every 8 hours, 3: At 3, midnight.', - }, - { - 'name': 'cron_minute', - 'label': 'Minute', - 'advanced': True, - 'default': random.randint(0, 59), - 'type': 'string', - 'description': "Just keep it random, so the providers don't get DDOSed by every CP user on a 'full' hour." - }, - ], }, ], }, { diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py new file mode 100644 index 00000000..772a8acf --- /dev/null +++ b/couchpotato/core/media/_base/searcher/main.py @@ -0,0 +1,207 @@ +from couchpotato import get_session +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.encoding import simplifyString, toUnicode +from couchpotato.core.helpers.variable import md5, getTitle +from couchpotato.core.logger import CPLog +from couchpotato.core.plugins.base import Plugin +from couchpotato.core.settings.model import Movie, Release, ReleaseInfo +from couchpotato.environment import Env +from inspect import ismethod, isfunction +import datetime +import re +import time +import traceback + +log = CPLog(__name__) + + +class Searcher(Plugin): + + def __init__(self): + addEvent('searcher.get_types', self.getSearchTypes) + addEvent('searcher.contains_other_quality', self.containsOtherQuality) + addEvent('searcher.correct_year', self.correctYear) + addEvent('searcher.correct_name', self.correctName) + addEvent('searcher.download', self.download) + + def download(self, data, movie, manual = False): + + # Test to see if any downloaders are enabled for this type + downloader_enabled = fireEvent('download.enabled', manual, data, single = True) + + if downloader_enabled: + + snatched_status = fireEvent('status.get', 'snatched', single = True) + + # Download movie to temp + filedata = None + if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): + filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) + if filedata == 'try_next': + return filedata + + download_result = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True) + log.debug('Downloader result: %s', download_result) + + if download_result: + try: + # Mark release as snatched + db = get_session() + rls = db.query(Release).filter_by(identifier = md5(data['url'])).first() + if rls: + renamer_enabled = Env.setting('enabled', 'renamer') + + done_status = fireEvent('status.get', 'done', single = True) + rls.status_id = done_status.get('id') if not renamer_enabled else snatched_status.get('id') + + # Save download-id info if returned + if isinstance(download_result, dict): + for key in download_result: + rls_info = ReleaseInfo( + identifier = 'download_%s' % key, + value = toUnicode(download_result.get(key)) + ) + rls.info.append(rls_info) + db.commit() + + log_movie = '%s (%s) in %s' % (getTitle(movie['library']), movie['library']['year'], rls.quality.label) + snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) + log.info(snatch_message) + fireEvent('movie.snatched', message = snatch_message, data = rls.to_dict()) + + # If renamer isn't used, mark movie done + if not renamer_enabled: + active_status = fireEvent('status.get', 'active', single = True) + done_status = fireEvent('status.get', 'done', single = True) + try: + if movie['status_id'] == active_status.get('id'): + for profile_type in movie['profile']['types']: + if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: + log.info('Renamer disabled, marking movie as finished: %s', log_movie) + + # Mark release done + rls.status_id = done_status.get('id') + rls.last_edit = int(time.time()) + db.commit() + + # Mark movie done + mvie = db.query(Movie).filter_by(id = movie['id']).first() + mvie.status_id = done_status.get('id') + mvie.last_edit = int(time.time()) + db.commit() + except: + log.error('Failed marking movie finished, renamer disabled: %s', traceback.format_exc()) + + except: + log.error('Failed marking movie finished: %s', traceback.format_exc()) + + return True + + log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('type', ''))) + + return False + + def getSearchTypes(self): + + download_types = fireEvent('download.enabled_types', merge = True) + provider_types = fireEvent('provider.enabled_types', merge = True) + + if download_types and len(list(set(provider_types) & set(download_types))) == 0: + log.error('There aren\'t any providers enabled for your downloader (%s). Check your settings.', ','.join(download_types)) + return [] + + for useless_provider in list(set(provider_types) - set(download_types)): + log.debug('Provider for "%s" enabled, but no downloader.', useless_provider) + + search_types = download_types + + if len(search_types) == 0: + log.error('There aren\'t any downloaders enabled. Please pick one in settings.') + return [] + + return search_types + + def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = {}): + + name = nzb['name'] + size = nzb.get('size', 0) + nzb_words = re.split('\W+', simplifyString(name)) + + qualities = fireEvent('quality.all', single = True) + + found = {} + for quality in qualities: + # Main in words + if quality['identifier'] in nzb_words: + found[quality['identifier']] = True + + # Alt in words + if list(set(nzb_words) & set(quality['alternative'])): + found[quality['identifier']] = True + + # Try guessing via quality tags + guess = fireEvent('quality.guess', [nzb.get('name')], single = True) + if guess: + found[guess['identifier']] = True + + # Hack for older movies that don't contain quality tag + year_name = fireEvent('scanner.name_year', name, single = True) + if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): + if size > 3000: # Assume dvdr + log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', (size)) + found['dvdr'] = True + else: # Assume dvdrip + log.info('Quality was missing in name, assuming it\'s a DVD-Rip based on the size: %s', (size)) + found['dvdrip'] = True + + # Allow other qualities + for allowed in preferred_quality.get('allow'): + if found.get(allowed): + del found[allowed] + + return not (found.get(preferred_quality['identifier']) and len(found) == 1) + + def correctYear(self, haystack, year, year_range): + + if not isinstance(haystack, (list, tuple, set)): + haystack = [haystack] + + for string in haystack: + + year_name = fireEvent('scanner.name_year', string, single = True) + + if year_name and ((year - year_range) <= year_name.get('year') <= (year + year_range)): + log.debug('Movie year matches range: %s looking for %s', (year_name.get('year'), year)) + return True + + log.debug('Movie year doesn\'t matche range: %s looking for %s', (year_name.get('year'), year)) + return False + + def correctName(self, check_name, movie_name): + + check_names = [check_name] + + # Match names between " + try: check_names.append(re.search(r'([\'"])[^\1]*\1', check_name).group(0)) + except: pass + + # Match longest name between [] + try: check_names.append(max(check_name.split('['), key = len)) + except: pass + + for check_name in list(set(check_names)): + check_movie = fireEvent('scanner.name_year', check_name, single = True) + + try: + check_words = filter(None, re.split('\W+', check_movie.get('name', ''))) + movie_words = filter(None, re.split('\W+', simplifyString(movie_name))) + + if len(check_words) > 0 and len(movie_words) > 0 and len(list(set(check_words) - set(movie_words))) == 0: + return True + except: + pass + + return False + +class SearchSetupError(Exception): + pass diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index f5b5a2d5..20956a0f 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -29,14 +29,14 @@ var Movie = new Class({ self.update.delay(2000, self, notification); }); - ['movie.busy', 'searcher.started'].each(function(listener){ + ['movie.busy', 'movie.searcher.started'].each(function(listener){ App.addEvent(listener+'.'+self.data.id, function(notification){ if(notification.data) self.busy(true) }); }) - App.addEvent('searcher.ended.'+self.data.id, function(notification){ + App.addEvent('movie.searcher.ended.'+self.data.id, function(notification){ if(notification.data) self.busy(false) }); @@ -53,7 +53,7 @@ var Movie = new Class({ // Remove events App.removeEvents('movie.update.'+self.data.id); - ['movie.busy', 'searcher.started'].each(function(listener){ + ['movie.busy', 'movie.searcher.started'].each(function(listener){ App.removeEvents(listener+'.'+self.data.id); }) }, diff --git a/couchpotato/core/media/movie/searcher/__init__.py b/couchpotato/core/media/movie/searcher/__init__.py new file mode 100644 index 00000000..2962caae --- /dev/null +++ b/couchpotato/core/media/movie/searcher/__init__.py @@ -0,0 +1,60 @@ +from .main import Searcher +import random + +def start(): + return Searcher() + +config = [{ + 'name': 'searcher', + 'order': 20, + 'groups': [ + { + 'tab': 'searcher', + 'name': 'movie_searcher', + 'label': 'Movie search', + 'description': 'Search options for movies', + 'advanced': True, + 'options': [ + { + 'name': 'always_search', + 'default': False, + 'type': 'bool', + 'label': 'Always search', + 'description': 'Search for movies even before there is a ETA. Enabling this will probably get you a lot of fakes.', + }, + { + 'name': 'run_on_launch', + 'label': 'Run on launch', + 'advanced': True, + 'default': 0, + 'type': 'bool', + 'description': 'Force run the searcher after (re)start.', + }, + { + 'name': 'cron_day', + 'label': 'Day', + 'advanced': True, + 'default': '*', + 'type': 'string', + 'description': '*: Every day, */2: Every 2 days, 1: Every first of the month. See APScheduler for details.', + }, + { + 'name': 'cron_hour', + 'label': 'Hour', + 'advanced': True, + 'default': random.randint(0, 23), + 'type': 'string', + 'description': '*: Every hour, */8: Every 8 hours, 3: At 3, midnight.', + }, + { + 'name': 'cron_minute', + 'label': 'Minute', + 'advanced': True, + 'default': random.randint(0, 59), + 'type': 'string', + 'description': "Just keep it random, so the providers don't get DDOSed by every CP user on a 'full' hour." + }, + ], + }, + ], +}] diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py similarity index 67% rename from couchpotato/core/plugins/searcher/main.py rename to couchpotato/core/media/movie/searcher/main.py index b55e7201..dbfd6794 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -3,13 +3,12 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import simplifyString, toUnicode from couchpotato.core.helpers.variable import md5, getTitle, splitString, \ - possibleTitles + possibleTitles, getImdb from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Movie, Release, ReleaseInfo from couchpotato.environment import Env from datetime import date -from inspect import ismethod, isfunction from sqlalchemy.exc import InterfaceError import datetime import random @@ -25,25 +24,24 @@ class Searcher(Plugin): in_progress = False def __init__(self): - addEvent('searcher.all', self.allMovies) - addEvent('searcher.single', self.single) - addEvent('searcher.correct_movie', self.correctMovie) - addEvent('searcher.download', self.download) - addEvent('searcher.try_next_release', self.tryNextRelease) - addEvent('searcher.could_be_released', self.couldBeReleased) + addEvent('movie.searcher.all', self.searchAll) + addEvent('movie.searcher.single', self.single) + addEvent('movie.searcher.correct_movie', self.correctMovie) + addEvent('movie.searcher.try_next_release', self.tryNextRelease) + addEvent('movie.searcher.could_be_released', self.couldBeReleased) - addApiView('searcher.try_next', self.tryNextReleaseView, docs = { + addApiView('movie.searcher.try_next', self.tryNextReleaseView, docs = { 'desc': 'Marks the snatched results as ignored and try the next best release', 'params': { 'id': {'desc': 'The id of the movie'}, }, }) - addApiView('searcher.full_search', self.allMoviesView, docs = { + addApiView('movie.searcher.full_search', self.searchAllView, docs = { 'desc': 'Starts a full search for all wanted movies', }) - addApiView('searcher.progress', self.getProgress, docs = { + addApiView('movie.searcher.progress', self.getProgress, docs = { 'desc': 'Get the progress of current full search', 'return': {'type': 'object', 'example': """{ 'progress': False || object, total & to_go, @@ -51,7 +49,7 @@ class Searcher(Plugin): }) if self.conf('run_on_launch'): - addEvent('app.load', self.allMovies) + addEvent('app.load', self.searchAll) addEvent('app.load', self.setCrons) addEvent('setting.save.searcher.cron_day.after', self.setCrons) @@ -59,16 +57,16 @@ class Searcher(Plugin): addEvent('setting.save.searcher.cron_minute.after', self.setCrons) def setCrons(self): - fireEvent('schedule.cron', 'searcher.all', self.allMovies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + fireEvent('schedule.cron', 'movie.searcher.all', self.searchAll, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) - def allMoviesView(self, **kwargs): + def searchAllView(self, **kwargs): in_progress = self.in_progress if not in_progress: - fireEventAsync('searcher.all') - fireEvent('notify.frontend', type = 'searcher.started', data = True, message = 'Full search started') + fireEventAsync('movie.searcher.all') + fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') else: - fireEvent('notify.frontend', type = 'searcher.already_started', data = True, message = 'Full search already in progress') + fireEvent('notify.frontend', type = 'movie.searcher.already_started', data = True, message = 'Full search already in progress') return { 'success': not in_progress @@ -80,7 +78,7 @@ class Searcher(Plugin): 'progress': self.in_progress } - def allMovies(self): + def searchAll(self): if self.in_progress: log.info('Search already in progress') @@ -101,7 +99,7 @@ class Searcher(Plugin): } try: - search_types = self.getSearchTypes() + search_types = fireEvent('searcher.get_types', single = True) for movie in movies: movie_dict = movie.to_dict({ @@ -136,7 +134,7 @@ class Searcher(Plugin): # Find out search type try: if not search_types: - search_types = self.getSearchTypes() + search_types = fireEvent('searcher.get_types', single = True) except SearchSetupError: return @@ -161,7 +159,7 @@ class Searcher(Plugin): fireEvent('movie.delete', movie['id'], single = True) return - fireEvent('notify.frontend', type = 'searcher.started.%s' % movie['id'], data = True, message = 'Searching for "%s"' % default_title) + fireEvent('notify.frontend', type = 'movie.searcher.started.%s' % movie['id'], data = True, message = 'Searching for "%s"' % default_title) ret = False @@ -253,7 +251,7 @@ class Searcher(Plugin): log.info('Ignored, score to low: %s', nzb['name']) continue - downloaded = self.download(data = nzb, movie = movie) + downloaded = fireEvent('searcher.download', data = nzb, movie = movie, single = True) if downloaded is True: ret = True break @@ -277,107 +275,10 @@ class Searcher(Plugin): if len(too_early_to_search) > 0: log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) - fireEvent('notify.frontend', type = 'searcher.ended.%s' % movie['id'], data = True) + fireEvent('notify.frontend', type = 'movie.searcher.ended.%s' % movie['id'], data = True) return ret - def download(self, data, movie, manual = False): - - # Test to see if any downloaders are enabled for this type - downloader_enabled = fireEvent('download.enabled', manual, data, single = True) - - if downloader_enabled: - - snatched_status = fireEvent('status.get', 'snatched', single = True) - - # Download movie to temp - filedata = None - if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): - filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) - if filedata == 'try_next': - return filedata - - download_result = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True) - log.debug('Downloader result: %s', download_result) - - if download_result: - try: - # Mark release as snatched - db = get_session() - rls = db.query(Release).filter_by(identifier = md5(data['url'])).first() - if rls: - renamer_enabled = Env.setting('enabled', 'renamer') - - done_status = fireEvent('status.get', 'done', single = True) - rls.status_id = done_status.get('id') if not renamer_enabled else snatched_status.get('id') - - # Save download-id info if returned - if isinstance(download_result, dict): - for key in download_result: - rls_info = ReleaseInfo( - identifier = 'download_%s' % key, - value = toUnicode(download_result.get(key)) - ) - rls.info.append(rls_info) - db.commit() - - log_movie = '%s (%s) in %s' % (getTitle(movie['library']), movie['library']['year'], rls.quality.label) - snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) - log.info(snatch_message) - fireEvent('movie.snatched', message = snatch_message, data = rls.to_dict()) - - # If renamer isn't used, mark movie done - if not renamer_enabled: - active_status = fireEvent('status.get', 'active', single = True) - done_status = fireEvent('status.get', 'done', single = True) - try: - if movie['status_id'] == active_status.get('id'): - for profile_type in movie['profile']['types']: - if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: - log.info('Renamer disabled, marking movie as finished: %s', log_movie) - - # Mark release done - rls.status_id = done_status.get('id') - rls.last_edit = int(time.time()) - db.commit() - - # Mark movie done - mvie = db.query(Movie).filter_by(id = movie['id']).first() - mvie.status_id = done_status.get('id') - mvie.last_edit = int(time.time()) - db.commit() - except: - log.error('Failed marking movie finished, renamer disabled: %s', traceback.format_exc()) - - except: - log.error('Failed marking movie finished: %s', traceback.format_exc()) - - return True - - log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('type', ''))) - - return False - - def getSearchTypes(self): - - download_types = fireEvent('download.enabled_types', merge = True) - provider_types = fireEvent('provider.enabled_types', merge = True) - - if download_types and len(list(set(provider_types) & set(download_types))) == 0: - log.error('There aren\'t any providers enabled for your downloader (%s). Check your settings.', ','.join(download_types)) - raise NoProviders - - for useless_provider in list(set(provider_types) - set(download_types)): - log.debug('Provider for "%s" enabled, but no downloader.', useless_provider) - - search_types = download_types - - if len(search_types) == 0: - log.error('There aren\'t any downloaders enabled. Please pick one in settings.') - raise NoDownloaders - - return search_types - def correctMovie(self, nzb = None, movie = None, quality = None, **kwargs): imdb_results = kwargs.get('imdb_results', False) @@ -430,7 +331,7 @@ class Searcher(Plugin): preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True) # Contains lower quality string - if self.containsOtherQuality(nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality): + if fireEvent('searcher.contains_other_quality', nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single = True): log.info2('Wrong: %s, looking for %s', (nzb['name'], quality['label'])) return False @@ -460,20 +361,20 @@ class Searcher(Plugin): return True # Check if nzb contains imdb link - if self.checkIMDB([nzb.get('description', '')], movie['library']['identifier']): + if getImdb(nzb.get('description', '')) == movie['library']['identifier']: return True for raw_title in movie['library']['titles']: for movie_title in possibleTitles(raw_title['title']): movie_words = re.split('\W+', simplifyString(movie_title)) - if self.correctName(nzb['name'], movie_title): + if fireEvent('searcher.correct_name', nzb['name'], movie_title, single = True): # if no IMDB link, at least check year range 1 - if len(movie_words) > 2 and self.correctYear([nzb['name']], movie['library']['year'], 1): + if len(movie_words) > 2 and fireEvent('searcher.correct_year', nzb['name'], movie['library']['year'], 1, single = True): return True # if no IMDB link, at least check year - if len(movie_words) <= 2 and self.correctYear([nzb['name']], movie['library']['year'], 0): + if len(movie_words) <= 2 and fireEvent('searcher.correct_year', nzb['name'], movie['library']['year'], 0, single = True): return True log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], movie_name, movie['library']['year'])) @@ -527,45 +428,6 @@ class Searcher(Plugin): return False - def correctYear(self, haystack, year, year_range): - - for string in haystack: - - year_name = fireEvent('scanner.name_year', string, single = True) - - if year_name and ((year - year_range) <= year_name.get('year') <= (year + year_range)): - log.debug('Movie year matches range: %s looking for %s', (year_name.get('year'), year)) - return True - - log.debug('Movie year doesn\'t matche range: %s looking for %s', (year_name.get('year'), year)) - return False - - def correctName(self, check_name, movie_name): - - check_names = [check_name] - - # Match names between " - try: check_names.append(re.search(r'([\'"])[^\1]*\1', check_name).group(0)) - except: pass - - # Match longest name between [] - try: check_names.append(max(check_name.split('['), key = len)) - except: pass - - for check_name in list(set(check_names)): - check_movie = fireEvent('scanner.name_year', check_name, single = True) - - try: - check_words = filter(None, re.split('\W+', check_movie.get('name', ''))) - movie_words = filter(None, re.split('\W+', simplifyString(movie_name))) - - if len(check_words) > 0 and len(movie_words) > 0 and len(list(set(check_words) - set(movie_words))) == 0: - return True - except: - pass - - return False - def couldBeReleased(self, is_pre_release, dates, year = None): now = int(time.time()) @@ -626,7 +488,7 @@ class Searcher(Plugin): movie_dict = fireEvent('movie.get', movie_id, single = True) log.info('Trying next release for: %s', getTitle(movie_dict['library'])) - fireEvent('searcher.single', movie_dict) + fireEvent('movie.searcher.single', movie_dict) return True @@ -636,9 +498,3 @@ class Searcher(Plugin): class SearchSetupError(Exception): pass - -class NoDownloaders(SearchSetupError): - pass - -class NoProviders(SearchSetupError): - pass diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 67bae1d1..80e12850 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -36,4 +36,4 @@ class Automation(Plugin): for movie_id in movie_ids: movie_dict = fireEvent('movie.get', movie_id, single = True) - fireEvent('searcher.single', movie_dict) + fireEvent('movie.searcher.single', movie_dict) diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index 939b4154..df6f9757 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -70,9 +70,9 @@ class Dashboard(Plugin): coming_soon = False # Theater quality - if pp.get('theater') and fireEvent('searcher.could_be_released', True, eta, movie.library.year, single = True): + if pp.get('theater') and fireEvent('movie.searcher.could_be_released', True, eta, movie.library.year, single = True): coming_soon = True - if pp.get('dvd') and fireEvent('searcher.could_be_released', False, eta, movie.library.year, single = True): + if pp.get('dvd') and fireEvent('movie.searcher.could_be_released', False, eta, movie.library.year, single = True): coming_soon = True # Skip if movie is snatched/downloaded/available diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 30ce4088..8d1a8186 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -715,7 +715,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) db.commit() if self.conf('next_on_failed'): - fireEvent('searcher.try_next_release', movie_id = rel.movie_id) + fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id) elif item['status'] == 'completed': log.info('Download of %s completed!', item['name']) if self.statusInfoComplete(item): diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index cb7b16da..d7ac7d16 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -257,7 +257,7 @@ class ResultList(list): new_result = self.fillResult(result) - is_correct_movie = fireEvent('searcher.correct_movie', + is_correct_movie = fireEvent('movie.searcher.correct_movie', nzb = new_result, movie = self.movie, quality = self.quality, imdb_results = self.kwargs.get('imdb_results', False), single = True) diff --git a/couchpotato/static/scripts/page/wanted.js b/couchpotato/static/scripts/page/wanted.js index eabd1465..41bd4bcd 100644 --- a/couchpotato/static/scripts/page/wanted.js +++ b/couchpotato/static/scripts/page/wanted.js @@ -40,7 +40,7 @@ Page.Wanted = new Class({ if(!self.search_in_progress){ - Api.request('searcher.full_search'); + Api.request('movie.searcher.full_search'); self.startProgressInterval(); } @@ -53,7 +53,7 @@ Page.Wanted = new Class({ var start_text = self.manual_search.get('text'); self.progress_interval = setInterval(function(){ if(self.search_progress && self.search_progress.running) return; - self.search_progress = Api.request('searcher.progress', { + self.search_progress = Api.request('movie.searcher.progress', { 'onComplete': function(json){ self.search_in_progress = true; if(!json.progress){ From 2824c55231b0cb4ab862321653d8000f004fe9ec Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 15 Aug 2013 23:46:55 +0200 Subject: [PATCH 096/209] Give moviesearcher a unique name --- couchpotato/core/media/movie/searcher/__init__.py | 4 ++-- couchpotato/core/media/movie/searcher/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/media/movie/searcher/__init__.py b/couchpotato/core/media/movie/searcher/__init__.py index 2962caae..791bff2e 100644 --- a/couchpotato/core/media/movie/searcher/__init__.py +++ b/couchpotato/core/media/movie/searcher/__init__.py @@ -1,8 +1,8 @@ -from .main import Searcher +from .main import MovieSearcher import random def start(): - return Searcher() + return MovieSearcher() config = [{ 'name': 'searcher', diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index dbfd6794..ced3bcc8 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -19,7 +19,7 @@ import traceback log = CPLog(__name__) -class Searcher(Plugin): +class MovieSearcher(Plugin): in_progress = False From f7da408f83b34725b1ddf4d494a6844d3460eab5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 16 Aug 2013 10:21:44 +0200 Subject: [PATCH 097/209] Searcher conf section --- couchpotato/core/media/movie/searcher/main.py | 14 ++++++++------ couchpotato/core/plugins/base.py | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index ced3bcc8..21803cc6 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -48,7 +48,7 @@ class MovieSearcher(Plugin): }"""}, }) - if self.conf('run_on_launch'): + if self.conf('run_on_launch', section = 'searcher'): addEvent('app.load', self.searchAll) addEvent('app.load', self.setCrons) @@ -57,7 +57,9 @@ class MovieSearcher(Plugin): addEvent('setting.save.searcher.cron_minute.after', self.setCrons) def setCrons(self): - fireEvent('schedule.cron', 'movie.searcher.all', self.searchAll, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + + fireEvent('schedule.cron', 'movie.searcher.all', self.searchAll, + day = self.conf('cron_day', section = 'searcher'), hour = self.conf('cron_hour', section = 'searcher'), minute = self.conf('cron_minute', section = 'searcher')) def searchAllView(self, **kwargs): @@ -164,7 +166,7 @@ class MovieSearcher(Plugin): ret = False for quality_type in movie['profile']['types']: - if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): + if not self.conf('always_search', section = 'searcher') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): too_early_to_search.append(quality_type['quality']['identifier']) continue @@ -191,7 +193,7 @@ class MovieSearcher(Plugin): if len(sorted_results) == 0: log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label'])) - download_preference = self.conf('preferred_method') + download_preference = self.conf('preferred_method', section = 'searcher') if download_preference != 'both': sorted_results = sorted(sorted_results, key = lambda k: k['type'][:3], reverse = (download_preference == 'torrent')) @@ -294,7 +296,7 @@ class MovieSearcher(Plugin): nzb_words = re.split('\W+', nzb_name) # Make sure it has required words - required_words = splitString(self.conf('required_words').lower()) + required_words = splitString(self.conf('required_words', section = 'searcher').lower()) try: required_words = list(set(required_words + splitString(movie['category']['required'].lower()))) except: pass @@ -308,7 +310,7 @@ class MovieSearcher(Plugin): return False # Ignore releases - ignored_words = splitString(self.conf('ignored_words').lower()) + ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower()))) except: pass diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index aa2a99e9..f81d8a1b 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -40,8 +40,8 @@ class Plugin(object): addEvent('plugin.running', self.isRunning) self._running = [] - def conf(self, attr, value = None, default = None): - return Env.setting(attr, self.getName().lower(), value = value, default = default) + def conf(self, attr, value = None, default = None, section = None): + return Env.setting(attr, section = section if section else self.getName().lower(), value = value, default = default) def getName(self): return self.__class__.__name__ From 91856f11597de2614756de4f39bc2a723b3292b5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 16 Aug 2013 16:52:12 +0200 Subject: [PATCH 098/209] Searcher base Re-usable cronjob code --- couchpotato/core/media/__init__.py | 9 +- couchpotato/core/media/_base/searcher/base.py | 51 +++++++++++ couchpotato/core/media/_base/searcher/main.py | 29 +++++- couchpotato/core/media/movie/__init__.py | 6 ++ couchpotato/core/media/movie/_base/main.py | 9 +- .../core/media/movie/searcher/__init__.py | 7 +- couchpotato/core/media/movie/searcher/main.py | 88 +++---------------- couchpotato/static/scripts/page/wanted.js | 4 +- 8 files changed, 112 insertions(+), 91 deletions(-) create mode 100644 couchpotato/core/media/_base/searcher/base.py diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index 2d339e5a..8187f98a 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -2,16 +2,13 @@ from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -log = CPLog(__name__) - class MediaBase(Plugin): - identifier = None - - def __init__(self): + _type = None + def initType(self): addEvent('media.types', self.getType) def getType(self): - return self.identifier + return self._type diff --git a/couchpotato/core/media/_base/searcher/base.py b/couchpotato/core/media/_base/searcher/base.py new file mode 100644 index 00000000..ab294397 --- /dev/null +++ b/couchpotato/core/media/_base/searcher/base.py @@ -0,0 +1,51 @@ +from couchpotato.api import addApiView +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.logger import CPLog +from couchpotato.core.plugins.base import Plugin + +log = CPLog(__name__) + + +class SearcherBase(Plugin): + + in_progress = False + + def __init__(self): + super(SearcherBase, self).__init__() + + + addEvent('searcher.progress', self.getProgress) + addEvent('%s.searcher.progress' % self.getType(), self.getProgress) + + self.initCron() + + + """ Set the searcher cronjob + Make sure to reset cronjob after setting has changed + + """ + def initCron(self): + + _type = self.getType() + + def setCrons(): + + fireEvent('schedule.cron', '%s.searcher.all' % _type, self.searchAll, + day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + + addEvent('app.load', setCrons) + addEvent('setting.save.%s_searcher.cron_day.after' % _type, setCrons) + addEvent('setting.save.%s_searcher.cron_hour.after' % _type, setCrons) + addEvent('setting.save.%s_searcher.cron_minute.after' % _type, setCrons) + + + """ Return progress of current searcher + + """ + def getProgress(self, **kwargs): + + progress = {} + progress[self.getType()] = self.in_progress + + return progress + diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 772a8acf..55dfe3e3 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -1,9 +1,10 @@ from couchpotato import get_session +from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import simplifyString, toUnicode from couchpotato.core.helpers.variable import md5, getTitle from couchpotato.core.logger import CPLog -from couchpotato.core.plugins.base import Plugin +from couchpotato.core.media._base.searcher.base import SearcherBase from couchpotato.core.settings.model import Movie, Release, ReleaseInfo from couchpotato.environment import Env from inspect import ismethod, isfunction @@ -15,7 +16,7 @@ import traceback log = CPLog(__name__) -class Searcher(Plugin): +class Searcher(SearcherBase): def __init__(self): addEvent('searcher.get_types', self.getSearchTypes) @@ -24,6 +25,30 @@ class Searcher(Plugin): addEvent('searcher.correct_name', self.correctName) addEvent('searcher.download', self.download) + addApiView('searcher.full_search', self.searchAllView, docs = { + 'desc': 'Starts a full search for all media', + }) + + addApiView('searcher.progress', self.getProgressForAll, docs = { + 'desc': 'Get the progress of all media searches', + 'return': {'type': 'object', 'example': """{ + 'movie': False || object, total & to_go, + 'show': False || object, total & to_go, +}"""}, + }) + + def searchAllView(self): + + results = {} + for _type in fireEvent('media.types'): + results[_type] = fireEvent('%s.searcher.all_view' % _type) + + return results + + def getProgressForAll(self): + progress = fireEvent('searcher.progress', merge = True) + return progress + def download(self, data, movie, manual = False): # Test to see if any downloaders are enabled for this type diff --git a/couchpotato/core/media/movie/__init__.py b/couchpotato/core/media/movie/__init__.py index e69de29b..898529c1 100644 --- a/couchpotato/core/media/movie/__init__.py +++ b/couchpotato/core/media/movie/__init__.py @@ -0,0 +1,6 @@ +from couchpotato.core.media import MediaBase + + +class MovieTypeBase(MediaBase): + + _type = 'movie' diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index fc537ef0..448e9985 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -4,7 +4,7 @@ from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode, simplifyString from couchpotato.core.helpers.variable import getImdb, splitString, tryInt from couchpotato.core.logger import CPLog -from couchpotato.core.media import MediaBase +from couchpotato.core.media.movie import MovieTypeBase from couchpotato.core.settings.model import Library, LibraryTitle, Movie, \ Release from couchpotato.environment import Env @@ -16,9 +16,7 @@ import time log = CPLog(__name__) -class MovieBase(MediaBase): - - identifier = 'movie' +class MovieBase(MovieTypeBase): default_dict = { 'profile': {'types': {'quality': {}}}, @@ -29,7 +27,10 @@ class MovieBase(MediaBase): } def __init__(self): + + # Initialize this type super(MovieBase, self).__init__() + self.initType() addApiView('movie.search', self.search, docs = { 'desc': 'Search the movie providers for a movie', diff --git a/couchpotato/core/media/movie/searcher/__init__.py b/couchpotato/core/media/movie/searcher/__init__.py index 791bff2e..bf6ff218 100644 --- a/couchpotato/core/media/movie/searcher/__init__.py +++ b/couchpotato/core/media/movie/searcher/__init__.py @@ -5,7 +5,7 @@ def start(): return MovieSearcher() config = [{ - 'name': 'searcher', + 'name': 'moviesearcher', 'order': 20, 'groups': [ { @@ -18,12 +18,14 @@ config = [{ { 'name': 'always_search', 'default': False, + 'migrate_from': 'searcher', 'type': 'bool', 'label': 'Always search', 'description': 'Search for movies even before there is a ETA. Enabling this will probably get you a lot of fakes.', }, { 'name': 'run_on_launch', + 'migrate_from': 'searcher', 'label': 'Run on launch', 'advanced': True, 'default': 0, @@ -32,6 +34,7 @@ config = [{ }, { 'name': 'cron_day', + 'migrate_from': 'searcher', 'label': 'Day', 'advanced': True, 'default': '*', @@ -40,6 +43,7 @@ config = [{ }, { 'name': 'cron_hour', + 'migrate_from': 'searcher', 'label': 'Hour', 'advanced': True, 'default': random.randint(0, 23), @@ -48,6 +52,7 @@ config = [{ }, { 'name': 'cron_minute', + 'migrate_from': 'searcher', 'label': 'Minute', 'advanced': True, 'default': random.randint(0, 59), diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 21803cc6..30f27d7f 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -5,12 +5,12 @@ from couchpotato.core.helpers.encoding import simplifyString, toUnicode from couchpotato.core.helpers.variable import md5, getTitle, splitString, \ possibleTitles, getImdb from couchpotato.core.logger import CPLog -from couchpotato.core.plugins.base import Plugin +from couchpotato.core.media._base.searcher.base import SearcherBase +from couchpotato.core.media.movie import MovieTypeBase from couchpotato.core.settings.model import Movie, Release, ReleaseInfo from couchpotato.environment import Env from datetime import date from sqlalchemy.exc import InterfaceError -import datetime import random import re import time @@ -19,12 +19,15 @@ import traceback log = CPLog(__name__) -class MovieSearcher(Plugin): +class MovieSearcher(SearcherBase, MovieTypeBase): in_progress = False def __init__(self): + super(MovieSearcher, self).__init__() + addEvent('movie.searcher.all', self.searchAll) + addEvent('movie.searcher.all_view', self.searchAllView) addEvent('movie.searcher.single', self.single) addEvent('movie.searcher.correct_movie', self.correctMovie) addEvent('movie.searcher.try_next_release', self.tryNextRelease) @@ -48,45 +51,26 @@ class MovieSearcher(Plugin): }"""}, }) - if self.conf('run_on_launch', section = 'searcher'): + if self.conf('run_on_launch'): addEvent('app.load', self.searchAll) - addEvent('app.load', self.setCrons) - addEvent('setting.save.searcher.cron_day.after', self.setCrons) - addEvent('setting.save.searcher.cron_hour.after', self.setCrons) - addEvent('setting.save.searcher.cron_minute.after', self.setCrons) - - def setCrons(self): - - fireEvent('schedule.cron', 'movie.searcher.all', self.searchAll, - day = self.conf('cron_day', section = 'searcher'), hour = self.conf('cron_hour', section = 'searcher'), minute = self.conf('cron_minute', section = 'searcher')) - def searchAllView(self, **kwargs): - in_progress = self.in_progress - if not in_progress: - fireEventAsync('movie.searcher.all') - fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') - else: - fireEvent('notify.frontend', type = 'movie.searcher.already_started', data = True, message = 'Full search already in progress') + fireEventAsync('movie.searcher.all') return { - 'success': not in_progress - } - - def getProgress(self, **kwargs): - - return { - 'progress': self.in_progress + 'success': not self.in_progress } def searchAll(self): if self.in_progress: log.info('Search already in progress') + fireEvent('notify.frontend', type = 'movie.searcher.already_started', data = True, message = 'Full search already in progress') return self.in_progress = True + fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') db = get_session() @@ -166,7 +150,7 @@ class MovieSearcher(Plugin): ret = False for quality_type in movie['profile']['types']: - if not self.conf('always_search', section = 'searcher') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): + if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): too_early_to_search.append(quality_type['quality']['identifier']) continue @@ -382,54 +366,6 @@ class MovieSearcher(Plugin): log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], movie_name, movie['library']['year'])) return False - def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = {}): - - name = nzb['name'] - size = nzb.get('size', 0) - nzb_words = re.split('\W+', simplifyString(name)) - - qualities = fireEvent('quality.all', single = True) - - found = {} - for quality in qualities: - # Main in words - if quality['identifier'] in nzb_words: - found[quality['identifier']] = True - - # Alt in words - if list(set(nzb_words) & set(quality['alternative'])): - found[quality['identifier']] = True - - # Try guessing via quality tags - guess = fireEvent('quality.guess', [nzb.get('name')], single = True) - if guess: - found[guess['identifier']] = True - - # Hack for older movies that don't contain quality tag - year_name = fireEvent('scanner.name_year', name, single = True) - if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): - if size > 3000: # Assume dvdr - log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', (size)) - found['dvdr'] = True - else: # Assume dvdrip - log.info('Quality was missing in name, assuming it\'s a DVD-Rip based on the size: %s', (size)) - found['dvdrip'] = True - - # Allow other qualities - for allowed in preferred_quality.get('allow'): - if found.get(allowed): - del found[allowed] - - return not (found.get(preferred_quality['identifier']) and len(found) == 1) - - def checkIMDB(self, haystack, imdbId): - - for string in haystack: - if 'imdb.com/title/' + imdbId in string: - return True - - return False - def couldBeReleased(self, is_pre_release, dates, year = None): now = int(time.time()) diff --git a/couchpotato/static/scripts/page/wanted.js b/couchpotato/static/scripts/page/wanted.js index 41bd4bcd..6adffbd5 100644 --- a/couchpotato/static/scripts/page/wanted.js +++ b/couchpotato/static/scripts/page/wanted.js @@ -56,13 +56,13 @@ Page.Wanted = new Class({ self.search_progress = Api.request('movie.searcher.progress', { 'onComplete': function(json){ self.search_in_progress = true; - if(!json.progress){ + if(!json.movie){ clearInterval(self.progress_interval); self.search_in_progress = false; self.manual_search.set('text', start_text); } else { - var progress = json.progress; + var progress = json.movie; self.manual_search.set('text', 'Searching.. (' + (((progress.total-progress.to_go)/progress.total)*100).round() + '%)'); } } From 4d5ba65254fe85104236858bbee795a4cbc9e2e1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 16 Aug 2013 17:23:40 +0200 Subject: [PATCH 099/209] Migrate options --- couchpotato/core/settings/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index cdf58aa2..e08adb87 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -2,7 +2,7 @@ from __future__ import with_statement from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import isInt, toUnicode -from couchpotato.core.helpers.variable import mergeDicts, tryInt +from couchpotato.core.helpers.variable import mergeDicts, tryInt, tryFloat from couchpotato.core.settings.model import Properties import ConfigParser import os.path @@ -77,9 +77,17 @@ class Settings(object): def registerDefaults(self, section_name, options = {}, save = True): self.addSection(section_name) + for option_name, option in options.iteritems(): self.setDefault(section_name, option_name, option.get('default', '')) + # Migrate old settings from old location to the new location + if option.get('migrate_from'): + if self.p.has_option(option.get('migrate_from'), option_name): + previous_value = self.p.get(option.get('migrate_from'), option_name) + self.p.set(section_name, option_name, previous_value) + self.p.remove_option(option.get('migrate_from'), option_name) + if option.get('type'): self.setType(section_name, option_name, option.get('type')) @@ -122,7 +130,7 @@ class Settings(object): try: return self.p.getfloat(section, option) except: - return tryInt(self.p.get(section, option)) + return tryFloat(self.p.get(section, option)) def getUnicode(self, section, option): value = self.p.get(section, option).decode('unicode_escape') From c73ed8a4c56cd448c863f20e4c09b4282b62f116 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 16 Aug 2013 20:05:30 +0200 Subject: [PATCH 100/209] Add multiple categories for BRRIP on TPB. fix #2025 --- couchpotato/core/providers/torrent/thepiratebay/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index 10608157..82cbbe9e 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -15,12 +15,13 @@ class ThePirateBay(TorrentMagnetProvider): urls = { 'detail': '%s/torrent/%s', - 'search': '%s/search/%s/%s/7/%d' + 'search': '%s/search/%s/%s/7/%s' } cat_ids = [ ([207], ['720p', '1080p']), - ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']), + ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), + ([201, 207], ['brrip']), ([202], ['dvdr']) ] @@ -50,10 +51,11 @@ class ThePirateBay(TorrentMagnetProvider): page = 0 total_pages = 1 + cats = self.getCatId(quality['identifier']) while page < total_pages: - search_url = self.urls['search'] % (self.getDomain(), tryUrlencode('"%s" %s' % (title, movie['library']['year'])), page, self.getCatId(quality['identifier'])[0]) + search_url = self.urls['search'] % (self.getDomain(), tryUrlencode('"%s" %s' % (title, movie['library']['year'])), page, ','.join(str(x) for x in cats)) page += 1 data = self.getHTMLData(search_url) From 3af6623a919769bfde5e407bfd60fff7c777ba93 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 18 Aug 2013 00:22:36 +0200 Subject: [PATCH 101/209] Move registerPlugin to __new__ magic --- couchpotato/core/loader.py | 7 +------ couchpotato/core/plugins/base.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index 8aef89a4..9d04632b 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -111,12 +111,7 @@ class Loader(object): def loadPlugins(self, module, name): try: - klass = module.start() - klass.registerPlugin() - - if klass and getattr(klass, 'auto_register_static'): - klass.registerStatic(module.__file__) - + module.start() return True except Exception, e: log.error('Failed loading plugin "%s": %s', (module.__file__, traceback.format_exc())) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index f81d8a1b..9e97c804 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -12,6 +12,7 @@ from urlparse import urlparse import cookielib import glob import gzip +import inspect import math import os.path import re @@ -35,11 +36,20 @@ class Plugin(object): http_failed_request = {} http_failed_disabled = {} + def __new__(typ, *args, **kwargs): + new_plugin = super(Plugin, typ).__new__(typ, *args, **kwargs) + new_plugin.registerPlugin() + + return new_plugin + def registerPlugin(self): addEvent('app.do_shutdown', self.doShutdown) addEvent('plugin.running', self.isRunning) self._running = [] + if self.auto_register_static: + self.registerStatic(inspect.getfile(self.__class__)) + def conf(self, attr, value = None, default = None, section = None): return Env.setting(attr, section = section if section else self.getName().lower(), value = value, default = default) From 62b571d5f1f232dab633201b1ed0f256fe5e1a38 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 18 Aug 2013 11:44:00 +0200 Subject: [PATCH 102/209] Rename type to protocol --- couchpotato/core/downloaders/base.py | 20 ++++++------- .../core/downloaders/blackhole/main.py | 24 ++++++++-------- couchpotato/core/downloaders/nzbget/main.py | 2 +- .../core/downloaders/nzbvortex/main.py | 2 +- .../core/downloaders/pneumatic/main.py | 4 +-- couchpotato/core/downloaders/sabnzbd/main.py | 2 +- couchpotato/core/downloaders/synology/main.py | 28 +++++++++---------- .../core/downloaders/transmission/main.py | 8 +++--- couchpotato/core/downloaders/utorrent/main.py | 10 +++---- couchpotato/core/media/_base/searcher/main.py | 22 +++++++-------- couchpotato/core/media/movie/searcher/main.py | 18 ++++++------ couchpotato/core/providers/base.py | 17 ++++++----- couchpotato/core/providers/nzb/base.py | 3 +- couchpotato/core/providers/torrent/base.py | 4 +-- 14 files changed, 84 insertions(+), 80 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index b820b9ff..cc0d59ea 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -11,7 +11,7 @@ log = CPLog(__name__) class Downloader(Provider): - type = [] + protocol = [] http_time_between_calls = 0 torrent_sources = [ @@ -36,16 +36,16 @@ class Downloader(Provider): def __init__(self): addEvent('download', self._download) addEvent('download.enabled', self._isEnabled) - addEvent('download.enabled_types', self.getEnabledDownloadType) + addEvent('download.enabled_protocols', self.getEnabledProtocol) addEvent('download.status', self._getAllDownloadStatus) addEvent('download.remove_failed', self._removeFailed) addEvent('download.pause', self._pause) addEvent('download.process_complete', self._processComplete) - def getEnabledDownloadType(self): - for download_type in self.type: - if self.isEnabled(manual = True, data = {'type': download_type}): - return self.type + def getEnabledProtocol(self): + for download_protocol in self.protocol: + if self.isEnabled(manual = True, data = {'protocol': download_protocol}): + return self.protocol return [] @@ -91,11 +91,11 @@ class Downloader(Provider): def processComplete(self, item, delete_files): return - def isCorrectType(self, item_type): - is_correct = item_type in self.type + def isCorrectProtocol(self, item_protocol): + is_correct = item_protocol in self.protocol if not is_correct: - log.debug("Downloader doesn't support this type") + log.debug("Downloader doesn't support this protocol") return is_correct @@ -140,7 +140,7 @@ class Downloader(Provider): d_manual = self.conf('manual', default = False) return super(Downloader, self).isEnabled() and \ ((d_manual and manual) or (d_manual is False)) and \ - (not data or self.isCorrectType(data.get('type'))) + (not data or self.isCorrectProtocol(data.get('protocol'))) def _pause(self, item, pause = True): if self.isDisabled(manual = True, data = {}): diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index 82b07276..9d2a5261 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -10,20 +10,20 @@ log = CPLog(__name__) class Blackhole(Downloader): - type = ['nzb', 'torrent', 'torrent_magnet'] + protocol = ['nzb', 'torrent', 'torrent_magnet'] def download(self, data = {}, movie = {}, filedata = None): directory = self.conf('directory') if not directory or not os.path.isdir(directory): - log.error('No directory set for blackhole %s download.', data.get('type')) + log.error('No directory set for blackhole %s download.', data.get('protocol')) else: try: if not filedata or len(filedata) < 50: try: - if data.get('type') == 'torrent_magnet': + if data.get('protocol') == 'torrent_magnet': filedata = self.magnetToTorrent(data.get('url')) - data['type'] = 'torrent' + data['protocol'] = 'torrent' except: log.error('Failed download torrent via magnet url: %s', traceback.format_exc()) @@ -35,7 +35,7 @@ class Blackhole(Downloader): try: if not os.path.isfile(fullPath): - log.info('Downloading %s to %s.', (data.get('type'), fullPath)) + log.info('Downloading %s to %s.', (data.get('protocol'), fullPath)) with open(fullPath, 'wb') as f: f.write(filedata) os.chmod(fullPath, Env.getPermission('file')) @@ -54,20 +54,20 @@ class Blackhole(Downloader): return False - def getEnabledDownloadType(self): + def getEnabledProtocol(self): if self.conf('use_for') == 'both': - return super(Blackhole, self).getEnabledDownloadType() + return super(Blackhole, self).getEnabledProtocol() elif self.conf('use_for') == 'torrent': return ['torrent', 'torrent_magnet'] else: return ['nzb'] def isEnabled(self, manual, data = {}): - for_type = ['both'] - if data and 'torrent' in data.get('type'): - for_type.append('torrent') + for_protocol = ['both'] + if data and 'torrent' in data.get('protocol'): + for_protocol.append('torrent') elif data: - for_type.append(data.get('type')) + for_protocol.append(data.get('protocol')) return super(Blackhole, self).isEnabled(manual, data) and \ - ((self.conf('use_for') in for_type)) + ((self.conf('use_for') in for_protocol)) diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index ef9d4efa..ba3b2618 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -15,7 +15,7 @@ log = CPLog(__name__) class NZBGet(Downloader): - type = ['nzb'] + protocol = ['nzb'] url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc' diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index 805c4598..b8817aca 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -19,7 +19,7 @@ log = CPLog(__name__) class NZBVortex(Downloader): - type = ['nzb'] + protocol = ['nzb'] api_level = None session_id = None diff --git a/couchpotato/core/downloaders/pneumatic/main.py b/couchpotato/core/downloaders/pneumatic/main.py index 5564dca7..25923e08 100644 --- a/couchpotato/core/downloaders/pneumatic/main.py +++ b/couchpotato/core/downloaders/pneumatic/main.py @@ -9,7 +9,7 @@ log = CPLog(__name__) class Pneumatic(Downloader): - type = ['nzb'] + protocol = ['nzb'] strm_syntax = 'plugin://plugin.program.pneumatic/?mode=strm&type=add_file&nzb=%s&nzbname=%s' def download(self, data = {}, movie = {}, filedata = None): @@ -27,7 +27,7 @@ class Pneumatic(Downloader): try: if not os.path.isfile(fullPath): - log.info('Downloading %s to %s.', (data.get('type'), fullPath)) + log.info('Downloading %s to %s.', (data.get('protocol'), fullPath)) with open(fullPath, 'wb') as f: f.write(filedata) diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 776749be..468f30b3 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -13,7 +13,7 @@ log = CPLog(__name__) class Sabnzbd(Downloader): - type = ['nzb'] + protocol = ['nzb'] def download(self, data = {}, movie = {}, filedata = None): diff --git a/couchpotato/core/downloaders/synology/main.py b/couchpotato/core/downloaders/synology/main.py index 87212749..362577fa 100644 --- a/couchpotato/core/downloaders/synology/main.py +++ b/couchpotato/core/downloaders/synology/main.py @@ -9,13 +9,13 @@ log = CPLog(__name__) class Synology(Downloader): - type = ['nzb', 'torrent', 'torrent_magnet'] + protocol = ['nzb', 'torrent', 'torrent_magnet'] log = CPLog(__name__) def download(self, data, movie, filedata = None): response = False - log.error('Sending "%s" (%s) to Synology.', (data['name'], data['type'])) + log.error('Sending "%s" (%s) to Synology.', (data['name'], data['protocol'])) # Load host from config and split out port. host = self.conf('host').split(':') @@ -26,38 +26,38 @@ class Synology(Downloader): try: # Send request to Synology srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password')) - if data['type'] == 'torrent_magnet': + if data['protocol'] == 'torrent_magnet': log.info('Adding torrent URL %s', data['url']) response = srpc.create_task(url = data['url']) - elif data['type'] in ['nzb', 'torrent']: - log.info('Adding %s' % data['type']) + elif data['protocol'] in ['nzb', 'torrent']: + log.info('Adding %s' % data['protocol']) if not filedata: - log.error('No %s data found' % data['type']) + log.error('No %s data found' % data['protocol']) else: - filename = data['name'] + '.' + data['type'] + filename = data['name'] + '.' + data['protocol'] response = srpc.create_task(filename = filename, filedata = filedata) except Exception, err: log.error('Exception while adding torrent: %s', err) finally: return response - def getEnabledDownloadType(self): + def getEnabledProtocol(self): if self.conf('use_for') == 'both': - return super(Synology, self).getEnabledDownloadType() + return super(Synology, self).getEnabledProtocol() elif self.conf('use_for') == 'torrent': return ['torrent', 'torrent_magnet'] else: return ['nzb'] def isEnabled(self, manual, data = {}): - for_type = ['both'] - if data and 'torrent' in data.get('type'): - for_type.append('torrent') + for_protocol = ['both'] + if data and 'torrent' in data.get('protocol'): + for_protocol.append('torrent') elif data: - for_type.append(data.get('type')) + for_protocol.append(data.get('protocol')) return super(Synology, self).isEnabled(manual, data) and\ - ((self.conf('use_for') in for_type)) + ((self.conf('use_for') in for_protocol)) class SynologyRPC(object): diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index a619d411..89c7099b 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -16,7 +16,7 @@ log = CPLog(__name__) class Transmission(Downloader): - type = ['torrent', 'torrent_magnet'] + protocol = ['torrent', 'torrent_magnet'] log = CPLog(__name__) trpc = None @@ -34,12 +34,12 @@ class Transmission(Downloader): def download(self, data, movie, filedata = None): - log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) + log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('protocol'))) if not self.connect(): return False - if not filedata and data.get('type') == 'torrent': + if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False @@ -64,7 +64,7 @@ class Transmission(Downloader): torrent_params['seedIdleMode'] = 1 # Send request to Transmission - if data.get('type') == 'torrent_magnet': + if data.get('protocol') == 'torrent_magnet': remote_torrent = self.trpc.add_torrent_uri(data.get('url'), arguments = params) torrent_params['trackerAdd'] = self.torrent_trackers else: diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index be6ff107..d5cc64f7 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -20,7 +20,7 @@ log = CPLog(__name__) class uTorrent(Downloader): - type = ['torrent', 'torrent_magnet'] + protocol = ['torrent', 'torrent_magnet'] utorrent_api = None def connect(self): @@ -36,7 +36,7 @@ class uTorrent(Downloader): def download(self, data, movie, filedata = None): - log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('type'))) + log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('protocol'))) if not self.connect(): return False @@ -63,11 +63,11 @@ class uTorrent(Downloader): if self.conf('label'): torrent_params['label'] = self.conf('label') - if not filedata and data.get('type') == 'torrent': + if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False - if data.get('type') == 'torrent_magnet': + if data.get('protocol') == 'torrent_magnet': torrent_hash = re.findall('urn:btih:([\w]{32,40})', data.get('url'))[0].upper() torrent_params['trackers'] = '%0D%0A%0D%0A'.join(self.torrent_trackers) else: @@ -88,7 +88,7 @@ class uTorrent(Downloader): torrent_hash = b16encode(b32decode(torrent_hash)) # Send request to uTorrent - if data.get('type') == 'torrent_magnet': + if data.get('protocol') == 'torrent_magnet': self.utorrent_api.add_torrent_uri(data.get('url')) else: self.utorrent_api.add_torrent_file(torrent_filename, filedata) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 55dfe3e3..7d84a58c 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -19,7 +19,7 @@ log = CPLog(__name__) class Searcher(SearcherBase): def __init__(self): - addEvent('searcher.get_types', self.getSearchTypes) + addEvent('searcher.protocols', self.getSearchProtocols) addEvent('searcher.contains_other_quality', self.containsOtherQuality) addEvent('searcher.correct_year', self.correctYear) addEvent('searcher.correct_name', self.correctName) @@ -122,29 +122,29 @@ class Searcher(SearcherBase): return True - log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('type', ''))) + log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol', ''))) return False - def getSearchTypes(self): + def getSearchProtocols(self): - download_types = fireEvent('download.enabled_types', merge = True) - provider_types = fireEvent('provider.enabled_types', merge = True) + download_protocols = fireEvent('download.enabled_protocols', merge = True) + provider_protocols = fireEvent('provider.enabled_protocols', merge = True) - if download_types and len(list(set(provider_types) & set(download_types))) == 0: - log.error('There aren\'t any providers enabled for your downloader (%s). Check your settings.', ','.join(download_types)) + if download_protocols and len(list(set(provider_protocols) & set(download_protocols))) == 0: + log.error('There aren\'t any providers enabled for your downloader (%s). Check your settings.', ','.join(download_protocols)) return [] - for useless_provider in list(set(provider_types) - set(download_types)): + for useless_provider in list(set(provider_protocols) - set(download_protocols)): log.debug('Provider for "%s" enabled, but no downloader.', useless_provider) - search_types = download_types + search_protocols = download_protocols - if len(search_types) == 0: + if len(search_protocols) == 0: log.error('There aren\'t any downloaders enabled. Please pick one in settings.') return [] - return search_types + return search_protocols def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = {}): diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 30f27d7f..8fb4acf1 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -85,7 +85,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): } try: - search_types = fireEvent('searcher.get_types', single = True) + search_protocols = fireEvent('searcher.protocols', single = True) for movie in movies: movie_dict = movie.to_dict({ @@ -97,7 +97,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): }) try: - self.single(movie_dict, search_types) + self.single(movie_dict, search_protocols) except IndexError: log.error('Forcing library update for %s, if you see this often, please report: %s', (movie_dict['library']['identifier'], traceback.format_exc())) fireEvent('library.update', movie_dict['library']['identifier'], force = True) @@ -115,12 +115,12 @@ class MovieSearcher(SearcherBase, MovieTypeBase): self.in_progress = False - def single(self, movie, search_types = None): + def single(self, movie, search_protocols = None): # Find out search type try: - if not search_types: - search_types = fireEvent('searcher.get_types', single = True) + if not search_protocols: + search_protocols = fireEvent('searcher.protocols', single = True) except SearchSetupError: return @@ -168,10 +168,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True) results = [] - for search_type in search_types: - type_results = fireEvent('%s.search' % search_type, movie, quality, merge = True) - if type_results: - results += type_results + for search_protocol in search_protocols: + protocol_results = fireEvent('provider.search.%s.movie' % search_protocol, movie, quality, merge = True) + if protocol_results: + results += protocol_results sorted_results = sorted(results, key = lambda k: k['score'], reverse = True) if len(sorted_results) == 0: diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index d7ac7d16..08b4c6e5 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -19,7 +19,7 @@ log = CPLog(__name__) class Provider(Plugin): - type = None # movie, nzb, torrent, subtitle, trailer + type = None # movie, show, subtitle, trailer, ... http_time_between_calls = 10 # Default timeout for url requests last_available_check = {} @@ -79,7 +79,10 @@ class Provider(Plugin): class YarrProvider(Provider): - cat_ids = [] + protocol = None # nzb, torrent, torrent_magnet + + cat_ids = {} + cat_backup_id = None sizeGb = ['gb', 'gib'] sizeMb = ['mb', 'mib'] @@ -89,14 +92,13 @@ class YarrProvider(Provider): last_login_check = 0 def __init__(self): - addEvent('provider.enabled_types', self.getEnabledProviderType) + addEvent('provider.enabled_protocols', self.getEnabledProtocol) addEvent('provider.belongs_to', self.belongsTo) - addEvent('yarr.search', self.search) - addEvent('%s.search' % self.type, self.search) + addEvent('provider.search.%s.%s' % (self.protocol, self.type), self.search) - def getEnabledProviderType(self): + def getEnabledProtocol(self): if self.isEnabled(): - return self.type + return self.protocol else: return [] @@ -273,6 +275,7 @@ class ResultList(list): defaults = { 'id': 0, + 'protocol': self.provider.protocol, 'type': self.provider.type, 'provider': self.provider.getName(), 'download': self.provider.loginDownload if self.provider.urls.get('login') else self.provider.download, diff --git a/couchpotato/core/providers/nzb/base.py b/couchpotato/core/providers/nzb/base.py index f11382ba..53c73af0 100644 --- a/couchpotato/core/providers/nzb/base.py +++ b/couchpotato/core/providers/nzb/base.py @@ -3,7 +3,8 @@ import time class NZBProvider(YarrProvider): - type = 'nzb' + + protocol = 'nzb' def calculateAge(self, unix): return int(time.time() - unix) / 24 / 60 / 60 diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index 453954c9..3e7ddde8 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -7,7 +7,7 @@ log = CPLog(__name__) class TorrentProvider(YarrProvider): - type = 'torrent' + protocol = 'torrent' def imdbMatch(self, url, imdbId): if getImdb(url) == imdbId: @@ -27,6 +27,6 @@ class TorrentProvider(YarrProvider): class TorrentMagnetProvider(TorrentProvider): - type = 'torrent_magnet' + protocol = 'torrent_magnet' download = None From 3dff598d03e9feb456ee639ede6128529c730a7e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 18 Aug 2013 11:45:45 +0200 Subject: [PATCH 103/209] Add multiprovider for provider grouping --- couchpotato/core/plugins/base.py | 10 ++++++++-- couchpotato/core/providers/base.py | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 9e97c804..84ecc451 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -25,6 +25,8 @@ log = CPLog(__name__) class Plugin(object): + _class_name = None + enabled_option = 'enabled' auto_register_static = True @@ -51,10 +53,14 @@ class Plugin(object): self.registerStatic(inspect.getfile(self.__class__)) def conf(self, attr, value = None, default = None, section = None): - return Env.setting(attr, section = section if section else self.getName().lower(), value = value, default = default) + class_name = self.getName().lower().split(':') + return Env.setting(attr, section = section if section else class_name[0].lower(), value = value, default = default) def getName(self): - return self.__class__.__name__ + return self._class_name or self.__class__.__name__ + + def setName(self, name): + self._class_name = name def renderTemplate(self, parent_file, templ, **params): diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 08b4c6e5..f2db8da6 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -13,10 +13,29 @@ import traceback import urllib2 import xml.etree.ElementTree as XMLTree - log = CPLog(__name__) +class MultiProvider(Plugin): + + def __init__(self): + self._classes = [] + + for Type in self.getTypes(): + klass = Type() + + # Overwrite name so logger knows what we're talking about + klass.setName('%s:%s' % (self.getName(), klass.getName())) + + self._classes.append(klass) + + def getTypes(self): + return [] + + def getClasses(self): + return self._classes + + class Provider(Plugin): type = None # movie, show, subtitle, trailer, ... From 9860a1c138f42e8b776714800fc76b68c10806d7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 18 Aug 2013 13:17:40 +0200 Subject: [PATCH 104/209] Default to movie type --- couchpotato/core/providers/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index f2db8da6..e6a9cb00 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -99,6 +99,7 @@ class Provider(Plugin): class YarrProvider(Provider): protocol = None # nzb, torrent, torrent_magnet + type = 'movie' cat_ids = {} cat_backup_id = None From 8a298edd4e442e7d3d68ef1907d6eab524f051a7 Mon Sep 17 00:00:00 2001 From: Techmunk Date: Wed, 21 Aug 2013 23:52:54 +1000 Subject: [PATCH 105/209] Implementation of Deluge downloader. --- .../core/downloaders/deluge/__init__.py | 89 ++++ couchpotato/core/downloaders/deluge/main.py | 241 ++++++++++ .../deluge/synchronousdeluge/__init__.py | 24 + .../deluge/synchronousdeluge/client.py | 135 ++++++ .../deluge/synchronousdeluge/exceptions.py | 11 + .../deluge/synchronousdeluge/protocol.py | 38 ++ .../deluge/synchronousdeluge/rencode.py | 433 ++++++++++++++++++ .../deluge/synchronousdeluge/transfer.py | 54 +++ 8 files changed, 1025 insertions(+) create mode 100644 couchpotato/core/downloaders/deluge/__init__.py create mode 100644 couchpotato/core/downloaders/deluge/main.py create mode 100644 couchpotato/core/downloaders/deluge/synchronousdeluge/__init__.py create mode 100644 couchpotato/core/downloaders/deluge/synchronousdeluge/client.py create mode 100644 couchpotato/core/downloaders/deluge/synchronousdeluge/exceptions.py create mode 100644 couchpotato/core/downloaders/deluge/synchronousdeluge/protocol.py create mode 100644 couchpotato/core/downloaders/deluge/synchronousdeluge/rencode.py create mode 100644 couchpotato/core/downloaders/deluge/synchronousdeluge/transfer.py diff --git a/couchpotato/core/downloaders/deluge/__init__.py b/couchpotato/core/downloaders/deluge/__init__.py new file mode 100644 index 00000000..4b122b38 --- /dev/null +++ b/couchpotato/core/downloaders/deluge/__init__.py @@ -0,0 +1,89 @@ +from .main import Deluge + +def start(): + return Deluge() + +config = [{ + 'name': 'deluge', + 'groups': [ + { + 'tab': 'downloaders', + 'list': 'download_providers', + 'name': 'deluge', + 'label': 'Deluge', + 'description': 'Use Deluge to download torrents.', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + 'radio_group': 'torrent', + }, + { + 'name': 'host', + 'default': 'localhost:58846', + 'description': 'Hostname with port. Usually localhost:58846', + }, + { + 'name': 'username', + }, + { + 'name': 'password', + 'type': 'password', + }, + { + 'name': 'paused', + 'type': 'bool', + 'default': False, + 'description': 'Add the torrent paused.', + }, + { + 'name': 'directory', + 'type': 'directory', + 'description': 'Download to this directory. Keep empty for default Deluge download directory.', + }, + { + 'name': 'completed_directory', + 'type': 'directory', + 'description': 'Move completed torrent to this directory. Keep empty for default Deluge options.', + 'advanced': True, + }, + { + 'name': 'label', + 'description': 'Label to add to torrents in the Deluge UI.', + }, + { + 'name': 'remove_complete', + 'label': 'Remove torrent', + 'type': 'bool', + 'default': True, + 'advanced': True, + 'description': 'Remove the torrent from Deluge after it has finished seeding.', + }, + { + 'name': 'delete_files', + 'label': 'Remove files', + 'default': True, + 'type': 'bool', + 'advanced': True, + 'description': 'Also remove the leftover files.', + }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, + { + 'name': 'delete_failed', + 'default': True, + 'advanced': True, + 'type': 'bool', + 'description': 'Delete a release after the download has failed.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/downloaders/deluge/main.py b/couchpotato/core/downloaders/deluge/main.py new file mode 100644 index 00000000..a990b175 --- /dev/null +++ b/couchpotato/core/downloaders/deluge/main.py @@ -0,0 +1,241 @@ +from base64 import b64encode +from couchpotato.core.helpers.variable import tryInt, tryFloat +from couchpotato.core.downloaders.base import Downloader, StatusList +from couchpotato.core.helpers.encoding import isInt +from couchpotato.core.logger import CPLog +from couchpotato.environment import Env +from datetime import timedelta + +from synchronousdeluge import DelugeClient + +import os.path +import traceback + +log = CPLog(__name__) + +class Deluge(Downloader): + + protocol = ['torrent', 'torrent_magnet'] + log = CPLog(__name__) + drpc = None + + def connect(self): + # Load host from config and split out port. + host = self.conf('host').split(':') + if not isInt(host[1]): + log.error('Config properties are not filled in correctly, port is missing.') + return False + + if not self.drpc: + self.drpc = DelugeRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) + + return self.drpc + + def download(self, data, movie, filedata = None): + + log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol'))) + + if not self.connect(): + return False + + if not filedata and data.get('protocol') == 'torrent': + log.error('Failed sending torrent, no data') + return False + + # Set parameters for Deluge + options = { + 'add_paused': self.conf('paused', default = 0), + 'label': self.conf('label') + } + + if self.conf('directory'): + if os.path.isdir(self.conf('directory')): + options['download_location'] = self.conf('directory') + else: + log.error('Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory')) + + if self.conf('completed_directory'): + if os.path.isdir(self.conf('completed_directory')): + options['move_completed'] = 1 + options['move_completed_path'] = self.conf('completed_directory') + else: + log.error('Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory')) + + if data.get('seed_ratio'): + options['stop_at_ratio'] = 1 + options['stop_ratio'] = tryFloat(data.get('seed_ratio')) + +# Deluge only has seed time as a global option. Might be added in +# in a future API release. +# if data.get('seed_time'): + + # Send request to Deluge + if data.get('protocol') == 'torrent_magnet': + remote_torrent = self.drpc.add_torrent_magnet(data.get('url'), options) + else: + remote_torrent = self.drpc.add_torrent_file(movie, b64encode(filedata), options) + + if not remote_torrent: + log.error('Failed sending torrent to Deluge') + return False + + log.info('Torrent sent to Deluge successfully.') + return self.downloadReturnId(remote_torrent) + + def getAllDownloadStatus(self): + + log.debug('Checking Deluge download status.') + + if not self.connect(): + return False + + statuses = StatusList(self) + + queue = self.drpc.get_alltorrents() + + if not (queue and queue.get('torrents')): + log.debug('Nothing in queue or error') + return False + + for torrent_id in queue: + item = queue[torrent_id] + log.debug('name=%s / id=%s / save_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / conf_ratio=%s/ is_seed=%s / is_finished=%s', (item['name'], item['hash'], item['save_path'], item['hash'], item['progress'], item['state'], item['eta'], item['ratio'], self.conf('ratio'), item['is_seed'], item['is_finished'])) + + if not os.path.isdir(Env.setting('from', 'renamer')): + log.error('Renamer "from" folder doesn\'t to exist.') + return + + status = 'busy' + # Deluge seems to set both is_seed and is_finished once everything has been downloaded. + if item['is_seed'] or item['is_finished']: + status = 'seeding' + elif item['is_seed'] and item['is_finished'] and item['paused']: + status = 'completed' + + download_dir = item['save_path'] + if item['move_on_completed']: + download_dir = item['move_completed_path'] + + statuses.append({ + 'id': item['hash'], + 'name': item['name'], + 'status': status, + 'original_status': item['state'], + 'seed_ratio': item['ratio'], + 'timeleft': str(timedelta(seconds = item['eta'])), + 'folder': os.path.join(download_dir, item['name']), + }) + + return statuses + + def pause(self, item, pause = True): + if pause: + return self.drpc.pause_torrent([item['id']]) + else: + return self.drpc.resume_torrent([item['id']]) + + def removeFailed(self, item): + log.info('%s failed downloading, deleting...', item['name']) + return self.drpc.remove_torrent(item['id'], True) + + def processComplete(self, item, delete_files = False): + log.debug('Requesting Deluge to remove the torrent %s%s.', (item['name'], ' and cleanup the downloaded files' if delete_files else '')) + return self.drpc.remove_torrent(item['id'], remove_local_data = delete_files) + +class DelugeRPC(object): + + host = 'localhost' + port = 58846 + username = None + password = None + client = None + + def __init__(self, host = 'localhost', port = 58846, username = None, password = None): + super(DelugeRPC, self).__init__() + + self.host = host + self.port = port + self.username = username + self.password = password + + def connect(self): + self.client = DelugeClient() + self.client.connect(self.host, int(self.port), self.username, self.password) + + def add_torrent_magnet(self, torrent, options): + torrent_id = False + try: + self.connect() + torrent_id = self.client.core.add_torrent_magnet(torrent, options).get() + if options['label']: + self.client.label.set_torrent(torrent_id, options['label']).get() + except Exception, err: + log.error('Failed to add torrent magnet: %s %s', err, traceback.format_exc()) + finally: + if self.client: + self.disconnect() + + return torrent_id + + def add_torrent_file(self, movie, torrent, options): + torrent_id = False + try: + self.connect() + torrent_id = self.client.core.add_torrent_file(movie, torrent, options).get() + if options['label']: + self.client.label.set_torrent(torrent_id, options['label']).get() + except Exception, err: + log.error('Failed to add torrent file: %s %s', err, traceback.format_exc()) + finally: + if self.client: + self.disconnect() + + return torrent_id + + def get_alltorrents(self): + ret = False + try: + self.connect() + ret = self.client.core.get_torrents_status({}, {}).get() + except Exception, err: + log.error('Failed to get all torrents: %s %s', err, traceback.format_exc()) + finally: + if self.client: + self.disconnect() + return ret + + def pause_torrent(self, torrent_ids): + try: + self.connect() + self.client.core.pause_torrent(torrent_ids).get() + except Exception, err: + log.error('Failed to pause torrent: %s %s', err, traceback.format_exc()) + finally: + if self.client: + self.disconnect() + + def resume_torrent(self, torrent_ids): + try: + self.connect() + self.client.core.resume_torrent(torrent_ids).get() + except Exception, err: + log.error('Failed to resume torrent: %s %s', err, traceback.format_exc()) + finally: + if self.client: + self.disconnect() + + def remove_torrent(self, torrent_id, remove_local_data): + ret = False + try: + self.connect() + ret = self.client.core.remove_torrent(torrent_id, remove_local_data).get() + except Exception, err: + log.error('Failed to remove torrent: %s %s', err, traceback.format_exc()) + finally: + if self.client: + self.disconnect() + return ret + + def disconnect(self): + self.client.disconnect() + diff --git a/couchpotato/core/downloaders/deluge/synchronousdeluge/__init__.py b/couchpotato/core/downloaders/deluge/synchronousdeluge/__init__.py new file mode 100644 index 00000000..a6fbcdd8 --- /dev/null +++ b/couchpotato/core/downloaders/deluge/synchronousdeluge/__init__.py @@ -0,0 +1,24 @@ +"""A synchronous implementation of the Deluge RPC protocol + based on gevent-deluge by Christopher Rosell. + + https://github.com/chrippa/gevent-deluge + +Example usage: + + from synchronousdeluge import DelgueClient + + client = DelugeClient() + client.connect() + + # Wait for value + download_location = client.core.get_config_value("download_location").get() +""" + + +__title__ = "synchronous-deluge" +__version__ = "0.1" +__author__ = "Christian Dale" + +from .client import DelugeClient +from .exceptions import DelugeRPCError + diff --git a/couchpotato/core/downloaders/deluge/synchronousdeluge/client.py b/couchpotato/core/downloaders/deluge/synchronousdeluge/client.py new file mode 100644 index 00000000..363bd855 --- /dev/null +++ b/couchpotato/core/downloaders/deluge/synchronousdeluge/client.py @@ -0,0 +1,135 @@ +import os + +from collections import defaultdict +from itertools import imap + +from .exceptions import DelugeRPCError +from .protocol import DelugeRPCRequest, DelugeRPCResponse +from .transfer import DelugeTransfer + +__all__ = ["DelugeClient"] + + +RPC_RESPONSE = 1 +RPC_ERROR = 2 +RPC_EVENT = 3 + + +class DelugeClient(object): + def __init__(self): + """A deluge client session.""" + self.transfer = DelugeTransfer() + self.modules = [] + self._request_counter = 0 + + def _get_local_auth(self): + xdg_config = os.path.expanduser(os.environ.get("XDG_CONFIG_HOME", "~/.config")) + config_home = os.path.join(xdg_config, "deluge") + auth_file = os.path.join(config_home, "auth") + + username = password = "" + with open(auth_file) as fd: + for line in fd: + if line.startswith("#"): + continue + + auth = line.split(":") + if len(auth) >= 2 and auth[0] == "localclient": + username, password = auth[0], auth[1] + break + + return username, password + + def _create_module_method(self, module, method): + fullname = "{0}.{1}".format(module, method) + + def func(obj, *args, **kwargs): + return self.remote_call(fullname, *args, **kwargs) + + func.__name__ = method + + return func + + def _introspect(self): + self.modules = [] + + methods = self.remote_call("daemon.get_method_list").get() + methodmap = defaultdict(dict) + splitter = lambda v: v.split(".") + + for module, method in imap(splitter, methods): + methodmap[module][method] = self._create_module_method(module, method) + + for module, methods in methodmap.items(): + clsname = "DelugeModule{0}".format(module.capitalize()) + cls = type(clsname, (), methods) + setattr(self, module, cls()) + self.modules.append(module) + + def remote_call(self, method, *args, **kwargs): + req = DelugeRPCRequest(self._request_counter, method, *args, **kwargs) + message = next(self.transfer.send_request(req)) + + response = DelugeRPCResponse() + + if not isinstance(message, tuple): + return + + if len(message) < 3: + return + + message_type = message[0] + +# if message_type == RPC_EVENT: +# event = message[1] +# values = message[2] +# +# if event in self._event_handlers: +# for handler in self._event_handlers[event]: +# gevent.spawn(handler, *values) +# +# elif message_type in (RPC_RESPONSE, RPC_ERROR): + if message_type in (RPC_RESPONSE, RPC_ERROR): + request_id = message[1] + value = message[2] + + if request_id == self._request_counter : + if message_type == RPC_RESPONSE: + response.set(value) + elif message_type == RPC_ERROR: + err = DelugeRPCError(*value) + response.set_exception(err) + + self._request_counter += 1 + return response + + def connect(self, host="127.0.0.1", port=58846, username="", password=""): + """Connects to a daemon process. + + :param host: str, the hostname of the daemon + :param port: int, the port of the daemon + :param username: str, the username to login with + :param password: str, the password to login with + """ + + # Connect transport + self.transfer.connect((host, port)) + + # Attempt to fetch local auth info if needed + if not username and host in ("127.0.0.1", "localhost"): + username, password = self._get_local_auth() + + # Authenticate + self.remote_call("daemon.login", username, password).get() + + # Introspect available methods + self._introspect() + + @property + def connected(self): + return self.transfer.connected + + def disconnect(self): + """Disconnects from the daemon.""" + self.transfer.disconnect() + diff --git a/couchpotato/core/downloaders/deluge/synchronousdeluge/exceptions.py b/couchpotato/core/downloaders/deluge/synchronousdeluge/exceptions.py new file mode 100644 index 00000000..da6cf022 --- /dev/null +++ b/couchpotato/core/downloaders/deluge/synchronousdeluge/exceptions.py @@ -0,0 +1,11 @@ +__all__ = ["DelugeRPCError"] + +class DelugeRPCError(Exception): + def __init__(self, name, msg, traceback): + self.name = name + self.msg = msg + self.traceback = traceback + + def __str__(self): + return "{0}: {1}: {2}".format(self.__class__.__name__, self.name, self.msg) + diff --git a/couchpotato/core/downloaders/deluge/synchronousdeluge/protocol.py b/couchpotato/core/downloaders/deluge/synchronousdeluge/protocol.py new file mode 100644 index 00000000..756d4dfc --- /dev/null +++ b/couchpotato/core/downloaders/deluge/synchronousdeluge/protocol.py @@ -0,0 +1,38 @@ +__all__ = ["DelugeRPCRequest", "DelugeRPCResponse"] + +class DelugeRPCRequest(object): + def __init__(self, request_id, method, *args, **kwargs): + self.request_id = request_id + self.method = method + self.args = args + self.kwargs = kwargs + + def format(self): + return (self.request_id, self.method, self.args, self.kwargs) + +class DelugeRPCResponse(object): + def __init__(self): + self.value = None + self._exception = None + + def successful(self): + return self._exception is None + + @property + def exception(self): + if self._exception is not None: + return self._exception + + def set(self, value=None): + self.value = value + self._exception = None + + def set_exception(self, exception): + self._exception = exception + + def get(self): + if self._exception is None: + return self.value + else: + raise self._exception + diff --git a/couchpotato/core/downloaders/deluge/synchronousdeluge/rencode.py b/couchpotato/core/downloaders/deluge/synchronousdeluge/rencode.py new file mode 100644 index 00000000..e58c7154 --- /dev/null +++ b/couchpotato/core/downloaders/deluge/synchronousdeluge/rencode.py @@ -0,0 +1,433 @@ + +""" +rencode -- Web safe object pickling/unpickling. + +Public domain, Connelly Barnes 2006-2007. + +The rencode module is a modified version of bencode from the +BitTorrent project. For complex, heterogeneous data structures with +many small elements, r-encodings take up significantly less space than +b-encodings: + + >>> len(rencode.dumps({'a':0, 'b':[1,2], 'c':99})) + 13 + >>> len(bencode.bencode({'a':0, 'b':[1,2], 'c':99})) + 26 + +The rencode format is not standardized, and may change with different +rencode module versions, so you should check that you are using the +same rencode version throughout your project. +""" + +__version__ = '1.0.1' +__all__ = ['dumps', 'loads'] + +# Original bencode module by Petru Paler, et al. +# +# Modifications by Connelly Barnes: +# +# - Added support for floats (sent as 32-bit or 64-bit in network +# order), bools, None. +# - Allowed dict keys to be of any serializable type. +# - Lists/tuples are always decoded as tuples (thus, tuples can be +# used as dict keys). +# - Embedded extra information in the 'typecodes' to save some space. +# - Added a restriction on integer length, so that malicious hosts +# cannot pass us large integers which take a long time to decode. +# +# Licensed by Bram Cohen under the "MIT license": +# +# "Copyright (C) 2001-2002 Bram Cohen +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# The Software is provided "AS IS", without warranty of any kind, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose and +# noninfringement. In no event shall the authors or copyright holders +# be liable for any claim, damages or other liability, whether in an +# action of contract, tort or otherwise, arising from, out of or in +# connection with the Software or the use or other dealings in the +# Software." +# +# (The rencode module is licensed under the above license as well). +# + +import struct +import string +from threading import Lock + +# Default number of bits for serialized floats, either 32 or 64 (also a parameter for dumps()). +DEFAULT_FLOAT_BITS = 32 + +# Maximum length of integer when written as base 10 string. +MAX_INT_LENGTH = 64 + +# The bencode 'typecodes' such as i, d, etc have been extended and +# relocated on the base-256 character set. +CHR_LIST = chr(59) +CHR_DICT = chr(60) +CHR_INT = chr(61) +CHR_INT1 = chr(62) +CHR_INT2 = chr(63) +CHR_INT4 = chr(64) +CHR_INT8 = chr(65) +CHR_FLOAT32 = chr(66) +CHR_FLOAT64 = chr(44) +CHR_TRUE = chr(67) +CHR_FALSE = chr(68) +CHR_NONE = chr(69) +CHR_TERM = chr(127) + +# Positive integers with value embedded in typecode. +INT_POS_FIXED_START = 0 +INT_POS_FIXED_COUNT = 44 + +# Dictionaries with length embedded in typecode. +DICT_FIXED_START = 102 +DICT_FIXED_COUNT = 25 + +# Negative integers with value embedded in typecode. +INT_NEG_FIXED_START = 70 +INT_NEG_FIXED_COUNT = 32 + +# Strings with length embedded in typecode. +STR_FIXED_START = 128 +STR_FIXED_COUNT = 64 + +# Lists with length embedded in typecode. +LIST_FIXED_START = STR_FIXED_START+STR_FIXED_COUNT +LIST_FIXED_COUNT = 64 + +def decode_int(x, f): + f += 1 + newf = x.index(CHR_TERM, f) + if newf - f >= MAX_INT_LENGTH: + raise ValueError('overflow') + try: + n = int(x[f:newf]) + except (OverflowError, ValueError): + n = long(x[f:newf]) + if x[f] == '-': + if x[f + 1] == '0': + raise ValueError + elif x[f] == '0' and newf != f+1: + raise ValueError + return (n, newf+1) + +def decode_intb(x, f): + f += 1 + return (struct.unpack('!b', x[f:f+1])[0], f+1) + +def decode_inth(x, f): + f += 1 + return (struct.unpack('!h', x[f:f+2])[0], f+2) + +def decode_intl(x, f): + f += 1 + return (struct.unpack('!l', x[f:f+4])[0], f+4) + +def decode_intq(x, f): + f += 1 + return (struct.unpack('!q', x[f:f+8])[0], f+8) + +def decode_float32(x, f): + f += 1 + n = struct.unpack('!f', x[f:f+4])[0] + return (n, f+4) + +def decode_float64(x, f): + f += 1 + n = struct.unpack('!d', x[f:f+8])[0] + return (n, f+8) + +def decode_string(x, f): + colon = x.index(':', f) + try: + n = int(x[f:colon]) + except (OverflowError, ValueError): + n = long(x[f:colon]) + if x[f] == '0' and colon != f+1: + raise ValueError + colon += 1 + s = x[colon:colon+n] + try: + t = s.decode("utf8") + if len(t) != len(s): + s = t + except UnicodeDecodeError: + pass + return (s, colon+n) + +def decode_list(x, f): + r, f = [], f+1 + while x[f] != CHR_TERM: + v, f = decode_func[x[f]](x, f) + r.append(v) + return (tuple(r), f + 1) + +def decode_dict(x, f): + r, f = {}, f+1 + while x[f] != CHR_TERM: + k, f = decode_func[x[f]](x, f) + r[k], f = decode_func[x[f]](x, f) + return (r, f + 1) + +def decode_true(x, f): + return (True, f+1) + +def decode_false(x, f): + return (False, f+1) + +def decode_none(x, f): + return (None, f+1) + +decode_func = {} +decode_func['0'] = decode_string +decode_func['1'] = decode_string +decode_func['2'] = decode_string +decode_func['3'] = decode_string +decode_func['4'] = decode_string +decode_func['5'] = decode_string +decode_func['6'] = decode_string +decode_func['7'] = decode_string +decode_func['8'] = decode_string +decode_func['9'] = decode_string +decode_func[CHR_LIST ] = decode_list +decode_func[CHR_DICT ] = decode_dict +decode_func[CHR_INT ] = decode_int +decode_func[CHR_INT1 ] = decode_intb +decode_func[CHR_INT2 ] = decode_inth +decode_func[CHR_INT4 ] = decode_intl +decode_func[CHR_INT8 ] = decode_intq +decode_func[CHR_FLOAT32] = decode_float32 +decode_func[CHR_FLOAT64] = decode_float64 +decode_func[CHR_TRUE ] = decode_true +decode_func[CHR_FALSE ] = decode_false +decode_func[CHR_NONE ] = decode_none + +def make_fixed_length_string_decoders(): + def make_decoder(slen): + def f(x, f): + s = x[f+1:f+1+slen] + try: + t = s.decode("utf8") + if len(t) != len(s): + s = t + except UnicodeDecodeError: + pass + return (s, f+1+slen) + return f + for i in range(STR_FIXED_COUNT): + decode_func[chr(STR_FIXED_START+i)] = make_decoder(i) + +make_fixed_length_string_decoders() + +def make_fixed_length_list_decoders(): + def make_decoder(slen): + def f(x, f): + r, f = [], f+1 + for i in range(slen): + v, f = decode_func[x[f]](x, f) + r.append(v) + return (tuple(r), f) + return f + for i in range(LIST_FIXED_COUNT): + decode_func[chr(LIST_FIXED_START+i)] = make_decoder(i) + +make_fixed_length_list_decoders() + +def make_fixed_length_int_decoders(): + def make_decoder(j): + def f(x, f): + return (j, f+1) + return f + for i in range(INT_POS_FIXED_COUNT): + decode_func[chr(INT_POS_FIXED_START+i)] = make_decoder(i) + for i in range(INT_NEG_FIXED_COUNT): + decode_func[chr(INT_NEG_FIXED_START+i)] = make_decoder(-1-i) + +make_fixed_length_int_decoders() + +def make_fixed_length_dict_decoders(): + def make_decoder(slen): + def f(x, f): + r, f = {}, f+1 + for j in range(slen): + k, f = decode_func[x[f]](x, f) + r[k], f = decode_func[x[f]](x, f) + return (r, f) + return f + for i in range(DICT_FIXED_COUNT): + decode_func[chr(DICT_FIXED_START+i)] = make_decoder(i) + +make_fixed_length_dict_decoders() + +def encode_dict(x,r): + r.append(CHR_DICT) + for k, v in x.items(): + encode_func[type(k)](k, r) + encode_func[type(v)](v, r) + r.append(CHR_TERM) + + +def loads(x): + try: + r, l = decode_func[x[0]](x, 0) + except (IndexError, KeyError): + raise ValueError + if l != len(x): + raise ValueError + return r + +from types import StringType, IntType, LongType, DictType, ListType, TupleType, FloatType, NoneType, UnicodeType + +def encode_int(x, r): + if 0 <= x < INT_POS_FIXED_COUNT: + r.append(chr(INT_POS_FIXED_START+x)) + elif -INT_NEG_FIXED_COUNT <= x < 0: + r.append(chr(INT_NEG_FIXED_START-1-x)) + elif -128 <= x < 128: + r.extend((CHR_INT1, struct.pack('!b', x))) + elif -32768 <= x < 32768: + r.extend((CHR_INT2, struct.pack('!h', x))) + elif -2147483648 <= x < 2147483648: + r.extend((CHR_INT4, struct.pack('!l', x))) + elif -9223372036854775808 <= x < 9223372036854775808: + r.extend((CHR_INT8, struct.pack('!q', x))) + else: + s = str(x) + if len(s) >= MAX_INT_LENGTH: + raise ValueError('overflow') + r.extend((CHR_INT, s, CHR_TERM)) + +def encode_float32(x, r): + r.extend((CHR_FLOAT32, struct.pack('!f', x))) + +def encode_float64(x, r): + r.extend((CHR_FLOAT64, struct.pack('!d', x))) + +def encode_bool(x, r): + r.extend({False: CHR_FALSE, True: CHR_TRUE}[bool(x)]) + +def encode_none(x, r): + r.extend(CHR_NONE) + +def encode_string(x, r): + if len(x) < STR_FIXED_COUNT: + r.extend((chr(STR_FIXED_START + len(x)), x)) + else: + r.extend((str(len(x)), ':', x)) + +def encode_unicode(x, r): + encode_string(x.encode("utf8"), r) + +def encode_list(x, r): + if len(x) < LIST_FIXED_COUNT: + r.append(chr(LIST_FIXED_START + len(x))) + for i in x: + encode_func[type(i)](i, r) + else: + r.append(CHR_LIST) + for i in x: + encode_func[type(i)](i, r) + r.append(CHR_TERM) + +def encode_dict(x,r): + if len(x) < DICT_FIXED_COUNT: + r.append(chr(DICT_FIXED_START + len(x))) + for k, v in x.items(): + encode_func[type(k)](k, r) + encode_func[type(v)](v, r) + else: + r.append(CHR_DICT) + for k, v in x.items(): + encode_func[type(k)](k, r) + encode_func[type(v)](v, r) + r.append(CHR_TERM) + +encode_func = {} +encode_func[IntType] = encode_int +encode_func[LongType] = encode_int +encode_func[StringType] = encode_string +encode_func[ListType] = encode_list +encode_func[TupleType] = encode_list +encode_func[DictType] = encode_dict +encode_func[NoneType] = encode_none +encode_func[UnicodeType] = encode_unicode + +lock = Lock() + +try: + from types import BooleanType + encode_func[BooleanType] = encode_bool +except ImportError: + pass + +def dumps(x, float_bits=DEFAULT_FLOAT_BITS): + """ + Dump data structure to str. + + Here float_bits is either 32 or 64. + """ + lock.acquire() + try: + if float_bits == 32: + encode_func[FloatType] = encode_float32 + elif float_bits == 64: + encode_func[FloatType] = encode_float64 + else: + raise ValueError('Float bits (%d) is not 32 or 64' % float_bits) + r = [] + encode_func[type(x)](x, r) + finally: + lock.release() + return ''.join(r) + +def test(): + f1 = struct.unpack('!f', struct.pack('!f', 25.5))[0] + f2 = struct.unpack('!f', struct.pack('!f', 29.3))[0] + f3 = struct.unpack('!f', struct.pack('!f', -0.6))[0] + L = (({'a':15, 'bb':f1, 'ccc':f2, '':(f3,(),False,True,'')},('a',10**20),tuple(range(-100000,100000)),'b'*31,'b'*62,'b'*64,2**30,2**33,2**62,2**64,2**30,2**33,2**62,2**64,False,False, True, -1, 2, 0),) + assert loads(dumps(L)) == L + d = dict(zip(range(-100000,100000),range(-100000,100000))) + d.update({'a':20, 20:40, 40:41, f1:f2, f2:f3, f3:False, False:True, True:False}) + L = (d, {}, {5:6}, {7:7,True:8}, {9:10, 22:39, 49:50, 44: ''}) + assert loads(dumps(L)) == L + L = ('', 'a'*10, 'a'*100, 'a'*1000, 'a'*10000, 'a'*100000, 'a'*1000000, 'a'*10000000) + assert loads(dumps(L)) == L + L = tuple([dict(zip(range(n),range(n))) for n in range(100)]) + ('b',) + assert loads(dumps(L)) == L + L = tuple([dict(zip(range(n),range(-n,0))) for n in range(100)]) + ('b',) + assert loads(dumps(L)) == L + L = tuple([tuple(range(n)) for n in range(100)]) + ('b',) + assert loads(dumps(L)) == L + L = tuple(['a'*n for n in range(1000)]) + ('b',) + assert loads(dumps(L)) == L + L = tuple(['a'*n for n in range(1000)]) + (None,True,None) + assert loads(dumps(L)) == L + assert loads(dumps(None)) == None + assert loads(dumps({None:None})) == {None:None} + assert 1e-10 Date: Sat, 20 Jul 2013 13:55:07 +0200 Subject: [PATCH 106/209] Fix untagDir and hastagDir Changes in commit 8a252bff64b2bb2d376673a0b00e9624d44aaf4c broke the tagging functionality --- couchpotato/core/plugins/renamer/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index d1daf183..1121691f 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -517,22 +517,22 @@ Remove it if you want it to be renamed (again, or at least let it try again) if ignore_file: self.createFile(ignore_file, text) - def untagDir(self, folder, tag = None): + def untagDir(self, folder, tag = ''): if not os.path.isdir(folder): return # Remove any .ignore files for root, dirnames, filenames in os.walk(folder): - for filename in fnmatch.filter(filenames, '%s.ignore' % tag if tag else '*'): + for filename in fnmatch.filter(filenames, '*%s.ignore' % tag): os.remove((os.path.join(root, filename))) - def hastagDir(self, folder, tag = None): + def hastagDir(self, folder, tag = ''): if not os.path.isdir(folder): return False # Find any .ignore files for root, dirnames, filenames in os.walk(folder): - if fnmatch.filter(filenames, '%s.ignore' % tag if tag else '*'): + if fnmatch.filter(filenames, '*%s.ignore' % tag): return True return False From d0735a6d5885d821ace5743b9ccc1bf334f5bf3b Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 20 Jul 2013 16:01:37 +0200 Subject: [PATCH 107/209] Add failsafe for symlink errors E.g. on Windows you need Admin rights to symlink... --- couchpotato/core/plugins/renamer/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 1121691f..d068c483 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -552,7 +552,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) shutil.copy(old, dest) elif self.conf('file_action') == 'move_symlink': shutil.move(old, dest) - symlink(dest, old) + try: + symlink(dest, old) + except: + log.error('Couldn\'t symlink file "%s" to "%s". Copying the file back. Error: %s. ', (old, dest, traceback.format_exc())) + shutil.copy(dest, old) else: shutil.move(old, dest) From 695cdea4476aea8bc462b5adb0b40286f3c98138 Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Sat, 3 Aug 2013 01:13:36 +0200 Subject: [PATCH 108/209] Remove 'move' exception No need to remove files when 'move' is selected as the downloaders do this themselves now when cleaning up --- couchpotato/core/downloaders/transmission/__init__.py | 2 +- couchpotato/core/downloaders/utorrent/__init__.py | 2 +- couchpotato/core/plugins/renamer/main.py | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index d0e8279e..f96e628e 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -47,7 +47,7 @@ config = [{ { 'name': 'remove_complete', 'label': 'Remove torrent', - 'default': False, + 'default': True, 'advanced': True, 'type': 'bool', 'description': 'Remove the torrent from Transmission after it finished seeding.', diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index 6a1da36b..d45e2e6c 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -39,7 +39,7 @@ config = [{ { 'name': 'remove_complete', 'label': 'Remove torrent', - 'default': False, + 'default': True, 'advanced': True, 'type': 'bool', 'description': 'Remove the torrent from uTorrent after it finished seeding.', diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index d068c483..dfe1b621 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -204,7 +204,7 @@ class Renamer(Plugin): # Move nfo depending on settings if file_type is 'nfo' and not self.conf('rename_nfo'): log.debug('Skipping, renaming of %s disabled', file_type) - if self.conf('cleanup') and not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)): + if self.conf('cleanup') and not self.downloadIsTorrent(download_info): for current_file in group['files'][file_type]: remove_files.append(current_file) continue @@ -387,7 +387,7 @@ class Renamer(Plugin): # Remove leftover files if self.conf('cleanup') and not self.conf('move_leftover') and remove_leftovers and \ - not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)): + not self.downloadIsTorrent(download_info): log.debug('Removing leftover files') for current_file in group['files']['leftover']: remove_files.append(current_file) @@ -444,8 +444,7 @@ class Renamer(Plugin): self.tagDir(group, 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent - if self.movieInFromFolder(movie_folder) and \ - self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info): + if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(download_info): self.tagDir(group, 'renamed_already') # Remove matching releases @@ -456,8 +455,7 @@ class Renamer(Plugin): except: log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) - if group['dirname'] and group['parentdir'] and \ - not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)): + if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(download_info): try: log.info('Deleting folder: %s', group['parentdir']) self.deleteEmptyFolder(group['parentdir']) From 70bc2a6656427fa5405889e3c219e7256e55fecc Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Wed, 21 Aug 2013 20:49:01 +0200 Subject: [PATCH 109/209] use right variable for pause fixes #2049 --- couchpotato/core/downloaders/transmission/main.py | 4 ++-- couchpotato/core/downloaders/utorrent/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index a619d411..12082f64 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -129,9 +129,9 @@ class Transmission(Downloader): def pause(self, item, pause = True): if pause: - return self.trpc.stop_torrent(item['hashString']) + return self.trpc.stop_torrent(item['id']) else: - return self.trpc.start_torrent(item['hashString']) + return self.trpc.start_torrent(item['id']) def removeFailed(self, item): log.info('%s failed downloading, deleting...', item['name']) diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 79f9f5b9..588d7587 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -149,10 +149,10 @@ class uTorrent(Downloader): return statuses - def pause(self, download_info, pause = True): + def pause(self, item, pause = True): if not self.connect(): return False - return self.utorrent_api.pause_torrent(download_info['id'], pause) + return self.utorrent_api.pause_torrent(item['id'], pause) def removeFailed(self, item): log.info('%s failed downloading, deleting...', item['name']) From bf6bcaed723f9930b7594559d8f2e84f8278804e Mon Sep 17 00:00:00 2001 From: mano3m <-> Date: Thu, 22 Aug 2013 21:20:02 +0200 Subject: [PATCH 110/209] provide more info in case no movie is found Several users reported an issue with "more than one group found (0)", and it was unclear to them what it meant. This might help. --- couchpotato/core/plugins/scanner/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index e48d2747..743b1a56 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -329,14 +329,17 @@ class Scanner(Plugin): del movie_files + total_found = len(valid_files) + # Make sure only one movie was found if a download ID is provided - if download_info and not len(valid_files) == 1: + if download_info and total_found == 0: + log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', download_info.get('imdb_id')) + elif download_info and total_found > 1: log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_info.get('imdb_id'), len(valid_files))) download_info = None # Determine file types processed_movies = {} - total_found = len(valid_files) while True and not self.shuttingDown(): try: identifier, group = valid_files.popitem() From 6aec5a9a606447526cc6169ddeca723ff4e937af Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 12:13:45 +0200 Subject: [PATCH 111/209] Cleanup IMDB provider --- .../providers/automation/imdb/__init__.py | 12 +- .../core/providers/automation/imdb/main.py | 131 ++++++++---------- 2 files changed, 67 insertions(+), 76 deletions(-) diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py index ee804af1..546cba97 100644 --- a/couchpotato/core/providers/automation/imdb/__init__.py +++ b/couchpotato/core/providers/automation/imdb/__init__.py @@ -38,21 +38,23 @@ config = [{ 'description': 'Import movies from IMDB Charts', 'options': [ { - 'name': 'automation_enabled', + 'name': 'automation_providers_enabled', 'default': False, 'type': 'enabler', }, { - 'name': 'automation_charts_theaters_use', - 'type': 'checkbox', + 'name': 'automation_charts_theater', + 'type': 'bool', 'label': 'In Theaters', 'description': 'New Movies In-Theaters chart', + 'default': True, }, { - 'name': 'automation_charts_top250_use', - 'type': 'checkbox', + 'name': 'automation_charts_top250', + 'type': 'bool', 'label': 'TOP 250', 'description': 'IMDB TOP 250 chart', + 'default': True, }, ], }, diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index 0d494949..c4aef7f1 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -1,90 +1,41 @@ +import traceback + from bs4 import BeautifulSoup +from couchpotato import fireEvent from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import getImdb, splitString, tryInt + from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation -import re -import traceback + +from couchpotato.core.providers.base import MultiProvider + log = CPLog(__name__) -class IMDB(Automation, RSS): +class IMDB(MultiProvider): + + def getTypes(self): + return [IMDBWatchlist, IMDBAutomation] + + +class IMDBBase(Automation, RSS): interval = 1800 - chart_urls = { - 'theater': 'http://www.imdb.com/movies-in-theaters/', - 'top250': 'http://www.imdb.com/chart/top', - } + def getInfo(self, imdb_id): + return fireEvent('movie.info', identifier = imdb_id, merge = True) +class IMDBWatchlist(IMDBBase): + + enabled_option = 'automation_enabled' + def getIMDBids(self): movies = [] - # Handle Chart URLs - if self.conf('automation_charts_theaters_use'): - log.debug('Started IMDB chart: %s', self.chart_urls['theater']) - data = self.getHTMLData(self.chart_urls['theater']) - if data: - html = BeautifulSoup(data) - - try: - result_div = html.find('div', attrs = {'id': 'main'}) - - entries = result_div.find_all('div', attrs = {'itemtype': 'http://schema.org/Movie'}) - - for entry in entries: - title = entry.find('h4', attrs = {'itemprop': 'name'}).getText() - - log.debug('Identified title: %s', title) - result = re.search('(.*) \((.*)\)', title) - - if result: - name = result.group(1) - year = result.group(2) - - imdb = self.search(name, year) - - if imdb and self.isMinimalMovie(imdb): - movies.append(imdb['imdb']) - - except: - log.error('Failed loading IMDB chart results from %s: %s', (self.chart_urls['theater'], traceback.format_exc())) - - if self.conf('automation_charts_top250_use'): - log.debug('Started IMDB chart: %s', self.chart_urls['top250']) - data = self.getHTMLData(self.chart_urls['top250']) - if data: - html = BeautifulSoup(data) - - try: - result_div = html.find('div', attrs = {'id': 'main'}) - - result_table = result_div.find_all('table')[1] - entries = result_table.find_all('tr') - - for entry in entries[1:]: - title = entry.find_all('td')[2].getText() - - log.debug('Identified title: %s', title) - result = re.search('(.*) \((.*)\)', title) - - if result: - name = result.group(1) - year = result.group(2) - - imdb = self.search(name, year) - - if imdb and self.isMinimalMovie(imdb): - movies.append(imdb['imdb']) - - except: - log.error('Failed loading IMDB chart results from %s: %s', (self.chart_urls['theater'], traceback.format_exc())) - - - # Handle Watchlists watchlist_enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))] watchlist_urls = splitString(self.conf('automation_urls')) @@ -103,9 +54,47 @@ class IMDB(Automation, RSS): for imdb in imdbs: movies.append(imdb) + if self.shuttingDown(): + break + except: log.error('Failed loading IMDB watchlist: %s %s', (url, traceback.format_exc())) - - # Return the combined resultset + return movies + + +class IMDBAutomation(IMDBBase): + + enabled_option = 'automation_providers_enabled' + + chart_urls = { + 'theater': 'http://www.imdb.com/movies-in-theaters/', + 'top250': 'http://www.imdb.com/chart/top', + } + + def getIMDBids(self): + + movies = [] + + for url in self.chart_urls: + if self.conf('automation_charts_%s' % url): + data = self.getHTMLData(self.chart_urls[url]) + if data: + html = BeautifulSoup(data) + + try: + result_div = html.find('div', attrs = {'id': 'main'}) + imdb_ids = getImdb(str(result_div), multiple = True) + + for imdb_id in imdb_ids: + info = self.getInfo(imdb_id) + if info and self.isMinimalMovie(info): + movies.append(imdb_id) + + if self.shuttingDown(): + break + + except: + log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc())) + return movies From 7e44af936d7186473e5c5fb781df97f102fd5243 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 12:14:02 +0200 Subject: [PATCH 112/209] Watch shutdown when adding automation movies --- couchpotato/core/plugins/automation/main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 80e12850..92547cb0 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -26,6 +26,10 @@ class Automation(Plugin): movie_ids = [] for imdb_id in movies: + + if self.shuttingDown(): + break + prop_name = 'automation.added.%s' % imdb_id added = Env.prop(prop_name, default = False) if not added: @@ -35,5 +39,11 @@ class Automation(Plugin): Env.prop(prop_name, True) for movie_id in movie_ids: + + if self.shuttingDown(): + break + movie_dict = fireEvent('movie.get', movie_id, single = True) fireEvent('movie.searcher.single', movie_dict) + + return True \ No newline at end of file From cef5b04eb1633b1edde308286f9d8484e2edb1d8 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 12:14:15 +0200 Subject: [PATCH 113/209] Return unique imdb list --- couchpotato/core/helpers/variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 381889c0..90caf848 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -128,7 +128,7 @@ def getImdb(txt, check_inside = True, multiple = False): try: ids = re.findall('(tt\d{7})', txt) if multiple: - return ids if len(ids) > 0 else [] + return list(set(ids)) if len(ids) > 0 else [] return ids[0] except IndexError: pass From ed0e5ef497d3bef71bca0288ea8db8af48522da1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 12:24:15 +0200 Subject: [PATCH 114/209] XMBC notification, better remote folder description --- couchpotato/core/notifications/xbmc/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/notifications/xbmc/__init__.py b/couchpotato/core/notifications/xbmc/__init__.py index f0167ce8..dafa0f63 100644 --- a/couchpotato/core/notifications/xbmc/__init__.py +++ b/couchpotato/core/notifications/xbmc/__init__.py @@ -44,7 +44,7 @@ config = [{ 'default': 0, 'type': 'bool', 'advanced': True, - 'description': 'Scan new movie folder at remote XBMC servers, only works if movie location is the same.', + 'description': 'Only scan new movie folder at remote XBMC servers. Works if movie location is the same.', }, { 'name': 'on_snatch', From e2bd6a91cd467e7464ab3c6503bebf072abefc58 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 13:21:39 +0200 Subject: [PATCH 115/209] MPAA rating for renamer --- couchpotato/core/plugins/renamer/__init__.py | 1 + couchpotato/core/plugins/renamer/main.py | 1 + couchpotato/core/providers/movie/_modifier/main.py | 1 + couchpotato/core/providers/movie/omdbapi/main.py | 1 + couchpotato/core/providers/movie/themoviedb/main.py | 1 + 5 files changed, 5 insertions(+) mode change 100644 => 100755 couchpotato/core/plugins/renamer/__init__.py mode change 100644 => 100755 couchpotato/core/plugins/renamer/main.py mode change 100644 => 100755 couchpotato/core/providers/movie/omdbapi/main.py diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py old mode 100644 new mode 100755 index 04cd970d..50cda078 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -27,6 +27,7 @@ rename_options = { 'imdb_id': 'IMDB id (tt0123456)', 'cd': 'CD number (cd1)', 'cd_nr': 'Just the cd nr. (1)', + 'mpaa': 'MPAA Rating', }, } diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py old mode 100644 new mode 100755 index 8d1a8186..4f435882 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -205,6 +205,7 @@ class Renamer(Plugin): 'imdb_id': library['identifier'], 'cd': '', 'cd_nr': '', + 'mpaa': library['info'].get('mpaa', ''), } for file_type in group['files']: diff --git a/couchpotato/core/providers/movie/_modifier/main.py b/couchpotato/core/providers/movie/_modifier/main.py index e4d70221..835cce04 100644 --- a/couchpotato/core/providers/movie/_modifier/main.py +++ b/couchpotato/core/providers/movie/_modifier/main.py @@ -28,6 +28,7 @@ class MovieResultModifier(Plugin): 'tagline': '', 'imdb': '', 'genres': [], + 'mpaa': None } def __init__(self): diff --git a/couchpotato/core/providers/movie/omdbapi/main.py b/couchpotato/core/providers/movie/omdbapi/main.py old mode 100644 new mode 100755 index 89990747..c9f4d927 --- a/couchpotato/core/providers/movie/omdbapi/main.py +++ b/couchpotato/core/providers/movie/omdbapi/main.py @@ -95,6 +95,7 @@ class OMDBAPI(MovieProvider): #'rotten': (tryFloat(movie.get('tomatoRating', 0)), tryInt(movie.get('tomatoReviews', '').replace(',', ''))), }, 'imdb': str(movie.get('imdbID', '')), + 'mpaa': str(movie.get('Rated', '')), 'runtime': self.runtimeToMinutes(movie.get('Runtime', '')), 'released': movie.get('Released'), 'year': year if isinstance(year, (int)) else None, diff --git a/couchpotato/core/providers/movie/themoviedb/main.py b/couchpotato/core/providers/movie/themoviedb/main.py index 735419c3..241fc6b0 100644 --- a/couchpotato/core/providers/movie/themoviedb/main.py +++ b/couchpotato/core/providers/movie/themoviedb/main.py @@ -167,6 +167,7 @@ class TheMovieDb(MovieProvider): 'backdrop_original': [backdrop_original] if backdrop_original else [], }, 'imdb': movie.get('imdb_id'), + 'mpaa': movie.get('certification', ''), 'runtime': movie.get('runtime'), 'released': movie.get('released'), 'year': year, From 08554889fd63f54e46f9cca4ef34e765c15b2bed Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 13:34:45 +0200 Subject: [PATCH 116/209] Add the old rottentomatoes to default enabled list --- .../core/providers/automation/rottentomatoes/__init__.py | 4 +++- couchpotato/core/providers/automation/rottentomatoes/main.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 52b1c882..4675fac2 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -21,19 +21,21 @@ config = [{ { 'name': 'automation_urls_use', 'label': 'Use', + 'default': '1', }, { 'name': 'automation_urls', 'label': 'url', 'type': 'combined', 'combine': ['automation_urls_use', 'automation_urls'], + 'default': 'http://www.rottentomatoes.com/syndication/rss/in_theaters.xml', }, { 'name': 'tomatometer_percent', 'default': '80', 'label': 'Tomatometer', 'description': 'Use as extra scoring requirement', - } + }, ], }, ], diff --git a/couchpotato/core/providers/automation/rottentomatoes/main.py b/couchpotato/core/providers/automation/rottentomatoes/main.py index 40f72a5c..69611705 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/main.py +++ b/couchpotato/core/providers/automation/rottentomatoes/main.py @@ -12,8 +12,6 @@ class Rottentomatoes(Automation, RSS): interval = 1800 - - def getIMDBids(self): movies = [] From 8e9e7b49eabfa40f8479d081e3e78a52f84587e0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 14:03:17 +0200 Subject: [PATCH 117/209] Simplify linking Thanks @mano3m --- couchpotato/core/plugins/renamer/__init__.py | 6 ++-- couchpotato/core/plugins/renamer/main.py | 30 +++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) mode change 100755 => 100644 couchpotato/core/plugins/renamer/main.py diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index 50cda078..56672b8e 100755 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -120,10 +120,10 @@ config = [{ { 'name': 'file_action', 'label': 'Torrent File Action', - 'default': 'move', + 'default': 'link', 'type': 'dropdown', - 'values': [('Move', 'move'), ('Copy', 'copy'), ('Hard link', 'hardlink'), ('Move & Sym link', 'move_symlink')], - 'description': 'Define which kind of file operation you want to use for torrents. Before you start using hard links or sym links, PLEASE read about their possible drawbacks.', + 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], + 'description': 'Link or Copy after downloading completed (and allow for seeding), or Move after seeding completed. Link first tries hard link, then sym link and falls back to Copy.', 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py old mode 100755 new mode 100644 index 508eab24..2b73590b --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -548,21 +548,23 @@ Remove it if you want it to be renamed (again, or at least let it try again) try: if forcemove: shutil.move(old, dest) - elif self.conf('file_action') == 'hardlink': - try: - link(old, dest) - except: - log.error('Couldn\'t hardlink file "%s" to "%s". Copying instead. Error: %s. ', (old, dest, traceback.format_exc())) - shutil.copy(old, dest) elif self.conf('file_action') == 'copy': shutil.copy(old, dest) - elif self.conf('file_action') == 'move_symlink': - shutil.move(old, dest) + elif self.conf('file_action') == 'link': + # First try to hardlink try: - symlink(dest, old) + log.debug('Hardlinking file "%s" to "%s"...', (old, dest)) + link(old, dest) except: - log.error('Couldn\'t symlink file "%s" to "%s". Copying the file back. Error: %s. ', (old, dest, traceback.format_exc())) - shutil.copy(dest, old) + # Try to simlink next + log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s. ', (old, dest, traceback.format_exc())) + shutil.copy(old, dest) + try: + symlink(dest, old + '.link') + os.unlink(old) + os.rename(old + '.link', old) + except: + log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc())) else: shutil.move(old, dest) @@ -767,10 +769,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) for item in scan_items: # Ask the renamer to scan the item if item['scan']: - if item['pause'] and self.conf('file_action') == 'move_symlink': + if item['pause'] and self.conf('file_action') == 'link': fireEvent('download.pause', item = item, pause = True, single = True) fireEvent('renamer.scan', download_info = item) - if item['pause'] and self.conf('file_action') == 'move_symlink': + if item['pause'] and self.conf('file_action') == 'link': fireEvent('download.pause', item = item, pause = False, single = True) if item['process_complete']: #First make sure the files were succesfully processed @@ -829,6 +831,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) def statusInfoComplete(self, item): return item['id'] and item['downloader'] and item['folder'] - + def movieInFromFolder(self, movie_folder): return movie_folder and self.conf('from') in movie_folder or not movie_folder From 770590e4f2361feaac7fb03106c9f7af17060438 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 14:08:05 +0200 Subject: [PATCH 118/209] Match default ports Thanks @cpg --- couchpotato/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 0c0127fa..f49ba3e5 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -212,7 +212,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # app.debug = development config = { 'use_reloader': reloader, - 'port': tryInt(Env.setting('port', default = 5000)), + 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), From 20aa78105f56a74ba3ad83e2282cd1f27d6e3eae Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 24 Aug 2013 14:22:15 +0200 Subject: [PATCH 119/209] Do window size check inside load event --- couchpotato/templates/index.html | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/couchpotato/templates/index.html b/couchpotato/templates/index.html index f9bc4634..d45dcb9b 100644 --- a/couchpotato/templates/index.html +++ b/couchpotato/templates/index.html @@ -22,17 +22,18 @@