From ed6a46e9c082954830d0e551f8a390bf3bc84f66 Mon Sep 17 00:00:00 2001 From: dumaresq Date: Sun, 17 Aug 2014 16:28:47 -0400 Subject: [PATCH 01/38] Added putioDownloader --- couchpotato/core/downloaders/putioDownload.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 couchpotato/core/downloaders/putioDownload.py diff --git a/couchpotato/core/downloaders/putioDownload.py b/couchpotato/core/downloaders/putioDownload.py new file mode 100644 index 00000000..4202e7bc --- /dev/null +++ b/couchpotato/core/downloaders/putioDownload.py @@ -0,0 +1,124 @@ +from __future__ import with_statement +import os +import traceback +import putio + +from couchpotato.api import addApiView +from couchpotato.core.event import addEvent +from couchpotato.core._base.downloader.main import DownloaderBase +from couchpotato.core.helpers.encoding import sp +from couchpotato.core.helpers.variable import getDownloadDir +from couchpotato.core.logger import CPLog +from couchpotato.environment import Env + +log = CPLog(__name__) + +autoload = 'Putiodownload' + + +class Putiodownload(DownloaderBase): + + protocol = ['torrent', 'torrent_magnet'] + status_support = False + + def __init__(self): + addApiView('putiodownload.getfrom', self.getFromPutio, docs = { + 'desc': 'Allows you to download file from prom Put.io', + }) + return super(Putiodownload,self).__init__() + + + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + log.info ('Sending "%s" to put.io', data.get('name')) + url = data.get('url') + OAUTH_TOKEN = self.conf('oauth_token') + client = putio.Client(OAUTH_TOKEN) + # Need to constuct a the API url a better way. + callbackurl = None + if self.conf('download'): + callbackurl = 'http://'+self.conf('callback_host')+'/'+self.conf('url_base', section='core')+'/api/'+self.conf('api_key', section='core')+'/putiodownload.getfrom/' + client.Transfer.add_url(url,callback_url=callbackurl) + return True + + def test(self): + OAUTH_TOKEN = self.conf('oauth_token') + try: + client = putio.Client(OAUTH_TOKEN) + if client.File.list(): + return True + except: + log.info('Failed to get file listing, check OAUTH_TOKEN') + return False + + def getFromPutio(self, **kwargs): + log.info('Put.io Download has been called') + OAUTH_TOKEN = self.conf('oauth_token') + client = putio.Client(OAUTH_TOKEN) + files = client.File.list() + delete = self.conf('detele_file') + downloaddir = self.conf('download_dir') + tempdownloaddir = '/export/nas/Downloads/incomplete' + for f in files: + if str(f.id) == str(kwargs.get('file_id')): + # Need to read this in from somewhere + client.File.download(f, dest=tempdownloaddir, delete_after_download=delete) + shutil.move(tempdownloaddir+"/"+str(f.name),downloaddir) + return True + +config = [{ + 'name': 'putiodownload', + 'groups': [ + { + 'tab': 'downloaders', + 'list': 'download_providers', + 'name': 'putiodownload', + 'label': 'put.io Download', + 'description': 'This will start a torrent download on Put.io.
Note: you must have a putio account and API', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + 'radio_group': 'torrent', + }, + { + 'name': 'oauth_token', + 'label': 'oauth_token', + 'description': 'This is the OAUTH_TOKEN from your putio API', + }, + { + 'name': 'callback_host', + 'description': 'This is used to generate the callback url', + }, + { + 'name': 'download', + 'description': 'Set this to have CouchPotato download the file from Put.io', + 'type': 'bool', + 'default': 0, + }, + { + 'name': 'detele_file', + 'description': 'Set this to remove the file from putio after sucessful download Note: does nothing if you don\'t select download', + 'type': 'bool', + 'default': 0, + }, + { + 'name': 'download_dir', + 'label': 'Download Directory', + 'description': 'The Directory to download files to, does nothing if you don\'t select download', + 'default': '/', + }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, + ], + } + ], +}] From 5acab980254a16145d05adab48af5c934c4c7427 Mon Sep 17 00:00:00 2001 From: dumaresq Date: Tue, 19 Aug 2014 19:32:00 -0400 Subject: [PATCH 02/38] fixed hardcoded directory --- couchpotato/core/downloaders/putioDownload.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/putioDownload.py b/couchpotato/core/downloaders/putioDownload.py index 4202e7bc..4eaa24f8 100644 --- a/couchpotato/core/downloaders/putioDownload.py +++ b/couchpotato/core/downloaders/putioDownload.py @@ -59,7 +59,7 @@ class Putiodownload(DownloaderBase): files = client.File.list() delete = self.conf('detele_file') downloaddir = self.conf('download_dir') - tempdownloaddir = '/export/nas/Downloads/incomplete' + tempdownloaddir = self.conf('tempdownload_dir') for f in files: if str(f.id) == str(kwargs.get('file_id')): # Need to read this in from somewhere @@ -111,6 +111,12 @@ config = [{ 'description': 'The Directory to download files to, does nothing if you don\'t select download', 'default': '/', }, + { + 'name': 'tempdownload_dir', + 'label': 'Temporary Download Directory', + 'description': 'The Temporary Directory to download files to, does nothing if you don\'t select download', + 'default': '/', + }, { 'name': 'manual', 'default': 0, From bb73cb8eecbd9d020c43ca5d30c12b7dab7cfacc Mon Sep 17 00:00:00 2001 From: dumaresq Date: Sun, 24 Aug 2014 18:19:01 -0400 Subject: [PATCH 03/38] Fixed missing library --- couchpotato/core/downloaders/putioDownload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/downloaders/putioDownload.py b/couchpotato/core/downloaders/putioDownload.py index 4eaa24f8..bc27a665 100644 --- a/couchpotato/core/downloaders/putioDownload.py +++ b/couchpotato/core/downloaders/putioDownload.py @@ -2,6 +2,7 @@ from __future__ import with_statement import os import traceback import putio +import shutil from couchpotato.api import addApiView from couchpotato.core.event import addEvent From 53e7e383a313e7f57de03605dc1e46832adf037d Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 29 Aug 2014 11:38:28 +0200 Subject: [PATCH 04/38] put.io rename --- couchpotato/core/downloaders/{putioDownload.py => putio.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename couchpotato/core/downloaders/{putioDownload.py => putio.py} (100%) diff --git a/couchpotato/core/downloaders/putioDownload.py b/couchpotato/core/downloaders/putio.py similarity index 100% rename from couchpotato/core/downloaders/putioDownload.py rename to couchpotato/core/downloaders/putio.py From d0f1e7c6a36006815ae17786504452c86d582c41 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 29 Aug 2014 12:30:31 +0200 Subject: [PATCH 05/38] Update put.io code --- couchpotato/core/downloaders/putio.py | 131 --------- .../core/downloaders/putio/__init__.py | 69 +++++ couchpotato/core/downloaders/putio/api.py | 271 ++++++++++++++++++ couchpotato/core/downloaders/putio/main.py | 87 ++++++ .../core/downloaders/putio/static/putio.js | 68 +++++ 5 files changed, 495 insertions(+), 131 deletions(-) delete mode 100644 couchpotato/core/downloaders/putio.py create mode 100644 couchpotato/core/downloaders/putio/__init__.py create mode 100644 couchpotato/core/downloaders/putio/api.py create mode 100644 couchpotato/core/downloaders/putio/main.py create mode 100644 couchpotato/core/downloaders/putio/static/putio.js diff --git a/couchpotato/core/downloaders/putio.py b/couchpotato/core/downloaders/putio.py deleted file mode 100644 index bc27a665..00000000 --- a/couchpotato/core/downloaders/putio.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import with_statement -import os -import traceback -import putio -import shutil - -from couchpotato.api import addApiView -from couchpotato.core.event import addEvent -from couchpotato.core._base.downloader.main import DownloaderBase -from couchpotato.core.helpers.encoding import sp -from couchpotato.core.helpers.variable import getDownloadDir -from couchpotato.core.logger import CPLog -from couchpotato.environment import Env - -log = CPLog(__name__) - -autoload = 'Putiodownload' - - -class Putiodownload(DownloaderBase): - - protocol = ['torrent', 'torrent_magnet'] - status_support = False - - def __init__(self): - addApiView('putiodownload.getfrom', self.getFromPutio, docs = { - 'desc': 'Allows you to download file from prom Put.io', - }) - return super(Putiodownload,self).__init__() - - - def download(self, data = None, media = None, filedata = None): - if not media: media = {} - if not data: data = {} - log.info ('Sending "%s" to put.io', data.get('name')) - url = data.get('url') - OAUTH_TOKEN = self.conf('oauth_token') - client = putio.Client(OAUTH_TOKEN) - # Need to constuct a the API url a better way. - callbackurl = None - if self.conf('download'): - callbackurl = 'http://'+self.conf('callback_host')+'/'+self.conf('url_base', section='core')+'/api/'+self.conf('api_key', section='core')+'/putiodownload.getfrom/' - client.Transfer.add_url(url,callback_url=callbackurl) - return True - - def test(self): - OAUTH_TOKEN = self.conf('oauth_token') - try: - client = putio.Client(OAUTH_TOKEN) - if client.File.list(): - return True - except: - log.info('Failed to get file listing, check OAUTH_TOKEN') - return False - - def getFromPutio(self, **kwargs): - log.info('Put.io Download has been called') - OAUTH_TOKEN = self.conf('oauth_token') - client = putio.Client(OAUTH_TOKEN) - files = client.File.list() - delete = self.conf('detele_file') - downloaddir = self.conf('download_dir') - tempdownloaddir = self.conf('tempdownload_dir') - for f in files: - if str(f.id) == str(kwargs.get('file_id')): - # Need to read this in from somewhere - client.File.download(f, dest=tempdownloaddir, delete_after_download=delete) - shutil.move(tempdownloaddir+"/"+str(f.name),downloaddir) - return True - -config = [{ - 'name': 'putiodownload', - 'groups': [ - { - 'tab': 'downloaders', - 'list': 'download_providers', - 'name': 'putiodownload', - 'label': 'put.io Download', - 'description': 'This will start a torrent download on Put.io.
Note: you must have a putio account and API', - 'wizard': True, - 'options': [ - { - 'name': 'enabled', - 'default': 0, - 'type': 'enabler', - 'radio_group': 'torrent', - }, - { - 'name': 'oauth_token', - 'label': 'oauth_token', - 'description': 'This is the OAUTH_TOKEN from your putio API', - }, - { - 'name': 'callback_host', - 'description': 'This is used to generate the callback url', - }, - { - 'name': 'download', - 'description': 'Set this to have CouchPotato download the file from Put.io', - 'type': 'bool', - 'default': 0, - }, - { - 'name': 'detele_file', - 'description': 'Set this to remove the file from putio after sucessful download Note: does nothing if you don\'t select download', - 'type': 'bool', - 'default': 0, - }, - { - 'name': 'download_dir', - 'label': 'Download Directory', - 'description': 'The Directory to download files to, does nothing if you don\'t select download', - 'default': '/', - }, - { - 'name': 'tempdownload_dir', - 'label': 'Temporary Download Directory', - 'description': 'The Temporary Directory to download files to, does nothing if you don\'t select download', - 'default': '/', - }, - { - 'name': 'manual', - 'default': 0, - 'type': 'bool', - 'advanced': True, - 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', - }, - ], - } - ], -}] diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py new file mode 100644 index 00000000..1f4865ec --- /dev/null +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -0,0 +1,69 @@ +from .main import PutIO + + +def autoload(): + return PutIO() + + +config = [{ + 'name': 'putio', + 'groups': [ + { + 'tab': 'downloaders', + 'list': 'download_providers', + 'name': 'putio', + 'label': 'put.io', + 'description': 'This will start a torrent download on Put.io.', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + 'radio_group': 'torrent', + }, + { + 'name': 'oauth_token', + 'label': 'oauth_token', + 'description': 'This is the OAUTH_TOKEN from your putio API', + 'advanced': True, + }, + { + 'name': 'callback_host', + 'description': 'This is used to generate the callback url', + }, + { + 'name': 'download', + 'description': 'Set this to have CouchPotato download the file from Put.io', + 'type': 'bool', + 'default': 0, + }, + { + 'name': 'delete_file', + 'description': 'Set this to remove the file from putio after sucessful download Note: does nothing if you don\'t select download', + 'type': 'bool', + 'default': 0, + }, + { + 'name': 'download_dir', + 'type': 'directory', + 'label': 'Download Directory', + 'description': 'The Directory to download files to, does nothing if you don\'t select download', + }, + { + 'name': 'tempdownload_dir', + 'type': 'directory', + 'label': 'Temporary Download Directory', + 'description': 'The Temporary Directory to download files to, does nothing if you don\'t select download', + }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/downloaders/putio/api.py b/couchpotato/core/downloaders/putio/api.py new file mode 100644 index 00000000..0f2a2c66 --- /dev/null +++ b/couchpotato/core/downloaders/putio/api.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- + +# Changed +# Removed iso8601 library requirement +# Added CP logging + +import os +import re +import json +import webbrowser +from urllib import urlencode +from couchpotato import CPLog +from dateutil.parser import parse + +import requests + +BASE_URL = 'https://api.put.io/v2' +ACCESS_TOKEN_URL = 'https://api.put.io/v2/oauth2/access_token' +AUTHENTICATION_URL = 'https://api.put.io/v2/oauth2/authenticate' + +log = CPLog(__name__) + + +class AuthHelper(object): + + def __init__(self, client_id, client_secret, redirect_uri, type='code'): + self.client_id = client_id + self.client_secret = client_secret + self.callback_url = redirect_uri + self.type = type + + @property + def authentication_url(self): + """Redirect your users to here to authenticate them.""" + params = { + 'client_id': self.client_id, + 'response_type': self.type, + 'redirect_uri': self.callback_url + } + return AUTHENTICATION_URL + "?" + urlencode(params) + + def open_authentication_url(self): + webbrowser.open(self.authentication_url) + + def get_access_token(self, code): + params = { + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'grant_type': 'authorization_code', + 'redirect_uri': self.callback_url, + 'code': code + } + response = requests.get(ACCESS_TOKEN_URL, params=params) + log.debug(response) + assert response.status_code == 200 + return response.json()['access_token'] + + +class Client(object): + + def __init__(self, access_token): + self.access_token = access_token + self.session = requests.session() + + # Keep resource classes as attributes of client. + # Pass client to resource classes so resource object + # can use the client. + attributes = {'client': self} + self.File = type('File', (_File,), attributes) + self.Transfer = type('Transfer', (_Transfer,), attributes) + self.Account = type('Account', (_Account,), attributes) + + def request(self, path, method='GET', params=None, data=None, files=None, + headers=None, raw=False, stream=False): + """ + Wrapper around requests.request() + + Prepends BASE_URL to path. + Inserts oauth_token to query params. + Parses response as JSON and returns it. + + """ + if not params: + params = {} + + if not headers: + headers = {} + + # All requests must include oauth_token + params['oauth_token'] = self.access_token + + headers['Accept'] = 'application/json' + + url = BASE_URL + path + log.debug('url: %s', url) + + response = self.session.request( + method, url, params=params, data=data, files=files, + headers=headers, allow_redirects=True, stream=stream) + log.debug('response: %s', response) + if raw: + return response + + log.debug('content: %s', response.content) + try: + response = json.loads(response.content) + except ValueError: + raise Exception('Server didn\'t send valid JSON:\n%s\n%s' % ( + response, response.content)) + + if response['status'] == 'ERROR': + raise Exception(response['error_type']) + + return response + + +class _BaseResource(object): + + client = None + + def __init__(self, resource_dict): + """Constructs the object from a dict.""" + # All resources must have id and name attributes + self.id = None + self.name = None + self.__dict__.update(resource_dict) + try: + self.created_at = parse(self.created_at) + except AttributeError: + self.created_at = None + + def __str__(self): + return self.name.encode('utf-8') + + def __repr__(self): + # shorten name for display + name = self.name[:17] + '...' if len(self.name) > 20 else self.name + return '<%s id=%r, name="%r">' % ( + self.__class__.__name__, self.id, name) + + +class _File(_BaseResource): + + @classmethod + def get(cls, id): + d = cls.client.request('/files/%i' % id, method='GET') + t = d['file'] + return cls(t) + + @classmethod + def list(cls, parent_id=0): + d = cls.client.request('/files/list', params={'parent_id': parent_id}) + files = d['files'] + return [cls(f) for f in files] + + @classmethod + def upload(cls, path, name=None): + with open(path) as f: + if name: + files = {'file': (name, f)} + else: + files = {'file': f} + d = cls.client.request('/files/upload', method='POST', files=files) + + f = d['file'] + return cls(f) + + def dir(self): + """List the files under directory.""" + return self.list(parent_id=self.id) + + def download(self, dest='.', delete_after_download=False): + if self.content_type == 'application/x-directory': + self._download_directory(dest, delete_after_download) + else: + self._download_file(dest, delete_after_download) + + def _download_directory(self, dest='.', delete_after_download=False): + name = self.name + if isinstance(name, unicode): + name = name.encode('utf-8', 'replace') + + dest = os.path.join(dest, name) + if not os.path.exists(dest): + os.mkdir(dest) + + for sub_file in self.dir(): + sub_file.download(dest, delete_after_download) + + if delete_after_download: + self.delete() + + def _download_file(self, dest='.', delete_after_download=False): + response = self.client.request( + '/files/%s/download' % self.id, raw=True, stream=True) + + filename = re.match( + 'attachment; filename=(.*)', + response.headers['content-disposition']).groups()[0] + # If file name has spaces, it must have quotes around. + filename = filename.strip('"') + + with open(os.path.join(dest, filename), 'wb') as f: + for chunk in response.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + + if delete_after_download: + self.delete() + + def delete(self): + return self.client.request('/files/delete', method='POST', + data={'file_ids': str(self.id)}) + + def move(self, parent_id): + return self.client.request('/files/move', method='POST', + data={'file_ids': str(self.id), 'parent_id': str(parent_id)}) + + def rename(self, name): + return self.client.request('/files/rename', method='POST', + data={'file_id': str(self.id), 'name': str(name)}) + + +class _Transfer(_BaseResource): + + @classmethod + def list(cls): + d = cls.client.request('/transfers/list') + transfers = d['transfers'] + return [cls(t) for t in transfers] + + @classmethod + def get(cls, id): + d = cls.client.request('/transfers/%i' % id, method='GET') + t = d['transfer'] + return cls(t) + + @classmethod + def add_url(cls, url, parent_id=0, extract=False, callback_url=None): + d = cls.client.request('/transfers/add', method='POST', data=dict( + url=url, parent_id=parent_id, extract=extract, + callback_url=callback_url)) + t = d['transfer'] + return cls(t) + + @classmethod + def add_torrent(cls, path, parent_id=0, extract=False, callback_url=None): + with open(path) as f: + files = {'file': f} + d = cls.client.request('/files/upload', method='POST', files=files, + data=dict(parent_id=parent_id, + extract=extract, + callback_url=callback_url)) + t = d['transfer'] + return cls(t) + + @classmethod + def clean(cls): + return cls.client.request('/transfers/clean', method='POST') + + +class _Account(_BaseResource): + + @classmethod + def info(cls): + return cls.client.request('/account/info', method='GET') + + @classmethod + def settings(cls): + return cls.client.request('/account/settings', method='GET') diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py new file mode 100644 index 00000000..10e6aa11 --- /dev/null +++ b/couchpotato/core/downloaders/putio/main.py @@ -0,0 +1,87 @@ +import shutil + +from couchpotato.api import addApiView +from couchpotato.core._base.downloader.main import DownloaderBase +from couchpotato.core.logger import CPLog +import api as pio + +log = CPLog(__name__) + +autoload = 'Putiodownload' + + +class PutIO(DownloaderBase): + protocol = ['torrent', 'torrent_magnet'] + status_support = False + + def __init__(self): + addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { + 'desc': 'Allows you to download file from prom Put.io', + }) + + addApiView('downloader.putio.auth_url', self.getAuthorizationUrl) + + return super(PutIO, self).__init__() + + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + + log.info('Sending "%s" to put.io', data.get('name')) + url = data.get('url') + + client = pio.Client(self.conf('oauth_token')) + + # Need to constuct a the API url a better way. + callbackurl = None + if self.conf('download'): + callbackurl = 'http://' + self.conf('callback_host') + '/' + self.conf('url_base', + section = 'core') + '/api/' + self.conf( + 'api_key', section = 'core') + '/downloader.putiodownload.getfrom/' + client.Transfer.add_url(url, callback_url = callbackurl) + + return True + + def test(self): + try: + client = pio.Client(self.conf('oauth_token')) + if client.File.list(): + return True + except: + log.info('Failed to get file listing, check OAUTH_TOKEN') + return False + + def getAuthorizationUrl(self): + # See notification/twitter + pass + + def getCredentials(self): + # Save oauth_token here to settings + pass + + def getAllDownloadStatus(self, ids): + # See other downloaders for examples + + # Check putio for status + + # Check "getFromPutio" progress + pass + + def getFromPutio(self, **kwargs): + + log.info('Put.io Download has been called') + client = pio.Client(self.conf('oauth_token')) + files = client.File.list() + + tempdownloaddir = self.conf('tempdownload_dir') + downloaddir = self.conf('download_dir') + + for f in files: + if str(f.id) == str(kwargs.get('file_id')): + # Need to read this in from somewhere + client.File.download(f, dest = tempdownloaddir, delete_after_download = self.conf('delete_file')) + shutil.move(tempdownloaddir + "/" + str(f.name), downloaddir) + + # Mark status of file_id as "done" here for getAllDownloadStatus + + return True diff --git a/couchpotato/core/downloaders/putio/static/putio.js b/couchpotato/core/downloaders/putio/static/putio.js new file mode 100644 index 00000000..1b71c263 --- /dev/null +++ b/couchpotato/core/downloaders/putio/static/putio.js @@ -0,0 +1,68 @@ +var PutIODownloader = new Class({ + + initialize: function(){ + var self = this; + + App.addEvent('loadSettings', self.addRegisterButton.bind(self)); + }, + + addRegisterButton: function(){ + var self = this; + + var setting_page = App.getPage('Settings'); + setting_page.addEvent('create', function(){ + + var fieldset = setting_page.tabs.downloaders.groups.putio, + l = window.location; + + var putio_set = 0; + fieldset.getElements('input[type=text]').each(function(el){ + putio_set += +(el.get('value') != ''); + }); + + new Element('.ctrlHolder').adopt( + + // Unregister button + (putio_set > 0) ? + [ + self.unregister = new Element('a.button.red', { + 'text': 'Unregister "'+fieldset.getElement('input[name*=screen_name]').get('value')+'"', + 'events': { + 'click': function(){ + fieldset.getElements('input[type=text]').set('value', '').fireEvent('change'); + + self.unregister.destroy(); + self.unregister_or.destroy(); + } + } + }), + self.unregister_or = new Element('span[text=or]') + ] + : null, + + // Register button + new Element('a.button', { + 'text': putio_set > 0 ? 'Register a different account' : 'Register your put.io account', + 'events': { + 'click': function(){ + Api.request('downloader.putio.auth_url', { + 'data': { + 'host': l.protocol + '//' + l.hostname + (l.port ? ':' + l.port : '') + }, + 'onComplete': function(json){ + window.location = json.url; + } + }); + } + } + }) + ).inject(fieldset.getElement('.test_button'), 'before'); + }) + + } + +}); + +window.addEvent('domready', function(){ + new PutIODownloader(); +}); From 872a4f4650d40bb8189723f1fd785d2fa8c57c78 Mon Sep 17 00:00:00 2001 From: dumaresq Date: Sun, 7 Sep 2014 17:59:16 -0400 Subject: [PATCH 06/38] Worked on geting Oauth and adding download status --- couchpotato/core/downloaders/putio/main.py | 73 ++++++++++++++++------ 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 10e6aa11..5e56b103 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -1,8 +1,10 @@ import shutil from couchpotato.api import addApiView -from couchpotato.core._base.downloader.main import DownloaderBase +from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList +from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog +from couchpotato.environment import Env import api as pio log = CPLog(__name__) @@ -12,7 +14,9 @@ autoload = 'Putiodownload' class PutIO(DownloaderBase): protocol = ['torrent', 'torrent_magnet'] - status_support = False + status_support = True + client_id = '1575' + client_secret = '132qbpseq1ymwn83wus4' def __init__(self): addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { @@ -20,6 +24,7 @@ class PutIO(DownloaderBase): }) addApiView('downloader.putio.auth_url', self.getAuthorizationUrl) + addApiView('downloader.putio.credentials', self.getCredentials) return super(PutIO, self).__init__() @@ -35,12 +40,10 @@ class PutIO(DownloaderBase): # Need to constuct a the API url a better way. callbackurl = None if self.conf('download'): - callbackurl = 'http://' + self.conf('callback_host') + '/' + self.conf('url_base', - section = 'core') + '/api/' + self.conf( - 'api_key', section = 'core') + '/downloader.putiodownload.getfrom/' - client.Transfer.add_url(url, callback_url = callbackurl) - - return True + callbackurl = 'http://' + self.conf('callback_host') + '/' + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) + resp = client.Transfer.add_url(url, callback_url = callbackurl) + log.debug('resp is %s', resp.id); + return self.downloadReturnId(resp.id) def test(self): try: @@ -51,21 +54,53 @@ class PutIO(DownloaderBase): log.info('Failed to get file listing, check OAUTH_TOKEN') return False - def getAuthorizationUrl(self): - # See notification/twitter - pass + def getAuthorizationUrl(self, host = None, **kwargs): + callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) + log.info('callback_url is %s', callback_url) + #I can't figure out how to pass this into getCredentials so I'm saving it here + self.apicallhost = host; + oauth = pio.AuthHelper(client_id=self.client_id,client_secret=self.client_secret,redirect_uri=callback_url) + resp = oauth.authentication_url + log.info ('reps is %s,', resp) + return { + 'success': True, + 'url': resp, + } - def getCredentials(self): - # Save oauth_token here to settings - pass + + def getCredentials(self, **kwargs): + code = kwargs.get('code') + log.info('getCredentials Called with code: %s', code) + callback_url = cleanHost(self.apicallhost) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) + log.info('callback is %s',callback_url) + oauth = pio.AuthHelper(client_id=self.client_id,client_secret=self.client_secret,redirect_uri=callback_url) + oauth_token = oauth.get_access_token(code) + log.info('oauth_token is: %s', oauth_token) + self.conf('oauth_token', value = oauth_token); + return 'redirect', Env.get('web_base') + 'settings/downloaders/' def getAllDownloadStatus(self, ids): - # See other downloaders for examples - - # Check putio for status - + log.debug('Checking putio download status.') + client = pio.Client(self.conf('oauth_token')) + transfers = client.Transfer.list() + log.debug(transfers); + release_downloads = ReleaseDownloadList(self) + for t in transfers: + if t.status == "COMPLETED" and self.conf('download') == False : + status = 'completed' + else: + #status = t.status.lower() + status = 'busy' + release_downloads.append({ + 'id' : t.id, + 'name': t.name, + 'status': status, + 'timeleft': t.estimated_time, + }) + + log.info(release_downloads) # Check "getFromPutio" progress - pass + return release_downloads def getFromPutio(self, **kwargs): From c77b270fa80da360747dcd2e1ba720f121334b3b Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Thu, 18 Sep 2014 06:00:09 -0400 Subject: [PATCH 07/38] Cleaned up OAUTH and made the download asyc --- couchpotato/core/downloaders/putio/main.py | 45 +++++++++++++--------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 5e56b103..764d5c1d 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -1,6 +1,7 @@ import shutil from couchpotato.api import addApiView +from couchpotato.core.event import addEvent, fireEventAsync from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog @@ -25,6 +26,7 @@ class PutIO(DownloaderBase): addApiView('downloader.putio.auth_url', self.getAuthorizationUrl) addApiView('downloader.putio.credentials', self.getCredentials) + addEvent('putio.download', self.putioDownloader) return super(PutIO, self).__init__() @@ -57,24 +59,18 @@ class PutIO(DownloaderBase): def getAuthorizationUrl(self, host = None, **kwargs): callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) log.info('callback_url is %s', callback_url) - #I can't figure out how to pass this into getCredentials so I'm saving it here - self.apicallhost = host; - oauth = pio.AuthHelper(client_id=self.client_id,client_secret=self.client_secret,redirect_uri=callback_url) - resp = oauth.authentication_url - log.info ('reps is %s,', resp) + target_url = "http://sabnzbd.dumaresq.ca/index.cgi?target=" + callback_url + log.info('target_url is %s', target_url) return { 'success': True, - 'url': resp, + 'url': target_url, } def getCredentials(self, **kwargs): - code = kwargs.get('code') - log.info('getCredentials Called with code: %s', code) - callback_url = cleanHost(self.apicallhost) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) - log.info('callback is %s',callback_url) - oauth = pio.AuthHelper(client_id=self.client_id,client_secret=self.client_secret,redirect_uri=callback_url) - oauth_token = oauth.get_access_token(code) + oauth_token = kwargs.get('oauth') + if not oauth_token: + return 'redirect', Env.get('web_base') + 'settigs/downloaders/' log.info('oauth_token is: %s', oauth_token) self.conf('oauth_token', value = oauth_token); return 'redirect', Env.get('web_base') + 'settings/downloaders/' @@ -102,17 +98,16 @@ class PutIO(DownloaderBase): # Check "getFromPutio" progress return release_downloads - def getFromPutio(self, **kwargs): - - log.info('Put.io Download has been called') + def putioDownloader(self, fid): + log.info('Put.io Real downloader called with file_id: %s',fid) client = pio.Client(self.conf('oauth_token')) + log.debug('About to get file List') files = client.File.list() - + log.debug('File list is %s',files) tempdownloaddir = self.conf('tempdownload_dir') downloaddir = self.conf('download_dir') - for f in files: - if str(f.id) == str(kwargs.get('file_id')): + if str(f.id) == str(fid): # Need to read this in from somewhere client.File.download(f, dest = tempdownloaddir, delete_after_download = self.conf('delete_file')) shutil.move(tempdownloaddir + "/" + str(f.name), downloaddir) @@ -120,3 +115,17 @@ class PutIO(DownloaderBase): # Mark status of file_id as "done" here for getAllDownloadStatus return True + + def getFromPutio(self, **kwargs): + # This needs some checking so that we don't download the same file multiple times. + # This needs to have a way to query if the download is still going + # would be nice to be albe to check how much is downloaded and report that. + file_id = str(kwargs.get('file_id')) + log.info('Put.io Download has been called file_id is %s', file_id) + fireEventAsync('putio.download',fid = file_id) + # Mark status of file_id as "done" here for getAllDownloadStatus + + return { + 'success': True, + } + From ef2b8e88b4b05307bbc65a0d58e36c513e93694c Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Fri, 19 Sep 2014 07:07:23 -0400 Subject: [PATCH 08/38] better download checking --- couchpotato/core/downloaders/putio/main.py | 68 +++++++++++++--------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 764d5c1d..6ddea4e2 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -1,4 +1,4 @@ -import shutil +import datetime from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEventAsync @@ -18,6 +18,7 @@ class PutIO(DownloaderBase): status_support = True client_id = '1575' client_secret = '132qbpseq1ymwn83wus4' + downloadingList = [] def __init__(self): addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { @@ -27,7 +28,6 @@ class PutIO(DownloaderBase): addApiView('downloader.putio.auth_url', self.getAuthorizationUrl) addApiView('downloader.putio.credentials', self.getCredentials) addEvent('putio.download', self.putioDownloader) - return super(PutIO, self).__init__() def download(self, data = None, media = None, filedata = None): @@ -82,19 +82,34 @@ class PutIO(DownloaderBase): log.debug(transfers); release_downloads = ReleaseDownloadList(self) for t in transfers: - if t.status == "COMPLETED" and self.conf('download') == False : - status = 'completed' - else: - #status = t.status.lower() - status = 'busy' - release_downloads.append({ - 'id' : t.id, - 'name': t.name, - 'status': status, - 'timeleft': t.estimated_time, - }) + if t.id in ids: + log.debug('id is %s', t.id) + log.debug('P.Status is %s',t.id) + log.debug('downloading list is %s', self.downloadingList) + if t.status == "COMPLETED" and self.conf('download') == False : + status = 'completed' + # This is Ugly but if we are set to download, and the thing is complete, we need to check the dowlading status + # Becuase putio changed the IDs the only thing we can check is the name. + elif t.status == "COMPLETED" and self.conf('download') == True: + status = 'busy' + # This is not ideal, right now if we are downloading anything we can't mark anything as completed + # The name and ID don't match currently so I can't use those... + if not self.downloadingList: + now = datetime.datetime.now() + log.debug ('now is %s', now) + log.debug ('t.finished_at is %s',t.finished_at) + if (now - t.finished_at) > datetime.timedelta(5,0): + status = 'completed' + else: + status = 'busy' + release_downloads.append({ + 'id' : t.id, + 'name': t.name, + 'status': status, + 'timeleft': t.estimated_time, + }) - log.info(release_downloads) + log.debug(release_downloads) # Check "getFromPutio" progress return release_downloads @@ -104,28 +119,25 @@ class PutIO(DownloaderBase): log.debug('About to get file List') files = client.File.list() log.debug('File list is %s',files) - tempdownloaddir = self.conf('tempdownload_dir') downloaddir = self.conf('download_dir') for f in files: if str(f.id) == str(fid): - # Need to read this in from somewhere - client.File.download(f, dest = tempdownloaddir, delete_after_download = self.conf('delete_file')) - shutil.move(tempdownloaddir + "/" + str(f.name), downloaddir) - - # Mark status of file_id as "done" here for getAllDownloadStatus + client.File.download(f, dest = downloaddir, delete_after_download = self.conf('delete_file')) + # Once the download is complete we need to remove it from the running list. + self.downloadingList.remove(fid) return True def getFromPutio(self, **kwargs): - # This needs some checking so that we don't download the same file multiple times. - # This needs to have a way to query if the download is still going - # would be nice to be albe to check how much is downloaded and report that. file_id = str(kwargs.get('file_id')) log.info('Put.io Download has been called file_id is %s', file_id) - fireEventAsync('putio.download',fid = file_id) - # Mark status of file_id as "done" here for getAllDownloadStatus - + if file_id not in self.downloadingList: + self.downloadingList.append(file_id) + fireEventAsync('putio.download',fid = file_id) + return { + 'success': True, + } return { - 'success': True, - } + 'success': False, + } From fba228fd9de9bb924016202f2e4b372826fe5f04 Mon Sep 17 00:00:00 2001 From: dumaresq Date: Fri, 19 Sep 2014 20:26:54 -0400 Subject: [PATCH 09/38] fixing check function --- couchpotato/core/downloaders/putio/main.py | 28 ++++++++++++---------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 6ddea4e2..ad206ccf 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -16,20 +16,21 @@ autoload = 'Putiodownload' class PutIO(DownloaderBase): protocol = ['torrent', 'torrent_magnet'] status_support = True - client_id = '1575' - client_secret = '132qbpseq1ymwn83wus4' downloadingList = [] + # This is the location on the Internet of the Oauth helper server + oauthServerURL = "http://sabnzb.dumaresq.ca/index.cgi" + def __init__(self): addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { 'desc': 'Allows you to download file from prom Put.io', }) - addApiView('downloader.putio.auth_url', self.getAuthorizationUrl) addApiView('downloader.putio.credentials', self.getCredentials) addEvent('putio.download', self.putioDownloader) return super(PutIO, self).__init__() + def download(self, data = None, media = None, filedata = None): if not media: media = {} if not data: data = {} @@ -40,6 +41,7 @@ class PutIO(DownloaderBase): client = pio.Client(self.conf('oauth_token')) # Need to constuct a the API url a better way. + # Note callback_host is NOT our address, it's the internet host that putio can call too callbackurl = None if self.conf('download'): callbackurl = 'http://' + self.conf('callback_host') + '/' + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) @@ -47,6 +49,7 @@ class PutIO(DownloaderBase): log.debug('resp is %s', resp.id); return self.downloadReturnId(resp.id) + def test(self): try: client = pio.Client(self.conf('oauth_token')) @@ -59,7 +62,7 @@ class PutIO(DownloaderBase): def getAuthorizationUrl(self, host = None, **kwargs): callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) log.info('callback_url is %s', callback_url) - target_url = "http://sabnzbd.dumaresq.ca/index.cgi?target=" + callback_url + target_url = oauthServerURL + "?target=" + callback_url log.info('target_url is %s', target_url) return { 'success': True, @@ -75,6 +78,7 @@ class PutIO(DownloaderBase): self.conf('oauth_token', value = oauth_token); return 'redirect', Env.get('web_base') + 'settings/downloaders/' + def getAllDownloadStatus(self, ids): log.debug('Checking putio download status.') client = pio.Client(self.conf('oauth_token')) @@ -83,22 +87,19 @@ class PutIO(DownloaderBase): release_downloads = ReleaseDownloadList(self) for t in transfers: if t.id in ids: - log.debug('id is %s', t.id) - log.debug('P.Status is %s',t.id) log.debug('downloading list is %s', self.downloadingList) if t.status == "COMPLETED" and self.conf('download') == False : status = 'completed' - # This is Ugly but if we are set to download, and the thing is complete, we need to check the dowlading status - # Becuase putio changed the IDs the only thing we can check is the name. + # So check if we are trying to download something elif t.status == "COMPLETED" and self.conf('download') == True: status = 'busy' # This is not ideal, right now if we are downloading anything we can't mark anything as completed # The name and ID don't match currently so I can't use those... if not self.downloadingList: - now = datetime.datetime.now() - log.debug ('now is %s', now) - log.debug ('t.finished_at is %s',t.finished_at) - if (now - t.finished_at) > datetime.timedelta(5,0): + now = datetime.datetime.utcnow() + date_time = datetime.datetime.strptime(t.finished_at,"%Y-%m-%dT%H:%M:%S") + # We need to make sure a race condition didn't happen + if (now - date_time) > datetime.timedelta(minutes=5): status = 'completed' else: status = 'busy' @@ -110,9 +111,9 @@ class PutIO(DownloaderBase): }) log.debug(release_downloads) - # Check "getFromPutio" progress return release_downloads + def putioDownloader(self, fid): log.info('Put.io Real downloader called with file_id: %s',fid) client = pio.Client(self.conf('oauth_token')) @@ -128,6 +129,7 @@ class PutIO(DownloaderBase): return True + def getFromPutio(self, **kwargs): file_id = str(kwargs.get('file_id')) log.info('Put.io Download has been called file_id is %s', file_id) From 2c40db307433210324a011873c67ab6d62ad26a2 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Fri, 19 Sep 2014 20:28:03 -0400 Subject: [PATCH 10/38] removed un-needed variable --- couchpotato/core/downloaders/putio/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index 1f4865ec..af05e360 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -50,12 +50,6 @@ config = [{ 'label': 'Download Directory', 'description': 'The Directory to download files to, does nothing if you don\'t select download', }, - { - 'name': 'tempdownload_dir', - 'type': 'directory', - 'label': 'Temporary Download Directory', - 'description': 'The Temporary Directory to download files to, does nothing if you don\'t select download', - }, { 'name': 'manual', 'default': 0, From 3e58378490a57bf0bc1e40166521cce4d24db788 Mon Sep 17 00:00:00 2001 From: dumaresq Date: Fri, 19 Sep 2014 21:41:58 -0400 Subject: [PATCH 11/38] figured out how to make the check work better --- couchpotato/core/downloaders/putio/main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index ad206ccf..3d55504c 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -92,15 +92,21 @@ class PutIO(DownloaderBase): status = 'completed' # So check if we are trying to download something elif t.status == "COMPLETED" and self.conf('download') == True: - status = 'busy' + # Assume we are done + status = 'completed' # This is not ideal, right now if we are downloading anything we can't mark anything as completed # The name and ID don't match currently so I can't use those... if not self.downloadingList: now = datetime.datetime.utcnow() date_time = datetime.datetime.strptime(t.finished_at,"%Y-%m-%dT%H:%M:%S") # We need to make sure a race condition didn't happen - if (now - date_time) > datetime.timedelta(minutes=5): - status = 'completed' + if (now - date_time) < datetime.timedelta(minutes=5): + #5 minutes haven't passed so we wait + status = 'busy' + else: + # If we have the file_id in the downloadingList mark it as busy + if str(t.file_id) in self.downloadingList: + status = 'busy' else: status = 'busy' release_downloads.append({ @@ -119,7 +125,6 @@ class PutIO(DownloaderBase): client = pio.Client(self.conf('oauth_token')) log.debug('About to get file List') files = client.File.list() - log.debug('File list is %s',files) downloaddir = self.conf('download_dir') for f in files: if str(f.id) == str(fid): From 4aa9801be41713a15001a61a9287020b4627c044 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sat, 20 Sep 2014 19:39:12 -0400 Subject: [PATCH 12/38] general code cleanup --- couchpotato/core/downloaders/putio/main.py | 87 +++++++++++----------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 3d55504c..8184b27d 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -1,5 +1,3 @@ -import datetime - from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEventAsync from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList @@ -7,6 +5,7 @@ from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from couchpotato.environment import Env import api as pio +import datetime log = CPLog(__name__) @@ -18,7 +17,7 @@ class PutIO(DownloaderBase): status_support = True downloadingList = [] # This is the location on the Internet of the Oauth helper server - oauthServerURL = "http://sabnzb.dumaresq.ca/index.cgi" + oauthServerURL = "http://sabnzbd.dumaresq.ca/index.cgi" def __init__(self): @@ -39,8 +38,7 @@ class PutIO(DownloaderBase): url = data.get('url') client = pio.Client(self.conf('oauth_token')) - - # Need to constuct a the API url a better way. + # It might be possible to call getFromPutio from the renamer if we can then we don't need to do this. # Note callback_host is NOT our address, it's the internet host that putio can call too callbackurl = None if self.conf('download'): @@ -59,11 +57,12 @@ class PutIO(DownloaderBase): log.info('Failed to get file listing, check OAUTH_TOKEN') return False + def getAuthorizationUrl(self, host = None, **kwargs): callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) - log.info('callback_url is %s', callback_url) - target_url = oauthServerURL + "?target=" + callback_url - log.info('target_url is %s', target_url) + log.debug('callback_url is %s', callback_url) + target_url = self.oauthServerURL + "?target=" + callback_url + log.debug('target_url is %s', target_url) return { 'success': True, 'url': target_url, @@ -71,10 +70,11 @@ class PutIO(DownloaderBase): def getCredentials(self, **kwargs): - oauth_token = kwargs.get('oauth') - if not oauth_token: - return 'redirect', Env.get('web_base') + 'settigs/downloaders/' - log.info('oauth_token is: %s', oauth_token) + try: + oauth_token = kwargs.get('oauth') + except: + return 'redirect', Env.get('web_base') + 'settigs/downloaders/' + log.debug('oauth_token is: %s', oauth_token) self.conf('oauth_token', value = oauth_token); return 'redirect', Env.get('web_base') + 'settings/downloaders/' @@ -86,37 +86,33 @@ class PutIO(DownloaderBase): log.debug(transfers); release_downloads = ReleaseDownloadList(self) for t in transfers: - if t.id in ids: - log.debug('downloading list is %s', self.downloadingList) - if t.status == "COMPLETED" and self.conf('download') == False : - status = 'completed' - # So check if we are trying to download something - elif t.status == "COMPLETED" and self.conf('download') == True: - # Assume we are done - status = 'completed' - # This is not ideal, right now if we are downloading anything we can't mark anything as completed - # The name and ID don't match currently so I can't use those... - if not self.downloadingList: - now = datetime.datetime.utcnow() - date_time = datetime.datetime.strptime(t.finished_at,"%Y-%m-%dT%H:%M:%S") - # We need to make sure a race condition didn't happen - if (now - date_time) < datetime.timedelta(minutes=5): - #5 minutes haven't passed so we wait - status = 'busy' + if t.id in ids: + log.debug('downloading list is %s', self.downloadingList) + if t.status == "COMPLETED" and self.conf('download') == False : + status = 'completed' + # So check if we are trying to download something + elif t.status == "COMPLETED" and self.conf('download') == True: + # Assume we are done + status = 'completed' + if not self.downloadingList: + now = datetime.datetime.utcnow() + date_time = datetime.datetime.strptime(t.finished_at,"%Y-%m-%dT%H:%M:%S") + # We need to make sure a race condition didn't happen + if (now - date_time) < datetime.timedelta(minutes=5): + # 5 minutes haven't passed so we wait + status = 'busy' + else: + # If we have the file_id in the downloadingList mark it as busy + if str(t.file_id) in self.downloadingList: + status = 'busy' else: - # If we have the file_id in the downloadingList mark it as busy - if str(t.file_id) in self.downloadingList: - status = 'busy' - else: - status = 'busy' - release_downloads.append({ + status = 'busy' + release_downloads.append({ 'id' : t.id, 'name': t.name, 'status': status, 'timeleft': t.estimated_time, - }) - - log.debug(release_downloads) + }) return release_downloads @@ -136,14 +132,19 @@ class PutIO(DownloaderBase): def getFromPutio(self, **kwargs): - file_id = str(kwargs.get('file_id')) + try: + file_id = str(kwargs.get('file_id')) + except: + return { + 'success' : False, + } log.info('Put.io Download has been called file_id is %s', file_id) if file_id not in self.downloadingList: - self.downloadingList.append(file_id) - fireEventAsync('putio.download',fid = file_id) - return { + self.downloadingList.append(file_id) + fireEventAsync('putio.download',fid = file_id) + return { 'success': True, - } + } return { 'success': False, } From 8de5fcdac657ee52108f07ef45ca22f18bb26a33 Mon Sep 17 00:00:00 2001 From: Andrew Dumaresq Date: Sat, 20 Sep 2014 19:39:35 -0400 Subject: [PATCH 13/38] fixed button name --- couchpotato/core/downloaders/putio/static/putio.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/putio/static/putio.js b/couchpotato/core/downloaders/putio/static/putio.js index 1b71c263..f58292ae 100644 --- a/couchpotato/core/downloaders/putio/static/putio.js +++ b/couchpotato/core/downloaders/putio/static/putio.js @@ -26,10 +26,10 @@ var PutIODownloader = new Class({ (putio_set > 0) ? [ self.unregister = new Element('a.button.red', { - 'text': 'Unregister "'+fieldset.getElement('input[name*=screen_name]').get('value')+'"', + 'text': 'Unregister "'+fieldset.getElement('input[name*=oauth_token]').get('value')+'"', 'events': { 'click': function(){ - fieldset.getElements('input[type=text]').set('value', '').fireEvent('change'); + fieldset.getElements('input[name*=oauth_token]').set('value', '').fireEvent('change'); self.unregister.destroy(); self.unregister_or.destroy(); From 2e52c8124a875423a50e6d1648145273458e84b8 Mon Sep 17 00:00:00 2001 From: Viktor Elofsson Date: Thu, 2 Oct 2014 20:34:43 +0200 Subject: [PATCH 14/38] Implemented a downloader for Hadouken. --- couchpotato/core/downloaders/hadouken.py | 337 +++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 couchpotato/core/downloaders/hadouken.py diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py new file mode 100644 index 00000000..ac6eae12 --- /dev/null +++ b/couchpotato/core/downloaders/hadouken.py @@ -0,0 +1,337 @@ +from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList +from couchpotato.core.helpers.encoding import isInt, ss, sp +from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost +from couchpotato.core.logger import CPLog + +from base64 import b16encode, b32decode, b64encode +from distutils.version import LooseVersion +import httplib +import json +import os +import re +import urllib +import urllib2 + +log = CPLog(__name__) + +autoload = 'Hadouken' + +class Hadouken(DownloaderBase): + protocol = ['torrent', 'torrent_magnet'] + hadouken_api = None + + def connect(self): + # Load host from config and split out port. + host = cleanHost(self.conf('host'), protocol = False).split(':') + + if not isInt(host[1]): + log.error('Config properties are not filled in correctly, port is missing.') + return False + + if not self.conf('apikey'): + log.error('Config properties are not filled in correctly, API key is missing.') + return False + + self.hadouken_api = HadoukenAPI(host[0], port = host[1], apiKey = self.conf('apikey')) + + return True + + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + + log.debug("Sending '%s' (%s) to Hadouken.", (data.get('name'), data.get('protocol'))) + + if not self.connect(): + return False + + torrent_params = {} + + if self.conf('label'): + torrent_params['label'] = self.conf('label') + + torrent_filename = self.createFileName(data, filedata, media) + + if data.get('protocol') == 'torrent_magnet': + torrent_hash = re.findall('urn:btih:([\w]{32,40})', data.get('url'))[0].upper() + torrent_params['trackers'] = self.torrent_trackers + torrent_params['name'] = torrent_filename + else: + torrent_hash = sha1(benc(info)).hexdigest().upper() + + # Convert base 32 to hex + if len(torrent_hash) == 32: + torrent_hash = b16encode(b32decode(torrent_hash)) + + # Send request to Hadouken + if data.get('protocol') == 'torrent_magnet': + self.hadouken_api.add_magnet_link(data.get('url'), torrent_params) + else: + self.hadouken_api.add_file(filedata, torrent_params) + + return self.downloadReturnId(torrent_hash) + + def test(self): + """ Tests the given host:port and API key """ + + if not self.connect(): + return False + + version = self.hadouken_api.get_version() + + if not version: + log.error('Could not get Hadouken version.') + return False + + if LooseVersion(version) >= LooseVersion('4.4.1'): + return True + + log.error('Hadouken v4.1.1 (or newer) required. Found v%s', version) + return False + + def getAllDownloadStatus(self, ids): + log.debug('Checking Hadouken download status.') + + if not self.connect(): + return [] + + release_downloads = ReleaseDownloadList(self) + queue = self.hadouken_api.get_by_hash_list(ids) + + if not queue: + return [] + + for torrent in queue: + if torrent is None: + continue + + torrent_dir = os.path.join(torrent['SavePath'], torrent['Name']) + torrent_files = [] + status = 'busy' + + for torrent_file in torrent['Files']: + torrent_files.append(sp(os.path.join(torrent['SavePath'], torrent_file['Path']))) + + if os.path.isdir(torrent_dir): + torrent['SavePath'] = torrent_dir + + release_downloads.append({ + 'id': torrent['InfoHash'].upper(), + 'name': torrent['Name'], + 'status': self.get_torrent_status(torrent), + 'seed_ratio': self.get_seed_ratio(torrent), + 'original_status': torrent['State'], + 'timeleft': -1, + 'folder': sp(torrent['SavePath']), + 'files': torrent_files + }) + + return release_downloads + + def get_seed_ratio(self, torrent): + """ Returns the seed ratio for a given torrent. + + Keyword arguments: + torrent -- The torrent to calculate seed ratio for. + """ + + up = torrent['TotalUploadedBytes'] + down = torrent['TotalDownloadedBytes'] + + if up > 0 and down > 0: + return up / down + + return 0 + + def get_torrent_status(self, torrent): + """ Returns the CouchPotato status for a given torrent. + + Keyword arguments: + torrent -- The torrent to translate status for. + """ + + if torrent['IsSeeding'] and torrent['IsFinished'] and torrent['Paused']: + return 'completed' + + if torrent['IsSeeding']: + return 'seeding' + + return 'busy' + + def pause(self, release_download, pause = True): + """ Pauses or resumes the torrent specified by the ID field + in release_download. + + Keyword arguments: + release_download -- The CouchPotato release_download to pause/resume. + pause -- Boolean indicating whether to pause or resume. + """ + + if not self.connect(): + return False + + return self.hadouken_api.pause(release_download['id'], pause) + + def removeFailed(self, release_download): + """ Removes a failed torrent and also remove the data associated with it. + + Keyword arguments: + release_download -- The CouchPotato release_download to remove. + """ + + log.info('%s failed downloading, deleting...', release_download['name']) + + if not self.connect(): + return False + + return self.hadouken_api.remove(release_download['id'], remove_data = True) + + def processComplete(self, release_download, delete_files = False): + """ Removes the completed torrent from Hadouken and optionally removes the data + associated with it. + + Keyword arguments: + release_download -- The CouchPotato release_download to remove. + delete_files: Boolean indicating whether to remove the associated data. + """ + + log.debug('Requesting Hadouken to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) + + if not self.connect(): + return False + + return self.hadouken_api.remove(release_download['id'], remove_data = delete_files) + +class HadoukenAPI(object): + def __init__(self, host = 'localhost', port = 7890, apiKey = None): + self.url = 'http://' + str(host) + ':' + str(port) + self.apiKey = apiKey + self.requestId = 0; + + self.opener = urllib2.build_opener() + self.opener.addheaders = [('User-agent', 'couchpotato-hadouken-client/1.0'), ('Accept', 'application/json')] + + if not apiKey: + log.error('API key missing.') + + def add_file(self, filedata, torrent_params): + data = { + 'method': 'torrents.addFile', + 'params': [ b64encode(filedata), torrent_params ] + } + + return self._request('/jsonrpc', data) + + def add_magnet_link(self, magnetLink, torrent_params): + data = { + 'method': 'torrents.addUrl', + 'params': [ magnetLink, torrent_params ] + } + + return self._request('/jsonrpc', data) + + def get_by_hash_list(self, infoHashList): + data = { + 'method': 'torrents.getByInfoHashList', + 'params': [ infoHashList ] + } + + return self._request('/jsonrpc', data) + + def get_version(self): + data = { + 'method': 'core.getVersion', + 'params': None + } + + result = self._request('/jsonrpc', data) + + if not result: + return False + + return result['Version'] + + def pause(self, id, pause): + data = { + 'method': 'torrents.pause', + 'params': [ id ] + } + + if not pause: + data['method'] = 'torrents.resume' + + return self._request('/jsonrpc', data) + + def remove(self, id, remove_data = False): + data = { + 'method': 'torrents.remove', + 'params': [ id, remove_data ] + } + + return self._request('/jsonrpc', data) + + + def _request(self, url, data): + self.requestId += 1 + + data['jsonrpc'] = '2.0' + data['id'] = self.requestId + + request = urllib2.Request(self.url + url, data = json.dumps(data)) + request.add_header('Authorization', 'Token ' + self.apiKey) + request.add_header('Content-Type', 'application/json') + + try: + f = self.opener.open(request) + response = f.read() + f.close() + + obj = json.loads(response) + + if not 'error' in obj.keys(): + return obj['result'] + + log.error('JSONRPC error, %s: %s', obj['error']['code'], obj['error']['message']) + except httplib.InvalidURL as err: + log.error('Invalid Hadouken host, check your config %s', err) + except urllib2.HTTPError as err: + if err.code == 401: + log.error('Invalid Hadouken API key, check your config') + else: + log.error('Hadouken HTTPError: %s', err) + except urllib2.URLError as err: + log.error('Unable to connect to Hadouken %s', err) + + return False + + +config = [{ + 'name': 'hadouken', + 'groups': [ + { + 'tab': 'downloaders', + 'list': 'download_providers', + 'name': 'hadouken', + 'label': 'Hadouken', + 'description': 'Use Hadouken (>= v4.4.1) to download torrents.', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + 'radio_group': 'torrent' + }, + { + 'name': 'host', + 'default': 'localhost:7890' + }, + { + 'name': 'apikey', + 'label': 'API key', + 'type': 'password' + } + ] + } + ] +}] \ No newline at end of file From 61f634a21ed0c9429965d072258ae4a52a113817 Mon Sep 17 00:00:00 2001 From: Viktor Elofsson Date: Tue, 21 Oct 2014 16:52:28 +0200 Subject: [PATCH 15/38] Refactored Hadouken downloader. --- couchpotato/core/downloaders/hadouken.py | 121 +++++++++++++++++------ 1 file changed, 93 insertions(+), 28 deletions(-) diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index ac6eae12..b27a645a 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -4,7 +4,9 @@ from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost from couchpotato.core.logger import CPLog from base64 import b16encode, b32decode, b64encode +from bencode import bencode as benc, bdecode from distutils.version import LooseVersion +from hashlib import sha1 import httplib import json import os @@ -32,7 +34,7 @@ class Hadouken(DownloaderBase): log.error('Config properties are not filled in correctly, API key is missing.') return False - self.hadouken_api = HadoukenAPI(host[0], port = host[1], apiKey = self.conf('apikey')) + self.hadouken_api = HadoukenAPI(host[0], port = host[1], api_key = self.conf('api_key')) return True @@ -57,6 +59,7 @@ class Hadouken(DownloaderBase): torrent_params['trackers'] = self.torrent_trackers torrent_params['name'] = torrent_filename else: + info = bdecode(filedata)['info'] torrent_hash = sha1(benc(info)).hexdigest().upper() # Convert base 32 to hex @@ -83,10 +86,11 @@ class Hadouken(DownloaderBase): log.error('Could not get Hadouken version.') return False - if LooseVersion(version) >= LooseVersion('4.4.1'): + # The minimum required version of Hadouken is 4.5.6. + if LooseVersion(version) >= LooseVersion('4.5.6'): return True - log.error('Hadouken v4.1.1 (or newer) required. Found v%s', version) + log.error('Hadouken v4.5.6 (or newer) required. Found v%s', version) return False def getAllDownloadStatus(self, ids): @@ -105,15 +109,27 @@ class Hadouken(DownloaderBase): if torrent is None: continue - torrent_dir = os.path.join(torrent['SavePath'], torrent['Name']) + torrent_filelist = self.hadouken_api.get_files_by_hash(torrent['InfoHash']) torrent_files = [] - status = 'busy' - for torrent_file in torrent['Files']: - torrent_files.append(sp(os.path.join(torrent['SavePath'], torrent_file['Path']))) + save_path = torrent['SavePath'] - if os.path.isdir(torrent_dir): - torrent['SavePath'] = torrent_dir + # The 'Path' key for each file_item contains + # the full path to the single file relative to the + # torrents save path. + + # For a single file torrent the result would be, + # - Save path: "C:\Downloads" + # - file_item['Path'] = "file1.iso" + # Resulting path: "C:\Downloads\file1.iso" + + # For a multi file torrent the result would be, + # - Save path: "C:\Downloads" + # - file_item['Path'] = "dirname/file1.iso" + # Resulting path: "C:\Downloads\dirname/file1.iso" + + for file_item in torrent_filelist: + torrent_files.append(sp(os.path.join(save_path, file_item['Path']))) release_downloads.append({ 'id': torrent['InfoHash'].upper(), @@ -122,7 +138,7 @@ class Hadouken(DownloaderBase): 'seed_ratio': self.get_seed_ratio(torrent), 'original_status': torrent['State'], 'timeleft': -1, - 'folder': sp(torrent['SavePath']), + 'folder': sp(save_path if len(torrent_files == 1) else os.path.join(save_path, torrent['Name'])), 'files': torrent_files }) @@ -203,82 +219,127 @@ class Hadouken(DownloaderBase): return self.hadouken_api.remove(release_download['id'], remove_data = delete_files) class HadoukenAPI(object): - def __init__(self, host = 'localhost', port = 7890, apiKey = None): + def __init__(self, host = 'localhost', port = 7890, api_key = None): self.url = 'http://' + str(host) + ':' + str(port) - self.apiKey = apiKey + self.api_key = api_key self.requestId = 0; self.opener = urllib2.build_opener() self.opener.addheaders = [('User-agent', 'couchpotato-hadouken-client/1.0'), ('Accept', 'application/json')] - if not apiKey: + if not api_key: log.error('API key missing.') def add_file(self, filedata, torrent_params): + """ Add a file to Hadouken with the specified parameters. + + Keyword arguments: + filedata -- The binary torrent data. + torrent_params -- Additional parameters for the file. + """ data = { 'method': 'torrents.addFile', 'params': [ b64encode(filedata), torrent_params ] } - return self._request('/jsonrpc', data) + return self._request(data) def add_magnet_link(self, magnetLink, torrent_params): + """ Add a magnet link to Hadouken with the specified parameters. + + Keyword arguments: + magnetLink -- The magnet link to send. + torrent_params -- Additional parameters for the magnet link. + """ data = { 'method': 'torrents.addUrl', 'params': [ magnetLink, torrent_params ] } - return self._request('/jsonrpc', data) + return self._request(data) def get_by_hash_list(self, infoHashList): + """ Gets a list of torrents filtered by the given info hash list. + + Keyword arguments: + infoHashList -- A list of info hashes. + """ data = { 'method': 'torrents.getByInfoHashList', 'params': [ infoHashList ] } - return self._request('/jsonrpc', data) + return self._request(data) + + def get_files_by_hash(self, infoHash): + """ Gets a list of files for the torrent identified by the + given info hash. + + Keyword arguments: + infoHash -- The info hash of the torrent to return files for. + """ + data = { + 'method': 'torrents.getFiles', + 'params': [ infoHash ] + } + + return self._request(data) def get_version(self): + """ Gets the version, commitish and build date of Hadouken. """ data = { 'method': 'core.getVersion', 'params': None } - result = self._request('/jsonrpc', data) + result = self._request(data) if not result: return False return result['Version'] - def pause(self, id, pause): + def pause(self, infoHash, pause): + """ Pauses/unpauses the torrent identified by the given info hash. + + Keyword arguments: + infoHash -- The info hash of the torrent to operate on. + pause -- If true, pauses the torrent. Otherwise resumes. + """ data = { 'method': 'torrents.pause', - 'params': [ id ] + 'params': [ infoHash ] } if not pause: data['method'] = 'torrents.resume' - return self._request('/jsonrpc', data) + return self._request(data) - def remove(self, id, remove_data = False): + def remove(self, infoHash, remove_data = False): + """ Removes the torrent identified by the given info hash and + optionally removes the data as well. + + Keyword arguments: + infoHash -- The info hash of the torrent to remove. + remove_data -- If true, removes the data associated with the torrent. + """ data = { 'method': 'torrents.remove', - 'params': [ id, remove_data ] + 'params': [ infoHash, remove_data ] } - return self._request('/jsonrpc', data) + return self._request(data) - def _request(self, url, data): + def _request(self, data): self.requestId += 1 data['jsonrpc'] = '2.0' data['id'] = self.requestId - request = urllib2.Request(self.url + url, data = json.dumps(data)) - request.add_header('Authorization', 'Token ' + self.apiKey) + request = urllib2.Request(self.url + '/jsonrpc', data = json.dumps(data)) + request.add_header('Authorization', 'Token ' + self.api_key) request.add_header('Content-Type', 'application/json') try: @@ -313,7 +374,7 @@ config = [{ 'list': 'download_providers', 'name': 'hadouken', 'label': 'Hadouken', - 'description': 'Use Hadouken (>= v4.4.1) to download torrents.', + 'description': 'Use Hadouken (>= v4.5.6) to download torrents.', 'wizard': True, 'options': [ { @@ -327,9 +388,13 @@ config = [{ 'default': 'localhost:7890' }, { - 'name': 'apikey', + 'name': 'api_key', 'label': 'API key', 'type': 'password' + }, + { + 'name': 'label', + 'description': 'Label to add torrent as.' } ] } From 28019b0a096bdc36e2cef259cf4925c7b84de9e0 Mon Sep 17 00:00:00 2001 From: Mathew Paret Date: Mon, 10 Nov 2014 18:39:58 +0530 Subject: [PATCH 16/38] Transmission status 16 is for "Stopped". So we need to detect a download as completed even if it is stopped but percent done is 100 --- couchpotato/core/downloaders/transmission.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index d6112a91..400e00ce 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -119,6 +119,8 @@ class Transmission(DownloaderBase): status = 'failed' elif torrent['status'] == 0 and torrent['percentDone'] == 1: status = 'completed' + elif torrent['status'] == 16 and torrent['percentDone'] == 1: + status = 'completed' elif torrent['status'] in [5, 6]: status = 'seeding' From 87338760ad2f8773330b1934946bbd23e35274ac Mon Sep 17 00:00:00 2001 From: Mathew Paret Date: Mon, 10 Nov 2014 18:47:37 +0530 Subject: [PATCH 17/38] Feature #3967 - Added IMDB link to download complete tweet --- .../core/notifications/twitter/main.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/notifications/twitter/main.py b/couchpotato/core/notifications/twitter/main.py index 0d02191e..fd7d2844 100644 --- a/couchpotato/core/notifications/twitter/main.py +++ b/couchpotato/core/notifications/twitter/main.py @@ -34,11 +34,13 @@ class Twitter(Notification): def notify(self, message = '', data = None, listener = None): if not data: data = {} - + log.debug('Data in notification is %s', data['identifier']) api = Api(self.consumer_key, self.consumer_secret, self.conf('access_token_key'), self.conf('access_token_secret')) direct_message = self.conf('direct_message') direct_message_users = self.conf('screen_name') + + message = '%s%s' % (message,' - http://www.imdb.com/title/' + data['identifier'] if data['identifier'] else '') mention = self.conf('mention') mention_tag = None @@ -59,11 +61,20 @@ class Twitter(Notification): update_message = '[%s] %s' % (self.default_title, message) if len(update_message) > 140: if mention_tag: - api.PostUpdate(update_message[:135 - len(mention_tag)] + ('%s 1/2 ' % mention_tag)) - api.PostUpdate(update_message[135 - len(mention_tag):] + ('%s 2/2 ' % mention_tag)) + if update_message.find(" - http://www.imdb.com/title/") < 141 and update_message.find(" - http://www.imdb.com/title/") > 0: + api.PostUpdate(update_message[:update_message.find(" - http://www.imdb.com/title/") - len(mention_tag)] + ('%s 1/2 ' % mention_tag)) + api.PostUpdate(update_message[update_message.find(" - http://www.imdb.com/title/") - len(mention_tag):] + ('%s 2/2 ' % mention_tag)) + else: + api.PostUpdate(update_message[:135 - len(mention_tag)] + ('%s 1/2 ' % mention_tag)) + api.PostUpdate(update_message[135 - len(mention_tag):] + ('%s 2/2 ' % mention_tag)) else: - api.PostUpdate(update_message[:135] + ' 1/2') - api.PostUpdate(update_message[135:] + ' 2/2') + if update_message.find(" - http://www.imdb.com/title/") < 141 and update_message.find(" - http://www.imdb.com/title/") > 0: + api.PostUpdate(update_message[:update_message.find(" - http://www.imdb.com/title/")] + ' 1/2') + api.PostUpdate(update_message[update_message.find(" - http://www.imdb.com/title/"):] + ' 2/2') + else: + api.PostUpdate(update_message[:135] + ' 1/2') + api.PostUpdate(update_message[135:] + ' 2/2') + else: api.PostUpdate(update_message) except Exception as e: From c1b6811b8aaf9cb38485ff0c8699beece2512e12 Mon Sep 17 00:00:00 2001 From: Paul Saab Date: Sat, 15 Nov 2014 15:48:45 -0800 Subject: [PATCH 18/38] Tornado requires two sockets to support IPv6 Tornado sets setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) to force IPv6 sockets to only be used for IPv6 connections. create a separate socket to allow for CouchPotato to be used over IPv6. --- couchpotato/runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index b7803976..7b834149 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -244,11 +244,13 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Basic config host = Env.setting('host', default = '0.0.0.0') + host6 = Env.setting('host6', default = '::') # app.debug = development config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', + 'host6': host6 if host6 and len(host6) > 0 else '::', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), } @@ -331,6 +333,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En while try_restart: try: server.listen(config['port'], config['host']) + server.listen(config['port'], config['host6']) loop.start() server.close_all_connections() server.stop() From 52478a00db0bbaf986d7a49a2a0d77fe367bd23c Mon Sep 17 00:00:00 2001 From: Mathew Paret Date: Thu, 27 Nov 2014 18:13:41 +0530 Subject: [PATCH 19/38] Revert "Feature #3967 - Added IMDB link to download complete tweet" This reverts commit 87338760ad2f8773330b1934946bbd23e35274ac. --- .../core/notifications/twitter/main.py | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/couchpotato/core/notifications/twitter/main.py b/couchpotato/core/notifications/twitter/main.py index fd7d2844..0d02191e 100644 --- a/couchpotato/core/notifications/twitter/main.py +++ b/couchpotato/core/notifications/twitter/main.py @@ -34,13 +34,11 @@ class Twitter(Notification): def notify(self, message = '', data = None, listener = None): if not data: data = {} - log.debug('Data in notification is %s', data['identifier']) + api = Api(self.consumer_key, self.consumer_secret, self.conf('access_token_key'), self.conf('access_token_secret')) direct_message = self.conf('direct_message') direct_message_users = self.conf('screen_name') - - message = '%s%s' % (message,' - http://www.imdb.com/title/' + data['identifier'] if data['identifier'] else '') mention = self.conf('mention') mention_tag = None @@ -61,20 +59,11 @@ class Twitter(Notification): update_message = '[%s] %s' % (self.default_title, message) if len(update_message) > 140: if mention_tag: - if update_message.find(" - http://www.imdb.com/title/") < 141 and update_message.find(" - http://www.imdb.com/title/") > 0: - api.PostUpdate(update_message[:update_message.find(" - http://www.imdb.com/title/") - len(mention_tag)] + ('%s 1/2 ' % mention_tag)) - api.PostUpdate(update_message[update_message.find(" - http://www.imdb.com/title/") - len(mention_tag):] + ('%s 2/2 ' % mention_tag)) - else: - api.PostUpdate(update_message[:135 - len(mention_tag)] + ('%s 1/2 ' % mention_tag)) - api.PostUpdate(update_message[135 - len(mention_tag):] + ('%s 2/2 ' % mention_tag)) + api.PostUpdate(update_message[:135 - len(mention_tag)] + ('%s 1/2 ' % mention_tag)) + api.PostUpdate(update_message[135 - len(mention_tag):] + ('%s 2/2 ' % mention_tag)) else: - if update_message.find(" - http://www.imdb.com/title/") < 141 and update_message.find(" - http://www.imdb.com/title/") > 0: - api.PostUpdate(update_message[:update_message.find(" - http://www.imdb.com/title/")] + ' 1/2') - api.PostUpdate(update_message[update_message.find(" - http://www.imdb.com/title/"):] + ' 2/2') - else: - api.PostUpdate(update_message[:135] + ' 1/2') - api.PostUpdate(update_message[135:] + ' 2/2') - + api.PostUpdate(update_message[:135] + ' 1/2') + api.PostUpdate(update_message[135:] + ' 2/2') else: api.PostUpdate(update_message) except Exception as e: From 0dca34958c9e2ca8d69918864678ed162ea30ebf Mon Sep 17 00:00:00 2001 From: voidstarstar Date: Thu, 27 Nov 2014 21:43:19 -0500 Subject: [PATCH 20/38] Added a parameter to the renamer API. Fixes #3845. The renamer now has a new 'to_folder' parameter. This parameter specifies where movies are moved to. --- couchpotato/core/plugins/renamer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index d6381a3e..bd8f14f4 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -35,6 +35,7 @@ class Renamer(Plugin): 'desc': 'For the renamer to check for new files to rename in a folder', 'params': { 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'}, + 'to_folder': {'desc': 'Optional: The folder to move releases to. Leave empty for default folder.'}, 'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'}, 'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'}, 'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'}, @@ -72,6 +73,7 @@ class Renamer(Plugin): async = tryInt(kwargs.get('async', 0)) base_folder = kwargs.get('base_folder') media_folder = sp(kwargs.get('media_folder')) + to_folder = kwargs.get('to_folder') # Backwards compatibility, to be removed after a few versions :) if not media_folder: @@ -95,13 +97,13 @@ class Renamer(Plugin): }) fire_handle = fireEvent if not async else fireEventAsync - fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download) + fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download, to_folder = to_folder) return { 'success': True } - def scan(self, base_folder = None, release_download = None): + def scan(self, base_folder = None, release_download = None, to_folder = None): if not release_download: release_download = {} if self.isDisabled(): @@ -115,7 +117,9 @@ class Renamer(Plugin): base_folder = sp(self.conf('from')) from_folder = sp(self.conf('from')) - to_folder = sp(self.conf('to')) + + if not to_folder: + to_folder = sp(self.conf('to')) # Get media folder to process media_folder = sp(release_download.get('folder')) From 15a0131587a271a6d357fe678b4a881e53cbf7e0 Mon Sep 17 00:00:00 2001 From: voidstarstar Date: Thu, 27 Nov 2014 21:51:30 -0500 Subject: [PATCH 21/38] Added renamer.progress API function. Fixes #4211. This function reports the status of the renamer. Progress value True means the renamer is currently running. Progress value False means the renamer is not currently running. --- couchpotato/core/plugins/renamer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index d6381a3e..a52e8ba3 100755 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -44,6 +44,13 @@ class Renamer(Plugin): }, }) + addApiView('renamer.progress', self.getProgress, docs = { + 'desc': 'Get the progress of current renamer scan', + 'return': {'type': 'object', 'example': """{ + 'progress': False || True, +}"""}, + }) + addEvent('renamer.scan', self.scan) addEvent('renamer.check_snatched', self.checkSnatched) @@ -67,6 +74,11 @@ class Renamer(Plugin): return True + def getProgress(self, **kwargs): + return { + 'progress': self.renaming_started + } + def scanView(self, **kwargs): async = tryInt(kwargs.get('async', 0)) From df2d7ec9c2e4533fb314933abee9abebc318a52c Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Dec 2014 15:39:33 +0100 Subject: [PATCH 22/38] Remove debug code --- couchpotato/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 7b834149..4e53562e 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -245,7 +245,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Basic config host = Env.setting('host', default = '0.0.0.0') host6 = Env.setting('host6', default = '::') - # app.debug = development + config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), From c6d326f97352891b17100d1aa551db21aea49ef5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Dec 2014 15:50:03 +0100 Subject: [PATCH 23/38] Move put.io API --- libs/pio/__init__.py | 0 {couchpotato/core/downloaders/putio => libs/pio}/api.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 libs/pio/__init__.py rename {couchpotato/core/downloaders/putio => libs/pio}/api.py (100%) diff --git a/libs/pio/__init__.py b/libs/pio/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/couchpotato/core/downloaders/putio/api.py b/libs/pio/api.py similarity index 100% rename from couchpotato/core/downloaders/putio/api.py rename to libs/pio/api.py From fe56a69e8f1860e3517f3853a5c3710e20157510 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Dec 2014 16:49:27 +0100 Subject: [PATCH 24/38] Put.IO cleanup --- couchpotato/core/downloaders/putio/main.py | 35 ++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 8184b27d..44fbf1fd 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -4,7 +4,7 @@ from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownlo from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from couchpotato.environment import Env -import api as pio +from pio import api as pio import datetime log = CPLog(__name__) @@ -13,12 +13,14 @@ autoload = 'Putiodownload' class PutIO(DownloaderBase): + protocol = ['torrent', 'torrent_magnet'] status_support = True downloadingList = [] - # This is the location on the Internet of the Oauth helper server - oauthServerURL = "http://sabnzbd.dumaresq.ca/index.cgi" + # This is the location on the Internet of the Oauth helper server + #oauthServerURL = 'https://api.couchpota.to/validate/putio/' + oauthServerURL = 'http://localhost:3000/authorize/putio/' def __init__(self): addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { @@ -27,8 +29,8 @@ class PutIO(DownloaderBase): addApiView('downloader.putio.auth_url', self.getAuthorizationUrl) addApiView('downloader.putio.credentials', self.getCredentials) addEvent('putio.download', self.putioDownloader) - return super(PutIO, self).__init__() + return super(PutIO, self).__init__() def download(self, data = None, media = None, filedata = None): if not media: media = {} @@ -47,7 +49,6 @@ class PutIO(DownloaderBase): log.debug('resp is %s', resp.id); return self.downloadReturnId(resp.id) - def test(self): try: client = pio.Client(self.conf('oauth_token')) @@ -57,39 +58,44 @@ class PutIO(DownloaderBase): log.info('Failed to get file listing, check OAUTH_TOKEN') return False - def getAuthorizationUrl(self, host = None, **kwargs): + callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) log.debug('callback_url is %s', callback_url) + target_url = self.oauthServerURL + "?target=" + callback_url log.debug('target_url is %s', target_url) + return { 'success': True, 'url': target_url, } - def getCredentials(self, **kwargs): try: oauth_token = kwargs.get('oauth') except: - return 'redirect', Env.get('web_base') + 'settigs/downloaders/' + return 'redirect', Env.get('web_base') + 'settings/downloaders/' log.debug('oauth_token is: %s', oauth_token) self.conf('oauth_token', value = oauth_token); return 'redirect', Env.get('web_base') + 'settings/downloaders/' - def getAllDownloadStatus(self, ids): - log.debug('Checking putio download status.') + + log.debug('Checking putio download status.') client = pio.Client(self.conf('oauth_token')) + transfers = client.Transfer.list() + log.debug(transfers); release_downloads = ReleaseDownloadList(self) for t in transfers: if t.id in ids: + log.debug('downloading list is %s', self.downloadingList) if t.status == "COMPLETED" and self.conf('download') == False : status = 'completed' + # So check if we are trying to download something elif t.status == "COMPLETED" and self.conf('download') == True: # Assume we are done @@ -113,15 +119,18 @@ class PutIO(DownloaderBase): 'status': status, 'timeleft': t.estimated_time, }) + return release_downloads - def putioDownloader(self, fid): + log.info('Put.io Real downloader called with file_id: %s',fid) client = pio.Client(self.conf('oauth_token')) + log.debug('About to get file List') files = client.File.list() downloaddir = self.conf('download_dir') + for f in files: if str(f.id) == str(fid): client.File.download(f, dest = downloaddir, delete_after_download = self.conf('delete_file')) @@ -130,14 +139,15 @@ class PutIO(DownloaderBase): return True - def getFromPutio(self, **kwargs): + try: file_id = str(kwargs.get('file_id')) except: return { 'success' : False, } + log.info('Put.io Download has been called file_id is %s', file_id) if file_id not in self.downloadingList: self.downloadingList.append(file_id) @@ -145,6 +155,7 @@ class PutIO(DownloaderBase): return { 'success': True, } + return { 'success': False, } From defe256f1b505e7be1342c749b677f99db1aa989 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Dec 2014 16:52:43 +0100 Subject: [PATCH 25/38] Correct url --- couchpotato/core/downloaders/putio/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index 44fbf1fd..fca7032f 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -19,8 +19,8 @@ class PutIO(DownloaderBase): downloadingList = [] # This is the location on the Internet of the Oauth helper server - #oauthServerURL = 'https://api.couchpota.to/validate/putio/' - oauthServerURL = 'http://localhost:3000/authorize/putio/' + oauthServerURL = 'https://api.couchpota.to/validate/putio/' + # oauthServerURL = 'http://localhost:3000/authorize/putio/' def __init__(self): addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { From 6a174716af1c091b5c9262f48deac64fea68b8c3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Dec 2014 22:52:10 +0100 Subject: [PATCH 26/38] underscored variables --- .../core/downloaders/putio/__init__.py | 2 +- couchpotato/core/downloaders/putio/main.py | 32 ++++++++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index af05e360..cabf9a3a 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -40,7 +40,7 @@ config = [{ }, { 'name': 'delete_file', - 'description': 'Set this to remove the file from putio after sucessful download Note: does nothing if you don\'t select download', + 'description': ('Set this to remove the file from putio after sucessful download','Does nothing if you don\'t select download'), 'type': 'bool', 'default': 0, }, diff --git a/couchpotato/core/downloaders/putio/main.py b/couchpotato/core/downloaders/putio/main.py index fca7032f..76ac2033 100644 --- a/couchpotato/core/downloaders/putio/main.py +++ b/couchpotato/core/downloaders/putio/main.py @@ -15,12 +15,8 @@ autoload = 'Putiodownload' class PutIO(DownloaderBase): protocol = ['torrent', 'torrent_magnet'] - status_support = True - downloadingList = [] - - # This is the location on the Internet of the Oauth helper server - oauthServerURL = 'https://api.couchpota.to/validate/putio/' - # oauthServerURL = 'http://localhost:3000/authorize/putio/' + downloading_list = [] + oauth_authenticate = 'https://api.couchpota.to/authorize/putio/' def __init__(self): addApiView('downloader.putio.getfrom', self.getFromPutio, docs = { @@ -44,10 +40,10 @@ class PutIO(DownloaderBase): # Note callback_host is NOT our address, it's the internet host that putio can call too callbackurl = None if self.conf('download'): - callbackurl = 'http://' + self.conf('callback_host') + '/' + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) + callbackurl = 'http://' + self.conf('callback_host') + '/' + '%sdownloader.putio.getfrom/' %Env.get('api_base'.strip('/')) resp = client.Transfer.add_url(url, callback_url = callbackurl) log.debug('resp is %s', resp.id); - return self.downloadReturnId(resp.id) + return self.downloadReturnId(resp.id) def test(self): try: @@ -63,10 +59,10 @@ class PutIO(DownloaderBase): callback_url = cleanHost(host) + '%sdownloader.putio.credentials/' % (Env.get('api_base').lstrip('/')) log.debug('callback_url is %s', callback_url) - target_url = self.oauthServerURL + "?target=" + callback_url + target_url = self.oauth_authenticate + "?target=" + callback_url log.debug('target_url is %s', target_url) - return { + return { 'success': True, 'url': target_url, } @@ -92,7 +88,7 @@ class PutIO(DownloaderBase): for t in transfers: if t.id in ids: - log.debug('downloading list is %s', self.downloadingList) + log.debug('downloading list is %s', self.downloading_list) if t.status == "COMPLETED" and self.conf('download') == False : status = 'completed' @@ -100,7 +96,7 @@ class PutIO(DownloaderBase): elif t.status == "COMPLETED" and self.conf('download') == True: # Assume we are done status = 'completed' - if not self.downloadingList: + if not self.downloading_list: now = datetime.datetime.utcnow() date_time = datetime.datetime.strptime(t.finished_at,"%Y-%m-%dT%H:%M:%S") # We need to make sure a race condition didn't happen @@ -108,8 +104,8 @@ class PutIO(DownloaderBase): # 5 minutes haven't passed so we wait status = 'busy' else: - # If we have the file_id in the downloadingList mark it as busy - if str(t.file_id) in self.downloadingList: + # If we have the file_id in the downloading_list mark it as busy + if str(t.file_id) in self.downloading_list: status = 'busy' else: status = 'busy' @@ -135,7 +131,7 @@ class PutIO(DownloaderBase): if str(f.id) == str(fid): client.File.download(f, dest = downloaddir, delete_after_download = self.conf('delete_file')) # Once the download is complete we need to remove it from the running list. - self.downloadingList.remove(fid) + self.downloading_list.remove(fid) return True @@ -149,8 +145,8 @@ class PutIO(DownloaderBase): } log.info('Put.io Download has been called file_id is %s', file_id) - if file_id not in self.downloadingList: - self.downloadingList.append(file_id) + if file_id not in self.downloading_list: + self.downloading_list.append(file_id) fireEventAsync('putio.download',fid = file_id) return { 'success': True, @@ -158,5 +154,5 @@ class PutIO(DownloaderBase): return { 'success': False, - } + } From fc1c95fefb6d6148a5e5c7e5441547a87ce8d704 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 1 Dec 2014 23:00:59 +0100 Subject: [PATCH 27/38] Description --- couchpotato/core/downloaders/putio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/putio/__init__.py b/couchpotato/core/downloaders/putio/__init__.py index cabf9a3a..114ad6d8 100644 --- a/couchpotato/core/downloaders/putio/__init__.py +++ b/couchpotato/core/downloaders/putio/__init__.py @@ -30,7 +30,7 @@ config = [{ }, { 'name': 'callback_host', - 'description': 'This is used to generate the callback url', + 'description': 'External reachable url to CP so put.io can do it\'s thing', }, { 'name': 'download', From 1d73fd9d7eab8a68212fec708b3bfd26a84996c0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 2 Dec 2014 11:15:29 +0100 Subject: [PATCH 28/38] Import optimize --- couchpotato/core/downloaders/hadouken.py | 35 +++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index b27a645a..2570b591 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -1,23 +1,24 @@ -from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList -from couchpotato.core.helpers.encoding import isInt, ss, sp -from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost -from couchpotato.core.logger import CPLog - from base64 import b16encode, b32decode, b64encode -from bencode import bencode as benc, bdecode from distutils.version import LooseVersion from hashlib import sha1 import httplib import json import os import re -import urllib import urllib2 +from couchpotato.core._base.downloader.main import DownloaderBase, ReleaseDownloadList +from couchpotato.core.helpers.encoding import isInt, sp +from couchpotato.core.helpers.variable import cleanHost +from couchpotato.core.logger import CPLog +from bencode import bencode as benc, bdecode + + log = CPLog(__name__) autoload = 'Hadouken' + class Hadouken(DownloaderBase): protocol = ['torrent', 'torrent_magnet'] hadouken_api = None @@ -211,13 +212,15 @@ class Hadouken(DownloaderBase): delete_files: Boolean indicating whether to remove the associated data. """ - log.debug('Requesting Hadouken to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) + log.debug('Requesting Hadouken to remove the torrent %s%s.', + (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) if not self.connect(): return False return self.hadouken_api.remove(release_download['id'], remove_data = delete_files) + class HadoukenAPI(object): def __init__(self, host = 'localhost', port = 7890, api_key = None): self.url = 'http://' + str(host) + ':' + str(port) @@ -239,7 +242,7 @@ class HadoukenAPI(object): """ data = { 'method': 'torrents.addFile', - 'params': [ b64encode(filedata), torrent_params ] + 'params': [b64encode(filedata), torrent_params] } return self._request(data) @@ -253,7 +256,7 @@ class HadoukenAPI(object): """ data = { 'method': 'torrents.addUrl', - 'params': [ magnetLink, torrent_params ] + 'params': [magnetLink, torrent_params] } return self._request(data) @@ -266,7 +269,7 @@ class HadoukenAPI(object): """ data = { 'method': 'torrents.getByInfoHashList', - 'params': [ infoHashList ] + 'params': [infoHashList] } return self._request(data) @@ -280,7 +283,7 @@ class HadoukenAPI(object): """ data = { 'method': 'torrents.getFiles', - 'params': [ infoHash ] + 'params': [infoHash] } return self._request(data) @@ -308,7 +311,7 @@ class HadoukenAPI(object): """ data = { 'method': 'torrents.pause', - 'params': [ infoHash ] + 'params': [infoHash] } if not pause: @@ -326,7 +329,7 @@ class HadoukenAPI(object): """ data = { 'method': 'torrents.remove', - 'params': [ infoHash, remove_data ] + 'params': [infoHash, remove_data] } return self._request(data) @@ -362,9 +365,9 @@ class HadoukenAPI(object): log.error('Hadouken HTTPError: %s', err) except urllib2.URLError as err: log.error('Unable to connect to Hadouken %s', err) - + return False - + config = [{ 'name': 'hadouken', From 752191bc23cdde9de9d32da4531a12211291af95 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 2 Dec 2014 11:43:10 +0100 Subject: [PATCH 29/38] Comments --- couchpotato/core/downloaders/blackhole.py | 15 +++++++++++++++ couchpotato/core/downloaders/hadouken.py | 15 +++++++++++++++ couchpotato/core/downloaders/nzbget.py | 15 +++++++++++++++ couchpotato/core/downloaders/nzbvortex.py | 15 +++++++++++++++ couchpotato/core/downloaders/pneumatic.py | 15 +++++++++++++++ couchpotato/core/downloaders/qbittorrent_.py | 15 +++++++++++++++ couchpotato/core/downloaders/rtorrent_.py | 15 +++++++++++++++ couchpotato/core/downloaders/sabnzbd.py | 15 +++++++++++++++ couchpotato/core/downloaders/synology.py | 15 +++++++++++++++ couchpotato/core/downloaders/transmission.py | 15 +++++++++++++++ couchpotato/core/downloaders/utorrent.py | 15 +++++++++++++++ 11 files changed, 165 insertions(+) diff --git a/couchpotato/core/downloaders/blackhole.py b/couchpotato/core/downloaders/blackhole.py index 262776a8..5d4e54be 100644 --- a/couchpotato/core/downloaders/blackhole.py +++ b/couchpotato/core/downloaders/blackhole.py @@ -20,6 +20,21 @@ class Blackhole(DownloaderBase): status_support = False def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index 2570b591..0841832e 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -40,6 +40,21 @@ class Hadouken(DownloaderBase): return True def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/nzbget.py b/couchpotato/core/downloaders/nzbget.py index 54725bd5..2f3e686e 100644 --- a/couchpotato/core/downloaders/nzbget.py +++ b/couchpotato/core/downloaders/nzbget.py @@ -23,6 +23,21 @@ class NZBGet(DownloaderBase): rpc = 'xmlrpc' def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/nzbvortex.py b/couchpotato/core/downloaders/nzbvortex.py index 4f28ed45..df3e3719 100644 --- a/couchpotato/core/downloaders/nzbvortex.py +++ b/couchpotato/core/downloaders/nzbvortex.py @@ -24,6 +24,21 @@ class NZBVortex(DownloaderBase): session_id = None def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/pneumatic.py b/couchpotato/core/downloaders/pneumatic.py index 8cf1aebb..957a76dd 100644 --- a/couchpotato/core/downloaders/pneumatic.py +++ b/couchpotato/core/downloaders/pneumatic.py @@ -19,6 +19,21 @@ class Pneumatic(DownloaderBase): status_support = False def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/qbittorrent_.py b/couchpotato/core/downloaders/qbittorrent_.py index d4bfced1..a9e8cf4d 100644 --- a/couchpotato/core/downloaders/qbittorrent_.py +++ b/couchpotato/core/downloaders/qbittorrent_.py @@ -47,6 +47,21 @@ class qBittorrent(DownloaderBase): return False def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/rtorrent_.py b/couchpotato/core/downloaders/rtorrent_.py index 7474697f..4de952fa 100644 --- a/couchpotato/core/downloaders/rtorrent_.py +++ b/couchpotato/core/downloaders/rtorrent_.py @@ -94,6 +94,21 @@ class rTorrent(DownloaderBase): def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/sabnzbd.py b/couchpotato/core/downloaders/sabnzbd.py index cd51cb87..d6e3e1d2 100644 --- a/couchpotato/core/downloaders/sabnzbd.py +++ b/couchpotato/core/downloaders/sabnzbd.py @@ -21,6 +21,21 @@ class Sabnzbd(DownloaderBase): protocol = ['nzb'] def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/synology.py b/couchpotato/core/downloaders/synology.py index 2c12536f..3e4ead2f 100644 --- a/couchpotato/core/downloaders/synology.py +++ b/couchpotato/core/downloaders/synology.py @@ -19,6 +19,21 @@ class Synology(DownloaderBase): status_support = False def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index 3a04cbbc..8f274915 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -34,6 +34,21 @@ class Transmission(DownloaderBase): return self.trpc def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} diff --git a/couchpotato/core/downloaders/utorrent.py b/couchpotato/core/downloaders/utorrent.py index 3164681c..f4535cad 100644 --- a/couchpotato/core/downloaders/utorrent.py +++ b/couchpotato/core/downloaders/utorrent.py @@ -51,6 +51,21 @@ class uTorrent(DownloaderBase): return self.utorrent_api def download(self, data = None, media = None, filedata = None): + """ + Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} From 4d329d6a36c8dbfd8212a57aa8b291374eed397f Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 2 Dec 2014 11:45:17 +0100 Subject: [PATCH 30/38] Revert "Remove torrentleech" This reverts commit dacc3d8f470e7cc4c294bab86bd7d9b6afe4d947. --- .../_base/providers/torrent/torrentleech.py | 126 ++++++++++++++++++ .../movie/providers/torrent/torrentleech.py | 27 ++++ 2 files changed, 153 insertions(+) create mode 100644 couchpotato/core/media/_base/providers/torrent/torrentleech.py create mode 100644 couchpotato/core/media/movie/providers/torrent/torrentleech.py diff --git a/couchpotato/core/media/_base/providers/torrent/torrentleech.py b/couchpotato/core/media/_base/providers/torrent/torrentleech.py new file mode 100644 index 00000000..83eb5f1f --- /dev/null +++ b/couchpotato/core/media/_base/providers/torrent/torrentleech.py @@ -0,0 +1,126 @@ +import traceback + +from bs4 import BeautifulSoup +from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.torrent.base import TorrentProvider +import six + + +log = CPLog(__name__) + + +class Base(TorrentProvider): + + urls = { + 'test': 'https://www.torrentleech.org/', + 'login': 'https://www.torrentleech.org/user/account/login/', + 'login_check': 'https://torrentleech.org/user/messages', + 'detail': 'https://www.torrentleech.org/torrent/%s', + 'search': 'https://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', + 'download': 'https://www.torrentleech.org%s', + } + + http_time_between_calls = 1 # Seconds + cat_backup_id = None + + def _searchOnTitle(self, title, media, quality, results): + + url = self.urls['search'] % self.buildUrl(title, media, quality) + + data = self.getHTMLData(url) + + if data: + html = BeautifulSoup(data) + + try: + result_table = html.find('table', attrs = {'id': 'torrenttable'}) + if not result_table: + return + + entries = result_table.find_all('tr') + + for result in entries[1:]: + + link = result.find('td', attrs = {'class': 'name'}).find('a') + url = result.find('td', attrs = {'class': 'quickdownload'}).find('a') + details = result.find('td', attrs = {'class': 'name'}).find('a') + + results.append({ + 'id': link['href'].replace('/torrent/', ''), + 'name': six.text_type(link.string), + 'url': self.urls['download'] % url['href'], + 'detail_url': self.urls['download'] % details['href'], + 'size': self.parseSize(result.find_all('td')[4].string), + 'seeders': tryInt(result.find('td', attrs = {'class': 'seeders'}).string), + 'leechers': tryInt(result.find('td', attrs = {'class': 'leechers'}).string), + }) + + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + + def getLoginParams(self): + return { + 'username': self.conf('username'), + 'password': self.conf('password'), + 'remember_me': 'on', + 'login': 'submit', + } + + def loginSuccess(self, output): + return '/user/account/logout' in output.lower() or 'welcome back' in output.lower() + + loginCheckSuccess = loginSuccess + + +config = [{ + 'name': 'torrentleech', + 'groups': [ + { + 'tab': 'searcher', + 'list': 'torrent_providers', + 'name': 'TorrentLeech', + 'description': 'TorrentLeech', + 'wizard': True, + 'icon': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAACHUlEQVR4AZVSO48SYRSdGTCBEMKzILLAWiybkKAGMZRUUJEoDZX7B9zsbuQPYEEjNLTQkYgJDwsoSaxspEBsCITXjjNAIKi8AkzceXgmbHQ1NJ5iMufmO9/9zrmXlCSJ+B8o75J8Pp/NZj0eTzweBy0Wi4PBYD6f12o1r9ebTCZx+22HcrnMsuxms7m6urTZ7LPZDMVYLBZ8ZV3yo8aq9Pq0wzCMTqe77dDv9y8uLyAWBH6xWOyL0K/56fcb+rrPgPZ6PZfLRe1fsl6vCUmGKIqoqNXqdDr9Dbjps9znUV0uTqdTjuPkDoVCIfcuJ4gizjMMm8u9vW+1nr04czqdK56c37CbKY9j2+1WEARZ0Gq1RFHAz2q1qlQqXxoN69HRcDjUarW8ZD6QUigUOnY8uKYH8N1sNkul9yiGw+F6vS4Rxn8EsodEIqHRaOSnq9T7ajQazWQycEIR1AEBYDabSZJyHDucJyegwWBQr9ebTCaKvHd4cCQANUU9evwQ1Ofz4YvUKUI43GE8HouSiFiNRhOowWBIpVLyHITJkuW3PwgAEf3pgIwxF5r+OplMEsk3CPT5szCMnY7EwUdhwUh/CXiej0Qi3idPz89fdrpdbsfBzH7S3Q9K5pP4c0sAKpVKoVAQGO1ut+t0OoFAQHkH2Da/3/+but3uarWK0ZMQoNdyucRutdttmqZxMTzY7XaYxsrgtUjEZrNhkSwWyy/0NCatZumrNQAAAABJRU5ErkJggg==', + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': False, + }, + { + 'name': 'username', + 'default': '', + }, + { + 'name': 'password', + 'default': '', + 'type': 'password', + }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'type': 'float', + 'default': 1, + 'description': 'Will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'type': 'int', + 'default': 40, + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', + }, + { + 'name': 'extra_score', + 'advanced': True, + 'label': 'Extra Score', + 'type': 'int', + 'default': 20, + 'description': 'Starting score for each release found via this provider.', + } + ], + }, + ], +}] diff --git a/couchpotato/core/media/movie/providers/torrent/torrentleech.py b/couchpotato/core/media/movie/providers/torrent/torrentleech.py new file mode 100644 index 00000000..d72f4257 --- /dev/null +++ b/couchpotato/core/media/movie/providers/torrent/torrentleech.py @@ -0,0 +1,27 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.torrent.torrentleech import Base +from couchpotato.core.media.movie.providers.base import MovieProvider + +log = CPLog(__name__) + +autoload = 'TorrentLeech' + + +class TorrentLeech(MovieProvider, Base): + + cat_ids = [ + ([13], ['720p', '1080p', 'bd50']), + ([8], ['cam']), + ([9], ['ts', 'tc']), + ([10], ['r5', 'scr']), + ([11], ['dvdrip']), + ([14], ['brrip']), + ([12], ['dvdr']), + ] + + def buildUrl(self, title, media, quality): + return ( + tryUrlencode(title.replace(':', '')), + self.getCatId(quality)[0] + ) From 2e907e93e71e96d6dfadb47f02ebad570c803f16 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 2 Dec 2014 12:02:49 +0100 Subject: [PATCH 31/38] Whiteline --- couchpotato/core/downloaders/hadouken.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index 0841832e..98a2b21c 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -54,7 +54,7 @@ class Hadouken(DownloaderBase): :return: boolean One faile returns false, but the downloaded should log his own errors """ - + if not media: media = {} if not data: data = {} From ff43df9ef19ac74b3a7158736ede466f889efb1e Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 2 Dec 2014 15:38:55 +0100 Subject: [PATCH 32/38] Comments comments comments --- couchpotato/core/downloaders/blackhole.py | 25 +++++++++++++++-- couchpotato/core/downloaders/deluge.py | 29 ++++++++++++++++++++ couchpotato/core/downloaders/hadouken.py | 11 ++++++-- couchpotato/core/downloaders/nzbget.py | 14 ++++++++-- couchpotato/core/downloaders/nzbvortex.py | 14 ++++++++-- couchpotato/core/downloaders/pneumatic.py | 7 +++-- couchpotato/core/downloaders/qbittorrent_.py | 15 ++++++++-- couchpotato/core/downloaders/rtorrent_.py | 15 ++++++++-- couchpotato/core/downloaders/sabnzbd.py | 12 ++++++++ couchpotato/core/downloaders/synology.py | 4 +++ couchpotato/core/downloaders/transmission.py | 11 ++++++++ couchpotato/core/downloaders/utorrent.py | 11 ++++++++ 12 files changed, 154 insertions(+), 14 deletions(-) diff --git a/couchpotato/core/downloaders/blackhole.py b/couchpotato/core/downloaders/blackhole.py index 5d4e54be..22ed9ada 100644 --- a/couchpotato/core/downloaders/blackhole.py +++ b/couchpotato/core/downloaders/blackhole.py @@ -20,8 +20,7 @@ class Blackhole(DownloaderBase): status_support = False def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -39,10 +38,13 @@ class Blackhole(DownloaderBase): if not data: data = {} directory = self.conf('directory') + + # The folder needs to exist if not directory or not os.path.isdir(directory): log.error('No directory set for blackhole %s download.', data.get('protocol')) else: try: + # Filedata can be empty, which probably means it a magnet link if not filedata or len(filedata) < 50: try: if data.get('protocol') == 'torrent_magnet': @@ -51,13 +53,16 @@ class Blackhole(DownloaderBase): except: log.error('Failed download torrent via magnet url: %s', traceback.format_exc()) + # If it's still empty, don't know what to do! if not filedata or len(filedata) < 50: log.error('No nzb/torrent available: %s', data.get('url')) return False + # Create filename with imdb id and other nice stuff file_name = self.createFileName(data, filedata, media) full_path = os.path.join(directory, file_name) + # People want thinks nice and tidy, create a subdir if self.conf('create_subdir'): try: new_path = os.path.splitext(full_path)[0] @@ -68,6 +73,8 @@ class Blackhole(DownloaderBase): log.error('Couldnt create sub dir, reverting to old one: %s', full_path) try: + + # Make sure the file doesn't exist yet, no need in overwriting it if not os.path.isfile(full_path): log.info('Downloading %s to %s.', (data.get('protocol'), full_path)) with open(full_path, 'wb') as f: @@ -89,6 +96,10 @@ class Blackhole(DownloaderBase): return False def test(self): + """ Test and see if the directory is writable + :return: boolean + """ + directory = self.conf('directory') if directory and os.path.isdir(directory): @@ -103,6 +114,10 @@ class Blackhole(DownloaderBase): return False def getEnabledProtocol(self): + """ What protocols is this downloaded used for + :return: list with protocols + """ + if self.conf('use_for') == 'both': return super(Blackhole, self).getEnabledProtocol() elif self.conf('use_for') == 'torrent': @@ -111,6 +126,12 @@ class Blackhole(DownloaderBase): return ['nzb'] def isEnabled(self, manual = False, data = None): + """ Check if protocol is used (and enabled) + :param manual: The user has clicked to download a link through the webUI + :param data: dict returned from provider + Contains the release information + :return: boolean + """ if not data: data = {} for_protocol = ['both'] if data and 'torrent' in data.get('protocol'): diff --git a/couchpotato/core/downloaders/deluge.py b/couchpotato/core/downloaders/deluge.py index 1230cd6e..3bcbfb63 100644 --- a/couchpotato/core/downloaders/deluge.py +++ b/couchpotato/core/downloaders/deluge.py @@ -25,6 +25,11 @@ class Deluge(DownloaderBase): drpc = None def connect(self, reconnect = False): + """ Connect to the delugeRPC, re-use connection when already available + :param reconnect: force reconnect + :return: DelugeRPC instance + """ + # Load host from config and split out port. host = cleanHost(self.conf('host'), protocol = False).split(':') @@ -42,6 +47,20 @@ class Deluge(DownloaderBase): return self.drpc def download(self, data = None, media = None, filedata = None): + """ Send a torrent/nzb file to the downloader + + :param data: dict returned from provider + Contains the release information + :param media: media dict with information + Used for creating the filename when possible + :param filedata: downloaded torrent/nzb filedata + The file gets downloaded in the searcher and send to this function + This is done to have failed checking before using the downloader, so the downloader + doesn't need to worry about that + :return: boolean + One faile returns false, but the downloaded should log his own errors + """ + if not media: media = {} if not data: data = {} @@ -96,11 +115,21 @@ class Deluge(DownloaderBase): return self.downloadReturnId(remote_torrent) def test(self): + """ Check if connection works + :return: bool + """ if self.connect(True) and self.drpc.test(): return True return False def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ log.debug('Checking Deluge download status.') diff --git a/couchpotato/core/downloaders/hadouken.py b/couchpotato/core/downloaders/hadouken.py index 98a2b21c..c7dddbe7 100644 --- a/couchpotato/core/downloaders/hadouken.py +++ b/couchpotato/core/downloaders/hadouken.py @@ -40,8 +40,7 @@ class Hadouken(DownloaderBase): return True def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -110,6 +109,14 @@ class Hadouken(DownloaderBase): return False def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ + log.debug('Checking Hadouken download status.') if not self.connect(): diff --git a/couchpotato/core/downloaders/nzbget.py b/couchpotato/core/downloaders/nzbget.py index 2f3e686e..9fbed734 100644 --- a/couchpotato/core/downloaders/nzbget.py +++ b/couchpotato/core/downloaders/nzbget.py @@ -23,8 +23,7 @@ class NZBGet(DownloaderBase): rpc = 'xmlrpc' def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -86,6 +85,10 @@ class NZBGet(DownloaderBase): return False def test(self): + """ Check if connection works + :return: bool + """ + rpc = self.getRPC() try: @@ -106,6 +109,13 @@ class NZBGet(DownloaderBase): return True def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ log.debug('Checking NZBGet download status.') diff --git a/couchpotato/core/downloaders/nzbvortex.py b/couchpotato/core/downloaders/nzbvortex.py index df3e3719..f98f0f95 100644 --- a/couchpotato/core/downloaders/nzbvortex.py +++ b/couchpotato/core/downloaders/nzbvortex.py @@ -24,8 +24,7 @@ class NZBVortex(DownloaderBase): session_id = None def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -60,6 +59,10 @@ class NZBVortex(DownloaderBase): return False def test(self): + """ Check if connection works + :return: bool + """ + try: login_result = self.login() except: @@ -68,6 +71,13 @@ class NZBVortex(DownloaderBase): return login_result def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ raw_statuses = self.call('nzb') diff --git a/couchpotato/core/downloaders/pneumatic.py b/couchpotato/core/downloaders/pneumatic.py index 957a76dd..df53fe64 100644 --- a/couchpotato/core/downloaders/pneumatic.py +++ b/couchpotato/core/downloaders/pneumatic.py @@ -19,8 +19,7 @@ class Pneumatic(DownloaderBase): status_support = False def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -78,6 +77,10 @@ class Pneumatic(DownloaderBase): return False def test(self): + """ Check if connection works + :return: bool + """ + directory = self.conf('directory') if directory and os.path.isdir(directory): diff --git a/couchpotato/core/downloaders/qbittorrent_.py b/couchpotato/core/downloaders/qbittorrent_.py index a9e8cf4d..9cfae4dd 100644 --- a/couchpotato/core/downloaders/qbittorrent_.py +++ b/couchpotato/core/downloaders/qbittorrent_.py @@ -41,14 +41,17 @@ class qBittorrent(DownloaderBase): return self.qb def test(self): + """ Check if connection works + :return: bool + """ + if self.connect(): return True return False def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -110,6 +113,14 @@ class qBittorrent(DownloaderBase): return 'busy' def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ + log.debug('Checking qBittorrent download status.') if not self.connect(): diff --git a/couchpotato/core/downloaders/rtorrent_.py b/couchpotato/core/downloaders/rtorrent_.py index 4de952fa..d754022f 100644 --- a/couchpotato/core/downloaders/rtorrent_.py +++ b/couchpotato/core/downloaders/rtorrent_.py @@ -84,6 +84,10 @@ class rTorrent(DownloaderBase): return self.rt def test(self): + """ Check if connection works + :return: bool + """ + if self.connect(True): return True @@ -94,8 +98,7 @@ class rTorrent(DownloaderBase): def download(self, data = None, media = None, filedata = None): - """ - Send a torrent/nzb file to the downloader + """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information @@ -176,6 +179,14 @@ class rTorrent(DownloaderBase): return 'completed' def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ + log.debug('Checking rTorrent download status.') if not self.connect(): diff --git a/couchpotato/core/downloaders/sabnzbd.py b/couchpotato/core/downloaders/sabnzbd.py index d6e3e1d2..4859209e 100644 --- a/couchpotato/core/downloaders/sabnzbd.py +++ b/couchpotato/core/downloaders/sabnzbd.py @@ -84,6 +84,11 @@ class Sabnzbd(DownloaderBase): return False def test(self): + """ Check if connection works + Return message if an old version of SAB is used + :return: bool + """ + try: sab_data = self.call({ 'mode': 'version', @@ -104,6 +109,13 @@ class Sabnzbd(DownloaderBase): return True def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ log.debug('Checking SABnzbd download status.') diff --git a/couchpotato/core/downloaders/synology.py b/couchpotato/core/downloaders/synology.py index 3e4ead2f..b5327ccb 100644 --- a/couchpotato/core/downloaders/synology.py +++ b/couchpotato/core/downloaders/synology.py @@ -65,6 +65,10 @@ class Synology(DownloaderBase): return self.downloadReturnId('') if response else False def test(self): + """ Check if connection works + :return: bool + """ + host = cleanHost(self.conf('host'), protocol = False).split(':') try: srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password')) diff --git a/couchpotato/core/downloaders/transmission.py b/couchpotato/core/downloaders/transmission.py index 8f274915..697f22ac 100644 --- a/couchpotato/core/downloaders/transmission.py +++ b/couchpotato/core/downloaders/transmission.py @@ -103,11 +103,22 @@ class Transmission(DownloaderBase): return self.downloadReturnId(data['hashString']) def test(self): + """ Check if connection works + :return: bool + """ + if self.connect() and self.trpc.get_session(): return True return False def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ log.debug('Checking Transmission download status.') diff --git a/couchpotato/core/downloaders/utorrent.py b/couchpotato/core/downloaders/utorrent.py index f4535cad..847eaf11 100644 --- a/couchpotato/core/downloaders/utorrent.py +++ b/couchpotato/core/downloaders/utorrent.py @@ -135,6 +135,10 @@ class uTorrent(DownloaderBase): return self.downloadReturnId(torrent_hash) def test(self): + """ Check if connection works + :return: bool + """ + if self.connect(): build_version = self.utorrent_api.get_build() if not build_version: @@ -146,6 +150,13 @@ class uTorrent(DownloaderBase): return False def getAllDownloadStatus(self, ids): + """ Get status of all active downloads + + :param ids: list of (mixed) downloader ids + Used to match the releases for this downloader as there could be + other downloaders active that it should ignore + :return: list of releases + """ log.debug('Checking uTorrent download status.') From 766f819c0b1869daada1e788c5834ea6998eb518 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 14 Dec 2014 12:06:03 +0100 Subject: [PATCH 33/38] Userscript for RT not parsing URL correctly --- .../core/media/movie/providers/userscript/rottentomatoes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/userscript/rottentomatoes.py b/couchpotato/core/media/movie/providers/userscript/rottentomatoes.py index 902192e2..a61c3131 100644 --- a/couchpotato/core/media/movie/providers/userscript/rottentomatoes.py +++ b/couchpotato/core/media/movie/providers/userscript/rottentomatoes.py @@ -12,7 +12,7 @@ autoload = 'RottenTomatoes' class RottenTomatoes(UserscriptBase): - includes = ['*://www.rottentomatoes.com/m/*/'] + includes = ['*://www.rottentomatoes.com/m/*'] excludes = ['*://www.rottentomatoes.com/m/*/*/'] version = 2 From 814ddfb79f7cbbf4095d3a61679f15767cfe4c29 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 14 Dec 2014 12:33:28 +0100 Subject: [PATCH 34/38] Don't return password fields fix #4300 --- couchpotato/core/settings.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/couchpotato/core/settings.py b/couchpotato/core/settings.py index 4315ec18..ffc142ae 100644 --- a/couchpotato/core/settings.py +++ b/couchpotato/core/settings.py @@ -157,7 +157,15 @@ class Settings(object): values[section] = {} for option in self.p.items(section): (option_name, option_value) = option + + is_password = False + try: is_password = self.types[section][option_name] == 'password' + except: pass + values[section][option_name] = self.get(option_name, section) + if is_password and values[section][option_name]: + values[section][option_name] = len(values[section][option_name]) * '*' + return values def save(self): From cb8d24ef1f7f17c3a7fffcfeed5ce0398c6eab1b Mon Sep 17 00:00:00 2001 From: mano3m Date: Mon, 15 Dec 2014 21:55:44 +0100 Subject: [PATCH 35/38] Fix TorrentShack size --- couchpotato/core/media/_base/providers/torrent/torrentshack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/torrentshack.py b/couchpotato/core/media/_base/providers/torrent/torrentshack.py index f56017f5..b65222b3 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentshack.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentshack.py @@ -42,6 +42,7 @@ class Base(TorrentProvider): link = result.find('span', attrs = {'class': 'torrent_name_link'}).parent url = result.find('td', attrs = {'class': 'torrent_td'}).find('a') + size = result.find('td', attrs = {'class': 'size'}).contents[0].strip('\n ') tds = result.find_all('td') results.append({ @@ -49,7 +50,7 @@ class Base(TorrentProvider): 'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}), 'url': self.urls['download'] % url['href'], 'detail_url': self.urls['download'] % link['href'], - 'size': self.parseSize(result.find_all('td')[5].string), + 'size': self.parseSize(size), 'seeders': tryInt(tds[len(tds)-2].string), 'leechers': tryInt(tds[len(tds)-1].string), }) From ddf575a86e40092fb5443bebcdfd02b85e97b486 Mon Sep 17 00:00:00 2001 From: Rami Taibah Date: Wed, 17 Dec 2014 13:00:54 +0300 Subject: [PATCH 36/38] Change Readd in tooltip to Re-add. Former is confusing and not an English word --- couchpotato/core/media/movie/_base/static/movie.actions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index 09a998f3..273df5ae 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -696,7 +696,7 @@ MA.Readd = new Class({ if(movie_done || snatched && snatched > 0) self.el = new Element('a.readd', { - 'title': 'Readd the movie and mark all previous snatched/downloaded as ignored', + 'title': 'Re-add the movie and mark all previous snatched/downloaded as ignored', 'events': { 'click': self.doReadd.bind(self) } From 576bcb9f4b389d271da6a88addb50405c8f21dff Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Dec 2014 08:57:24 +0100 Subject: [PATCH 37/38] Give response back to the main thread on api calls fix #4337 --- couchpotato/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/api.py b/couchpotato/api.py index cd01197a..36025134 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -7,6 +7,7 @@ import urllib from couchpotato.core.helpers.request import getParams from couchpotato.core.logger import CPLog +from tornado.ioloop import IOLoop from tornado.web import RequestHandler, asynchronous @@ -33,7 +34,7 @@ def run_async(func): def run_handler(route, kwargs, callback = None): try: res = api[route](**kwargs) - callback(res, route) + IOLoop.instance().add_callback(callback, res, route) except: log.error('Failed doing api request "%s": %s', (route, traceback.format_exc())) callback({'success': False, 'error': 'Failed returning results'}, route) From eea9f40501573c8554f0abc089bb0c8431fab7a3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 19 Dec 2014 09:01:52 +0100 Subject: [PATCH 38/38] Use current --- couchpotato/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/api.py b/couchpotato/api.py index 36025134..2ce4312c 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -34,7 +34,7 @@ def run_async(func): def run_handler(route, kwargs, callback = None): try: res = api[route](**kwargs) - IOLoop.instance().add_callback(callback, res, route) + IOLoop.current().add_callback(callback, res, route) except: log.error('Failed doing api request "%s": %s', (route, traceback.format_exc())) callback({'success': False, 'error': 'Failed returning results'}, route)