Add and make use of renamer.scanfolder in downloaders

This is the next step in closing the loop between the downloaders and CPS. The download_id and folder from the downloader are used to find the downloaded files and start the renamer. This is done by adding an additional API call: renamer.scanfolder.

I tested this for SabNZBd only (!) and everything works as expected.

I also added transmission with thanks @manusfreedom for setting this up in f1cf0d91da. @manusfreedom, please check if this works as expected. Note that transmission now has a feature which is not in the other torrent providers: it waits until the seed ratio is met and then removes the torrent. I opened a topic in the forum to discuss how we want to deal with torrents: https://couchpota.to/forum/thread-1704.html

couchpotato/core/downloaders/base.py

@@ -150,6 +150,7 @@ class StatusList(list):
             'id': 0,
             'status': 'busy',
             'downloader': self.provider.getName(),
+            'folder': '',
         }
 
         return mergeDicts(defaults, result)

couchpotato/core/downloaders/nzbget/main.py

@@ -120,6 +120,7 @@ class NZBGet(Downloader):
                 'status': 'completed' if item['ParStatus'] == 'SUCCESS' and item['ScriptStatus'] == 'SUCCESS' else 'failed',
                 'original_status': item['ParStatus'] + ', ' + item['ScriptStatus'],
                 'timeleft': str(timedelta(seconds = 0)),
+                'folder': item['DestDir']
             })
 
         return statuses

couchpotato/core/downloaders/nzbvortex/main.py

@@ -55,7 +55,8 @@ class NZBVortex(Downloader):
                 'name': item['uiTitle'],
                 'status': status,
                 'original_status': item['state'],
-                'timeleft':-1,
+                'timeleft': -1,
+                'folder': item['destinationPath'],
             })
 
         return statuses

couchpotato/core/downloaders/sabnzbd/main.py

@@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode, ss
 from couchpotato.core.helpers.variable import cleanHost, mergeDicts
 from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
+from datetime import timedelta
 from urllib2 import URLError
 import json
 import traceback
@@ -46,19 +47,15 @@ class Sabnzbd(Downloader):
             log.error('Failed sending release, use API key, NOT the NZB key: %s', traceback.format_exc(0))
             return False
 
-        if sab_data.get('error'):
-            log.error('Error getting data from SABNZBd: %s', sab_data.get('error'))
-            return False
-
         log.debug('Result from SAB: %s', sab_data)
-        if sab_data.get('status'):
+        if sab_data.get('status') and not sab_data.get('error'):
             log.info('NZB sent to SAB successfully.')
             if filedata:
                 return self.downloadReturnId(sab_data.get('nzo_ids')[0])
             else:
                 return True
         else:
-            log.error(sab_data)
+            log.error('Error getting data from SABNZBd: %s', sab_data)
             return False
 
     def getAllDownloadStatus(self):
@@ -109,7 +106,8 @@ class Sabnzbd(Downloader):
                 'name': item['name'],
                 'status': status,
                 'original_status': item['status'],
-                'timeleft': 0,
+                'timeleft': str(timedelta(seconds = 0)),
+                'folder': item['storage'],
             })
 
         return statuses

couchpotato/core/downloaders/transmission/__init__.py

@@ -41,15 +41,22 @@ config = [{
                 {
                     'name': 'directory',
                     'type': 'directory',
-                    'description': 'Where should Transmission saved the downloaded files?',
+                    'description': 'Where should Transmission save the downloaded files?',
                 },
                 {
                     'name': 'ratio',
                     'default': 10,
-                    'type': 'int',
+                    'type': 'float',
                     'advanced': True,
                     'description': 'Stop transfer when reaching ratio',
                 },
+                {
+                    'name': 'ratiomode',
+                    'default': 0,
+                    'type': 'int',
+                    'advanced': True,
+                    'description': '0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.',
+                },
                 {
                     'name': 'manual',
                     'default': 0,

couchpotato/core/downloaders/transmission/main.py

@@ -1,12 +1,15 @@
 from base64 import b64encode
-from couchpotato.core.downloaders.base import Downloader
+from couchpotato.core.downloaders.base import Downloader, StatusList
 from couchpotato.core.helpers.encoding import isInt
 from couchpotato.core.logger import CPLog
+from couchpotato.environment import Env
+from datetime import timedelta
 import httplib
 import json
 import os.path
 import re
 import urllib2
+import shutil
 
 log = CPLog(__name__)
 
@@ -18,7 +21,7 @@ class Transmission(Downloader):
 
     def download(self, data, movie, filedata = None):
 
-        log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type')))
+        log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type')))
 
         # Load host from config and split out port.
         host = self.conf('host').split(':')
@@ -30,7 +33,7 @@ class Transmission(Downloader):
         folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1]
         folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep)
 
-        # Create the empty folder to download too
+        # Create the empty folder to download to
         self.makeDir(folder_path)
 
         params = {
@@ -42,7 +45,7 @@ class Transmission(Downloader):
         if self.conf('ratio'):
             torrent_params = {
                 'seedRatioLimit': self.conf('ratio'),
-                'seedRatioMode': self.conf('ratio')
+                'seedRatioMode': self.conf('ratiomode')
             }
 
         if not filedata and data.get('type') == 'torrent':
@@ -62,11 +65,99 @@ class Transmission(Downloader):
             if torrent_params:
                 trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params)
 
+            log.info('Torrent sent to Transmission successfully.')
             return self.downloadReturnId(remote_torrent['torrent-added']['hashString'])
         except Exception, err:
             log.error('Failed to change settings for transfer: %s', err)
             return False
 
+    def getAllDownloadStatus(self):
+
+        log.debug('Checking Transmission download status.')
+
+        # Load host from config and split out port.
+        host = self.conf('host').split(':')
+        if not isInt(host[1]):
+            log.error('Config properties are not filled in correctly, port is missing.')
+            return False
+
+        # Go through Queue
+        try:
+            trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password'))
+            return_params = {
+                'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isFinished', 'downloadDir', 'uploadRatio']
+            }
+            queue = trpc.get_alltorrents(return_params)
+
+        except Exception, err:
+            log.error('Failed getting queue: %s', err)
+            return False
+
+        statuses = StatusList(self)
+
+        # Get torrents status
+            # CouchPotato Status
+                #status = 'busy'
+                #status = 'failed'
+                #status = 'completed'
+            # Transmission Status
+                #status = 0 => "Torrent is stopped"
+                #status = 1 => "Queued to check files"
+                #status = 2 => "Checking files"
+                #status = 3 => "Queued to download"
+                #status = 4 => "Downloading"
+                #status = 4 => "Queued to seed"
+                #status = 6 => "Seeding"
+        #To do :
+        #   add checking file
+        #   manage no peer in a range time => fail
+
+        for item in queue['torrents']:
+            log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / confRatio=%s / isFinished=%s', (item['name'], item['id'], item['downloadDir'], item['hashString'], item['percentDone'], item['status'], item['eta'], item['uploadRatio'], self.conf('ratio'), item['isFinished'] ))
+
+            if not os.path.isdir(Env.setting('from', 'renamer')):
+                log.debug('Renamer folder has to exist.')
+                return
+
+            if (item['percentDone'] * 100) >= 100 and (item['status'] == 6 or item['status'] == 0) and item['uploadRatio'] > self.conf('ratio'):
+                try:
+                    trpc.stop_torrent(item['hashString'], {})
+
+                    if not os.path.isdir(item['downloadDir']):
+                        raise Exception('Missing folder: %s' % item['downloadDir']) 
+                        
+                    else:
+                        log.info('Moving folder from "%s" to "%s"', (item['downloadDir'], Env.setting('from', 'renamer')))
+                        shutil.move(item['downloadDir'], Env.setting('from', 'renamer'))
+                    
+                    statuses.append({
+                        'id': item['hashString'],
+                        'name': item['downloadDir'],
+                        'status': 'completed',
+                        'original_status': item['status'],
+                        'timeleft': str(timedelta(seconds = 0)),
+                        'folder': os.path.join(Env.setting('from', 'renamer'), os.path.basename(item['downloadDir'].rstrip(os.path.sep))),
+                    })
+                    trpc.remove_torrent(item['hashString'], True, {})
+                except Exception, err:
+                    log.error('Failed to stop and remove torrent "%s" with error: %s', (item['name'], err))
+                    statuses.append({
+                        'id': item['hashString'],
+                        'name': item['downloadDir'],
+                        'status': 'failed',
+                        'original_status': item['status'],
+                        'timeleft': str(timedelta(seconds = 0)),
+                    })
+            else:
+                statuses.append({
+                    'id': item['hashString'],
+                    'name': item['downloadDir'],
+                    'status': 'busy',
+                    'original_status': item['status'],
+                    'timeleft': str(timedelta(seconds = item['eta'])), # Is ETA in seconds??
+                })
+
+        return statuses
 
 class TransmissionRPC(object):
 
@@ -97,6 +188,7 @@ class TransmissionRPC(object):
         try:
             open_request = urllib2.urlopen(request)
             response = json.loads(open_request.read())
+            log.debug('request: %s', json.dumps(ojson))
             log.debug('response: %s', json.dumps(response))
             if response['result'] == 'success':
                 log.debug('Transmission action successfull')
@@ -146,3 +238,18 @@ class TransmissionRPC(object):
         arguments['ids'] = torrent_id
         post_data = {'arguments': arguments, 'method': 'torrent-set', 'tag': self.tag}
         return self._request(post_data)
+
+    def get_alltorrents(self, arguments):
+        post_data = {'arguments': arguments, 'method': 'torrent-get', 'tag': self.tag}
+        return self._request(post_data)
+
+    def stop_torrent(self, torrent_id, arguments):
+        arguments['ids'] = torrent_id
+        post_data = {'arguments': arguments, 'method': 'torrent-stop', 'tag': self.tag}
+        return self._request(post_data)
+
+    def remove_torrent(self, torrent_id, remove_local_data, arguments):
+        arguments['ids'] = torrent_id
+        arguments['delete-local-data'] = remove_local_data
+        post_data = {'arguments': arguments, 'method': 'torrent-remove', 'tag': self.tag}
+        return self._request(post_data)

couchpotato/core/downloaders/utorrent/main.py

@@ -5,6 +5,7 @@ from couchpotato.core.helpers.encoding import isInt, ss
 from couchpotato.core.logger import CPLog
 from hashlib import sha1
 from multipartpost import MultipartPostHandler
+from datetime import timedelta
 import cookielib
 import httplib
 import json
@@ -118,7 +119,8 @@ class uTorrent(Downloader):
                 'name': item[2],
                 'status':  status,
                 'original_status': item[1],
-                'timeleft': item[10],
+                'timeleft': str(timedelta(seconds = item[10])),
+                'folder': '', #no fucntion to get folder, but can be deduced with getSettings function.
             })
 
         return statuses

couchpotato/core/plugins/renamer/main.py

@@ -2,7 +2,7 @@ from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import toUnicode, ss
-from couchpotato.core.helpers.request import jsonified
+from couchpotato.core.helpers.request import getParams, jsonified, getParam
 from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
     getImdb
 from couchpotato.core.logger import CPLog
@@ -31,6 +31,17 @@ class Renamer(Plugin):
         })
 
         addEvent('renamer.scan', self.scan)
+        
+        addApiView('renamer.scanfolder', self.scanfolderView, docs = {
+            'desc': 'For the renamer to check for new files to rename in a specified folder',
+            'params': {
+                'movie_folder': {'desc': 'The folder of the movie to scan'},
+                'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'},
+                'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'},
+            },
+        })
+
+        addEvent('renamer.scanfolder', self.scanfolder)
         addEvent('renamer.check_snatched', self.checkSnatched)
 
         addEvent('app.load', self.scan)
@@ -51,6 +62,26 @@ class Renamer(Plugin):
         })
 
     def scan(self):
+        self.scanfolder()
+
+    def scanfolderView(self):
+    
+        params = getParams()
+        movie_folder = params.get('movie_folder', None)
+        downloader = params.get('downloader', None)
+        download_id = params.get('download_id', None)
+
+        fireEventAsync('renamer.scanfolder', 
+            movie_folder = movie_folder, 
+            downloader = downloader, 
+            download_id = download_id
+        )
+
+        return jsonified({
+            'success': True
+        })
+        
+    def scanfolder(self, movie_folder = None, downloader = None, download_id = None):
 
         if self.isDisabled():
             return
@@ -59,17 +90,42 @@ class Renamer(Plugin):
             log.info('Renamer is already running, if you see this often, check the logs above for errors.')
             return
 
+        self.renaming_started = True
+
         # Check to see if the "to" folder is inside the "from" folder.
-        if not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')):
+        if movie_folder and not os.path.isdir(movie_folder): # or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')):
             log.debug('"To" and "From" have to exist.')
             return
         elif self.conf('from') in self.conf('to'):
             log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.')
             return
+        elif (movie_folder and movie_folder in [self.conf('to'), self.conf('from')]):
+            log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.')
+            return
 
-        groups = fireEvent('scanner.scan', folder = self.conf('from'), single = True)
+        # make sure the movie folder name is included in the search
+        folder = None 
+        movie_files = []
+        if movie_folder:
+            log.info('Scanning movie folder %s...', movie_folder)
+            movie_folder = movie_folder.rstrip(os.path.sep)
+            folder = os.path.dirname(movie_folder)
 
-        self.renaming_started = True
+            # Get all files from the specified folder
+            try:
+                for root, folders, names in os.walk(movie_folder):
+                    movie_files.extend([os.path.join(root, name) for name in names])
+            except:
+                log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc()))
+
+        groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, downloader = downloader, download_id = download_id, single = True)
+
+        # Make sure only one movie was found if a download ID is provided
+        if downloader and download_id and not len(groups) == 1:
+            log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_id, len(groups)))
+            downloader = None
+            download_id = None
+            groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'), files = movie_files, single = True)
                 
         destination = self.conf('to')
         folder_name = self.conf('folder_name')
@@ -597,7 +653,10 @@ class Renamer(Plugin):
                                         db.commit()
                                 elif item['status'] == 'completed':
                                     log.info('Download of %s completed!', item['name'])
-                                    scan_required = True
+                                    if item['id'] and item['downloader'] and item['folder']:
+                                        fireEventAsync('renamer.scanfolder', movie_folder = item['folder'], downloader = item['downloader'], download_id = item['id'])
+                                    else:
+                                        scan_required = True
 
                                 found = True
                                 break

couchpotato/core/plugins/scanner/main.py

@@ -4,7 +4,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss
 from couchpotato.core.helpers.variable import getExt, getImdb, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
-from couchpotato.core.settings.model import File, Movie
+from couchpotato.core.settings.model import File, Movie, Release, ReleaseInfo
 from enzyme.exceptions import NoParserError, ParseError
 from guessit import guess_movie_info
 from subliminal.videos import Video
@@ -101,7 +101,7 @@ class Scanner(Plugin):
         addEvent('scanner.name_year', self.getReleaseNameYear)
         addEvent('scanner.partnumber', self.getPartNumber)
 
-    def scan(self, folder = None, files = None, simple = False, newer_than = 0, on_found = None):
+    def scan(self, folder = None, files = None, downloader = None, download_id = None, simple = False, newer_than = 0, on_found = None):
 
         folder = ss(os.path.normpath(folder))
 
@@ -119,8 +119,7 @@ class Scanner(Plugin):
             try:
                 files = []
                 for root, dirs, walk_files in os.walk(folder):
-                    for filename in walk_files:
+                    files.extend(os.path.join(root, filename) for filename in walk_files)
-                        files.append(os.path.join(root, filename))
             except:
                 log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
         else:
@@ -129,6 +128,24 @@ class Scanner(Plugin):
 
         db = get_session()
 
+        # Get the release with the downloader ID that was downloded by the downloader
+        download_quality = None
+        download_imdb_id = None
+        if downloader and download_id:
+            # NOTE TO RUUD: Don't really know how to do this better... but there must be a way...?
+            rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = downloader)
+            rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_id)
+            for rlsnfo_dwnld in rlsnfo_dwnlds:
+                for rlsnfo_id in rlsnfo_ids:
+                    if rlsnfo_id.release == rlsnfo_dwnld.release:
+                        rls = rlsnfo_id.release
+
+            if rls:
+                download_imdb_id = rls.movie.library.identifier
+                download_quality = rls.quality.identifier
+            else:
+                log.error('Download ID %s from downloader %s not found in releases', (download_id, downloader))
+
         for file_path in files:
 
             if not os.path.exists(file_path):
@@ -346,7 +363,7 @@ class Scanner(Plugin):
                 continue
 
             log.debug('Getting metadata for %s', identifier)
-            group['meta_data'] = self.getMetaData(group, folder = folder)
+            group['meta_data'] = self.getMetaData(group, folder = folder, download_quality = download_quality)
 
             # Subtitle meta
             group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {}
@@ -376,7 +393,7 @@ class Scanner(Plugin):
             del group['unsorted_files']
 
             # Determine movie
-            group['library'] = self.determineMovie(group)
+            group['library'] = self.determineMovie(group, download_imdb_id = download_imdb_id)
             if not group['library']:
                 log.error('Unable to determine movie: %s', group['identifiers'])
             else:
@@ -401,7 +418,7 @@ class Scanner(Plugin):
 
         return processed_movies
 
-    def getMetaData(self, group, folder = ''):
+    def getMetaData(self, group, folder = '', download_quality = None):
 
         data = {}
         files = list(group['files']['movie'])
@@ -423,9 +440,13 @@ class Scanner(Plugin):
 
             if data.get('audio'): break
 
+        # Use the quality guess first, if that failes use the quality we wanted to download
         data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True)
         if not data['quality']:
-            data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True)
+            if download_quality:
+                data['quality'] = fireEvent('quality.single', download_quality, single = True)
+            else:
+                data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True)
         
         data['quality_type'] = 'HD' if data.get('resolution_width', 0) >= 1280 or data['quality'].get('hd') else 'SD'
 
@@ -501,17 +522,19 @@ class Scanner(Plugin):
 
         return detected_languages
 
-    def determineMovie(self, group):
+    def determineMovie(self, group, download_imdb_id = None):
-        imdb_id = None
+        # Get imdb id from downloader
+        imdb_id = download_imdb_id
 
         files = group['files']
 
         # Check for CP(imdb_id) string in the file paths
-        for cur_file in files['movie']:
+        if not imdb_id:
-            imdb_id = self.getCPImdb(cur_file)
+            for cur_file in files['movie']:
-            if imdb_id:
+                imdb_id = self.getCPImdb(cur_file)
-                log.debug('Found movie via CP tag: %s', cur_file)
+                if imdb_id:
-                break
+                    log.debug('Found movie via CP tag: %s', cur_file)
+                    break
 
         # Check and see if nfo contains the imdb-id
         if not imdb_id: