dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refs/heads/develop' into tv

Browse Source 
Conflicts:
	couchpotato/core/helpers/variable.py
	couchpotato/core/media/_base/searcher/main.py
	couchpotato/core/media/movie/searcher/main.py
	couchpotato/core/plugins/quality/main.py
	couchpotato/core/plugins/release/main.py
	couchpotato/core/plugins/renamer/main.py
This commit is contained in:
Ruud
2013-11-24 23:13:25 +01:00
parent 611c159373 3d5b33856f
commit 3e035f84b1
263 changed files with 27457 additions and 10287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/plugins/base.py
+3 -3
View File
@@ -1,7 +1,7 @@
from StringIO import StringIO
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import tryUrlencode, ss, toSafeString, \
toUnicode
toUnicode, sp
from couchpotato.core.helpers.variable import getExt, md5, isLocalIP
from couchpotato.core.logger import CPLog
from couchpotato.environment import Env
@@ -291,10 +291,10 @@ class Plugin(object):
def createNzbName(self, data, movie):
tag = self.cpTag(movie)
return '%s%s' % (toSafeString(data.get('name')[:127 - len(tag)]), tag)
return '%s%s' % (toSafeString(toUnicode(data.get('name'))[:127 - len(tag)]), tag)
def createFileName(self, data, filedata, movie):
name = os.path.join(self.createNzbName(data, movie))
name = sp(os.path.join(self.createNzbName(data, movie)))
if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '</nzb>' not in filedata:
return '%s.%s' % (name, 'rar')
return '%s.%s' % (name, data.get('protocol'))
couchpotato/core/plugins/manage/main.py
+1
View File
@@ -79,6 +79,7 @@ class Manage(Plugin):
try:
directories = self.directories()
directories.sort()
added_identifiers = []
# Add some progress
couchpotato/core/plugins/quality/main.py
+127 -44
View File
@@ -2,7 +2,7 @@ from couchpotato import get_session
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent
from couchpotato.core.helpers.encoding import toUnicode, ss
from couchpotato.core.helpers.variable import mergeDicts, md5, getExt
from couchpotato.core.helpers.variable import mergeDicts, getExt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Quality, Profile, ProfileType
@@ -38,6 +38,9 @@ class QualityPlugin(Plugin):
]
pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
cached_qualities = None
cached_order = None
def __init__(self):
addEvent('quality.all', self.all)
addEvent('quality.single', self.single)
@@ -55,6 +58,8 @@ class QualityPlugin(Plugin):
addEvent('app.initialize', self.fill, priority = 10)
addEvent('app.test', self.doTest)
def preReleases(self):
return self.pre_releases
@@ -67,6 +72,9 @@ class QualityPlugin(Plugin):
def all(self):
if self.cached_qualities:
return self.cached_qualities
db = get_session()
qualities = db.query(Quality).all()
@@ -76,6 +84,7 @@ class QualityPlugin(Plugin):
q = mergeDicts(self.getQuality(quality.identifier), quality.to_dict())
temp.append(q)
self.cached_qualities = temp
return temp
def single(self, identifier = ''):
@@ -104,6 +113,8 @@ class QualityPlugin(Plugin):
setattr(quality, kwargs.get('value_type'), kwargs.get('value'))
db.commit()
self.cached_qualities = None
return {
'success': True
}
@@ -164,77 +175,149 @@ class QualityPlugin(Plugin):
if not extra: extra = {}
# Create hash for cache
cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files]))
cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files])
cached = self.getCache(cache_key)
if cached and len(extra) == 0: return cached
if cached and len(extra) == 0:
return cached
qualities = self.all()
# Start with 0
score = {}
for quality in qualities:
score[quality.get('identifier')] = 0
for cur_file in files:
words = re.split('\W+', cur_file.lower())
found = {}
for quality in qualities:
contains = self.containsTag(quality, words, cur_file)
if contains:
found[quality['identifier']] = True
for quality in qualities:
# Check identifier
if quality['identifier'] in words:
if len(found) == 0 or len(found) == 1 and found.get(quality['identifier']):
log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file))
return self.setCache(cache_key, quality)
# Check alt and tags
contains = self.containsTag(quality, words, cur_file)
if contains:
return self.setCache(cache_key, quality)
contains_score = self.containsTagScore(quality, words, cur_file)
self.calcScore(score, quality, contains_score)
# Try again with loose testing
quality = self.guessLoose(cache_key, files = files, extra = extra)
if quality:
return self.setCache(cache_key, quality)
for quality in qualities:
loose_score = self.guessLooseScore(quality, files = files, extra = extra)
self.calcScore(score, quality, loose_score)
# Return nothing if all scores are 0
has_non_zero = 0
for s in score:
if score[s] > 0:
has_non_zero += 1
if not has_non_zero:
return None
heighest_quality = max(score, key = score.get)
if heighest_quality:
for quality in qualities:
if quality.get('identifier') == heighest_quality:
return self.setCache(cache_key, quality)
log.debug('Could not identify quality for: %s', files)
return None
def containsTag(self, quality, words, cur_file = ''):
def containsTagScore(self, quality, words, cur_file = ''):
cur_file = ss(cur_file)
score = 0
points = {
'identifier': 10,
'label': 10,
'alternative': 9,
'tags': 9,
'ext': 3,
}
# Check alt and tags
for tag_type in ['alternative', 'tags', 'label']:
for tag_type in ['identifier', 'alternative', 'tags', 'label']:
qualities = quality.get(tag_type, [])
qualities = [qualities] if isinstance(qualities, (str, unicode)) else qualities
for alt in qualities:
if (isinstance(alt, tuple) and '.'.join(alt) in '.'.join(words)) or (isinstance(alt, (str, unicode)) and ss(alt.lower()) in cur_file.lower()):
if (isinstance(alt, tuple)):
if len(set(words) & set(alt)) == len(alt):
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
score += points.get(tag_type)
if (isinstance(alt, (str, unicode)) and ss(alt.lower()) in cur_file.lower()):
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
return True
score += points.get(tag_type) / 2
if list(set(qualities) & set(words)):
log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file))
return True
score += points.get(tag_type)
return
# Check extention
for ext in quality.get('ext', []):
if ext == words[-1]:
log.debug('Found %s extension in %s', (ext, cur_file))
score += points['ext']
def guessLoose(self, cache_key, files = None, extra = None):
return score
def guessLooseScore(self, quality, files = None, extra = None):
score = 0
if extra:
for quality in self.all():
# Check width resolution, range 20
if quality.get('width') and (quality.get('width') - 20) <= extra.get('resolution_width', 0) <= (quality.get('width') + 20):
log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width'), extra.get('resolution_width', 0)))
return self.setCache(cache_key, quality)
# Check width resolution, range 20
if quality.get('width') and (quality.get('width') - 20) <= extra.get('resolution_width', 0) <= (quality.get('width') + 20):
log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width'), extra.get('resolution_width', 0)))
score += 5
# Check height resolution, range 20
if quality.get('height') and (quality.get('height') - 20) <= extra.get('resolution_height', 0) <= (quality.get('height') + 20):
log.debug('Found %s via resolution_height: %s == %s', (quality['identifier'], quality.get('height'), extra.get('resolution_height', 0)))
return self.setCache(cache_key, quality)
# Check height resolution, range 20
if quality.get('height') and (quality.get('height') - 20) <= extra.get('resolution_height', 0) <= (quality.get('height') + 20):
log.debug('Found %s via resolution_height: %s == %s', (quality['identifier'], quality.get('height'), extra.get('resolution_height', 0)))
score += 5
if quality.get('identifier') == 'dvdrip' and 480 <= extra.get('resolution_width', 0) <= 720:
log.debug('Add point for correct dvdrip resolutions')
score += 1
return score
def calcScore(self, score, quality, add_score):
score[quality['identifier']] += add_score
# Set order for allow calculation (and cache)
if not self.cached_order:
self.cached_order = {}
for q in self.qualities:
self.cached_order[q.get('identifier')] = self.qualities.index(q)
if add_score != 0:
for allow in quality.get('allow', []):
score[allow] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5
def doTest(self):
tests = {
'Movie Name (1999)-DVD-Rip.avi': 'dvdrip',
'Movie Name 1999 720p Bluray.mkv': '720p',
'Movie Name 1999 BR-Rip 720p.avi': 'brrip',
'Movie Name 1999 720p Web Rip.avi': 'scr',
'Movie Name 1999 Web DL.avi': 'brrip',
'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr',
'Movie.Name.1999.DVDRip-Group': 'dvdrip',
'Movie.Name.1999.DVD-Rip-Group': 'dvdrip',
'Movie.Name.1999.DVD-R-Group': 'dvdr',
}
correct = 0
for name in tests:
success = self.guess([name]).get('identifier') == tests[name]
if not success:
log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier')))
correct += success
if correct == len(tests):
log.info('Quality test successful')
return True
else:
log.error('Quality test failed: %s out of %s succeeded', (correct, len(tests)))
if 480 <= extra.get('resolution_width', 0) <= 720:
log.debug('Found as dvdrip')
return self.setCache(cache_key, self.single('dvdrip'))
return None
couchpotato/core/plugins/release/main.py
+159 -15
View File
@@ -2,10 +2,14 @@ from couchpotato import get_session, md5
from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import ss, toUnicode
from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.plugins.scanner.main import Scanner
from couchpotato.core.settings.model import File, Release as Relea, Media, ReleaseInfo
from couchpotato.core.settings.model import File, Release as Relea, Media, \
ReleaseInfo
from couchpotato.environment import Env
from inspect import ismethod, isfunction
from sqlalchemy.exc import InterfaceError
from sqlalchemy.orm import joinedload_all
from sqlalchemy.sql.expression import and_, or_
@@ -21,7 +25,7 @@ class Release(Plugin):
def __init__(self):
addEvent('release.add', self.add)
addApiView('release.download', self.download, docs = {
addApiView('release.manual_download', self.manualDownload, docs = {
'desc': 'Send a release manually to the downloaders',
'params': {
'id': {'type': 'id', 'desc': 'ID of the release object in release-table'}
@@ -46,12 +50,46 @@ class Release(Plugin):
}
})
addEvent('release.download', self.download)
addEvent('release.try_download_result', self.tryDownloadResult)
addEvent('release.create_from_search', self.createFromSearch)
addEvent('release.for_movie', self.forMovie)
addEvent('release.delete', self.delete)
addEvent('release.clean', self.clean)
addEvent('release.update_status', self.updateStatus)
# Clean releases that didn't have activity in the last week
addEvent('app.load', self.cleanDone)
fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 4)
def cleanDone(self):
log.debug('Removing releases from dashboard')
now = time.time()
week = 262080
done_status, available_status, snatched_status, downloaded_status, ignored_status = \
fireEvent('status.get', ['done', 'available', 'snatched', 'downloaded', 'ignored'], single = True)
db = get_session()
# get movies last_edit more than a week ago
media = db.query(Media) \
.filter(Media.status_id == done_status.get('id'), Media.last_edit < (now - week)) \
.all()
for item in media:
for rel in item.releases:
# Remove all available releases
if rel.status_id in [available_status.get('id')]:
fireEvent('release.delete', id = rel.id, single = True)
# Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move
elif rel.status_id in [snatched_status.get('id'), downloaded_status.get('id')]:
self.updateStatus(id = rel.id, status = ignored_status)
db.expire_all()
def add(self, group):
db = get_session()
@@ -108,7 +146,6 @@ class Release(Plugin):
return True
def saveFile(self, filepath, type = 'unknown', include_media_info = False):
properties = {}
@@ -169,19 +206,17 @@ class Release(Plugin):
'success': True
}
def download(self, id = None, **kwargs):
def manualDownload(self, id = None, **kwargs):
db = get_session()
snatched_status, done_status = fireEvent('status.get', ['snatched', 'done'], single = True)
rel = db.query(Relea).filter_by(id = id).first()
if rel:
item = {}
for info in rel.info:
item[info.identifier] = info.value
fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Snatching "%s"' % item['name'])
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name'])
# Get matching provider
provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True)
@@ -193,18 +228,18 @@ class Release(Plugin):
if item.get('protocol') != 'torrent_magnet':
item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download
success = fireEvent('searcher.download', data = item, media = rel.media.to_dict({
success = self.download(data = item, media = rel.movie.to_dict({
'profile': {'types': {'quality': {}}},
'releases': {'status': {}, 'quality': {}},
'library': {'titles': {}, 'files':{}},
'files': {}
}), manual = True, single = True)
}), manual = True)
if success:
db.expunge_all()
rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again??
fireEvent('notify.frontend', type = 'release.download', data = True, message = 'Successfully snatched "%s"' % item['name'])
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
return {
'success': success
}
@@ -215,9 +250,108 @@ class Release(Plugin):
'success': False
}
def download(self, data, media, manual = False):
if not data.get('protocol'):
data['protocol'] = data['type']
data['type'] = 'movie'
# Test to see if any downloaders are enabled for this type
downloader_enabled = fireEvent('download.enabled', manual, data, single = True)
if downloader_enabled:
snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True)
# Download release to temp
filedata = None
if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))):
filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id'))
if filedata == 'try_next':
return filedata
download_result = fireEvent('download', data = data, movie = media, manual = manual, filedata = filedata, single = True)
log.debug('Downloader result: %s', download_result)
if download_result:
try:
# Mark release as snatched
db = get_session()
rls = db.query(Relea).filter_by(identifier = md5(data['url'])).first()
if rls:
renamer_enabled = Env.setting('enabled', 'renamer')
# Save download-id info if returned
if isinstance(download_result, dict):
for key in download_result:
rls_info = ReleaseInfo(
identifier = 'download_%s' % key,
value = toUnicode(download_result.get(key))
)
rls.info.append(rls_info)
db.commit()
log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label)
snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie)
log.info(snatch_message)
fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls.to_dict())
# If renamer isn't used, mark media done
if not renamer_enabled:
try:
if media['status_id'] == active_status.get('id'):
for profile_type in media['profile']['types']:
if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
log.info('Renamer disabled, marking media as finished: %s', log_movie)
# Mark release done
self.updateStatus(rls.id, status = done_status)
# Mark media done
mdia = db.query(Media).filter_by(id = media['id']).first()
mdia.status_id = done_status.get('id')
mdia.last_edit = int(time.time())
db.commit()
except:
log.error('Failed marking media finished, renamer disabled: %s', traceback.format_exc())
else:
self.updateStatus(rls.id, status = snatched_status)
except:
log.error('Failed marking media finished: %s', traceback.format_exc())
return True
log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol')))
return False
def tryDownloadResult(self, results, media, quality_type, manual = False):
ignored_status, failed_status = fireEvent('status.get', ['ignored', 'failed'], single = True)
for rel in results:
if not quality_type.get('finish', False) and quality_type.get('wait_for', 0) > 0 and rel.get('age') <= quality_type.get('wait_for', 0):
log.info('Ignored, waiting %s days: %s', (quality_type.get('wait_for'), rel['name']))
continue
if rel['status_id'] in [ignored_status.get('id'), failed_status.get('id')]:
log.info('Ignored: %s', rel['name'])
continue
if rel['score'] <= 0:
log.info('Ignored, score to low: %s', rel['name'])
continue
downloaded = fireEvent('release.download', data = rel, media = media, manual = manual, single = True)
if downloaded is True:
return True
elif downloaded != 'try_next':
break
return False
def createFromSearch(self, search_results, media, quality_type):
available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True)
available_status = fireEvent('status.get', ['available'], single = True)
db = get_session()
found_releases = []
@@ -231,7 +365,8 @@ class Release(Plugin):
if not rls:
rls = Relea(
identifier = rel_identifier,
media_id = media.get('id'),
movie_id = media.get('id'),
#media_id = media.get('id'),
quality_id = quality_type.get('quality_id'),
status_id = available_status.get('id')
)
@@ -286,7 +421,7 @@ class Release(Plugin):
}
def updateStatus(self, id, status = None):
if not status: return
if not status: return False
db = get_session()
@@ -297,11 +432,20 @@ class Release(Plugin):
for info in rel.info:
item[info.identifier] = info.value
if rel.files:
for file_item in rel.files:
if file_item.type.identifier == 'movie':
release_name = os.path.basename(file_item.path)
break
else:
release_name = item['name']
#update status in Db
log.debug('Marking release %s as %s', (item['name'], status.get("label")))
log.debug('Marking release %s as %s', (release_name, status.get("label")))
rel.status_id = status.get('id')
rel.last_edit = int(time.time())
db.commit()
#Update all movie info as there is no release update function
fireEvent('notify.frontend', type = 'release.update_status.%s' % rel.id, data = status.get('id'))
fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict())
return True
couchpotato/core/plugins/renamer/__init__.py
+1
View File
@@ -28,6 +28,7 @@ rename_options = {
'cd': 'CD number (cd1)',
'cd_nr': 'Just the cd nr. (1)',
'mpaa': 'MPAA Rating',
'category': 'Category label',
},
}
couchpotato/core/plugins/renamer/main.py
+303 -159
View File
@@ -1,9 +1,9 @@
from couchpotato import get_session
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss
from couchpotato.core.helpers.encoding import toUnicode, ss, sp
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb, link, symlink, tryInt
getImdb, link, symlink, tryInt, splitString
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, Release, \
@@ -31,8 +31,10 @@ class Renamer(Plugin):
'params': {
'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'},
'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'},
'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'},
'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'},
'files': {'desc': 'Optional: Provide the release files if more releases are in the same movie_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'},
'downloader' : {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'},
'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in movie_folder. \'downloader\' is required with this option.'},
'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''},
},
})
@@ -62,23 +64,26 @@ class Renamer(Plugin):
def scanView(self, **kwargs):
async = tryInt(kwargs.get('async', 0))
movie_folder = kwargs.get('movie_folder')
movie_folder = sp(kwargs.get('movie_folder'))
downloader = kwargs.get('downloader')
download_id = kwargs.get('download_id')
files = '|'.join([sp(filename) for filename in splitString(kwargs.get('files'), '|')])
status = kwargs.get('status', 'completed')
download_info = {'folder': movie_folder} if movie_folder else None
if download_info:
download_info.update({'id': download_id, 'downloader': downloader} if download_id else {})
release_download = {'folder': movie_folder} if movie_folder else None
if release_download:
release_download.update({'id': download_id, 'downloader': downloader, 'status': status, 'files': files} if download_id else {})
fire_handle = fireEvent if not async else fireEventAsync
fire_handle('renamer.scan', download_info)
fire_handle('renamer.scan', release_download)
return {
'success': True
}
def scan(self, download_info = None):
def scan(self, release_download = None):
if not release_download: release_download = {}
if self.isDisabled():
return
@@ -87,22 +92,66 @@ class Renamer(Plugin):
log.info('Renamer is already running, if you see this often, check the logs above for errors.')
return
movie_folder = download_info and download_info.get('folder')
from_folder = sp(self.conf('from'))
to_folder = sp(self.conf('to'))
# Check to see if the "to" folder is inside the "from" folder.
if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')):
l = log.debug if movie_folder else log.error
l('Both the "To" and "From" have to exist.')
return
elif self.conf('from') in self.conf('to'):
log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.')
return
elif movie_folder and movie_folder in [self.conf('to'), self.conf('from')]:
log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.')
# Get movie folder to process
movie_folder = release_download.get('folder')
# Get all folders that should not be processed
no_process = [to_folder]
cat_list = fireEvent('category.all', single = True) or []
no_process.extend([item['destination'] for item in cat_list])
try:
if Env.setting('library', section = 'manage').strip():
no_process.extend([sp(manage_folder) for manage_folder in splitString(Env.setting('library', section = 'manage'), '::')])
except:
pass
# Check to see if the no_process folders are inside the "from" folder.
if not os.path.isdir(from_folder) or not os.path.isdir(to_folder):
log.error('Both the "To" and "From" have to exist.')
return
else:
for item in no_process:
if from_folder in item:
log.error('To protect your data, the movie libraries can\'t be inside of or the same as the "from" folder.')
return
# Check to see if the no_process folders are inside the provided movie_folder
if movie_folder and not os.path.isdir(movie_folder):
log.debug('The provided movie folder %s does not exist. Trying to find it in the \'from\' folder.', movie_folder)
# Update to the from folder
if len(splitString(release_download.get('files'), '|')) == 1:
new_movie_folder = from_folder
else:
new_movie_folder = os.path.join(from_folder, os.path.basename(movie_folder))
if not os.path.isdir(new_movie_folder):
log.error('The provided movie folder %s does not exist and could also not be found in the \'from\' folder.', movie_folder)
return
# Update the files
new_files = [os.path.join(new_movie_folder, os.path.relpath(filename, movie_folder)) for filename in splitString(release_download.get('files'), '|')]
if new_files and not os.path.isfile(new_files[0]):
log.error('The provided movie folder %s does not exist and its files could also not be found in the \'from\' folder.', movie_folder)
return
# Update release_download info to the from folder
log.debug('Release %s found in the \'from\' folder.', movie_folder)
release_download['folder'] = new_movie_folder
release_download['files'] = '|'.join(new_files)
movie_folder = new_movie_folder
if movie_folder:
for item in no_process:
if movie_folder in item:
log.error('To protect your data, the movie libraries can\'t be inside of or the same as the provided movie folder.')
return
# Make sure a checkSnatched marked all downloads/seeds as such
if not download_info and self.conf('run_every') > 0:
if not release_download and self.conf('run_every') > 0:
fireEvent('renamer.check_snatched')
self.renaming_started = True
@@ -112,29 +161,35 @@ class Renamer(Plugin):
files = []
if movie_folder:
log.info('Scanning movie folder %s...', movie_folder)
movie_folder = movie_folder.rstrip(os.path.sep)
folder = os.path.dirname(movie_folder)
# Get all files from the specified folder
try:
for root, folders, names in os.walk(movie_folder):
files.extend([os.path.join(root, name) for name in names])
except:
log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc()))
if release_download.get('files', ''):
files = splitString(release_download['files'], '|')
# If there is only one file in the torrent, the downloader did not create a subfolder
if len(files) == 1:
folder = movie_folder
else:
# Get all files from the specified folder
try:
for root, folders, names in os.walk(movie_folder):
files.extend([sp(os.path.join(root, name)) for name in names])
except:
log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc()))
db = get_session()
# Extend the download info with info stored in the downloaded release
download_info = self.extendDownloadInfo(download_info)
release_download = self.extendReleaseDownload(release_download)
# Unpack any archives
extr_files = None
if self.conf('unrar'):
folder, movie_folder, files, extr_files = self.extractFiles(folder = folder, movie_folder = movie_folder, files = files,
cleanup = self.conf('cleanup') and not self.downloadIsTorrent(download_info))
cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download))
groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'),
files = files, download_info = download_info, return_ignored = False, single = True)
groups = fireEvent('scanner.scan', folder = folder if folder else from_folder,
files = files, release_download = release_download, return_ignored = False, single = True) or []
folder_name = self.conf('folder_name')
file_name = self.conf('file_name')
@@ -142,9 +197,9 @@ class Renamer(Plugin):
nfo_name = self.conf('nfo_name')
separator = self.conf('separator')
# Statusses
done_status, active_status, downloaded_status, snatched_status = \
fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched'], single = True)
# Statuses
done_status, active_status, downloaded_status, snatched_status, seeding_status = \
fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched', 'seeding'], single = True)
for group_identifier in groups:
@@ -157,7 +212,7 @@ class Renamer(Plugin):
# Add _UNKNOWN_ if no library item is connected
if not group['library'] or not movie_title:
self.tagDir(group, 'unknown')
self.tagRelease(group = group, tag = 'unknown')
continue
# Rename the files using the library data
else:
@@ -172,8 +227,13 @@ class Renamer(Plugin):
movie_title = getTitle(library)
# Overwrite destination when set in category
destination = self.conf('to')
for movie in library_ent.media:
destination = to_folder
category_label = ''
for movie in library_ent.movies:
if movie.category and movie.category.label:
category_label = movie.category.label
if movie.category and movie.category.destination and len(movie.category.destination) > 0 and movie.category.destination != 'None':
destination = movie.category.destination
log.debug('Setting category destination for "%s": %s' % (movie_title, destination))
@@ -190,7 +250,7 @@ class Renamer(Plugin):
if extr_files:
group['before_rename'].extend(extr_files)
# Remove weird chars from moviename
# Remove weird chars from movie name
movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title)
# Put 'The' at the end
@@ -217,6 +277,7 @@ class Renamer(Plugin):
'cd': '',
'cd_nr': '',
'mpaa': library['info'].get('mpaa', ''),
'category': category_label,
}
for file_type in group['files']:
@@ -225,7 +286,7 @@ class Renamer(Plugin):
if file_type is 'nfo' and not self.conf('rename_nfo'):
log.debug('Skipping, renaming of %s disabled', file_type)
for current_file in group['files'][file_type]:
if self.conf('cleanup') and (not self.downloadIsTorrent(download_info) or self.fileIsAdded(current_file, group)):
if self.conf('cleanup') and (not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)):
remove_files.append(current_file)
continue
@@ -385,7 +446,7 @@ class Renamer(Plugin):
log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label))
# Add exists tag to the .ignore file
self.tagDir(group, 'exists')
self.tagRelease(group = group, tag = 'exists')
# Notify on rename fail
download_message = 'Renaming of %s (%s) cancelled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label)
@@ -393,10 +454,20 @@ class Renamer(Plugin):
remove_leftovers = False
break
elif release.status_id is snatched_status.get('id'):
if release.quality.id is group['meta_data']['quality']['id']:
# Set the release to downloaded
fireEvent('release.update_status', release.id, status = downloaded_status, single = True)
elif release.status_id in [snatched_status.get('id'), seeding_status.get('id')]:
if release_download and release_download.get('rls_id'):
if release_download['rls_id'] == release.id:
if release_download['status'] == 'completed':
# Set the release to downloaded
fireEvent('release.update_status', release.id, status = downloaded_status, single = True)
elif release_download['status'] == 'seeding':
# Set the release to seeding
fireEvent('release.update_status', release.id, status = seeding_status, single = True)
elif release.quality.id is group['meta_data']['quality']['id']:
# Set the release to downloaded
fireEvent('release.update_status', release.id, status = downloaded_status, single = True)
# Remove leftover files
if not remove_leftovers: # Don't remove anything
@@ -405,7 +476,7 @@ class Renamer(Plugin):
log.debug('Removing leftover files')
for current_file in group['files']['leftover']:
if self.conf('cleanup') and not self.conf('move_leftover') and \
(not self.downloadIsTorrent(download_info) or self.fileIsAdded(current_file, group)):
(not self.downloadIsTorrent(release_download) or self.fileIsAdded(current_file, group)):
remove_files.append(current_file)
# Remove files
@@ -421,17 +492,17 @@ class Renamer(Plugin):
log.info('Removing "%s"', src)
try:
src = ss(src)
src = sp(src)
if os.path.isfile(src):
os.remove(src)
parent_dir = os.path.normpath(os.path.dirname(src))
if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not parent_dir in [destination, movie_folder] and not self.conf('from') in parent_dir:
parent_dir = os.path.dirname(src)
if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not parent_dir in [destination, movie_folder] and not from_folder in parent_dir:
delete_folders.append(parent_dir)
except:
log.error('Failed removing %s: %s', (src, traceback.format_exc()))
self.tagDir(group, 'failed_remove')
self.tagRelease(group = group, tag = 'failed_remove')
# Delete leftover folder from older releases
for delete_folder in delete_folders:
@@ -451,15 +522,15 @@ class Renamer(Plugin):
self.makeDir(os.path.dirname(dst))
try:
self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(download_info) or self.fileIsAdded(src, group))
self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group))
group['renamed_files'].append(dst)
except:
log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
self.tagDir(group, 'failed_rename')
self.tagRelease(group = group, tag = 'failed_rename')
# Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent
if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(download_info):
self.tagDir(group, 'renamed_already')
if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(release_download):
self.tagRelease(group = group, tag = 'renamed_already')
# Remove matching releases
for release in remove_releases:
@@ -469,13 +540,13 @@ class Renamer(Plugin):
except:
log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc()))
if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(download_info):
if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download):
if movie_folder:
# Delete the movie folder
group_folder = movie_folder
else:
# Delete the first empty subfolder in the tree relative to the 'from' folder
group_folder = os.path.join(self.conf('from'), os.path.relpath(group['parentdir'], self.conf('from')).split(os.path.sep)[0])
group_folder = sp(os.path.join(from_folder, os.path.relpath(group['parentdir'], from_folder).split(os.path.sep)[0]))
try:
log.info('Deleting folder: %s', group_folder)
@@ -516,18 +587,9 @@ class Renamer(Plugin):
return rename_files
# This adds a file to ignore / tag a release so it is ignored later
def tagDir(self, group, tag):
ignore_file = None
if isinstance(group, dict):
for movie_file in sorted(list(group['files']['movie'])):
ignore_file = '%s.%s.ignore' % (os.path.splitext(movie_file)[0], tag)
break
else:
if not os.path.isdir(group) or not tag:
return
ignore_file = os.path.join(group, '%s.ignore' % tag)
def tagRelease(self, tag, group = None, release_download = None):
if not tag:
return
text = """This file is from CouchPotato
It has marked this release as "%s"
@@ -535,25 +597,88 @@ This file hides the release from the renamer
Remove it if you want it to be renamed (again, or at least let it try again)
""" % tag
if ignore_file:
self.createFile(ignore_file, text)
tag_files = []
def untagDir(self, folder, tag = ''):
if not os.path.isdir(folder):
# Tag movie files if they are known
if isinstance(group, dict):
tag_files = [sorted(list(group['files']['movie']))[0]]
elif isinstance(release_download, dict):
# Tag download_files if they are known
if release_download['files']:
tag_files = splitString(release_download['files'], '|')
# Tag all files in release folder
else:
for root, folders, names in os.walk(release_download['folder']):
tag_files.extend([os.path.join(root, name) for name in names])
for filename in tag_files:
tag_filename = '%s.%s.ignore' % (os.path.splitext(filename)[0], tag)
if not os.path.isfile(tag_filename):
self.createFile(tag_filename, text)
def untagRelease(self, release_download, tag = ''):
if not release_download:
return
# Remove any .ignore files
for root, dirnames, filenames in os.walk(folder):
for filename in fnmatch.filter(filenames, '*%s.ignore' % tag):
os.remove((os.path.join(root, filename)))
tag_files = []
def hastagDir(self, folder, tag = ''):
folder = release_download['folder']
if not os.path.isdir(folder):
return False
# Find any .ignore files
# Untag download_files if they are known
if release_download['files']:
tag_files = splitString(release_download['files'], '|')
# Untag all files in release folder
else:
for root, folders, names in os.walk(release_download['folder']):
tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
# Find all .ignore files in folder
ignore_files = []
for root, dirnames, filenames in os.walk(folder):
if fnmatch.filter(filenames, '*%s.ignore' % tag):
ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
# Match all found ignore files with the tag_files and delete if found
for tag_file in tag_files:
ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (re.escape(os.path.splitext(tag_file)[0]), tag if tag else '*'))
for filename in ignore_file:
try:
os.remove(filename)
except:
log.debug('Unable to remove ignore file: %s. Error: %s.' % (filename, traceback.format_exc()))
def hastagRelease(self, release_download, tag = ''):
if not release_download:
return False
folder = release_download['folder']
if not os.path.isdir(folder):
return False
tag_files = []
ignore_files = []
# Find tag on download_files if they are known
if release_download['files']:
tag_files = splitString(release_download['files'], '|')
# Find tag on all files in release folder
else:
for root, folders, names in os.walk(release_download['folder']):
tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore'])
# Find all .ignore files in folder
for root, dirnames, filenames in os.walk(folder):
ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag))
# Match all found ignore files with the tag_files and return True found
for tag_file in tag_files:
ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*'))
if ignore_file:
return True
return False
@@ -572,7 +697,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
link(old, dest)
except:
# Try to simlink next
log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s. ', (old, dest, traceback.format_exc()))
log.debug('Couldn\'t hardlink file "%s" to "%s". Simlinking instead. Error: %s.', (old, dest, traceback.format_exc()))
shutil.copy(old, dest)
try:
symlink(dest, old + '.link')
@@ -616,22 +741,38 @@ Remove it if you want it to be renamed (again, or at least let it try again)
replaced = toUnicode(string)
for x, r in replacements.iteritems():
if x in ['thename', 'namethe']:
continue
if r is not None:
replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r))
else:
#If information is not available, we don't want the tag in the filename
replaced = replaced.replace('<' + x + '>', '')
replaced = self.replaceDoubles(replaced.lstrip('. '))
for x, r in replacements.iteritems():
if x in ['thename', 'namethe']:
replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r))
replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced)
sep = self.conf('foldersep') if folder else self.conf('separator')
return self.replaceDoubles(replaced.lstrip('. ')).replace(' ', ' ' if not sep else sep)
return replaced.replace(' ', ' ' if not sep else sep)
def replaceDoubles(self, string):
return string.replace(' ', ' ').replace(' .', '.')
replaces = [
('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '),
('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'),
]
for r in replaces:
reg, replace_with = r
string = re.sub(reg, replace_with, string)
return string
def deleteEmptyFolder(self, folder, show_error = True):
folder = ss(folder)
folder = sp(folder)
loge = log.error if show_error else log.debug
for root, dirs, files in os.walk(folder):
@@ -657,22 +798,22 @@ Remove it if you want it to be renamed (again, or at least let it try again)
self.checking_snatched = True
snatched_status, ignored_status, failed_status, done_status, seeding_status, downloaded_status, missing_status = \
fireEvent('status.get', ['snatched', 'ignored', 'failed', 'done', 'seeding', 'downloaded', 'missing'], single = True)
snatched_status, ignored_status, failed_status, seeding_status, downloaded_status, missing_status = \
fireEvent('status.get', ['snatched', 'ignored', 'failed', 'seeding', 'downloaded', 'missing'], single = True)
db = get_session()
rels = db.query(Release).filter(
Release.status_id.in_([snatched_status.get('id'), seeding_status.get('id'), missing_status.get('id')])
).all()
scan_items = []
scan_releases = []
scan_required = False
if rels:
log.debug('Checking status snatched releases...')
statuses = fireEvent('download.status', merge = True)
if not statuses:
release_downloads = fireEvent('download.status', merge = True)
if not release_downloads:
log.debug('Download status functionality is not implemented for active downloaders.')
scan_required = True
else:
@@ -680,91 +821,91 @@ Remove it if you want it to be renamed (again, or at least let it try again)
for rel in rels:
rel_dict = rel.to_dict({'info': {}})
movie_dict = fireEvent('movie.get', rel.media_id, single = True)
if not isinstance(rel_dict['info'], (dict)):
log.error('Faulty release found without any info, ignoring.')
fireEvent('release.update_status', rel.id, status = ignored_status, single = True)
continue
# check status
nzbname = self.createNzbName(rel_dict['info'], movie_dict)
found = False
for item in statuses:
for release_download in release_downloads:
found_release = False
if rel_dict['info'].get('download_id'):
if item['id'] == rel_dict['info']['download_id'] and item['downloader'] == rel_dict['info']['download_downloader']:
log.debug('Found release by id: %s', item['id'])
if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']:
log.debug('Found release by id: %s', release_download['id'])
found_release = True
else:
if item['name'] == nzbname or rel_dict['info']['name'] in item['name'] or getImdb(item['name']) == movie_dict['library']['identifier']:
if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']:
found_release = True
if found_release:
timeleft = 'N/A' if item['timeleft'] == -1 else item['timeleft']
log.debug('Found %s: %s, time to go: %s', (item['name'], item['status'].upper(), timeleft))
timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft']
log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft))
if item['status'] == 'busy':
if release_download['status'] == 'busy':
# Set the release to snatched if it was missing before
fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
# Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading
if item['folder'] and self.conf('from') in item['folder']:
self.tagDir(item['folder'], 'downloading')
elif item['status'] == 'seeding':
# Set the release to seeding
fireEvent('release.update_status', rel.id, status = seeding_status, single = True)
if self.movieInFromFolder(release_download['folder']):
self.tagRelease(release_download = release_download, tag = 'downloading')
elif release_download['status'] == 'seeding':
#If linking setting is enabled, process release
if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(item):
log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (item['name'], item['seed_ratio']))
if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download):
log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio']))
# Remove the downloading tag
self.untagDir(item['folder'], 'downloading')
self.untagRelease(release_download = release_download, tag = 'downloading')
# Scan and set the torrent to paused if required
item.update({'pause': True, 'scan': True, 'process_complete': False})
scan_items.append(item)
release_download.update({'pause': True, 'scan': True, 'process_complete': False})
scan_releases.append(release_download)
else:
#let it seed
log.debug('%s is seeding with ratio: %s', (item['name'], item['seed_ratio']))
log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio']))
elif item['status'] == 'failed':
# Set the release to seeding
fireEvent('release.update_status', rel.id, status = seeding_status, single = True)
elif release_download['status'] == 'failed':
# Set the release to failed
fireEvent('release.update_status', rel.id, status = failed_status, single = True)
fireEvent('download.remove_failed', item, single = True)
fireEvent('download.remove_failed', release_download, single = True)
if self.conf('next_on_failed'):
fireEvent('movie.searcher.try_next_release', media_id = rel.media_id)
elif item['status'] == 'completed':
log.info('Download of %s completed!', item['name'])
if self.statusInfoComplete(item):
fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id)
elif release_download['status'] == 'completed':
log.info('Download of %s completed!', release_download['name'])
if self.statusInfoComplete(release_download):
# If the release has been seeding, process now the seeding is done
if rel.status_id == seeding_status.get('id'):
if rel.movie.status_id == done_status.get('id'):
if self.conf('file_action') != 'move':
# Set the release to done as the movie has already been renamed
fireEvent('release.update_status', rel.id, status = downloaded_status, single = True)
# Allow the downloader to clean-up
item.update({'pause': False, 'scan': False, 'process_complete': True})
scan_items.append(item)
release_download.update({'pause': False, 'scan': False, 'process_complete': True})
scan_releases.append(release_download)
else:
# Set the release to snatched so that the renamer can process the release as if it was never seeding
fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
# Scan and Allow the downloader to clean-up
item.update({'pause': False, 'scan': True, 'process_complete': True})
scan_items.append(item)
release_download.update({'pause': False, 'scan': True, 'process_complete': True})
scan_releases.append(release_download)
else:
# Set the release to snatched if it was missing before
fireEvent('release.update_status', rel.id, status = snatched_status, single = True)
# Remove the downloading tag
self.untagDir(item['folder'], 'downloading')
self.untagRelease(release_download = release_download, tag = 'downloading')
# Scan and Allow the downloader to clean-up
item.update({'pause': False, 'scan': True, 'process_complete': True})
scan_items.append(item)
release_download.update({'pause': False, 'scan': True, 'process_complete': True})
scan_releases.append(release_download)
else:
scan_required = True
@@ -786,21 +927,21 @@ Remove it if you want it to be renamed (again, or at least let it try again)
log.error('Failed checking for release in downloader: %s', traceback.format_exc())
# The following can either be done here, or inside the scanner if we pass it scan_items in one go
for item in scan_items:
for release_download in scan_releases:
# Ask the renamer to scan the item
if item['scan']:
if item['pause'] and self.conf('file_action') == 'link':
fireEvent('download.pause', item = item, pause = True, single = True)
fireEvent('renamer.scan', download_info = item)
if item['pause'] and self.conf('file_action') == 'link':
fireEvent('download.pause', item = item, pause = False, single = True)
if item['process_complete']:
if release_download['scan']:
if release_download['pause'] and self.conf('file_action') == 'link':
fireEvent('download.pause', release_download = release_download, pause = True, single = True)
fireEvent('renamer.scan', release_download = release_download)
if release_download['pause'] and self.conf('file_action') == 'link':
fireEvent('download.pause', release_download = release_download, pause = False, single = True)
if release_download['process_complete']:
#First make sure the files were succesfully processed
if not self.hastagDir(item['folder'], 'failed_rename'):
if not self.hastagRelease(release_download = release_download, tag = 'failed_rename'):
# Remove the seeding tag if it exists
self.untagDir(item['folder'], 'renamed_already')
self.untagRelease(release_download = release_download, tag = 'renamed_already')
# Ask the downloader to process the item
fireEvent('download.process_complete', item = item, single = True)
fireEvent('download.process_complete', release_download = release_download, single = True)
if scan_required:
fireEvent('renamer.scan')
@@ -809,16 +950,16 @@ Remove it if you want it to be renamed (again, or at least let it try again)
return True
def extendDownloadInfo(self, download_info):
def extendReleaseDownload(self, release_download):
rls = None
if download_info and download_info.get('id') and download_info.get('downloader'):
if release_download and release_download.get('id') and release_download.get('downloader'):
db = get_session()
rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = download_info.get('downloader')).all()
rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = download_info.get('id')).all()
rlsnfo_dwnlds = db.query(ReleaseInfo).filter_by(identifier = 'download_downloader', value = release_download.get('downloader')).all()
rlsnfo_ids = db.query(ReleaseInfo).filter_by(identifier = 'download_id', value = release_download.get('id')).all()
for rlsnfo_dwnld in rlsnfo_dwnlds:
for rlsnfo_id in rlsnfo_ids:
@@ -828,32 +969,33 @@ Remove it if you want it to be renamed (again, or at least let it try again)
if rls: break
if not rls:
log.error('Download ID %s from downloader %s not found in releases', (download_info.get('id'), download_info.get('downloader')))
log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader')))
if rls:
rls_dict = rls.to_dict({'info':{}})
download_info.update({
release_download.update({
'imdb_id': rls.movie.library.identifier,
'quality': rls.quality.identifier,
'protocol': rls_dict.get('info', {}).get('protocol') or rls_dict.get('info', {}).get('type'),
'rls_id': rls.id,
})
return download_info
return release_download
def downloadIsTorrent(self, download_info):
return download_info and download_info.get('protocol') in ['torrent', 'torrent_magnet']
def downloadIsTorrent(self, release_download):
return release_download and release_download.get('protocol') in ['torrent', 'torrent_magnet']
def fileIsAdded(self, src, group):
if not group or not group.get('before_rename'):
return False
return src in group['before_rename']
def statusInfoComplete(self, item):
return item['id'] and item['downloader'] and item['folder']
def statusInfoComplete(self, release_download):
return release_download['id'] and release_download['downloader'] and release_download['folder']
def movieInFromFolder(self, movie_folder):
return movie_folder and self.conf('from') in movie_folder or not movie_folder
return movie_folder and sp(self.conf('from')) in sp(movie_folder) or not movie_folder
def extractFiles(self, folder = None, movie_folder = None, files = None, cleanup = False):
if not files: files = []
@@ -863,9 +1005,11 @@ Remove it if you want it to be renamed (again, or at least let it try again)
restfile_regex = '(^%s\.(?:part(?!0*1\.rar$)\d+\.rar$|[rstuvw]\d+$))'
extr_files = []
from_folder = sp(self.conf('from'))
# Check input variables
if not folder:
folder = self.conf('from')
folder = from_folder
check_file_date = True
if movie_folder:
@@ -873,7 +1017,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
if not files:
for root, folders, names in os.walk(folder):
files.extend([os.path.join(root, name) for name in names])
files.extend([sp(os.path.join(root, name)) for name in names])
# Find all archive files
archives = [re.search(archive_regex, name).groupdict() for name in files if re.search(archive_regex, name)]
@@ -881,7 +1025,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
#Extract all found archives
for archive in archives:
# Check if it has already been processed by CPS
if self.hastagDir(os.path.dirname(archive['file'])):
if self.hastagRelease(release_download = {'folder': os.path.dirname(archive['file']), 'files': archive['file']}):
continue
# Find all related archive files
@@ -919,13 +1063,13 @@ Remove it if you want it to be renamed (again, or at least let it try again)
log.info('Archive %s found. Extracting...', os.path.basename(archive['file']))
try:
rar_handle = RarFile(archive['file'])
extr_path = os.path.join(self.conf('from'), os.path.relpath(os.path.dirname(archive['file']), folder))
extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder))
self.makeDir(extr_path)
for packedinfo in rar_handle.infolist():
if not packedinfo.isdir and not os.path.isfile(os.path.join(extr_path, os.path.basename(packedinfo.filename))):
if not packedinfo.isdir and not os.path.isfile(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))):
log.debug('Extracting %s...', packedinfo.filename)
rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False)
extr_files.append(os.path.join(extr_path, os.path.basename(packedinfo.filename)))
extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename))))
del rar_handle
except Exception, e:
log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc()))
@@ -942,9 +1086,9 @@ Remove it if you want it to be renamed (again, or at least let it try again)
files.remove(filename)
# Move the rest of the files and folders if any files are extracted to the from folder (only if folder was provided)
if extr_files and os.path.normpath(os.path.normcase(folder)) != os.path.normpath(os.path.normcase(self.conf('from'))):
if extr_files and folder != from_folder:
for leftoverfile in list(files):
move_to = os.path.join(self.conf('from'), os.path.relpath(leftoverfile, folder))
move_to = os.path.join(from_folder, os.path.relpath(leftoverfile, folder))
try:
self.makeDir(os.path.dirname(move_to))
@@ -967,8 +1111,8 @@ Remove it if you want it to be renamed (again, or at least let it try again)
log.debug('Removing old movie folder %s...', movie_folder)
self.deleteEmptyFolder(movie_folder)
movie_folder = os.path.join(self.conf('from'), os.path.relpath(movie_folder, folder))
folder = self.conf('from')
movie_folder = os.path.join(from_folder, os.path.relpath(movie_folder, folder))
folder = from_folder
if extr_files:
files.extend(extr_files)
couchpotato/core/plugins/scanner/main.py
+57 -61
View File
@@ -1,6 +1,6 @@
from couchpotato import get_session
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss
from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss, sp
from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \
splitString
from couchpotato.core.logger import CPLog
@@ -21,10 +21,6 @@ log = CPLog(__name__)
class Scanner(Plugin):
minimal_filesize = {
'media': 314572800, # 300MB
'trailer': 1048576, # 1MB
}
ignored_in_path = [os.path.sep + 'extracted' + os.path.sep, 'extracting', '_unpack', '_failed_', '_unknown_', '_exists_', '_failed_remove_',
'_failed_rename_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo',
'thumbs.db', 'ehthumbs.db', 'desktop.ini'] #unpacking, smb-crap, hidden files
@@ -52,6 +48,12 @@ class Scanner(Plugin):
'leftover': ('leftover', 'leftover'),
}
file_sizes = { # in MB
'movie': {'min': 300},
'trailer': {'min': 2, 'max': 250},
'backdrop': {'min': 0, 'max': 5},
}
codecs = {
'audio': ['dts', 'ac3', 'ac3d', 'mp3'],
'video': ['x264', 'h264', 'divx', 'xvid']
@@ -104,9 +106,9 @@ class Scanner(Plugin):
addEvent('scanner.name_year', self.getReleaseNameYear)
addEvent('scanner.partnumber', self.getPartNumber)
def scan(self, folder = None, files = None, download_info = None, simple = False, newer_than = 0, return_ignored = True, on_found = None):
def scan(self, folder = None, files = None, release_download = None, simple = False, newer_than = 0, return_ignored = True, on_found = None):
folder = ss(os.path.normpath(folder))
folder = sp(folder)
if not folder or not os.path.isdir(folder):
log.error('Folder doesn\'t exists: %s', folder)
@@ -122,7 +124,7 @@ class Scanner(Plugin):
try:
files = []
for root, dirs, walk_files in os.walk(folder):
files.extend(os.path.join(root, filename) for filename in walk_files)
files.extend([sp(os.path.join(root, filename)) for filename in walk_files])
# Break if CP wants to shut down
if self.shuttingDown():
@@ -132,7 +134,7 @@ class Scanner(Plugin):
log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
else:
check_file_date = False
files = [ss(x) for x in files]
files = [sp(x) for x in files]
for file_path in files:
@@ -148,7 +150,7 @@ class Scanner(Plugin):
continue
is_dvd_file = self.isDVDFile(file_path)
if os.path.getsize(file_path) > self.minimal_filesize['media'] or is_dvd_file: # Minimal 300MB files or is DVD file
if self.filesizeBetween(file_path, self.file_sizes['movie']) or is_dvd_file: # Minimal 300MB files or is DVD file
# Normal identifier
identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file)
@@ -182,7 +184,6 @@ class Scanner(Plugin):
# files will be grouped first.
leftovers = set(sorted(leftovers, reverse = True))
# Group files minus extension
ignored_identifiers = []
for identifier, group in movie_files.iteritems():
@@ -191,7 +192,7 @@ class Scanner(Plugin):
log.debug('Grouping files: %s', identifier)
has_ignored = 0
for file_path in group['unsorted_files']:
for file_path in list(group['unsorted_files']):
ext = getExt(file_path)
wo_ext = file_path[:-(len(ext) + 1)]
found_files = set([i for i in leftovers if wo_ext in i])
@@ -200,6 +201,11 @@ class Scanner(Plugin):
has_ignored += 1 if ext == 'ignore' else 0
if has_ignored == 0:
for file_path in list(group['unsorted_files']):
ext = getExt(file_path)
has_ignored += 1 if ext == 'ignore' else 0
if has_ignored > 0:
ignored_identifiers.append(identifier)
@@ -232,10 +238,6 @@ class Scanner(Plugin):
# Remove the found files from the leftover stack
leftovers = leftovers - set(found_files)
exts = [getExt(ff) for ff in found_files]
if 'ignore' in exts:
ignored_identifiers.append(identifier)
# Break if CP wants to shut down
if self.shuttingDown():
break
@@ -262,14 +264,14 @@ class Scanner(Plugin):
# Remove the found files from the leftover stack
leftovers = leftovers - set([ff])
ext = getExt(ff)
if ext == 'ignore':
ignored_identifiers.append(new_identifier)
# Break if CP wants to shut down
if self.shuttingDown():
break
# leftovers should be empty
if leftovers:
log.debug('Some files are still left over: %s', leftovers)
# Cleaning up used
for identifier in delete_identifiers:
if path_identifiers.get(identifier):
@@ -339,11 +341,11 @@ class Scanner(Plugin):
total_found = len(valid_files)
# Make sure only one movie was found if a download ID is provided
if download_info and total_found == 0:
log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', download_info.get('imdb_id'))
elif download_info and total_found > 1:
log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (download_info.get('imdb_id'), len(valid_files)))
download_info = None
if release_download and total_found == 0:
log.info('Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', release_download.get('imdb_id'))
elif release_download and total_found > 1:
log.info('Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (release_download.get('imdb_id'), len(valid_files)))
release_download = None
# Determine file types
db = get_session()
@@ -379,7 +381,7 @@ class Scanner(Plugin):
continue
log.debug('Getting metadata for %s', identifier)
group['meta_data'] = self.getMetaData(group, folder = folder, download_info = download_info)
group['meta_data'] = self.getMetaData(group, folder = folder, release_download = release_download)
# Subtitle meta
group['subtitle_language'] = self.getSubtitleLanguage(group) if not simple else {}
@@ -411,7 +413,7 @@ class Scanner(Plugin):
del group['unsorted_files']
# Determine movie
group['library'] = self.determineMovie(group, download_info = download_info)
group['library'] = self.determineMovie(group, release_download = release_download)
if not group['library']:
log.error('Unable to determine movie: %s', group['identifiers'])
else:
@@ -436,13 +438,13 @@ class Scanner(Plugin):
return processed_movies
def getMetaData(self, group, folder = '', download_info = None):
def getMetaData(self, group, folder = '', release_download = None):
data = {}
files = list(group['files']['movie'])
for cur_file in files:
if os.path.getsize(cur_file) < self.minimal_filesize['media']: continue # Ignore smaller files
if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files
meta = self.getMeta(cur_file)
@@ -461,8 +463,8 @@ class Scanner(Plugin):
# Use the quality guess first, if that failes use the quality we wanted to download
data['quality'] = None
if download_info and download_info.get('quality'):
data['quality'] = fireEvent('quality.single', download_info.get('quality'), single = True)
if release_download and release_download.get('quality'):
data['quality'] = fireEvent('quality.single', release_download.get('quality'), single = True)
if not data['quality']:
data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True)
@@ -546,12 +548,12 @@ class Scanner(Plugin):
return detected_languages
def determineMovie(self, group, download_info = None):
def determineMovie(self, group, release_download = None):
# Get imdb id from downloader
imdb_id = download_info and download_info.get('imdb_id')
imdb_id = release_download and release_download.get('imdb_id')
if imdb_id:
log.debug('Found movie via imdb id from it\'s download id: %s', download_info.get('imdb_id'))
log.debug('Found movie via imdb id from it\'s download id: %s', release_download.get('imdb_id'))
files = group['files']
@@ -652,7 +654,7 @@ class Scanner(Plugin):
def getMediaFiles(self, files):
def test(s):
return self.filesizeBetween(s, 300, 100000) and getExt(s.lower()) in self.extensions['movie'] and not self.isSampleFile(s)
return self.filesizeBetween(s, self.file_sizes['movie']) and getExt(s.lower()) in self.extensions['movie'] and not self.isSampleFile(s)
return set(filter(test, files))
@@ -677,7 +679,7 @@ class Scanner(Plugin):
def getTrailers(self, files):
def test(s):
return re.search('(^|[\W_])trailer\d*[\W_]', s.lower()) and self.filesizeBetween(s, 2, 250)
return re.search('(^|[\W_])trailer\d*[\W_]', s.lower()) and self.filesizeBetween(s, self.file_sizes['trailer'])
return set(filter(test, files))
@@ -688,7 +690,7 @@ class Scanner(Plugin):
files = set(filter(test, files))
images = {
'backdrop': set(filter(lambda s: re.search('(^|[\W_])fanart|backdrop\d*[\W_]', s.lower()) and self.filesizeBetween(s, 0, 5), files))
'backdrop': set(filter(lambda s: re.search('(^|[\W_])fanart|backdrop\d*[\W_]', s.lower()) and self.filesizeBetween(s, self.file_sizes['backdrop']), files))
}
# Rest
@@ -716,16 +718,6 @@ class Scanner(Plugin):
log.debug('Ignored "%s" contains "%s".', (filename, i))
return False
# Sample file
if self.isSampleFile(filename):
log.debug('Is sample file "%s".', filename)
return False
# Minimal size
if self.filesizeBetween(filename, self.minimal_filesize['media']):
log.debug('File to small: %s', filename)
return False
# All is OK
return True
@@ -734,9 +726,9 @@ class Scanner(Plugin):
if is_sample: log.debug('Is sample file: %s', filename)
return is_sample
def filesizeBetween(self, file, min = 0, max = 100000):
def filesizeBetween(self, file, file_size = []):
try:
return (min * 1048576) < os.path.getsize(file) < (max * 1048576)
return (file_size.get('min', 0) * 1048576) < os.path.getsize(file) < (file_size.get('max', 100000) * 1048576)
except:
log.error('Couldn\'t get filesize of %s.', file)
@@ -830,19 +822,21 @@ class Scanner(Plugin):
def findYear(self, text):
# Search year inside () or [] first
matches = re.search('(\(|\[)(?P<year>19[0-9]{2}|20[0-9]{2})(\]|\))', text)
matches = re.findall('(\(|\[)(?P<year>19[0-9]{2}|20[0-9]{2})(\]|\))', text)
if matches:
return matches.group('year')
return matches[-1][1]
# Search normal
matches = re.search('(?P<year>19[0-9]{2}|20[0-9]{2})', text)
matches = re.findall('(?P<year>19[0-9]{2}|20[0-9]{2})', text)
if matches:
return matches.group('year')
return matches[-1]
return ''
def getReleaseNameYear(self, release_name, file_name = None):
release_name = release_name.strip(' .-_')
# Use guessit first
guess = {}
if file_name:
@@ -860,7 +854,7 @@ class Scanner(Plugin):
cleaned = ' '.join(re.split('\W+', simplifyString(release_name)))
cleaned = re.sub(self.clean, ' ', cleaned)
for year_str in [file_name, cleaned]:
for year_str in [file_name, release_name, cleaned]:
if not year_str: continue
year = self.findYear(year_str)
if year:
@@ -870,19 +864,21 @@ class Scanner(Plugin):
if year: # Split name on year
try:
movie_name = cleaned.split(year).pop(0).strip()
cp_guess = {
'name': movie_name,
'year': int(year),
}
movie_name = cleaned.rsplit(year, 1).pop(0).strip()
if movie_name:
cp_guess = {
'name': movie_name,
'year': int(year),
}
except:
pass
else: # Split name on multiple spaces
if not cp_guess: # Split name on multiple spaces
try:
movie_name = cleaned.split(' ').pop(0).strip()
cp_guess = {
'name': movie_name,
'year': int(year),
'year': int(year) if movie_name[:4] != year else 0,
}
except:
pass
couchpotato/core/plugins/score/main.py
+5 -2
View File
@@ -1,11 +1,11 @@
from couchpotato.core.event import addEvent
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.variable import getTitle, splitString
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \
sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \
halfMultipartScore
halfMultipartScore, sceneScore
from couchpotato.environment import Env
log = CPLog(__name__)
@@ -62,4 +62,7 @@ class Score(Plugin):
if extra_score:
score += extra_score(nzb)
# Scene / Nuke scoring
score += sceneScore(nzb['name'])
return score
couchpotato/core/plugins/score/scores.py
+40
View File
@@ -1,8 +1,13 @@
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.environment import Env
import re
import traceback
log = CPLog(__name__)
name_scores = [
# Tags
@@ -160,3 +165,38 @@ def halfMultipartScore(nzb_name):
return -30
return 0
def sceneScore(nzb_name):
check_names = [nzb_name]
# Match names between "
try: check_names.append(re.search(r'([\'"])[^\1]*\1', nzb_name).group(0))
except: pass
# Match longest name between []
try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', nzb_name), key = len).strip())
except: pass
for name in check_names:
# Strip twice, remove possible file extensions
name = name.lower().strip(' "\'\.-_\[\]')
name = re.sub('\.([a-z0-9]{0,4})$', '', name)
name = name.strip(' "\'\.-_\[\]')
# Make sure year and groupname is in there
year = re.findall('(?P<year>19[0-9]{2}|20[0-9]{2})', name)
group = re.findall('\-([a-z0-9]+)$', name)
if len(year) > 0 and len(group) > 0:
try:
validate = fireEvent('release.validate', name, single = True)
if validate and tryInt(validate.get('score')) != 0:
log.debug('Release "%s" scored %s, reason: %s', (nzb_name, validate['score'], validate['reasons']))
return tryInt(validate.get('score'))
except:
log.error('Failed scoring scene: %s', traceback.format_exc())
return 0
couchpotato/core/plugins/subtitle/main.py
+4 -4
View File
@@ -1,6 +1,6 @@
from couchpotato import get_session
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.encoding import toUnicode, sp
from couchpotato.core.helpers.variable import splitString
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
@@ -58,9 +58,9 @@ class Subtitle(Plugin):
for d_sub in downloaded:
log.info('Found subtitle (%s): %s', (d_sub.language.alpha2, files))
group['files']['subtitle'].append(d_sub.path)
group['before_rename'].append(d_sub.path)
group['subtitle_language'][d_sub.path] = [d_sub.language.alpha2]
group['files']['subtitle'].append(sp(d_sub.path))
group['before_rename'].append(sp(d_sub.path))
group['subtitle_language'][sp(d_sub.path)] = [d_sub.language.alpha2]
return True
couchpotato/core/plugins/userscript/static/userscript.css
+6 -6
View File
@@ -14,25 +14,25 @@
padding: 20px;
}
.page.userscript .movie_result {
.page.userscript .media_result {
height: 140px;
}
.page.userscript .movie_result .thumbnail {
.page.userscript .media_result .thumbnail {
width: 90px;
}
.page.userscript .movie_result .options {
.page.userscript .media_result .options {
left: 90px;
padding: 54px 15px;
}
.page.userscript .movie_result .year {
.page.userscript .media_result .year {
display: none;
}
.page.userscript .movie_result .options select[name="title"] {
.page.userscript .media_result .options select[name="title"] {
width: 190px;
}
.page.userscript .movie_result .options select[name="profile"] {
.page.userscript .media_result .options select[name="profile"] {
width: 70px;
}
couchpotato/core/plugins/userscript/static/userscript.js
+1 -1
View File
@@ -34,7 +34,7 @@ Page.Userscript = new Class({
if(json.error)
self.frame.set('html', json.error);
else {
var item = new Block.Search.Item(json.movie);
var item = new Block.Search.MovieItem(json.movie);
self.frame.adopt(item);
item.showOptions();
}