dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved manage scanning

Browse Source 
Expire after db get
This commit is contained in:
Ruud
2014-01-24 22:33:22 +01:00
parent cbd8981ee2
commit 88c328af8e
15 changed files with 117 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
couchpotato/core/helpers/variable.py
View File

@@ -278,4 +278,4 @@ def dictIsSubset(a, b):
def isSubFolder(sub_folder, base_folder):
# Returns True if sub_folder is the same as or inside base_folder
return base_folder and sub_folder and os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep in os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep
return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep)

4
couchpotato/core/logger.py
View File

@@ -38,7 +38,7 @@ class CPLog(object):
def safeMessage(self, msg, replace_tuple = ()):
from couchpotato.environment import Env
from couchpotato.core.helpers.encoding import ss
from couchpotato.core.helpers.encoding import ss, toUnicode
msg = ss(msg)
@@ -67,4 +67,4 @@ class CPLog(object):
except:
pass
return msg
return toUnicode(msg)

10
couchpotato/core/media/_base/media/main.py
View File

@@ -70,8 +70,6 @@ class MediaPlugin(MediaBase):
addEvent('media.restatus', self.restatus)
def refresh(self, id = '', **kwargs):
db = get_session()
handlers = []
ids = splitString(id)
@@ -97,11 +95,14 @@ class MediaPlugin(MediaBase):
default_title = getTitle(media.library)
identifier = media.library.identifier
db.expire_all()
event = 'library.update.%s' % media.type
def handler():
fireEvent('library.update.%s' % media.type, identifier = identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(id))
fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id))
db.expire_all()
if handler:
return handler
def addSingleRefreshView(self):
@@ -467,6 +468,7 @@ class MediaPlugin(MediaBase):
m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id')
db.commit()
db.expire_all()
return True

99
couchpotato/core/media/movie/library/movie/main.py
View File

@@ -60,7 +60,7 @@ class MovieLibraryPlugin(LibraryBase):
db.expire_all()
return library_dict
def update(self, identifier, default_title = '', force = False):
def update(self, identifier, default_title = '', extended = False):
if self.shuttingDown():
return
@@ -69,13 +69,7 @@ class MovieLibraryPlugin(LibraryBase):
library = db.query(Library).filter_by(identifier = identifier).first()
done_status = fireEvent('status.get', 'done', single = True)
library_dict = None
if library:
library_dict = library.to_dict(self.default_dict)
do_update = True
info = fireEvent('movie.info', merge = True, identifier = identifier)
info = fireEvent('movie.info', merge = True, extended = extended, identifier = identifier)
# Don't need those here
try: del info['in_wanted']
@@ -88,55 +82,54 @@ class MovieLibraryPlugin(LibraryBase):
return False
# Main info
if do_update:
library.plot = toUnicode(info.get('plot', ''))
library.tagline = toUnicode(info.get('tagline', ''))
library.year = info.get('year', 0)
library.status_id = done_status.get('id')
library.info.update(info)
db.commit()
library.plot = toUnicode(info.get('plot', ''))
library.tagline = toUnicode(info.get('tagline', ''))
library.year = info.get('year', 0)
library.status_id = done_status.get('id')
library.info.update(info)
db.commit()
# Titles
[db.delete(title) for title in library.titles]
db.commit()
# Titles
[db.delete(title) for title in library.titles]
db.commit()
titles = info.get('titles', [])
log.debug('Adding titles: %s', titles)
counter = 0
for title in titles:
if not title:
titles = info.get('titles', [])
log.debug('Adding titles: %s', titles)
counter = 0
for title in titles:
if not title:
continue
title = toUnicode(title)
t = LibraryTitle(
title = title,
simple_title = self.simplifyTitle(title),
default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title)
)
library.titles.append(t)
counter += 1
db.commit()
# Files
images = info.get('images', [])
for image_type in ['poster']:
for image in images.get(image_type, []):
if not isinstance(image, (str, unicode)):
continue
title = toUnicode(title)
t = LibraryTitle(
title = title,
simple_title = self.simplifyTitle(title),
default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title)
)
library.titles.append(t)
counter += 1
db.commit()
file_path = fireEvent('file.download', url = image, single = True)
if file_path:
file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True)
try:
file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
library.files.append(file_obj)
db.commit()
# Files
images = info.get('images', [])
for image_type in ['poster']:
for image in images.get(image_type, []):
if not isinstance(image, (str, unicode)):
continue
break
except:
log.debug('Failed to attach to library: %s', traceback.format_exc())
file_path = fireEvent('file.download', url = image, single = True)
if file_path:
file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True)
try:
file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
library.files.append(file_obj)
db.commit()
break
except:
log.debug('Failed to attach to library: %s', traceback.format_exc())
library_dict = library.to_dict(self.default_dict)
library_dict = library.to_dict(self.default_dict)
db.expire_all()
return library_dict
@@ -147,14 +140,14 @@ class MovieLibraryPlugin(LibraryBase):
library = db.query(Library).filter_by(identifier = identifier).first()
if not library.info:
library_dict = self.update(identifier, force = True)
library_dict = self.update(identifier)
dates = library_dict.get('info', {}).get('release_date')
else:
dates = library.info.get('release_date')
if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates:
dates = fireEvent('movie.release_date', identifier = identifier, merge = True)
library.info.update({'release_date': dates })
library.info.update({'release_date': dates})
db.commit()
db.expire_all()

35
couchpotato/core/media/movie/searcher/main.py
View File

@@ -73,11 +73,24 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
db = get_session()
movies = db.query(Media).filter(
movies_raw = db.query(Media).filter(
Media.status.has(identifier = 'active')
).all()
random.shuffle(movies)
movies = []
for m in movies_raw:
movies.append(m.to_dict({
'category': {},
'profile': {'types': {'quality': {}}},
'releases': {'status': {}, 'quality': {}},
'library': {'titles': {}, 'files': {}},
'files': {},
}))
db.expire_all()
self.in_progress = {
'total': len(movies),
'to_go': len(movies),
@@ -87,21 +100,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
search_protocols = fireEvent('searcher.protocols', single = True)
for movie in movies:
movie_dict = movie.to_dict({
'category': {},
'profile': {'types': {'quality': {}}},
'releases': {'status': {}, 'quality': {}},
'library': {'titles': {}, 'files': {}},
'files': {},
})
try:
self.single(movie_dict, search_protocols)
self.single(movie, search_protocols)
except IndexError:
log.error('Forcing library update for %s, if you see this often, please report: %s', (movie_dict['library']['identifier'], traceback.format_exc()))
fireEvent('library.update.movie', movie_dict['library']['identifier'], force = True)
log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['library']['identifier'], traceback.format_exc()))
fireEvent('library.update.movie', movie['library']['identifier'])
except:
log.error('Search failed for %s: %s', (movie_dict['library']['identifier'], traceback.format_exc()))
log.error('Search failed for %s: %s', (movie['library']['identifier'], traceback.format_exc()))
self.in_progress['to_go'] -= 1
@@ -133,8 +139,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.')
return
db = get_session()
pre_releases = fireEvent('quality.pre_releases', single = True)
release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True)
available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True)
@@ -150,6 +154,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title)
db = get_session()
ret = False
for quality_type in movie['profile']['types']:
@@ -204,6 +209,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']})
db.expire_all()
return ret
def correctRelease(self, nzb = None, media = None, quality = None, **kwargs):
@@ -336,6 +342,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
for rel in rels:
rel.status_id = ignored_status.get('id')
db.commit()
db.expire_all()
movie_dict = fireEvent('media.get', media_id = media_id, single = True)
log.info('Trying next release for: %s', getTitle(movie_dict['library']))

2
couchpotato/core/media/movie/suggestion/main.py
View File

@@ -33,6 +33,7 @@ class Suggestion(Plugin):
.options(joinedload_all('library')) \
.filter(or_(*[Media.status.has(identifier = s) for s in ['active', 'done']])).all()
movies = [x.library.identifier for x in active_movies]
db.expire_all()
if not ignored or len(ignored) == 0:
ignored = splitString(Env.prop('suggest_ignore', default = ''))
@@ -97,6 +98,7 @@ class Suggestion(Plugin):
.filter(Media.status_id.in_([active_status.get('id'), done_status.get('id')])).all()
movies = [x[0] for x in active_movies]
movies.extend(seen)
db.expire_all()
ignored.extend([x.get('imdb') for x in cached_suggestion])
suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True)

2
couchpotato/core/plugins/category/main.py
View File

@@ -119,3 +119,5 @@ class CategoryPlugin(Plugin):
for movie in movies:
movie.category_id = None
db.commit()
db.expire_all()

1
couchpotato/core/plugins/dashboard/main.py
View File

@@ -119,6 +119,7 @@ class Dashboard(Plugin):
'files': {},
}))
db.expire_all()
return {
'success': True,
'empty': len(movies) == 0,

27
couchpotato/core/plugins/manage/main.py
View File

@@ -1,7 +1,7 @@
from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
from couchpotato.core.helpers.encoding import sp
from couchpotato.core.helpers.variable import splitString, getTitle
from couchpotato.core.helpers.variable import splitString, getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
@@ -59,6 +59,7 @@ class Manage(Plugin):
fireEventAsync('manage.update', full = True if full == '1' else False)
return {
'progress': self.in_progress,
'success': True
}
@@ -84,15 +85,17 @@ class Manage(Plugin):
added_identifiers = []
# Add some progress
self.in_progress = {}
for directory in directories:
self.in_progress[os.path.normpath(directory)] = {
'started': False,
'eta': -1,
'total': None,
'to_go': None,
}
for directory in directories:
folder = os.path.normpath(directory)
self.in_progress[os.path.normpath(directory)]['started'] = tryInt(time.time())
if not os.path.isdir(folder):
if len(directory) > 0:
@@ -102,6 +105,7 @@ class Manage(Plugin):
log.info('Updating manage library: %s', folder)
fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
onFound = self.createAddToLibrary(folder, added_identifiers)
fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True)
@@ -175,10 +179,10 @@ class Manage(Plugin):
def addToLibrary(group, total_found, to_go):
if self.in_progress[folder]['total'] is None:
self.in_progress[folder] = {
self.in_progress[folder].update({
'total': total_found,
'to_go': total_found,
}
})
if group['library'] and group['library'].get('identifier'):
identifier = group['library'].get('identifier')
@@ -186,9 +190,9 @@ class Manage(Plugin):
# Add it to release and update the info
fireEvent('release.add', group = group)
fireEventAsync('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
fireEvent('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
else:
self.in_progress[folder]['to_go'] -= 1
self.updateProgress(folder)
return addToLibrary
@@ -199,7 +203,7 @@ class Manage(Plugin):
if not self.in_progress or self.shuttingDown():
return
self.in_progress[folder]['to_go'] -= 1
self.updateProgress(folder)
total = self.in_progress[folder]['total']
movie_dict = fireEvent('media.get', identifier, single = True)
@@ -207,6 +211,15 @@ class Manage(Plugin):
return afterUpdate
def updateProgress(self, folder):
pr = self.in_progress[folder]
pr['to_go'] -= 1
avg = (time.time() - pr['started'])/(pr['total'] - pr['to_go'])
pr['eta'] = tryInt(avg * pr['to_go'])
def directories(self):
try:
if self.conf('library', default = '').strip():

2
couchpotato/core/plugins/profile/main.py
View File

@@ -46,6 +46,8 @@ class ProfilePlugin(Plugin):
movie.profile_id = default_profile.get('id')
db.commit()
db.expire_all()
def allView(self, **kwargs):
return {

4
couchpotato/core/plugins/release/main.py
View File

@@ -143,6 +143,7 @@ class Release(Plugin):
fireEvent('media.restatus', media.id)
db.expire_all()
return True
def saveFile(self, filepath, type = 'unknown', include_media_info = False):
@@ -338,6 +339,8 @@ class Release(Plugin):
except:
log.error('Failed storing download status: %s', traceback.format_exc())
return False
finally:
db.expire_all()
return True
@@ -466,4 +469,5 @@ class Release(Plugin):
#Update all movie info as there is no release update function
fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict())
db.expire_all()
return True

6
couchpotato/core/plugins/scanner/main.py
View File

@@ -134,6 +134,8 @@ class Scanner(Plugin):
except:
log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
log.debug('Found %s files to scan and group in %s', (len(files), folder))
else:
check_file_date = False
files = [sp(x) for x in files]
@@ -421,6 +423,7 @@ class Scanner(Plugin):
else:
movie = db.query(Media).filter_by(library_id = group['library']['id']).first()
group['movie_id'] = None if not movie else movie.id
db.expire_all()
processed_movies[identifier] = group
@@ -604,6 +607,7 @@ class Scanner(Plugin):
break
except:
pass
db.expire_all()
# Search based on identifiers
if not imdb_id:
@@ -637,7 +641,7 @@ class Scanner(Plugin):
try:
m = re.search(self.cp_imdb, string.lower())
id = m.group('id')
if id: return id
if id: return id
except AttributeError:
pass

2
couchpotato/core/providers/metadata/base.py
View File

@@ -25,7 +25,7 @@ class MetaDataBase(Plugin):
# Update library to get latest info
try:
updated_library = fireEvent('library.update.movie', group['library']['identifier'], force = True, single = True)
updated_library = fireEvent('library.update.movie', group['library']['identifier'], extended = True, single = True)
group['library'] = mergeDicts(group['library'], updated_library)
except:
log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())

2
couchpotato/environment.py
View File

@@ -67,7 +67,7 @@ class Env(object):
@staticmethod
def getEngine():
return create_engine(Env.get('db_path'), echo = False, pool_recycle = 30)
return create_engine(Env.get('db_path'), echo = False)
@staticmethod
def setting(attr, section = 'core', value = None, default = '', type = None):

4
couchpotato/static/scripts/page/manage.js
View File

@@ -119,7 +119,9 @@ Page.Manage = new Class({
sorted_table.each(function(folder){
var folder_progress = progress[folder]
new Element('div').adopt(
new Element('span.folder', {'text': folder}),
new Element('span.folder', {'text': folder +
(folder_progress.eta > 0 ? ', ' + new Date ().increment('second', folder_progress.eta).timeDiffInWords().replace('from now', 'to go') : '')
}),
new Element('span.percentage', {'text': folder_progress.total ? (((folder_progress.total-folder_progress.to_go)/folder_progress.total)*100).round() + '%' : '0%'})
).inject(self.progress_container)
});