Improved manage scanning

Expire after db get

couchpotato/core/plugins/category/main.py

@@ -119,3 +119,5 @@ class CategoryPlugin(Plugin):
             for movie in movies:
                 movie.category_id = None
                 db.commit()
+
+        db.expire_all()

couchpotato/core/plugins/dashboard/main.py

@@ -119,6 +119,7 @@ class Dashboard(Plugin):
                         'files': {},
                     }))
 
+        db.expire_all()
         return {
             'success': True,
             'empty': len(movies) == 0,

couchpotato/core/plugins/manage/main.py

@@ -1,7 +1,7 @@
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import sp
-from couchpotato.core.helpers.variable import splitString, getTitle
+from couchpotato.core.helpers.variable import splitString, getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
@@ -59,6 +59,7 @@ class Manage(Plugin):
         fireEventAsync('manage.update', full = True if full == '1' else False)
 
         return {
+            'progress': self.in_progress,
             'success': True
         }
 
@@ -84,15 +85,17 @@ class Manage(Plugin):
             added_identifiers = []
 
             # Add some progress
-            self.in_progress = {}
             for directory in directories:
                 self.in_progress[os.path.normpath(directory)] = {
+                    'started': False,
+                    'eta': -1,
                     'total': None,
                     'to_go': None,
                 }
 
             for directory in directories:
                 folder = os.path.normpath(directory)
+                self.in_progress[os.path.normpath(directory)]['started'] = tryInt(time.time())
 
                 if not os.path.isdir(folder):
                     if len(directory) > 0:
@@ -102,6 +105,7 @@ class Manage(Plugin):
                 log.info('Updating manage library: %s', folder)
                 fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder)
 
+
                 onFound = self.createAddToLibrary(folder, added_identifiers)
                 fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True)
 
@@ -175,10 +179,10 @@ class Manage(Plugin):
 
         def addToLibrary(group, total_found, to_go):
             if self.in_progress[folder]['total'] is None:
-                self.in_progress[folder] = {
+                self.in_progress[folder].update({
                     'total': total_found,
                     'to_go': total_found,
-                }
+                })
 
             if group['library'] and group['library'].get('identifier'):
                 identifier = group['library'].get('identifier')
@@ -186,9 +190,9 @@ class Manage(Plugin):
 
                 # Add it to release and update the info
                 fireEvent('release.add', group = group)
-                fireEventAsync('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
+                fireEvent('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
             else:
-                self.in_progress[folder]['to_go'] -= 1
+                self.updateProgress(folder)
 
         return addToLibrary
 
@@ -199,7 +203,7 @@ class Manage(Plugin):
             if not self.in_progress or self.shuttingDown():
                 return
 
-            self.in_progress[folder]['to_go'] -= 1
+            self.updateProgress(folder)
             total = self.in_progress[folder]['total']
             movie_dict = fireEvent('media.get', identifier, single = True)
 
@@ -207,6 +211,15 @@ class Manage(Plugin):
 
         return afterUpdate
 
+    def updateProgress(self, folder):
+
+        pr = self.in_progress[folder]
+        pr['to_go'] -= 1
+
+        avg = (time.time() - pr['started'])/(pr['total'] - pr['to_go'])
+        pr['eta'] = tryInt(avg * pr['to_go'])
+
+
     def directories(self):
         try:
             if self.conf('library', default = '').strip():

couchpotato/core/plugins/profile/main.py

@@ -46,6 +46,8 @@ class ProfilePlugin(Plugin):
                 movie.profile_id = default_profile.get('id')
                 db.commit()
 
+        db.expire_all()
+
     def allView(self, **kwargs):
 
         return {

couchpotato/core/plugins/release/main.py

@@ -143,6 +143,7 @@ class Release(Plugin):
 
         fireEvent('media.restatus', media.id)
 
+        db.expire_all()
         return True
 
     def saveFile(self, filepath, type = 'unknown', include_media_info = False):
@@ -338,6 +339,8 @@ class Release(Plugin):
         except:
             log.error('Failed storing download status: %s', traceback.format_exc())
             return False
+        finally:
+            db.expire_all()
 
         return True
 
@@ -466,4 +469,5 @@ class Release(Plugin):
             #Update all movie info as there is no release update function
             fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict())
 
+        db.expire_all()
         return True

couchpotato/core/plugins/scanner/main.py

@@ -134,6 +134,8 @@ class Scanner(Plugin):
 
             except:
                 log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
+
+            log.debug('Found %s files to scan and group in %s', (len(files), folder))
         else:
             check_file_date = False
             files = [sp(x) for x in files]
@@ -421,6 +423,7 @@ class Scanner(Plugin):
             else:
                 movie = db.query(Media).filter_by(library_id = group['library']['id']).first()
                 group['movie_id'] = None if not movie else movie.id
+                db.expire_all()
 
             processed_movies[identifier] = group
 
@@ -604,6 +607,7 @@ class Scanner(Plugin):
                     break
                 except:
                     pass
+            db.expire_all()
 
         # Search based on identifiers
         if not imdb_id:
@@ -637,7 +641,7 @@ class Scanner(Plugin):
         try:
             m = re.search(self.cp_imdb, string.lower())
             id = m.group('id')
-            if id:  return id
+            if id: return id
         except AttributeError:
             pass