nosql

couchpotato/core/plugins/file/main.py

@@ -1,12 +1,12 @@
-from couchpotato import get_session
+from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import md5, getExt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
-from couchpotato.core.settings.model import File
 from couchpotato.environment import Env
+from scandir import scandir
 from tornado.web import StaticFileHandler
 import os.path
 import time
@@ -28,22 +28,30 @@ class FileManager(Plugin):
             'return': {'type': 'file'}
         })
 
+        addEvent('app.load', self.cleanup)
+
     def cleanup(self):
-        # TODO: unused
 
         # Wait a bit after starting before cleanup
-        time.sleep(3)
+        time.sleep(2)
         log.debug('Cleaning up unused files')
 
         try:
-            db = get_session()
-            for root, dirs, walk_files in os.walk(Env.get('cache_dir')):
-                for filename in walk_files:
-                    if os.path.splitext(filename)[1] in ['.png', '.jpg', '.jpeg']:
-                        file_path = os.path.join(root, filename)
-                        f = db.query(File).filter(File.path == toUnicode(file_path)).first()
-                        if not f:
-                            os.remove(file_path)
+            db = get_db()
+            cache_dir = Env.get('cache_dir')
+            medias = db.all('media', with_doc = True)
+
+            files = []
+            for media in medias:
+                file_dict = media['doc'].get('files', {})
+                for x in file_dict.keys():
+                    files.extend(file_dict[x])
+
+            for file in scandir.scandir(cache_dir):
+                if os.path.splitext(file.name)[1] in ['.png', '.jpg', '.jpeg']:
+                    file_path = os.path.join(cache_dir, file.name)
+                    if toUnicode(file_path) not in files:
+                        os.remove(file_path)
         except:
             log.error('Failed removing unused file: %s', traceback.format_exc())
 

couchpotato/core/plugins/manage/main.py

@@ -48,6 +48,7 @@ class Manage(Plugin):
 
         if not Env.get('dev') and self.conf('startup_scan'):
             addEvent('app.load', self.updateLibraryQuick)
+        addEvent('app.load', self.updateLibrary)
 
     def getProgress(self, **kwargs):
         return {
@@ -67,7 +68,7 @@ class Manage(Plugin):
         return self.updateLibrary(full = False)
 
     def updateLibrary(self, full = True):
-        last_update = float(Env.prop('manage.last_update', default = 0))
+        last_update = 0 #float(Env.prop('manage.last_update', default = 0))
 
         if self.in_progress:
             log.info('Already updating library: %s', self.in_progress)
@@ -184,13 +185,12 @@ class Manage(Plugin):
                     'to_go': total_found,
                 })
 
-            if group['media'] and group['media'].get('identifier'):
-                identifier = group['media'].get('identifier')
-                added_identifiers.append(identifier)
+            if group['media'] and group['identifier']:
+                added_identifiers.append(group['identifier'])
 
                 # Add it to release and update the info
                 fireEvent('release.add', group = group)
-                fireEvent('movie.update_info', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier))
+                fireEvent('movie.update_info', identifier = group['identifier'], on_complete = self.createAfterUpdate(folder, group['identifier']))
             else:
                 self.updateProgress(folder)
 

couchpotato/core/plugins/profile/main.py

@@ -1,13 +1,11 @@
 import traceback
-from couchpotato import get_session, get_db, tryInt
+from couchpotato import get_db, tryInt
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.helpers.variable import splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from .index import ProfileIndex
-from couchpotato.core.settings.model import Profile, ProfileType
 
 log = CPLog(__name__)
 
@@ -82,6 +80,7 @@ class ProfilePlugin(Plugin):
             db = get_db()
 
             profile = {
+                '_t': 'profile',
                 'label': toUnicode(kwargs.get('label')),
                 'order': tryInt(kwargs.get('order', 999)),
                 'core': kwargs.get('core', False),
@@ -101,11 +100,11 @@ class ProfilePlugin(Plugin):
             id = kwargs.get('id')
             try:
                 p = db.get('id', id)
+                profile['order'] = tryInt(kwargs.get('order', p.get('order', 999)))
             except:
                 p = db.insert(profile)
-                p.update(profile)
 
-            p['order'] = tryInt(kwargs.get('order', p.get('order', 999)))
+            p.update(profile)
             db.update(p)
 
             return {
@@ -151,15 +150,14 @@ class ProfilePlugin(Plugin):
     def delete(self, id = None, **kwargs):
 
         try:
-            db = get_session()
+            db = get_db()
 
             success = False
             message = ''
-            try:
-                p = db.query(Profile).filter_by(id = id).first()
 
+            try:
+                p = db.get('id', id)
                 db.delete(p)
-                db.commit()
 
                 # Force defaults on all empty profile movies
                 self.forceDefaults()

couchpotato/core/plugins/quality/main.py

@@ -1,5 +1,5 @@
 import traceback
-from couchpotato import get_session, get_db
+from couchpotato import get_db
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode, ss
@@ -7,7 +7,6 @@ from couchpotato.core.helpers.variable import mergeDicts, getExt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.plugins.quality.index import QualityIndex
-from couchpotato.core.settings.model import Quality
 import re
 
 log = CPLog(__name__)
@@ -124,12 +123,12 @@ class QualityPlugin(Plugin):
     def saveSize(self, **kwargs):
 
         try:
-            db = get_session()
-            quality = db.query(Quality).filter_by(identifier = kwargs.get('identifier')).first()
+            db = get_db()
+            quality = db.get('quality', kwargs.get('identifier'), with_doc = True)
 
             if quality:
-                setattr(quality, kwargs.get('value_type'), kwargs.get('value'))
-                db.commit()
+                quality['doc'][kwargs.get('value_type')] = kwargs.get('value')
+                db.update(quality['doc'])
 
             self.cached_qualities = None
 

couchpotato/core/plugins/release/index.py

@@ -9,11 +9,11 @@ class ReleaseIndex(TreeBasedIndex):
         super(ReleaseIndex, self).__init__(*args, **kwargs)
 
     def make_key(self, key):
-        return md5(key).hexdigest()
+        return key
 
     def make_key_value(self, data):
         if data.get('_t') == 'release' and data.get('media_id'):
-            return md5(data['media_id']).hexdigest(), {'media_id': data.get('media_id')}
+            return data['media_id'], None
 
     def run_for_media(self, db, media_id):
         for release in db.get_many('release', media_id, with_doc = True):
@@ -49,6 +49,7 @@ class ReleaseIDIndex(TreeBasedIndex):
         super(ReleaseIDIndex, self).__init__(*args, **kwargs)
 
     def make_key(self, key):
+        print key
         return md5(key).hexdigest()
 
     def make_key_value(self, data):

couchpotato/core/plugins/release/main.py

@@ -12,7 +12,6 @@ from couchpotato.core.settings.model import Release as Relea, Media, \
 from couchpotato.environment import Env
 from inspect import ismethod, isfunction
 from sqlalchemy.exc import InterfaceError
-from sqlalchemy.orm import joinedload_all
 from sqlalchemy.sql.expression import and_, or_
 import os
 import time
@@ -110,44 +109,43 @@ class Release(Plugin):
     def add(self, group):
 
         try:
-            db = get_session()
+            db = get_db()
 
             identifier = '%s.%s.%s' % (group['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier'])
 
-            # Add movie
-            media = db.query(Media).filter_by(library_id = group['library'].get('id')).first()
-            if not media:
-                media = Media(
-                    library_id = group['library'].get('id'),
-                    profile_id = 0,
-                    status = 'done'
-                )
-                db.add(media)
-                db.commit()
+            # Add movie if it doesn't exist
+            try:
+                media = db.get('media', group['identifier'], with_doc = True)['doc']
+                media['status'] = 'done'
+                db.update(media)
+            except:
+                media = {
+                    '_t': 'media',
+                    'identifier': group['identifier'],
+                    'profile_id': None,
+                    'status': 'done'
+                }
+                m = db.insert(media)
+                media.update(m)
 
             # Add Release
-            rel = db.query(Relea).filter(
-                or_(
-                    Relea.identifier == identifier,
-                    and_(Relea.identifier.startswith(group['identifier']), Relea.status == 'snatched')
-                )
-            ).first()
-            if not rel:
-                rel = Relea(
-                    identifier = identifier,
-                    movie = media,
-                    quality_id = group['meta_data']['quality'].get('id'),
-                    status = 'done'
-                )
-                db.add(rel)
-                db.commit()
+            release = {
+                '_t': 'release',
+                'media_id': media['_id'],
+                'identifier': identifier,
+                'quality': group['meta_data']['quality'].get('identifier'),
+                'status': 'done'
+            }
+            try:
+                r = db.get('release_identifier', identifier, with_doc = True)['doc']
+                release.update(r)
+                db.update(release)
+            except:
+                r = db.insert(release)
+                release.update(r)
 
             # Add each file type
-            rel['files'] = []
-            for type in group['files']:
-                for cur_file in group['files'][type]:
-                    added_file = self.saveFile(cur_file, type = type)
-                    rel['files'].append(added_file.get('id'))
+            release['files'] = dict((k, v) for k, v in group['files'].items() if v)
 
             fireEvent('media.restatus', media['_id'])
 
@@ -157,15 +155,6 @@ class Release(Plugin):
 
         return False
 
-    def saveFile(self, filepath, type = 'unknown', include_media_info = False):
-
-        # Check database and update/insert if necessary
-        return {
-            'type': '%s_%s' % Scanner.file_types.get(type),
-            'path': filepath,
-            'part': fireEvent('scanner.partnumber', file, single = True),
-        }
-
     def deleteView(self, id = None, **kwargs):
 
         return {

couchpotato/core/plugins/scanner/main.py

@@ -1,3 +1,4 @@
+from couchpotato import get_db
 from couchpotato.core.event import fireEvent, addEvent
 from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp
 from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \
@@ -6,6 +7,7 @@ from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from enzyme.exceptions import NoParserError, ParseError
 from guessit import guess_movie_info
+from scandir import scandir
 from subliminal.videos import Video
 import enzyme
 import os
@@ -123,7 +125,7 @@ class Scanner(Plugin):
             check_file_date = True
             try:
                 files = []
-                for root, dirs, walk_files in os.walk(folder):
+                for root, dirs, walk_files in scandir.walk(folder):
                     files.extend([sp(os.path.join(root, filename)) for filename in walk_files])
 
                     # Break if CP wants to shut down
@@ -138,7 +140,6 @@ class Scanner(Plugin):
             check_file_date = False
             files = [sp(x) for x in files]
 
-
         for file_path in files:
 
             if not os.path.exists(file_path):
@@ -418,7 +419,7 @@ class Scanner(Plugin):
             if not group['media']:
                 log.error('Unable to determine media: %s', group['identifiers'])
             else:
-                group['identifier'] = group['media']['identifier']
+                group['identifier'] = group['media']['imdb']
 
             processed_movies[identifier] = group
 
@@ -610,9 +611,12 @@ class Scanner(Plugin):
                     log.debug('Identifier to short to use for search: %s', identifier)
 
         if imdb_id:
-            return fireEvent('library.add.movie', attrs = {
-                'identifier': imdb_id
-            }, update_after = False, single = True)
+            try:
+                db = get_db()
+                return db.get('media', imdb_id, with_doc = True)['doc']['info']
+            except:
+                log.debug('Movie "%s" not in library, just getting info', imdb_id)
+                return fireEvent('movie.info', identifier = imdb_id, merge = True, extended = False)
 
         log.error('No imdb_id found for %s. Add a NFO file with IMDB id or add the year to the filename.', group['identifiers'])
         return {}