dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Try NoSQL

Browse Source
This commit is contained in:
Ruud
2014-01-29 17:49:54 +01:00
parent aab10fb599
commit a254886bad
65 changed files with 7372 additions and 761 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/plugins/automation/main.py
+2 -2
View File
@@ -10,10 +10,10 @@ class Automation(Plugin):
def __init__(self):
addEvent('app.load', self.setCrons)
addEvent('app.load2', self.setCrons)
if not Env.get('dev'):
addEvent('app.load', self.addMovies)
addEvent('app.load2', self.addMovies)
addEvent('setting.save.automation.hour.after', self.setCrons)
couchpotato/core/plugins/category/main.py
+8 -8
View File
@@ -42,7 +42,7 @@ class CategoryPlugin(Plugin):
for category in categories:
temp.append(category.to_dict())
db.close()
pass #db.close()
return temp
def save(self, **kwargs):
@@ -66,7 +66,7 @@ class CategoryPlugin(Plugin):
category_dict = c.to_dict()
db.close()
pass #db.close()
return {
'success': True,
'category': category_dict
@@ -75,7 +75,7 @@ class CategoryPlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False,
@@ -96,7 +96,7 @@ class CategoryPlugin(Plugin):
db.commit()
db.close()
pass #db.close()
return {
'success': True
}
@@ -104,7 +104,7 @@ class CategoryPlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False
@@ -129,7 +129,7 @@ class CategoryPlugin(Plugin):
except Exception as e:
message = log.error('Failed deleting category: %s', e)
db.close()
pass #db.close()
return {
'success': success,
'message': message
@@ -138,7 +138,7 @@ class CategoryPlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False
@@ -158,4 +158,4 @@ class CategoryPlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
couchpotato/core/plugins/custom/main.py
+1 -1
View File
@@ -10,7 +10,7 @@ log = CPLog(__name__)
class Custom(Plugin):
def __init__(self):
addEvent('app.load', self.createStructure)
addEvent('app.load2', self.createStructure)
def createStructure(self):
couchpotato/core/plugins/dashboard/main.py
+1 -1
View File
@@ -119,7 +119,7 @@ class Dashboard(Plugin):
'files': {},
}))
db.close()
pass #db.close()
return {
'success': True,
'empty': len(movies) == 0,
couchpotato/core/plugins/file/main.py
+1 -107
View File
@@ -19,9 +19,7 @@ log = CPLog(__name__)
class FileManager(Plugin):
def __init__(self):
addEvent('file.add', self.add)
addEvent('file.download', self.download)
addEvent('file.types', self.getTypes)
addApiView('file.cache/(.*)', self.showCacheFile, static = True, docs = {
'desc': 'Return a file from the cp_data/cache directory',
@@ -31,36 +29,8 @@ class FileManager(Plugin):
'return': {'type': 'file'}
})
addApiView('file.types', self.getTypesView, docs = {
'desc': 'Return a list of all the file types and their ids.',
'return': {'type': 'object', 'example': """{
'types': [
{
"identifier": "poster_original",
"type": "image",
"id": 1,
"name": "Poster_original"
},
{
"identifier": "poster",
"type": "image",
"id": 2,
"name": "Poster"
},
etc
]
}"""}
})
addEvent('app.load', self.cleanup)
addEvent('app.load', self.init)
def init(self):
for type_tuple in Scanner.file_types.values():
self.getType(type_tuple)
def cleanup(self):
# TODO: unused
# Wait a bit after starting before cleanup
time.sleep(3)
@@ -77,9 +47,6 @@ class FileManager(Plugin):
os.remove(file_path)
except:
log.error('Failed removing unused file: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
def showCacheFile(self, route, **kwargs):
Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})])
@@ -101,76 +68,3 @@ class FileManager(Plugin):
self.createFile(dest, filedata, binary = True)
return dest
def add(self, path = '', part = 1, type_tuple = (), available = 1, properties = None):
if not properties: properties = {}
try:
db = get_session()
type_id = self.getType(type_tuple).get('id')
f = db.query(File).filter(File.path == toUnicode(path)).first()
if not f:
f = File()
db.add(f)
f.path = toUnicode(path)
f.part = part
f.available = available
f.type_id = type_id
db.commit()
file_dict = f.to_dict()
return file_dict
except:
log.error('Failed adding file: %s, %s', (path, traceback.format_exc()))
db.rollback()
finally:
db.close()
def getType(self, type_tuple):
try:
db = get_session()
type_type, type_identifier = type_tuple
ft = db.query(FileType).filter_by(identifier = type_identifier).first()
if not ft:
ft = FileType(
type = toUnicode(type_type),
identifier = type_identifier,
name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:])
)
db.add(ft)
db.commit()
type_dict = ft.to_dict()
return type_dict
except:
log.error('Failed getting type: %s, %s', (type_tuple, traceback.format_exc()))
db.rollback()
finally:
db.close()
def getTypes(self):
db = get_session()
results = db.query(FileType).all()
types = []
for type_object in results:
types.append(type_object.to_dict())
db.close()
return types
def getTypesView(self, **kwargs):
return {
'types': self.getTypes()
}
couchpotato/core/plugins/manage/main.py
+1 -1
View File
@@ -47,7 +47,7 @@ class Manage(Plugin):
})
if not Env.get('dev') and self.conf('startup_scan'):
addEvent('app.load', self.updateLibraryQuick)
addEvent('app.load2', self.updateLibraryQuick)
def getProgress(self, **kwargs):
return {
couchpotato/core/plugins/nosql/__init__.py
+7
View File
@@ -0,0 +1,7 @@
from .main import NoSQL
def start():
return NoSQL()
config = []
couchpotato/core/plugins/nosql/index.py
+20
View File
@@ -0,0 +1,20 @@
from hashlib import md5
from CodernityDB.tree_index import TreeBasedIndex
class ReleaseIndex(TreeBasedIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '16s'
super(ReleaseIndex, self).__init__(*args, **kwargs)
def make_key(self, key):
return md5(key).digest()
def make_key_value(self, data):
if data.get('type') == 'release':
return md5(data['media_id']).digest(), None
def run_for_media(self, db, media_id):
for release in db.get_many('release', media_id, with_doc=True):
yield release['doc']
couchpotato/core/plugins/nosql/main.py
+72
View File
@@ -0,0 +1,72 @@
import time
from couchpotato import CPLog
from couchpotato.core.plugins.base import Plugin
from .index import ReleaseIndex, MediaIMDBIndex, TitleIndex
log = CPLog(__name__)
class NoSQL(Plugin):
db = None
def test(self):
db = self.db
try: db.add_index(ReleaseIndex(db.path, 'release'))
except: log.debug('Index already exists')
for id in range(10):
media = db.insert({
'type': 'media',
'tmdb': id,
'imdb': 'tt%s' % id,
'last_edit': 0,
'status': 'active',
'title': 'Lord of the Rings: The Return of the King',
'year': 2011,
'profile_id': 0,
'category_id': 0,
})
db.insert({
'media_id': media['_id'],
'type': 'title',
'title': 'Lord of the Rings: The Return of the King',
})
for x in range(40):
db.insert({
'media_id': media['_id'],
'type': 'release',
'name': 'Release %s' % x
})
print db.count(db.all, 'media')
m = db.get('media', 'tt0')
db.get('id', m['_id'])
start = time.time()
print list(db.get_many('media_title', 'lord of'))
print time.time() - start
return
return
for media in db.all('media', with_doc = True):
doc = media['doc']
for r in db.run('release', 'for_media', media['_id']):
db.delete(r)
db.delete(doc)
break
start = time.time()
db.reindex()
print time.time() - start
print db.count(db.all, 'media')
print db.count(db.all, 'release')
couchpotato/core/plugins/profile/main.py
+19 -29
View File
@@ -1,5 +1,5 @@
import traceback
from couchpotato import get_session
from couchpotato import get_session, get_db
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import toUnicode
@@ -31,27 +31,24 @@ class ProfilePlugin(Plugin):
})
addEvent('app.initialize', self.fill, priority = 90)
addEvent('app.load', self.forceDefaults)
addEvent('app.load2', self.forceDefaults)
def forceDefaults(self):
# Get all active movies without profile
active_status = fireEvent('status.get', 'active', single = True)
try:
db = get_session()
movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all()
db = get_db()
medias = db.run('media', 'with_status', ['active'])
if len(movies) > 0:
default_profile = self.default()
for movie in movies:
movie.profile_id = default_profile.get('id')
db.commit()
profile_ids = [x.get('_id') for x in self.all()]
for media in medias:
if media['profile_id'] not in profile_ids:
default_profile = self.default()
media['profile_id'] = default_profile.get('id')
db.update(media)
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
def allView(self, **kwargs):
@@ -62,17 +59,10 @@ class ProfilePlugin(Plugin):
def all(self):
db = get_session()
profiles = db.query(Profile) \
.options(joinedload_all('types')) \
.all()
db = get_db()
profiles = db.all('profile', with_doc = True)
temp = []
for profile in profiles:
temp.append(profile.to_dict(self.to_dict))
db.close()
return temp
return list(profiles)
def save(self, **kwargs):
@@ -115,7 +105,7 @@ class ProfilePlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False
@@ -128,7 +118,7 @@ class ProfilePlugin(Plugin):
.options(joinedload_all('types')) \
.first()
default_dict = default.to_dict(self.to_dict)
db.close()
pass #db.close()
return default_dict
@@ -154,7 +144,7 @@ class ProfilePlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False
@@ -188,7 +178,7 @@ class ProfilePlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False
@@ -243,6 +233,6 @@ class ProfilePlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return False
couchpotato/core/plugins/quality/main.py
+4 -4
View File
@@ -79,7 +79,7 @@ class QualityPlugin(Plugin):
self.cached_qualities = temp
db.close()
pass #db.close()
return temp
def single(self, identifier = ''):
@@ -91,7 +91,7 @@ class QualityPlugin(Plugin):
if quality:
quality_dict = dict(self.getQuality(quality.identifier), **quality.to_dict())
db.close()
pass #db.close()
return quality_dict
def getQuality(self, identifier):
@@ -119,7 +119,7 @@ class QualityPlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return {
'success': False
@@ -181,7 +181,7 @@ class QualityPlugin(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return False
couchpotato/core/plugins/release/main.py
+58 -75
View File
@@ -1,4 +1,4 @@
from couchpotato import get_session, md5
from couchpotato import get_session, md5, get_db
from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import ss, toUnicode
@@ -72,34 +72,32 @@ class Release(Plugin):
done_status, available_status, snatched_status, downloaded_status, ignored_status = \
fireEvent('status.get', ['done', 'available', 'snatched', 'downloaded', 'ignored'], single = True)
db = get_session()
db = get_db()
# get movies last_edit more than a week ago
media = db.query(Media) \
.filter(Media.status_id == done_status.get('id'), Media.last_edit < (now - week)) \
.all()
medias = db.run('media', 'with_status', ['done'])
for item in media:
for rel in item.releases:
for media in medias:
if media['last_edit'] > (now - week):
continue
for rel in db.run('release', 'for_media', media['_id']):
# Remove all available releases
if rel.status_id in [available_status.get('id')]:
fireEvent('release.delete', id = rel.id, single = True)
# Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move
elif rel.status_id in [snatched_status.get('id'), downloaded_status.get('id')]:
self.updateStatus(id = rel.id, status = ignored_status)
if rel['status'] in ['available']:
fireEvent('release.delete', id = rel['_id'], single = True)
db.close()
# Set all snatched and downloaded releases to ignored to make sure they are ignored when re-adding the move
elif rel['status'] in ['snatched', 'downloaded']:
self.updateStatus(id = rel['id'], status = ignored_status)
def add(self, group):
try:
db = get_session()
db = get_db()
identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier'])
done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True)
# Add movie
media = db.query(Media).filter_by(library_id = group['library'].get('id')).first()
if not media:
@@ -129,41 +127,28 @@ class Release(Plugin):
db.commit()
# Add each file type
added_files = []
rel['files'] = []
for type in group['files']:
for cur_file in group['files'][type]:
added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie')
added_files.append(added_file.get('id'))
added_file = self.saveFile(cur_file, type = type)
rel['files'].append(added_file.get('id'))
# Add the release files in batch
try:
added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all()
rel.files.extend(added_files)
db.commit()
except:
log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc()))
fireEvent('media.restatus', media.id)
fireEvent('media.restatus', media['_id'])
return True
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
return False
def saveFile(self, filepath, type = 'unknown', include_media_info = False):
properties = {}
# Get media info for files
if include_media_info:
properties = {}
# Check database and update/insert if necessary
return fireEvent('file.add', path = filepath, part = fireEvent('scanner.partnumber', file, single = True), type_tuple = Scanner.file_types.get(type), properties = properties, single = True)
return {
'type': '%s_%s' % Scanner.file_types.get(type),
'path': filepath,
'part': fireEvent('scanner.partnumber', file, single = True),
}
def deleteView(self, id = None, **kwargs):
@@ -171,60 +156,58 @@ class Release(Plugin):
'success': self.delete(id)
}
def delete(self, id):
def delete(self, release_id):
try:
db = get_session()
db = get_db()
rel = db.query(Relea).filter_by(id = id).first()
if rel:
db.delete(rel)
db.commit()
return True
rel = db.get('release', release_id, with_doc = True)
db.delete(rel['doc'])
return True
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
return False
def clean(self, id):
def clean(self, release_id):
try:
db = get_session()
db = get_db()
rel = db.query(Relea).filter_by(id = id).first()
if rel:
for release_file in rel.files:
if not os.path.isfile(ss(release_file.path)):
db.delete(release_file)
db.commit()
rel = db.get('release', release_id, with_doc = True)
files = []
for release_file in rel['files']:
if os.path.isfile(ss(release_file['path'])):
files.append(release_file)
if len(rel.files) == 0:
self.delete(id)
if len(rel['files']) == 0:
self.delete(rel['_id'])
else:
rel['files'] = files
db.update(rel)
return True
return True
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
return False
def ignore(self, id = None, **kwargs):
def ignore(self, release_id = None, **kwargs):
db = get_session()
db = get_db()
rel = db.query(Relea).filter_by(id = id).first()
if rel:
ignored_status, failed_status, available_status = fireEvent('status.get', ['ignored', 'failed', 'available'], single = True)
self.updateStatus(id, available_status if rel.status_id in [ignored_status.get('id'), failed_status.get('id')] else ignored_status)
try:
rel = db.get('release', release_id, with_doc = True)['doc']
self.updateStatus(release_id, 'available' if rel['status'] in ['ignored', 'failed'] else 'ignored')
return {
'success': True
}
except:
log.error('Failed: %s', traceback.format_exc())
db.close()
return {
'success': True
'success': False
}
def manualDownload(self, id = None, **kwargs):
@@ -265,7 +248,7 @@ class Release(Plugin):
if success:
fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
db.close()
pass #db.close()
return {
'success': success == True
}
@@ -361,7 +344,7 @@ class Release(Plugin):
db.rollback()
return False
finally:
db.close()
pass #db.close()
return True
@@ -441,7 +424,7 @@ class Release(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return []
@@ -458,7 +441,7 @@ class Release(Plugin):
releases = [r.to_dict({'info': {}, 'files': {}}) for r in releases_raw]
releases = sorted(releases, key = lambda k: k['info'].get('score', 0), reverse = True)
db.close()
pass #db.close()
return releases
def forMovieView(self, id = None, **kwargs):
@@ -506,6 +489,6 @@ class Release(Plugin):
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
pass #db.close()
return False
couchpotato/core/plugins/renamer/main.py
+3 -3
View File
@@ -598,7 +598,7 @@ class Renamer(Plugin):
break
self.renaming_started = False
db.close()
pass #db.close()
def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False):
if not group: group = {}
@@ -1045,7 +1045,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
self.checking_snatched = False
return True
finally:
db.close()
pass #db.close()
def extendReleaseDownload(self, release_download):
@@ -1068,7 +1068,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
if not rls:
log.error('Download ID %s from downloader %s not found in releases', (release_download.get('id'), release_download.get('downloader')))
db.close()
pass #db.close()
if rls:
couchpotato/core/plugins/scanner/main.py
+2 -2
View File
@@ -441,7 +441,7 @@ class Scanner(Plugin):
else:
log.debug('Found no movies in the folder %s', folder)
db.close()
pass #db.close()
return processed_movies
def getMetaData(self, group, folder = '', release_download = None):
@@ -609,7 +609,7 @@ class Scanner(Plugin):
break
except:
pass
db.close()
pass #db.close()
# Search based on identifiers
if not imdb_id:
couchpotato/core/plugins/status/__init__.py
-7
View File
@@ -1,7 +0,0 @@
from .main import StatusPlugin
def start():
return StatusPlugin()
config = []
couchpotato/core/plugins/status/main.py
-137
View File
@@ -1,137 +0,0 @@
import traceback
from couchpotato import get_session
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Status
log = CPLog(__name__)
class StatusPlugin(Plugin):
statuses = {
'needs_update': 'Needs update',
'active': 'Active',
'done': 'Done',
'downloaded': 'Downloaded',
'wanted': 'Wanted',
'snatched': 'Snatched',
'failed': 'Failed',
'deleted': 'Deleted',
'ignored': 'Ignored',
'available': 'Available',
'suggest': 'Suggest',
'seeding': 'Seeding',
'missing': 'Missing',
}
status_cached = {}
def __init__(self):
addEvent('status.get', self.get)
addEvent('status.get_by_id', self.getById)
addEvent('status.all', self.all)
addEvent('app.initialize', self.fill)
addEvent('app.load', self.all) # Cache all statuses
addApiView('status.list', self.list, docs = {
'desc': 'Check for available update',
'return': {'type': 'object', 'example': """{
'success': True,
'list': array, statuses
}"""}
})
def list(self, **kwargs):
return {
'success': True,
'list': self.all()
}
def getById(self, id):
db = get_session()
status = db.query(Status).filter_by(id = id).first()
status_dict = status.to_dict()
db.close()
return status_dict
def all(self):
db = get_session()
statuses = db.query(Status).all()
temp = []
for status in statuses:
s = status.to_dict()
temp.append(s)
# Update cache
self.status_cached[status.identifier] = s
db.close()
return temp
def get(self, identifiers):
if not isinstance(identifiers, list):
identifiers = [identifiers]
try:
db = get_session()
return_list = []
for identifier in identifiers:
if self.status_cached.get(identifier):
return_list.append(self.status_cached.get(identifier))
continue
s = db.query(Status).filter_by(identifier = identifier).first()
if not s:
s = Status(
identifier = identifier,
label = toUnicode(identifier.capitalize())
)
db.add(s)
db.commit()
status_dict = s.to_dict()
self.status_cached[identifier] = status_dict
return_list.append(status_dict)
return return_list if len(identifiers) > 1 else return_list[0]
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
def fill(self):
try:
db = get_session()
for identifier, label in self.statuses.items():
s = db.query(Status).filter_by(identifier = identifier).first()
if not s:
log.info('Creating status: %s', label)
s = Status(
identifier = identifier,
label = toUnicode(label)
)
db.add(s)
s.label = toUnicode(label)
db.commit()
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
couchpotato/core/plugins/status/static/status.js
-17
View File
@@ -1,17 +0,0 @@
var StatusBase = new Class({
setup: function(statuses){
var self = this;
self.statuses = statuses;
},
get: function(id){
return this.statuses.filter(function(status){
return status.id == id
}).pick()
},
});
window.Status = new StatusBase();
couchpotato/core/plugins/subtitle/main.py
+1 -1
View File
@@ -41,7 +41,7 @@ class Subtitle(Plugin):
# get subtitles for those files
subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services)
db.close()
pass #db.close()
def searchSingle(self, group):
if self.isDisabled(): return