database manage init

This commit is contained in:
Ruud
2014-02-11 22:19:55 +01:00
parent 8724076601
commit 96e8a909d8
26 changed files with 382 additions and 183 deletions
+8
View File
@@ -75,8 +75,16 @@ def apiDocs():
addView('docs', apiDocs)
# Database debug manager
def databaseManage():
return template_loader.load('database.html').generate(fireEvent = fireEvent, Env = Env)
addView('database', databaseManage)
# Make non basic auth option to get api key
class KeyHandler(RequestHandler):
def get(self, *args, **kwargs):
api_key = None
+3
View File
@@ -93,6 +93,7 @@ class ApiHandler(RequestHandler):
# Split array arguments
kwargs = getParams(kwargs)
kwargs['_request'] = self
# Remove t random string
try: del kwargs['t']
@@ -127,6 +128,8 @@ class ApiHandler(RequestHandler):
api_locks[route].release()
post = get
def addApiView(route, func, static = False, docs = None, **kwargs):
+109
View File
@@ -0,0 +1,109 @@
import json
import time
import traceback
from couchpotato import CPLog
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent
log = CPLog(__name__)
class Database(object):
indexes = []
db = None
def __init__(self):
addApiView('database.list_documents', self.listDocuments)
addApiView('database.document.update', self.updateDocument)
addApiView('database.document.delete', self.deleteDocument)
addEvent('database.setup_index', self.setupIndex)
def test():
time.sleep(1)
self.listDocuments()
addEvent('app.load', test)
def getDB(self):
if not self.db:
from couchpotato import get_db
self.db = get_db()
return self.db
def setupIndex(self, index_name, klass):
self.indexes.append(index_name)
db = self.getDB()
# Category index
try:
db.add_index(klass(db.path, index_name))
db.reindex_index(index_name)
except:
previous_version = db.indexes_names[index_name]._version
current_version = klass._version
# Only edit index if versions are different
if previous_version < current_version:
log.debug('Index "%s" already exists, updating and reindexing', index_name)
db.edit_index(klass(db.path, index_name), reindex = True)
def deleteDocument(self, **kwargs):
db = self.getDB()
try:
document = db.get(id)
db.delete(document)
return {
'success': True
}
except:
return {
'success': False,
'error': traceback.format_exc()
}
def updateDocument(self, **kwargs):
db = self.getDB()
try:
document = json.loads(kwargs.get('_request').get_argument('document'))
d = db.update(document)
document.update(d)
return {
'success': True,
'document': document
}
except:
return {
'success': False,
'error': traceback.format_exc()
}
def listDocuments(self, **kwargs):
db = self.getDB()
results = {
'unknown': []
}
for document in db.all('id'):
key = document.get('_t', 'unknown')
if not results.get(key):
results[key] = []
results[key].append(document)
return results
-9
View File
@@ -10,15 +10,6 @@ class MediaBase(Plugin):
_type = None
default_dict = {
'profile': {'types': {'quality': {}}},
'releases': {'status': {}, 'quality': {}, 'files': {}, 'info': {}},
'library': {'titles': {}, 'files': {}},
'files': {},
'status': {},
'category': {},
}
def initType(self):
addEvent('media.types', self.getType)
+8 -2
View File
@@ -7,6 +7,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString
class MediaIMDBIndex(HashIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = 'I'
@@ -19,8 +20,8 @@ class MediaIMDBIndex(HashIndex):
if data.get('_t') == 'media' and data.get('identifier'):
return int(data['identifier'].strip('t')), None
def run_to_dict(self, db, media_id, dict = None):
if not dict: dict = {}
def run_to_dict(self, db, media_id, dict_dept = None):
if not dict_dept: dict_dept = {}
return db.get('id', media_id)
@@ -34,6 +35,7 @@ class MediaIMDBIndex(HashIndex):
class MediaStatusIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
@@ -48,6 +50,7 @@ class MediaStatusIndex(TreeBasedIndex):
class MediaTypeIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
@@ -62,6 +65,7 @@ class MediaTypeIndex(TreeBasedIndex):
class TitleSearchIndex(MultiTreeBasedIndex):
_version = 1
custom_header = """from CodernityDB.tree_index import MultiTreeBasedIndex
from itertools import izip
@@ -93,6 +97,7 @@ from couchpotato.core.helpers.encoding import simplifyString"""
class TitleIndex(TreeBasedIndex):
_version = 1
custom_header = """from CodernityDB.tree_index import TreeBasedIndex
from string import ascii_letters
@@ -125,6 +130,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
class StartsWithIndex(TreeBasedIndex):
_version = 1
custom_header = """from CodernityDB.tree_index import TreeBasedIndex
from string import ascii_letters
+9 -47
View File
@@ -14,6 +14,15 @@ log = CPLog(__name__)
class MediaPlugin(MediaBase):
_database = {
'media': MediaIMDBIndex,
'media_search_title': MediaStatusIndex,
'media_status': MediaTypeIndex,
'media_by_type': TitleSearchIndex,
'media_title': TitleIndex,
'media_startswith': StartsWithIndex,
}
def __init__(self):
addApiView('media.refresh', self.refresh, docs = {
@@ -57,8 +66,6 @@ class MediaPlugin(MediaBase):
addApiView('media.available_chars', self.charView)
addEvent('database.setup', self.databaseSetup)
addEvent('app.load', self.addSingleRefreshView, priority = 100)
addEvent('app.load', self.addSingleListView, priority = 100)
addEvent('app.load', self.addSingleCharView, priority = 100)
@@ -69,51 +76,6 @@ class MediaPlugin(MediaBase):
addEvent('media.delete', self.delete)
addEvent('media.restatus', self.restatus)
def databaseSetup(self):
db = get_db()
# IMDB index
try:
db.add_index(MediaIMDBIndex(db.path, 'media'))
except:
log.debug('Index already exists')
db.edit_index(MediaIMDBIndex(db.path, 'media'))
# Title index
try:
db.add_index(TitleSearchIndex(db.path, 'media_search_title'))
except:
log.debug('Index already exists')
db.edit_index(TitleSearchIndex(db.path, 'media_search_title'))
# Status index
try:
db.add_index(MediaStatusIndex(db.path, 'media_status'))
except:
log.debug('Index already exists')
db.edit_index(MediaStatusIndex(db.path, 'media_status'))
# Type index
try:
db.add_index(MediaTypeIndex(db.path, 'media_by_type'))
except:
log.debug('Index already exists')
db.edit_index(MediaTypeIndex(db.path, 'media_by_type'))
# Title index
try: db.add_index(TitleIndex(db.path, 'media_title'))
except:
log.debug('Index already exists')
db.edit_index(TitleIndex(db.path, 'media_title'))
# Startswith index
try: db.add_index(StartsWithIndex(db.path, 'media_startswith'))
except:
log.debug('Index already exists')
db.edit_index(StartsWithIndex(db.path, 'media_startswith'))
def refresh(self, id = '', **kwargs):
handlers = []
ids = splitString(id)
@@ -3,6 +3,7 @@ from CodernityDB.tree_index import TreeBasedIndex
class NotificationIndex(TreeBasedIndex):
_version = 1
custom_header = """from CodernityDB.tree_index import TreeBasedIndex
import time"""
@@ -23,6 +24,7 @@ import time"""
class NotificationUnreadIndex(TreeBasedIndex):
_version = 1
custom_header = """from CodernityDB.tree_index import TreeBasedIndex
import time"""
+5 -18
View File
@@ -18,6 +18,11 @@ log = CPLog(__name__)
class CoreNotifier(Notification):
_database = {
'notification': NotificationIndex,
'notification_unread': NotificationUnreadIndex
}
m_lock = None
listen_to = [
@@ -60,28 +65,10 @@ class CoreNotifier(Notification):
addEvent('app.load', self.clean)
addEvent('app.load', self.checkMessages)
addEvent('database.setup', self.databaseSetup)
self.messages = []
self.listeners = []
self.m_lock = threading.Lock()
def databaseSetup(self):
db = get_db()
try:
db.add_index(NotificationIndex(db.path, 'notification'))
except:
log.debug('Index already exists')
db.edit_index(NotificationIndex(db.path, 'notification'))
try:
db.add_index(NotificationUnreadIndex(db.path, 'notification_unread'))
except:
log.debug('Index already exists')
db.edit_index(NotificationUnreadIndex(db.path, 'notification_unread'))
def clean(self):
try:
db = get_db()
+18
View File
@@ -1,3 +1,4 @@
from couchpotato import get_db
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import ss, toSafeString, \
toUnicode, sp
@@ -24,6 +25,7 @@ log = CPLog(__name__)
class Plugin(object):
_class_name = None
_database = None
plugin_path = None
enabled_option = 'enabled'
@@ -53,6 +55,22 @@ class Plugin(object):
if self.auto_register_static:
self.registerStatic(inspect.getfile(self.__class__))
# Setup database
if self._database:
addEvent('database.setup', self.databaseSetup)
def databaseSetup(self):
db = get_db()
for index_name in self._database:
klass = self._database[index_name]
fireEvent('database.setup_index', index_name, klass)
def afterDatabaseSetup(self):
print self._database_indexes
def conf(self, attr, value = None, default = None, section = None):
class_name = self.getName().lower().split(':')[0].lower()
return Env.setting(attr, section = section if section else class_name, value = value, default = default)
@@ -2,6 +2,7 @@ from CodernityDB.tree_index import TreeBasedIndex
class CategoryIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = 'i'
@@ -16,6 +17,7 @@ class CategoryIndex(TreeBasedIndex):
class CategoryMediaIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
+5 -19
View File
@@ -12,6 +12,11 @@ log = CPLog(__name__)
class CategoryPlugin(Plugin):
_database = {
'category': CategoryIndex,
'category_media': CategoryMediaIndex,
}
def __init__(self):
addApiView('category.save', self.save)
addApiView('category.save_order', self.saveOrder)
@@ -25,25 +30,6 @@ class CategoryPlugin(Plugin):
})
addEvent('category.all', self.all)
addEvent('database.setup', self.databaseSetup)
def databaseSetup(self):
db = get_db()
# Category index
try:
db.add_index(CategoryIndex(db.path, 'category'))
except:
log.debug('Index already exists')
db.edit_index(CategoryIndex(db.path, 'category'))
# Category media_id index
try:
db.add_index(CategoryMediaIndex(db.path, 'category_media'))
except:
log.debug('Index already exists')
db.edit_index(CategoryMediaIndex(db.path, 'category_media'))
def allView(self, **kwargs):
+3 -3
View File
@@ -47,9 +47,9 @@ class FileManager(Plugin):
for x in file_dict.keys():
files.extend(file_dict[x])
for file in scandir.scandir(cache_dir):
if os.path.splitext(file.name)[1] in ['.png', '.jpg', '.jpeg']:
file_path = os.path.join(cache_dir, file.name)
for f in scandir.scandir(cache_dir):
if os.path.splitext(f.name)[1] in ['.png', '.jpg', '.jpeg']:
file_path = os.path.join(cache_dir, f.name)
if toUnicode(file_path) not in files:
os.remove(file_path)
except:
@@ -2,6 +2,7 @@ from CodernityDB.tree_index import TreeBasedIndex
class ProfileIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = 'i'
+3 -13
View File
@@ -12,7 +12,9 @@ log = CPLog(__name__)
class ProfilePlugin(Plugin):
to_dict = {'types': {}}
_database = {
'profile': ProfileIndex
}
def __init__(self):
addEvent('profile.all', self.all)
@@ -29,21 +31,9 @@ class ProfilePlugin(Plugin):
}"""}
})
addEvent('database.setup', self.databaseSetup)
addEvent('app.initialize', self.fill, priority = 90)
addEvent('app.load', self.forceDefaults)
def databaseSetup(self):
db = get_db()
try:
db.add_index(ProfileIndex(db.path, 'profile'))
except:
log.debug('Index already exists')
db.edit_index(ProfileIndex(db.path, 'profile'))
def forceDefaults(self):
# Get all active movies without profile
@@ -3,6 +3,7 @@ from hashlib import md5
class QualityIndex(HashIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
+4 -12
View File
@@ -14,6 +14,10 @@ log = CPLog(__name__)
class QualityPlugin(Plugin):
_database = {
'quality': QualityIndex
}
qualities = [
{'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]},
{'identifier': '1080p', 'hd': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']},
@@ -49,7 +53,6 @@ class QualityPlugin(Plugin):
})
addEvent('app.initialize', self.fill, priority = 10)
addEvent('database.setup', self.databaseSetup)
addEvent('app.test', self.doTest)
@@ -63,17 +66,6 @@ class QualityPlugin(Plugin):
def getOrder(self):
return self.order
def databaseSetup(self):
db = get_db()
# Quality index
try:
db.add_index(QualityIndex(db.path, 'quality'))
except:
log.debug('Index already exists')
db.edit_index(QualityIndex(db.path, 'quality'))
def preReleases(self):
return self.pre_releases
@@ -4,6 +4,7 @@ from CodernityDB.tree_index import TreeBasedIndex
class ReleaseIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
@@ -30,6 +31,7 @@ class ReleaseIndex(TreeBasedIndex):
class ReleaseStatusIndex(TreeBasedIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
@@ -44,6 +46,7 @@ class ReleaseStatusIndex(TreeBasedIndex):
class ReleaseIDIndex(HashIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
@@ -58,6 +61,7 @@ class ReleaseIDIndex(HashIndex):
class ReleaseDownloadIndex(HashIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
+7 -34
View File
@@ -17,6 +17,13 @@ log = CPLog(__name__)
class Release(Plugin):
_database = {
'release': ReleaseIndex,
'release_status': ReleaseStatusIndex,
'release_identifier': ReleaseIDIndex,
'release_download': ReleaseDownloadIndex
}
def __init__(self):
addApiView('release.manual_download', self.manualDownload, docs = {
'desc': 'Send a release manually to the downloaders',
@@ -45,44 +52,10 @@ class Release(Plugin):
addEvent('release.clean', self.clean)
addEvent('release.update_status', self.updateStatus)
addEvent('database.setup', self.databaseSetup)
# Clean releases that didn't have activity in the last week
addEvent('app.load', self.cleanDone)
fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 4)
def databaseSetup(self):
db = get_db()
# Release media_id index
try:
db.add_index(ReleaseIndex(db.path, 'release'))
except:
log.debug('Index already exists')
db.edit_index(ReleaseIndex(db.path, 'release'))
# Release status index
try:
db.add_index(ReleaseStatusIndex(db.path, 'release_status'))
except:
log.debug('Index already exists')
db.edit_index(ReleaseStatusIndex(db.path, 'release_status'))
# Release identifier index
try:
db.add_index(ReleaseIDIndex(db.path, 'release_identifier'))
except:
log.debug('Index already exists')
db.edit_index(ReleaseIDIndex(db.path, 'release_identifier'))
# Release identifier index
try:
db.add_index(ReleaseDownloadIndex(db.path, 'release_download'))
except:
log.debug('Index already exists')
db.edit_index(ReleaseDownloadIndex(db.path, 'release_download'))
def cleanDone(self):
log.debug('Removing releases from dashboard')
@@ -141,4 +141,4 @@ if(document.location.href.indexOf(host) == -1)
else
setVersion();
}
}
@@ -75,7 +75,7 @@ class TorrentPotato(TorrentProvider):
pass_keys = splitString(self.conf('pass_key'), clean = False)
extra_score = splitString(self.conf('extra_score'), clean = False)
list = []
host_list = []
for nr in range(len(hosts)):
try: key = pass_keys[nr]
@@ -93,7 +93,7 @@ class TorrentPotato(TorrentProvider):
try: seed_time = seed_times[nr]
except: seed_time = ''
list.append({
host_list.append({
'use': uses[nr],
'host': host,
'name': name,
@@ -103,7 +103,7 @@ class TorrentPotato(TorrentProvider):
'extra_score': tryInt(extra_score[nr]) if len(extra_score) > nr else 0
})
return list
return host_list
def belongsTo(self, url, provider = None, host = None):
@@ -1,11 +1,12 @@
from __future__ import with_statement
import traceback
from CodernityDB.hash_index import HashIndex
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.variable import mergeDicts, tryInt, tryFloat
from couchpotato.core.settings.index import PropertyIndex
import ConfigParser
from hashlib import md5
class Settings(object):
@@ -73,7 +74,7 @@ class Settings(object):
try:
db.add_index(PropertyIndex(db.path, 'property'))
except:
self.log.debug('Index already exists')
self.log.debug('Index for properties already exists')
db.edit_index(PropertyIndex(db.path, 'property'))
def parser(self):
@@ -250,3 +251,17 @@ class Settings(object):
'identifier': identifier,
'value': toUnicode(value),
})
class PropertyIndex(HashIndex):
_version = 1
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
super(PropertyIndex, self).__init__(*args, **kwargs)
def make_key(self, key):
return md5(key).hexdigest()
def make_key_value(self, data):
if data.get('_t') == 'property':
return md5(data['identifier']).hexdigest(), None
-16
View File
@@ -1,16 +0,0 @@
from CodernityDB.hash_index import UniqueHashIndex, HashIndex
from hashlib import md5
class PropertyIndex(HashIndex):
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
super(PropertyIndex, self).__init__(*args, **kwargs)
def make_key(self, key):
return md5(key).hexdigest()
def make_key_value(self, data):
if data.get('_t') == 'property':
return md5(data['identifier']).hexdigest(), None
+2
View File
@@ -1,3 +1,4 @@
from couchpotato.core.database import Database
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.loader import Loader
from couchpotato.core.settings import Settings
@@ -14,6 +15,7 @@ class Env(object):
_debug = False
_dev = False
_settings = Settings()
_database = Database()
_loader = Loader()
_cache = None
_options = None
+49 -1
View File
@@ -92,4 +92,52 @@ pre {
.api .return {
float: left;
width: 700px;
}
}
.database table {
font-size: 11px;
}
.database table th {
text-align: left;
}
.database table tr:hover {
position: relative;
z-index: 20;
}
.database table td {
vertical-align: top;
position: relative;
}
.database table .id {
width: 100px;
}
.database table ._rev {
width: 60px;
}
.database table ._t {
width: 60px;
}
.database table .form {
width: 600px;
}
.database table form {
width: 600px;
}
.database textarea {
font-size: 12px;
width: 100%;
height: 200px;
}
.database input[type=submit] {
display: block;
}
+3 -3
View File
@@ -10,7 +10,7 @@
<h1>CouchPotato API Documentation</h1>
<div class="api">
You can access the API via <pre>{{ Env.get('api_base') }}</pre>
To see it in action, have a look at the webinterface with Firebug (on firefox) or the development tools included in Chrome.
To see it in action, have a look at the webinterface with Firebug (on firefox) or the development tools included in Chrome.
All the data that you see there are from the API.
<br />
<br />
@@ -26,7 +26,7 @@
Will return {"api_key": "XXXXXXXXXX", "success": true}. When username or password is empty you don't need to md5 it.
<br />
</div>
{% for route in routes %}
{% if api_docs.get(route) %}
<div class="api">
@@ -68,4 +68,4 @@
</div>
</body>
</html>
</html>
+115
View File
@@ -0,0 +1,115 @@
{% autoescape None %}
<!doctype html>
<html>
<head>
<link rel="stylesheet" href="{{ Env.get('static_path') }}style/api.css" type="text/css">
<script type="text/javascript" src="{{ Env.get('static_path') }}scripts/library/mootools.js"></script>
<script>
var api_base = '{{ Env.get('api_base') }}';
var createList = function(name, documents){
var list;
var el = new Element('div').adopt(
new Element('h2', {'text': name}),
list = new Element('table.documents').grab(
new Element('tr').adopt(
new Element('th.id', {'text': 'id'}),
new Element('th._rev', {'text': 'rev'}),
new Element('th._t', {'text': 'type'}),
new Element('th.form', {'text': 'document'}),
new Element('th.actions', {'text': ''})
)
)
);
documents.each(function(doc, nr){
new Element('tr.document').adopt(
new Element('td.id', {'text': doc['_id']}),
new Element('td._rev', {'text': doc['_rev']}),
new Element('td._t', {'text': doc['_t']}),
new Element('td.form').grab(
new Element('form', {'action': '', 'method': 'post'}).adopt(
new Element('textarea.document', {'text': JSON.stringify(doc, null, 4)}),
new Element('input.submit', {'text': 'save', 'type': 'submit', 'value': 'save'})
)
),
new Element('td.actions').grab(
new Element('a.delete', {'text': 'delete', 'data-id': doc['_id']})
)
).inject(list)
});
if(documents.length > 0)
return el;
}
$(window).addEvent('domready', function(){
var c = $('container');
// Delete
c.addEvent('click:relay(.delete)', function(e){
(e).stop();
if(confirm('Are you sure?')){
new Request.JSON({
'url': api_base + 'database.document.delete',
'data': this.get('data-id'),
'onSuccess': function(){
console.log(arguments);
}
}).send();
}
});
// Form submit
c.addEvent('submit:relay(form)', function(e){
(e).stop();
var form = this;
new Request.JSON({
'url': api_base + 'database.document.update',
'data': {
'document': form.getElement('textarea').get('value')
},
'onSuccess': function(response){
form.getElement('textarea').set('value', JSON.stringify(response.document, null, 4));
form.getParent('tr').getElement('._rev').set('text', response.document['_rev']);
}
}).send();
});
new Request.JSON({
'url': api_base + 'database.list_documents',
'method': 'get',
'onSuccess': function(data){
Object.each(data, function(documents, name){
var list = createList(name, documents);
if(list)
list.inject(c);
})
},
'onFailure': function(){
alert('Something went wrong retrieving all documents')
}
}).send();
})
</script>
<title>CouchPotato Database Management</title>
</head>
<body>
<h1>CouchPotato Database Management</h1>
<div id="container" class="database"></div>
</body>
</html>