dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scheduler

Browse Source 
NZBmatrix, newznab
Scores
This commit is contained in:
Ruud
2011-04-17 23:36:23 +02:00
parent 9524306739
commit 7837adec1d
38 changed files with 856 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/cli.py
+2 -3
View File
@@ -1,7 +1,7 @@
from argparse import ArgumentParser
from couchpotato import web
from couchpotato.api import api
from couchpotato.core.event import fireEvent
from couchpotato.core.event import fireEventAsync
from libs.daemon import createDaemon
from logging import handlers
from werkzeug.contrib.cache import FileSystemCache
@@ -90,7 +90,6 @@ def cmd_couchpotato(base_path, args):
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = base_path)
loader.addModule(0, 'core', 'couchpotato.core', 'core')
loader.run()
@@ -116,7 +115,7 @@ def cmd_couchpotato(base_path, args):
from couchpotato.core.settings.model import setup
setup()
fireEvent('app.load')
fireEventAsync('app.load')
# Create app
from couchpotato import app
couchpotato/core/__init__.py
-73
View File
@@ -1,74 +1 @@
from uuid import uuid4
def start():
pass
config = [{
'name': 'core',
'groups': [
{
'tab': 'general',
'name': 'basics',
'description': 'Needs restart before changes take effect.',
'options': [
{
'name': 'username',
'default': '',
},
{
'name': 'password',
'default': '',
'type': 'password',
},
{
'name': 'host',
'advanced': True,
'default': '0.0.0.0',
'label': 'IP',
'description': 'Host that I should listen to. "0.0.0.0" listens to all ips.',
},
{
'name': 'port',
'default': 5000,
'type': 'int',
'description': 'The port I should listen to.',
},
{
'name': 'launch_browser',
'default': 1,
'type': 'bool',
'label': 'Launch Browser',
'description': 'Launch the browser when I start.',
},
],
},
{
'tab': 'general',
'name': 'advanced',
'description': "For those who know what the're doing",
'advanced': True,
'options': [
{
'name': 'api_key',
'default': uuid4().hex,
'readonly': 1,
'label': 'Api Key',
'description': "This is top-secret! Don't share this!",
},
{
'name': 'debug',
'default': 0,
'type': 'bool',
'label': 'Debug',
'description': 'Enable debugging.',
},
{
'name': 'url_base',
'default': '',
'label': 'Url Base',
'description': 'When using mod_proxy use this to append the url with this.',
},
],
},
],
}]
couchpotato/core/_base/__init__.py
View File
couchpotato/core/_base/_core/__init__.py
+74
View File
@@ -0,0 +1,74 @@
from uuid import uuid4
def start():
pass
config = [{
'name': 'core',
'groups': [
{
'tab': 'general',
'name': 'basics',
'description': 'Needs restart before changes take effect.',
'options': [
{
'name': 'username',
'default': '',
},
{
'name': 'password',
'default': '',
'type': 'password',
},
{
'name': 'host',
'advanced': True,
'default': '0.0.0.0',
'label': 'IP',
'description': 'Host that I should listen to. "0.0.0.0" listens to all ips.',
},
{
'name': 'port',
'default': 5000,
'type': 'int',
'description': 'The port I should listen to.',
},
{
'name': 'launch_browser',
'default': 1,
'type': 'bool',
'label': 'Launch Browser',
'description': 'Launch the browser when I start.',
},
],
},
{
'tab': 'general',
'name': 'advanced',
'description': "For those who know what the're doing",
'advanced': True,
'options': [
{
'name': 'api_key',
'default': uuid4().hex,
'readonly': 1,
'label': 'Api Key',
'description': "This is top-secret! Don't share this!",
},
{
'name': 'debug',
'default': 0,
'type': 'bool',
'label': 'Debug',
'description': 'Enable debugging.',
},
{
'name': 'url_base',
'default': '',
'label': 'Url Base',
'description': 'When using mod_proxy use this to append the url with this.',
},
],
},
],
}]
couchpotato/core/plugins/_clientscript/__init__.py → couchpotato/core/_base/clientscript/__init__.py
View File
couchpotato/core/plugins/_clientscript/main.py → couchpotato/core/_base/clientscript/main.py
View File
couchpotato/core/_base/scheduler/__init__.py
+6
View File
@@ -0,0 +1,6 @@
from .main import Scheduler
def start():
return Scheduler()
config = []
couchpotato/core/_base/scheduler/main.py
+85
View File
@@ -0,0 +1,85 @@
from apscheduler.scheduler import Scheduler as Sched
from couchpotato.core.event import addEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
log = CPLog(__name__)
class Scheduler(Plugin):
crons = {}
intervals = {}
started = False
def __init__(self):
addEvent('schedule.cron', self.cron)
addEvent('schedule.interval', self.interval)
addEvent('schedule.start', self.start)
addEvent('schedule.restart', self.start)
addEvent('app.load', self.start)
self.sched = Sched(misfire_grace_time = 60)
def remove(self, identifier):
for type in ['interval', 'cron']:
try:
self.sched.unschedule_job(getattr(self, type)[identifier]['job'])
log.debug('%s unscheduled %s' % (type.capitalize(), identifier))
except:
pass
def start(self):
# Stop all running
self.stop()
# Crons
for identifier in self.crons:
self.remove(identifier)
cron = self.crons[identifier]
job = self.sched.add_cron_job(cron['handle'], day = cron['day'], hour = cron['hour'], minute = cron['minute'])
cron['job'] = job
# Intervals
for identifier in self.intervals:
self.remove(identifier)
interval = self.intervals[identifier]
job = self.sched.add_interval_job(interval['handle'], hours = interval['hours'], minutes = interval['minutes'], seconds = interval['seconds'], repeat = interval['repeat'])
interval['job'] = job
# Start it
self.sched.start()
self.started = True
def stop(self):
if self.started:
self.sched.shutdown()
self.started = False
def cron(self, identifier = '', handle = None, day = '*', hour = '*', minute = '*'):
log.info('Scheduling "%s", cron: day = %s, hour = %s, minute = %s' % (identifier, day, hour, minute))
self.remove(identifier)
self.crons[identifier] = {
'handle': handle,
'day': day,
'hour': hour,
'minute': minute,
}
def interval(self, identifier = '', handle = None, hours = 0, minutes = 0, seconds = 0, repeat = 0):
log.info('Scheduling %s, interval: hours = %s, minutes = %s, seconds = %s, repeat = %s' % (identifier, hours, minutes, seconds, repeat))
self.remove(identifier)
self.intervals[identifier] = {
'handle': handle,
'repeat': repeat,
'hours': hours,
'minutes': minutes,
'seconds': seconds,
}
couchpotato/core/event.py
+15 -5
View File
@@ -1,6 +1,7 @@
from axl.axel import Event
from couchpotato.core.helpers.variable import merge_dicts
from couchpotato.core.helpers.variable import mergeDicts
from couchpotato.core.logger import CPLog
import threading
import traceback
log = CPLog(__name__)
@@ -11,7 +12,7 @@ def addEvent(name, handler):
if events.get(name):
e = events[name]
else:
e = events[name] = Event(threads = 20, exc_info = True, traceback = True)
e = events[name] = Event(threads = 20, exc_info = True, traceback = True, lock = threading.RLock())
e += handler
@@ -51,15 +52,24 @@ def fireEvent(name, *args, **kwargs):
else:
errorHandler(r[1])
# Merge the results
if merge:
# Merge dict
if merge and type(results[0]) == dict:
merged = {}
for result in results:
merged = merge_dicts(merged, result)
merged = mergeDicts(merged, result)
results = merged
# Merg lists
elif merge and type(results[0]) == list:
merged = []
for result in results:
merged += result
results = merged
return results
except KeyError:
pass
except Exception, e:
log.error('%s: %s' % (name, e))
couchpotato/core/helpers/encoding.py
+1 -1
View File
@@ -29,7 +29,7 @@ def toUnicode(original, *args):
return unicode(ascii_text)
def is_int(value):
def isInt(value):
try:
int(value)
return True
couchpotato/core/helpers/rss.py
+12 -2
View File
@@ -5,7 +5,7 @@ log = CPLog(__name__)
class RSS():
def gettextelements(self, xml, path):
def getTextElements(self, xml, path):
''' Find elements and return tree'''
textelements = []
@@ -17,7 +17,17 @@ class RSS():
textelements.append(element.text)
return textelements
def gettextelement(self, xml, path):
def getElements(self, xml, path):
elements = []
try:
elements = xml.findall(path)
except:
pass
return elements
def getTextElement(self, xml, path):
''' Find element and return text'''
try:
couchpotato/core/helpers/variable.py
+4 -4
View File
@@ -1,12 +1,12 @@
import hashlib
import os.path
def is_dict(object):
def isDict(object):
return isinstance(object, dict)
def merge_dicts(a, b):
assert is_dict(a), is_dict(b)
def mergeDicts(a, b):
assert isDict(a), isDict(b)
dst = a.copy()
stack = [(dst, b)]
@@ -16,7 +16,7 @@ def merge_dicts(a, b):
if key not in current_dst:
current_dst[key] = current_src[key]
else:
if is_dict(current_src[key]) and is_dict(current_dst[key]) :
if isDict(current_src[key]) and isDict(current_dst[key]) :
stack.append((current_dst[key], current_src[key]))
else:
current_dst[key] = current_src[key]
couchpotato/core/loader.py
+3 -2
View File
@@ -18,7 +18,8 @@ class Loader:
providers = os.path.join(root, 'couchpotato', 'core', 'providers')
self.paths = {
'plugin' : (0, 'couchpotato.core.plugins', os.path.join(core, 'plugins')),
'core' : (0, 'couchpotato.core._base', os.path.join(core, '_base')),
'plugin' : (1, 'couchpotato.core.plugins', os.path.join(core, 'plugins')),
'notifications' : (20, 'couchpotato.core.notifications', os.path.join(core, 'notifications')),
'downloaders' : (20, 'couchpotato.core.downloaders', os.path.join(core, 'downloaders')),
'movie_provider' : (20, 'couchpotato.core.providers.movie', os.path.join(providers, 'movie')),
@@ -48,7 +49,7 @@ class Loader:
self.loadPlugins(m, plugin.get('name'))
except Exception, e:
log.error('Can\'t import %s: %s' % (plugin.get('name'), e))
log.error('Can\'t import %s: %s' % (module_name, e))
if did_save:
fireEvent('settings.save')
couchpotato/core/plugins/__init__.py
+74
View File
@@ -1 +1,75 @@
from uuid import uuid4
def start():
pass
config = [{
'name': 'core',
'groups': [
{
'tab': 'general',
'name': 'basics',
'description': 'Needs restart before changes take effect.',
'options': [
{
'name': 'username',
'default': '',
},
{
'name': 'password',
'default': '',
'type': 'password',
},
{
'name': 'host',
'advanced': True,
'default': '0.0.0.0',
'label': 'IP',
'description': 'Host that I should listen to. "0.0.0.0" listens to all ips.',
},
{
'name': 'port',
'default': 5000,
'type': 'int',
'description': 'The port I should listen to.',
},
{
'name': 'launch_browser',
'default': 1,
'type': 'bool',
'label': 'Launch Browser',
'description': 'Launch the browser when I start.',
},
],
},
{
'tab': 'general',
'name': 'advanced',
'description': "For those who know what the're doing",
'advanced': True,
'options': [
{
'name': 'api_key',
'default': uuid4().hex,
'readonly': 1,
'label': 'Api Key',
'description': "This is top-secret! Don't share this!",
},
{
'name': 'debug',
'default': 0,
'type': 'bool',
'label': 'Debug',
'description': 'Enable debugging.',
},
{
'name': 'url_base',
'default': '',
'label': 'Url Base',
'description': 'When using mod_proxy use this to append the url with this.',
},
],
},
],
}]
couchpotato/core/plugins/base.py
+1 -1
View File
@@ -37,4 +37,4 @@ class Plugin():
return not self.isEnabled()
def isEnabled(self):
return self.conf('enabled', True)
return self.conf('enabled')
couchpotato/core/plugins/file_browser/__init__.py → couchpotato/core/plugins/browser/__init__.py
View File
couchpotato/core/plugins/file_browser/main.py → couchpotato/core/plugins/browser/main.py
View File
couchpotato/core/plugins/quality/main.py
+9
View File
@@ -27,6 +27,7 @@ class QualityPlugin(Plugin):
def __init__(self):
addEvent('quality.all', self.all)
addEvent('quality.single', self.single)
addEvent('app.load', self.fill)
path = self.registerStatic(__file__)
@@ -46,6 +47,14 @@ class QualityPlugin(Plugin):
return temp
def single(self, identifier = ''):
db = get_session()
quality = db.query(Quality).filter_by(identifier = identifier).first()
return dict(self.getQuality(quality.identifier), **quality.to_dict())
def getQuality(self, identifier):
for q in self.qualities:
couchpotato/core/plugins/renamer/main.py
+9
View File
@@ -1,3 +1,4 @@
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
@@ -8,3 +9,11 @@ class Renamer(Plugin):
def __init__(self):
pass
addEvent('renamer.scan', self.scan)
addEvent('app.load', self.scan)
fireEvent('schedule.interval', 'renamer.scan', self.scan, minutes = self.conf('run_every'))
def scan(self):
print 'scan'
couchpotato/core/plugins/score/__init__.py
+6
View File
@@ -0,0 +1,6 @@
from .main import Score
def start():
return Score()
config = []
couchpotato/core/plugins/score/main.py
+22
View File
@@ -0,0 +1,22 @@
from couchpotato.core.event import addEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore
log = CPLog(__name__)
class Score(Plugin):
def __init__(self):
addEvent('score.calculate', self.calculate)
def calculate(self, nzb, movie):
''' Calculate the score of a NZB, used for sorting later '''
score = nameScore(nzb['name'], movie['library']['year'])
for movie_title in movie['library']['titles']:
score += nameRatioScore(nzb['name'], movie_title['title'])
return score
couchpotato/core/plugins/score/scores.py
+51
View File
@@ -0,0 +1,51 @@
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.environment import Env
import re
name_scores = [
'proper:2', 'repack:2',
'unrated:1',
'x264:1',
'DTS:4', 'AC3:2',
'720p:10', '1080p:10', 'bluray:10', 'dvd:1', 'dvdrip:1', 'brrip:1', 'bdrip:1',
'metis:1', 'diamond:1', 'wiki:1', 'CBGB:1',
'german:-10', 'french:-10', 'spanish:-10', 'swesub:-20', 'danish:-10'
]
def nameScore(name, year):
''' Calculate score for words in the NZB name '''
score = 0
name = name.lower()
#give points for the cool stuff
for value in name_scores:
v = value.split(':')
add = int(v.pop())
if v.pop() in name:
score = score + add
#points if the year is correct
if str(year) in name:
score = score + 1
# Contains preferred word
nzb_words = re.split('\W+', simplifyString(name))
preferred_words = Env.setting('preferred_words', section = 'searcher').split(',')
for word in preferred_words:
if word.strip() and word.strip().lower() in nzb_words:
score = score + 100
return score
def nameRatioScore(nzb_name, movie_name):
nzb_words = re.split('\W+', simplifyString(nzb_name))
movie_words = re.split('\W+', simplifyString(movie_name))
# Replace .,-_ with space
left_over = len(nzb_words) - len(movie_words)
if 2 <= left_over <= 6:
return 4
else:
return 0
couchpotato/core/plugins/searcher/__init__.py
+78
View File
@@ -0,0 +1,78 @@
from .main import Searcher
import random
def start():
return Searcher()
config = [{
'name': 'searcher',
'groups': [
{
'tab': 'searcher',
'name': 'searcher',
'label': 'Search',
'description': 'Options for the searchers',
'options': [
{
'name': 'required_words',
'label': 'Required words',
'default': '',
},
{
'name': 'ignored_words',
'label': 'Ignored words',
'default': '',
},
],
}, {
'tab': 'searcher',
'name': 'searcher',
'label': 'Cronjob',
'advanced': True,
'description': 'Cron settings for the searcher see: <a href="http://packages.python.org/APScheduler/cronschedule.html">APScheduler</a> for details.',
'options': [
{
'name': 'cron_day',
'label': 'Day',
'advanced': True,
'default': '*',
'type': 'string',
'description': '<strong>*</strong>: Every day, <strong>*/2</strong>: Every 2 days, <strong>1</strong>: Every first of the month.',
},
{
'name': 'cron_hour',
'label': 'Hour',
'advanced': True,
'default': random.randint(0, 23),
'type': 'string',
'description': '<strong>*</strong>: Every hour, <strong>*/8</strong>: Every 8 hours, <strong>3</strong>: At 3, midnight.',
},
{
'name': 'cron_minute',
'label': 'Minute',
'advanced': True,
'default': random.randint(0, 59),
'type': 'string',
'description': "Just keep it random, so the providers don't get DDOSed by every CP user on a 'full' hour."
},
],
},
],
}, {
'name': 'nzb',
'groups': [
{
'tab': 'searcher',
'name': 'nzb',
'label': 'NZB',
'options': [
{
'name': 'retention',
'default': 350,
'type': 'int',
'unit': 'days'
},
],
},
],
}]
couchpotato/core/plugins/searcher/main.py
+156
View File
@@ -0,0 +1,156 @@
from couchpotato import get_session
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Movie
from couchpotato.environment import Env
import re
log = CPLog(__name__)
class Searcher(Plugin):
def __init__(self):
addEvent('searcher.all', self.all)
addEvent('searcher.single', self.single)
addEvent('searcher.correct_movie', self.correctMovie)
# Schedule cronjob
fireEvent('schedule.cron', 'searcher.all', self.all, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute'))
def all(self):
db = get_session()
movies = db.query(Movie).filter(
Movie.status.has(identifier = 'active')
).all()
for movie in movies:
self.single(movie.to_dict(deep = {
'profile': {'types': {'quality': {}}},
'releases': {'status': {}, 'quality': {}},
'library': {'titles': {}, 'files':{}},
'files': {}
}))
def single(self, movie):
for type in movie['profile']['types']:
results = fireEvent('provider.yarr.search', movie, type['quality'], merge = True)
sorted_results = sorted(results, key = lambda k: k['score'], reverse = True)
for nzb in sorted_results:
print nzb['name']
def correctMovie(self, nzb = {}, movie = {}, quality = {}, **kwargs):
imdb_results = kwargs.get('imdb_results', False)
single_category = kwargs.get('single_category', False)
retention = Env.setting('retention', section = 'nzb')
if retention < nzb.get('age', 0):
log.info('Wrong: Outside retention, age = %s, needs = %s: %s' % (nzb['age'], retention, nzb['name']))
return False
nzb_words = re.split('\W+', simplifyString(nzb['name']))
required_words = self.conf('required_words').split(',')
if self.conf('required_words') and not list(set(nzb_words) & set(required_words)):
log.info("NZB doesn't contain any of the required words.")
return False
ignored_words = self.conf('ignored_words').split(',')
blacklisted = list(set(nzb_words) & set(ignored_words))
if self.conf('ignored_words') and blacklisted:
log.info("NZB '%s' contains the following blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted)))
return False
#qualities = fireEvent('quality.all', single = True)
preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True)
# Contains lower quality string
if self.containsOtherQuality(nzb['name'], preferred_quality, single_category):
log.info('Wrong: %s, looking for %s' % (nzb['name'], quality['label']))
return False
"""
# File to small
minSize = q.minimumSize(qualityType)
if minSize > item.size:
log.info('"%s" is too small to be %s. %sMB instead of the minimal of %sMB.' % (item.name, type['label'], item.size, minSize))
return False
# File to large
maxSize = q.maximumSize(qualityType)
if maxSize < item.size:
log.info('"%s" is too large to be %s. %sMB instead of the maximum of %sMB.' % (item.name, type['label'], item.size, maxSize))
return False
"""
if imdb_results:
return True
# Check if nzb contains imdb link
if self.checkIMDB([nzb['description']], movie['library']['identifier']):
return True
for movie_title in movie['library']['titles']:
movie_words = re.split('\W+', simplifyString(movie_title['title']))
if self.correctName(nzb['name'], movie_title['title']):
# if no IMDB link, at least check year range 1
if len(movie_words) > 2 and self.correctYear([nzb['name']], movie['library']['year'], 1):
return True
# if no IMDB link, at least check year
if len(movie_words) == 2 and self.correctYear([nzb['name']], movie['library']['year'], 0):
return True
return False
def containsOtherQuality(self, name, preferred_quality = {}, single_category = False):
nzb_words = re.split('\W+', simplifyString(name))
qualities = fireEvent('quality.all', single = True)
found = {}
for quality in qualities:
# Main in words
if quality['identifier'] in nzb_words:
found[quality['identifier']] = True
# Alt in words
if list(set(nzb_words) & set(quality['alternative'])):
found[quality['identifier']] = True
# Allow other qualities
for allowed in preferred_quality.get('allow'):
if found.get(allowed):
del found[allowed]
if (len(found) == 0 and single_category):
return False
return not (found.get(preferred_quality['identifier']) and len(found) == 1)
def checkIMDB(self, haystack, imdbId):
for string in haystack:
if 'imdb.com/title/' + imdbId in string:
return True
return False
def correctYear(self, haystack, year, range):
for string in haystack:
if str(year) in string or str(int(year) + range) in string or str(int(year) - range) in string: # 1 year of is fine too
return True
return False
couchpotato/core/plugins/wizard/__init__.py
+1 -1
View File
@@ -13,7 +13,7 @@ config = [{
{
'name': 'show_wizard',
'label': 'Run the wizard',
'default': True,
'default': 1,
'type': 'bool',
},
],
couchpotato/core/plugins/wizard/static/spotlight.js
+2 -2
View File
@@ -193,7 +193,7 @@ var Spotlight = new Class({
'position': 'absolute',
'background-color': 'rgba('+self.options.fillColor.join(',')+', '+self.options.fillOpacity+')',
'display': 'block',
'z-index': 2,
'z-index': 999,
'top': self.top[nr],
'left': self.left[nr],
'height': self.height[nr],
@@ -276,7 +276,7 @@ var Spotlight = new Class({
var self = this;
var soften = self.options.soften;
var edge = new Element('div', {
var edge = new Element('div.edge', {
'styles': Object.merge({
'position': 'absolute',
'width': soften,
couchpotato/core/plugins/wizard/static/wizard.js
+10 -1
View File
@@ -6,13 +6,22 @@ var WizardBase = new Class({
var self = this;
self.steps = steps;
self.start();
self.spotlight = new Spotlight([], {
'fillColor': [0,0,0],
'soften': 0
});
window.addEvent('resize', self.spotlight.create.bind(self.spotlight))
},
start: function(){
var self = this;
var els = $(document.body).getElements("[name='core[username]'], [name='core[password]'], [name='core[launch_browser]']").getParent('.ctrlHolder')
p(els, self.spotlight.setElements(els))
self.spotlight.create();
},
couchpotato/core/providers/base.py
+114 -6
View File
@@ -1,5 +1,12 @@
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from urllib2 import URLError
import math
import re
import socket
import time
import urllib2
log = CPLog(__name__)
@@ -7,24 +14,125 @@ log = CPLog(__name__)
class Provider(Plugin):
type = None # movie, nzb, torrent, subtitle, trailer
timeout = 10 # Default timeout for url requests
time_between_searches = 10 # Default timeout for url requests
last_use = 0
last_available_check = 0
is_available = 0
def getCache(self, cache_key):
cache = Env.get('cache').get(cache_key)
if cache:
log.debug('Getting cache %s' % cache_key)
return cache
def setCache(self, cache_key, value):
log.debug('Setting cache %s' % cache_key)
Env.get('cache').set(cache_key, value)
def isAvailable(self, test_url):
if Env.get('debug'): return True
now = time.time()
if self.last_available_check < now - 900:
self.last_available_check = now
try:
self.urlopen(test_url, 30)
self.is_available = True
except (IOError, URLError):
log.error('%s unavailable, trying again in an 15 minutes.' % self.name)
self.is_available = False
return self.is_available
def urlopen(self, url, timeout = 10, username = None, password = None):
socket.setdefaulttimeout(timeout)
self.wait()
try:
if username and password:
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, username, password)
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(authhandler)
data = opener.open(url).read()
else:
data = urllib2.urlopen(url).read()
except IOError, e:
log.debug(e)
data = ''
self.last_use = time.time()
return data
def wait(self):
now = time.time()
wait = math.ceil(self.last_use - now + self.time_between_searches)
if wait > 0:
log.debug('Waiting for %s, %d seconds' % (self.getName(), wait))
time.sleep(self.last_use - now + self.time_between_searches)
class MovieProvider(Provider):
type = 'movie'
class NZBProvider(Provider):
class YarrProvider(Provider):
cat_ids = []
sizeGb = ['gb', 'gib']
sizeMb = ['mb', 'mib']
sizeKb = ['kb', 'kib']
def parseSize(self, size):
sizeRaw = size.lower()
size = re.sub(r'[^0-9.]', '', size).strip()
for s in self.sizeGb:
if s in sizeRaw:
return float(size) * 1024
for s in self.sizeMb:
if s in sizeRaw:
return float(size)
for s in self.sizeKb:
if s in sizeRaw:
return float(size) / 1024
return 0
def getCatId(self, identifier):
for cats in self.cat_ids:
ids, qualities = cats
if identifier in qualities:
return ids
return False
def found(self, new):
log.info('Found, score(%(score)s): %(name)s' % new)
class NZBProvider(YarrProvider):
type = 'nzb'
time_between_searches = 10 # Seconds
def isEnabled(self):
return True # nzb_downloaded is enabled check
def calculateAge(self, unix):
return int(time.time() - unix) / 24 / 60 / 60
class TorrentProvider(Provider):
class TorrentProvider(YarrProvider):
type = 'torrent'
couchpotato/core/providers/nzb/newzbin/main.py
+2 -3
View File
@@ -29,13 +29,12 @@ class Newzbin(NZBProvider):
def __init__(self):
addEvent('provider.nzb.search', self.search)
addEvent('provider.yarr.search', self.search)
def search(self, movie, quality):
self.cleanCache();
results = []
if not self.enabled() or not self.isAvailable(self.searchUrl):
if self.isDisabled() or not self.isAvailable(self.searchUrl):
return results
formatId = self.getFormatId(type)
couchpotato/core/providers/nzb/newznab/main.py
+46 -50
View File
@@ -1,20 +1,24 @@
from couchpotato.core.event import addEvent
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
from urllib import urlencode
from urllib2 import URLError
import time
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
class Newznab(NZBProvider):
class Newznab(NZBProvider, RSS):
urls = {
'download': 'get&id=%s%s',
'download': 'get&id=%s',
'detail': 'details&id=%s',
'search': 'movie',
}
cat_ids = [
@@ -29,43 +33,35 @@ class Newznab(NZBProvider):
def __init__(self):
addEvent('provider.nzb.search', self.search)
addEvent('provider.yarr.search', self.search)
def getUrl(self, type):
return cleanHost(self.conf('host')) + 'api?t=' + type
def search(self, movie, quality):
self.cleanCache();
results = []
if not self.enabled() or not self.isAvailable(self.getUrl(self.searchUrl)):
if self.isDisabled() or not self.isAvailable(self.getUrl(self.urls['search'])):
return results
catId = self.getCatId(type)
cat_id = self.getCatId(quality['identifier'])
arguments = urlencode({
'imdbid': movie.imdb.replace('tt', ''),
'cat': catId,
'apikey': self.conf('apikey'),
't': self.searchUrl,
'imdbid': movie['library']['identifier'].replace('tt', ''),
'cat': cat_id[0],
'apikey': self.conf('api_key'),
't': self.urls['search'],
'extended': 1
})
url = "%s&%s" % (self.getUrl(self.searchUrl), arguments)
cacheId = str(movie.imdb) + '-' + str(catId)
singleCat = (len(self.catIds.get(catId)) == 1 and catId != self.catBackupId)
url = "%s&%s" % (self.getUrl(self.urls['search']), arguments)
cache_key = '%s-%s' % (movie['library']['identifier'], cat_id[0])
single_cat = (len(cat_id) == 1 and cat_id[0] != self.cat_backup_id)
try:
cached = False
if(self.cache.get(cacheId)):
data = True
cached = True
log.info('Getting RSS from cache: %s.' % cacheId)
else:
log.info('Searching: %s' % url)
data = self.getCache(cache_key)
if not data:
data = self.urlopen(url)
self.cache[cacheId] = {
'time': time.time()
}
self.setCache(cache_key, data)
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
@@ -73,17 +69,14 @@ class Newznab(NZBProvider):
if data:
try:
try:
if cached:
xml = self.cache[cacheId]['xml']
else:
xml = self.getItems(data)
self.cache[cacheId]['xml'] = xml
except:
log.debug('No valid xml or to many requests.' % self.name)
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
return results
results = []
for nzb in xml:
for nzb in nzbs:
for item in nzb:
if item.attrib.get('name') == 'size':
@@ -91,20 +84,26 @@ class Newznab(NZBProvider):
elif item.attrib.get('name') == 'usenetdate':
date = item.attrib.get('value')
new = self.feedItem()
new.id = self.gettextelement(nzb, "guid").split('/')[-1:].pop()
new.type = 'nzb'
new.name = self.gettextelement(nzb, "title")
new.date = int(time.mktime(parse(date).timetuple()))
new.size = int(size) / 1024 / 1024
new.url = self.downloadLink(new.id)
new.detailUrl = self.detailLink(new.id)
new.content = self.gettextelement(nzb, "description")
new.score = self.calcScore(new, movie)
id = self.getTextElement(nzb, "guid").split('/')[-1:].pop()
new = {
'id': id,
'type': 'nzb',
'name': self.getTextElement(nzb, "title"),
'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
'size': int(size) / 1024 / 1024,
'url': (self.getUrl(self.urls['download']) % id) + self.getApiExt(),
'detail_url': (self.getUrl(self.urls['detail']) % id) + self.getApiExt(),
'content': self.getTextElement(nzb, "description"),
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
if new.date > time.time() - (int(self.config.get('NZB', 'retention')) * 24 * 60 * 60) and self.isCorrectMovie(new, movie, type, imdbResults = True, singleCategory = singleCat):
is_correct_movie = fireEvent('searcher.correct_movie',
nzb = new, movie = movie, quality = quality,
imdb_results = True, single_category = single_cat, single = True)
if is_correct_movie:
results.append(new)
log.info('Found: %s' % new.name)
self.found(new)
return results
except SyntaxError:
@@ -114,13 +113,10 @@ class Newznab(NZBProvider):
return results
def isEnabled(self):
return NZBProvider.isEnabled(self) and self.conf('enabled') and self.conf('host') and self.conf('apikey')
return NZBProvider.isEnabled(self) and self.conf('host') and self.conf('api_key')
def getApiExt(self):
return '&apikey=%s' % self.conf('apikey')
def downloadLink(self, id):
return self.getUrl(self.downloadUrl) % (id, self.getApiExt())
return '&apikey=%s' % self.conf('api_key')
def detailLink(self, id):
return self.getUrl(self.detailUrl) % id
couchpotato/core/providers/nzb/nzbmatrix/__init__.py
+7
View File
@@ -23,6 +23,13 @@ config = [{
'default': '',
'label': 'Api Key',
},
{
'name': 'english_only',
'default': 1,
'type': 'bool',
'label': 'English only',
'description': 'Only search for English spoken movies on NZBMatrix',
},
],
},
],
couchpotato/core/providers/nzb/nzbmatrix/main.py
+54 -56
View File
@@ -1,24 +1,27 @@
from couchpotato.core.event import addEvent
from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
from urllib import urlencode
from urllib2 import URLError
import time
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
class NZBMatrix(NZBProvider):
class NZBMatrix(NZBProvider, RSS):
urls = {
'download': 'https://api.nzbmatrix.com/v1.1/download.php?id=%s%s',
'download': 'https://api.nzbmatrix.com/v1.1/download.php?id=%s',
'detail': 'https://nzbmatrix.com/nzb-details.php?id=%s&hit=1',
'search': 'http://rss.nzbmatrix.com/rss.php',
}
cat_ids = [
([42], ['720p', '1080p']),
([42, 53], ['720p', '1080p']),
([2], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']),
([54], ['brrip']),
([1], ['dvdr']),
@@ -27,41 +30,37 @@ class NZBMatrix(NZBProvider):
def __init__(self):
addEvent('provider.nzb.search', self.search)
addEvent('provider.yarr.search', self.search)
def search(self, movie, quality):
self.cleanCache();
results = []
if not self.enabled() or not self.isAvailable(self.searchUrl):
if self.isDisabled() or not self.isAvailable(self.urls['search']):
return results
catId = self.getCatId(type)
cat_ids = ','.join(['%s' % x for x in self.getCatId(quality.get('identifier'))])
arguments = urlencode({
'term': movie.imdb,
'subcat': catId,
'term': movie['library']['identifier'],
'subcat': cat_ids,
'username': self.conf('username'),
'apikey': self.conf('apikey'),
'apikey': self.conf('api_key'),
'searchin': 'weblink',
'english': 1 if self.conf('english') else 0,
'age': Env.setting('retention', section = 'nzb'),
'english': self.conf('english_only'),
})
url = "%s?%s" % (self.searchUrl, arguments)
cacheId = str(movie.imdb) + '-' + str(catId)
singleCat = (len(self.catIds.get(catId)) == 1 and catId != self.catBackupId)
url = "%s?%s" % (self.urls['search'], arguments)
log.info('Searching: %s' % url)
cache_key = '%s-%s' % (movie['library'].get('identifier'), cat_ids)
single_cat = True
try:
cached = False
if(self.cache.get(cacheId)):
data = True
cached = True
log.info('Getting RSS from cache: %s.' % cacheId)
else:
log.info('Searching: %s' % url)
data = self.getCache(cache_key)
if not data:
data = self.urlopen(url)
self.cache[cacheId] = {
'time': time.time()
}
self.setCache(cache_key, data)
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
@@ -69,42 +68,41 @@ class NZBMatrix(NZBProvider):
if data:
try:
try:
if cached:
xml = self.cache[cacheId]['xml']
else:
xml = self.getItems(data)
self.cache[cacheId]['xml'] = xml
except:
log.debug('No valid xml or to many requests.. You never know with %s.' % self.name)
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
return results
for nzb in xml:
for nzb in nzbs:
title = self.gettextelement(nzb, "title")
title = self.getTextElement(nzb, "title")
if 'error' in title.lower(): continue
id = int(self.gettextelement(nzb, "link").split('&')[0].partition('id=')[2])
size = self.gettextelement(nzb, "description").split('<br /><b>')[2].split('> ')[1]
date = str(self.gettextelement(nzb, "description").split('<br /><b>')[3].partition('Added:</b> ')[2])
id = int(self.getTextElement(nzb, "link").split('&')[0].partition('id=')[2])
size = self.getTextElement(nzb, "description").split('<br /><b>')[2].split('> ')[1]
date = str(self.getTextElement(nzb, "description").split('<br /><b>')[3].partition('Added:</b> ')[2])
new = self.feedItem()
new.id = id
new.type = 'nzb'
new.name = title
new.date = int(time.mktime(parse(date).timetuple()))
new.size = self.parseSize(size)
new.url = self.downloadLink(id)
new.detailUrl = self.detailLink(id)
new.content = self.gettextelement(nzb, "description")
new.score = self.calcScore(new, movie)
new.checkNZB = True
new = {
'id': id,
'type': 'nzb',
'name': title,
'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
'size': self.parseSize(size),
'url': self.urls['download'] % id + self.getApiExt(),
'detail_url': self.urls['detail'] % id,
'description': self.getTextElement(nzb, "description"),
'check_nzb': True,
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
if new.date > time.time() - (int(self.config.get('NZB', 'retention')) * 24 * 60 * 60):
if self.isCorrectMovie(new, movie, type, imdbResults = True, singleCategory = singleCat):
results.append(new)
log.info('Found: %s' % new.name)
else:
log.info('Found outside retention: %s' % new.name)
is_correct_movie = fireEvent('searcher.correct_movie',
nzb = new, movie = movie, quality = quality,
imdb_results = True, single_category = single_cat, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
except SyntaxError:
@@ -116,4 +114,4 @@ class NZBMatrix(NZBProvider):
return '&username=%s&apikey=%s' % (self.conf('username'), self.conf('apikey'))
def isEnabled(self):
return NZBProvider.isEnabled(self) and self.conf('enabled') and self.conf('username') and self.conf('apikey')
return NZBProvider.isEnabled(self) and self.conf('username') and self.conf('api_key')
couchpotato/core/providers/nzb/nzbs/main.py
+2 -3
View File
@@ -29,13 +29,12 @@ class Nzbs(NZBProvider):
def __init__(self):
addEvent('provider.nzb.search', self.search)
addEvent('provider.yarr.search', self.search)
def search(self, movie, quality):
self.cleanCache();
results = []
if not self.enabled() or not self.isAvailable(self.apiUrl + '?test' + self.getApiExt()):
if self.isDisabled() or not self.isAvailable(self.apiUrl + '?test' + self.getApiExt()):
return results
catId = self.getCatId(type)
couchpotato/core/settings/__init__.py
+2 -2
View File
@@ -1,7 +1,7 @@
from __future__ import with_statement
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent
from couchpotato.core.helpers.encoding import is_int
from couchpotato.core.helpers.encoding import isInt
from couchpotato.core.helpers.request import getParams, jsonified
import ConfigParser
import os.path
@@ -61,7 +61,7 @@ class Settings():
return default
def cleanValue(self, value):
if(is_int(value)):
if(isInt(value)):
return int(value)
if str(value).lower() in self.bool:
couchpotato/static/scripts/page/settings.js
+4 -2
View File
@@ -7,6 +7,7 @@ Page.Settings = new Class({
tabs: {
'general': {},
'searcher': {},
'providers': {},
'downloaders': {},
'notifications': {},
@@ -177,7 +178,7 @@ Page.Settings = new Class({
'text': (group.label || group.name).capitalize()
}).adopt(
new Element('span.hint', {
'text': group.description
'html': group.description
})
)
)
@@ -243,7 +244,7 @@ var OptionBase = new Class({
var self = this;
if(self.options.description)
new Element('p.formHint', {
'text': self.options.description
'html': self.options.description
}).inject(self.el);
},
@@ -380,6 +381,7 @@ Option.Checkbox = new Class({
self.el.adopt(
self.createLabel().set('for', randomId),
self.input = new Element('input', {
'name': self.postName(),
'type': 'checkbox',
'checked': self.getSettingValue(),
'id': randomId
couchpotato/static/style/main.css
+1 -1
View File
@@ -138,7 +138,7 @@ form {
-webkit-box-shadow: 0 0 30px rgba(0,0,0,0.1);
position: fixed;
width: 99%;
z-index: 9999;
z-index: 2;
}
.header > div {
couchpotato/static/style/page/settings.css
+1 -1
View File
@@ -19,7 +19,7 @@
padding: 10px 15px;
border: 1px solid transparent;
position: relative;
z-index: 200;
z-index: 1;
margin-right: -1px;
}
.page.settings .tabs .active a {
couchpotato/templates/_desktop.html
+2
View File
@@ -56,6 +56,8 @@
App.setup({
'base_url': '{{ request.path }}'
});
//Wizard.start.delay(100, Wizard);
})
</script>
<title>CouchPotato</title>