diff --git a/CouchPotato.py b/CouchPotato.py index e777f9bf..375a1d41 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function from logging import handlers from os.path import dirname import logging @@ -132,14 +133,15 @@ if __name__ == '__main__': pass except SystemExit: raise - except socket.error as (nr, msg): + except socket.error as e: # log when socket receives SIGINT, but continue. # previous code would have skipped over other types of IO errors too. + nr, msg = e if nr != 4: try: l.log.critical(traceback.format_exc()) except: - print traceback.format_exc() + print(traceback.format_exc()) raise except: try: @@ -148,7 +150,7 @@ if __name__ == '__main__': if l: l.log.critical(traceback.format_exc()) else: - print traceback.format_exc() + print(traceback.format_exc()) except: - print traceback.format_exc() + print(traceback.format_exc()) raise diff --git a/contributing.md b/contributing.md index ef8546f0..d5db0b42 100644 --- a/contributing.md +++ b/contributing.md @@ -1,15 +1,25 @@ -#So you feel like posting a bug, sending me a pull request or just telling me how awesome I am. No problem! +## Got a issue/feature request or submitting a pull request? -##Just make sure you think of the following things: +Make sure you think of the following things: - * Search through the existing (and closed) issues first. See if you can get your answer there. +## Issue + * Search through the existing (and closed) issues first, see if you can get your answer there. * Double check the result manually, because it could be an external issue. * Post logs! Without seeing what is going on, I can't reproduce the error. - * What is the movie + quality you are searching for. - * What are you settings for the specific problem. - * What providers are you using. (While your logs include these, scanning through hundred of lines of log isn't my hobby). - * Give me a short step by step of how to reproduce. + * Also check the logs before submitting, obvious errors like permission or http errors are often not related to CP. + * What is the movie + quality you are searching for? + * What are you're settings for the specific problem? + * What providers are you using? (While you're logs include these, scanning through hundred of lines of log isn't our hobby) + * Post the logs from config directory, please do not copy paste the UI. Use pastebin to store these logs! + * Give a short step by step of how to reproduce the error. * What hardware / OS are you using and what are the limits? NAS can be slow and maybe have a different python installed then when you use CP on OSX or Windows for example. - * I will mark issues with the "can't reproduce" tag. Don't go asking me "why closed" if it clearly says the issue in the tag ;) + * I will mark issues with the "can't reproduce" tag. Don't go asking "why closed" if it clearly says the issue in the tag ;) + * If you're running on a NAS (QNAP, Austor etc..) with pre-made packages, make sure these are setup to use our source repo (RuudBurger/CouchPotatoServer) and nothing else!! -**If I don't get enough info, the chance of the issue getting closed is a lot bigger ;)** +## Pull Request + * Make sure you're pull request is made for develop branch (or relevant feature branch) + * Have you tested your PR? If not, why? + * Are there any limitations of your PR we should know of? + * Make sure to keep you're PR up-to-date with the branch you're trying to push into. + +**If we don't get enough info, the chance of the issue getting closed is a lot bigger ;)** diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index b8aa3ab9..6b8cfd36 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -9,13 +9,12 @@ import os import time import traceback -log = CPLog(__name__) +log = CPLog(__name__) views = {} template_loader = template.Loader(os.path.join(os.path.dirname(__file__), 'templates')) - class BaseHandler(RequestHandler): def get_current_user(self): @@ -24,9 +23,10 @@ class BaseHandler(RequestHandler): if username and password: return self.get_secure_cookie('user') - else: # Login when no username or password are set + else: # Login when no username or password are set return True + # Main web handler class WebHandler(BaseHandler): @@ -43,11 +43,13 @@ class WebHandler(BaseHandler): log.error("Failed doing web request '%s': %s", (route, traceback.format_exc())) self.write({'success': False, 'error': 'Failed returning results'}) + def addView(route, func, static = False): views[route] = func -def get_session(engine = None): - return Env.getSession(engine) + +def get_session(): + return Env.getSession() # Web view @@ -55,12 +57,10 @@ def index(): return template_loader.load('index.html').generate(sep = os.sep, fireEvent = fireEvent, Env = Env) addView('', index) + # API docs def apiDocs(): - routes = [] - - for route in api.iterkeys(): - routes.append(route) + routes = list(api.keys()) if api_docs.get(''): del api_docs[''] @@ -70,21 +70,22 @@ def apiDocs(): addView('docs', apiDocs) + # Make non basic auth option to get api key class KeyHandler(RequestHandler): def get(self, *args, **kwargs): - api = None + api_key = None try: username = Env.setting('username') password = Env.setting('password') if (self.get_argument('u') == md5(username) or not username) and (self.get_argument('p') == password or not password): - api = Env.setting('api_key') + api_key = Env.setting('api_key') self.write({ - 'success': api is not None, - 'api_key': api + 'success': api_key is not None, + 'api_key': api_key }) except: log.error('Failed doing key request: %s', (traceback.format_exc())) @@ -102,20 +103,21 @@ class LoginHandler(BaseHandler): def post(self, *args, **kwargs): - api = None + api_key = None username = Env.setting('username') password = Env.setting('password') if (self.get_argument('username') == username or not username) and (md5(self.get_argument('password')) == password or not password): - api = Env.setting('api_key') + api_key = Env.setting('api_key') - if api: + if api_key: remember_me = tryInt(self.get_argument('remember_me', default = 0)) - self.set_secure_cookie('user', api, expires_days = 30 if remember_me > 0 else None) + self.set_secure_cookie('user', api_key, expires_days = 30 if remember_me > 0 else None) self.redirect(Env.get('web_base')) + class LogoutHandler(BaseHandler): def get(self, *args, **kwargs): @@ -136,4 +138,3 @@ def page_not_found(rh): rh.set_status(404) rh.write('Wrong API key used') - diff --git a/couchpotato/api.py b/couchpotato/api.py index e86b127f..ba7f7b69 100644 --- a/couchpotato/api.py +++ b/couchpotato/api.py @@ -20,6 +20,7 @@ api_nonblock = {} api_docs = {} api_docs_missing = [] + def run_async(func): @wraps(func) def async_func(*args, **kwargs): @@ -29,6 +30,7 @@ def run_async(func): return async_func + # NonBlock API handler class NonBlockHandler(RequestHandler): @@ -61,6 +63,7 @@ class NonBlockHandler(RequestHandler): self.stopper = None + def addNonBlockApiView(route, func_tuple, docs = None, **kwargs): api_nonblock[route] = func_tuple @@ -69,6 +72,7 @@ def addNonBlockApiView(route, func_tuple, docs = None, **kwargs): else: api_docs_missing.append(route) + # Blocking API handler class ApiHandler(RequestHandler): @@ -98,11 +102,12 @@ class ApiHandler(RequestHandler): @run_async def run_handler(callback): try: - result = api[route](**kwargs) - callback(result) + res = api[route](**kwargs) + callback(res) except: log.error('Failed doing api request "%s": %s', (route, traceback.format_exc())) callback({'success': False, 'error': 'Failed returning results'}) + result = yield tornado.gen.Task(run_handler) # Check JSONP callback @@ -122,6 +127,7 @@ class ApiHandler(RequestHandler): api_locks[route].release() + def addApiView(route, func, static = False, docs = None, **kwargs): if static: func(route) diff --git a/couchpotato/core/_base/_core/__init__.py b/couchpotato/core/_base/_core/__init__.py index 4d1a6840..58965bbb 100644 --- a/couchpotato/core/_base/_core/__init__.py +++ b/couchpotato/core/_base/_core/__init__.py @@ -1,6 +1,7 @@ from .main import Core from uuid import uuid4 + def start(): return Core() diff --git a/couchpotato/core/_base/_core/main.py b/couchpotato/core/_base/_core/main.py index 803ac5a3..02e21f2d 100644 --- a/couchpotato/core/_base/_core/main.py +++ b/couchpotato/core/_base/_core/main.py @@ -55,6 +55,10 @@ class Core(Plugin): if not Env.get('desktop'): self.signalHandler() + # Set default urlopen timeout + import socket + socket.setdefaulttimeout(30) + def md5Password(self, value): return md5(value) if value else '' @@ -113,7 +117,7 @@ class Core(Plugin): if len(still_running) == 0: break - elif starttime < time.time() - 30: # Always force break after 30s wait + elif starttime < time.time() - 30: # Always force break after 30s wait break running = list(set(still_running) - set(self.ignore_restart)) diff --git a/couchpotato/core/_base/clientscript/__init__.py b/couchpotato/core/_base/clientscript/__init__.py index 8490eae7..8070044e 100644 --- a/couchpotato/core/_base/clientscript/__init__.py +++ b/couchpotato/core/_base/clientscript/__init__.py @@ -1,5 +1,6 @@ from .main import ClientScript + def start(): return ClientScript() diff --git a/couchpotato/core/_base/clientscript/main.py b/couchpotato/core/_base/clientscript/main.py index 1b7f1636..c1be7e73 100644 --- a/couchpotato/core/_base/clientscript/main.py +++ b/couchpotato/core/_base/clientscript/main.py @@ -34,6 +34,8 @@ class ClientScript(Plugin): 'scripts/library/question.js', 'scripts/library/scrollspy.js', 'scripts/library/spin.js', + 'scripts/library/Array.stableSort.js', + 'scripts/library/async.js', 'scripts/couchpotato.js', 'scripts/api.js', 'scripts/library/history.js', @@ -47,13 +49,14 @@ class ClientScript(Plugin): 'scripts/page/settings.js', 'scripts/page/about.js', 'scripts/page/manage.js', + 'scripts/misc/downloaders.js', ], } - urls = {'style': {}, 'script': {}, } - minified = {'style': {}, 'script': {}, } - paths = {'style': {}, 'script': {}, } + urls = {'style': {}, 'script': {}} + minified = {'style': {}, 'script': {}} + paths = {'style': {}, 'script': {}} comment = { 'style': '/*** %s:%d ***/\n', 'script': '// %s:%d\n' diff --git a/couchpotato/core/_base/desktop/__init__.py b/couchpotato/core/_base/desktop/__init__.py index 064492f2..e59ca523 100644 --- a/couchpotato/core/_base/desktop/__init__.py +++ b/couchpotato/core/_base/desktop/__init__.py @@ -1,5 +1,6 @@ from .main import Desktop + def start(): return Desktop() diff --git a/couchpotato/core/_base/scheduler/__init__.py b/couchpotato/core/_base/scheduler/__init__.py index aa1c5c90..abfc2305 100644 --- a/couchpotato/core/_base/scheduler/__init__.py +++ b/couchpotato/core/_base/scheduler/__init__.py @@ -1,5 +1,6 @@ from .main import Scheduler + def start(): return Scheduler() diff --git a/couchpotato/core/_base/scheduler/main.py b/couchpotato/core/_base/scheduler/main.py index 2c97e1b4..3d835f94 100644 --- a/couchpotato/core/_base/scheduler/main.py +++ b/couchpotato/core/_base/scheduler/main.py @@ -17,6 +17,7 @@ class Scheduler(Plugin): addEvent('schedule.cron', self.cron) addEvent('schedule.interval', self.interval) addEvent('schedule.remove', self.remove) + addEvent('schedule.queue', self.queue) self.sched = Sched(misfire_grace_time = 60) self.sched.start() @@ -31,13 +32,13 @@ class Scheduler(Plugin): pass def doShutdown(self): - super(Scheduler, self).doShutdown() self.stop() + return super(Scheduler, self).doShutdown() def stop(self): if self.started: log.debug('Stopping scheduler') - self.sched.shutdown() + self.sched.shutdown(wait = False) log.debug('Scheduler stopped') self.started = False @@ -64,3 +65,14 @@ class Scheduler(Plugin): 'seconds': seconds, 'job': self.sched.add_interval_job(handle, hours = hours, minutes = minutes, seconds = seconds) } + + def queue(self, handlers = None): + if not handlers: handlers = [] + + for h in handlers: + h() + + if self.shuttingDown(): + break + + return True diff --git a/couchpotato/core/_base/updater/__init__.py b/couchpotato/core/_base/updater/__init__.py index a304f9e7..7ad30d27 100644 --- a/couchpotato/core/_base/updater/__init__.py +++ b/couchpotato/core/_base/updater/__init__.py @@ -2,6 +2,7 @@ from .main import Updater from couchpotato.environment import Env import os + def start(): return Updater() diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index aecf0c4f..ef595ad7 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -15,6 +15,7 @@ import time import traceback import version import zipfile +from six.moves import filter log = CPLog(__name__) @@ -32,6 +33,7 @@ class Updater(Plugin): else: self.updater = SourceUpdater() + addEvent('app.load', self.logVersion, priority = 10000) addEvent('app.load', self.setCrons) addEvent('updater.info', self.info) @@ -53,12 +55,16 @@ class Updater(Plugin): addEvent('setting.save.updater.enabled.after', self.setCrons) + def logVersion(self): + info = self.info() + log.info('=== VERSION %s, using %s ===', (info.get('version', {}).get('repr', 'UNKNOWN'), self.updater.getName())) + def setCrons(self): fireEvent('schedule.remove', 'updater.check', single = True) if self.isEnabled(): fireEvent('schedule.interval', 'updater.check', self.autoUpdate, hours = 6) - self.autoUpdate() # Check after enabling + self.autoUpdate() # Check after enabling def autoUpdate(self): if self.isEnabled() and self.check() and self.conf('automatic') and not self.updater.update_failed: @@ -146,6 +152,9 @@ class BaseUpdater(Plugin): 'branch': self.branch, } + def getVersion(self): + pass + def check(self): pass @@ -174,7 +183,6 @@ class BaseUpdater(Plugin): log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc())) - class GitUpdater(BaseUpdater): def __init__(self, git_command): @@ -183,9 +191,6 @@ class GitUpdater(BaseUpdater): def doUpdate(self): try: - log.debug('Stashing local changes') - self.repo.saveStash() - log.info('Updating to latest version') self.repo.pull() @@ -204,14 +209,15 @@ class GitUpdater(BaseUpdater): if not self.version: try: - output = self.repo.getHead() # Yes, please + output = self.repo.getHead() # Yes, please log.debug('Git version output: %s', output.hash) self.version = { + 'repr': 'git:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.branch, output.hash[:8], datetime.fromtimestamp(output.getDate())), 'hash': output.hash[:8], 'date': output.getDate(), 'type': 'git', } - except Exception, e: + except Exception as e: log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e) return 'No GIT' @@ -234,7 +240,7 @@ class GitUpdater(BaseUpdater): local = self.repo.getHead() remote = branch.getHead() - log.info('Versions, local:%s, remote:%s', (local.hash[:8], remote.hash[:8])) + log.debug('Versions, local:%s, remote:%s', (local.hash[:8], remote.hash[:8])) if local.getDate() < remote.getDate(): self.update_version = { @@ -247,7 +253,6 @@ class GitUpdater(BaseUpdater): return False - class SourceUpdater(BaseUpdater): def __init__(self): @@ -273,9 +278,9 @@ class SourceUpdater(BaseUpdater): # Extract if download_data.get('type') == 'zip': - zip = zipfile.ZipFile(destination) - zip.extractall(extracted_path) - zip.close() + zip_file = zipfile.ZipFile(destination) + zip_file.extractall(extracted_path) + zip_file.close() else: tar = tarfile.open(destination) tar.extractall(path = extracted_path) @@ -342,13 +347,12 @@ class SourceUpdater(BaseUpdater): return True - def removeDir(self, path): try: if os.path.isdir(path): shutil.rmtree(path) - except OSError, inst: - os.chmod(inst.filename, 0777) + except OSError as inst: + os.chmod(inst.filename, 0o777) self.removeDir(path) def getVersion(self): @@ -362,7 +366,8 @@ class SourceUpdater(BaseUpdater): log.debug('Source version output: %s', output) self.version = output self.version['type'] = 'source' - except Exception, e: + self.version['repr'] = 'source:(%s:%s % s) %s (%s)' % (self.repo_user, self.repo_name, self.branch, output.get('hash', '')[:8], datetime.fromtimestamp(output.get('date', 0))) + except Exception as e: log.error('Failed using source updater. %s', e) return {} @@ -392,7 +397,7 @@ class SourceUpdater(BaseUpdater): return { 'hash': commit['sha'], - 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), + 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), } except: log.error('Failed getting latest request from github: %s', traceback.format_exc()) @@ -437,7 +442,7 @@ class DesktopUpdater(BaseUpdater): if latest and latest != current_version.get('hash'): self.update_version = { 'hash': latest, - 'date': None, + 'date': None, 'changelog': self.desktop._changelogURL, } @@ -449,6 +454,7 @@ class DesktopUpdater(BaseUpdater): def getVersion(self): return { + 'repr': 'desktop: %s' % self.desktop._esky.active_version, 'hash': self.desktop._esky.active_version, 'date': None, 'type': 'desktop', diff --git a/couchpotato/core/_base/updater/static/updater.js b/couchpotato/core/_base/updater/static/updater.js index 0577c783..be436ed2 100644 --- a/couchpotato/core/_base/updater/static/updater.js +++ b/couchpotato/core/_base/updater/static/updater.js @@ -24,7 +24,7 @@ var UpdaterBase = new Class({ self.doUpdate(); else { App.unBlockPage(); - App.fireEvent('message', 'No updates available'); + App.trigger('message', ['No updates available']); } } }) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 9e24d914..3bcf1f31 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -1,4 +1,5 @@ from base64 import b32decode, b16encode +from couchpotato.api import addApiView from couchpotato.core.event import addEvent from couchpotato.core.helpers.variable import mergeDicts from couchpotato.core.logger import CPLog @@ -13,6 +14,7 @@ class Downloader(Provider): protocol = [] http_time_between_calls = 0 + status_support = True torrent_sources = [ 'http://torrage.com/torrent/%s.torrent', @@ -41,6 +43,7 @@ class Downloader(Provider): addEvent('download.remove_failed', self._removeFailed) addEvent('download.pause', self._pause) addEvent('download.process_complete', self._processComplete) + addApiView('download.%s.test' % self.getName().lower(), self._test) def getEnabledProtocol(self): for download_protocol in self.protocol: @@ -49,22 +52,27 @@ class Downloader(Provider): return [] - def _download(self, data = None, movie = None, manual = False, filedata = None): - if not movie: movie = {} + def _download(self, data = None, media = None, manual = False, filedata = None): + if not media: media = {} if not data: data = {} if self.isDisabled(manual, data): return - return self.download(data = data, movie = movie, filedata = filedata) + return self.download(data = data, media = media, filedata = filedata) - def _getAllDownloadStatus(self): + def _getAllDownloadStatus(self, download_ids): if self.isDisabled(manual = True, data = {}): return - return self.getAllDownloadStatus() + ids = [download_id['id'] for download_id in download_ids if download_id['downloader'] == self.getName()] - def getAllDownloadStatus(self): - return + if ids: + return self.getAllDownloadStatus(ids) + else: + return + + def getAllDownloadStatus(self, ids): + return [] def _removeFailed(self, release_download): if self.isDisabled(manual = True, data = {}): @@ -128,6 +136,7 @@ class Downloader(Provider): def downloadReturnId(self, download_id): return { 'downloader': self.getName(), + 'status_support': self.status_support, 'id': download_id } @@ -151,6 +160,15 @@ class Downloader(Provider): (d_manual and manual or d_manual is False) and \ (not data or self.isCorrectProtocol(data.get('protocol'))) + def _test(self): + t = self.test() + if isinstance(t, tuple): + return {'success': t[0], 'msg': t[1]} + return {'success': t} + + def test(self): + return False + def _pause(self, release_download, pause = True): if self.isDisabled(manual = True, data = {}): return diff --git a/couchpotato/core/downloaders/blackhole/__init__.py b/couchpotato/core/downloaders/blackhole/__init__.py index 6b5279a1..92d18e7f 100644 --- a/couchpotato/core/downloaders/blackhole/__init__.py +++ b/couchpotato/core/downloaders/blackhole/__init__.py @@ -1,6 +1,7 @@ from .main import Blackhole from couchpotato.core.helpers.variable import getDownloadDir + def start(): return Blackhole() @@ -13,7 +14,7 @@ config = [{ 'list': 'download_providers', 'name': 'blackhole', 'label': 'Black hole', - 'description': 'Download the NZB/Torrent to a specific folder.', + 'description': 'Download the NZB/Torrent to a specific folder. Note: Seeding and copying/linking features do not work with Black hole.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index 854860cd..9a018354 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -1,5 +1,6 @@ from __future__ import with_statement from couchpotato.core.downloaders.base import Downloader +from couchpotato.core.helpers.encoding import sp from couchpotato.core.logger import CPLog from couchpotato.environment import Env import os @@ -11,9 +12,10 @@ log = CPLog(__name__) class Blackhole(Downloader): protocol = ['nzb', 'torrent', 'torrent_magnet'] + status_support = False - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} directory = self.conf('directory') @@ -33,7 +35,7 @@ class Blackhole(Downloader): log.error('No nzb/torrent available: %s', data.get('url')) return False - file_name = self.createFileName(data, filedata, movie) + file_name = self.createFileName(data, filedata, media) full_path = os.path.join(directory, file_name) if self.conf('create_subdir'): @@ -51,10 +53,10 @@ class Blackhole(Downloader): with open(full_path, 'wb') as f: f.write(filedata) os.chmod(full_path, Env.getPermission('file')) - return True + return self.downloadReturnId('') else: log.info('File %s already exists.', full_path) - return True + return self.downloadReturnId('') except: log.error('Failed to download to blackhole %s', traceback.format_exc()) @@ -66,6 +68,20 @@ class Blackhole(Downloader): return False + def test(self): + directory = self.conf('directory') + if directory and os.path.isdir(directory): + + test_file = sp(os.path.join(directory, 'couchpotato_test.txt')) + + # Check if folder is writable + self.createFile(test_file, 'This is a test file') + if os.path.isfile(test_file): + os.remove(test_file) + return True + + return False + def getEnabledProtocol(self): if self.conf('use_for') == 'both': return super(Blackhole, self).getEnabledProtocol() diff --git a/couchpotato/core/downloaders/deluge/__init__.py b/couchpotato/core/downloaders/deluge/__init__.py index c7aa26e6..09fae751 100644 --- a/couchpotato/core/downloaders/deluge/__init__.py +++ b/couchpotato/core/downloaders/deluge/__init__.py @@ -1,5 +1,6 @@ from .main import Deluge + def start(): return Deluge() diff --git a/couchpotato/core/downloaders/deluge/main.py b/couchpotato/core/downloaders/deluge/main.py index f3a1238f..59300958 100644 --- a/couchpotato/core/downloaders/deluge/main.py +++ b/couchpotato/core/downloaders/deluge/main.py @@ -2,7 +2,7 @@ from base64 import b64encode, b16encode, b32decode from bencode import bencode as benc, bdecode from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, sp -from couchpotato.core.helpers.variable import tryFloat +from couchpotato.core.helpers.variable import tryFloat, cleanHost from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 @@ -20,19 +20,22 @@ class Deluge(Downloader): log = CPLog(__name__) drpc = None - def connect(self): + def connect(self, reconnect = False): # Load host from config and split out port. - host = self.conf('host').split(':') + host = cleanHost(self.conf('host'), protocol = False).split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False - if not self.drpc: + if not self.drpc or reconnect: self.drpc = DelugeRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password')) return self.drpc - def download(self, data, movie, filedata = None): + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol'))) if not self.connect(): @@ -73,7 +76,7 @@ class Deluge(Downloader): if data.get('protocol') == 'torrent_magnet': remote_torrent = self.drpc.add_torrent_magnet(data.get('url'), options) else: - filename = self.createFileName(data, filedata, movie) + filename = self.createFileName(data, filedata, media) remote_torrent = self.drpc.add_torrent_file(filename, filedata, options) if not remote_torrent: @@ -83,24 +86,34 @@ class Deluge(Downloader): log.info('Torrent sent to Deluge successfully.') return self.downloadReturnId(remote_torrent) - def getAllDownloadStatus(self): + def test(self): + if self.connect(True) and self.drpc.test(): + return True + return False + + def getAllDownloadStatus(self, ids): log.debug('Checking Deluge download status.') if not self.connect(): - return False + return [] release_downloads = ReleaseDownloadList(self) - queue = self.drpc.get_alltorrents() + queue = self.drpc.get_alltorrents(ids) if not queue: log.debug('Nothing in queue or error') - return False + return [] for torrent_id in queue: torrent = queue[torrent_id] - log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) + + if not 'hash' in torrent: + # When given a list of ids, deluge will return an empty item for a non-existant torrent. + continue + + log.debug('name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_on_completed'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) # Deluge has no easy way to work out if a torrent is stalled or failing. #status = 'failed' @@ -149,6 +162,7 @@ class Deluge(Downloader): log.debug('Requesting Deluge to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) return self.drpc.remove_torrent(release_download['id'], remove_local_data = delete_files) + class DelugeRPC(object): host = 'localhost' @@ -169,6 +183,13 @@ class DelugeRPC(object): self.client = DelugeClient() self.client.connect(self.host, int(self.port), self.username, self.password) + def test(self): + try: + self.connect() + except: + return False + return True + def add_torrent_magnet(self, torrent, options): torrent_id = False try: @@ -179,7 +200,7 @@ class DelugeRPC(object): if torrent_id and options['label']: self.client.label.set_torrent(torrent_id, options['label']).get() - except Exception, err: + except Exception as err: log.error('Failed to add torrent magnet %s: %s %s', (torrent, err, traceback.format_exc())) finally: if self.client: @@ -197,7 +218,7 @@ class DelugeRPC(object): if torrent_id and options['label']: self.client.label.set_torrent(torrent_id, options['label']).get() - except Exception, err: + except Exception as err: log.error('Failed to add torrent file %s: %s %s', (filename, err, traceback.format_exc())) finally: if self.client: @@ -205,12 +226,12 @@ class DelugeRPC(object): return torrent_id - def get_alltorrents(self): + def get_alltorrents(self, ids): ret = False try: self.connect() - ret = self.client.core.get_torrents_status({}, {}).get() - except Exception, err: + ret = self.client.core.get_torrents_status({'id': ids}, ('name', 'hash', 'save_path', 'move_completed_path', 'progress', 'state', 'eta', 'ratio', 'stop_ratio', 'is_seed', 'is_finished', 'paused', 'move_on_completed', 'files')).get() + except Exception as err: log.error('Failed to get all torrents: %s %s', (err, traceback.format_exc())) finally: if self.client: @@ -221,7 +242,7 @@ class DelugeRPC(object): try: self.connect() self.client.core.pause_torrent(torrent_ids).get() - except Exception, err: + except Exception as err: log.error('Failed to pause torrent: %s %s', (err, traceback.format_exc())) finally: if self.client: @@ -231,7 +252,7 @@ class DelugeRPC(object): try: self.connect() self.client.core.resume_torrent(torrent_ids).get() - except Exception, err: + except Exception as err: log.error('Failed to resume torrent: %s %s', (err, traceback.format_exc())) finally: if self.client: @@ -242,7 +263,7 @@ class DelugeRPC(object): try: self.connect() ret = self.client.core.remove_torrent(torrent_id, remove_local_data).get() - except Exception, err: + except Exception as err: log.error('Failed to remove torrent: %s %s', (err, traceback.format_exc())) finally: if self.client: diff --git a/couchpotato/core/downloaders/nzbget/__init__.py b/couchpotato/core/downloaders/nzbget/__init__.py index 00763cfb..551eb42c 100644 --- a/couchpotato/core/downloaders/nzbget/__init__.py +++ b/couchpotato/core/downloaders/nzbget/__init__.py @@ -1,5 +1,6 @@ from .main import NZBGet + def start(): return NZBGet() @@ -25,6 +26,13 @@ config = [{ 'default': 'localhost:6789', 'description': 'Hostname with port. Usually localhost:6789', }, + { + 'name': 'ssl', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Use HyperText Transfer Protocol Secure, or https', + }, { 'name': 'username', 'default': 'nzbget', diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index f8506134..3dad8670 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -1,7 +1,7 @@ from base64 import standard_b64encode from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import ss, sp -from couchpotato.core.helpers.variable import tryInt, md5 +from couchpotato.core.helpers.variable import tryInt, md5, cleanHost from couchpotato.core.logger import CPLog from datetime import timedelta import re @@ -16,11 +16,10 @@ log = CPLog(__name__) class NZBGet(Downloader): protocol = ['nzb'] + rpc = 'xmlrpc' - url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc' - - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} if not filedata: @@ -29,10 +28,10 @@ class NZBGet(Downloader): log.info('Sending "%s" to NZBGet.', data.get('name')) - url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} - nzb_name = ss('%s.nzb' % self.createNzbName(data, movie)) + nzb_name = ss('%s.nzb' % self.createNzbName(data, media)) + + rpc = self.getRPC() - rpc = xmlrpclib.ServerProxy(url) try: if rpc.writelog('INFO', 'CouchPotato connected to drop off %s.' % nzb_name): log.debug('Successfully connected to NZBGet') @@ -41,7 +40,7 @@ class NZBGet(Downloader): except socket.error: log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') return False - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: if e.errcode == 401: log.error('Password is incorrect.') else: @@ -55,7 +54,7 @@ class NZBGet(Downloader): if xml_response: log.info('NZB sent successfully to NZBGet') - nzb_id = md5(data['url']) # about as unique as they come ;) + nzb_id = md5(data['url']) # about as unique as they come ;) couchpotato_id = "couchpotato=" + nzb_id groups = rpc.listgroups() file_id = [item['LastID'] for item in groups if item['NZBFilename'] == nzb_name] @@ -67,13 +66,32 @@ class NZBGet(Downloader): log.error('NZBGet could not add %s to the queue.', nzb_name) return False - def getAllDownloadStatus(self): + def test(self): + rpc = self.getRPC() + + try: + if rpc.writelog('INFO', 'CouchPotato connected to test connection'): + log.debug('Successfully connected to NZBGet') + else: + log.info('Successfully connected to NZBGet, but unable to send a message') + except socket.error: + log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') + return False + except xmlrpclib.ProtocolError as e: + if e.errcode == 401: + log.error('Password is incorrect.') + else: + log.error('Protocol Error: %s', e) + return False + + return True + + def getAllDownloadStatus(self, ids): log.debug('Checking NZBGet download status.') - url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} + rpc = self.getRPC() - rpc = xmlrpclib.ServerProxy(url) try: if rpc.writelog('INFO', 'CouchPotato connected to check status'): log.debug('Successfully connected to NZBGet') @@ -81,13 +99,13 @@ class NZBGet(Downloader): log.info('Successfully connected to NZBGet, but unable to send a message') except socket.error: log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') - return False - except xmlrpclib.ProtocolError, e: + return [] + except xmlrpclib.ProtocolError as e: if e.errcode == 401: log.error('Password is incorrect.') else: log.error('Protocol Error: %s', e) - return False + return [] # Get NZBGet data try: @@ -97,56 +115,59 @@ class NZBGet(Downloader): history = rpc.history() except: log.error('Failed getting data: %s', traceback.format_exc(1)) - return False + return [] release_downloads = ReleaseDownloadList(self) for nzb in groups: - log.debug('Found %s in NZBGet download queue', nzb['NZBFilename']) try: nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0] except: nzb_id = nzb['NZBID'] + if nzb_id in ids: + log.debug('Found %s in NZBGet download queue', nzb['NZBFilename']) + timeleft = -1 + try: + if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']): + timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) + except: + pass - timeleft = -1 - try: - if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']): - timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) - except: - pass - - release_downloads.append({ - 'id': nzb_id, - 'name': nzb['NZBFilename'], - 'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED', - # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item - 'timeleft': timeleft, - }) + release_downloads.append({ + 'id': nzb_id, + 'name': nzb['NZBFilename'], + 'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED', + # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item + 'timeleft': timeleft, + }) for nzb in queue: # 'Parameters' is not passed in rpc.postqueue - log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename']) - release_downloads.append({ - 'id': nzb['NZBID'], - 'name': nzb['NZBFilename'], - 'original_status': nzb['Stage'], - 'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1, - }) + if nzb['NZBID'] in ids: + log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename']) + release_downloads.append({ + 'id': nzb['NZBID'], + 'name': nzb['NZBFilename'], + 'original_status': nzb['Stage'], + 'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1, + }) for nzb in history: - log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log'])) try: nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0] except: nzb_id = nzb['NZBID'] - release_downloads.append({ - 'id': nzb_id, - 'name': nzb['NZBFilename'], - 'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed', - 'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'], - 'timeleft': str(timedelta(seconds = 0)), - 'folder': sp(nzb['DestDir']) - }) + + if nzb_id in ids: + log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log'])) + release_downloads.append({ + 'id': nzb_id, + 'name': nzb['NZBFilename'], + 'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed', + 'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'], + 'timeleft': str(timedelta(seconds = 0)), + 'folder': sp(nzb['DestDir']) + }) return release_downloads @@ -154,9 +175,8 @@ class NZBGet(Downloader): log.info('%s failed downloading, deleting...', release_download['name']) - url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} + rpc = self.getRPC() - rpc = xmlrpclib.ServerProxy(url) try: if rpc.writelog('INFO', 'CouchPotato connected to delete some history'): log.debug('Successfully connected to NZBGet') @@ -165,7 +185,7 @@ class NZBGet(Downloader): except socket.error: log.error('NZBGet is not responding. Please ensure that NZBGet is running and host setting is correct.') return False - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: if e.errcode == 401: log.error('Password is incorrect.') else: @@ -190,3 +210,7 @@ class NZBGet(Downloader): return False return True + + def getRPC(self): + url = cleanHost(host = self.conf('host'), ssl = self.conf('ssl'), username = self.conf('username'), password = self.conf('password')) + self.rpc + return xmlrpclib.ServerProxy(url) diff --git a/couchpotato/core/downloaders/nzbvortex/__init__.py b/couchpotato/core/downloaders/nzbvortex/__init__.py index 3b95698e..1c2d699e 100644 --- a/couchpotato/core/downloaders/nzbvortex/__init__.py +++ b/couchpotato/core/downloaders/nzbvortex/__init__.py @@ -1,5 +1,6 @@ from .main import NZBVortex + def start(): return NZBVortex() @@ -22,7 +23,15 @@ config = [{ }, { 'name': 'host', - 'default': 'https://localhost:4321', + 'default': 'localhost:4321', + 'description': 'Hostname with port. Usually localhost:4321', + }, + { + 'name': 'ssl', + 'default': 1, + 'type': 'bool', + 'advanced': True, + 'description': 'Use HyperText Transfer Protocol Secure, or https', }, { 'name': 'api_key', diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index f4e233be..d1525c89 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -8,9 +8,11 @@ from uuid import uuid4 import hashlib import httplib import json +import os import socket import ssl import sys +import time import traceback import urllib2 @@ -23,44 +25,54 @@ class NZBVortex(Downloader): api_level = None session_id = None - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} # Send the nzb try: - nzb_filename = self.createFileName(data, filedata, movie) - self.call('nzb/add', params = {'file': (nzb_filename, filedata)}, multipart = True) + nzb_filename = self.createFileName(data, filedata, media) + self.call('nzb/add', files = {'file': (nzb_filename, filedata)}) + time.sleep(10) raw_statuses = self.call('nzb') - nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if nzb['name'] == nzb_filename][0] + nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if os.path.basename(nzb['nzbFileName']) == nzb_filename][0] return self.downloadReturnId(nzb_id) except: log.error('Something went wrong sending the NZB file: %s', traceback.format_exc()) return False - def getAllDownloadStatus(self): + def test(self): + try: + login_result = self.login() + except: + return False + + return login_result + + def getAllDownloadStatus(self, ids): raw_statuses = self.call('nzb') release_downloads = ReleaseDownloadList(self) for nzb in raw_statuses.get('nzbs', []): + if nzb['id'] in ids: - # Check status - status = 'busy' - if nzb['state'] == 20: - status = 'completed' - elif nzb['state'] in [21, 22, 24]: - status = 'failed' + # Check status + status = 'busy' + if nzb['state'] == 20: + status = 'completed' + elif nzb['state'] in [21, 22, 24]: + status = 'failed' - release_downloads.append({ - 'id': nzb['id'], - 'name': nzb['uiTitle'], - 'status': status, - 'original_status': nzb['state'], - 'timeleft':-1, - 'folder': sp(nzb['destinationPath']), - }) + release_downloads.append({ + 'id': nzb['id'], + 'name': nzb['uiTitle'], + 'status': status, + 'original_status': nzb['state'], + 'timeleft': -1, + 'folder': sp(nzb['destinationPath']), + }) return release_downloads @@ -98,7 +110,6 @@ class NZBVortex(Downloader): log.error('Login failed, please check you api-key') return False - def call(self, call, parameters = None, repeat = False, auth = True, *args, **kwargs): # Login first @@ -112,15 +123,14 @@ class NZBVortex(Downloader): params = tryUrlencode(parameters) - url = cleanHost(self.conf('host')) + 'api/' + call - url_opener = urllib2.build_opener(HTTPSHandler()) + url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api/' + call try: - data = self.urlopen('%s?%s' % (url, params), opener = url_opener, *args, **kwargs) + data = self.urlopen('%s?%s' % (url, params), *args, **kwargs) if data: return json.loads(data) - except URLError, e: + except URLError as e: if hasattr(e, 'code') and e.code == 403: # Try login and do again if not repeat: @@ -138,12 +148,11 @@ class NZBVortex(Downloader): if not self.api_level: url = cleanHost(self.conf('host')) + 'api/app/apilevel' - url_opener = urllib2.build_opener(HTTPSHandler()) try: - data = self.urlopen(url, opener = url_opener, show_error = False) + data = self.urlopen(url, show_error = False) self.api_level = float(json.loads(data).get('apilevel')) - except URLError, e: + except URLError as e: if hasattr(e, 'code') and e.code == 403: log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher') else: @@ -173,6 +182,7 @@ class HTTPSConnection(httplib.HTTPSConnection): self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1) + class HTTPSHandler(urllib2.HTTPSHandler): def https_open(self, req): return self.do_open(HTTPSConnection, req) diff --git a/couchpotato/core/downloaders/pneumatic/__init__.py b/couchpotato/core/downloaders/pneumatic/__init__.py index 96574a7a..698643fb 100644 --- a/couchpotato/core/downloaders/pneumatic/__init__.py +++ b/couchpotato/core/downloaders/pneumatic/__init__.py @@ -1,5 +1,6 @@ from .main import Pneumatic + def start(): return Pneumatic() diff --git a/couchpotato/core/downloaders/pneumatic/main.py b/couchpotato/core/downloaders/pneumatic/main.py index 643350e1..bc1f6d04 100644 --- a/couchpotato/core/downloaders/pneumatic/main.py +++ b/couchpotato/core/downloaders/pneumatic/main.py @@ -1,5 +1,6 @@ from __future__ import with_statement from couchpotato.core.downloaders.base import Downloader +from couchpotato.core.helpers.encoding import sp from couchpotato.core.logger import CPLog import os import traceback @@ -11,9 +12,10 @@ class Pneumatic(Downloader): protocol = ['nzb'] strm_syntax = 'plugin://plugin.program.pneumatic/?mode=strm&type=add_file&nzb=%s&nzbname=%s' + status_support = False - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} directory = self.conf('directory') @@ -25,27 +27,27 @@ class Pneumatic(Downloader): log.error('No nzb available!') return False - fullPath = os.path.join(directory, self.createFileName(data, filedata, movie)) + full_path = os.path.join(directory, self.createFileName(data, filedata, media)) try: - if not os.path.isfile(fullPath): - log.info('Downloading %s to %s.', (data.get('protocol'), fullPath)) - with open(fullPath, 'wb') as f: + if not os.path.isfile(full_path): + log.info('Downloading %s to %s.', (data.get('protocol'), full_path)) + with open(full_path, 'wb') as f: f.write(filedata) - nzb_name = self.createNzbName(data, movie) + nzb_name = self.createNzbName(data, media) strm_path = os.path.join(directory, nzb_name) strm_file = open(strm_path + '.strm', 'wb') - strmContent = self.strm_syntax % (fullPath, nzb_name) + strmContent = self.strm_syntax % (full_path, nzb_name) strm_file.write(strmContent) strm_file.close() - return True + return self.downloadReturnId('') else: - log.info('File %s already exists.', fullPath) - return True + log.info('File %s already exists.', full_path) + return self.downloadReturnId('') except: log.error('Failed to download .strm: %s', traceback.format_exc()) @@ -55,3 +57,17 @@ class Pneumatic(Downloader): log.info('Failed to download file %s: %s', (data.get('name'), traceback.format_exc())) return False return False + + def test(self): + directory = self.conf('directory') + if directory and os.path.isdir(directory): + + test_file = sp(os.path.join(directory, 'couchpotato_test.txt')) + + # Check if folder is writable + self.createFile(test_file, 'This is a test file') + if os.path.isfile(test_file): + os.remove(test_file) + return True + + return False diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index 026a56c6..f793cad1 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -1,5 +1,6 @@ from .main import rTorrent + def start(): return rTorrent() @@ -20,11 +21,32 @@ config = [{ 'type': 'enabler', 'radio_group': 'torrent', }, +# @RuudBurger: How do I migrate this? +# { +# 'name': 'url', +# 'default': 'http://localhost:80/RPC2', +# 'description': 'XML-RPC Endpoint URI. Usually scgi://localhost:5000 ' +# 'or http://localhost:80/RPC2' +# }, { - 'name': 'url', - 'default': 'http://localhost:80/RPC2', - 'description': 'XML-RPC Endpoint URI. Usually scgi://localhost:5000 ' - 'or http://localhost:80/RPC2' + 'name': 'host', + 'default': 'localhost:80', + 'description': 'RPC Communication URI. Usually scgi://localhost:5000, ' + 'httprpc://localhost/rutorrent or localhost:80' + }, + { + 'name': 'ssl', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Use HyperText Transfer Protocol Secure, or https', + }, + { + 'name': 'rpc_url', + 'type': 'string', + 'default': 'RPC2', + 'advanced': True, + 'description': 'Change if your RPC mount is at a different path.', }, { 'name': 'username', @@ -58,14 +80,6 @@ config = [{ 'advanced': True, 'description': 'Also remove the leftover files.', }, - { - 'name': 'append_label', - 'label': 'Append Label', - 'default': False, - 'advanced': True, - 'type': 'bool', - 'description': 'Append label to download location. Requires you to set the download location above.', - }, { 'name': 'paused', 'type': 'bool', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index d7ae589f..08d34213 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -1,12 +1,15 @@ +from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList +from couchpotato.core.event import fireEvent, addEvent +from couchpotato.core.helpers.encoding import sp +from couchpotato.core.helpers.variable import cleanHost, splitString +from couchpotato.core.logger import CPLog from base64 import b16encode, b32decode from bencode import bencode, bdecode -from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList -from couchpotato.core.helpers.encoding import sp -from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 from rtorrent import RTorrent from rtorrent.err import MethodError +from urlparse import urlparse import os log = CPLog(__name__) @@ -16,29 +19,75 @@ class rTorrent(Downloader): protocol = ['torrent', 'torrent_magnet'] rt = None + error_msg = '' - def connect(self): + # Migration url to host options + def __init__(self): + super(rTorrent, self).__init__() + + addEvent('app.load', self.migrate) + addEvent('setting.save.rtorrent.*.after', self.settingsChanged) + + def migrate(self): + + url = self.conf('url') + if url: + host_split = splitString(url.split('://')[-1], split_on = '/') + + self.conf('ssl', value = url.startswith('https')) + self.conf('host', value = host_split[0].strip()) + self.conf('rpc_url', value = '/'.join(host_split[1:])) + + self.deleteConf('url') + + def settingsChanged(self): + # Reset active connection if settings have changed + if self.rt: + log.debug('Settings have changed, closing active connection') + + self.rt = None + return True + + def connect(self, reconnect = False): # Already connected? - if self.rt is not None: + if not reconnect and self.rt is not None: return self.rt - # Ensure url is set - if not self.conf('url'): - log.error('Config properties are not filled in correctly, url is missing.') - return False + url = cleanHost(self.conf('host'), protocol = True, ssl = self.conf('ssl')) + parsed = urlparse(url) + + # rpc_url is only used on http/https scgi pass-through + if parsed.scheme in ['http', 'https']: + url += self.conf('rpc_url') if self.conf('username') and self.conf('password'): self.rt = RTorrent( - self.conf('url'), + url, self.conf('username'), self.conf('password') ) else: - self.rt = RTorrent(self.conf('url')) + self.rt = RTorrent(url) + + self.error_msg = '' + try: + self.rt._verify_conn() + except AssertionError as e: + self.error_msg = e.message + self.rt = None return self.rt - def _update_provider_group(self, name, data): + def test(self): + if self.connect(True): + return True + + if self.error_msg: + return False, 'Connection failed: ' + self.error_msg + + return False + + def updateProviderGroup(self, name, data): if data.get('seed_time'): log.info('seeding time ignored, not supported') @@ -70,28 +119,30 @@ class rTorrent(Downloader): # Reset group action and disable it group.set_command() group.disable() - except MethodError, err: + except MethodError as err: log.error('Unable to set group options: %s', err.msg) return False return True - def download(self, data, movie, filedata = None): + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + log.debug('Sending "%s" to rTorrent.', (data.get('name'))) if not self.connect(): return False group_name = 'cp_' + data.get('provider').lower() - if not self._update_provider_group(group_name, data): + if not self.updateProviderGroup(group_name, data): return False torrent_params = {} if self.conf('label'): torrent_params['label'] = self.conf('label') - if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False @@ -115,7 +166,7 @@ class rTorrent(Downloader): # Send request to rTorrent try: # Send torrent to rTorrent - torrent = self.rt.load_torrent(filedata) + torrent = self.rt.load_torrent(filedata, verify_retries=10) if not torrent: log.error('Unable to find the torrent, did it fail to load?') @@ -125,9 +176,7 @@ class rTorrent(Downloader): if self.conf('label'): torrent.set_custom(1, self.conf('label')) - if self.conf('directory') and self.conf('append_label'): - torrent.set_directory(os.path.join(self.conf('directory'), self.conf('label'))) - elif self.conf('directory'): + if self.conf('directory'): torrent.set_directory(self.conf('directory')) # Set Ratio Group @@ -138,15 +187,30 @@ class rTorrent(Downloader): torrent.start() return self.downloadReturnId(torrent_hash) - except Exception, err: + except Exception as err: log.error('Failed to send torrent to rTorrent: %s', err) return False - def getAllDownloadStatus(self): + def getTorrentStatus(self, torrent): + if torrent.hashing or torrent.hash_checking or torrent.message: + return 'busy' + + if not torrent.complete: + return 'busy' + + if not torrent.open: + return 'completed' + + if torrent.state and torrent.active: + return 'seeding' + + return 'busy' + + def getAllDownloadStatus(self, ids): log.debug('Checking rTorrent download status.') if not self.connect(): - return False + return [] try: torrents = self.rt.get_torrents() @@ -154,33 +218,34 @@ class rTorrent(Downloader): release_downloads = ReleaseDownloadList(self) for torrent in torrents: - torrent_files = [] - for file_item in torrent.get_files(): - torrent_files.append(sp(os.path.join(torrent.directory, file_item.path))) + if torrent.info_hash in ids: + torrent_directory = os.path.normpath(torrent.directory) + torrent_files = [] - status = 'busy' - if torrent.complete: - if torrent.active: - status = 'seeding' - else: - status = 'completed' + for file in torrent.get_files(): + if not os.path.normpath(file.path).startswith(torrent_directory): + file_path = os.path.join(torrent_directory, file.path.lstrip('/')) + else: + file_path = file.path - release_downloads.append({ - 'id': torrent.info_hash, - 'name': torrent.name, - 'status': status, - 'seed_ratio': torrent.ratio, - 'original_status': torrent.state, - 'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1, - 'folder': sp(torrent.directory), - 'files': '|'.join(torrent_files) - }) + torrent_files.append(sp(file_path)) + + release_downloads.append({ + 'id': torrent.info_hash, + 'name': torrent.name, + 'status': self.getTorrentStatus(torrent), + 'seed_ratio': torrent.ratio, + 'original_status': torrent.state, + 'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1, + 'folder': sp(torrent.directory), + 'files': '|'.join(torrent_files) + }) return release_downloads - except Exception, err: + except Exception as err: log.error('Failed to get status from rTorrent: %s', err) - return False + return [] def pause(self, release_download, pause = True): if not self.connect(): diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 48692dae..2990078a 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -1,5 +1,6 @@ from .main import Sabnzbd + def start(): return Sabnzbd() @@ -24,6 +25,13 @@ config = [{ 'name': 'host', 'default': 'localhost:8080', }, + { + 'name': 'ssl', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Use HyperText Transfer Protocol Secure, or https', + }, { 'name': 'api_key', 'label': 'Api Key', diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index aba21231..ba58c090 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -16,8 +16,8 @@ class Sabnzbd(Downloader): protocol = ['nzb'] - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} log.info('Sending "%s" to SABnzbd.', data.get('name')) @@ -25,7 +25,7 @@ class Sabnzbd(Downloader): req_params = { 'cat': self.conf('category'), 'mode': 'addurl', - 'nzbname': self.createNzbName(data, movie), + 'nzbname': self.createNzbName(data, media), 'priority': self.conf('priority'), } @@ -36,14 +36,14 @@ class Sabnzbd(Downloader): return False # If it's a .rar, it adds the .rar extension, otherwise it stays .nzb - nzb_filename = self.createFileName(data, filedata, movie) + nzb_filename = self.createFileName(data, filedata, media) req_params['mode'] = 'addfile' else: req_params['name'] = data.get('url') try: if nzb_filename and req_params.get('mode') is 'addfile': - sab_data = self.call(req_params, params = {'nzbfile': (ss(nzb_filename), filedata)}, multipart = True) + sab_data = self.call(req_params, files = {'nzbfile': (ss(nzb_filename), filedata)}) else: sab_data = self.call(req_params) except URLError: @@ -64,7 +64,27 @@ class Sabnzbd(Downloader): log.error('Error getting data from SABNZBd: %s', sab_data) return False - def getAllDownloadStatus(self): + def test(self): + try: + sab_data = self.call({ + 'mode': 'version', + }) + v = sab_data.split('.') + if int(v[0]) == 0 and int(v[1]) < 7: + return False, 'Your Sabnzbd client is too old, please update to newest version.' + + # the version check will work even with wrong api key, so we need the next check as well + sab_data = self.call({ + 'mode': 'qstatus', + }) + if not sab_data: + return False + except: + return False + + return True + + def getAllDownloadStatus(self, ids): log.debug('Checking SABnzbd download status.') @@ -75,7 +95,7 @@ class Sabnzbd(Downloader): }) except: log.error('Failed getting queue: %s', traceback.format_exc(1)) - return False + return [] # Go through history items try: @@ -85,41 +105,42 @@ class Sabnzbd(Downloader): }) except: log.error('Failed getting history json: %s', traceback.format_exc(1)) - return False + return [] release_downloads = ReleaseDownloadList(self) # Get busy releases for nzb in queue.get('slots', []): - status = 'busy' - if 'ENCRYPTED / ' in nzb['filename']: - status = 'failed' + if nzb['nzo_id'] in ids: + status = 'busy' + if 'ENCRYPTED / ' in nzb['filename']: + status = 'failed' - release_downloads.append({ - 'id': nzb['nzo_id'], - 'name': nzb['filename'], - 'status': status, - 'original_status': nzb['status'], - 'timeleft': nzb['timeleft'] if not queue['paused'] else -1, - }) + release_downloads.append({ + 'id': nzb['nzo_id'], + 'name': nzb['filename'], + 'status': status, + 'original_status': nzb['status'], + 'timeleft': nzb['timeleft'] if not queue['paused'] else -1, + }) # Get old releases for nzb in history.get('slots', []): + if nzb['nzo_id'] in ids: + status = 'busy' + if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()): + status = 'failed' + elif nzb['status'] == 'Completed': + status = 'completed' - status = 'busy' - if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()): - status = 'failed' - elif nzb['status'] == 'Completed': - status = 'completed' - - release_downloads.append({ - 'id': nzb['nzo_id'], - 'name': nzb['name'], - 'status': status, - 'original_status': nzb['status'], - 'timeleft': str(timedelta(seconds = 0)), - 'folder': sp(os.path.dirname(nzb['storage']) if os.path.isfile(nzb['storage']) else nzb['storage']), - }) + release_downloads.append({ + 'id': nzb['nzo_id'], + 'name': nzb['name'], + 'status': status, + 'original_status': nzb['status'], + 'timeleft': str(timedelta(seconds = 0)), + 'folder': sp(os.path.dirname(nzb['storage']) if os.path.isfile(nzb['storage']) else nzb['storage']), + }) return release_downloads @@ -164,9 +185,9 @@ class Sabnzbd(Downloader): def call(self, request_params, use_json = True, **kwargs): - url = cleanHost(self.conf('host')) + 'api?' + tryUrlencode(mergeDicts(request_params, { - 'apikey': self.conf('api_key'), - 'output': 'json' + url = cleanHost(self.conf('host'), ssl = self.conf('ssl')) + 'api?' + tryUrlencode(mergeDicts(request_params, { + 'apikey': self.conf('api_key'), + 'output': 'json' })) data = self.urlopen(url, timeout = 60, show_error = False, headers = {'User-Agent': Env.getIdentifier()}, **kwargs) diff --git a/couchpotato/core/downloaders/synology/__init__.py b/couchpotato/core/downloaders/synology/__init__.py index 8be16f61..d0c57c2f 100644 --- a/couchpotato/core/downloaders/synology/__init__.py +++ b/couchpotato/core/downloaders/synology/__init__.py @@ -1,5 +1,6 @@ from .main import Synology + def start(): return Synology() diff --git a/couchpotato/core/downloaders/synology/main.py b/couchpotato/core/downloaders/synology/main.py index 0721085c..7e5b6098 100644 --- a/couchpotato/core/downloaders/synology/main.py +++ b/couchpotato/core/downloaders/synology/main.py @@ -1,5 +1,6 @@ from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt +from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog import json import requests @@ -11,17 +12,17 @@ log = CPLog(__name__) class Synology(Downloader): protocol = ['nzb', 'torrent', 'torrent_magnet'] - log = CPLog(__name__) + status_support = False - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} response = False log.error('Sending "%s" (%s) to Synology.', (data['name'], data['protocol'])) # Load host from config and split out port. - host = self.conf('host').split(':') + host = cleanHost(self.conf('host'), protocol = False).split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False @@ -42,7 +43,17 @@ class Synology(Downloader): except: log.error('Exception while adding torrent: %s', traceback.format_exc()) finally: - return response + return self.downloadReturnId('') if response else False + + def test(self): + host = cleanHost(self.conf('host'), protocol = False).split(':') + try: + srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password')) + test_result = srpc.test() + except: + return False + + return test_result def getEnabledProtocol(self): if self.conf('use_for') == 'both': @@ -64,6 +75,7 @@ class Synology(Downloader): return super(Synology, self).isEnabled(manual, data) and\ ((self.conf('use_for') in for_protocol)) + class SynologyRPC(object): """SynologyRPC lite library""" @@ -106,11 +118,11 @@ class SynologyRPC(object): if response['success']: log.info('Synology action successfull') return response - except requests.ConnectionError, err: + except requests.ConnectionError as err: log.error('Synology connection error, check your config %s', err) - except requests.HTTPError, err: + except requests.HTTPError as err: log.error('SynologyRPC HTTPError: %s', err) - except Exception, err: + except Exception as err: log.error('Exception: %s', err) finally: return response @@ -145,3 +157,6 @@ class SynologyRPC(object): self._logout() return result + + def test(self): + return bool(self._login()) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index f96e628e..4c9b4aad 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -1,5 +1,6 @@ from .main import Transmission + def start(): return Transmission() diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 2eabb2e8..4c42bf0f 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -1,7 +1,7 @@ from base64 import b64encode from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, sp -from couchpotato.core.helpers.variable import tryInt, tryFloat +from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost from couchpotato.core.logger import CPLog from datetime import timedelta import httplib @@ -19,19 +19,21 @@ class Transmission(Downloader): log = CPLog(__name__) trpc = None - def connect(self): + def connect(self, reconnect = False): # Load host from config and split out port. - host = self.conf('host').split(':') + host = cleanHost(self.conf('host'), protocol = False).split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False - if not self.trpc: - self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url'), username = self.conf('username'), password = self.conf('password')) + if not self.trpc or reconnect: + self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password')) return self.trpc - def download(self, data, movie, filedata = None): + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('protocol'))) @@ -81,12 +83,17 @@ class Transmission(Downloader): log.info('Torrent sent to Transmission successfully.') return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) - def getAllDownloadStatus(self): + def test(self): + if self.connect(True) and self.trpc.get_session(): + return True + return False + + def getAllDownloadStatus(self, ids): log.debug('Checking Transmission download status.') if not self.connect(): - return False + return [] release_downloads = ReleaseDownloadList(self) @@ -94,37 +101,44 @@ class Transmission(Downloader): 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit', 'files'] } + session = self.trpc.get_session() queue = self.trpc.get_alltorrents(return_params) if not (queue and queue.get('torrents')): log.debug('Nothing in queue or error') - return False + return [] for torrent in queue['torrents']: - log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', - (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent['eta'], torrent['uploadRatio'], torrent['isFinished'])) + if torrent['hashString'] in ids: + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / isStalled=%s / eta=%s / uploadRatio=%s / isFinished=%s / incomplete-dir-enabled=%s / incomplete-dir=%s', + (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent.get('isStalled', 'N/A'), torrent['eta'], torrent['uploadRatio'], torrent['isFinished'], session['incomplete-dir-enabled'], session['incomplete-dir'])) - torrent_files = [] - for file_item in torrent['files']: - torrent_files.append(sp(os.path.join(torrent['downloadDir'], file_item['name']))) + status = 'busy' + if torrent.get('isStalled') and not torrent['percentDone'] == 1 and self.conf('stalled_as_failed'): + status = 'failed' + elif torrent['status'] == 0 and torrent['percentDone'] == 1: + status = 'completed' + elif torrent['status'] in [5, 6]: + status = 'seeding' - status = 'busy' - if torrent.get('isStalled') and self.conf('stalled_as_failed'): - status = 'failed' - elif torrent['status'] == 0 and torrent['percentDone'] == 1: - status = 'completed' - elif torrent['status'] in [5, 6]: - status = 'seeding' + if session['incomplete-dir-enabled'] and status == 'busy': + torrent_folder = session['incomplete-dir'] + else: + torrent_folder = torrent['downloadDir'] - release_downloads.append({ - 'id': torrent['hashString'], - 'name': torrent['name'], - 'status': status, - 'original_status': torrent['status'], - 'seed_ratio': torrent['uploadRatio'], - 'timeleft': str(timedelta(seconds = torrent['eta'])), - 'folder': sp(torrent['downloadDir'] if len(torrent_files) == 1 else os.path.join(torrent['downloadDir'], torrent['name'])), - 'files': '|'.join(torrent_files) - }) + torrent_files = [] + for file_item in torrent['files']: + torrent_files.append(sp(os.path.join(torrent_folder, file_item['name']))) + + release_downloads.append({ + 'id': torrent['hashString'], + 'name': torrent['name'], + 'status': status, + 'original_status': torrent['status'], + 'seed_ratio': torrent['uploadRatio'], + 'timeleft': str(timedelta(seconds = torrent['eta'])), + 'folder': sp(torrent_folder if len(torrent_files) == 1 else os.path.join(torrent_folder, torrent['name'])), + 'files': '|'.join(torrent_files) + }) return release_downloads @@ -178,10 +192,10 @@ class TransmissionRPC(object): else: log.debug('Unknown failure sending command to Transmission. Return text is: %s', response['result']) return False - except httplib.InvalidURL, err: + except httplib.InvalidURL as err: log.error('Invalid Transmission host, check your config %s', err) return False - except urllib2.HTTPError, err: + except urllib2.HTTPError as err: if err.code == 401: log.error('Invalid Transmission Username or Password, check your config') return False @@ -199,7 +213,7 @@ class TransmissionRPC(object): log.error('Unable to get Transmission Session-Id %s', err) else: log.error('TransmissionRPC HTTPError: %s', err) - except urllib2.URLError, err: + except urllib2.URLError as err: log.error('Unable to connect to Transmission %s', err) def get_session(self): diff --git a/couchpotato/core/downloaders/utorrent/__init__.py b/couchpotato/core/downloaders/utorrent/__init__.py index d45e2e6c..da160956 100644 --- a/couchpotato/core/downloaders/utorrent/__init__.py +++ b/couchpotato/core/downloaders/utorrent/__init__.py @@ -1,5 +1,6 @@ from .main import uTorrent + def start(): return uTorrent() @@ -23,7 +24,7 @@ config = [{ { 'name': 'host', 'default': 'localhost:8000', - 'description': 'Hostname with port. Usually localhost:8000', + 'description': 'Port can be found in settings when enabling WebUI.', }, { 'name': 'username', diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 1db1b8a3..6a5e4257 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -2,7 +2,7 @@ from base64 import b16encode, b32decode from bencode import bencode as benc, bdecode from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, ss, sp -from couchpotato.core.helpers.variable import tryInt, tryFloat +from couchpotato.core.helpers.variable import tryInt, tryFloat, cleanHost from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 @@ -24,10 +24,20 @@ class uTorrent(Downloader): protocol = ['torrent', 'torrent_magnet'] utorrent_api = None + status_flags = { + 'STARTED' : 1, + 'CHECKING' : 2, + 'CHECK-START' : 4, + 'CHECKED' : 8, + 'ERROR' : 16, + 'PAUSED' : 32, + 'QUEUED' : 64, + 'LOADED' : 128 + } def connect(self): # Load host from config and split out port. - host = self.conf('host').split(':') + host = cleanHost(self.conf('host'), protocol = False).split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False @@ -36,11 +46,11 @@ class uTorrent(Downloader): return self.utorrent_api - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} - log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('protocol'))) + log.debug("Sending '%s' (%s) to uTorrent.", (data.get('name'), data.get('protocol'))) if not self.connect(): return False @@ -56,7 +66,7 @@ class uTorrent(Downloader): new_settings['seed_prio_limitul_flag'] = True log.info('Updated uTorrent settings to set a torrent to complete after it the seeding requirements are met.') - if settings.get('bt.read_only_on_complete'): #This doesn't work as this option seems to be not available through the api. Mitigated with removeReadOnly function + if settings.get('bt.read_only_on_complete'): #This doesn't work as this option seems to be not available through the api. Mitigated with removeReadOnly function new_settings['bt.read_only_on_complete'] = False log.info('Updated uTorrent settings to not set the files to read only after completing.') @@ -75,9 +85,10 @@ class uTorrent(Downloader): torrent_hash = re.findall('urn:btih:([\w]{32,40})', data.get('url'))[0].upper() torrent_params['trackers'] = '%0D%0A%0D%0A'.join(self.torrent_trackers) else: - info = bdecode(filedata)["info"] + info = bdecode(filedata)['info'] torrent_hash = sha1(benc(info)).hexdigest().upper() - torrent_filename = self.createFileName(data, filedata, movie) + + torrent_filename = self.createFileName(data, filedata, media) if data.get('seed_ratio'): torrent_params['seed_override'] = 1 @@ -104,72 +115,73 @@ class uTorrent(Downloader): return self.downloadReturnId(torrent_hash) - def getAllDownloadStatus(self): + def test(self): + if self.connect(): + build_version = self.utorrent_api.get_build() + if not build_version: + return False + if build_version < 25406: # This build corresponds to version 3.0.0 stable + return False, 'Your uTorrent client is too old, please update to newest version.' + return True + + return False + + def getAllDownloadStatus(self, ids): log.debug('Checking uTorrent download status.') if not self.connect(): - return False + return [] release_downloads = ReleaseDownloadList(self) data = self.utorrent_api.get_status() if not data: log.error('Error getting data from uTorrent') - return False + return [] queue = json.loads(data) if queue.get('error'): log.error('Error getting data from uTorrent: %s', queue.get('error')) - return False + return [] if not queue.get('torrents'): log.debug('Nothing in queue') - return False + return [] # Get torrents for torrent in queue['torrents']: + if torrent[0] in ids: - #Get files of the torrent - torrent_files = [] - try: - torrent_files = json.loads(self.utorrent_api.get_files(torrent[0])) - torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]] - except: - log.debug('Failed getting files from torrent: %s', torrent[2]) + #Get files of the torrent + torrent_files = [] + try: + torrent_files = json.loads(self.utorrent_api.get_files(torrent[0])) + torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]] + except: + log.debug('Failed getting files from torrent: %s', torrent[2]) - status_flags = { - "STARTED" : 1, - "CHECKING" : 2, - "CHECK-START" : 4, - "CHECKED" : 8, - "ERROR" : 16, - "PAUSED" : 32, - "QUEUED" : 64, - "LOADED" : 128 - } + status = 'busy' + if (torrent[1] & self.status_flags['STARTED'] or torrent[1] & self.status_flags['QUEUED']) and torrent[4] == 1000: + status = 'seeding' + elif (torrent[1] & self.status_flags['ERROR']): + status = 'failed' + elif torrent[4] == 1000: + status = 'completed' - status = 'busy' - if (torrent[1] & status_flags["STARTED"] or torrent[1] & status_flags["QUEUED"]) and torrent[4] == 1000: - status = 'seeding' - elif (torrent[1] & status_flags["ERROR"]): - status = 'failed' - elif torrent[4] == 1000: - status = 'completed' + if not status == 'busy': + self.removeReadOnly(torrent_files) - if not status == 'busy': - self.removeReadOnly(torrent_files) - - release_downloads.append({ - 'id': torrent[0], - 'name': torrent[2], - 'status': status, - 'seed_ratio': float(torrent[7]) / 1000, - 'original_status': torrent[1], - 'timeleft': str(timedelta(seconds = torrent[10])), - 'folder': sp(torrent[26]), - 'files': '|'.join(torrent_files) - }) + release_downloads.append({ + 'id': torrent[0], + 'name': torrent[2], + 'status': status, + 'seed_ratio': float(torrent[7]) / 1000, + 'original_status': torrent[1], + 'timeleft': str(timedelta(seconds = torrent[10])), + 'folder': sp(torrent[26]), + 'files': '|'.join(torrent_files) + }) return release_downloads @@ -222,7 +234,7 @@ class uTorrentAPI(object): if time.time() > self.last_time + 1800: self.last_time = time.time() self.token = self.get_token() - request = urllib2.Request(self.url + "?token=" + self.token + "&" + action, data) + request = urllib2.Request(self.url + '?token=' + self.token + '&' + action, data) try: open_request = self.opener.open(request) response = open_request.read() @@ -230,64 +242,64 @@ class uTorrentAPI(object): return response else: log.debug('Unknown failure sending command to uTorrent. Return text is: %s', response) - except httplib.InvalidURL, err: + except httplib.InvalidURL as err: log.error('Invalid uTorrent host, check your config %s', err) - except urllib2.HTTPError, err: + except urllib2.HTTPError as err: if err.code == 401: log.error('Invalid uTorrent Username or Password, check your config') else: log.error('uTorrent HTTPError: %s', err) - except urllib2.URLError, err: + except urllib2.URLError as err: log.error('Unable to connect to uTorrent %s', err) return False def get_token(self): - request = self.opener.open(self.url + "token.html") - token = re.findall("(.*?)(.*?) 1: @@ -37,13 +42,17 @@ def getParams(params): return dictToList(temp) + def dictToList(params): if type(params) is dict: new = {} - for x, value in params.iteritems(): + for x, value in params.items(): try: - new_value = [dictToList(value[k]) for k in sorted(value.iterkeys(), cmp = natcmp)] + convert = lambda text: int(text) if text.isdigit() else text.lower() + alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] + sorted_keys = sorted(value.keys(), key = alphanum_key) + new_value = [dictToList(value[k]) for k in sorted_keys] except: new_value = value diff --git a/couchpotato/core/helpers/rss.py b/couchpotato/core/helpers/rss.py index b840d862..1a4d37c2 100644 --- a/couchpotato/core/helpers/rss.py +++ b/couchpotato/core/helpers/rss.py @@ -3,6 +3,7 @@ import xml.etree.ElementTree as XMLTree log = CPLog(__name__) + class RSS(object): def getTextElements(self, xml, path): @@ -46,6 +47,6 @@ class RSS(object): def getItems(self, data, path = 'channel/item'): try: return XMLTree.parse(data).findall(path) - except Exception, e: + except Exception as e: log.error('Error parsing RSS. %s', e) return [] diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 7d35b997..8be6f299 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -2,29 +2,38 @@ from couchpotato.core.helpers.encoding import simplifyString, toSafeString, ss from couchpotato.core.logger import CPLog import collections import hashlib -import os.path +import os import platform import random import re import string import sys +import six +from six.moves import map, zip, filter log = CPLog(__name__) + +def fnEscape(pattern): + return pattern.replace('[', '[[').replace(']', '[]]').replace('[[', '[[]') + + def link(src, dst): if os.name == 'nt': import ctypes - if ctypes.windll.kernel32.CreateHardLinkW(unicode(dst), unicode(src), 0) == 0: raise ctypes.WinError() + if ctypes.windll.kernel32.CreateHardLinkW(six.text_type(dst), six.text_type(src), 0) == 0: raise ctypes.WinError() else: os.link(src, dst) + def symlink(src, dst): if os.name == 'nt': import ctypes - if ctypes.windll.kernel32.CreateSymbolicLinkW(unicode(dst), unicode(src), 1 if os.path.isdir(src) else 0) in [0, 1280]: raise ctypes.WinError() + if ctypes.windll.kernel32.CreateSymbolicLinkW(six.text_type(dst), six.text_type(src), 1 if os.path.isdir(src) else 0) in [0, 1280]: raise ctypes.WinError() else: os.symlink(src, dst) + def getUserDir(): try: import pwd @@ -34,6 +43,7 @@ def getUserDir(): return os.path.expanduser('~') + def getDownloadDir(): user_dir = getUserDir() @@ -46,6 +56,7 @@ def getDownloadDir(): return user_dir + def getDataDir(): # Windows @@ -65,8 +76,10 @@ def getDataDir(): # Linux return os.path.join(user_dir, '.couchpotato') -def isDict(object): - return isinstance(object, dict) + +def isDict(obj): + return isinstance(obj, dict) + def mergeDicts(a, b, prepend_list = False): assert isDict(a), isDict(b) @@ -88,6 +101,7 @@ def mergeDicts(a, b, prepend_list = False): current_dst[key] = current_src[key] return dst + def removeListDuplicates(seq): checked = [] for e in seq: @@ -95,35 +109,73 @@ def removeListDuplicates(seq): checked.append(e) return checked + def flattenList(l): if isinstance(l, list): return sum(map(flattenList, l)) else: return l + def md5(text): return hashlib.md5(ss(text)).hexdigest() + def sha1(text): return hashlib.sha1(text).hexdigest() + def isLocalIP(ip): ip = ip.lstrip('htps:/') regex = '/(^127\.)|(^192\.168\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^::1)$/' return re.search(regex, ip) is not None or 'localhost' in ip or ip[:4] == '127.' + def getExt(filename): return os.path.splitext(filename)[1][1:] -def cleanHost(host): - if not host.startswith(('http://', 'https://')): - host = 'http://' + host - host = host.rstrip('/') - host += '/' +def cleanHost(host, protocol = True, ssl = False, username = None, password = None): + """Return a cleaned up host with given url options set + + Changes protocol to https if ssl is set to True and http if ssl is set to false. + >>> cleanHost("localhost:80", ssl=True) + 'https://localhost:80/' + >>> cleanHost("localhost:80", ssl=False) + 'http://localhost:80/' + + Username and password is managed with the username and password variables + >>> cleanHost("localhost:80", username="user", password="passwd") + 'http://user:passwd@localhost:80/' + + Output without scheme (protocol) can be forced with protocol=False + >>> cleanHost("localhost:80", protocol=False) + 'localhost:80' + """ + + if not '://' in host and protocol: + host = ('https://' if ssl else 'http://') + host + + if not protocol: + host = host.split('://', 1)[-1] + + if protocol and username and password: + try: + auth = re.findall('^(?:.+?//)(.+?):(.+?)@(?:.+)$', host) + if auth: + log.error('Cleanhost error: auth already defined in url: %s, please remove BasicAuth from url.', host) + else: + host = host.replace('://', '://%s:%s@' % (username, password), 1) + except: + pass + + host = host.rstrip('/ ') + if protocol: + host += '/' return host + def getImdb(txt, check_inside = False, multiple = False): if not check_inside: @@ -140,7 +192,7 @@ def getImdb(txt, check_inside = False, multiple = False): ids = re.findall('(tt\d{4,7})', txt) if multiple: - return list(set(['tt%07d' % tryInt(x[2:]) for x in ids])) if len(ids) > 0 else [] + return removeDuplicate(['tt%07d' % tryInt(x[2:]) for x in ids]) if len(ids) > 0 else [] return 'tt%07d' % tryInt(ids[0][2:]) except IndexError: @@ -148,10 +200,12 @@ def getImdb(txt, check_inside = False, multiple = False): return False + def tryInt(s, default = 0): try: return int(s) except: return default + def tryFloat(s): try: if isinstance(s, str): @@ -160,17 +214,17 @@ def tryFloat(s): return float(s) except: return 0 -def natsortKey(s): - return map(tryInt, re.findall(r'(\d+|\D+)', s)) +def natsortKey(string_): + """See http://www.codinghorror.com/blog/archives/001018.html""" + return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] -def natcmp(a, b): - return cmp(natsortKey(a), natsortKey(b)) def toIterable(value): if isinstance(value, collections.Iterable): return value return [value] + def getTitle(library_dict): try: try: @@ -193,6 +247,7 @@ def getTitle(library_dict): log.error('Could not get title for library item: %s', library_dict) return None + def possibleTitles(raw_title): titles = [ @@ -205,14 +260,42 @@ def possibleTitles(raw_title): new_title = raw_title.replace('&', 'and') titles.append(simplifyString(new_title)) - return list(set(titles)) + return removeDuplicate(titles) + def randomString(size = 8, chars = string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for x in range(size)) + def splitString(str, split_on = ',', clean = True): - list = [x.strip() for x in str.split(split_on)] if str else [] - return filter(None, list) if clean else list + l = [x.strip() for x in str.split(split_on)] if str else [] + return removeEmpty(l) if clean else l + + +def removeEmpty(l): + return list(filter(None, l)) + + +def removeDuplicate(l): + seen = set() + return [x for x in l if x not in seen and not seen.add(x)] + def dictIsSubset(a, b): return all([k in b and b[k] == v for k, v in a.items()]) + + +def isSubFolder(sub_folder, base_folder): + # Returns True if sub_folder is the same as or inside base_folder + return base_folder and sub_folder and ss(os.path.normpath(base_folder).rstrip(os.path.sep) + os.path.sep) in ss(os.path.normpath(sub_folder).rstrip(os.path.sep) + os.path.sep) + +# From SABNZBD +re_password = [re.compile(r'([^/\\]+)[/\\](.+)'), re.compile(r'(.+){{([^{}]+)}}$'), re.compile(r'(.+)\s+password\s*=\s*(.+)$', re.I)] +def scanForPassword(name): + m = None + for reg in re_password: + m = reg.search(name) + if m: break + + if m: + return m.group(1).strip('. '), m.group(2).strip() diff --git a/couchpotato/core/loader.py b/couchpotato/core/loader.py index c14b55bd..6c3a719d 100644 --- a/couchpotato/core/loader.py +++ b/couchpotato/core/loader.py @@ -1,9 +1,10 @@ from couchpotato.core.event import fireEvent from couchpotato.core.logger import CPLog -from importlib import import_module +from importhelper import import_module import os import sys import traceback +import six log = CPLog(__name__) @@ -37,7 +38,7 @@ class Loader(object): self.paths['custom_plugins'] = (30, '', custom_plugin_dir) # Loop over all paths and add to module list - for plugin_type, plugin_tuple in self.paths.iteritems(): + for plugin_type, plugin_tuple in self.paths.items(): priority, module, dir_name = plugin_tuple self.addFromDir(plugin_type, priority, module, dir_name) @@ -45,7 +46,7 @@ class Loader(object): did_save = 0 for priority in sorted(self.modules): - for module_name, plugin in sorted(self.modules[priority].iteritems()): + for module_name, plugin in sorted(self.modules[priority].items()): # Load module try: @@ -81,7 +82,7 @@ class Loader(object): for filename in os.listdir(root_path): path = os.path.join(root_path, filename) if os.path.isdir(path) and filename[:2] != '__': - if u'__init__.py' in os.listdir(path): + if six.u('__init__.py') in os.listdir(path): new_base_path = ''.join(s + '.' for s in base_path) + filename self.paths[new_base_path.replace('.', '_')] = (priority, new_base_path, path) diff --git a/couchpotato/core/logger.py b/couchpotato/core/logger.py index 69a031f1..8223f146 100644 --- a/couchpotato/core/logger.py +++ b/couchpotato/core/logger.py @@ -1,6 +1,7 @@ import logging import re + class CPLog(object): context = '' @@ -37,7 +38,7 @@ class CPLog(object): def safeMessage(self, msg, replace_tuple = ()): from couchpotato.environment import Env - from couchpotato.core.helpers.encoding import ss + from couchpotato.core.helpers.encoding import ss, toUnicode msg = ss(msg) @@ -49,8 +50,8 @@ class CPLog(object): msg = msg % tuple([ss(x) for x in list(replace_tuple)]) else: msg = msg % ss(replace_tuple) - except Exception, e: - self.logger.error(u'Failed encoding stuff to log "%s": %s' % (msg, e)) + except Exception as e: + self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e)) if not Env.get('dev'): @@ -66,4 +67,4 @@ class CPLog(object): except: pass - return msg + return toUnicode(msg) diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index e6a249d5..cf9302b1 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -1,8 +1,11 @@ -from couchpotato import get_session +import traceback +from couchpotato import get_session, CPLog from couchpotato.core.event import addEvent, fireEventAsync, fireEvent from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Media +log = CPLog(__name__) + class MediaBase(Plugin): @@ -10,8 +13,8 @@ class MediaBase(Plugin): default_dict = { 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}, 'files':{}, 'info': {}}, - 'library': {'titles': {}, 'files':{}}, + 'releases': {'status': {}, 'quality': {}, 'files': {}, 'info': {}}, + 'library': {'titles': {}, 'files': {}}, 'files': {}, 'status': {}, 'category': {}, @@ -26,19 +29,33 @@ class MediaBase(Plugin): def createOnComplete(self, id): def onComplete(): - db = get_session() - media = db.query(Media).filter_by(id = id).first() - fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.default_dict), on_complete = self.createNotifyFront(id)) - db.expire_all() + try: + db = get_session() + media = db.query(Media).filter_by(id = id).first() + media_dict = media.to_dict(self.default_dict) + event_name = '%s.searcher.single' % media.type + + fireEvent(event_name, media_dict, on_complete = self.createNotifyFront(id)) + except: + log.error('Failed creating onComplete: %s', traceback.format_exc()) + finally: + db.close() return onComplete def createNotifyFront(self, media_id): def notifyFront(): - db = get_session() - media = db.query(Media).filter_by(id = media_id).first() - fireEvent('notify.frontend', type = '%s.update.%s' % (media.type, media.id), data = media.to_dict(self.default_dict)) - db.expire_all() + try: + db = get_session() + media = db.query(Media).filter_by(id = media_id).first() + media_dict = media.to_dict(self.default_dict) + event_name = '%s.update' % media.type + + fireEvent('notify.frontend', type = event_name, data = media_dict) + except: + log.error('Failed creating onComplete: %s', traceback.format_exc()) + finally: + db.close() return notifyFront diff --git a/couchpotato/core/media/_base/media/__init__.py b/couchpotato/core/media/_base/media/__init__.py index a9693a3d..e5f5a0ec 100644 --- a/couchpotato/core/media/_base/media/__init__.py +++ b/couchpotato/core/media/_base/media/__init__.py @@ -1,5 +1,6 @@ from .main import MediaPlugin + def start(): return MediaPlugin() diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 87afb82a..cbcd4245 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -1,10 +1,16 @@ -from couchpotato import get_session +import traceback +from couchpotato import get_session, tryInt from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent -from couchpotato.core.helpers.variable import splitString +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import mergeDicts, splitString, getImdb, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.media import MediaBase -from couchpotato.core.settings.model import Media +from couchpotato.core.settings.model import Library, LibraryTitle, Release, \ + Media +from sqlalchemy.orm import joinedload_all +from sqlalchemy.sql.expression import or_, asc, not_, desc +from string import ascii_lowercase log = CPLog(__name__) @@ -20,30 +26,455 @@ class MediaPlugin(MediaBase): } }) - addEvent('app.load', self.addSingleRefresh) + addApiView('media.list', self.listView, docs = { + 'desc': 'List media', + 'params': { + 'type': {'type': 'string', 'desc': 'Media type to filter on.'}, + 'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'}, + 'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'}, + 'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'}, + 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'}, + 'search': {'desc': 'Search movie title'}, + }, + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'empty': bool, any movies returned or not, + 'media': array, media found, +}"""} + }) + + addApiView('media.get', self.getView, docs = { + 'desc': 'Get media by id', + 'params': { + 'id': {'desc': 'The id of the media'}, + } + }) + + addApiView('media.delete', self.deleteView, docs = { + 'desc': 'Delete a media from the wanted list', + 'params': { + 'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'}, + 'delete_from': {'desc': 'Delete media from this page', 'type': 'string: all (default), wanted, manage'}, + } + }) + + addApiView('media.available_chars', self.charView) + + addEvent('app.load', self.addSingleRefreshView) + addEvent('app.load', self.addSingleListView) + addEvent('app.load', self.addSingleCharView) + addEvent('app.load', self.addSingleDeleteView) + + addEvent('media.get', self.get) + addEvent('media.list', self.list) + addEvent('media.delete', self.delete) + addEvent('media.restatus', self.restatus) def refresh(self, id = '', **kwargs): - db = get_session() + handlers = [] + ids = splitString(id) - for x in splitString(id): - media = db.query(Media).filter_by(id = x).first() + for x in ids: - if media: - # Get current selected title - default_title = '' - for title in media.library.titles: - if title.default: default_title = title.title + refresh_handler = self.createRefreshHandler(x) + if refresh_handler: + handlers.append(refresh_handler) - fireEvent('notify.frontend', type = '%s.busy.%s' % (media.type, x), data = True) - fireEventAsync('library.update.%s' % media.type, identifier = media.library.identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(x)) - - db.expire_all() + fireEvent('notify.frontend', type = 'media.busy', data = {'id': [tryInt(x) for x in ids]}) + fireEventAsync('schedule.queue', handlers = handlers) return { 'success': True, } - def addSingleRefresh(self): + def createRefreshHandler(self, id): + db = get_session() + + media = db.query(Media).filter_by(id = id).first() + + if media: + + default_title = getTitle(media.library) + identifier = media.library.identifier + event = 'library.update.%s' % media.type + + def handler(): + fireEvent(event, identifier = identifier, default_title = default_title, on_complete = self.createOnComplete(id)) + + if handler: + return handler + + def addSingleRefreshView(self): for media_type in fireEvent('media.types', merge = True): addApiView('%s.refresh' % media_type, self.refresh) + + def get(self, media_id): + + db = get_session() + + imdb_id = getImdb(str(media_id)) + + if imdb_id: + m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first() + else: + m = db.query(Media).filter_by(id = media_id).first() + + results = None + if m: + results = m.to_dict(self.default_dict) + + return results + + def getView(self, id = None, **kwargs): + + media = self.get(id) if id else None + + return { + 'success': media is not None, + 'media': media, + } + + def list(self, types = None, status = None, release_status = None, limit_offset = None, starts_with = None, search = None, order = None): + + db = get_session() + + # Make a list from string + if status and not isinstance(status, (list, tuple)): + status = [status] + if release_status and not isinstance(release_status, (list, tuple)): + release_status = [release_status] + if types and not isinstance(types, (list, tuple)): + types = [types] + + # query movie ids + q = db.query(Media) \ + .with_entities(Media.id) \ + .group_by(Media.id) + + # Filter on movie status + if status and len(status) > 0: + statuses = fireEvent('status.get', status, single = len(status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.filter(Media.status_id.in_(statuses)) + + # Filter on release status + if release_status and len(release_status) > 0: + q = q.join(Media.releases) + + statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.filter(Release.status_id.in_(statuses)) + + # Filter on type + if types and len(types) > 0: + try: q = q.filter(Media.type.in_(types)) + except: pass + + # Only join when searching / ordering + if starts_with or search or order != 'release_order': + q = q.join(Media.library, Library.titles) \ + .filter(LibraryTitle.default == True) + + # Add search filters + filter_or = [] + if starts_with: + starts_with = toUnicode(starts_with.lower()) + if starts_with in ascii_lowercase: + filter_or.append(LibraryTitle.simple_title.startswith(starts_with)) + else: + ignore = [] + for letter in ascii_lowercase: + ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter))) + filter_or.append(not_(or_(*ignore))) + + if search: + filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%')) + + if len(filter_or) > 0: + q = q.filter(or_(*filter_or)) + + total_count = q.count() + if total_count == 0: + return 0, [] + + if order == 'release_order': + q = q.order_by(desc(Release.last_edit)) + else: + q = q.order_by(asc(LibraryTitle.simple_title)) + + if limit_offset: + splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset + limit = splt[0] + offset = 0 if len(splt) is 1 else splt[1] + q = q.limit(limit).offset(offset) + + # Get all media_ids in sorted order + media_ids = [m.id for m in q.all()] + + # List release statuses + releases = db.query(Release) \ + .filter(Release.movie_id.in_(media_ids)) \ + .all() + + release_statuses = dict((m, set()) for m in media_ids) + releases_count = dict((m, 0) for m in media_ids) + for release in releases: + release_statuses[release.movie_id].add('%d,%d' % (release.status_id, release.quality_id)) + releases_count[release.movie_id] += 1 + + # Get main movie data + q2 = db.query(Media) \ + .options(joinedload_all('library.titles')) \ + .options(joinedload_all('library.files')) \ + .options(joinedload_all('status')) \ + .options(joinedload_all('files')) + + q2 = q2.filter(Media.id.in_(media_ids)) + + results = q2.all() + + # Create dict by movie id + movie_dict = {} + for movie in results: + movie_dict[movie.id] = movie + + # List movies based on media_ids order + movies = [] + for media_id in media_ids: + + releases = [] + for r in release_statuses.get(media_id): + x = splitString(r) + releases.append({'status_id': x[0], 'quality_id': x[1]}) + + # Merge releases with movie dict + movies.append(mergeDicts(movie_dict[media_id].to_dict({ + 'library': {'titles': {}, 'files': {}}, + 'files': {}, + }), { + 'releases': releases, + 'releases_count': releases_count.get(media_id), + })) + + return total_count, movies + + def listView(self, **kwargs): + + types = splitString(kwargs.get('types')) + status = splitString(kwargs.get('status')) + release_status = splitString(kwargs.get('release_status')) + limit_offset = kwargs.get('limit_offset') + starts_with = kwargs.get('starts_with') + search = kwargs.get('search') + order = kwargs.get('order') + + total_movies, movies = self.list( + types = types, + status = status, + release_status = release_status, + limit_offset = limit_offset, + starts_with = starts_with, + search = search, + order = order + ) + + return { + 'success': True, + 'empty': len(movies) == 0, + 'total': total_movies, + 'movies': movies, + } + + def addSingleListView(self): + + for media_type in fireEvent('media.types', merge = True): + def tempList(*args, **kwargs): + return self.listView(types = media_type, *args, **kwargs) + addApiView('%s.list' % media_type, tempList) + + def availableChars(self, types = None, status = None, release_status = None): + + types = types or [] + status = status or [] + release_status = release_status or [] + + db = get_session() + + # Make a list from string + if not isinstance(status, (list, tuple)): + status = [status] + if release_status and not isinstance(release_status, (list, tuple)): + release_status = [release_status] + if types and not isinstance(types, (list, tuple)): + types = [types] + + q = db.query(Media) + + # Filter on movie status + if status and len(status) > 0: + statuses = fireEvent('status.get', status, single = len(release_status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.filter(Media.status_id.in_(statuses)) + + # Filter on release status + if release_status and len(release_status) > 0: + + statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.join(Media.releases) \ + .filter(Release.status_id.in_(statuses)) + + # Filter on type + if types and len(types) > 0: + try: q = q.filter(Media.type.in_(types)) + except: pass + + q = q.join(Library, LibraryTitle) \ + .with_entities(LibraryTitle.simple_title) \ + .filter(LibraryTitle.default == True) + + titles = q.all() + + chars = set() + for title in titles: + try: + char = title[0][0] + char = char if char in ascii_lowercase else '#' + chars.add(str(char)) + except: + log.error('Failed getting title for %s', title.libraries_id) + + if len(chars) == 25: + break + + return ''.join(sorted(chars)) + + def charView(self, **kwargs): + + type = splitString(kwargs.get('type', 'movie')) + status = splitString(kwargs.get('status', None)) + release_status = splitString(kwargs.get('release_status', None)) + chars = self.availableChars(type, status, release_status) + + return { + 'success': True, + 'empty': len(chars) == 0, + 'chars': chars, + } + + def addSingleCharView(self): + + for media_type in fireEvent('media.types', merge = True): + def tempChar(*args, **kwargs): + return self.charView(types = media_type, *args, **kwargs) + addApiView('%s.available_chars' % media_type, tempChar) + + def delete(self, media_id, delete_from = None): + + try: + db = get_session() + + media = db.query(Media).filter_by(id = media_id).first() + if media: + deleted = False + if delete_from == 'all': + db.delete(media) + db.commit() + deleted = True + else: + done_status = fireEvent('status.get', 'done', single = True) + + total_releases = len(media.releases) + total_deleted = 0 + new_movie_status = None + for release in media.releases: + if delete_from in ['wanted', 'snatched', 'late']: + if release.status_id != done_status.get('id'): + db.delete(release) + total_deleted += 1 + new_movie_status = 'done' + elif delete_from == 'manage': + if release.status_id == done_status.get('id'): + db.delete(release) + total_deleted += 1 + new_movie_status = 'active' + db.commit() + + if total_releases == total_deleted: + db.delete(media) + db.commit() + deleted = True + elif new_movie_status: + new_status = fireEvent('status.get', new_movie_status, single = True) + media.profile_id = None + media.status_id = new_status.get('id') + db.commit() + else: + fireEvent('media.restatus', media.id, single = True) + + if deleted: + fireEvent('notify.frontend', type = 'movie.deleted', data = media.to_dict()) + except: + log.error('Failed deleting media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return True + + def deleteView(self, id = '', **kwargs): + + ids = splitString(id) + for media_id in ids: + self.delete(media_id, delete_from = kwargs.get('delete_from', 'all')) + + return { + 'success': True, + } + + def addSingleDeleteView(self): + + for media_type in fireEvent('media.types', merge = True): + def tempDelete(*args, **kwargs): + return self.deleteView(types = media_type, *args, **kwargs) + addApiView('%s.delete' % media_type, tempDelete) + + def restatus(self, media_id): + + active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True) + + try: + db = get_session() + + m = db.query(Media).filter_by(id = media_id).first() + if not m or len(m.library.titles) == 0: + log.debug('Can\'t restatus movie, doesn\'t seem to exist.') + return False + + log.debug('Changing status for %s', m.library.titles[0].title) + if not m.profile: + m.status_id = done_status.get('id') + else: + move_to_wanted = True + + for t in m.profile.types: + for release in m.releases: + if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): + move_to_wanted = False + + m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') + + db.commit() + + return True + except: + log.error('Failed restatus: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + diff --git a/couchpotato/core/media/_base/search/__init__.py b/couchpotato/core/media/_base/search/__init__.py index 4b2eae27..09bc84ef 100644 --- a/couchpotato/core/media/_base/search/__init__.py +++ b/couchpotato/core/media/_base/search/__init__.py @@ -1,5 +1,6 @@ from .main import Search + def start(): return Search() diff --git a/couchpotato/core/media/_base/searcher/__init__.py b/couchpotato/core/media/_base/searcher/__init__.py index 5e029a25..72c7d6ef 100644 --- a/couchpotato/core/media/_base/searcher/__init__.py +++ b/couchpotato/core/media/_base/searcher/__init__.py @@ -1,5 +1,6 @@ from .main import Searcher + def start(): return Searcher() diff --git a/couchpotato/core/media/_base/searcher/base.py b/couchpotato/core/media/_base/searcher/base.py index 368c6e2d..5322d850 100644 --- a/couchpotato/core/media/_base/searcher/base.py +++ b/couchpotato/core/media/_base/searcher/base.py @@ -12,7 +12,6 @@ class SearcherBase(Plugin): def __init__(self): super(SearcherBase, self).__init__() - addEvent('searcher.progress', self.getProgress) addEvent('%s.searcher.progress' % self.getType(), self.getProgress) @@ -26,9 +25,8 @@ class SearcherBase(Plugin): _type = self.getType() def setCrons(): - fireEvent('schedule.cron', '%s.searcher.all' % _type, self.searchAll, - day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) addEvent('app.load', setCrons) addEvent('setting.save.%s_searcher.cron_day.after' % _type, setCrons) diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index 3c73eb27..e7209b60 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -1,7 +1,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import simplifyString -from couchpotato.core.helpers.variable import splitString +from couchpotato.core.helpers.variable import splitString, removeEmpty, removeDuplicate from couchpotato.core.logger import CPLog from couchpotato.core.media._base.searcher.base import SearcherBase import datetime @@ -107,10 +107,10 @@ class Searcher(SearcherBase): # Hack for older movies that don't contain quality tag year_name = fireEvent('scanner.name_year', name, single = True) if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): - if size > 3000: # Assume dvdr + if size > 3000: # Assume dvdr log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', size) found['dvdr'] = True - else: # Assume dvdrip + else: # Assume dvdrip log.info('Quality was missing in name, assuming it\'s a DVD-Rip based on the size: %s', size) found['dvdrip'] = True @@ -150,12 +150,12 @@ class Searcher(SearcherBase): try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', check_name), key = len).strip()) except: pass - for check_name in list(set(check_names)): + for check_name in removeDuplicate(check_names): check_movie = fireEvent('scanner.name_year', check_name, single = True) try: - check_words = filter(None, re.split('\W+', check_movie.get('name', ''))) - movie_words = filter(None, re.split('\W+', simplifyString(movie_name))) + check_words = removeEmpty(re.split('\W+', check_movie.get('name', ''))) + movie_words = removeEmpty(re.split('\W+', simplifyString(movie_name))) if len(check_words) > 0 and len(movie_words) > 0 and len(list(set(check_words) - set(movie_words))) == 0: return True @@ -173,7 +173,7 @@ class Searcher(SearcherBase): # Make sure it has required words required_words = splitString(self.conf('required_words', section = 'searcher').lower()) - try: required_words = list(set(required_words + splitString(media['category']['required'].lower()))) + try: required_words = removeDuplicate(required_words + splitString(media['category']['required'].lower())) except: pass req_match = 0 @@ -187,7 +187,7 @@ class Searcher(SearcherBase): # Ignore releases ignored_words = splitString(self.conf('ignored_words', section = 'searcher').lower()) - try: ignored_words = list(set(ignored_words + splitString(media['category']['ignored'].lower()))) + try: ignored_words = removeDuplicate(ignored_words + splitString(media['category']['ignored'].lower())) except: pass ignored_match = 0 diff --git a/couchpotato/core/media/movie/_base/__init__.py b/couchpotato/core/media/movie/_base/__init__.py index 4be3b127..22211332 100644 --- a/couchpotato/core/media/movie/_base/__init__.py +++ b/couchpotato/core/media/movie/_base/__init__.py @@ -1,5 +1,6 @@ from .main import MovieBase + def start(): return MovieBase() diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 817b0a38..a7ecf2d2 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -1,16 +1,12 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import getImdb, splitString, tryInt, \ - mergeDicts +from couchpotato.core.helpers.variable import splitString, tryInt, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.media.movie import MovieTypeBase -from couchpotato.core.settings.model import Library, LibraryTitle, Media, \ - Release -from sqlalchemy.orm import joinedload_all -from sqlalchemy.sql.expression import or_, asc, not_, desc -from string import ascii_lowercase +from couchpotato.core.settings.model import Media import time log = CPLog(__name__) @@ -26,33 +22,12 @@ class MovieBase(MovieTypeBase): super(MovieBase, self).__init__() self.initType() - addApiView('movie.list', self.listView, docs = { - 'desc': 'List movies in wanted list', - 'params': { - 'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'}, - 'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'}, - 'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'}, - 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'}, - 'search': {'desc': 'Search movie title'}, - }, - 'return': {'type': 'object', 'example': """{ - 'success': True, - 'empty': bool, any movies returned or not, - 'movies': array, movies found, -}"""} - }) - addApiView('movie.get', self.getView, docs = { - 'desc': 'Get a movie by id', - 'params': { - 'id': {'desc': 'The id of the movie'}, - } - }) - addApiView('movie.available_chars', self.charView) addApiView('movie.add', self.addView, docs = { 'desc': 'Add new movie to the wanted list', 'params': { 'identifier': {'desc': 'IMDB id of the movie your want to add.'}, 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'}, + 'category_id': {'desc': 'ID of category you want the add the movie in. If empty will use no category.'}, 'title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'}, } }) @@ -61,258 +36,12 @@ class MovieBase(MovieTypeBase): 'params': { 'id': {'desc': 'Movie ID(s) you want to edit.', 'type': 'int (comma separated)'}, 'profile_id': {'desc': 'ID of quality profile you want the edit the movie to.'}, + 'category_id': {'desc': 'ID of category you want the add the movie in. If empty will use no category.'}, 'default_title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'}, } }) - addApiView('movie.delete', self.deleteView, docs = { - 'desc': 'Delete a movie from the wanted list', - 'params': { - 'id': {'desc': 'Movie ID(s) you want to delete.', 'type': 'int (comma separated)'}, - 'delete_from': {'desc': 'Delete movie from this page', 'type': 'string: all (default), wanted, manage'}, - } - }) addEvent('movie.add', self.add) - addEvent('movie.delete', self.delete) - addEvent('movie.get', self.get) - addEvent('movie.list', self.list) - addEvent('movie.restatus', self.restatus) - - def getView(self, id = None, **kwargs): - - movie = self.get(id) if id else None - - return { - 'success': movie is not None, - 'movie': movie, - } - - def get(self, movie_id): - - db = get_session() - - imdb_id = getImdb(str(movie_id)) - - if imdb_id: - m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first() - else: - m = db.query(Media).filter_by(id = movie_id).first() - - results = None - if m: - results = m.to_dict(self.default_dict) - - db.expire_all() - return results - - def list(self, status = None, release_status = None, limit_offset = None, starts_with = None, search = None, order = None): - - db = get_session() - - # Make a list from string - if status and not isinstance(status, (list, tuple)): - status = [status] - if release_status and not isinstance(release_status, (list, tuple)): - release_status = [release_status] - - # query movie ids - q = db.query(Media) \ - .with_entities(Media.id) \ - .group_by(Media.id) - - # Filter on movie status - if status and len(status) > 0: - statuses = fireEvent('status.get', status, single = len(status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.filter(Media.status_id.in_(statuses)) - - # Filter on release status - if release_status and len(release_status) > 0: - q = q.join(Media.releases) - - statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.filter(Release.status_id.in_(statuses)) - - # Only join when searching / ordering - if starts_with or search or order != 'release_order': - q = q.join(Media.library, Library.titles) \ - .filter(LibraryTitle.default == True) - - # Add search filters - filter_or = [] - if starts_with: - starts_with = toUnicode(starts_with.lower()) - if starts_with in ascii_lowercase: - filter_or.append(LibraryTitle.simple_title.startswith(starts_with)) - else: - ignore = [] - for letter in ascii_lowercase: - ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter))) - filter_or.append(not_(or_(*ignore))) - - if search: - filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%')) - - if len(filter_or) > 0: - q = q.filter(or_(*filter_or)) - - total_count = q.count() - if total_count == 0: - return 0, [] - - if order == 'release_order': - q = q.order_by(desc(Release.last_edit)) - else: - q = q.order_by(asc(LibraryTitle.simple_title)) - - if limit_offset: - splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset - limit = splt[0] - offset = 0 if len(splt) is 1 else splt[1] - q = q.limit(limit).offset(offset) - - # Get all movie_ids in sorted order - movie_ids = [m.id for m in q.all()] - - # List release statuses - releases = db.query(Release) \ - .filter(Release.movie_id.in_(movie_ids)) \ - .all() - - release_statuses = dict((m, set()) for m in movie_ids) - releases_count = dict((m, 0) for m in movie_ids) - for release in releases: - release_statuses[release.movie_id].add('%d,%d' % (release.status_id, release.quality_id)) - releases_count[release.movie_id] += 1 - - # Get main movie data - q2 = db.query(Media) \ - .options(joinedload_all('library.titles')) \ - .options(joinedload_all('library.files')) \ - .options(joinedload_all('status')) \ - .options(joinedload_all('files')) - - q2 = q2.filter(Media.id.in_(movie_ids)) - - results = q2.all() - - # Create dict by movie id - movie_dict = {} - for movie in results: - movie_dict[movie.id] = movie - - # List movies based on movie_ids order - movies = [] - for movie_id in movie_ids: - - releases = [] - for r in release_statuses.get(movie_id): - x = splitString(r) - releases.append({'status_id': x[0], 'quality_id': x[1]}) - - # Merge releases with movie dict - movies.append(mergeDicts(movie_dict[movie_id].to_dict({ - 'library': {'titles': {}, 'files':{}}, - 'files': {}, - }), { - 'releases': releases, - 'releases_count': releases_count.get(movie_id), - })) - - db.expire_all() - return total_count, movies - - def availableChars(self, status = None, release_status = None): - - status = status or [] - release_status = release_status or [] - - db = get_session() - - # Make a list from string - if not isinstance(status, (list, tuple)): - status = [status] - if release_status and not isinstance(release_status, (list, tuple)): - release_status = [release_status] - - q = db.query(Media) - - # Filter on movie status - if status and len(status) > 0: - statuses = fireEvent('status.get', status, single = len(release_status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.filter(Media.status_id.in_(statuses)) - - # Filter on release status - if release_status and len(release_status) > 0: - - statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.join(Media.releases) \ - .filter(Release.status_id.in_(statuses)) - - q = q.join(Library, LibraryTitle) \ - .with_entities(LibraryTitle.simple_title) \ - .filter(LibraryTitle.default == True) - - titles = q.all() - - chars = set() - for title in titles: - try: - char = title[0][0] - char = char if char in ascii_lowercase else '#' - chars.add(str(char)) - except: - log.error('Failed getting title for %s', title.libraries_id) - - if len(chars) == 25: - break - - db.expire_all() - return ''.join(sorted(chars)) - - def listView(self, **kwargs): - - status = splitString(kwargs.get('status')) - release_status = splitString(kwargs.get('release_status')) - limit_offset = kwargs.get('limit_offset') - starts_with = kwargs.get('starts_with') - search = kwargs.get('search') - order = kwargs.get('order') - - total_movies, movies = self.list( - status = status, - release_status = release_status, - limit_offset = limit_offset, - starts_with = starts_with, - search = search, - order = order - ) - - return { - 'success': True, - 'empty': len(movies) == 0, - 'total': total_movies, - 'movies': movies, - } - - def charView(self, **kwargs): - - status = splitString(kwargs.get('status', None)) - release_status = splitString(kwargs.get('release_status', None)) - chars = self.availableChars(status, release_status) - - return { - 'success': True, - 'empty': len(chars) == 0, - 'chars': chars, - } def add(self, params = None, force_readd = True, search_after = True, update_library = False, status_id = None): if not params: params = {} @@ -333,7 +62,6 @@ class MovieBase(MovieTypeBase): except: pass - library = fireEvent('library.add.movie', single = True, attrs = params, update_after = update_library) # Status @@ -343,68 +71,81 @@ class MovieBase(MovieTypeBase): default_profile = fireEvent('profile.default', single = True) cat_id = params.get('category_id') - db = get_session() - m = db.query(Media).filter_by(library_id = library.get('id')).first() - added = True - do_search = False - search_after = search_after and self.conf('search_on_add', section = 'moviesearcher') - if not m: - m = Media( - library_id = library.get('id'), - profile_id = params.get('profile_id', default_profile.get('id')), - status_id = status_id if status_id else status_active.get('id'), - category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, - ) - db.add(m) - db.commit() - - onComplete = None - if search_after: - onComplete = self.createOnComplete(m.id) - - fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) - search_after = False - elif force_readd: - - # Clean snatched history - for release in m.releases: - if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]: - if params.get('ignore_previous', False): - release.status_id = ignored_status.get('id') - else: - fireEvent('release.delete', release.id, single = True) - - m.profile_id = params.get('profile_id', default_profile.get('id')) - m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m.category_id or None) - else: - log.debug('Movie already exists, not updating: %s', params) - added = False - - if force_readd: - m.status_id = status_id if status_id else status_active.get('id') - m.last_edit = int(time.time()) - do_search = True - - db.commit() - - # Remove releases - available_status = fireEvent('status.get', 'available', single = True) - for rel in m.releases: - if rel.status_id is available_status.get('id'): - db.delete(rel) + try: + db = get_session() + m = db.query(Media).filter_by(library_id = library.get('id')).first() + added = True + do_search = False + search_after = search_after and self.conf('search_on_add', section = 'moviesearcher') + if not m: + m = Media( + library_id = library.get('id'), + profile_id = params.get('profile_id', default_profile.get('id')), + status_id = status_id if status_id else status_active.get('id'), + category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, + ) + db.add(m) db.commit() - movie_dict = m.to_dict(self.default_dict) + onComplete = None + if search_after: + onComplete = self.createOnComplete(m.id) - if do_search and search_after: - onComplete = self.createOnComplete(m.id) - onComplete() + fireEventAsync('library.update.movie', params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) + search_after = False + elif force_readd: - if added: - fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = 'Successfully added "%s" to your wanted list.' % params.get('title', '')) + # Clean snatched history + for release in m.releases: + if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]: + if params.get('ignore_previous', False): + release.status_id = ignored_status.get('id') + else: + fireEvent('release.delete', release.id, single = True) - db.expire_all() - return movie_dict + m.profile_id = params.get('profile_id', default_profile.get('id')) + m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else (m.category_id or None) + else: + log.debug('Movie already exists, not updating: %s', params) + added = False + + if force_readd: + m.status_id = status_id if status_id else status_active.get('id') + m.last_edit = int(time.time()) + do_search = True + + db.commit() + + # Remove releases + available_status = fireEvent('status.get', 'available', single = True) + for rel in m.releases: + if rel.status_id is available_status.get('id'): + db.delete(rel) + db.commit() + + movie_dict = m.to_dict(self.default_dict) + + if do_search and search_after: + onComplete = self.createOnComplete(m.id) + onComplete() + + if added: + if params.get('title'): + message = 'Successfully added "%s" to your wanted list.' % params.get('title', '') + else: + title = getTitle(m.library) + if title: + message = 'Successfully added "%s" to your wanted list.' % title + else: + message = 'Succesfully added to your wanted list.' + fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = message) + + return movie_dict + except: + log.error('Failed deleting media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def addView(self, **kwargs): add_dict = self.add(params = kwargs) @@ -416,128 +157,51 @@ class MovieBase(MovieTypeBase): def edit(self, id = '', **kwargs): - db = get_session() + try: + db = get_session() - available_status = fireEvent('status.get', 'available', single = True) + available_status = fireEvent('status.get', 'available', single = True) - ids = splitString(id) - for movie_id in ids: + ids = splitString(id) + for media_id in ids: - m = db.query(Media).filter_by(id = movie_id).first() - if not m: - continue + m = db.query(Media).filter_by(id = media_id).first() + if not m: + continue - m.profile_id = kwargs.get('profile_id') + m.profile_id = kwargs.get('profile_id') - cat_id = kwargs.get('category_id') - if cat_id is not None: - m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None + cat_id = kwargs.get('category_id') + if cat_id is not None: + m.category_id = tryInt(cat_id) if tryInt(cat_id) > 0 else None - # Remove releases - for rel in m.releases: - if rel.status_id is available_status.get('id'): - db.delete(rel) - db.commit() + # Remove releases + for rel in m.releases: + if rel.status_id is available_status.get('id'): + db.delete(rel) + db.commit() - # Default title - if kwargs.get('default_title'): - for title in m.library.titles: - title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower() + # Default title + if kwargs.get('default_title'): + for title in m.library.titles: + title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower() - db.commit() - - fireEvent('movie.restatus', m.id) - - movie_dict = m.to_dict(self.default_dict) - fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id)) - - db.expire_all() - return { - 'success': True, - } - - def deleteView(self, id = '', **kwargs): - - ids = splitString(id) - for movie_id in ids: - self.delete(movie_id, delete_from = kwargs.get('delete_from', 'all')) - - return { - 'success': True, - } - - def delete(self, movie_id, delete_from = None): - - db = get_session() - - movie = db.query(Media).filter_by(id = movie_id).first() - if movie: - deleted = False - if delete_from == 'all': - db.delete(movie) - db.commit() - deleted = True - else: - done_status = fireEvent('status.get', 'done', single = True) - - total_releases = len(movie.releases) - total_deleted = 0 - new_movie_status = None - for release in movie.releases: - if delete_from in ['wanted', 'snatched', 'late']: - if release.status_id != done_status.get('id'): - db.delete(release) - total_deleted += 1 - new_movie_status = 'done' - elif delete_from == 'manage': - if release.status_id == done_status.get('id'): - db.delete(release) - total_deleted += 1 - new_movie_status = 'active' db.commit() - if total_releases == total_deleted: - db.delete(movie) - db.commit() - deleted = True - elif new_movie_status: - new_status = fireEvent('status.get', new_movie_status, single = True) - movie.profile_id = None - movie.status_id = new_status.get('id') - db.commit() - else: - fireEvent('movie.restatus', movie.id, single = True) + fireEvent('media.restatus', m.id) - if deleted: - fireEvent('notify.frontend', type = 'movie.deleted', data = movie.to_dict()) + movie_dict = m.to_dict(self.default_dict) + fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id)) - db.expire_all() - return True + return { + 'success': True, + } + except: + log.error('Failed deleting media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - def restatus(self, movie_id): - - active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True) - - db = get_session() - - m = db.query(Media).filter_by(id = movie_id).first() - if not m or len(m.library.titles) == 0: - log.debug('Can\'t restatus movie, doesn\'t seem to exist.') - return False - - log.debug('Changing status for %s', m.library.titles[0].title) - if not m.profile: - m.status_id = done_status.get('id') - else: - move_to_wanted = True - - for t in m.profile.types: - for release in m.releases: - if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): - move_to_wanted = False - - m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') - - db.commit() - - return True + return { + 'success': False, + } diff --git a/couchpotato/core/media/movie/_base/static/list.js b/couchpotato/core/media/movie/_base/static/list.js index aaa8be12..85dee2e5 100644 --- a/couchpotato/core/media/movie/_base/static/list.js +++ b/couchpotato/core/media/movie/_base/static/list.js @@ -52,8 +52,8 @@ var MovieList = new Class({ self.getMovies(); - App.addEvent('movie.added', self.movieAdded.bind(self)) - App.addEvent('movie.deleted', self.movieDeleted.bind(self)) + App.on('movie.added', self.movieAdded.bind(self)) + App.on('movie.deleted', self.movieDeleted.bind(self)) }, movieDeleted: function(notification){ @@ -65,6 +65,7 @@ var MovieList = new Class({ movie.destroy(); delete self.movies_added[notification.data.id]; self.setCounter(self.counter_count-1); + self.total_movies--; } }) } @@ -75,6 +76,7 @@ var MovieList = new Class({ movieAdded: function(notification){ var self = this; + self.fireEvent('movieAdded', notification); if(self.options.add_new && !self.movies_added[notification.data.id] && notification.data.status.identifier == self.options.status){ window.scroll(0,0); self.createMovie(notification.data, 'top'); @@ -279,7 +281,7 @@ var MovieList = new Class({ // Get available chars and highlight if(!available_chars && (self.navigation.isDisplayed() || self.navigation.isVisible())) - Api.request('movie.available_chars', { + Api.request('media.available_chars', { 'data': Object.merge({ 'status': self.options.status }, self.filter), @@ -370,7 +372,7 @@ var MovieList = new Class({ 'click': function(e){ (e).preventDefault(); this.set('text', 'Deleting..') - Api.request('movie.delete', { + Api.request('media.delete', { 'data': { 'id': ids.join(','), 'delete_from': self.options.identifier @@ -390,6 +392,7 @@ var MovieList = new Class({ self.movies.erase(movie); movie.destroy(); self.setCounter(self.counter_count-1); + self.total_movies--; }); self.calculateSelected(); @@ -547,8 +550,9 @@ var MovieList = new Class({ } - Api.request(self.options.api_call || 'movie.list', { + Api.request(self.options.api_call || 'media.list', { 'data': Object.merge({ + 'type': 'movie', 'status': self.options.status, 'limit_offset': self.options.limit ? self.options.limit + ',' + self.offset : null }, self.filter), diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index f6e0f542..66c84c68 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -126,7 +126,9 @@ MA.Release = new Class({ else self.showHelper(); - App.addEvent('movie.searcher.ended.'+self.movie.data.id, function(notification){ + App.on('movie.searcher.ended', function(notification){ + if(self.movie.data.id != notification.data.id) return; + self.releases = null; if(self.options_container){ self.options_container.destroy(); @@ -250,12 +252,14 @@ MA.Release = new Class({ else if(!self.next_release && status.identifier == 'available'){ self.next_release = release; } - + var update_handle = function(notification) { - var q = self.movie.quality.getElement('.q_id' + release.quality_id), + if(notification.data.id != release.id) return; + + var q = self.movie.quality.getElement('.q_id' + release.quality_id), status = Status.get(release.status_id), - new_status = Status.get(notification.data); - + new_status = Status.get(notification.data.status_id); + release.status_id = new_status.id release.el.set('class', 'item ' + new_status.identifier); @@ -272,7 +276,7 @@ MA.Release = new Class({ } } - App.addEvent('release.update_status.' + release.id, update_handle); + App.on('release.update_status', update_handle); }); @@ -285,7 +289,7 @@ MA.Release = new Class({ if(self.next_release || (self.last_release && ['ignored', 'failed'].indexOf(self.last_release.status.identifier) === false)){ self.trynext_container = new Element('div.buttons.try_container').inject(self.release_container, 'top'); - + var nr = self.next_release, lr = self.last_release; @@ -427,7 +431,7 @@ MA.Release = new Class({ markMovieDone: function(){ var self = this; - Api.request('movie.delete', { + Api.request('media.delete', { 'data': { 'id': self.movie.get('id'), 'delete_from': 'wanted' @@ -446,7 +450,7 @@ MA.Release = new Class({ }, - tryNextRelease: function(movie_id){ + tryNextRelease: function(){ var self = this; Api.request('movie.searcher.try_next', { @@ -817,7 +821,7 @@ MA.Delete = new Class({ self.callChain(); }, function(){ - Api.request('movie.delete', { + Api.request('media.delete', { 'data': { 'id': self.movie.get('id'), 'delete_from': self.movie.list.options.identifier diff --git a/couchpotato/core/media/movie/_base/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css index c013bd80..a88a2077 100644 --- a/couchpotato/core/media/movie/_base/static/movie.css +++ b/couchpotato/core/media/movie/_base/static/movie.css @@ -1036,7 +1036,7 @@ text-overflow: ellipsis; overflow: hidden; width: 85%; - direction: rtl; + direction: ltr; vertical-align: middle; } diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index a865325b..3ca1912c 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -23,23 +23,49 @@ var Movie = new Class({ addEvents: function(){ var self = this; - App.addEvent('movie.update.'+self.data.id, function(notification){ + self.global_events = {} + + // Do refresh with new data + self.global_events['movie.update'] = function(notification){ + if(self.data.id != notification.data.id) return; + self.busy(false); self.removeView(); self.update.delay(2000, self, notification); - }); + } + App.on('movie.update', self.global_events['movie.update']); - ['movie.busy', 'movie.searcher.started'].each(function(listener){ - App.addEvent(listener+'.'+self.data.id, function(notification){ - if(notification.data) - self.busy(true) - }); + // Add spinner on load / search + ['media.busy', 'movie.searcher.started'].each(function(listener){ + self.global_events[listener] = function(notification){ + if(notification.data && (self.data.id == notification.data.id || (typeOf(notification.data.id) == 'array' && notification.data.id.indexOf(self.data.id) > -1))) + self.busy(true); + } + App.on(listener, self.global_events[listener]); }) - App.addEvent('movie.searcher.ended.'+self.data.id, function(notification){ - if(notification.data) + // Remove spinner + self.global_events['movie.searcher.ended'] = function(notification){ + if(notification.data && self.data.id == notification.data.id) self.busy(false) - }); + } + App.on('movie.searcher.ended', self.global_events['movie.searcher.ended']); + + // Reload when releases have updated + self.global_events['release.update_status'] = function(notification){ + var data = notification.data + if(data && self.data.id == data.movie_id){ + + if(!self.data.releases) + self.data.releases = []; + + self.data.releases.push({'quality_id': data.quality_id, 'status_id': data.status_id}); + self.updateReleases(); + } + } + + App.on('release.update_status', self.global_events['release.update_status']); + }, destroy: function(){ @@ -52,10 +78,9 @@ var Movie = new Class({ self.list.checkIfEmpty(); // Remove events - App.removeEvents('movie.update.'+self.data.id); - ['movie.busy', 'movie.searcher.started'].each(function(listener){ - App.removeEvents(listener+'.'+self.data.id); - }) + Object.each(self.global_events, function(handle, listener){ + App.off(listener, handle); + }); }, busy: function(set_busy, timeout){ @@ -179,21 +204,7 @@ var Movie = new Class({ }); // Add releases - if(self.data.releases) - self.data.releases.each(function(release){ - - var q = self.quality.getElement('.q_id'+ release.quality_id), - status = Status.get(release.status_id); - - if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done')) - var q = self.addQuality(release.quality_id) - - if (status && q && !q.hasClass(status.identifier)){ - q.addClass(status.identifier); - q.set('title', (q.get('title') ? q.get('title') : '') + ' status: '+ status.label) - } - - }); + self.updateReleases(); Object.each(self.options.actions, function(action, key){ self.action[key.toLowerCase()] = action = new self.options.actions[key](self) @@ -203,6 +214,26 @@ var Movie = new Class({ }, + updateReleases: function(){ + var self = this; + if(!self.data.releases || self.data.releases.length == 0) return; + + self.data.releases.each(function(release){ + + var q = self.quality.getElement('.q_id'+ release.quality_id), + status = Status.get(release.status_id); + + if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done')) + var q = self.addQuality(release.quality_id) + + if (status && q && !q.hasClass(status.identifier)){ + q.addClass(status.identifier); + q.set('title', (q.get('title') ? q.get('title') : '') + ' status: '+ status.label) + } + + }); + }, + addQuality: function(quality_id){ var self = this; @@ -298,4 +329,4 @@ var Movie = new Class({ return this.el; } -}); \ No newline at end of file +}); diff --git a/couchpotato/core/media/movie/library/movie/__init__.py b/couchpotato/core/media/movie/library/movie/__init__.py index 03494a11..98ed54c0 100644 --- a/couchpotato/core/media/movie/library/movie/__init__.py +++ b/couchpotato/core/media/movie/library/movie/__init__.py @@ -1,5 +1,6 @@ from .main import MovieLibraryPlugin + def start(): return MovieLibraryPlugin() diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index b0d05202..034a8fb0 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -7,13 +7,14 @@ from couchpotato.core.settings.model import Library, LibraryTitle, File from string import ascii_letters import time import traceback +import six log = CPLog(__name__) class MovieLibraryPlugin(LibraryBase): - default_dict = {'titles': {}, 'files':{}} + default_dict = {'titles': {}, 'files': {}} def __init__(self): addEvent('library.add.movie', self.add) @@ -25,69 +26,70 @@ class MovieLibraryPlugin(LibraryBase): primary_provider = attrs.get('primary_provider', 'imdb') - db = get_session() + try: + db = get_session() - l = db.query(Library).filter_by(identifier = attrs.get('identifier')).first() - if not l: - status = fireEvent('status.get', 'needs_update', single = True) - l = Library( - year = attrs.get('year'), - identifier = attrs.get('identifier'), - plot = toUnicode(attrs.get('plot')), - tagline = toUnicode(attrs.get('tagline')), - status_id = status.get('id'), - info = {} - ) + l = db.query(Library).filter_by(identifier = attrs.get('identifier')).first() + if not l: + status = fireEvent('status.get', 'needs_update', single = True) + l = Library( + year = attrs.get('year'), + identifier = attrs.get('identifier'), + plot = toUnicode(attrs.get('plot')), + tagline = toUnicode(attrs.get('tagline')), + status_id = status.get('id'), + info = {} + ) - title = LibraryTitle( - title = toUnicode(attrs.get('title')), - simple_title = self.simplifyTitle(attrs.get('title')), - ) + title = LibraryTitle( + title = toUnicode(attrs.get('title')), + simple_title = self.simplifyTitle(attrs.get('title')), + ) - l.titles.append(title) + l.titles.append(title) - db.add(l) - db.commit() + db.add(l) + db.commit() - # Update library info - if update_after is not False: - handle = fireEventAsync if update_after is 'async' else fireEvent - handle('library.update.movie', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) + # Update library info + if update_after is not False: + handle = fireEventAsync if update_after is 'async' else fireEvent + handle('library.update.movie', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) - library_dict = l.to_dict(self.default_dict) + library_dict = l.to_dict(self.default_dict) + return library_dict + except: + log.error('Failed adding media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() - return library_dict + return {} - def update(self, identifier, default_title = '', force = False): + def update(self, identifier, default_title = '', extended = False): if self.shuttingDown(): return - db = get_session() - library = db.query(Library).filter_by(identifier = identifier).first() - done_status = fireEvent('status.get', 'done', single = True) + try: + db = get_session() - library_dict = None - if library: - library_dict = library.to_dict(self.default_dict) + library = db.query(Library).filter_by(identifier = identifier).first() + done_status = fireEvent('status.get', 'done', single = True) - do_update = True + info = fireEvent('movie.info', merge = True, extended = extended, identifier = identifier) - info = fireEvent('movie.info', merge = True, identifier = identifier) + # Don't need those here + try: del info['in_wanted'] + except: pass + try: del info['in_library'] + except: pass - # Don't need those here - try: del info['in_wanted'] - except: pass - try: del info['in_library'] - except: pass + if not info or len(info) == 0: + log.error('Could not update, no movie info to work with: %s', identifier) + return False - if not info or len(info) == 0: - log.error('Could not update, no movie info to work with: %s', identifier) - return False - - # Main info - if do_update: + # Main info library.plot = toUnicode(info.get('plot', '')) library.tagline = toUnicode(info.get('tagline', '')) library.year = info.get('year', 0) @@ -102,6 +104,17 @@ class MovieLibraryPlugin(LibraryBase): titles = info.get('titles', []) log.debug('Adding titles: %s', titles) counter = 0 + + def_title = None + for title in titles: + if (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title): + def_title = toUnicode(title) + break + counter += 1 + + if not def_title: + def_title = toUnicode(titles[0]) + for title in titles: if not title: continue @@ -109,10 +122,9 @@ class MovieLibraryPlugin(LibraryBase): t = LibraryTitle( title = title, simple_title = self.simplifyTitle(title), - default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title) + default = title == def_title ) library.titles.append(t) - counter += 1 db.commit() @@ -134,30 +146,43 @@ class MovieLibraryPlugin(LibraryBase): break except: log.debug('Failed to attach to library: %s', traceback.format_exc()) + db.rollback() library_dict = library.to_dict(self.default_dict) + return library_dict + except: + log.error('Failed update media: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() - return library_dict + return {} def updateReleaseDate(self, identifier): - db = get_session() - library = db.query(Library).filter_by(identifier = identifier).first() + try: + db = get_session() + library = db.query(Library).filter_by(identifier = identifier).first() - if not library.info: - library_dict = self.update(identifier, force = True) - dates = library_dict.get('info', {}).get('release_date') - else: - dates = library.info.get('release_date') + if not library.info: + library_dict = self.update(identifier) + dates = library_dict.get('info', {}).get('release_date') + else: + dates = library.info.get('release_date') - if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: - dates = fireEvent('movie.release_date', identifier = identifier, merge = True) - library.info.update({'release_date': dates }) - db.commit() + if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: + dates = fireEvent('movie.release_date', identifier = identifier, merge = True) + library.info.update({'release_date': dates}) + db.commit() - db.expire_all() - return dates + return dates + except: + log.error('Failed updating release dates: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return {} def simplifyTitle(self, title): diff --git a/couchpotato/core/media/movie/searcher/__init__.py b/couchpotato/core/media/movie/searcher/__init__.py index bae18902..4ae1ed32 100644 --- a/couchpotato/core/media/movie/searcher/__init__.py +++ b/couchpotato/core/media/movie/searcher/__init__.py @@ -1,6 +1,7 @@ from .main import MovieSearcher import random + def start(): return MovieSearcher() diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 93441c59..1c22e18c 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -73,10 +73,21 @@ class MovieSearcher(SearcherBase, MovieTypeBase): db = get_session() - movies = db.query(Media).filter( + movies_raw = db.query(Media).filter( Media.status.has(identifier = 'active') ).all() - random.shuffle(movies) + + random.shuffle(movies_raw) + + movies = [] + for m in movies_raw: + movies.append(m.to_dict({ + 'category': {}, + 'profile': {'types': {'quality': {}}}, + 'releases': {'status': {}, 'quality': {}}, + 'library': {'titles': {}, 'files': {}}, + 'files': {}, + })) self.in_progress = { 'total': len(movies), @@ -87,21 +98,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): search_protocols = fireEvent('searcher.protocols', single = True) for movie in movies: - movie_dict = movie.to_dict({ - 'category': {}, - 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}}, - 'library': {'titles': {}, 'files':{}}, - 'files': {}, - }) try: - self.single(movie_dict, search_protocols) + self.single(movie, search_protocols) except IndexError: - log.error('Forcing library update for %s, if you see this often, please report: %s', (movie_dict['library']['identifier'], traceback.format_exc())) - fireEvent('library.update.movie', movie_dict['library']['identifier'], force = True) + log.error('Forcing library update for %s, if you see this often, please report: %s', (movie['library']['identifier'], traceback.format_exc())) + fireEvent('library.update.movie', movie['library']['identifier']) except: - log.error('Search failed for %s: %s', (movie_dict['library']['identifier'], traceback.format_exc())) + log.error('Search failed for %s: %s', (movie['library']['identifier'], traceback.format_exc())) self.in_progress['to_go'] -= 1 @@ -117,7 +121,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): def single(self, movie, search_protocols = None, manual = False): # movies don't contain 'type' yet, so just set to default here - if not movie.has_key('type'): + if 'type' not in movie: movie['type'] = 'movie' # Find out search type @@ -133,8 +137,6 @@ class MovieSearcher(SearcherBase, MovieTypeBase): log.debug('Movie doesn\'t have a profile or already done, assuming in manage tab.') return - db = get_session() - pre_releases = fireEvent('quality.pre_releases', single = True) release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True) available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True) @@ -145,11 +147,12 @@ class MovieSearcher(SearcherBase, MovieTypeBase): default_title = getTitle(movie['library']) if not default_title: log.error('No proper info found for movie, removing it from library to cause it from having more issues.') - fireEvent('movie.delete', movie['id'], single = True) + fireEvent('media.delete', movie['id'], single = True) return - fireEvent('notify.frontend', type = 'movie.searcher.started.%s' % movie['id'], data = True, message = 'Searching for "%s"' % default_title) + fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title) + db = get_session() ret = False for quality_type in movie['profile']['types']: @@ -192,7 +195,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): else: log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title)) - fireEvent('movie.restatus', movie['id']) + fireEvent('media.restatus', movie['id']) break # Break if CP wants to shut down @@ -202,7 +205,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if len(too_early_to_search) > 0: log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) - fireEvent('notify.frontend', type = 'movie.searcher.ended.%s' % movie['id'], data = True) + fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']}) return ret @@ -279,11 +282,17 @@ class MovieSearcher(SearcherBase, MovieTypeBase): now = int(time.time()) now_year = date.today().year + now_month = date.today().month if (year is None or year < now_year - 1) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)): return True else: + # Don't allow movies with years to far in the future + add_year = 1 if now_month > 10 else 0 # Only allow +1 year if end of the year + if year is not None and year > (now_year + add_year): + return False + # For movies before 1972 if not dates or dates.get('theater', 0) < 0 or dates.get('dvd', 0) < 0: return True @@ -318,14 +327,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): 'success': trynext } - def tryNextRelease(self, movie_id, manual = False): + def tryNextRelease(self, media_id, manual = False): snatched_status, done_status, ignored_status = fireEvent('status.get', ['snatched', 'done', 'ignored'], single = True) try: db = get_session() rels = db.query(Release) \ - .filter_by(movie_id = movie_id) \ + .filter_by(movie_id = media_id) \ .filter(Release.status_id.in_([snatched_status.get('id'), done_status.get('id')])) \ .all() @@ -333,7 +342,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): rel.status_id = ignored_status.get('id') db.commit() - movie_dict = fireEvent('movie.get', movie_id, single = True) + movie_dict = fireEvent('media.get', media_id = media_id, single = True) log.info('Trying next release for: %s', getTitle(movie_dict['library'])) fireEvent('movie.searcher.single', movie_dict, manual = manual) @@ -341,7 +350,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): except: log.error('Failed searching for next release: %s', traceback.format_exc()) + db.rollback() return False + finally: + db.close() def getSearchTitle(self, media): if media['type'] == 'movie': diff --git a/couchpotato/core/media/movie/suggestion/__init__.py b/couchpotato/core/media/movie/suggestion/__init__.py index b63b5b13..50083fe7 100644 --- a/couchpotato/core/media/movie/suggestion/__init__.py +++ b/couchpotato/core/media/movie/suggestion/__init__.py @@ -1,5 +1,6 @@ from .main import Suggestion + def start(): return Suggestion() diff --git a/couchpotato/core/media/movie/suggestion/main.py b/couchpotato/core/media/movie/suggestion/main.py index f29281ea..22e23fe2 100644 --- a/couchpotato/core/media/movie/suggestion/main.py +++ b/couchpotato/core/media/movie/suggestion/main.py @@ -1,7 +1,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.variable import splitString +from couchpotato.core.helpers.variable import splitString, removeDuplicate from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Media, Library from couchpotato.environment import Env @@ -40,7 +40,7 @@ class Suggestion(Plugin): movies.extend(splitString(Env.prop('suggest_seen', default = ''))) suggestions = fireEvent('movie.suggest', movies = movies, ignore = ignored, single = True) - self.setCache('suggestion_cached', suggestions, timeout = 6048000) # Cache for 10 weeks + self.setCache('suggestion_cached', suggestions, timeout = 6048000) # Cache for 10 weeks return { 'success': True, @@ -79,8 +79,10 @@ class Suggestion(Plugin): seen = [] if not seen else seen if ignore_imdb: + suggested_imdbs = [] for cs in cached_suggestion: - if cs.get('imdb') != ignore_imdb: + if cs.get('imdb') != ignore_imdb and cs.get('imdb') not in suggested_imdbs: + suggested_imdbs.append(cs.get('imdb')) new_suggestions.append(cs) # Get new results and add them @@ -97,7 +99,7 @@ class Suggestion(Plugin): movies.extend(seen) ignored.extend([x.get('imdb') for x in cached_suggestion]) - suggestions = fireEvent('movie.suggest', movies = movies, ignore = list(set(ignored)), single = True) + suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True) if suggestions: new_suggestions.extend(suggestions) diff --git a/couchpotato/core/media/movie/suggestion/static/suggest.js b/couchpotato/core/media/movie/suggestion/static/suggest.js index cb09ef4a..c4e5630a 100644 --- a/couchpotato/core/media/movie/suggestion/static/suggest.js +++ b/couchpotato/core/media/movie/suggestion/static/suggest.js @@ -101,7 +101,7 @@ var SuggestList = new Class({ // Add rating m.info_container.adopt( - m.rating = m.info.rating && m.info.rating.imdb.length == 2 && parseFloat(m.info.rating.imdb[0]) > 0 ? new Element('span.rating', { + m.rating = m.info.rating && m.info.rating.imdb && m.info.rating.imdb.length == 2 && parseFloat(m.info.rating.imdb[0]) > 0 ? new Element('span.rating', { 'text': parseFloat(m.info.rating.imdb[0]), 'title': parseInt(m.info.rating.imdb[1]) + ' votes' }) : null, diff --git a/couchpotato/core/migration/versions/002_Movie_category.py b/couchpotato/core/migration/versions/002_Movie_category.py index 234e1136..023e47c6 100644 --- a/couchpotato/core/migration/versions/002_Movie_category.py +++ b/couchpotato/core/migration/versions/002_Movie_category.py @@ -13,5 +13,6 @@ def upgrade(migrate_engine): create_column(category_column, movie) Index('ix_movie_category_id', movie.c.category_id).create() + def downgrade(migrate_engine): pass diff --git a/couchpotato/core/notifications/base.py b/couchpotato/core/notifications/base.py index 4c0d0992..63d2075e 100644 --- a/couchpotato/core/notifications/base.py +++ b/couchpotato/core/notifications/base.py @@ -17,7 +17,7 @@ class Notification(Provider): listen_to = [ 'renamer.after', 'movie.snatched', 'updater.available', 'updater.updated', - 'core.message', + 'core.message.important', ] dont_listen_to = [] diff --git a/couchpotato/core/notifications/boxcar/__init__.py b/couchpotato/core/notifications/boxcar/__init__.py index ab244c32..faab7a5c 100644 --- a/couchpotato/core/notifications/boxcar/__init__.py +++ b/couchpotato/core/notifications/boxcar/__init__.py @@ -1,5 +1,6 @@ from .main import Boxcar + def start(): return Boxcar() diff --git a/couchpotato/core/notifications/boxcar/main.py b/couchpotato/core/notifications/boxcar/main.py index 0fca749f..49aab316 100644 --- a/couchpotato/core/notifications/boxcar/main.py +++ b/couchpotato/core/notifications/boxcar/main.py @@ -16,14 +16,14 @@ class Boxcar(Notification): try: message = message.strip() - params = { + data = { 'email': self.conf('email'), 'notification[from_screen_name]': self.default_title, 'notification[message]': toUnicode(message), 'notification[from_remote_service_id]': int(time.time()), } - self.urlopen(self.url, params = params) + self.urlopen(self.url, data = data) except: log.error('Check your email and added services on boxcar.io') return False diff --git a/couchpotato/core/notifications/boxcar2/__init__.py b/couchpotato/core/notifications/boxcar2/__init__.py new file mode 100644 index 00000000..da7f99c0 --- /dev/null +++ b/couchpotato/core/notifications/boxcar2/__init__.py @@ -0,0 +1,34 @@ +from .main import Boxcar2 + + +def start(): + return Boxcar2() + +config = [{ + 'name': 'boxcar2', + 'groups': [ + { + 'tab': 'notifications', + 'list': 'notification_providers', + 'name': 'boxcar2', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'token', + 'description': ('Your Boxcar access token.', 'Can be found in the app under settings') + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/notifications/boxcar2/main.py b/couchpotato/core/notifications/boxcar2/main.py new file mode 100644 index 00000000..6633ca70 --- /dev/null +++ b/couchpotato/core/notifications/boxcar2/main.py @@ -0,0 +1,39 @@ +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification + +log = CPLog(__name__) + + +class Boxcar2(Notification): + + url = 'https://new.boxcar.io/api/notifications' + + def notify(self, message = '', data = None, listener = None): + if not data: data = {} + + try: + message = message.strip() + + long_message = '' + if listener == 'test': + long_message = 'This is a test message' + elif data.get('identifier'): + long_message = 'More movie info on IMDB' % data['identifier'] + + data = { + 'user_credentials': self.conf('token'), + 'notification[title]': toUnicode(message), + 'notification[long_message]': toUnicode(long_message), + } + + self.urlopen(self.url, data = data) + except: + log.error('Make sure the token provided is for the correct device') + return False + + log.info('Boxcar notification successful.') + return True + + def isEnabled(self): + return super(Boxcar2, self).isEnabled() and self.conf('token') diff --git a/couchpotato/core/notifications/core/__init__.py b/couchpotato/core/notifications/core/__init__.py index 6e923dac..b68a915a 100644 --- a/couchpotato/core/notifications/core/__init__.py +++ b/couchpotato/core/notifications/core/__init__.py @@ -1,5 +1,6 @@ from .main import CoreNotifier + def start(): return CoreNotifier() diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index 04acf284..93f94d6a 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -21,6 +21,12 @@ class CoreNotifier(Notification): m_lock = None + listen_to = [ + 'renamer.after', 'movie.snatched', + 'updater.available', 'updater.updated', + 'core.message', 'core.message.important', + ] + def __init__(self): super(CoreNotifier, self).__init__() @@ -61,28 +67,42 @@ class CoreNotifier(Notification): def clean(self): - db = get_session() - db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete() - db.commit() - + try: + db = get_session() + db.query(Notif).filter(Notif.added <= (int(time.time()) - 2419200)).delete() + db.commit() + except: + log.error('Failed cleaning notification: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def markAsRead(self, ids = None, **kwargs): ids = splitString(ids) if ids else None - db = get_session() + try: + db = get_session() - if ids: - q = db.query(Notif).filter(or_(*[Notif.id == tryInt(s) for s in ids])) - else: - q = db.query(Notif).filter_by(read = False) + if ids: + q = db.query(Notif).filter(or_(*[Notif.id == tryInt(s) for s in ids])) + else: + q = db.query(Notif).filter_by(read = False) - q.update({Notif.read: True}) + q.update({Notif.read: True}) + db.commit() - db.commit() + return { + 'success': True + } + except: + log.error('Failed mark as read: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def listView(self, limit_offset = None, **kwargs): @@ -121,7 +141,10 @@ class CoreNotifier(Notification): for message in messages: if message.get('time') > last_check: - fireEvent('core.message', message = message.get('message'), data = message) + message['sticky'] = True # Always sticky core messages + + message_type = 'core.message.important' if message.get('important') else 'core.message' + fireEvent(message_type, message = message.get('message'), data = message) if last_check < message.get('time'): last_check = message.get('time') @@ -131,24 +154,30 @@ class CoreNotifier(Notification): def notify(self, message = '', data = None, listener = None): if not data: data = {} - db = get_session() + try: + db = get_session() - data['notification_type'] = listener if listener else 'unknown' + data['notification_type'] = listener if listener else 'unknown' - n = Notif( - message = toUnicode(message), - data = data - ) - db.add(n) - db.commit() + n = Notif( + message = toUnicode(message), + data = data + ) + db.add(n) + db.commit() - ndict = n.to_dict() - ndict['type'] = 'notification' - ndict['time'] = time.time() + ndict = n.to_dict() + ndict['type'] = 'notification' + ndict['time'] = time.time() - self.frontend(type = listener, data = data) + self.frontend(type = listener, data = data) - return True + return True + except: + log.error('Failed notify: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def frontend(self, type = 'notification', data = None, message = None): if not data: data = {} diff --git a/couchpotato/core/notifications/core/static/notification.js b/couchpotato/core/notifications/core/static/notification.js index e485976e..18d09e76 100644 --- a/couchpotato/core/notifications/core/static/notification.js +++ b/couchpotato/core/notifications/core/static/notification.js @@ -10,8 +10,8 @@ var NotificationBase = new Class({ // Listener App.addEvent('unload', self.stopPoll.bind(self)); App.addEvent('reload', self.startInterval.bind(self, [true])); - App.addEvent('notification', self.notify.bind(self)); - App.addEvent('message', self.showMessage.bind(self)); + App.on('notification', self.notify.bind(self)); + App.on('message', self.showMessage.bind(self)); // Add test buttons to settings page App.addEvent('load', self.addTestButtons.bind(self)); @@ -50,9 +50,9 @@ var NotificationBase = new Class({ , 'top'); self.notifications.include(result); - if(result.data.important !== undefined && !result.read){ + if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){ var sticky = true - App.fireEvent('message', [result.message, sticky, result]) + App.trigger('message', [result.message, sticky, result]) } else if(!result.read){ self.setBadge(self.notifications.filter(function(n){ return !n.read}).length) @@ -147,7 +147,7 @@ var NotificationBase = new Class({ // Process data if(json){ Array.each(json.result, function(result){ - App.fireEvent(result.type, result); + App.trigger(result.type, [result]); if(result.message && result.read === undefined) self.showMessage(result.message); }) diff --git a/couchpotato/core/notifications/email/__init__.py b/couchpotato/core/notifications/email/__init__.py index 33c2f634..aaf087b9 100644 --- a/couchpotato/core/notifications/email/__init__.py +++ b/couchpotato/core/notifications/email/__init__.py @@ -1,5 +1,6 @@ from .main import Email + def start(): return Email() @@ -30,7 +31,7 @@ config = [{ }, { 'name': 'smtp_port', 'label': 'SMTP server port', - 'default': '25', + 'default': '25', 'type': 'int', }, { diff --git a/couchpotato/core/notifications/email/main.py b/couchpotato/core/notifications/email/main.py index c67ac97d..b8544016 100644 --- a/couchpotato/core/notifications/email/main.py +++ b/couchpotato/core/notifications/email/main.py @@ -4,6 +4,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from couchpotato.environment import Env from email.mime.text import MIMEText +from email.utils import formatdate, make_msgid import smtplib import traceback @@ -30,6 +31,8 @@ class Email(Notification): message['Subject'] = self.default_title message['From'] = from_address message['To'] = to_address + message['Date'] = formatdate(localtime = 1) + message['Message-ID'] = make_msgid() try: # Open the SMTP connection, via SSL if requested @@ -37,7 +40,7 @@ class Email(Notification): log.debug("SMTP over SSL %s", ("enabled" if ssl == 1 else "disabled")) mailserver = smtplib.SMTP_SSL(smtp_server) if ssl == 1 else smtplib.SMTP(smtp_server) - if (starttls): + if starttls: log.debug("Using StartTLS to initiate the connection with the SMTP server") mailserver.starttls() diff --git a/couchpotato/core/notifications/growl/__init__.py b/couchpotato/core/notifications/growl/__init__.py index 8e462236..dd01cb91 100644 --- a/couchpotato/core/notifications/growl/__init__.py +++ b/couchpotato/core/notifications/growl/__init__.py @@ -1,5 +1,6 @@ from .main import Growl + def start(): return Growl() diff --git a/couchpotato/core/notifications/growl/main.py b/couchpotato/core/notifications/growl/main.py index dabeea01..a3927ed2 100644 --- a/couchpotato/core/notifications/growl/main.py +++ b/couchpotato/core/notifications/growl/main.py @@ -37,7 +37,7 @@ class Growl(Notification): ) self.growl.register() self.registered = True - except Exception, e: + except Exception as e: if 'timed out' in str(e): self.registered = True else: diff --git a/couchpotato/core/notifications/nmj/__init__.py b/couchpotato/core/notifications/nmj/__init__.py index 08a21a3e..461a450e 100644 --- a/couchpotato/core/notifications/nmj/__init__.py +++ b/couchpotato/core/notifications/nmj/__init__.py @@ -1,5 +1,6 @@ from .main import NMJ + def start(): return NMJ() diff --git a/couchpotato/core/notifications/nmj/main.py b/couchpotato/core/notifications/nmj/main.py index 1479fb1b..967b70e7 100644 --- a/couchpotato/core/notifications/nmj/main.py +++ b/couchpotato/core/notifications/nmj/main.py @@ -86,18 +86,17 @@ class NMJ(Notification): 'arg3': '', } params = tryUrlencode(params) - UPDATE_URL = 'http://%(host)s:8008/metadata_database?%(params)s' - updateUrl = UPDATE_URL % {'host': host, 'params': params} + update_url = 'http://%(host)s:8008/metadata_database?%(params)s' % {'host': host, 'params': params} try: - response = self.urlopen(updateUrl) + response = self.urlopen(update_url) except: return False try: et = etree.fromstring(response) result = et.findtext('returnValue') - except SyntaxError, e: + except SyntaxError as e: log.error('Unable to parse XML returned from the Popcorn Hour: %s', e) return False diff --git a/couchpotato/core/notifications/notifymyandroid/__init__.py b/couchpotato/core/notifications/notifymyandroid/__init__.py index 9ee5d90a..7d4f4aeb 100644 --- a/couchpotato/core/notifications/notifymyandroid/__init__.py +++ b/couchpotato/core/notifications/notifymyandroid/__init__.py @@ -1,5 +1,6 @@ from .main import NotifyMyAndroid + def start(): return NotifyMyAndroid() diff --git a/couchpotato/core/notifications/notifymyandroid/main.py b/couchpotato/core/notifications/notifymyandroid/main.py index 92e59562..16465101 100644 --- a/couchpotato/core/notifications/notifymyandroid/main.py +++ b/couchpotato/core/notifications/notifymyandroid/main.py @@ -2,6 +2,7 @@ from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification import pynma +import six log = CPLog(__name__) @@ -26,7 +27,7 @@ class NotifyMyAndroid(Notification): successful = 0 for key in keys: - if not response[str(key)]['code'] == u'200': + if not response[str(key)]['code'] == six.u('200'): log.error('Could not send notification to NotifyMyAndroid (%s). %s', (key, response[key]['message'])) else: successful += 1 diff --git a/couchpotato/core/notifications/notifymywp/__init__.py b/couchpotato/core/notifications/notifymywp/__init__.py index 6e0bd06d..4fcf1a9a 100644 --- a/couchpotato/core/notifications/notifymywp/__init__.py +++ b/couchpotato/core/notifications/notifymywp/__init__.py @@ -1,5 +1,6 @@ from .main import NotifyMyWP + def start(): return NotifyMyWP() diff --git a/couchpotato/core/notifications/notifymywp/main.py b/couchpotato/core/notifications/notifymywp/main.py index 167b6eeb..74010441 100644 --- a/couchpotato/core/notifications/notifymywp/main.py +++ b/couchpotato/core/notifications/notifymywp/main.py @@ -2,13 +2,15 @@ from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from pynmwp import PyNMWP +import six log = CPLog(__name__) class NotifyMyWP(Notification): - def notify(self, message = '', data = {}, listener = None): + def notify(self, message = '', data = None, listener = None): + if not data: data = {} keys = splitString(self.conf('api_key')) p = PyNMWP(keys, self.conf('dev_key')) @@ -16,7 +18,7 @@ class NotifyMyWP(Notification): response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1) for key in keys: - if not response[key]['Code'] == u'200': + if not response[key]['Code'] == six.u('200'): log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message'])) return False diff --git a/couchpotato/core/notifications/plex/__init__.py b/couchpotato/core/notifications/plex/__init__.py index d68ddb19..0de92ca3 100755 --- a/couchpotato/core/notifications/plex/__init__.py +++ b/couchpotato/core/notifications/plex/__init__.py @@ -1,5 +1,6 @@ from .main import Plex + def start(): return Plex() diff --git a/couchpotato/core/notifications/plex/client.py b/couchpotato/core/notifications/plex/client.py index b873518e..8864230d 100644 --- a/couchpotato/core/notifications/plex/client.py +++ b/couchpotato/core/notifications/plex/client.py @@ -29,7 +29,7 @@ class PlexClientHTTP(PlexClientProtocol): try: self.plex.urlopen(url, headers = headers, timeout = 3, show_error = False) - except Exception, err: + except Exception as err: log.error("Couldn't sent command to Plex: %s", err) return False @@ -68,7 +68,7 @@ class PlexClientJSON(PlexClientProtocol): try: requests.post(url, headers = headers, timeout = 3, data = json.dumps(request)) - except Exception, err: + except Exception as err: log.error("Couldn't sent command to Plex: %s", err) return False diff --git a/couchpotato/core/notifications/plex/main.py b/couchpotato/core/notifications/plex/main.py index ce25c8f0..a6853b2f 100755 --- a/couchpotato/core/notifications/plex/main.py +++ b/couchpotato/core/notifications/plex/main.py @@ -23,9 +23,9 @@ class Plex(Notification): addEvent('renamer.after', self.addToLibrary) - - def addToLibrary(self, message = None, group = {}): + def addToLibrary(self, message = None, group = None): if self.isDisabled(): return + if not group: group = {} return self.server.refresh() @@ -57,7 +57,8 @@ class Plex(Notification): return success - def notify(self, message = '', data = {}, listener = None): + def notify(self, message = '', data = None, listener = None): + if not data: data = {} return self.notifyClients(message, self.getClientNames()) def test(self, **kwargs): diff --git a/couchpotato/core/notifications/prowl/__init__.py b/couchpotato/core/notifications/prowl/__init__.py index e0564289..3721a0ad 100644 --- a/couchpotato/core/notifications/prowl/__init__.py +++ b/couchpotato/core/notifications/prowl/__init__.py @@ -1,5 +1,6 @@ from .main import Prowl + def start(): return Prowl() diff --git a/couchpotato/core/notifications/prowl/main.py b/couchpotato/core/notifications/prowl/main.py index a8a3dda2..b3385863 100644 --- a/couchpotato/core/notifications/prowl/main.py +++ b/couchpotato/core/notifications/prowl/main.py @@ -22,11 +22,11 @@ class Prowl(Notification): 'priority': self.conf('priority'), } headers = { - 'Content-type': 'application/x-www-form-urlencoded' + 'Content-type': 'application/x-www-form-urlencoded' } try: - self.urlopen(self.urls['api'], headers = headers, params = data, multipart = True, show_error = False) + self.urlopen(self.urls['api'], headers = headers, data = data, show_error = False) log.info('Prowl notifications sent.') return True except: diff --git a/couchpotato/core/notifications/pushalot/__init__.py b/couchpotato/core/notifications/pushalot/__init__.py index a2a297a3..ad0c853f 100644 --- a/couchpotato/core/notifications/pushalot/__init__.py +++ b/couchpotato/core/notifications/pushalot/__init__.py @@ -1,5 +1,6 @@ from .main import Pushalot + def start(): return Pushalot() diff --git a/couchpotato/core/notifications/pushalot/main.py b/couchpotato/core/notifications/pushalot/main.py index 4e3b6e76..306ee1d1 100644 --- a/couchpotato/core/notifications/pushalot/main.py +++ b/couchpotato/core/notifications/pushalot/main.py @@ -5,6 +5,7 @@ import traceback log = CPLog(__name__) + class Pushalot(Notification): urls = { @@ -29,7 +30,7 @@ class Pushalot(Notification): } try: - self.urlopen(self.urls['api'], headers = headers, params = data, multipart = True, show_error = False) + self.urlopen(self.urls['api'], headers = headers, data = data, show_error = False) return True except: log.error('PushAlot failed: %s', traceback.format_exc()) diff --git a/couchpotato/core/notifications/pushbullet/__init__.py b/couchpotato/core/notifications/pushbullet/__init__.py new file mode 100644 index 00000000..c52e7781 --- /dev/null +++ b/couchpotato/core/notifications/pushbullet/__init__.py @@ -0,0 +1,40 @@ +from .main import Pushbullet + + +def start(): + return Pushbullet() + +config = [{ + 'name': 'pushbullet', + 'groups': [ + { + 'tab': 'notifications', + 'list': 'notification_providers', + 'name': 'pushbullet', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'api_key', + 'label': 'User API Key' + }, + { + 'name': 'devices', + 'default': '', + 'advanced': True, + 'description': 'IDs of devices to send notifications to, empty = all devices' + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/notifications/pushbullet/main.py b/couchpotato/core/notifications/pushbullet/main.py new file mode 100644 index 00000000..487fb3aa --- /dev/null +++ b/couchpotato/core/notifications/pushbullet/main.py @@ -0,0 +1,69 @@ +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import splitString +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification +import base64 +import json + +log = CPLog(__name__) + + +class Pushbullet(Notification): + + url = 'https://api.pushbullet.com/api/%s' + + def notify(self, message = '', data = None, listener = None): + if not data: data = {} + + devices = self.getDevices() + if devices is None: + return False + + # Get all the device IDs linked to this user + if not len(devices): + response = self.request('devices') + if not response: + return False + + devices += [device.get('id') for device in response['devices']] + + successful = 0 + for device in devices: + response = self.request( + 'pushes', + cache = False, + device_iden = device, + type = 'note', + title = self.default_title, + body = toUnicode(message) + ) + + if response: + successful += 1 + else: + log.error('Unable to push notification to Pushbullet device with ID %s' % device) + + return successful == len(devices) + + def getDevices(self): + return splitString(self.conf('devices')) + + def request(self, method, cache = True, **kwargs): + try: + base64string = base64.encodestring('%s:' % self.conf('api_key'))[:-1] + + headers = { + "Authorization": "Basic %s" % base64string + } + + if cache: + return self.getJsonData(self.url % method, headers = headers, data = kwargs) + else: + data = self.urlopen(self.url % method, headers = headers, data = kwargs) + return json.loads(data) + + except Exception as ex: + log.error('Pushbullet request failed') + log.debug(ex) + + return None diff --git a/couchpotato/core/notifications/pushover/__init__.py b/couchpotato/core/notifications/pushover/__init__.py index 1ea1d5c0..da764860 100644 --- a/couchpotato/core/notifications/pushover/__init__.py +++ b/couchpotato/core/notifications/pushover/__init__.py @@ -1,5 +1,6 @@ from .main import Pushover + def start(): return Pushover() diff --git a/couchpotato/core/notifications/pushover/main.py b/couchpotato/core/notifications/pushover/main.py index 76f730b6..ba954a54 100644 --- a/couchpotato/core/notifications/pushover/main.py +++ b/couchpotato/core/notifications/pushover/main.py @@ -30,9 +30,9 @@ class Pushover(Notification): }) http_handler.request('POST', - "/1/messages.json", - headers = {'Content-type': 'application/x-www-form-urlencoded'}, - body = tryUrlencode(api_data) + "/1/messages.json", + headers = {'Content-type': 'application/x-www-form-urlencoded'}, + body = tryUrlencode(api_data) ) response = http_handler.getresponse() diff --git a/couchpotato/core/notifications/synoindex/__init__.py b/couchpotato/core/notifications/synoindex/__init__.py index eb3a793f..89d07b06 100644 --- a/couchpotato/core/notifications/synoindex/__init__.py +++ b/couchpotato/core/notifications/synoindex/__init__.py @@ -1,5 +1,6 @@ from .main import Synoindex + def start(): return Synoindex() diff --git a/couchpotato/core/notifications/synoindex/main.py b/couchpotato/core/notifications/synoindex/main.py index 0f7775d6..ec7a64ef 100644 --- a/couchpotato/core/notifications/synoindex/main.py +++ b/couchpotato/core/notifications/synoindex/main.py @@ -26,7 +26,7 @@ class Synoindex(Notification): out = p.communicate() log.info('Result from synoindex: %s', str(out)) return True - except OSError, e: + except OSError as e: log.error('Unable to run synoindex: %s', e) return False diff --git a/couchpotato/core/notifications/toasty/__init__.py b/couchpotato/core/notifications/toasty/__init__.py index 8e2dae76..31e055a0 100644 --- a/couchpotato/core/notifications/toasty/__init__.py +++ b/couchpotato/core/notifications/toasty/__init__.py @@ -1,5 +1,6 @@ from .main import Toasty + def start(): return Toasty() diff --git a/couchpotato/core/notifications/toasty/main.py b/couchpotato/core/notifications/toasty/main.py index c65b6b42..ea1f2192 100644 --- a/couchpotato/core/notifications/toasty/main.py +++ b/couchpotato/core/notifications/toasty/main.py @@ -5,6 +5,7 @@ import traceback log = CPLog(__name__) + class Toasty(Notification): urls = { diff --git a/couchpotato/core/notifications/trakt/__init__.py b/couchpotato/core/notifications/trakt/__init__.py index b119736c..20e2e3f9 100644 --- a/couchpotato/core/notifications/trakt/__init__.py +++ b/couchpotato/core/notifications/trakt/__init__.py @@ -1,5 +1,6 @@ from .main import Trakt + def start(): return Trakt() diff --git a/couchpotato/core/notifications/trakt/main.py b/couchpotato/core/notifications/trakt/main.py index 99d55530..399f76d8 100644 --- a/couchpotato/core/notifications/trakt/main.py +++ b/couchpotato/core/notifications/trakt/main.py @@ -3,12 +3,14 @@ from couchpotato.core.notifications.base import Notification log = CPLog(__name__) + class Trakt(Notification): urls = { 'base': 'http://api.trakt.tv/%s', 'library': 'movie/library/%s', 'unwatchlist': 'movie/unwatchlist/%s', + 'test': 'account/test/%s', } listen_to = ['movie.downloaded'] @@ -16,26 +18,40 @@ class Trakt(Notification): def notify(self, message = '', data = None, listener = None): if not data: data = {} - post_data = { - 'username': self.conf('automation_username'), - 'password' : self.conf('automation_password'), - 'movies': [{ - 'imdb_id': data['library']['identifier'], - 'title': data['library']['titles'][0]['title'], - 'year': data['library']['year'] - }] if data else [] - } + if listener == 'test': - result = self.call((self.urls['library'] % self.conf('automation_api_key')), post_data) - if self.conf('remove_watchlist_enabled'): - result = result and self.call((self.urls['unwatchlist'] % self.conf('automation_api_key')), post_data) + post_data = { + 'username': self.conf('automation_username'), + 'password': self.conf('automation_password'), + } - return result + result = self.call((self.urls['test'] % self.conf('automation_api_key')), post_data) + + return result + + else: + + post_data = { + 'username': self.conf('automation_username'), + 'password': self.conf('automation_password'), + 'movies': [{ + 'imdb_id': data['library']['identifier'], + 'title': data['library']['titles'][0]['title'], + 'year': data['library']['year'] + }] if data else [] + } + + result = self.call((self.urls['library'] % self.conf('automation_api_key')), post_data) + if self.conf('remove_watchlist_enabled'): + result = result and self.call((self.urls['unwatchlist'] % self.conf('automation_api_key')), post_data) + + return result def call(self, method_url, post_data): try: - response = self.getJsonData(self.urls['base'] % method_url, params = post_data, cache_timeout = 1) + + response = self.getJsonData(self.urls['base'] % method_url, data = post_data, cache_timeout = 1) if response: if response.get('status') == "success": log.info('Successfully called Trakt') diff --git a/couchpotato/core/notifications/twitter/__init__.py b/couchpotato/core/notifications/twitter/__init__.py index 9db8dcb8..1b9c7699 100644 --- a/couchpotato/core/notifications/twitter/__init__.py +++ b/couchpotato/core/notifications/twitter/__init__.py @@ -1,5 +1,6 @@ from .main import Twitter + def start(): return Twitter() diff --git a/couchpotato/core/notifications/twitter/main.py b/couchpotato/core/notifications/twitter/main.py index ad4fc315..559c830f 100644 --- a/couchpotato/core/notifications/twitter/main.py +++ b/couchpotato/core/notifications/twitter/main.py @@ -64,7 +64,7 @@ class Twitter(Notification): api.PostUpdate(update_message[135:] + ' 2/2') else: api.PostUpdate(update_message) - except Exception, e: + except Exception as e: log.error('Error sending tweet: %s', e) return False diff --git a/couchpotato/core/notifications/xbmc/__init__.py b/couchpotato/core/notifications/xbmc/__init__.py index dafa0f63..34fed632 100644 --- a/couchpotato/core/notifications/xbmc/__init__.py +++ b/couchpotato/core/notifications/xbmc/__init__.py @@ -1,5 +1,6 @@ from .main import XBMC + def start(): return XBMC() @@ -46,6 +47,14 @@ config = [{ 'advanced': True, 'description': 'Only scan new movie folder at remote XBMC servers. Works if movie location is the same.', }, + { + 'name': 'force_full_scan', + 'label': 'Always do a full scan', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Do a full scan instead of only the new movie. Useful if the XBMC path is different from the path CPS uses.', + }, { 'name': 'on_snatch', 'default': 0, diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index dc185c41..bfda85e1 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -1,12 +1,13 @@ from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification -from urllib2 import URLError import base64 import json import socket import traceback import urllib +import requests +from requests.packages.urllib3.exceptions import MaxRetryError log = CPLog(__name__) @@ -36,7 +37,7 @@ class XBMC(Notification): if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0): param = {} - if self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0]): + if not self.conf('force_full_scan') and (self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0])): param = {'directory': data['destination_dir']} calls.append(('VideoLibrary.Scan', param)) @@ -44,7 +45,7 @@ class XBMC(Notification): max_successful += len(calls) response = self.request(host, calls) else: - response = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message}) + response = self.notifyXBMCnoJSON(host, {'title': self.default_title, 'message': message}) if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0): response += self.request(host, [('VideoLibrary.Scan', {})]) @@ -167,22 +168,18 @@ class XBMC(Notification): # manually fake expected response array return [{'result': 'Error'}] - except URLError, e: - if isinstance(e.reason, socket.timeout): - log.info('Couldn\'t send request to XBMC, assuming it\'s turned off') - return [{'result': 'Error'}] - else: - log.error('Failed sending non-JSON-type request to XBMC: %s', traceback.format_exc()) - return [{'result': 'Error'}] + except (MaxRetryError, requests.exceptions.Timeout): + log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') + return [{'result': 'Error'}] except: log.error('Failed sending non-JSON-type request to XBMC: %s', traceback.format_exc()) return [{'result': 'Error'}] - def request(self, host, requests): + def request(self, host, do_requests): server = 'http://%s/jsonrpc' % host data = [] - for req in requests: + for req in do_requests: method, kwargs = req data.append({ 'method': method, @@ -202,17 +199,13 @@ class XBMC(Notification): try: log.debug('Sending request to %s: %s', (host, data)) - response = self.getJsonData(server, headers = headers, params = data, timeout = 3, show_error = False) + response = self.getJsonData(server, headers = headers, data = data, timeout = 3, show_error = False) log.debug('Returned from request %s: %s', (host, response)) return response - except URLError, e: - if isinstance(e.reason, socket.timeout): - log.info('Couldn\'t send request to XBMC, assuming it\'s turned off') - return [] - else: - log.error('Failed sending request to XBMC: %s', traceback.format_exc()) - return [] + except (MaxRetryError, requests.exceptions.Timeout): + log.info2('Couldn\'t send request to XBMC, assuming it\'s turned off') + return [] except: log.error('Failed sending request to XBMC: %s', traceback.format_exc()) return [] diff --git a/couchpotato/core/notifications/xmpp/__init__.py b/couchpotato/core/notifications/xmpp/__init__.py index a52242ff..0e3e14d9 100644 --- a/couchpotato/core/notifications/xmpp/__init__.py +++ b/couchpotato/core/notifications/xmpp/__init__.py @@ -1,5 +1,6 @@ from .main import Xmpp + def start(): return Xmpp() diff --git a/couchpotato/core/plugins/automation/__init__.py b/couchpotato/core/plugins/automation/__init__.py index 440232b2..482a0090 100644 --- a/couchpotato/core/plugins/automation/__init__.py +++ b/couchpotato/core/plugins/automation/__init__.py @@ -1,5 +1,6 @@ from .main import Automation + def start(): return Automation() @@ -41,7 +42,7 @@ config = [{ 'label': 'Required Genres', 'default': '', 'placeholder': 'Example: Action, Crime & Drama', - 'description': 'Ignore movies that don\'t contain at least one set of genres. Sets are separated by "," and each word within a set must be separated with "&"' + 'description': ('Ignore movies that don\'t contain at least one set of genres.', 'Sets are separated by "," and each word within a set must be separated with "&"') }, { 'name': 'ignored_genres', diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 92547cb0..2edcd3be 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -43,7 +43,7 @@ class Automation(Plugin): if self.shuttingDown(): break - movie_dict = fireEvent('movie.get', movie_id, single = True) + movie_dict = fireEvent('media.get', movie_id, single = True) fireEvent('movie.searcher.single', movie_dict) - return True \ No newline at end of file + return True diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 649e359d..d7487a10 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -1,19 +1,17 @@ -from StringIO import StringIO from couchpotato.core.event import fireEvent, addEvent -from couchpotato.core.helpers.encoding import tryUrlencode, ss, toSafeString, \ +from couchpotato.core.helpers.encoding import ss, toSafeString, \ toUnicode, sp -from couchpotato.core.helpers.variable import getExt, md5, isLocalIP +from couchpotato.core.helpers.variable import getExt, md5, isLocalIP, scanForPassword, tryInt from couchpotato.core.logger import CPLog from couchpotato.environment import Env -from multipartpost import MultipartPostHandler +import requests +from requests.packages.urllib3 import Timeout +from requests.packages.urllib3.exceptions import MaxRetryError from tornado import template from tornado.web import StaticFileHandler from urlparse import urlparse -import cookielib import glob -import gzip import inspect -import math import os.path import re import time @@ -39,6 +37,7 @@ class Plugin(object): http_time_between_calls = 0 http_failed_request = {} http_failed_disabled = {} + http_opener = requests.Session() def __new__(typ, *args, **kwargs): new_plugin = super(Plugin, typ).__new__(typ) @@ -55,8 +54,11 @@ class Plugin(object): self.registerStatic(inspect.getfile(self.__class__)) def conf(self, attr, value = None, default = None, section = None): - class_name = self.getName().lower().split(':') - return Env.setting(attr, section = section if section else class_name[0].lower(), value = value, default = default) + class_name = self.getName().lower().split(':')[0].lower() + return Env.setting(attr, section = section if section else class_name, value = value, default = default) + + def deleteConf(self, attr): + return Env._settings.delete(attr, section = self.getName().lower().split(':')[0].lower()) def getName(self): return self._class_name or self.__class__.__name__ @@ -83,7 +85,7 @@ class Plugin(object): class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() # View path - path = 'static/plugin/%s/' % (class_name) + path = 'static/plugin/%s/' % class_name # Add handler to Tornado Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})]) @@ -100,13 +102,18 @@ class Plugin(object): self.makeDir(os.path.dirname(path)) + if os.path.exists(path): + log.debug('%s already exists, overwriting file with new version', path) + try: f = open(path, 'w+' if not binary else 'w+b') f.write(content) f.close() os.chmod(path, Env.getPermission('file')) - except Exception, e: - log.error('Unable writing to file "%s": %s', (path, e)) + except: + log.error('Unable writing to file "%s": %s', (path, traceback.format_exc())) + if os.path.isfile(path): + os.remove(path) def makeDir(self, path): path = ss(path) @@ -114,17 +121,17 @@ class Plugin(object): if not os.path.isdir(path): os.makedirs(path, Env.getPermission('folder')) return True - except Exception, e: + except Exception as e: log.error('Unable to create folder "%s": %s', (path, e)) return False # http request - def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True): + def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True): url = urllib2.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]") if not headers: headers = {} - if not params: params = {} + if not data: data = {} # Fill in some headers parsed_url = urlparse(url) @@ -137,6 +144,8 @@ class Plugin(object): headers['Connection'] = headers.get('Connection', 'keep-alive') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') + r = self.http_opener + # Don't try for failed requests if self.http_failed_disabled.get(host, 0) > 0: if self.http_failed_disabled[host] > (time.time() - 900): @@ -152,50 +161,26 @@ class Plugin(object): self.wait(host) try: - # Make sure opener has the correct headers - if opener: - opener.add_headers = headers + kwargs = { + 'headers': headers, + 'data': data if len(data) > 0 else None, + 'timeout': timeout, + 'files': files, + } + method = 'post' if len(data) > 0 or files else 'get' - if multipart: - log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) - request = urllib2.Request(url, params, headers) + log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data')) + response = r.request(method, url, verify = False, **kwargs) - if opener: - opener.add_handler(MultipartPostHandler()) - else: - cookies = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler) - - response = opener.open(request, timeout = timeout) + if response.status_code == requests.codes.ok: + data = response.content else: - log.info('Opening url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data')) - - if isinstance(params, (str, unicode)) and len(params) > 0: - data = params - else: - data = tryUrlencode(params) if len(params) > 0 else None - - request = urllib2.Request(url, data, headers) - - if opener: - response = opener.open(request, timeout = timeout) - else: - response = urllib2.urlopen(request, timeout = timeout) - - # unzip if needed - if response.info().get('Content-Encoding') == 'gzip': - buf = StringIO(response.read()) - f = gzip.GzipFile(fileobj = buf) - data = f.read() - f.close() - else: - data = response.read() - response.close() + response.raise_for_status() self.http_failed_request[host] = 0 - except IOError: + except (IOError, MaxRetryError, Timeout): if show_error: - log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(1))) + log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(0))) # Save failed requests by hosts try: @@ -218,15 +203,19 @@ class Plugin(object): return data def wait(self, host = ''): + if self.http_time_between_calls == 0: + return + now = time.time() last_use = self.http_last_use.get(host, 0) + if last_use > 0: - wait = math.ceil(last_use - now + self.http_time_between_calls) + wait = (last_use - now) + self.http_time_between_calls - if wait > 0: - log.debug('Waiting for %s, %d seconds', (self.getName(), wait)) - time.sleep(last_use - now + self.http_time_between_calls) + if wait > 0: + log.debug('Waiting for %s, %d seconds', (self.getName(), wait)) + time.sleep(wait) def beforeCall(self, handler): self.isRunning('%s.%s' % (self.getName(), handler.__name__)) @@ -257,30 +246,34 @@ class Plugin(object): except: log.error("Something went wrong when finishing the plugin function. Could not find the 'is_running' key") - def getCache(self, cache_key, url = None, **kwargs): - cache_key_md5 = md5(cache_key) - cache = Env.get('cache').get(cache_key_md5) - if cache: - if not Env.get('dev'): log.debug('Getting cache %s', cache_key) - return cache + + use_cache = not len(kwargs.get('data', {})) > 0 and not kwargs.get('files') + + if use_cache: + cache_key_md5 = md5(cache_key) + cache = Env.get('cache').get(cache_key_md5) + if cache: + if not Env.get('dev'): log.debug('Getting cache %s', cache_key) + return cache if url: try: cache_timeout = 300 - if kwargs.get('cache_timeout'): + if 'cache_timeout' in kwargs: cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] data = self.urlopen(url, **kwargs) - if data: + if data and cache_timeout > 0 and use_cache: self.setCache(cache_key, data, timeout = cache_timeout) return data except: if not kwargs.get('show_error', True): raise + log.debug('Failed getting cache: %s', (traceback.format_exc(0))) return '' def setCache(self, cache_key, value, timeout = 300): @@ -289,22 +282,68 @@ class Plugin(object): Env.get('cache').set(cache_key_md5, value, timeout) return value - def createNzbName(self, data, movie): - tag = self.cpTag(movie) - return '%s%s' % (toSafeString(toUnicode(data.get('name'))[:127 - len(tag)]), tag) + def createNzbName(self, data, media): + release_name = data.get('name') + tag = self.cpTag(media) - def createFileName(self, data, filedata, movie): - name = sp(os.path.join(self.createNzbName(data, movie))) + # Check if password is filename + name_password = scanForPassword(data.get('name')) + if name_password: + release_name, password = name_password + tag += '{{%s}}' % password + + max_length = 127 - len(tag) # Some filesystems don't support 128+ long filenames + return '%s%s' % (toSafeString(toUnicode(release_name)[:max_length]), tag) + + def createFileName(self, data, filedata, media): + name = sp(os.path.join(self.createNzbName(data, media))) if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '' not in filedata: return '%s.%s' % (name, 'rar') return '%s.%s' % (name, data.get('protocol')) - def cpTag(self, movie): + def cpTag(self, media): if Env.setting('enabled', 'renamer'): - return '.cp(' + movie['library'].get('identifier') + ')' if movie['library'].get('identifier') else '' + return '.cp(' + media['library'].get('identifier') + ')' if media['library'].get('identifier') else '' return '' + def checkFilesChanged(self, files, unchanged_for = 60): + now = time.time() + file_too_new = False + + for cur_file in files: + + # File got removed while checking + if not os.path.isfile(cur_file): + file_too_new = now + break + + # File has changed in last 60 seconds + file_time = self.getFileTimes(cur_file) + for t in file_time: + if t > now - unchanged_for: + file_too_new = tryInt(time.time() - t) + break + + if file_too_new: + break + + if file_too_new: + try: + time_string = time.ctime(file_time[0]) + except: + try: + time_string = time.ctime(file_time[1]) + except: + time_string = 'unknown' + + return file_too_new, time_string + + return False, None + + def getFileTimes(self, file_path): + return [os.path.getmtime(file_path), os.path.getctime(file_path) if os.name != 'posix' else 0] + def isDisabled(self): return not self.isEnabled() diff --git a/couchpotato/core/plugins/browser/__init__.py b/couchpotato/core/plugins/browser/__init__.py index 976fcd10..fae50657 100644 --- a/couchpotato/core/plugins/browser/__init__.py +++ b/couchpotato/core/plugins/browser/__init__.py @@ -1,5 +1,6 @@ from .main import FileBrowser + def start(): return FileBrowser() diff --git a/couchpotato/core/plugins/browser/main.py b/couchpotato/core/plugins/browser/main.py index 380e6826..956a7680 100644 --- a/couchpotato/core/plugins/browser/main.py +++ b/couchpotato/core/plugins/browser/main.py @@ -4,6 +4,7 @@ from couchpotato.core.plugins.base import Plugin import ctypes import os import string +import six if os.name == 'nt': import imp @@ -14,7 +15,7 @@ if os.name == 'nt': raise ImportError("Missing the win32file module, which is a part of the prerequisite \ pywin32 package. You can get it from http://sourceforge.net/projects/pywin32/files/pywin32/") else: - import win32file #@UnresolvedImport + import win32file #@UnresolvedImport class FileBrowser(Plugin): @@ -96,7 +97,7 @@ class FileBrowser(Plugin): def has_hidden_attribute(self, filepath): try: - attrs = ctypes.windll.kernel32.GetFileAttributesW(unicode(filepath)) #@UndefinedVariable + attrs = ctypes.windll.kernel32.GetFileAttributesW(six.text_type(filepath)) #@UndefinedVariable assert attrs != -1 result = bool(attrs & 2) except (AttributeError, AssertionError): diff --git a/couchpotato/core/plugins/category/__init__.py b/couchpotato/core/plugins/category/__init__.py index 6dc41df7..dcdae90b 100644 --- a/couchpotato/core/plugins/category/__init__.py +++ b/couchpotato/core/plugins/category/__init__.py @@ -1,5 +1,6 @@ from .main import CategoryPlugin + def start(): return CategoryPlugin() diff --git a/couchpotato/core/plugins/category/main.py b/couchpotato/core/plugins/category/main.py index 87cd0ea4..c7abaee4 100644 --- a/couchpotato/core/plugins/category/main.py +++ b/couchpotato/core/plugins/category/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent @@ -41,81 +42,116 @@ class CategoryPlugin(Plugin): for category in categories: temp.append(category.to_dict()) - db.expire_all() return temp def save(self, **kwargs): - db = get_session() + try: + db = get_session() - c = db.query(Category).filter_by(id = kwargs.get('id')).first() - if not c: - c = Category() - db.add(c) + c = db.query(Category).filter_by(id = kwargs.get('id')).first() + if not c: + c = Category() + db.add(c) - c.order = kwargs.get('order', c.order if c.order else 0) - c.label = toUnicode(kwargs.get('label', '')) - c.ignored = toUnicode(kwargs.get('ignored', '')) - c.preferred = toUnicode(kwargs.get('preferred', '')) - c.required = toUnicode(kwargs.get('required', '')) - c.destination = toUnicode(kwargs.get('destination', '')) + c.order = kwargs.get('order', c.order if c.order else 0) + c.label = toUnicode(kwargs.get('label', '')) + c.ignored = toUnicode(kwargs.get('ignored', '')) + c.preferred = toUnicode(kwargs.get('preferred', '')) + c.required = toUnicode(kwargs.get('required', '')) + c.destination = toUnicode(kwargs.get('destination', '')) - db.commit() + db.commit() - category_dict = c.to_dict() + category_dict = c.to_dict() + + return { + 'success': True, + 'category': category_dict + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True, - 'category': category_dict + 'success': False, + 'category': None } def saveOrder(self, **kwargs): - db = get_session() + try: + db = get_session() - order = 0 - for category_id in kwargs.get('ids', []): - c = db.query(Category).filter_by(id = category_id).first() - c.order = order + order = 0 + for category_id in kwargs.get('ids', []): + c = db.query(Category).filter_by(id = category_id).first() + c.order = order - order += 1 + order += 1 - db.commit() + db.commit() + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def delete(self, id = None, **kwargs): - db = get_session() - - success = False - message = '' try: - c = db.query(Category).filter_by(id = id).first() - db.delete(c) - db.commit() + db = get_session() - # Force defaults on all empty category movies - self.removeFromMovie(id) + success = False + message = '' + try: + c = db.query(Category).filter_by(id = id).first() + db.delete(c) + db.commit() - success = True - except Exception, e: - message = log.error('Failed deleting category: %s', e) + # Force defaults on all empty category movies + self.removeFromMovie(id) + + success = True + except Exception as e: + message = log.error('Failed deleting category: %s', e) + + return { + 'success': success, + 'message': message + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() return { - 'success': success, - 'message': message + 'success': False } def removeFromMovie(self, category_id): - db = get_session() - movies = db.query(Media).filter(Media.category_id == category_id).all() + try: + db = get_session() + movies = db.query(Media).filter(Media.category_id == category_id).all() - if len(movies) > 0: - for movie in movies: - movie.category_id = None - db.commit() + if len(movies) > 0: + for movie in movies: + movie.category_id = None + db.commit() + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/core/plugins/custom/__init__.py b/couchpotato/core/plugins/custom/__init__.py index 573cd99f..20a39351 100644 --- a/couchpotato/core/plugins/custom/__init__.py +++ b/couchpotato/core/plugins/custom/__init__.py @@ -1,5 +1,6 @@ from .main import Custom + def start(): return Custom() diff --git a/couchpotato/core/plugins/dashboard/__init__.py b/couchpotato/core/plugins/dashboard/__init__.py index 81279291..c43a44eb 100644 --- a/couchpotato/core/plugins/dashboard/__init__.py +++ b/couchpotato/core/plugins/dashboard/__init__.py @@ -1,5 +1,6 @@ from .main import Dashboard + def start(): return Dashboard() diff --git a/couchpotato/core/plugins/dashboard/main.py b/couchpotato/core/plugins/dashboard/main.py index 4f4d85ab..3367ffb7 100644 --- a/couchpotato/core/plugins/dashboard/main.py +++ b/couchpotato/core/plugins/dashboard/main.py @@ -1,3 +1,4 @@ +from datetime import date from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent @@ -65,6 +66,7 @@ class Dashboard(Plugin): active = q.all() movies = [] + now_year = date.today().year if len(active) > 0: @@ -91,8 +93,8 @@ class Dashboard(Plugin): if coming_soon: # Don't list older movies - if ((not late and (not eta.get('dvd') and not eta.get('theater') or eta.get('dvd') and eta.get('dvd') > (now - 2419200))) or - (late and (eta.get('dvd', 0) > 0 or eta.get('theater')) and eta.get('dvd') < (now - 2419200))): + if ((not late and (year >= now_year-1) and (not eta.get('dvd') and not eta.get('theater') or eta.get('dvd') and eta.get('dvd') > (now - 2419200))) or + (late and ((year < now_year-1) or ((eta.get('dvd', 0) > 0 or eta.get('theater')) and eta.get('dvd') < (now - 2419200))))): movie_ids.append(movie_id) if len(movie_ids) >= limit: @@ -115,7 +117,7 @@ class Dashboard(Plugin): for movie_id in movie_ids: movies.append(movie_dict[movie_id].to_dict({ - 'library': {'titles': {}, 'files':{}}, + 'library': {'titles': {}, 'files': {}}, 'files': {}, })) diff --git a/couchpotato/core/plugins/file/__init__.py b/couchpotato/core/plugins/file/__init__.py index 54d9cbe5..3dced3d0 100644 --- a/couchpotato/core/plugins/file/__init__.py +++ b/couchpotato/core/plugins/file/__init__.py @@ -1,5 +1,6 @@ from .main import FileManager + def start(): return FileManager() diff --git a/couchpotato/core/plugins/file/main.py b/couchpotato/core/plugins/file/main.py index fc63aca8..c52a9801 100644 --- a/couchpotato/core/plugins/file/main.py +++ b/couchpotato/core/plugins/file/main.py @@ -66,7 +66,6 @@ class FileManager(Plugin): time.sleep(3) log.debug('Cleaning up unused files') - python_cache = Env.get('cache')._path try: db = get_session() for root, dirs, walk_files in os.walk(Env.get('cache_dir')): @@ -78,11 +77,13 @@ class FileManager(Plugin): os.remove(file_path) except: log.error('Failed removing unused file: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def showCacheFile(self, route, **kwargs): Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), StaticFileHandler, {'path': Env.get('cache_dir')})]) - def download(self, url = '', dest = None, overwrite = False, urlopen_kwargs = None): if not urlopen_kwargs: urlopen_kwargs = {} @@ -104,42 +105,56 @@ class FileManager(Plugin): def add(self, path = '', part = 1, type_tuple = (), available = 1, properties = None): if not properties: properties = {} - type_id = self.getType(type_tuple).get('id') - db = get_session() + try: + db = get_session() + type_id = self.getType(type_tuple).get('id') - f = db.query(File).filter(File.path == toUnicode(path)).first() - if not f: - f = File() - db.add(f) + f = db.query(File).filter(File.path == toUnicode(path)).first() + if not f: + f = File() + db.add(f) - f.path = toUnicode(path) - f.part = part - f.available = available - f.type_id = type_id + f.path = toUnicode(path) + f.part = part + f.available = available + f.type_id = type_id - db.commit() + db.commit() - file_dict = f.to_dict() + file_dict = f.to_dict() - return file_dict + return file_dict + except: + log.error('Failed adding file: %s, %s', (path, traceback.format_exc())) + db.rollback() + finally: + db.close() def getType(self, type_tuple): - db = get_session() - type_type, type_identifier = type_tuple + try: + db = get_session() + type_type, type_identifier = type_tuple - ft = db.query(FileType).filter_by(identifier = type_identifier).first() - if not ft: - ft = FileType( - type = toUnicode(type_type), - identifier = type_identifier, - name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:]) - ) - db.add(ft) - db.commit() + ft = db.query(FileType).filter_by(identifier = type_identifier).first() + if not ft: + ft = FileType( + type = toUnicode(type_type), + identifier = type_identifier, + name = toUnicode(type_identifier[0].capitalize() + type_identifier[1:]) + ) + db.add(ft) + db.commit() + + type_dict = ft.to_dict() + + return type_dict + except: + log.error('Failed getting type: %s, %s', (type_tuple, traceback.format_exc())) + db.rollback() + finally: + db.close() - type_dict = ft.to_dict() - return type_dict def getTypes(self): diff --git a/couchpotato/core/plugins/log/__init__.py b/couchpotato/core/plugins/log/__init__.py index 33dcf338..f5d9d105 100644 --- a/couchpotato/core/plugins/log/__init__.py +++ b/couchpotato/core/plugins/log/__init__.py @@ -1,5 +1,6 @@ from .main import Logging + def start(): return Logging() diff --git a/couchpotato/core/plugins/log/main.py b/couchpotato/core/plugins/log/main.py index dc8f740f..2f471586 100644 --- a/couchpotato/core/plugins/log/main.py +++ b/couchpotato/core/plugins/log/main.py @@ -42,7 +42,7 @@ class Logging(Plugin): 'desc': 'Log errors', 'params': { 'type': {'desc': 'Type of logging, default "error"'}, - '**kwargs': {'type':'object', 'desc': 'All other params will be printed in the log string.'}, + '**kwargs': {'type': 'object', 'desc': 'All other params will be printed in the log string.'}, } }) diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index e9e4af05..159bfeaa 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -73,10 +73,10 @@ Page.Log = new Class({ .replace(/\u001b\[31m/gi, '') .replace(/\u001b\[36m/gi, '') .replace(/\u001b\[33m/gi, '') - .replace(/\u001b\[0m\n/gi, '') + .replace(/\u001b\[0m\n/gi, '
') .replace(/\u001b\[0m/gi, '') - return '' + text + ''; + return '
' + text + '
'; } -}) \ No newline at end of file +}) diff --git a/couchpotato/core/plugins/manage/__init__.py b/couchpotato/core/plugins/manage/__init__.py index 912296b4..c992dee6 100644 --- a/couchpotato/core/plugins/manage/__init__.py +++ b/couchpotato/core/plugins/manage/__init__.py @@ -1,5 +1,6 @@ from .main import Manage + def start(): return Manage() diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index e8ccaf7e..2f297491 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -1,7 +1,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent, fireEventAsync -from couchpotato.core.helpers.encoding import ss -from couchpotato.core.helpers.variable import splitString, getTitle +from couchpotato.core.helpers.encoding import sp +from couchpotato.core.helpers.variable import splitString, getTitle, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -14,6 +14,7 @@ import traceback log = CPLog(__name__) + class Manage(Plugin): in_progress = False @@ -58,6 +59,7 @@ class Manage(Plugin): fireEventAsync('manage.update', full = True if full == '1' else False) return { + 'progress': self.in_progress, 'success': True } @@ -79,18 +81,21 @@ class Manage(Plugin): try: directories = self.directories() + directories.sort() added_identifiers = [] # Add some progress - self.in_progress = {} for directory in directories: self.in_progress[os.path.normpath(directory)] = { + 'started': False, + 'eta': -1, 'total': None, 'to_go': None, } for directory in directories: folder = os.path.normpath(directory) + self.in_progress[os.path.normpath(directory)]['started'] = tryInt(time.time()) if not os.path.isdir(folder): if len(directory) > 0: @@ -100,6 +105,7 @@ class Manage(Plugin): log.info('Updating manage library: %s', folder) fireEvent('notify.frontend', type = 'manage.update', data = True, message = 'Scanning for movies in "%s"' % folder) + onFound = self.createAddToLibrary(folder, added_identifiers) fireEvent('scanner.scan', folder = folder, simple = True, newer_than = last_update if not full else 0, on_found = onFound, single = True) @@ -111,22 +117,20 @@ class Manage(Plugin): if self.conf('cleanup') and full and not self.shuttingDown(): # Get movies with done status - total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True) + total_movies, done_movies = fireEvent('media.list', types = 'movie', status = 'done', single = True) for done_movie in done_movies: if done_movie['library']['identifier'] not in added_identifiers: - fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all') + fireEvent('media.delete', media_id = done_movie['id'], delete_from = 'all') else: releases = fireEvent('release.for_movie', id = done_movie.get('id'), single = True) for release in releases: - if len(release.get('files', [])) == 0: - fireEvent('release.delete', release['id']) - else: + if len(release.get('files', [])) > 0: for release_file in release.get('files', []): # Remove release not available anymore - if not os.path.isfile(ss(release_file['path'])): + if not os.path.isfile(sp(release_file['path'])): fireEvent('release.clean', release['id']) break @@ -175,10 +179,10 @@ class Manage(Plugin): def addToLibrary(group, total_found, to_go): if self.in_progress[folder]['total'] is None: - self.in_progress[folder] = { + self.in_progress[folder].update({ 'total': total_found, 'to_go': total_found, - } + }) if group['library'] and group['library'].get('identifier'): identifier = group['library'].get('identifier') @@ -186,9 +190,9 @@ class Manage(Plugin): # Add it to release and update the info fireEvent('release.add', group = group) - fireEventAsync('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier)) + fireEvent('library.update.movie', identifier = identifier, on_complete = self.createAfterUpdate(folder, identifier)) else: - self.in_progress[folder]['to_go'] -= 1 + self.updateProgress(folder) return addToLibrary @@ -199,14 +203,23 @@ class Manage(Plugin): if not self.in_progress or self.shuttingDown(): return - self.in_progress[folder]['to_go'] -= 1 + self.updateProgress(folder) total = self.in_progress[folder]['total'] - movie_dict = fireEvent('movie.get', identifier, single = True) + movie_dict = fireEvent('media.get', identifier, single = True) fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict['library'])) return afterUpdate + def updateProgress(self, folder): + + pr = self.in_progress[folder] + pr['to_go'] -= 1 + + avg = (time.time() - pr['started'])/(pr['total'] - pr['to_go']) + pr['eta'] = tryInt(avg * pr['to_go']) + + def directories(self): try: if self.conf('library', default = '').strip(): @@ -223,7 +236,7 @@ class Manage(Plugin): groups = fireEvent('scanner.scan', folder = folder, files = files, single = True) if groups: - for group in groups.itervalues(): + for group in groups.values(): if group['library'] and group['library'].get('identifier'): fireEvent('release.add', group = group) diff --git a/couchpotato/core/plugins/profile/__init__.py b/couchpotato/core/plugins/profile/__init__.py index ac19b018..c07bc7c5 100644 --- a/couchpotato/core/plugins/profile/__init__.py +++ b/couchpotato/core/plugins/profile/__init__.py @@ -1,5 +1,6 @@ from .main import ProfilePlugin + def start(): return ProfilePlugin() diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 9ff3ead2..914d46f3 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent @@ -37,14 +38,20 @@ class ProfilePlugin(Plugin): # Get all active movies without profile active_status = fireEvent('status.get', 'active', single = True) - db = get_session() - movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all() + try: + db = get_session() + movies = db.query(Media).filter(Media.status_id == active_status.get('id'), Media.profile == None).all() - if len(movies) > 0: - default_profile = self.default() - for movie in movies: - movie.profile_id = default_profile.get('id') - db.commit() + if len(movies) > 0: + default_profile = self.default() + for movie in movies: + movie.profile_id = default_profile.get('id') + db.commit() + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def allView(self, **kwargs): @@ -64,44 +71,53 @@ class ProfilePlugin(Plugin): for profile in profiles: temp.append(profile.to_dict(self.to_dict)) - db.expire_all() return temp def save(self, **kwargs): - db = get_session() + try: + db = get_session() - p = db.query(Profile).filter_by(id = kwargs.get('id')).first() - if not p: - p = Profile() - db.add(p) + p = db.query(Profile).filter_by(id = kwargs.get('id')).first() + if not p: + p = Profile() + db.add(p) - p.label = toUnicode(kwargs.get('label')) - p.order = kwargs.get('order', p.order if p.order else 0) - p.core = kwargs.get('core', False) + p.label = toUnicode(kwargs.get('label')) + p.order = kwargs.get('order', p.order if p.order else 0) + p.core = kwargs.get('core', False) - #delete old types - [db.delete(t) for t in p.types] + #delete old types + [db.delete(t) for t in p.types] - order = 0 - for type in kwargs.get('types', []): - t = ProfileType( - order = order, - finish = type.get('finish') if order > 0 else 1, - wait_for = kwargs.get('wait_for'), - quality_id = type.get('quality_id') - ) - p.types.append(t) + order = 0 + for type in kwargs.get('types', []): + t = ProfileType( + order = order, + finish = type.get('finish') if order > 0 else 1, + wait_for = kwargs.get('wait_for'), + quality_id = type.get('quality_id') + ) + p.types.append(t) - order += 1 + order += 1 - db.commit() + db.commit() - profile_dict = p.to_dict(self.to_dict) + profile_dict = p.to_dict(self.to_dict) + + return { + 'success': True, + 'profile': profile_dict + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True, - 'profile': profile_dict + 'success': False } def default(self): @@ -112,93 +128,119 @@ class ProfilePlugin(Plugin): .first() default_dict = default.to_dict(self.to_dict) - db.expire_all() return default_dict def saveOrder(self, **kwargs): - db = get_session() + try: + db = get_session() - order = 0 - for profile in kwargs.get('ids', []): - p = db.query(Profile).filter_by(id = profile).first() - p.hide = kwargs.get('hidden')[order] - p.order = order + order = 0 + for profile in kwargs.get('ids', []): + p = db.query(Profile).filter_by(id = profile).first() + p.hide = kwargs.get('hidden')[order] + p.order = order - order += 1 + order += 1 - db.commit() + db.commit() + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def delete(self, id = None, **kwargs): - db = get_session() - - success = False - message = '' try: - p = db.query(Profile).filter_by(id = id).first() + db = get_session() - db.delete(p) - db.commit() + success = False + message = '' + try: + p = db.query(Profile).filter_by(id = id).first() - # Force defaults on all empty profile movies - self.forceDefaults() + db.delete(p) + db.commit() - success = True - except Exception, e: - message = log.error('Failed deleting Profile: %s', e) + # Force defaults on all empty profile movies + self.forceDefaults() + + success = True + except Exception as e: + message = log.error('Failed deleting Profile: %s', e) + + return { + 'success': success, + 'message': message + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - db.expire_all() return { - 'success': success, - 'message': message + 'success': False } def fill(self): - db = get_session() + try: + db = get_session() - profiles = [{ - 'label': 'Best', - 'qualities': ['720p', '1080p', 'brrip', 'dvdrip'] - }, { - 'label': 'HD', - 'qualities': ['720p', '1080p'] - }, { - 'label': 'SD', - 'qualities': ['dvdrip', 'dvdr'] - }] + profiles = [{ + 'label': 'Best', + 'qualities': ['720p', '1080p', 'brrip', 'dvdrip'] + }, { + 'label': 'HD', + 'qualities': ['720p', '1080p'] + }, { + 'label': 'SD', + 'qualities': ['dvdrip', 'dvdr'] + }] - # Create default quality profile - order = -2 - for profile in profiles: - log.info('Creating default profile: %s', profile.get('label')) - p = Profile( - label = toUnicode(profile.get('label')), - order = order - ) - db.add(p) - - quality_order = 0 - for quality in profile.get('qualities'): - quality = fireEvent('quality.single', identifier = quality, single = True) - profile_type = ProfileType( - quality_id = quality.get('id'), - profile = p, - finish = True, - wait_for = 0, - order = quality_order + # Create default quality profile + order = -2 + for profile in profiles: + log.info('Creating default profile: %s', profile.get('label')) + p = Profile( + label = toUnicode(profile.get('label')), + order = order ) - p.types.append(profile_type) + db.add(p) - quality_order += 1 + quality_order = 0 + for quality in profile.get('qualities'): + quality = fireEvent('quality.single', identifier = quality, single = True) + profile_type = ProfileType( + quality_id = quality.get('id'), + profile = p, + finish = True, + wait_for = 0, + order = quality_order + ) + p.types.append(profile_type) - order += 1 + quality_order += 1 - db.commit() + order += 1 - return True + db.commit() + + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return False diff --git a/couchpotato/core/plugins/quality/__init__.py b/couchpotato/core/plugins/quality/__init__.py index e1b97ad0..2630f1a3 100644 --- a/couchpotato/core/plugins/quality/__init__.py +++ b/couchpotato/core/plugins/quality/__init__.py @@ -1,5 +1,6 @@ from .main import QualityPlugin + def start(): return QualityPlugin() diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 0c0636e6..80773a84 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -1,8 +1,9 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode, ss -from couchpotato.core.helpers.variable import mergeDicts, md5, getExt +from couchpotato.core.helpers.variable import mergeDicts, getExt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Quality, Profile, ProfileType @@ -19,14 +20,14 @@ class QualityPlugin(Plugin): {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, {'identifier': '1080p', 'hd': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r')]}, - {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':['avi', 'mpg', 'mpeg'], 'tags': ['webrip', ('web', 'rip')]}, - {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']} + {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, + {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, + {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, + {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]}, + {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]}, + {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':[]} ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] @@ -50,6 +51,8 @@ class QualityPlugin(Plugin): addEvent('app.initialize', self.fill, priority = 10) + addEvent('app.test', self.doTest) + def preReleases(self): return self.pre_releases @@ -96,78 +99,97 @@ class QualityPlugin(Plugin): def saveSize(self, **kwargs): - db = get_session() - quality = db.query(Quality).filter_by(identifier = kwargs.get('identifier')).first() + try: + db = get_session() + quality = db.query(Quality).filter_by(identifier = kwargs.get('identifier')).first() - if quality: - setattr(quality, kwargs.get('value_type'), kwargs.get('value')) - db.commit() + if quality: + setattr(quality, kwargs.get('value_type'), kwargs.get('value')) + db.commit() - self.cached_qualities = None + self.cached_qualities = None + + return { + 'success': True + } + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return { - 'success': True + 'success': False } def fill(self): - db = get_session() + try: + db = get_session() - order = 0 - for q in self.qualities: + order = 0 + for q in self.qualities: - # Create quality - qual = db.query(Quality).filter_by(identifier = q.get('identifier')).first() + # Create quality + qual = db.query(Quality).filter_by(identifier = q.get('identifier')).first() - if not qual: - log.info('Creating quality: %s', q.get('label')) - qual = Quality() - qual.order = order - qual.identifier = q.get('identifier') - qual.label = toUnicode(q.get('label')) - qual.size_min, qual.size_max = q.get('size') + if not qual: + log.info('Creating quality: %s', q.get('label')) + qual = Quality() + qual.order = order + qual.identifier = q.get('identifier') + qual.label = toUnicode(q.get('label')) + qual.size_min, qual.size_max = q.get('size') - db.add(qual) + db.add(qual) - # Create single quality profile - prof = db.query(Profile).filter( + # Create single quality profile + prof = db.query(Profile).filter( Profile.core == True ).filter( Profile.types.any(quality = qual) ).all() - if not prof: - log.info('Creating profile: %s', q.get('label')) - prof = Profile( - core = True, - label = toUnicode(qual.label), - order = order - ) - db.add(prof) + if not prof: + log.info('Creating profile: %s', q.get('label')) + prof = Profile( + core = True, + label = toUnicode(qual.label), + order = order + ) + db.add(prof) - profile_type = ProfileType( - quality = qual, - profile = prof, - finish = True, - order = 0 - ) - prof.types.append(profile_type) + profile_type = ProfileType( + quality = qual, + profile = prof, + finish = True, + order = 0 + ) + prof.types.append(profile_type) - order += 1 + order += 1 - db.commit() + db.commit() - time.sleep(0.3) # Wait a moment + time.sleep(0.3) # Wait a moment - return True + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return False def guess(self, files, extra = None): if not extra: extra = {} # Create hash for cache - cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files])) + cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files]) cached = self.getCache(cache_key) - if cached and len(extra) == 0: return cached + if cached and len(extra) == 0: + return cached qualities = self.all() @@ -228,11 +250,6 @@ class QualityPlugin(Plugin): if len(set(words) & set(alt)) == len(alt): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) score += points.get(tag_type) - elif len(set(words) & set(alt)) > 0: - partial = list(set(words) & set(alt))[0] - if len(partial) > 2: - log.debug('Found %s via partial %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) - score += points.get(tag_type) / 3 if (isinstance(alt, (str, unicode)) and ss(alt.lower()) in cur_file.lower()): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) @@ -285,3 +302,36 @@ class QualityPlugin(Plugin): if add_score != 0: for allow in quality.get('allow', []): score[allow] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 + + def doTest(self): + + tests = { + 'Movie Name (1999)-DVD-Rip.avi': 'dvdrip', + 'Movie Name 1999 720p Bluray.mkv': '720p', + 'Movie Name 1999 BR-Rip 720p.avi': 'brrip', + 'Movie Name 1999 720p Web Rip.avi': 'scr', + 'Movie Name 1999 Web DL.avi': 'brrip', + 'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr', + 'Movie.Name.1999.DVDRip-Group': 'dvdrip', + 'Movie.Name.1999.DVD-Rip-Group': 'dvdrip', + 'Movie.Name.1999.DVD-R-Group': 'dvdr', + 'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': '720p', + 'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': '1080p', + 'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': '1080p', + } + + correct = 0 + for name in tests: + success = self.guess([name]).get('identifier') == tests[name] + if not success: + log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier'))) + + correct += success + + if correct == len(tests): + log.info('Quality test successful') + return True + else: + log.error('Quality test failed: %s out of %s succeeded', (correct, len(tests))) + + diff --git a/couchpotato/core/plugins/release/__init__.py b/couchpotato/core/plugins/release/__init__.py index b6a667c2..08c6a57c 100644 --- a/couchpotato/core/plugins/release/__init__.py +++ b/couchpotato/core/plugins/release/__init__.py @@ -1,5 +1,6 @@ from .main import Release + def start(): return Release() diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 009a60e9..a478b64d 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -88,63 +88,69 @@ class Release(Plugin): elif rel.status_id in [snatched_status.get('id'), downloaded_status.get('id')]: self.updateStatus(id = rel.id, status = ignored_status) - db.expire_all() def add(self, group): - db = get_session() - - identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier']) - - - done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) - - # Add movie - movie = db.query(Media).filter_by(library_id = group['library'].get('id')).first() - if not movie: - movie = Media( - library_id = group['library'].get('id'), - profile_id = 0, - status_id = done_status.get('id') - ) - db.add(movie) - db.commit() - - # Add Release - rel = db.query(Relea).filter( - or_( - Relea.identifier == identifier, - and_(Relea.identifier.startswith(group['library']['identifier']), Relea.status_id == snatched_status.get('id')) - ) - ).first() - if not rel: - rel = Relea( - identifier = identifier, - movie = movie, - quality_id = group['meta_data']['quality'].get('id'), - status_id = done_status.get('id') - ) - db.add(rel) - db.commit() - - # Add each file type - added_files = [] - for type in group['files']: - for cur_file in group['files'][type]: - added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie') - added_files.append(added_file.get('id')) - - # Add the release files in batch try: - added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all() - rel.files.extend(added_files) - db.commit() + db = get_session() + + identifier = '%s.%s.%s' % (group['library']['identifier'], group['meta_data'].get('audio', 'unknown'), group['meta_data']['quality']['identifier']) + + done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) + + # Add movie + media = db.query(Media).filter_by(library_id = group['library'].get('id')).first() + if not media: + media = Media( + library_id = group['library'].get('id'), + profile_id = 0, + status_id = done_status.get('id') + ) + db.add(media) + db.commit() + + # Add Release + rel = db.query(Relea).filter( + or_( + Relea.identifier == identifier, + and_(Relea.identifier.startswith(group['library']['identifier']), Relea.status_id == snatched_status.get('id')) + ) + ).first() + if not rel: + rel = Relea( + identifier = identifier, + movie = media, + quality_id = group['meta_data']['quality'].get('id'), + status_id = done_status.get('id') + ) + db.add(rel) + db.commit() + + # Add each file type + added_files = [] + for type in group['files']: + for cur_file in group['files'][type]: + added_file = self.saveFile(cur_file, type = type, include_media_info = type is 'movie') + added_files.append(added_file.get('id')) + + # Add the release files in batch + try: + added_files = db.query(File).filter(or_(*[File.id == x for x in added_files])).all() + rel.files.extend(added_files) + db.commit() + except: + log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc())) + + fireEvent('media.restatus', media.id) + + return True except: - log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc())) + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() - fireEvent('movie.restatus', movie.id) - - return True + return False def saveFile(self, filepath, type = 'unknown', include_media_info = False): @@ -165,31 +171,43 @@ class Release(Plugin): def delete(self, id): - db = get_session() + try: + db = get_session() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - rel.delete() - db.commit() - return True + rel = db.query(Relea).filter_by(id = id).first() + if rel: + rel.delete() + db.commit() + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return False def clean(self, id): - db = get_session() + try: + db = get_session() - rel = db.query(Relea).filter_by(id = id).first() - if rel: - for release_file in rel.files: - if not os.path.isfile(ss(release_file.path)): - db.delete(release_file) - db.commit() + rel = db.query(Relea).filter_by(id = id).first() + if rel: + for release_file in rel.files: + if not os.path.isfile(ss(release_file.path)): + db.delete(release_file) + db.commit() - if len(rel.files) == 0: - self.delete(id) + if len(rel.files) == 0: + self.delete(id) - return True + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() return False @@ -211,119 +229,139 @@ class Release(Plugin): db = get_session() rel = db.query(Relea).filter_by(id = id).first() - if rel: - item = {} - for info in rel.info: - item[info.identifier] = info.value - - fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name']) - - # Get matching provider - provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True) - - if not item.get('protocol'): - item['protocol'] = item['type'] - item['type'] = 'movie' - - if item.get('protocol') != 'torrent_magnet': - item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download - - success = self.download(data = item, media = rel.movie.to_dict({ - 'profile': {'types': {'quality': {}}}, - 'releases': {'status': {}, 'quality': {}}, - 'library': {'titles': {}, 'files':{}}, - 'files': {} - }), manual = True) - - if success: - db.expunge_all() - rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again?? - - fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name']) - return { - 'success': success - } - else: + if not rel: log.error('Couldn\'t find release with id: %s', id) + return { + 'success': False + } + + item = {} + for info in rel.info: + item[info.identifier] = info.value + + fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Snatching "%s"' % item['name']) + + # Get matching provider + provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True) + + # Backwards compatibility code + if not item.get('protocol'): + item['protocol'] = item['type'] + item['type'] = 'movie' + + if item.get('protocol') != 'torrent_magnet': + item['download'] = provider.loginDownload if provider.urls.get('login') else provider.download + + success = self.download(data = item, media = rel.movie.to_dict({ + 'profile': {'types': {'quality': {}}}, + 'releases': {'status': {}, 'quality': {}}, + 'library': {'titles': {}, 'files': {}}, + 'files': {} + }), manual = True) + + db.expunge_all() + + if success: + fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name']) return { - 'success': False + 'success': success == True } def download(self, data, media, manual = False): + # Backwards compatibility code if not data.get('protocol'): data['protocol'] = data['type'] data['type'] = 'movie' # Test to see if any downloaders are enabled for this type downloader_enabled = fireEvent('download.enabled', manual, data, single = True) + if not downloader_enabled: + log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', data.get('protocol')) + return False - if downloader_enabled: - snatched_status, done_status, active_status = fireEvent('status.get', ['snatched', 'done', 'active'], single = True) - - # Download release to temp - filedata = None - if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): + # Download NZB or torrent file + filedata = None + if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))): + try: filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id')) - if filedata == 'try_next': - return filedata + except: + log.error('Tried to download, but the "%s" provider gave an error: %s', (data.get('protocol'), traceback.format_exc())) + return False - download_result = fireEvent('download', data = data, movie = media, manual = manual, filedata = filedata, single = True) - log.debug('Downloader result: %s', download_result) + if filedata == 'try_next': + return filedata + elif not filedata: + return False - if download_result: - try: - # Mark release as snatched - db = get_session() - rls = db.query(Relea).filter_by(identifier = md5(data['url'])).first() - if rls: - renamer_enabled = Env.setting('enabled', 'renamer') + # Send NZB or torrent file to downloader + download_result = fireEvent('download', data = data, media = media, manual = manual, filedata = filedata, single = True) + if not download_result: + log.info('Tried to download, but the "%s" downloader gave an error', data.get('protocol')) + return False + log.debug('Downloader result: %s', download_result) - # Save download-id info if returned - if isinstance(download_result, dict): - for key in download_result: - rls_info = ReleaseInfo( - identifier = 'download_%s' % key, - value = toUnicode(download_result.get(key)) - ) - rls.info.append(rls_info) + snatched_status, done_status, downloaded_status, active_status = fireEvent('status.get', ['snatched', 'done', 'downloaded', 'active'], single = True) + + try: + db = get_session() + rls = db.query(Relea).filter_by(identifier = md5(data['url'])).first() + if not rls: + log.error('No release found to store download information in') + return False + + renamer_enabled = Env.setting('enabled', 'renamer') + + # Save download-id info if returned + if isinstance(download_result, dict): + for key in download_result: + rls_info = ReleaseInfo( + identifier = 'download_%s' % key, + value = toUnicode(download_result.get(key)) + ) + rls.info.append(rls_info) + db.commit() + + log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label) + snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) + log.info(snatch_message) + fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls.to_dict()) + + # Mark release as snatched + if renamer_enabled: + self.updateStatus(rls.id, status = snatched_status) + + # If renamer isn't used, mark media done if finished or release downloaded + else: + if media['status_id'] == active_status.get('id'): + finished = next((True for profile_type in media['profile']['types'] + if profile_type['quality_id'] == rls.quality.id and profile_type['finish']), False) + if finished: + log.info('Renamer disabled, marking media as finished: %s', log_movie) + + # Mark release done + self.updateStatus(rls.id, status = done_status) + + # Mark media done + mdia = db.query(Media).filter_by(id = media['id']).first() + mdia.status_id = done_status.get('id') + mdia.last_edit = int(time.time()) db.commit() - log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label) - snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) - log.info(snatch_message) - fireEvent('%s.snatched' % data['type'], message = snatch_message, data = rls.to_dict()) + return True - # If renamer isn't used, mark media done - if not renamer_enabled: - try: - if media['status_id'] == active_status.get('id'): - for profile_type in media['profile']['types']: - if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: - log.info('Renamer disabled, marking media as finished: %s', log_movie) + # Assume release downloaded + self.updateStatus(rls.id, status = downloaded_status) - # Mark release done - self.updateStatus(rls.id, status = done_status) + except: + log.error('Failed storing download status: %s', traceback.format_exc()) + db.rollback() + return False + finally: + db.close() - # Mark media done - mdia = db.query(Media).filter_by(id = media['id']).first() - mdia.status_id = done_status.get('id') - mdia.last_edit = int(time.time()) - db.commit() - except: - log.error('Failed marking media finished, renamer disabled: %s', traceback.format_exc()) - else: - self.updateStatus(rls.id, status = snatched_status) - - except: - log.error('Failed marking media finished: %s', traceback.format_exc()) - - return True - - log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol'))) - - return False + return True def tryDownloadResult(self, results, media, quality_type, manual = False): ignored_status, failed_status = fireEvent('status.get', ['ignored', 'failed'], single = True) @@ -352,49 +390,58 @@ class Release(Plugin): def createFromSearch(self, search_results, media, quality_type): available_status = fireEvent('status.get', ['available'], single = True) - db = get_session() - found_releases = [] + try: + db = get_session() - for rel in search_results: + found_releases = [] - rel_identifier = md5(rel['url']) - found_releases.append(rel_identifier) + for rel in search_results: - rls = db.query(Relea).filter_by(identifier = rel_identifier).first() - if not rls: - rls = Relea( - identifier = rel_identifier, - movie_id = media.get('id'), - #media_id = media.get('id'), - quality_id = quality_type.get('quality_id'), - status_id = available_status.get('id') - ) - db.add(rls) - else: - [db.delete(old_info) for old_info in rls.info] - rls.last_edit = int(time.time()) + rel_identifier = md5(rel['url']) + found_releases.append(rel_identifier) - db.commit() - - for info in rel: - try: - if not isinstance(rel[info], (str, unicode, int, long, float)): - continue - - rls_info = ReleaseInfo( - identifier = info, - value = toUnicode(rel[info]) + rls = db.query(Relea).filter_by(identifier = rel_identifier).first() + if not rls: + rls = Relea( + identifier = rel_identifier, + movie_id = media.get('id'), + #media_id = media.get('id'), + quality_id = quality_type.get('quality_id'), + status_id = available_status.get('id') ) - rls.info.append(rls_info) - except InterfaceError: - log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) + db.add(rls) + else: + [db.delete(old_info) for old_info in rls.info] + rls.last_edit = int(time.time()) - db.commit() + db.commit() - rel['status_id'] = rls.status_id + for info in rel: + try: + if not isinstance(rel[info], (str, unicode, int, long, float)): + continue - return found_releases + rls_info = ReleaseInfo( + identifier = info, + value = toUnicode(rel[info]) + ) + rls.info.append(rls_info) + except InterfaceError: + log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc())) + + db.commit() + + rel['status_id'] = rls.status_id + + return found_releases + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return [] def forMovie(self, id = None): @@ -406,7 +453,7 @@ class Release(Plugin): .filter(Relea.movie_id == id) \ .all() - releases = [r.to_dict({'info':{}, 'files':{}}) for r in releases_raw] + releases = [r.to_dict({'info': {}, 'files': {}}) for r in releases_raw] releases = sorted(releases, key = lambda k: k['info'].get('score', 0), reverse = True) return releases @@ -423,29 +470,39 @@ class Release(Plugin): def updateStatus(self, id, status = None): if not status: return False - db = get_session() + try: + db = get_session() - rel = db.query(Relea).filter_by(id = id).first() - if rel and status and rel.status_id != status.get('id'): + rel = db.query(Relea).filter_by(id = id).first() + if rel and status and rel.status_id != status.get('id'): - item = {} - for info in rel.info: - item[info.identifier] = info.value + item = {} + for info in rel.info: + item[info.identifier] = info.value - if rel.files: - for file_item in rel.files: - if file_item.type.identifier == 'movie': - release_name = os.path.basename(file_item.path) - break - else: - release_name = item['name'] - #update status in Db - log.debug('Marking release %s as %s', (release_name, status.get("label"))) - rel.status_id = status.get('id') - rel.last_edit = int(time.time()) - db.commit() + release_name = None + if rel.files: + for file_item in rel.files: + if file_item.type.identifier == 'movie': + release_name = os.path.basename(file_item.path) + break + else: + release_name = item['name'] - #Update all movie info as there is no release update function - fireEvent('notify.frontend', type = 'release.update_status.%s' % rel.id, data = status.get('id')) + #update status in Db + log.debug('Marking release %s as %s', (release_name, status.get("label"))) + rel.status_id = status.get('id') + rel.last_edit = int(time.time()) + db.commit() - return True + #Update all movie info as there is no release update function + fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict()) + + return True + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() + + return False diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index c8f6b37f..e238f5eb 100755 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -1,6 +1,7 @@ from couchpotato.core.plugins.renamer.main import Renamer import os + def start(): return Renamer() @@ -93,7 +94,7 @@ config = [{ 'default': 1, 'type': 'int', 'unit': 'min(s)', - 'description': 'Detect movie status every X minutes. Will start the renamer if movie is completed or handle failed download if these options are enabled', + 'description': ('Detect movie status every X minutes.', 'Will start the renamer if movie is completed or handle failed download if these options are enabled'), }, { 'advanced': True, @@ -122,13 +123,13 @@ config = [{ 'advanced': True, 'name': 'separator', 'label': 'File-Separator', - 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', + 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), }, { 'advanced': True, 'name': 'foldersep', 'label': 'Folder-Separator', - 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', + 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), }, { 'name': 'file_action', @@ -136,7 +137,8 @@ config = [{ 'default': 'link', 'type': 'dropdown', 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], - 'description': 'Link or Copy after downloading completed (and allow for seeding), or Move after seeding completed. Link first tries hard link, then sym link and falls back to Copy.', + 'description': ('Link, Copy or Move after download completed.', + 'Link first tries hard link, then sym link and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'), 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index f99c92c0..436107f7 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -3,7 +3,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt, splitString + getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ @@ -17,9 +17,12 @@ import re import shutil import time import traceback +import six +from six.moves import filter log = CPLog(__name__) + class Renamer(Plugin): renaming_started = False @@ -30,10 +33,11 @@ class Renamer(Plugin): 'desc': 'For the renamer to check for new files to rename in a folder', 'params': { 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'}, - 'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'}, - 'files': {'desc': 'Optional: Provide the release files if more releases are in the same movie_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'}, - 'downloader' : {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'}, - 'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in movie_folder. \'downloader\' is required with this option.'}, + 'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'}, + 'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'}, + 'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'}, + 'downloader': {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'}, + 'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in media_folder. \'downloader\' is required with this option.'}, 'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''}, }, }) @@ -64,25 +68,33 @@ class Renamer(Plugin): def scanView(self, **kwargs): async = tryInt(kwargs.get('async', 0)) - movie_folder = sp(kwargs.get('movie_folder')) + base_folder = kwargs.get('base_folder') + media_folder = sp(kwargs.get('media_folder')) + + # Backwards compatibility, to be removed after a few versions :) + if not media_folder: + media_folder = sp(kwargs.get('movie_folder')) + downloader = kwargs.get('downloader') download_id = kwargs.get('download_id') files = '|'.join([sp(filename) for filename in splitString(kwargs.get('files'), '|')]) status = kwargs.get('status', 'completed') - release_download = {'folder': movie_folder} if movie_folder else None - if release_download: + release_download = None + if not base_folder and media_folder: + release_download = {'folder': media_folder} release_download.update({'id': download_id, 'downloader': downloader, 'status': status, 'files': files} if download_id else {}) fire_handle = fireEvent if not async else fireEventAsync - fire_handle('renamer.scan', release_download) + fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download) return { 'success': True } - def scan(self, release_download = None): + def scan(self, base_folder = None, release_download = None): + if not release_download: release_download = {} if self.isDisabled(): return @@ -91,11 +103,14 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return + if not base_folder: + base_folder = self.conf('from') + from_folder = sp(self.conf('from')) to_folder = sp(self.conf('to')) - # Get movie folder to process - movie_folder = release_download and release_download.get('folder') + # Get media folder to process + media_folder = release_download.get('folder') # Get all folders that should not be processed no_process = [to_folder] @@ -108,73 +123,73 @@ class Renamer(Plugin): pass # Check to see if the no_process folders are inside the "from" folder. - if not os.path.isdir(from_folder) or not os.path.isdir(to_folder): - log.error('Both the "To" and "From" have to exist.') + if not os.path.isdir(base_folder) or not os.path.isdir(to_folder): + log.error('Both the "To" and "From" folder have to exist.') return else: for item in no_process: - if from_folder in item: - log.error('To protect your data, the movie libraries can\'t be inside of or the same as the "from" folder.') + if isSubFolder(item, base_folder): + log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.') return - # Check to see if the no_process folders are inside the provided movie_folder - if movie_folder and not os.path.isdir(movie_folder): - log.debug('The provided movie folder %s does not exist. Trying to find it in the \'from\' folder.', movie_folder) + # Check to see if the no_process folders are inside the provided media_folder + if media_folder and not os.path.isdir(media_folder): + log.debug('The provided media folder %s does not exist. Trying to find it in the \'from\' folder.', media_folder) # Update to the from folder - if len(release_download.get('files')) == 1: - new_movie_folder = from_folder + if len(splitString(release_download.get('files'), '|')) == 1: + new_media_folder = from_folder else: - new_movie_folder = os.path.join(from_folder, os.path.basename(movie_folder)) + new_media_folder = os.path.join(from_folder, os.path.basename(media_folder)) - if not os.path.isdir(new_movie_folder): - log.error('The provided movie folder %s does not exist and could also not be found in the \'from\' folder.', movie_folder) + if not os.path.isdir(new_media_folder): + log.error('The provided media folder %s does not exist and could also not be found in the \'from\' folder.', media_folder) return # Update the files - new_files = [os.path.join(new_movie_folder, os.path.relpath(filename, movie_folder)) for filename in splitString(release_download.get('files'), '|')] + new_files = [os.path.join(new_media_folder, os.path.relpath(filename, media_folder)) for filename in splitString(release_download.get('files'), '|')] if new_files and not os.path.isfile(new_files[0]): - log.error('The provided movie folder %s does not exist and its files could also not be found in the \'from\' folder.', movie_folder) + log.error('The provided media folder %s does not exist and its files could also not be found in the \'from\' folder.', media_folder) return # Update release_download info to the from folder - log.debug('Release %s found in the \'from\' folder.', movie_folder) - release_download['folder'] = new_movie_folder + log.debug('Release %s found in the \'from\' folder.', media_folder) + release_download['folder'] = new_media_folder release_download['files'] = '|'.join(new_files) - movie_folder = new_movie_folder + media_folder = new_media_folder - if movie_folder: + if media_folder: for item in no_process: - if movie_folder in item: - log.error('To protect your data, the movie libraries can\'t be inside of or the same as the provided movie folder.') + if isSubFolder(item, media_folder): + log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.') return # Make sure a checkSnatched marked all downloads/seeds as such if not release_download and self.conf('run_every') > 0: - fireEvent('renamer.check_snatched') + self.checkSnatched(fire_scan = False) self.renaming_started = True - # make sure the movie folder name is included in the search + # make sure the media folder name is included in the search folder = None files = [] - if movie_folder: - log.info('Scanning movie folder %s...', movie_folder) - folder = os.path.dirname(movie_folder) + if media_folder: + log.info('Scanning media folder %s...', media_folder) + folder = os.path.dirname(media_folder) if release_download.get('files', ''): files = splitString(release_download['files'], '|') # If there is only one file in the torrent, the downloader did not create a subfolder if len(files) == 1: - folder = movie_folder + folder = media_folder else: # Get all files from the specified folder try: - for root, folders, names in os.walk(movie_folder): + for root, folders, names in os.walk(media_folder): files.extend([sp(os.path.join(root, name)) for name in names]) except: - log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) + log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc())) db = get_session() @@ -184,10 +199,10 @@ class Renamer(Plugin): # Unpack any archives extr_files = None if self.conf('unrar'): - folder, movie_folder, files, extr_files = self.extractFiles(folder = folder, movie_folder = movie_folder, files = files, + folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files, cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download)) - groups = fireEvent('scanner.scan', folder = folder if folder else from_folder, + groups = fireEvent('scanner.scan', folder = folder if folder else base_folder, files = files, release_download = release_download, return_ignored = False, single = True) or [] folder_name = self.conf('folder_name') @@ -200,6 +215,10 @@ class Renamer(Plugin): done_status, active_status, downloaded_status, snatched_status, seeding_status = \ fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched', 'seeding'], single = True) + # Tag release folder as failed_rename in case no groups were found. This prevents check_snatched from removing the release from the downloader. + if not groups and self.statusInfoComplete(release_download): + self.tagRelease(release_download = release_download, tag = 'failed_rename') + for group_identifier in groups: group = groups[group_identifier] @@ -258,25 +277,25 @@ class Renamer(Plugin): name_the = movie_name[4:] + ', The' replacements = { - 'ext': 'mkv', - 'namethe': name_the.strip(), - 'thename': movie_name.strip(), - 'year': library['year'], - 'first': name_the[0].upper(), - 'quality': group['meta_data']['quality']['label'], - 'quality_type': group['meta_data']['quality_type'], - 'video': group['meta_data'].get('video'), - 'audio': group['meta_data'].get('audio'), - 'group': group['meta_data']['group'], - 'source': group['meta_data']['source'], - 'resolution_width': group['meta_data'].get('resolution_width'), - 'resolution_height': group['meta_data'].get('resolution_height'), - 'audio_channels': group['meta_data'].get('audio_channels'), - 'imdb_id': library['identifier'], - 'cd': '', - 'cd_nr': '', - 'mpaa': library['info'].get('mpaa', ''), - 'category': category_label, + 'ext': 'mkv', + 'namethe': name_the.strip(), + 'thename': movie_name.strip(), + 'year': library['year'], + 'first': name_the[0].upper(), + 'quality': group['meta_data']['quality']['label'], + 'quality_type': group['meta_data']['quality_type'], + 'video': group['meta_data'].get('video'), + 'audio': group['meta_data'].get('audio'), + 'group': group['meta_data']['group'], + 'source': group['meta_data']['source'], + 'resolution_width': group['meta_data'].get('resolution_width'), + 'resolution_height': group['meta_data'].get('resolution_height'), + 'audio_channels': group['meta_data'].get('audio_channels'), + 'imdb_id': library['identifier'], + 'cd': '', + 'cd_nr': '', + 'mpaa': library['info'].get('mpaa', ''), + 'category': category_label, } for file_type in group['files']: @@ -298,7 +317,7 @@ class Renamer(Plugin): cd = 1 if multiple else 0 for current_file in sorted(list(group['files'][file_type])): - current_file = toUnicode(current_file) + current_file = sp(current_file) # Original filename replacements['original'] = os.path.splitext(os.path.basename(current_file))[0] @@ -418,8 +437,9 @@ class Renamer(Plugin): movie.status_id = done_status.get('id') movie.last_edit = int(time.time()) db.commit() - except Exception, e: + except Exception as e: log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc())) + db.rollback() # Go over current movie releases for release in movie.releases: @@ -496,7 +516,10 @@ class Renamer(Plugin): os.remove(src) parent_dir = os.path.dirname(src) - if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not parent_dir in [destination, movie_folder] and not from_folder in parent_dir: + if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and \ + not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \ + not isSubFolder(parent_dir, base_folder): + delete_folders.append(parent_dir) except: @@ -507,11 +530,12 @@ class Renamer(Plugin): for delete_folder in delete_folders: try: self.deleteEmptyFolder(delete_folder, show_error = False) - except Exception, e: + except Exception as e: log.error('Failed to delete folder: %s %s', (e, traceback.format_exc())) # Rename all files marked group['renamed_files'] = [] + failed_rename = False for src in rename_files: if rename_files[src]: dst = rename_files[src] @@ -524,11 +548,20 @@ class Renamer(Plugin): self.moveFile(src, dst, forcemove = not self.downloadIsTorrent(release_download) or self.fileIsAdded(src, group)) group['renamed_files'].append(dst) except: - log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) - self.tagRelease(group = group, tag = 'failed_rename') + log.error('Failed ranaming the file "%s" : %s', (os.path.basename(src), traceback.format_exc())) + failed_rename = True + break + + # If renaming failed tag the release folder as failed and continue with next group. Note that all old files have already been deleted. + if failed_rename: + self.tagRelease(group = group, tag = 'failed_rename') + continue + # If renaming succeeded, make sure it is not tagged as failed (scanner didn't return a group, but a download_ID was provided in an earlier attempt) + else: + self.untagRelease(group = group, tag = 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent - if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(release_download): + if self.movieInFromFolder(media_folder) and self.downloadIsTorrent(release_download): self.tagRelease(group = group, tag = 'renamed_already') # Remove matching releases @@ -540,12 +573,12 @@ class Renamer(Plugin): log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download): - if movie_folder: + if media_folder: # Delete the movie folder - group_folder = movie_folder + group_folder = media_folder else: # Delete the first empty subfolder in the tree relative to the 'from' folder - group_folder = sp(os.path.join(from_folder, os.path.relpath(group['parentdir'], from_folder).split(os.path.sep)[0])) + group_folder = sp(os.path.join(base_folder, os.path.relpath(group['parentdir'], base_folder).split(os.path.sep)[0])) try: log.info('Deleting folder: %s', group_folder) @@ -574,7 +607,7 @@ class Renamer(Plugin): rename_files = {} def test(s): - return current_file[:-len(replacements['ext'])] in s + return current_file[:-len(replacements['ext'])] in sp(s) for extra in set(filter(test, group['files'][extra_type])): replacements['ext'] = getExt(extra) @@ -613,29 +646,47 @@ Remove it if you want it to be renamed (again, or at least let it try again) tag_files.extend([os.path.join(root, name) for name in names]) for filename in tag_files: + + # Dont tag .ignore files + if os.path.splitext(filename)[1] == '.ignore': + continue + tag_filename = '%s.%s.ignore' % (os.path.splitext(filename)[0], tag) if not os.path.isfile(tag_filename): self.createFile(tag_filename, text) - def untagRelease(self, release_download, tag = ''): + def untagRelease(self, group = None, release_download = None, tag = ''): if not release_download: return tag_files = [] + folder = None - folder = release_download['folder'] - if not os.path.isdir(folder): + # Tag movie files if they are known + if isinstance(group, dict): + tag_files = [sorted(list(group['files']['movie']))[0]] + + folder = group['parentdir'] + if not group.get('dirname') or not os.path.isdir(folder): + return False + + elif isinstance(release_download, dict): + # Untag download_files if they are known + if release_download['files']: + tag_files = splitString(release_download['files'], '|') + + # Untag all files in release folder + else: + for root, folders, names in os.walk(release_download['folder']): + tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore']) + + folder = release_download['folder'] + if not os.path.isdir(folder): + return False + + if not folder: return False - # Untag download_files if they are known - if release_download['files']: - tag_files = splitString(release_download['files'], '|') - - # Untag all files in release folder - else: - for root, folders, names in os.walk(release_download['folder']): - tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore']) - # Find all .ignore files in folder ignore_files = [] for root, dirnames, filenames in os.walk(folder): @@ -643,7 +694,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Match all found ignore files with the tag_files and delete if found for tag_file in tag_files: - ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (re.escape(os.path.splitext(tag_file)[0]), tag if tag else '*')) + ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*'))) for filename in ignore_file: try: os.remove(filename) @@ -676,7 +727,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Match all found ignore files with the tag_files and return True found for tag_file in tag_files: - ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')) + ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*'))) if ignore_file: return True @@ -685,8 +736,15 @@ Remove it if you want it to be renamed (again, or at least let it try again) def moveFile(self, old, dest, forcemove = False): dest = ss(dest) try: - if forcemove: - shutil.move(old, dest) + if forcemove or self.conf('file_action') not in ['copy', 'link']: + try: + shutil.move(old, dest) + except: + if os.path.exists(dest): + log.error('Successfully moved file "%s", but something went wrong: %s', (dest, traceback.format_exc())) + os.unlink(old) + else: + raise elif self.conf('file_action') == 'copy': shutil.copy(old, dest) elif self.conf('file_action') == 'link': @@ -704,8 +762,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) os.rename(old + '.link', old) except: log.error('Couldn\'t symlink file "%s" to "%s". Copied instead. Error: %s. ', (old, dest, traceback.format_exc())) - else: - shutil.move(old, dest) try: os.chmod(dest, Env.getPermission('file')) @@ -713,15 +769,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) os.popen('icacls "' + dest + '"* /reset /T') except: log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1))) - - except OSError, err: - # Copying from a filesystem with octal permission to an NTFS file system causes a permission error. In this case ignore it. - if not hasattr(os, 'chmod') or err.errno != errno.EPERM: - raise - else: - if os.path.exists(dest): - os.unlink(old) - except: log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc())) raise @@ -739,19 +786,19 @@ Remove it if you want it to be renamed (again, or at least let it try again) replacements['cd_nr'] = '' replaced = toUnicode(string) - for x, r in replacements.iteritems(): + for x, r in replacements.items(): if x in ['thename', 'namethe']: continue if r is not None: - replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r)) + replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r)) else: #If information is not available, we don't want the tag in the filename replaced = replaced.replace('<' + x + '>', '') replaced = self.replaceDoubles(replaced.lstrip('. ')) - for x, r in replacements.iteritems(): + for x, r in replacements.items(): if x in ['thename', 'namethe']: - replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r)) + replaced = replaced.replace(six.u('<%s>') % toUnicode(x), toUnicode(r)) replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced) sep = self.conf('foldersep') if folder else self.conf('separator') @@ -789,7 +836,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) except: loge('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc())) - def checkSnatched(self): + def checkSnatched(self, fire_scan = True): if self.checking_snatched: log.debug('Already checking snatched') @@ -805,126 +852,169 @@ Remove it if you want it to be renamed (again, or at least let it try again) Release.status_id.in_([snatched_status.get('id'), seeding_status.get('id'), missing_status.get('id')]) ).all() + if not rels: + #No releases found that need status checking + self.checking_snatched = False + return True + + # Collect all download information with the download IDs from the releases + download_ids = [] + no_status_support = [] + try: + for rel in rels: + rel_dict = rel.to_dict({'info': {}}) + if rel_dict['info'].get('download_id') and rel_dict['info'].get('download_downloader'): + download_ids.append({'id': rel_dict['info']['download_id'], 'downloader': rel_dict['info']['download_downloader']}) + + ds = rel_dict['info'].get('download_status_support') + if ds == False or ds == 'False': + no_status_support.append(ss(rel_dict['info'].get('download_downloader'))) + except: + log.error('Error getting download IDs from database') + self.checking_snatched = False + return False + + release_downloads = fireEvent('download.status', download_ids, merge = True) if download_ids else [] + + if len(no_status_support) > 0: + log.debug('Download status functionality is not implemented for one of the active downloaders: %s', no_status_support) + + if not release_downloads: + if fire_scan: + self.scan() + + self.checking_snatched = False + return True + scan_releases = [] scan_required = False - if rels: - log.debug('Checking status snatched releases...') + log.debug('Checking status snatched releases...') - release_downloads = fireEvent('download.status', merge = True) - if not release_downloads: - log.debug('Download status functionality is not implemented for active downloaders.') - scan_required = True - else: - try: - for rel in rels: - rel_dict = rel.to_dict({'info': {}}) - movie_dict = fireEvent('movie.get', rel.movie_id, single = True) + try: + for rel in rels: + rel_dict = rel.to_dict({'info': {}}) + movie_dict = fireEvent('media.get', media_id = rel.movie_id, single = True) - if not isinstance(rel_dict['info'], (dict)): - log.error('Faulty release found without any info, ignoring.') + if not isinstance(rel_dict['info'], dict): + log.error('Faulty release found without any info, ignoring.') + fireEvent('release.update_status', rel.id, status = ignored_status, single = True) + continue + + # Check if download ID is available + if not rel_dict['info'].get('download_id') or not rel_dict['info'].get('download_downloader'): + log.debug('Download status functionality is not implemented for downloader (%s) of release %s.', (rel_dict['info'].get('download_downloader', 'unknown'), rel_dict['info']['name'])) + scan_required = True + + # Continue with next release + continue + + # Find release in downloaders + nzbname = self.createNzbName(rel_dict['info'], movie_dict) + + found_release = False + for release_download in release_downloads: + found_release = False + if rel_dict['info'].get('download_id'): + if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']: + log.debug('Found release by id: %s', release_download['id']) + found_release = True + break + else: + if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']: + log.debug('Found release by release name or imdb ID: %s', release_download['name']) + found_release = True + break + + if not found_release: + log.info('%s not found in downloaders', nzbname) + + #Check status if already missing and for how long, if > 1 week, set to ignored else to missing + if rel.status_id == missing_status.get('id'): + if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60: fireEvent('release.update_status', rel.id, status = ignored_status, single = True) - continue + else: + # Set the release to missing + fireEvent('release.update_status', rel.id, status = missing_status, single = True) - # check status - nzbname = self.createNzbName(rel_dict['info'], movie_dict) + # Continue with next release + continue - found = False - for release_download in release_downloads: - found_release = False - if rel_dict['info'].get('download_id'): - if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']: - log.debug('Found release by id: %s', release_download['id']) - found_release = True + # Log that we found the release + timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft'] + log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft)) + + # Check status of release + if release_download['status'] == 'busy': + # Set the release to snatched if it was missing before + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) + + # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading + if self.movieInFromFolder(release_download['folder']): + self.tagRelease(release_download = release_download, tag = 'downloading') + + elif release_download['status'] == 'seeding': + #If linking setting is enabled, process release + if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download): + log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio'])) + + # Remove the downloading tag + self.untagRelease(release_download = release_download, tag = 'downloading') + + # Scan and set the torrent to paused if required + release_download.update({'pause': True, 'scan': True, 'process_complete': False}) + scan_releases.append(release_download) + else: + #let it seed + log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio'])) + + # Set the release to seeding + fireEvent('release.update_status', rel.id, status = seeding_status, single = True) + + elif release_download['status'] == 'failed': + # Set the release to failed + fireEvent('release.update_status', rel.id, status = failed_status, single = True) + + fireEvent('download.remove_failed', release_download, single = True) + + if self.conf('next_on_failed'): + fireEvent('movie.searcher.try_next_release', media_id = rel.movie_id) + + elif release_download['status'] == 'completed': + log.info('Download of %s completed!', release_download['name']) + + #Make sure the downloader sent over a path to look in + if self.statusInfoComplete(release_download): + + # If the release has been seeding, process now the seeding is done + if rel.status_id == seeding_status.get('id'): + if self.conf('file_action') != 'move': + # Set the release to done as the movie has already been renamed + fireEvent('release.update_status', rel.id, status = downloaded_status, single = True) + + # Allow the downloader to clean-up + release_download.update({'pause': False, 'scan': False, 'process_complete': True}) + scan_releases.append(release_download) else: - if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']: - found_release = True + # Scan and Allow the downloader to clean-up + release_download.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_releases.append(release_download) - if found_release: - timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft'] - log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft)) + else: + # Set the release to snatched if it was missing before + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) - if release_download['status'] == 'busy': - # Set the release to snatched if it was missing before - fireEvent('release.update_status', rel.id, status = snatched_status, single = True) + # Remove the downloading tag + self.untagRelease(release_download = release_download, tag = 'downloading') - # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading - if self.movieInFromFolder(release_download['folder']): - self.tagRelease(release_download = release_download, tag = 'downloading') + # Scan and Allow the downloader to clean-up + release_download.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_releases.append(release_download) + else: + scan_required = True - elif release_download['status'] == 'seeding': - #If linking setting is enabled, process release - if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download): - log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio'])) - - # Remove the downloading tag - self.untagRelease(release_download = release_download, tag = 'downloading') - - # Scan and set the torrent to paused if required - release_download.update({'pause': True, 'scan': True, 'process_complete': False}) - scan_releases.append(release_download) - else: - #let it seed - log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio'])) - - # Set the release to seeding - fireEvent('release.update_status', rel.id, status = seeding_status, single = True) - - elif release_download['status'] == 'failed': - # Set the release to failed - fireEvent('release.update_status', rel.id, status = failed_status, single = True) - - fireEvent('download.remove_failed', release_download, single = True) - - if self.conf('next_on_failed'): - fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id) - elif release_download['status'] == 'completed': - log.info('Download of %s completed!', release_download['name']) - if self.statusInfoComplete(release_download): - - # If the release has been seeding, process now the seeding is done - if rel.status_id == seeding_status.get('id'): - if self.conf('file_action') != 'move': - # Set the release to done as the movie has already been renamed - fireEvent('release.update_status', rel.id, status = downloaded_status, single = True) - - # Allow the downloader to clean-up - release_download.update({'pause': False, 'scan': False, 'process_complete': True}) - scan_releases.append(release_download) - else: - # Scan and Allow the downloader to clean-up - release_download.update({'pause': False, 'scan': True, 'process_complete': True}) - scan_releases.append(release_download) - - else: - # Set the release to snatched if it was missing before - fireEvent('release.update_status', rel.id, status = snatched_status, single = True) - - # Remove the downloading tag - self.untagRelease(release_download = release_download, tag = 'downloading') - - # Scan and Allow the downloader to clean-up - release_download.update({'pause': False, 'scan': True, 'process_complete': True}) - scan_releases.append(release_download) - else: - scan_required = True - - found = True - break - - if not found: - log.info('%s not found in downloaders', nzbname) - - #Check status if already missing and for how long, if > 1 week, set to ignored else to missing - if rel.status_id == missing_status.get('id'): - if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60: - fireEvent('release.update_status', rel.id, status = ignored_status, single = True) - else: - # Set the release to missing - fireEvent('release.update_status', rel.id, status = missing_status, single = True) - - except: - log.error('Failed checking for release in downloader: %s', traceback.format_exc()) + except: + log.error('Failed checking for release in downloader: %s', traceback.format_exc()) # The following can either be done here, or inside the scanner if we pass it scan_items in one go for release_download in scan_releases: @@ -932,7 +1022,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if release_download['scan']: if release_download['pause'] and self.conf('file_action') == 'link': fireEvent('download.pause', release_download = release_download, pause = True, single = True) - fireEvent('renamer.scan', release_download = release_download) + self.scan(release_download = release_download) if release_download['pause'] and self.conf('file_action') == 'link': fireEvent('download.pause', release_download = release_download, pause = False, single = True) if release_download['process_complete']: @@ -943,11 +1033,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Ask the downloader to process the item fireEvent('download.process_complete', release_download = release_download, single = True) - if scan_required: - fireEvent('renamer.scan') + if fire_scan and (scan_required or len(no_status_support) > 0): + self.scan() self.checking_snatched = False - return True def extendReleaseDownload(self, release_download): @@ -992,12 +1081,12 @@ Remove it if you want it to be renamed (again, or at least let it try again) return src in group['before_rename'] def statusInfoComplete(self, release_download): - return release_download['id'] and release_download['downloader'] and release_download['folder'] + return release_download.get('id') and release_download.get('downloader') and release_download.get('folder') - def movieInFromFolder(self, movie_folder): - return movie_folder and sp(self.conf('from')) in sp(movie_folder) or not movie_folder + def movieInFromFolder(self, media_folder): + return media_folder and isSubFolder(media_folder, sp(self.conf('from'))) or not media_folder - def extractFiles(self, folder = None, movie_folder = None, files = None, cleanup = False): + def extractFiles(self, folder = None, media_folder = None, files = None, cleanup = False): if not files: files = [] # RegEx for finding rar files @@ -1012,7 +1101,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) folder = from_folder check_file_date = True - if movie_folder: + if media_folder: check_file_date = False if not files: @@ -1034,29 +1123,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Check if archive is fresh and maybe still copying/moving/downloading, ignore files newer than 1 minute if check_file_date: - file_too_new = False - for cur_file in archive['files']: - if not os.path.isfile(cur_file): - file_too_new = time.time() - break - file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)] - for t in file_time: - if t > time.time() - 60: - file_too_new = tryInt(time.time() - t) - break - - if file_too_new: - break - - if file_too_new: - try: - time_string = time.ctime(file_time[0]) - except: - try: - time_string = time.ctime(file_time[1]) - except: - time_string = 'unknown' + files_too_new, time_string = self.checkFilesChanged(archive['files']) + if files_too_new: log.info('Archive seems to be still copying/moving/downloading or just copied/moved/downloaded (created on %s), ignoring for now: %s', (time_string, os.path.basename(archive['file']))) continue @@ -1071,7 +1140,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False) extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))) del rar_handle - except Exception, e: + except Exception as e: log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc())) continue @@ -1080,7 +1149,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) if cleanup: try: os.remove(filename) - except Exception, e: + except Exception as e: log.error('Failed to remove %s: %s %s', (filename, e, traceback.format_exc())) continue files.remove(filename) @@ -1093,7 +1162,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) try: self.makeDir(os.path.dirname(move_to)) self.moveFile(leftoverfile, move_to, cleanup) - except Exception, e: + except Exception as e: log.error('Failed moving left over file %s to %s: %s %s', (leftoverfile, move_to, e, traceback.format_exc())) # As we probably tried to overwrite the nfo file, check if it exists and then remove the original if os.path.isfile(move_to): @@ -1108,18 +1177,18 @@ Remove it if you want it to be renamed (again, or at least let it try again) if cleanup: # Remove all left over folders - log.debug('Removing old movie folder %s...', movie_folder) - self.deleteEmptyFolder(movie_folder) + log.debug('Removing old movie folder %s...', media_folder) + self.deleteEmptyFolder(media_folder) - movie_folder = os.path.join(from_folder, os.path.relpath(movie_folder, folder)) + media_folder = os.path.join(from_folder, os.path.relpath(media_folder, folder)) folder = from_folder if extr_files: files.extend(extr_files) - # Cleanup files and folder if movie_folder was not provided - if not movie_folder: + # Cleanup files and folder if media_folder was not provided + if not media_folder: files = [] folder = None - return folder, movie_folder, files, extr_files + return folder, media_folder, files, extr_files diff --git a/couchpotato/core/plugins/scanner/__init__.py b/couchpotato/core/plugins/scanner/__init__.py index 3d640465..66c6b39c 100644 --- a/couchpotato/core/plugins/scanner/__init__.py +++ b/couchpotato/core/plugins/scanner/__init__.py @@ -1,5 +1,6 @@ from .main import Scanner + def start(): return Scanner() diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index eb193ad8..3031236f 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -15,6 +15,7 @@ import re import threading import time import traceback +from six.moves import filter, map, zip log = CPLog(__name__) @@ -23,7 +24,7 @@ class Scanner(Plugin): ignored_in_path = [os.path.sep + 'extracted' + os.path.sep, 'extracting', '_unpack', '_failed_', '_unknown_', '_exists_', '_failed_remove_', '_failed_rename_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo', - 'thumbs.db', 'ehthumbs.db', 'desktop.ini'] #unpacking, smb-crap, hidden files + 'thumbs.db', 'ehthumbs.db', 'desktop.ini'] #unpacking, smb-crap, hidden files ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films', 'download', 'downloads', 'video_ts', 'audio_ts', 'bdmv', 'certificate'] extensions = { 'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img', 'mdf', 'ts', 'm4v'], @@ -48,7 +49,7 @@ class Scanner(Plugin): 'leftover': ('leftover', 'leftover'), } - file_sizes = { # in MB + file_sizes = { # in MB 'movie': {'min': 300}, 'trailer': {'min': 2, 'max': 250}, 'backdrop': {'min': 0, 'max': 5}, @@ -80,19 +81,20 @@ class Scanner(Plugin): 'hdtv': ['hdtv'] } - clean = '[ _\,\.\(\)\[\]\-](extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)' + clean = '[ _\,\.\(\)\[\]\-]?(extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \ + '|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)' multipart_regex = [ - '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 - '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 - '[ _\.-]+part[ _\.-]*([0-9a-d]+)', #*part1 - '[ _\.-]+dis[ck][ _\.-]*([0-9a-d]+)', #*disk1 - 'cd[ _\.-]*([0-9a-d]+)$', #cd1.ext - 'dvd[ _\.-]*([0-9a-d]+)$', #dvd1.ext - 'part[ _\.-]*([0-9a-d]+)$', #part1.mkv - 'dis[ck][ _\.-]*([0-9a-d]+)$', #disk1.mkv + '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 + '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 + '[ _\.-]+part[ _\.-]*([0-9a-d]+)', #*part1 + '[ _\.-]+dis[ck][ _\.-]*([0-9a-d]+)', #*disk1 + 'cd[ _\.-]*([0-9a-d]+)$', #cd1.ext + 'dvd[ _\.-]*([0-9a-d]+)$', #dvd1.ext + 'part[ _\.-]*([0-9a-d]+)$', #part1.mkv + 'dis[ck][ _\.-]*([0-9a-d]+)$', #disk1.mkv '()[ _\.-]+([0-9]*[abcd]+)(\.....?)$', '([a-z])([0-9]+)(\.....?)$', - '()([ab])(\.....?)$' #*a.mkv + '()([ab])(\.....?)$' #*a.mkv ] cp_imdb = '(.cp.(?Ptt[0-9{7}]+).)' @@ -132,6 +134,8 @@ class Scanner(Plugin): except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) + + log.debug('Found %s files to scan and group in %s', (len(files), folder)) else: check_file_date = False files = [sp(x) for x in files] @@ -186,7 +190,7 @@ class Scanner(Plugin): # Group files minus extension ignored_identifiers = [] - for identifier, group in movie_files.iteritems(): + for identifier, group in movie_files.items(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) @@ -227,7 +231,7 @@ class Scanner(Plugin): # Group the files based on the identifier delete_identifiers = [] - for identifier, found_files in path_identifiers.iteritems(): + for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) @@ -250,7 +254,7 @@ class Scanner(Plugin): # Group based on folder delete_identifiers = [] - for identifier, found_files in path_identifiers.iteritems(): + for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: @@ -262,7 +266,7 @@ class Scanner(Plugin): delete_identifiers.append(identifier) # Remove the found files from the leftover stack - leftovers = leftovers - set([ff]) + leftovers -= leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): @@ -287,41 +291,21 @@ class Scanner(Plugin): break # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute - file_too_new = False - for cur_file in group['unsorted_files']: - if not os.path.isfile(cur_file): - file_too_new = time.time() - break - file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)] - for t in file_time: - if t > time.time() - 60: - file_too_new = tryInt(time.time() - t) - break + if check_file_date: + files_too_new, time_string = self.checkFilesChanged(group['unsorted_files']) + if files_too_new: + log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier)) - if file_too_new: - break + # Delete the unsorted list + del group['unsorted_files'] - if check_file_date and file_too_new: - try: - time_string = time.ctime(file_time[0]) - except: - try: - time_string = time.ctime(file_time[1]) - except: - time_string = 'unknown' - - log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier)) - - # Delete the unsorted list - del group['unsorted_files'] - - continue + continue # Only process movies newer than x if newer_than and newer_than > 0: has_new_files = False for cur_file in group['unsorted_files']: - file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)] + file_time = self.getFileTimes(cur_file) if file_time[0] > newer_than or file_time[1] > newer_than: has_new_files = True break @@ -419,6 +403,7 @@ class Scanner(Plugin): else: movie = db.query(Media).filter_by(library_id = group['library']['id']).first() group['movie_id'] = None if not movie else movie.id + db.expire_all() processed_movies[identifier] = group @@ -444,7 +429,7 @@ class Scanner(Plugin): files = list(group['files']['movie']) for cur_file in files: - if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files + if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files meta = self.getMeta(cur_file) @@ -454,7 +439,7 @@ class Scanner(Plugin): data['resolution_width'] = meta.get('resolution_width', 720) data['resolution_height'] = meta.get('resolution_height', 480) data['audio_channels'] = meta.get('audio_channels', 2.0) - data['aspect'] = meta.get('resolution_width', 720) / meta.get('resolution_height', 480) + data['aspect'] = round(float(meta.get('resolution_width', 720)) / meta.get('resolution_height', 480), 2) except: log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc())) pass @@ -592,6 +577,7 @@ class Scanner(Plugin): # Check if path is already in db if not imdb_id: + db = get_session() for cf in files['movie']: f = db.query(File).filter_by(path = toUnicode(cf)).first() @@ -635,7 +621,7 @@ class Scanner(Plugin): try: m = re.search(self.cp_imdb, string.lower()) id = m.group('id') - if id: return id + if id: return id except AttributeError: pass @@ -726,7 +712,9 @@ class Scanner(Plugin): if is_sample: log.debug('Is sample file: %s', filename) return is_sample - def filesizeBetween(self, file, file_size = []): + def filesizeBetween(self, file, file_size = None): + if not file_size: file_size = [] + try: return (file_size.get('min', 0) * 1048576) < os.path.getsize(file) < (file_size.get('max', 100000) * 1048576) except: @@ -760,7 +748,8 @@ class Scanner(Plugin): # Year if year and identifier[:4] != year: - identifier = '%s %s' % (identifier.split(year)[0].strip(), year) + split_by = ':::' if ':::' in identifier else year + identifier = '%s %s' % (identifier.split(split_by)[0].strip(), year) else: identifier = identifier.split('::')[0] @@ -873,7 +862,7 @@ class Scanner(Plugin): except: pass - if not cp_guess: # Split name on multiple spaces + if not cp_guess: # Split name on multiple spaces try: movie_name = cleaned.split(' ').pop(0).strip() cp_guess = { diff --git a/couchpotato/core/plugins/score/__init__.py b/couchpotato/core/plugins/score/__init__.py index 2c367f89..a960081c 100644 --- a/couchpotato/core/plugins/score/__init__.py +++ b/couchpotato/core/plugins/score/__init__.py @@ -1,5 +1,6 @@ from .main import Score + def start(): return Score() diff --git a/couchpotato/core/plugins/score/main.py b/couchpotato/core/plugins/score/main.py index 54b6ca31..30e7baca 100644 --- a/couchpotato/core/plugins/score/main.py +++ b/couchpotato/core/plugins/score/main.py @@ -1,6 +1,6 @@ -from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import getTitle, splitString +from couchpotato.core.helpers.variable import getTitle, splitString, removeDuplicate from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \ @@ -21,7 +21,7 @@ class Score(Plugin): # Merge global and category preferred_words = splitString(Env.setting('preferred_words', section = 'searcher').lower()) - try: preferred_words = list(set(preferred_words + splitString(movie['category']['preferred'].lower()))) + try: preferred_words = removeDuplicate(preferred_words + splitString(movie['category']['preferred'].lower())) except: pass score = nameScore(toUnicode(nzb['name']), movie['library']['year'], preferred_words) @@ -35,8 +35,8 @@ class Score(Plugin): # Torrents only if nzb.get('seeders'): try: - score += nzb.get('seeders') / 5 - score += nzb.get('leechers') / 10 + score += nzb.get('seeders') * 100 / 15 + score += nzb.get('leechers') * 100 / 30 except: pass @@ -48,7 +48,7 @@ class Score(Plugin): # Merge global and category ignored_words = splitString(Env.setting('ignored_words', section = 'searcher').lower()) - try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower()))) + try: ignored_words = removeDuplicate(ignored_words + splitString(movie['category']['ignored'].lower())) except: pass # Partial ignored words diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index 895f5fc0..c1f5123a 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -51,6 +51,7 @@ def nameScore(name, year, preferred_words): return score + def nameRatioScore(nzb_name, movie_name): nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) movie_words = re.split('\W+', simplifyString(movie_name)) diff --git a/couchpotato/core/plugins/status/__init__.py b/couchpotato/core/plugins/status/__init__.py index fb5b4cc7..204fbee7 100644 --- a/couchpotato/core/plugins/status/__init__.py +++ b/couchpotato/core/plugins/status/__init__.py @@ -1,5 +1,6 @@ from .main import StatusPlugin + def start(): return StatusPlugin() diff --git a/couchpotato/core/plugins/status/main.py b/couchpotato/core/plugins/status/main.py index b3b37bdc..08f46984 100644 --- a/couchpotato/core/plugins/status/main.py +++ b/couchpotato/core/plugins/status/main.py @@ -1,3 +1,4 @@ +import traceback from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent @@ -33,7 +34,7 @@ class StatusPlugin(Plugin): addEvent('status.get_by_id', self.getById) addEvent('status.all', self.all) addEvent('app.initialize', self.fill) - addEvent('app.load', self.all) # Cache all statuses + addEvent('app.load', self.all) # Cache all statuses addApiView('status.list', self.list, docs = { 'desc': 'Check for available update', @@ -79,47 +80,57 @@ class StatusPlugin(Plugin): if not isinstance(identifiers, list): identifiers = [identifiers] - db = get_session() - return_list = [] + try: + db = get_session() + return_list = [] - for identifier in identifiers: + for identifier in identifiers: - if self.status_cached.get(identifier): - return_list.append(self.status_cached.get(identifier)) - continue + if self.status_cached.get(identifier): + return_list.append(self.status_cached.get(identifier)) + continue - s = db.query(Status).filter_by(identifier = identifier).first() - if not s: - s = Status( - identifier = identifier, - label = toUnicode(identifier.capitalize()) - ) - db.add(s) - db.commit() + s = db.query(Status).filter_by(identifier = identifier).first() + if not s: + s = Status( + identifier = identifier, + label = toUnicode(identifier.capitalize()) + ) + db.add(s) + db.commit() - status_dict = s.to_dict() + status_dict = s.to_dict() - self.status_cached[identifier] = status_dict - return_list.append(status_dict) + self.status_cached[identifier] = status_dict + return_list.append(status_dict) - return return_list if len(identifiers) > 1 else return_list[0] + return return_list if len(identifiers) > 1 else return_list[0] + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() def fill(self): - db = get_session() + try: + db = get_session() - for identifier, label in self.statuses.iteritems(): - s = db.query(Status).filter_by(identifier = identifier).first() - if not s: - log.info('Creating status: %s', label) - s = Status( - identifier = identifier, - label = toUnicode(label) - ) - db.add(s) + for identifier, label in self.statuses.items(): + s = db.query(Status).filter_by(identifier = identifier).first() + if not s: + log.info('Creating status: %s', label) + s = Status( + identifier = identifier, + label = toUnicode(label) + ) + db.add(s) - s.label = toUnicode(label) - db.commit() - - #db.close() + s.label = toUnicode(label) + db.commit() + except: + log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/core/plugins/subtitle/__init__.py b/couchpotato/core/plugins/subtitle/__init__.py index bbd40853..59847aee 100644 --- a/couchpotato/core/plugins/subtitle/__init__.py +++ b/couchpotato/core/plugins/subtitle/__init__.py @@ -1,5 +1,6 @@ from .main import Subtitle + def start(): return Subtitle() @@ -20,7 +21,7 @@ config = [{ }, { 'name': 'languages', - 'description': 'Comma separated, 2 letter country code. Example: en, nl. See the codes at on Wikipedia', + 'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at on Wikipedia'), }, # { # 'name': 'automatic', diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index 7504d6a9..56056c0a 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -45,7 +45,7 @@ class Subtitle(Plugin): if self.isDisabled(): return try: - available_languages = sum(group['subtitle_language'].itervalues(), []) + available_languages = sum(group['subtitle_language'].values(), []) downloaded = [] files = [toUnicode(x) for x in group['files']['movie']] log.debug('Searching for subtitles for: %s', files) diff --git a/couchpotato/core/plugins/trailer/__init__.py b/couchpotato/core/plugins/trailer/__init__.py index d8496b30..e7a6d26e 100644 --- a/couchpotato/core/plugins/trailer/__init__.py +++ b/couchpotato/core/plugins/trailer/__init__.py @@ -1,5 +1,6 @@ from .main import Trailer + def start(): return Trailer() diff --git a/couchpotato/core/plugins/trailer/main.py b/couchpotato/core/plugins/trailer/main.py index e27e3f9f..ba040058 100644 --- a/couchpotato/core/plugins/trailer/main.py +++ b/couchpotato/core/plugins/trailer/main.py @@ -28,7 +28,7 @@ class Trailer(Plugin): destination = os.path.join(group['destination_dir'], filename) if not os.path.isfile(destination): trailer_file = fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) - if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one + if os.path.getsize(trailer_file) < (1024 * 1024): # Don't trust small trailers (1MB), try next one os.unlink(trailer_file) continue else: diff --git a/couchpotato/core/plugins/userscript/__init__.py b/couchpotato/core/plugins/userscript/__init__.py index 5df5a801..184f5d79 100644 --- a/couchpotato/core/plugins/userscript/__init__.py +++ b/couchpotato/core/plugins/userscript/__init__.py @@ -1,5 +1,6 @@ from .main import Userscript + def start(): return Userscript() diff --git a/couchpotato/core/plugins/userscript/main.py b/couchpotato/core/plugins/userscript/main.py index 1b3cfe3d..113c0351 100644 --- a/couchpotato/core/plugins/userscript/main.py +++ b/couchpotato/core/plugins/userscript/main.py @@ -13,7 +13,7 @@ log = CPLog(__name__) class Userscript(Plugin): - version = 3 + version = 4 def __init__(self): addApiView('userscript.get/(.*)/(.*)', self.getUserScript, static = True) @@ -42,7 +42,7 @@ class Userscript(Plugin): 'excludes': fireEvent('userscript.get_excludes', merge = True), } - def getUserScript(self, route, **kwargs): + def getUserScript(self, script_route, **kwargs): klass = self @@ -63,8 +63,7 @@ class Userscript(Plugin): self.redirect(Env.get('api_base') + 'file.cache/couchpotato.user.js') - Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), route), UserscriptHandler)]) - + Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), script_route), UserscriptHandler)]) def getVersion(self): diff --git a/couchpotato/core/plugins/userscript/template.js b/couchpotato/core/plugins/userscript/template.js index 19840966..f5928b85 100644 --- a/couchpotato/core/plugins/userscript/template.js +++ b/couchpotato/core/plugins/userscript/template.js @@ -64,7 +64,7 @@ var addStyle = function(css) { // Styles addStyle('\ - #cp_popup { font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:9999; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \ + #cp_popup { font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:20000; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \ #cp_popup.opened { width: 492px; } \ #cp_popup a#add_to { cursor:pointer; text-align:center; text-decoration:none; color: #000; display:block; padding:5px 0 5px 5px; } \ #cp_popup a#close_button { cursor:pointer; float: right; padding:120px 10px 10px; } \ diff --git a/couchpotato/core/plugins/wizard/__init__.py b/couchpotato/core/plugins/wizard/__init__.py index 78876470..eda6f25a 100644 --- a/couchpotato/core/plugins/wizard/__init__.py +++ b/couchpotato/core/plugins/wizard/__init__.py @@ -1,5 +1,6 @@ from .main import Wizard + def start(): return Wizard() diff --git a/couchpotato/core/providers/automation/bluray/__init__.py b/couchpotato/core/providers/automation/bluray/__init__.py index ed270056..519a7119 100644 --- a/couchpotato/core/providers/automation/bluray/__init__.py +++ b/couchpotato/core/providers/automation/bluray/__init__.py @@ -1,5 +1,6 @@ from .main import Bluray + def start(): return Bluray() diff --git a/couchpotato/core/providers/automation/bluray/main.py b/couchpotato/core/providers/automation/bluray/main.py index d98557ec..ddd7b8ab 100644 --- a/couchpotato/core/providers/automation/bluray/main.py +++ b/couchpotato/core/providers/automation/bluray/main.py @@ -21,7 +21,7 @@ class Bluray(Automation, RSS): page = 0 while True: - page = page + 1 + page += 1 url = self.backlog_url % page data = self.getHTMLData(url) @@ -37,7 +37,7 @@ class Bluray(Automation, RSS): name = table.h3.get_text().lower().split('blu-ray')[0].strip() year = table.small.get_text().split('|')[1].strip() - if not name.find('/') == -1: # make sure it is not a double movie release + if not name.find('/') == -1: # make sure it is not a double movie release continue if tryInt(year) < self.getMinimal('year'): diff --git a/couchpotato/core/providers/automation/cp/__init__.py b/couchpotato/core/providers/automation/cp/__init__.py deleted file mode 100644 index a4b55a83..00000000 --- a/couchpotato/core/providers/automation/cp/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .main import CP - -def start(): - return CP() - -config = [] diff --git a/couchpotato/core/providers/automation/cp/main.py b/couchpotato/core/providers/automation/cp/main.py deleted file mode 100644 index 22b7942a..00000000 --- a/couchpotato/core/providers/automation/cp/main.py +++ /dev/null @@ -1,11 +0,0 @@ -from couchpotato.core.logger import CPLog -from couchpotato.core.providers.automation.base import Automation - -log = CPLog(__name__) - - -class CP(Automation): - - def getMovies(self): - - return [] diff --git a/couchpotato/core/providers/automation/flixster/__init__.py b/couchpotato/core/providers/automation/flixster/__init__.py index 1c6c4590..71bd83c0 100644 --- a/couchpotato/core/providers/automation/flixster/__init__.py +++ b/couchpotato/core/providers/automation/flixster/__init__.py @@ -1,5 +1,6 @@ from .main import Flixster + def start(): return Flixster() diff --git a/couchpotato/core/providers/automation/flixster/main.py b/couchpotato/core/providers/automation/flixster/main.py index 7fd2f717..f07ecd6b 100644 --- a/couchpotato/core/providers/automation/flixster/main.py +++ b/couchpotato/core/providers/automation/flixster/main.py @@ -42,6 +42,9 @@ class Flixster(Automation): data = self.getJsonData(self.url % user_id, decode_from = 'iso-8859-1') for movie in data: - movies.append({'title': movie['movie']['title'], 'year': movie['movie']['year'] }) + movies.append({ + 'title': movie['movie']['title'], + 'year': movie['movie']['year'] + }) return movies diff --git a/couchpotato/core/providers/automation/goodfilms/__init__.py b/couchpotato/core/providers/automation/goodfilms/__init__.py index 795e21da..e04ccd0d 100644 --- a/couchpotato/core/providers/automation/goodfilms/__init__.py +++ b/couchpotato/core/providers/automation/goodfilms/__init__.py @@ -1,5 +1,6 @@ from .main import Goodfilms + def start(): return Goodfilms() @@ -25,4 +26,4 @@ config = [{ ], }, ], -}] \ No newline at end of file +}] diff --git a/couchpotato/core/providers/automation/goodfilms/main.py b/couchpotato/core/providers/automation/goodfilms/main.py index e1125615..e668a4fb 100644 --- a/couchpotato/core/providers/automation/goodfilms/main.py +++ b/couchpotato/core/providers/automation/goodfilms/main.py @@ -7,7 +7,7 @@ log = CPLog(__name__) class Goodfilms(Automation): - url = 'http://goodfil.ms/%s/queue?page=%d&without_layout=1' + url = 'https://goodfil.ms/%s/queue?page=%d&without_layout=1' interval = 1800 @@ -35,9 +35,12 @@ class Goodfilms(Automation): data = self.getHTMLData(url) soup = BeautifulSoup(data) - this_watch_list = soup.find_all('div', attrs = { 'class': 'movie', 'data-film-title': True }) + this_watch_list = soup.find_all('div', attrs = { + 'class': 'movie', + 'data-film-title': True + }) - if not this_watch_list: # No Movies + if not this_watch_list: # No Movies break for movie in this_watch_list: diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py index 20e4f41b..f9baabf2 100644 --- a/couchpotato/core/providers/automation/imdb/__init__.py +++ b/couchpotato/core/providers/automation/imdb/__init__.py @@ -1,5 +1,6 @@ from .main import IMDB + def start(): return IMDB() @@ -11,7 +12,7 @@ config = [{ 'list': 'watchlist_providers', 'name': 'imdb_automation_watchlist', 'label': 'IMDB', - 'description': 'From any public IMDB watchlists. Url should be the CSV link.', + 'description': 'From any public IMDB watchlists.', 'options': [ { 'name': 'automation_enabled', @@ -59,7 +60,7 @@ config = [{ { 'name': 'automation_charts_boxoffice', 'type': 'bool', - 'label': 'Box offce TOP 10', + 'label': 'Box office TOP 10', 'description': 'IMDB Box office TOP 10 chart', 'default': True, }, diff --git a/couchpotato/core/providers/automation/imdb/main.py b/couchpotato/core/providers/automation/imdb/main.py index 76afb24c..6ca81b70 100644 --- a/couchpotato/core/providers/automation/imdb/main.py +++ b/couchpotato/core/providers/automation/imdb/main.py @@ -1,4 +1,5 @@ import traceback +import re from bs4 import BeautifulSoup from couchpotato import fireEvent @@ -25,7 +26,7 @@ class IMDBBase(Automation, RSS): interval = 1800 def getInfo(self, imdb_id): - return fireEvent('movie.info', identifier = imdb_id, merge = True) + return fireEvent('movie.info', identifier = imdb_id, extended = False, merge = True) class IMDBWatchlist(IMDBBase): @@ -42,23 +43,55 @@ class IMDBWatchlist(IMDBBase): index = -1 for watchlist_url in watchlist_urls: + try: + # Get list ID + ids = re.findall('(?:list/|list_id=)([a-zA-Z0-9\-_]{11})', watchlist_url) + if len(ids) == 1: + watchlist_url = 'http://www.imdb.com/list/%s/?view=compact&sort=created:asc' % ids[0] + # Try find user id with watchlist + else: + userids = re.findall('(ur\d{7,9})', watchlist_url) + if len(userids) == 1: + watchlist_url = 'http://www.imdb.com/user/%s/watchlist?view=compact&sort=created:asc' % userids[0] + except: + log.error('Failed getting id from watchlist: %s', traceback.format_exc()) + index += 1 if not watchlist_enablers[index]: continue - try: - log.debug('Started IMDB watchlists: %s', watchlist_url) - rss_data = self.getHTMLData(watchlist_url) - imdbs = getImdb(rss_data, multiple = True) if rss_data else [] + start = 0 + while True: + try: - for imdb in imdbs: - movies.append(imdb) + w_url = '%s&start=%s' % (watchlist_url, start) + log.debug('Started IMDB watchlists: %s', w_url) + html = self.getHTMLData(w_url) - if self.shuttingDown(): + try: + split = splitString(html, split_on="
")[1] + html = splitString(split, split_on="
")[0] + except: + pass + + imdbs = getImdb(html, multiple = True) if html else [] + + for imdb in imdbs: + if imdb not in movies: + movies.append(imdb) + + if self.shuttingDown(): + break + + log.debug('Found %s movies on %s', (len(imdbs), w_url)) + + if len(imdbs) < 250: break - except: - log.error('Failed loading IMDB watchlist: %s %s', (watchlist_url, traceback.format_exc())) + start += 250 + + except: + log.error('Failed loading IMDB watchlist: %s %s', (watchlist_url, traceback.format_exc())) return movies diff --git a/couchpotato/core/providers/automation/itunes/__init__.py b/couchpotato/core/providers/automation/itunes/__init__.py index cc5dddc7..13526f43 100644 --- a/couchpotato/core/providers/automation/itunes/__init__.py +++ b/couchpotato/core/providers/automation/itunes/__init__.py @@ -1,5 +1,6 @@ from .main import ITunes + def start(): return ITunes() diff --git a/couchpotato/core/providers/automation/itunes/main.py b/couchpotato/core/providers/automation/itunes/main.py index eb68e348..086c981d 100644 --- a/couchpotato/core/providers/automation/itunes/main.py +++ b/couchpotato/core/providers/automation/itunes/main.py @@ -22,7 +22,7 @@ class ITunes(Automation, RSS): urls = splitString(self.conf('automation_urls')) namespace = 'http://www.w3.org/2005/Atom' - namespaceIM = 'http://itunes.apple.com/rss' + namespace_im = 'https://rss.itunes.apple.com' index = -1 for url in urls: @@ -42,10 +42,10 @@ class ITunes(Automation, RSS): rss_movies = self.getElements(data, entry_tag) for movie in rss_movies: - name_tag = str(QName(namespaceIM, 'name')) + name_tag = str(QName(namespace_im, 'name')) name = self.getTextElement(movie, name_tag) - releaseDate_tag = str(QName(namespaceIM, 'releaseDate')) + releaseDate_tag = str(QName(namespace_im, 'releaseDate')) releaseDateText = self.getTextElement(movie, releaseDate_tag) year = datetime.datetime.strptime(releaseDateText, '%Y-%m-%dT00:00:00-07:00').strftime("%Y") diff --git a/couchpotato/core/providers/automation/kinepolis/__init__.py b/couchpotato/core/providers/automation/kinepolis/__init__.py index 24bd4ebb..cc4c5706 100644 --- a/couchpotato/core/providers/automation/kinepolis/__init__.py +++ b/couchpotato/core/providers/automation/kinepolis/__init__.py @@ -1,5 +1,6 @@ from .main import Kinepolis + def start(): return Kinepolis() diff --git a/couchpotato/core/providers/automation/letterboxd/__init__.py b/couchpotato/core/providers/automation/letterboxd/__init__.py index f2b8486b..88bfe6a1 100644 --- a/couchpotato/core/providers/automation/letterboxd/__init__.py +++ b/couchpotato/core/providers/automation/letterboxd/__init__.py @@ -1,5 +1,6 @@ from .main import Letterboxd + def start(): return Letterboxd() diff --git a/couchpotato/core/providers/automation/letterboxd/main.py b/couchpotato/core/providers/automation/letterboxd/main.py index 1f106dd1..dbbf53b1 100644 --- a/couchpotato/core/providers/automation/letterboxd/main.py +++ b/couchpotato/core/providers/automation/letterboxd/main.py @@ -1,5 +1,5 @@ from bs4 import BeautifulSoup -from couchpotato.core.helpers.variable import tryInt, splitString +from couchpotato.core.helpers.variable import tryInt, splitString, removeEmpty from couchpotato.core.logger import CPLog from couchpotato.core.providers.automation.base import Automation import re @@ -44,8 +44,8 @@ class Letterboxd(Automation): soup = BeautifulSoup(self.getHTMLData(self.url % username)) - for movie in soup.find_all('a', attrs = { 'class': 'frame' }): - match = filter(None, self.pattern.split(movie['title'])) + for movie in soup.find_all('a', attrs = {'class': 'frame'}): + match = removeEmpty(self.pattern.split(movie['title'])) movies.append({'title': match[0], 'year': match[1] }) return movies diff --git a/couchpotato/core/providers/automation/moviemeter/__init__.py b/couchpotato/core/providers/automation/moviemeter/__init__.py index aff5d09d..0e9a4edc 100644 --- a/couchpotato/core/providers/automation/moviemeter/__init__.py +++ b/couchpotato/core/providers/automation/moviemeter/__init__.py @@ -1,5 +1,6 @@ from .main import Moviemeter + def start(): return Moviemeter() diff --git a/couchpotato/core/providers/automation/movies_io/__init__.py b/couchpotato/core/providers/automation/movies_io/__init__.py index 9b280930..0361223b 100644 --- a/couchpotato/core/providers/automation/movies_io/__init__.py +++ b/couchpotato/core/providers/automation/movies_io/__init__.py @@ -1,5 +1,6 @@ from .main import MoviesIO + def start(): return MoviesIO() diff --git a/couchpotato/core/providers/automation/rottentomatoes/__init__.py b/couchpotato/core/providers/automation/rottentomatoes/__init__.py index 4675fac2..1d3026d3 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/automation/rottentomatoes/__init__.py @@ -1,5 +1,6 @@ from .main import Rottentomatoes + def start(): return Rottentomatoes() diff --git a/couchpotato/core/providers/automation/rottentomatoes/main.py b/couchpotato/core/providers/automation/rottentomatoes/main.py index 69611705..c873a8e1 100644 --- a/couchpotato/core/providers/automation/rottentomatoes/main.py +++ b/couchpotato/core/providers/automation/rottentomatoes/main.py @@ -8,6 +8,7 @@ import re log = CPLog(__name__) + class Rottentomatoes(Automation, RSS): interval = 1800 diff --git a/couchpotato/core/providers/automation/trakt/__init__.py b/couchpotato/core/providers/automation/trakt/__init__.py index cbaaece3..6ae2806b 100644 --- a/couchpotato/core/providers/automation/trakt/__init__.py +++ b/couchpotato/core/providers/automation/trakt/__init__.py @@ -1,5 +1,6 @@ from .main import Trakt + def start(): return Trakt() diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index da27d853..93e0900f 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -1,20 +1,20 @@ from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.encoding import ss from couchpotato.core.helpers.variable import tryFloat, mergeDicts, md5, \ possibleTitles, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env from urlparse import urlparse -import cookielib import json import re import time import traceback -import urllib2 import xml.etree.ElementTree as XMLTree log = CPLog(__name__) + class MultiProvider(Plugin): def __init__(self): @@ -37,8 +37,8 @@ class MultiProvider(Plugin): class Provider(Plugin): - type = None # movie, show, subtitle, trailer, ... - http_time_between_calls = 10 # Default timeout for url requests + type = None # movie, show, subtitle, trailer, ... + http_time_between_calls = 10 # Default timeout for url requests last_available_check = {} is_available = {} @@ -64,7 +64,7 @@ class Provider(Plugin): def getJsonData(self, url, decode_from = None, **kwargs): - cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {}))) + cache_key = md5(url) data = self.getCache(cache_key, url, **kwargs) if data: @@ -81,12 +81,12 @@ class Provider(Plugin): def getRSSData(self, url, item_path = 'channel/item', **kwargs): - cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {}))) + cache_key = md5(url) data = self.getCache(cache_key, url, **kwargs) if data and len(data) > 0: try: - data = XMLTree.fromstring(data) + data = XMLTree.fromstring(ss(data)) return self.getElements(data, item_path) except: log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) @@ -95,24 +95,23 @@ class Provider(Plugin): def getHTMLData(self, url, **kwargs): - cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {}))) + cache_key = md5(url) return self.getCache(cache_key, url, **kwargs) class YarrProvider(Provider): - protocol = None # nzb, torrent, torrent_magnet + protocol = None # nzb, torrent, torrent_magnet type = 'movie' cat_ids = {} cat_backup_id = None - sizeGb = ['gb', 'gib'] - sizeMb = ['mb', 'mib'] - sizeKb = ['kb', 'kib'] + size_gb = ['gb', 'gib'] + size_mb = ['mb', 'mib'] + size_kb = ['kb', 'kib'] - login_opener = None - last_login_check = 0 + last_login_check = None def __init__(self): addEvent('provider.enabled_protocols', self.getEnabledProtocol) @@ -129,35 +128,30 @@ class YarrProvider(Provider): # Check if we are still logged in every hour now = time.time() - if self.login_opener and self.last_login_check < (now - 3600): + if self.last_login_check and self.last_login_check < (now - 3600): try: - output = self.urlopen(self.urls['login_check'], opener = self.login_opener) + output = self.urlopen(self.urls['login_check']) if self.loginCheckSuccess(output): self.last_login_check = now return True - else: - self.login_opener = None - except: - self.login_opener = None + except: pass + self.last_login_check = None - if self.login_opener: + if self.last_login_check: return True try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - output = self.urlopen(self.urls['login'], params = self.getLoginParams(), opener = opener) + output = self.urlopen(self.urls['login'], data = self.getLoginParams()) if self.loginSuccess(output): self.last_login_check = now - self.login_opener = opener return True error = 'unknown' except: error = traceback.format_exc() - self.login_opener = None + self.last_login_check = None log.error('Failed to login %s: %s', (self.getName(), error)) return False @@ -171,12 +165,12 @@ class YarrProvider(Provider): try: if not self.login(): log.error('Failed downloading from %s', self.getName()) - return self.urlopen(url, opener = self.login_opener) + return self.urlopen(url) except: log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return '' + return {} def download(self, url = '', nzb_id = ''): try: @@ -230,19 +224,19 @@ class YarrProvider(Provider): def parseSize(self, size): - sizeRaw = size.lower() + size_raw = size.lower() size = tryFloat(re.sub(r'[^0-9.]', '', size).strip()) - for s in self.sizeGb: - if s in sizeRaw: + for s in self.size_gb: + if s in size_raw: return size * 1024 - for s in self.sizeMb: - if s in sizeRaw: + for s in self.size_mb: + if s in size_raw: return size - for s in self.sizeKb: - if s in sizeRaw: + for s in self.size_kb: + if s in size_raw: return size / 1024 return 0 @@ -264,14 +258,14 @@ class ResultList(list): result_ids = None provider = None - movie = None + media = None quality = None - def __init__(self, provider, movie, quality, **kwargs): + def __init__(self, provider, media, quality, **kwargs): self.result_ids = [] self.provider = provider - self.movie = movie + self.media = media self.quality = quality self.kwargs = kwargs @@ -285,13 +279,13 @@ class ResultList(list): new_result = self.fillResult(result) - is_correct = fireEvent('searcher.correct_release', new_result, self.movie, self.quality, - imdb_results = self.kwargs.get('imdb_results', False), single = True) + is_correct = fireEvent('searcher.correct_release', new_result, self.media, self.quality, + imdb_results = self.kwargs.get('imdb_results', False), single = True) if is_correct and new_result['id'] not in self.result_ids: is_correct_weight = float(is_correct) - new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True) + new_result['score'] += fireEvent('score.calculate', new_result, self.media, single = True) old_score = new_result['score'] new_result['score'] = int(old_score * is_correct_weight) diff --git a/couchpotato/core/providers/info/_modifier/__init__.py b/couchpotato/core/providers/info/_modifier/__init__.py index 3bdf5e0d..9dfab703 100644 --- a/couchpotato/core/providers/info/_modifier/__init__.py +++ b/couchpotato/core/providers/info/_modifier/__init__.py @@ -1,7 +1,7 @@ from .main import MovieResultModifier -def start(): +def start(): return MovieResultModifier() config = [] diff --git a/couchpotato/core/providers/info/_modifier/main.py b/couchpotato/core/providers/info/_modifier/main.py index 0bb2e6a4..88d4381c 100644 --- a/couchpotato/core/providers/info/_modifier/main.py +++ b/couchpotato/core/providers/info/_modifier/main.py @@ -21,14 +21,17 @@ class MovieResultModifier(Plugin): 'poster': [], 'backdrop': [], 'poster_original': [], - 'backdrop_original': [] + 'backdrop_original': [], + 'actors': {} }, 'runtime': 0, 'plot': '', 'tagline': '', 'imdb': '', 'genres': [], - 'mpaa': None + 'mpaa': None, + 'actors': [], + 'actor_roles': {} } def __init__(self): @@ -41,13 +44,13 @@ class MovieResultModifier(Plugin): new_results = {} for r in results: type_name = r.get('type', 'movie') + 's' - if not new_results.has_key(type_name): + if type_name not in new_results: new_results[type_name] = [] new_results[type_name].append(r) # Combine movies, needs a cleaner way.. - if new_results.has_key('movies'): + if 'movies' in new_results: new_results['movies'] = self.combineOnIMDB(new_results['movies']) return new_results @@ -93,11 +96,11 @@ class MovieResultModifier(Plugin): for movie in l.movies: if movie.status_id == active_status['id']: - temp['in_wanted'] = fireEvent('movie.get', movie.id, single = True) + temp['in_wanted'] = fireEvent('media.get', movie.id, single = True) for release in movie.releases: if release.status_id == done_status['id']: - temp['in_library'] = fireEvent('movie.get', movie.id, single = True) + temp['in_library'] = fireEvent('media.get', movie.id, single = True) except: log.error('Tried getting more info on searched movies: %s', traceback.format_exc()) diff --git a/couchpotato/core/providers/info/couchpotatoapi/__init__.py b/couchpotato/core/providers/info/couchpotatoapi/__init__.py index 37d9eca9..196dde6a 100644 --- a/couchpotato/core/providers/info/couchpotatoapi/__init__.py +++ b/couchpotato/core/providers/info/couchpotatoapi/__init__.py @@ -1,5 +1,6 @@ from .main import CouchPotatoApi + def start(): return CouchPotatoApi() diff --git a/couchpotato/core/providers/info/couchpotatoapi/main.py b/couchpotato/core/providers/info/couchpotatoapi/main.py index 4dd942e0..848cbf05 100644 --- a/couchpotato/core/providers/info/couchpotatoapi/main.py +++ b/couchpotato/core/providers/info/couchpotatoapi/main.py @@ -74,14 +74,14 @@ class CouchPotatoApi(MovieProvider): return True - def getInfo(self, identifier = None): + def getInfo(self, identifier = None, **kwargs): if not identifier: return result = self.getJsonData(self.urls['info'] % identifier, headers = self.getRequestHeaders()) if result: - return dict((k, v) for k, v in result.iteritems() if v) + return dict((k, v) for k, v in result.items() if v) return {} @@ -97,7 +97,7 @@ class CouchPotatoApi(MovieProvider): if not ignore: ignore = [] if not movies: movies = [] - suggestions = self.getJsonData(self.urls['suggest'], params = { + suggestions = self.getJsonData(self.urls['suggest'], data = { 'movies': ','.join(movies), 'ignore': ','.join(ignore), }, headers = self.getRequestHeaders()) @@ -110,5 +110,5 @@ class CouchPotatoApi(MovieProvider): 'X-CP-Version': fireEvent('app.version', single = True), 'X-CP-API': self.api_version, 'X-CP-Time': time.time(), - 'X-CP-Identifier': '+%s' % Env.setting('api_key', 'core')[:10], # Use first 10 as identifier, so we don't need to use IP address in api stats + 'X-CP-Identifier': '+%s' % Env.setting('api_key', 'core')[:10], # Use first 10 as identifier, so we don't need to use IP address in api stats } diff --git a/couchpotato/core/providers/info/omdbapi/__init__.py b/couchpotato/core/providers/info/omdbapi/__init__.py index 765662e9..b7ea3932 100644 --- a/couchpotato/core/providers/info/omdbapi/__init__.py +++ b/couchpotato/core/providers/info/omdbapi/__init__.py @@ -1,5 +1,6 @@ from .main import OMDBAPI + def start(): return OMDBAPI() diff --git a/couchpotato/core/providers/info/omdbapi/main.py b/couchpotato/core/providers/info/omdbapi/main.py index 47374f47..8f04d3b6 100755 --- a/couchpotato/core/providers/info/omdbapi/main.py +++ b/couchpotato/core/providers/info/omdbapi/main.py @@ -39,14 +39,14 @@ class OMDBAPI(MovieProvider): if cached: result = self.parseMovie(cached) if result.get('titles') and len(result.get('titles')) > 0: - log.info('Found: %s', result['titles'][0] + ' (' + str(result['year']) + ')') + log.info('Found: %s', result['titles'][0] + ' (' + str(result.get('year')) + ')') return [result] return [] return [] - def getInfo(self, identifier = None): + def getInfo(self, identifier = None, **kwargs): if not identifier: return {} @@ -107,7 +107,7 @@ class OMDBAPI(MovieProvider): 'writers': splitString(movie.get('Writer', '')), 'actors': splitString(movie.get('Actors', '')), } - movie_data = dict((k, v) for k, v in movie_data.iteritems() if v) + movie_data = dict((k, v) for k, v in movie_data.items() if v) except: log.error('Failed parsing IMDB API json: %s', traceback.format_exc()) diff --git a/couchpotato/core/providers/info/themoviedb/__init__.py b/couchpotato/core/providers/info/themoviedb/__init__.py index 66ac536a..b981950e 100644 --- a/couchpotato/core/providers/info/themoviedb/__init__.py +++ b/couchpotato/core/providers/info/themoviedb/__init__.py @@ -1,5 +1,6 @@ from .main import TheMovieDb + def start(): return TheMovieDb() diff --git a/couchpotato/core/providers/info/themoviedb/main.py b/couchpotato/core/providers/info/themoviedb/main.py index a7901351..d301db2b 100644 --- a/couchpotato/core/providers/info/themoviedb/main.py +++ b/couchpotato/core/providers/info/themoviedb/main.py @@ -1,5 +1,6 @@ from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import simplifyString, toUnicode +from couchpotato.core.helpers.encoding import simplifyString, toUnicode, ss +from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.info.base import MovieProvider import tmdb3 @@ -11,8 +12,8 @@ log = CPLog(__name__) class TheMovieDb(MovieProvider): def __init__(self): - addEvent('info.search', self.search, priority = 2) - addEvent('movie.search', self.search, priority = 2) + #addEvent('info.search', self.search, priority = 2) + #addEvent('movie.search', self.search, priority = 2) addEvent('movie.info', self.getInfo, priority = 2) addEvent('movie.info_by_tmdb', self.getInfo) @@ -45,7 +46,7 @@ class TheMovieDb(MovieProvider): nr = 0 for movie in raw: - results.append(self.parseMovie(movie, with_titles = False)) + results.append(self.parseMovie(movie, extended = False)) nr += 1 if nr == limit: @@ -55,34 +56,40 @@ class TheMovieDb(MovieProvider): self.setCache(cache_key, results) return results - except SyntaxError, e: + except SyntaxError as e: log.error('Failed to parse XML response: %s', e) return False return results - def getInfo(self, identifier = None): + def getInfo(self, identifier = None, extended = True): if not identifier: return {} - cache_key = 'tmdb.cache.%s' % identifier + cache_key = 'tmdb.cache.%s%s' % (identifier, '.ex' if extended else '') result = self.getCache(cache_key) if not result: try: log.debug('Getting info: %s', cache_key) movie = tmdb3.Movie(identifier) - result = self.parseMovie(movie) - self.setCache(cache_key, result) + try: exists = movie.title is not None + except: exists = False + + if exists: + result = self.parseMovie(movie, extended = extended) + self.setCache(cache_key, result) + else: + result = {} except: - pass + log.error('Failed getting info for %s: %s', (identifier, traceback.format_exc())) return result - def parseMovie(self, movie, with_titles = True): + def parseMovie(self, movie, extended = True): - cache_key = 'tmdb.cache.%s' % movie.id + cache_key = 'tmdb.cache.%s%s' % (movie.id, '.ex' if extended else '') movie_data = self.getCache(cache_key) if not movie_data: @@ -92,6 +99,14 @@ class TheMovieDb(MovieProvider): poster_original = self.getImage(movie, type = 'poster', size = 'original') backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original') + images = { + 'poster': [poster] if poster else [], + #'backdrop': [backdrop] if backdrop else [], + 'poster_original': [poster_original] if poster_original else [], + 'backdrop_original': [backdrop_original] if backdrop_original else [], + 'actors': {} + } + # Genres try: genres = [genre.name for genre in movie.genres] @@ -103,31 +118,37 @@ class TheMovieDb(MovieProvider): if not movie.releasedate or year == '1900' or year.lower() == 'none': year = None + # Gather actors data + actors = {} + if extended: + for cast_item in movie.cast: + try: + actors[toUnicode(cast_item.name)] = toUnicode(cast_item.character) + images['actors'][toUnicode(cast_item.name)] = self.getImage(cast_item, type = 'profile', size = 'original') + except: + log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) + movie_data = { 'type': 'movie', 'via_tmdb': True, 'tmdb_id': movie.id, 'titles': [toUnicode(movie.title)], 'original_title': movie.originaltitle, - 'images': { - 'poster': [poster] if poster else [], - #'backdrop': [backdrop] if backdrop else [], - 'poster_original': [poster_original] if poster_original else [], - 'backdrop_original': [backdrop_original] if backdrop_original else [], - }, + 'images': images, 'imdb': movie.imdb, 'runtime': movie.runtime, 'released': str(movie.releasedate), - 'year': year, + 'year': tryInt(year, None), 'plot': movie.overview, 'genres': genres, 'collection': getattr(movie.collection, 'name', None), + 'actor_roles': actors } - movie_data = dict((k, v) for k, v in movie_data.iteritems() if v) + movie_data = dict((k, v) for k, v in movie_data.items() if v) # Add alternative names - if with_titles: + if extended: movie_data['titles'].append(movie.originaltitle) for alt in movie.alternate_titles: alt_name = alt.title @@ -143,9 +164,9 @@ class TheMovieDb(MovieProvider): image_url = '' try: - image_url = getattr(movie, type).geturl(size = 'original') + image_url = getattr(movie, type).geturl(size = size) except: - log.debug('Failed getting %s.%s for "%s"', (type, size, movie.title)) + log.debug('Failed getting %s.%s for "%s"', (type, size, ss(str(movie)))) return image_url diff --git a/couchpotato/core/providers/metadata/base.py b/couchpotato/core/providers/metadata/base.py index f5610030..d1274adf 100644 --- a/couchpotato/core/providers/metadata/base.py +++ b/couchpotato/core/providers/metadata/base.py @@ -1,4 +1,5 @@ from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import mergeDicts from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -25,7 +26,7 @@ class MetaDataBase(Plugin): # Update library to get latest info try: - updated_library = fireEvent('library.update.movie', group['library']['identifier'], force = True, single = True) + updated_library = fireEvent('library.update.movie', group['library']['identifier'], extended = True, single = True) group['library'] = mergeDicts(group['library'], updated_library) except: log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc()) @@ -48,6 +49,9 @@ class MetaDataBase(Plugin): if content: log.debug('Creating %s file: %s', (file_type, name)) if os.path.isfile(content): + content = sp(content) + name = sp(name) + shutil.copy2(content, name) shutil.copyfile(content, name) @@ -59,7 +63,7 @@ class MetaDataBase(Plugin): group['renamed_files'].append(name) try: - os.chmod(name, Env.getPermission('file')) + os.chmod(sp(name), Env.getPermission('file')) except: log.debug('Failed setting permissions for %s: %s', (name, traceback.format_exc())) diff --git a/couchpotato/core/providers/metadata/wmc/__init__.py b/couchpotato/core/providers/metadata/wmc/__init__.py index 290436c6..167a24d7 100644 --- a/couchpotato/core/providers/metadata/wmc/__init__.py +++ b/couchpotato/core/providers/metadata/wmc/__init__.py @@ -1,5 +1,6 @@ from .main import WindowsMediaCenter + def start(): return WindowsMediaCenter() diff --git a/couchpotato/core/providers/metadata/wmc/main.py b/couchpotato/core/providers/metadata/wmc/main.py index 89258918..f84897b4 100644 --- a/couchpotato/core/providers/metadata/wmc/main.py +++ b/couchpotato/core/providers/metadata/wmc/main.py @@ -1,6 +1,7 @@ from couchpotato.core.providers.metadata.base import MetaDataBase import os + class WindowsMediaCenter(MetaDataBase): def getThumbnailName(self, name, root): diff --git a/couchpotato/core/providers/metadata/xbmc/__init__.py b/couchpotato/core/providers/metadata/xbmc/__init__.py index ea426dba..deb5c908 100644 --- a/couchpotato/core/providers/metadata/xbmc/__init__.py +++ b/couchpotato/core/providers/metadata/xbmc/__init__.py @@ -1,5 +1,6 @@ from .main import XBMC + def start(): return XBMC() diff --git a/couchpotato/core/providers/metadata/xbmc/main.py b/couchpotato/core/providers/metadata/xbmc/main.py index 7073363d..267b2822 100644 --- a/couchpotato/core/providers/metadata/xbmc/main.py +++ b/couchpotato/core/providers/metadata/xbmc/main.py @@ -65,7 +65,7 @@ class XBMC(MetaDataBase): name = type try: - if data['library'].get(type): + if movie_info.get(type): el = SubElement(nfoxml, name) el.text = toUnicode(movie_info.get(type, '')) except: @@ -89,10 +89,18 @@ class XBMC(MetaDataBase): genres.text = toUnicode(genre) # Actors - for actor in movie_info.get('actors', []): - actors = SubElement(nfoxml, 'actor') - name = SubElement(actors, 'name') - name.text = toUnicode(actor) + for actor_name in movie_info.get('actor_roles', {}): + role_name = movie_info['actor_roles'][actor_name] + + actor = SubElement(nfoxml, 'actor') + name = SubElement(actor, 'name') + name.text = toUnicode(actor_name) + if role_name: + role = SubElement(actor, 'role') + role.text = toUnicode(role_name) + if movie_info['images']['actors'].get(actor_name): + thumb = SubElement(actor, 'thumb') + thumb.text = toUnicode(movie_info['images']['actors'].get(actor_name)) # Directors for director_name in movie_info.get('directors', []): @@ -112,6 +120,51 @@ class XBMC(MetaDataBase): sorttitle = SubElement(nfoxml, 'sorttitle') sorttitle.text = '%s %s' % (toUnicode(collection_name), movie_info.get('year')) + # Images + for image_url in movie_info['images']['poster_original']: + image = SubElement(nfoxml, 'thumb') + image.text = toUnicode(image_url) + fanart = SubElement(nfoxml, 'fanart') + for image_url in movie_info['images']['backdrop_original']: + image = SubElement(fanart, 'thumb') + image.text = toUnicode(image_url) + + # Add trailer if found + trailer_found = False + if data.get('renamed_files'): + for filename in data.get('renamed_files'): + if 'trailer' in filename: + trailer = SubElement(nfoxml, 'trailer') + trailer.text = toUnicode(filename) + trailer_found = True + if not trailer_found and data['files'].get('trailer'): + trailer = SubElement(nfoxml, 'trailer') + trailer.text = toUnicode(data['files']['trailer'][0]) + + # Add file metadata + fileinfo = SubElement(nfoxml, 'fileinfo') + streamdetails = SubElement(fileinfo, 'streamdetails') + + # Video data + if data['meta_data'].get('video'): + video = SubElement(streamdetails, 'video') + codec = SubElement(video, 'codec') + codec.text = toUnicode(data['meta_data']['video']) + aspect = SubElement(video, 'aspect') + aspect.text = str(data['meta_data']['aspect']) + width = SubElement(video, 'width') + width.text = str(data['meta_data']['resolution_width']) + height = SubElement(video, 'height') + height.text = str(data['meta_data']['resolution_height']) + + # Audio data + if data['meta_data'].get('audio'): + audio = SubElement(streamdetails, 'audio') + codec = SubElement(audio, 'codec') + codec.text = toUnicode(data['meta_data'].get('audio')) + channels = SubElement(audio, 'channels') + channels.text = toUnicode(data['meta_data'].get('audio_channels')) + # Clean up the xml and return it nfoxml = xml.dom.minidom.parseString(tostring(nfoxml)) xml_string = nfoxml.toprettyxml(indent = ' ') diff --git a/couchpotato/core/providers/nzb/binsearch/__init__.py b/couchpotato/core/providers/nzb/binsearch/__init__.py index 1cfb0b73..c80ee6d9 100644 --- a/couchpotato/core/providers/nzb/binsearch/__init__.py +++ b/couchpotato/core/providers/nzb/binsearch/__init__.py @@ -1,5 +1,6 @@ from .main import BinSearch + def start(): return BinSearch() diff --git a/couchpotato/core/providers/nzb/binsearch/main.py b/couchpotato/core/providers/nzb/binsearch/main.py index db0fb5b8..c54dd435 100644 --- a/couchpotato/core/providers/nzb/binsearch/main.py +++ b/couchpotato/core/providers/nzb/binsearch/main.py @@ -18,7 +18,7 @@ class BinSearch(NZBProvider): 'search': 'https://www.binsearch.info/index.php?%s', } - http_time_between_calls = 4 # Seconds + http_time_between_calls = 4 # Seconds def _search(self, movie, quality, results): @@ -90,13 +90,13 @@ class BinSearch(NZBProvider): def download(self, url = '', nzb_id = ''): - params = { + data = { 'action': 'nzb', nzb_id: 'on' } try: - return self.urlopen(url, params = params, show_error = False) + return self.urlopen(url, data = data, show_error = False) except: log.error('Failed getting nzb from %s: %s', (self.getName(), traceback.format_exc())) diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py index 54359275..97f1cfad 100644 --- a/couchpotato/core/providers/nzb/newznab/__init__.py +++ b/couchpotato/core/providers/nzb/newznab/__init__.py @@ -1,5 +1,6 @@ from .main import Newznab + def start(): return Newznab() @@ -38,13 +39,20 @@ config = [{ 'default': '0,0,0,0,0,0', 'description': 'Starting score for each release found via this provider.', }, + { + 'name': 'custom_tag', + 'advanced': True, + 'label': 'Custom tag', + 'default': ',,,,,', + 'description': 'Add custom tags, for example add rls=1 to get only scene releases from nzbs.org', + }, { 'name': 'api_key', 'default': ',,,,,', 'label': 'Api Key', 'description': 'Can be found on your profile page', 'type': 'combined', - 'combine': ['use', 'host', 'api_key', 'extra_score'], + 'combine': ['use', 'host', 'api_key', 'extra_score', 'custom_tag'], }, ], }, diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index bd1b6c32..deadaa1c 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -10,6 +10,7 @@ from urllib2 import HTTPError from urlparse import urlparse import time import traceback +import urllib2 log = CPLog(__name__) @@ -24,7 +25,7 @@ class Newznab(NZBProvider, RSS): limits_reached = {} - http_time_between_calls = 1 # Seconds + http_time_between_calls = 1 # Seconds def search(self, movie, quality): hosts = self.getHosts() @@ -45,7 +46,7 @@ class Newznab(NZBProvider, RSS): 'imdbid': movie['library']['identifier'].replace('tt', ''), 'apikey': host['api_key'], 'extended': 1 - }) + }) + ('&%s' % host['custom_tag'] if host.get('custom_tag') else '') url = '%s&%s' % (self.getUrl(host['host'], self.urls['search']), arguments) nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) @@ -99,6 +100,7 @@ class Newznab(NZBProvider, RSS): hosts = splitString(self.conf('host'), clean = False) api_keys = splitString(self.conf('api_key'), clean = False) extra_score = splitString(self.conf('extra_score'), clean = False) + custom_tags = splitString(self.conf('custom_tag'), clean = False) list = [] for nr in range(len(hosts)): @@ -109,11 +111,18 @@ class Newznab(NZBProvider, RSS): try: host = hosts[nr] except: host = '' + try: score = tryInt(extra_score[nr]) + except: score = 0 + + try: custom_tag = custom_tags[nr] + except: custom_tag = '' + list.append({ 'use': uses[nr], 'host': host, 'api_key': key, - 'extra_score': tryInt(extra_score[nr]) if len(extra_score) > nr else 0 + 'extra_score': score, + 'custom_tag': custom_tag }) return list @@ -159,10 +168,19 @@ class Newznab(NZBProvider, RSS): return 'try_next' try: - data = self.urlopen(url, show_error = False) + # Get final redirected url + log.debug('Checking %s for redirects.', url) + req = urllib2.Request(url) + req.add_header('User-Agent', self.user_agent) + res = urllib2.urlopen(req) + finalurl = res.geturl() + if finalurl != url: + log.debug('Redirect url used: %s', finalurl) + + data = self.urlopen(finalurl, show_error = False) self.limits_reached[host] = False return data - except HTTPError, e: + except HTTPError as e: if e.code == 503: response = e.read().lower() if 'maximum api' in response or 'download limit' in response: diff --git a/couchpotato/core/providers/nzb/nzbclub/__init__.py b/couchpotato/core/providers/nzb/nzbclub/__init__.py index 95eeea13..02a69404 100644 --- a/couchpotato/core/providers/nzb/nzbclub/__init__.py +++ b/couchpotato/core/providers/nzb/nzbclub/__init__.py @@ -1,5 +1,6 @@ from .main import NZBClub + def start(): return NZBClub() diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py index 59382dfd..ce853cd5 100644 --- a/couchpotato/core/providers/nzb/nzbclub/main.py +++ b/couchpotato/core/providers/nzb/nzbclub/main.py @@ -13,17 +13,20 @@ log = CPLog(__name__) class NZBClub(NZBProvider, RSS): urls = { - 'search': 'http://www.nzbclub.com/nzbfeed.aspx?%s', + 'search': 'https://www.nzbclub.com/nzbfeeds.aspx?%s', } - http_time_between_calls = 4 #seconds + http_time_between_calls = 4 #seconds def _searchOnTitle(self, title, movie, quality, results): q = '"%s %s"' % (title, movie['library']['year']) - params = tryUrlencode({ + q_param = tryUrlencode({ 'q': q, + }) + + params = tryUrlencode({ 'ig': 1, 'rpp': 200, 'st': 5, @@ -31,7 +34,7 @@ class NZBClub(NZBProvider, RSS): 'ns': 1, }) - nzbs = self.getRSSData(self.urls['search'] % params) + nzbs = self.getRSSData(self.urls['search'] % ('%s&%s' % (q_param, params))) for nzb in nzbs: diff --git a/couchpotato/core/providers/nzb/nzbindex/__init__.py b/couchpotato/core/providers/nzb/nzbindex/__init__.py index 47461e63..acb53e19 100644 --- a/couchpotato/core/providers/nzb/nzbindex/__init__.py +++ b/couchpotato/core/providers/nzb/nzbindex/__init__.py @@ -1,5 +1,6 @@ from .main import NzbIndex + def start(): return NzbIndex() diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index 17b87fac..a143c199 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -19,7 +19,7 @@ class NzbIndex(NZBProvider, RSS): 'search': 'https://www.nzbindex.com/rss/?%s', } - http_time_between_calls = 1 # Seconds + http_time_between_calls = 1 # Seconds def _searchOnTitle(self, title, movie, quality, results): diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py b/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py index 933aff3e..2f3990de 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/__init__.py @@ -1,5 +1,6 @@ from .main import OMGWTFNZBs + def start(): return OMGWTFNZBs() diff --git a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py index 8cc4a3eb..93925752 100644 --- a/couchpotato/core/providers/nzb/omgwtfnzbs/main.py +++ b/couchpotato/core/providers/nzb/omgwtfnzbs/main.py @@ -18,7 +18,7 @@ class OMGWTFNZBs(NZBProvider, RSS): 'detail_url': 'https://omgwtfnzbs.org/details.php?id=%s', } - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds cat_ids = [ ([15], ['dvdrip']), diff --git a/couchpotato/core/providers/torrent/awesomehd/__init__.py b/couchpotato/core/providers/torrent/awesomehd/__init__.py index de6a2144..6f076703 100644 --- a/couchpotato/core/providers/torrent/awesomehd/__init__.py +++ b/couchpotato/core/providers/torrent/awesomehd/__init__.py @@ -1,5 +1,6 @@ from .main import AwesomeHD + def start(): return AwesomeHD() diff --git a/couchpotato/core/providers/torrent/awesomehd/main.py b/couchpotato/core/providers/torrent/awesomehd/main.py index 79482f2a..ca6a30df 100644 --- a/couchpotato/core/providers/torrent/awesomehd/main.py +++ b/couchpotato/core/providers/torrent/awesomehd/main.py @@ -11,10 +11,10 @@ log = CPLog(__name__) class AwesomeHD(TorrentProvider): urls = { - 'test' : 'https://awesome-hd.net/', - 'detail' : 'https://awesome-hd.net/torrents.php?torrentid=%s', - 'search' : 'https://awesome-hd.net/searchapi.php?action=imdbsearch&passkey=%s&imdb=%s&internal=%s', - 'download' : 'https://awesome-hd.net/torrents.php?action=download&id=%s&authkey=%s&torrent_pass=%s', + 'test': 'https://awesome-hd.net/', + 'detail': 'https://awesome-hd.net/torrents.php?torrentid=%s', + 'search': 'https://awesome-hd.net/searchapi.php?action=imdbsearch&passkey=%s&imdb=%s&internal=%s', + 'download': 'https://awesome-hd.net/torrents.php?action=download&id=%s&authkey=%s&torrent_pass=%s', } http_time_between_calls = 1 diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index c16e6c52..e134c8f3 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -1,3 +1,4 @@ +import traceback from couchpotato.core.helpers.variable import getImdb, md5, cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.providers.base import YarrProvider @@ -14,22 +15,6 @@ class TorrentProvider(YarrProvider): proxy_domain = None proxy_list = [] - def imdbMatch(self, url, imdbId): - if getImdb(url) == imdbId: - return True - - if url[:4] == 'http': - try: - cache_key = md5(url) - data = self.getCache(cache_key, url) - except IOError: - log.error('Failed to open %s.', url) - return False - - return getImdb(data) == imdbId - - return False - def getDomain(self, url = ''): forced_domain = self.conf('domain') @@ -48,7 +33,7 @@ class TorrentProvider(YarrProvider): try: data = self.urlopen(proxy, timeout = 3, show_error = False) except: - log.debug('Failed %s proxy %s', (self.getName(), proxy)) + log.debug('Failed %s proxy %s: %s', (self.getName(), proxy, traceback.format_exc())) if self.correctProxy(data): log.debug('Using proxy for %s: %s', (self.getName(), proxy)) @@ -63,9 +48,10 @@ class TorrentProvider(YarrProvider): return cleanHost(self.proxy_domain).rstrip('/') + url - def correctProxy(self): + def correctProxy(self, data): return True + class TorrentMagnetProvider(TorrentProvider): protocol = 'torrent_magnet' diff --git a/couchpotato/core/providers/torrent/bithdtv/__init__.py b/couchpotato/core/providers/torrent/bithdtv/__init__.py index 8c6f97a0..ffc5363f 100644 --- a/couchpotato/core/providers/torrent/bithdtv/__init__.py +++ b/couchpotato/core/providers/torrent/bithdtv/__init__.py @@ -1,5 +1,6 @@ from .main import BiTHDTV + def start(): return BiTHDTV() diff --git a/couchpotato/core/providers/torrent/bithdtv/main.py b/couchpotato/core/providers/torrent/bithdtv/main.py index 2cacff3d..90117de4 100644 --- a/couchpotato/core/providers/torrent/bithdtv/main.py +++ b/couchpotato/core/providers/torrent/bithdtv/main.py @@ -7,14 +7,15 @@ import traceback log = CPLog(__name__) + class BiTHDTV(TorrentProvider): urls = { - 'test' : 'http://www.bit-hdtv.com/', - 'login' : 'http://www.bit-hdtv.com/takelogin.php', + 'test': 'http://www.bit-hdtv.com/', + 'login': 'http://www.bit-hdtv.com/takelogin.php', 'login_check': 'http://www.bit-hdtv.com/messages.php', - 'detail' : 'http://www.bit-hdtv.com/details.php?id=%s', - 'search' : 'http://www.bit-hdtv.com/torrents.php?', + 'detail': 'http://www.bit-hdtv.com/details.php?id=%s', + 'search': 'http://www.bit-hdtv.com/torrents.php?', } # Searches for movies only - BiT-HDTV's subcategory and resolution search filters appear to be broken @@ -31,7 +32,7 @@ class BiTHDTV(TorrentProvider): url = "%s&%s" % (self.urls['search'], arguments) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: # Remove BiT-HDTV's output garbage so outdated BS4 versions successfully parse the HTML @@ -68,10 +69,10 @@ class BiTHDTV(TorrentProvider): log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), - }) + } def getMoreInfo(self, item): full_description = self.getCache('bithdtv.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) diff --git a/couchpotato/core/providers/torrent/bitsoup/__init__.py b/couchpotato/core/providers/torrent/bitsoup/__init__.py index a36ab08f..da07cc3b 100644 --- a/couchpotato/core/providers/torrent/bitsoup/__init__.py +++ b/couchpotato/core/providers/torrent/bitsoup/__init__.py @@ -1,5 +1,6 @@ from .main import Bitsoup + def start(): return Bitsoup() diff --git a/couchpotato/core/providers/torrent/bitsoup/main.py b/couchpotato/core/providers/torrent/bitsoup/main.py index 539ba43d..a709c5c1 100644 --- a/couchpotato/core/providers/torrent/bitsoup/main.py +++ b/couchpotato/core/providers/torrent/bitsoup/main.py @@ -12,7 +12,7 @@ class Bitsoup(TorrentProvider): urls = { 'test': 'https://www.bitsoup.me/', - 'login' : 'https://www.bitsoup.me/takelogin.php', + 'login': 'https://www.bitsoup.me/takelogin.php', 'login_check': 'https://www.bitsoup.me/my.php', 'search': 'https://www.bitsoup.me/browse.php?', 'baseurl': 'https://www.bitsoup.me/%s', @@ -28,13 +28,16 @@ class Bitsoup(TorrentProvider): }) url = "%s&%s" % (self.urls['search'], arguments) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) try: result_table = html.find('table', attrs = {'class': 'koptekst'}) + if not result_table or 'nothing found!' in data.lower(): + return + entries = result_table.find_all('tr') for result in entries[1:]: @@ -70,11 +73,11 @@ class Bitsoup(TorrentProvider): def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'ssl': 'yes', - }) + } def loginSuccess(self, output): diff --git a/couchpotato/core/providers/torrent/hdbits/__init__.py b/couchpotato/core/providers/torrent/hdbits/__init__.py index 07ea95d6..1e9aa3ce 100644 --- a/couchpotato/core/providers/torrent/hdbits/__init__.py +++ b/couchpotato/core/providers/torrent/hdbits/__init__.py @@ -1,5 +1,6 @@ from .main import HDBits + def start(): return HDBits() @@ -21,11 +22,6 @@ config = [{ 'name': 'username', 'default': '', }, - { - 'name': 'password', - 'default': '', - 'type': 'password', - }, { 'name': 'passkey', 'default': '', diff --git a/couchpotato/core/providers/torrent/hdbits/main.py b/couchpotato/core/providers/torrent/hdbits/main.py index 7b0444ba..ce17bbac 100644 --- a/couchpotato/core/providers/torrent/hdbits/main.py +++ b/couchpotato/core/providers/torrent/hdbits/main.py @@ -1,8 +1,9 @@ -from bs4 import BeautifulSoup -from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider + +import re +import json import traceback log = CPLog(__name__) @@ -11,48 +12,52 @@ log = CPLog(__name__) class HDBits(TorrentProvider): urls = { - 'test' : 'https://hdbits.org/', - 'login' : 'https://hdbits.org/login/doLogin/', - 'detail' : 'https://hdbits.org/details.php?id=%s&source=browse', - 'search' : 'https://hdbits.org/json_search.php?imdb=%s', - 'download' : 'https://hdbits.org/download.php/%s.torrent?id=%s&passkey=%s&source=details.browse', - 'login_check': 'http://hdbits.org/inbox.php', + 'test': 'https://hdbits.org/', + 'detail': 'https://hdbits.org/details.php?id=%s', + 'download': 'https://hdbits.org/download.php?id=%s&passkey=%s', + 'api': 'https://hdbits.org/api/torrents' } http_time_between_calls = 1 #seconds + def _post_query(self, **params): + + post_data = { + 'username': self.conf('username'), + 'passkey': self.conf('passkey') + } + post_data.update(params) + + try: + result = self.getJsonData(self.urls['api'], data = json.dumps(post_data)) + + if result: + if result['status'] != 0: + log.error('Error searching hdbits: %s' % result['message']) + else: + return result['data'] + except: + pass + + return None + def _search(self, movie, quality, results): - data = self.getJsonData(self.urls['search'] % movie['library']['identifier'], opener = self.login_opener) + match = re.match(r'tt(\d{7})', movie['library']['identifier']) + + data = self._post_query(imdb = {'id': match.group(1)}) if data: try: for result in data: results.append({ 'id': result['id'], - 'name': result['title'], - 'url': self.urls['download'] % (result['id'], result['id'], self.conf('passkey')), + 'name': result['name'], + 'url': self.urls['download'] % (result['id'], self.conf('passkey')), 'detail_url': self.urls['detail'] % result['id'], 'size': self.parseSize(result['size']), - 'seeders': tryInt(result['seeder']), - 'leechers': tryInt(result['leecher']) + 'seeders': tryInt(result['seeders']), + 'leechers': tryInt(result['leechers']) }) - except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) - - def getLoginParams(self): - data = self.getHTMLData('https://hdbits.org/login') - bs = BeautifulSoup(data) - secret = bs.find('input', attrs = {'name': 'lol'})['value'] - - return tryUrlencode({ - 'uname': self.conf('username'), - 'password': self.conf('password'), - 'lol': secret - }) - - def loginSuccess(self, output): - return '/logout.php' in output.lower() - - loginCheckSuccess = loginSuccess diff --git a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py index c6702d7f..f3f7b479 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/__init__.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/__init__.py @@ -1,4 +1,5 @@ -from main import ILoveTorrents +from .main import ILoveTorrents + def start(): return ILoveTorrents() @@ -18,14 +19,14 @@ config = [{ 'type': 'enabler', 'default': False }, - { + { 'name': 'username', 'label': 'Username', 'type': 'string', 'default': '', 'description': 'The user name for your ILT account', }, - { + { 'name': 'password', 'label': 'Password', 'type': 'password', diff --git a/couchpotato/core/providers/torrent/ilovetorrents/main.py b/couchpotato/core/providers/torrent/ilovetorrents/main.py index 8c060ec3..f8ed67a3 100644 --- a/couchpotato/core/providers/torrent/ilovetorrents/main.py +++ b/couchpotato/core/providers/torrent/ilovetorrents/main.py @@ -12,12 +12,12 @@ log = CPLog(__name__) class ILoveTorrents(TorrentProvider): urls = { - 'download': 'http://www.ilovetorrents.me/%s', - 'detail': 'http://www.ilovetorrents.me/%s', - 'search': 'http://www.ilovetorrents.me/browse.php?search=%s&page=%s&cat=%s', - 'test' : 'http://www.ilovetorrents.me/', - 'login' : 'http://www.ilovetorrents.me/takelogin.php', - 'login_check' : 'http://www.ilovetorrents.me' + 'download': 'https://www.ilovetorrents.me/%s', + 'detail': 'https//www.ilovetorrents.me/%s', + 'search': 'https://www.ilovetorrents.me/browse.php?search=%s&page=%s&cat=%s', + 'test': 'https://www.ilovetorrents.me/', + 'login': 'https://www.ilovetorrents.me/takelogin.php', + 'login_check': 'https://www.ilovetorrents.me' } cat_ids = [ @@ -42,7 +42,7 @@ class ILoveTorrents(TorrentProvider): search_url = self.urls['search'] % (movieTitle, page, cats[0]) page += 1 - data = self.getHTMLData(search_url, opener = self.login_opener) + data = self.getHTMLData(search_url) if data: try: soup = BeautifulSoup(data) @@ -96,11 +96,11 @@ class ILoveTorrents(TorrentProvider): log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'submit': 'Welcome to ILT', - }) + } def getMoreInfo(self, item): cache_key = 'ilt.%s' % item['id'] @@ -109,7 +109,7 @@ class ILoveTorrents(TorrentProvider): if not description: try: - full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener) + full_description = self.getHTMLData(item['detail_url']) html = BeautifulSoup(full_description) nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1] description = toUnicode(nfo_pre.text) if nfo_pre else '' diff --git a/couchpotato/core/providers/torrent/iptorrents/__init__.py b/couchpotato/core/providers/torrent/iptorrents/__init__.py index 6cb2dead..579d7974 100644 --- a/couchpotato/core/providers/torrent/iptorrents/__init__.py +++ b/couchpotato/core/providers/torrent/iptorrents/__init__.py @@ -1,5 +1,6 @@ from .main import IPTorrents + def start(): return IPTorrents() diff --git a/couchpotato/core/providers/torrent/iptorrents/main.py b/couchpotato/core/providers/torrent/iptorrents/main.py index d22f4bdd..4a2c6dbf 100644 --- a/couchpotato/core/providers/torrent/iptorrents/main.py +++ b/couchpotato/core/providers/torrent/iptorrents/main.py @@ -1,5 +1,5 @@ from bs4 import BeautifulSoup -from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.helpers.encoding import tryUrlencode, toSafeString from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider @@ -11,11 +11,11 @@ log = CPLog(__name__) class IPTorrents(TorrentProvider): urls = { - 'test' : 'http://www.iptorrents.com/', - 'base_url' : 'http://www.iptorrents.com', - 'login' : 'http://www.iptorrents.com/torrents/', - 'login_check': 'http://www.iptorrents.com/inbox.php', - 'search' : 'http://www.iptorrents.com/torrents/?l%d=1%s&q=%s&qf=ti&p=%d', + 'test': 'https://www.iptorrents.com/', + 'base_url': 'https://www.iptorrents.com', + 'login': 'https://www.iptorrents.com/torrents/', + 'login_check': 'https://www.iptorrents.com/inbox.php', + 'search': 'https://www.iptorrents.com/torrents/?l%d=1%s&q=%s&qf=ti&p=%d', } cat_ids = [ @@ -37,7 +37,7 @@ class IPTorrents(TorrentProvider): while current_page <= pages and not self.shuttingDown(): url = self.urls['search'] % (self.getCatId(quality['identifier'])[0], freeleech, tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), current_page) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) @@ -57,21 +57,27 @@ class IPTorrents(TorrentProvider): entries = result_table.find_all('tr') + columns = self.getColumns(entries) + + if 'seeders' not in columns or 'leechers' not in columns: + log.warning('Unrecognized table format returned') + return + for result in entries[1:]: - torrent = result.find_all('td') - if len(torrent) <= 1: + cells = result.find_all('td') + if len(cells) <= 1: break - torrent = torrent[1].find('a') + torrent = cells[1].find('a') torrent_id = torrent['href'].replace('/details.php?id=', '') torrent_name = torrent.string torrent_download_url = self.urls['base_url'] + (result.find_all('td')[3].find('a'))['href'].replace(' ', '.') torrent_details_url = self.urls['base_url'] + torrent['href'] torrent_size = self.parseSize(result.find_all('td')[5].string) - torrent_seeders = tryInt(result.find('td', attrs = {'class' : 'ac t_seeders'}).string) - torrent_leechers = tryInt(result.find('td', attrs = {'class' : 'ac t_leechers'}).string) + torrent_seeders = tryInt(cells[columns['seeders']].string) + torrent_leechers = tryInt(cells[columns['leechers']].string) results.append({ 'id': torrent_id, @@ -89,12 +95,26 @@ class IPTorrents(TorrentProvider): current_page += 1 + def getColumns(self, entries): + result = {} + + for x, col in enumerate(entries[0].find_all('th')): + name = col.text or col.find('img')['title'] + key = toSafeString(name).strip().lower() + + if not key: + continue + + result[key] = x + + return result + def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'login': 'submit', - }) + } def loginSuccess(self, output): return 'don\'t have an account' not in output.lower() diff --git a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py index 0b79c81a..ffe1040b 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/__init__.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/__init__.py @@ -1,5 +1,6 @@ from .main import KickAssTorrents + def start(): return KickAssTorrents() diff --git a/couchpotato/core/providers/torrent/kickasstorrents/main.py b/couchpotato/core/providers/torrent/kickasstorrents/main.py index 50f14ce2..f96e9812 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/main.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/main.py @@ -24,7 +24,7 @@ class KickAssTorrents(TorrentMagnetProvider): (['dvd'], ['dvdr']), ] - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds cat_backup_id = None proxy_list = [ @@ -45,7 +45,7 @@ class KickAssTorrents(TorrentMagnetProvider): try: html = BeautifulSoup(data) - resultdiv = html.find('div', attrs = {'class':'tabs'}) + resultdiv = html.find('div', attrs = {'class': 'tabs'}) for result in resultdiv.find_all('div', recursive = False): if result.get('id').lower().strip('tab-') not in cat_ids: continue @@ -107,7 +107,6 @@ class KickAssTorrents(TorrentMagnetProvider): return tryInt(age) - def isEnabled(self): return super(KickAssTorrents, self).isEnabled() and self.getDomain() diff --git a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py index 66b3ea76..a3e57c79 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/__init__.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/__init__.py @@ -1,4 +1,5 @@ -from main import PassThePopcorn +from .main import PassThePopcorn + def start(): return PassThePopcorn() diff --git a/couchpotato/core/providers/torrent/passthepopcorn/main.py b/couchpotato/core/providers/torrent/passthepopcorn/main.py index 42df76c8..57a36c27 100644 --- a/couchpotato/core/providers/torrent/passthepopcorn/main.py +++ b/couchpotato/core/providers/torrent/passthepopcorn/main.py @@ -8,6 +8,7 @@ import json import re import time import traceback +import six log = CPLog(__name__) @@ -15,12 +16,12 @@ log = CPLog(__name__) class PassThePopcorn(TorrentProvider): urls = { - 'domain': 'https://tls.passthepopcorn.me', - 'detail': 'https://tls.passthepopcorn.me/torrents.php?torrentid=%s', - 'torrent': 'https://tls.passthepopcorn.me/torrents.php', - 'login': 'https://tls.passthepopcorn.me/ajax.php?action=login', - 'login_check': 'https://tls.passthepopcorn.me/ajax.php?action=login', - 'search': 'https://tls.passthepopcorn.me/search/%s/0/7/%d' + 'domain': 'https://tls.passthepopcorn.me', + 'detail': 'https://tls.passthepopcorn.me/torrents.php?torrentid=%s', + 'torrent': 'https://tls.passthepopcorn.me/torrents.php', + 'login': 'https://tls.passthepopcorn.me/ajax.php?action=login', + 'login_check': 'https://tls.passthepopcorn.me/ajax.php?action=login', + 'search': 'https://tls.passthepopcorn.me/search/%s/0/7/%d' } http_time_between_calls = 2 @@ -65,7 +66,7 @@ class PassThePopcorn(TorrentProvider): }) url = '%s?json=noredirect&%s' % (self.urls['torrent'], tryUrlencode(params)) - res = self.getJsonData(url, opener = self.login_opener) + res = self.getJsonData(url) try: if not 'Movies' in res: @@ -88,11 +89,11 @@ class PassThePopcorn(TorrentProvider): if 'GoldenPopcorn' in torrent and torrent['GoldenPopcorn']: torrentdesc += ' HQ' if self.conf('prefer_golden'): - torrentscore += 200 + torrentscore += 5000 if 'Scene' in torrent and torrent['Scene']: torrentdesc += ' Scene' if self.conf('prefer_scene'): - torrentscore += 50 + torrentscore += 2000 if 'RemasterTitle' in torrent and torrent['RemasterTitle']: torrentdesc += self.htmlToASCII(' %s' % torrent['RemasterTitle']) @@ -178,7 +179,7 @@ class PassThePopcorn(TorrentProvider): except KeyError: pass return text # leave as is - return re.sub("&#?\w+;", fixup, u'%s' % text) + return re.sub("&#?\w+;", fixup, six.u('%s') % text) def unicodeToASCII(self, text): import unicodedata @@ -188,13 +189,13 @@ class PassThePopcorn(TorrentProvider): return self.unicodeToASCII(self.htmlToUnicode(text)) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'passkey': self.conf('passkey'), 'keeplogged': '1', 'login': 'Login' - }) + } def loginSuccess(self, output): try: diff --git a/couchpotato/core/providers/torrent/publichd/__init__.py b/couchpotato/core/providers/torrent/publichd/__init__.py index ace12880..3c20c51f 100644 --- a/couchpotato/core/providers/torrent/publichd/__init__.py +++ b/couchpotato/core/providers/torrent/publichd/__init__.py @@ -1,5 +1,6 @@ from .main import PublicHD + def start(): return PublicHD() diff --git a/couchpotato/core/providers/torrent/publichd/main.py b/couchpotato/core/providers/torrent/publichd/main.py index 7b497fd9..b7c32fba 100644 --- a/couchpotato/core/providers/torrent/publichd/main.py +++ b/couchpotato/core/providers/torrent/publichd/main.py @@ -76,7 +76,7 @@ class PublicHD(TorrentMagnetProvider): try: full_description = self.urlopen(item['detail_url']) html = BeautifulSoup(full_description) - nfo_pre = html.find('div', attrs = {'id':'torrmain'}) + nfo_pre = html.find('div', attrs = {'id': 'torrmain'}) description = toUnicode(nfo_pre.text) if nfo_pre else '' except: log.error('Failed getting more info for %s', item['name']) diff --git a/couchpotato/core/providers/torrent/sceneaccess/__init__.py b/couchpotato/core/providers/torrent/sceneaccess/__init__.py index 4b675573..3fa5d97f 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/__init__.py +++ b/couchpotato/core/providers/torrent/sceneaccess/__init__.py @@ -1,5 +1,6 @@ from .main import SceneAccess + def start(): return SceneAccess() diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 7e9ab896..c1c871ee 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -15,7 +15,7 @@ class SceneAccess(TorrentProvider): 'login': 'https://www.sceneaccess.eu/login', 'login_check': 'https://www.sceneaccess.eu/inbox', 'detail': 'https://www.sceneaccess.eu/details?id=%s', - 'search': 'https://www.sceneaccess.eu/browse?method=2&c%d=%d', + 'search': 'https://www.sceneaccess.eu/browse?c%d=%d', 'download': 'https://www.sceneaccess.eu/%s', } @@ -25,7 +25,7 @@ class SceneAccess(TorrentProvider): ([8], ['dvdr']), ] - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds def _search(self, movie, quality, results): @@ -40,12 +40,12 @@ class SceneAccess(TorrentProvider): arguments = tryUrlencode({ 'search': movie['library']['identifier'], - 'method': 1, + 'method': 3, }) url = "%s&%s" % (url, arguments) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) @@ -78,11 +78,11 @@ class SceneAccess(TorrentProvider): log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'submit': 'come on in', - }) + } def getMoreInfo(self, item): full_description = self.getCache('sceneaccess.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) diff --git a/couchpotato/core/providers/torrent/thepiratebay/__init__.py b/couchpotato/core/providers/torrent/thepiratebay/__init__.py index 8cf9f86c..8b3921cd 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/__init__.py +++ b/couchpotato/core/providers/torrent/thepiratebay/__init__.py @@ -1,4 +1,5 @@ -from main import ThePirateBay +from .main import ThePirateBay + def start(): return ThePirateBay() diff --git a/couchpotato/core/providers/torrent/thepiratebay/main.py b/couchpotato/core/providers/torrent/thepiratebay/main.py index b967d5f0..6ef5123a 100644 --- a/couchpotato/core/providers/torrent/thepiratebay/main.py +++ b/couchpotato/core/providers/torrent/thepiratebay/main.py @@ -5,6 +5,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentMagnetProvider import re import traceback +import six log = CPLog(__name__) @@ -12,15 +13,15 @@ log = CPLog(__name__) class ThePirateBay(TorrentMagnetProvider): urls = { - 'detail': '%s/torrent/%s', - 'search': '%s/search/%s/%s/7/%s' + 'detail': '%s/torrent/%s', + 'search': '%s/search/%s/%s/7/%s' } cat_ids = [ - ([207], ['720p', '1080p']), - ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), - ([201, 207], ['brrip']), - ([202], ['dvdr']) + ([207], ['720p', '1080p']), + ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']), + ([201, 207], ['brrip']), + ([202], ['dvdr']) ] cat_backup_id = 200 @@ -30,15 +31,13 @@ class ThePirateBay(TorrentMagnetProvider): proxy_list = [ 'https://tpb.ipredator.se', 'https://thepiratebay.se', - 'https://depiraatbaai.be', - 'https://piratereverse.info', - 'https://tpb.pirateparty.org.uk', - 'https://argumentomteemigreren.nl', - 'https://livepirate.com', + 'http://pirateproxy.ca', + 'http://tpb.al', + 'http://www.tpb.gr', + 'http://nl.tpb.li', + 'http://proxybay.eu', 'https://www.getpirate.com', - 'https://tpb.partipirate.org', - 'https://tpb.piraten.lu', - 'https://kuiken.co', + 'http://piratebay.io', ] def _searchOnTitle(self, title, movie, quality, results): @@ -73,7 +72,7 @@ class ThePirateBay(TorrentMagnetProvider): download = result.find(href = re.compile('magnet:')) try: - size = re.search('Size (?P.+),', unicode(result.select('font.detDesc')[0])).group('size') + size = re.search('Size (?P.+),', six.text_type(result.select('font.detDesc')[0])).group('size') except: continue @@ -111,7 +110,7 @@ class ThePirateBay(TorrentMagnetProvider): def getMoreInfo(self, item): full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) html = BeautifulSoup(full_description) - nfo_pre = html.find('div', attrs = {'class':'nfo'}) + nfo_pre = html.find('div', attrs = {'class': 'nfo'}) description = toUnicode(nfo_pre.text) if nfo_pre else '' item['description'] = description diff --git a/couchpotato/core/providers/torrent/torrentbytes/__init__.py b/couchpotato/core/providers/torrent/torrentbytes/__init__.py index 712eac85..79dec932 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/__init__.py +++ b/couchpotato/core/providers/torrent/torrentbytes/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentBytes + def start(): return TorrentBytes() diff --git a/couchpotato/core/providers/torrent/torrentbytes/main.py b/couchpotato/core/providers/torrent/torrentbytes/main.py index a5849a91..603da6e0 100644 --- a/couchpotato/core/providers/torrent/torrentbytes/main.py +++ b/couchpotato/core/providers/torrent/torrentbytes/main.py @@ -11,21 +11,21 @@ log = CPLog(__name__) class TorrentBytes(TorrentProvider): urls = { - 'test' : 'https://www.torrentbytes.net/', - 'login' : 'https://www.torrentbytes.net/takelogin.php', - 'login_check' : 'https://www.torrentbytes.net/inbox.php', - 'detail' : 'https://www.torrentbytes.net/details.php?id=%s', - 'search' : 'https://www.torrentbytes.net/browse.php?search=%s&cat=%d', - 'download' : 'https://www.torrentbytes.net/download.php?id=%s&name=%s', + 'test': 'https://www.torrentbytes.net/', + 'login': 'https://www.torrentbytes.net/takelogin.php', + 'login_check': 'https://www.torrentbytes.net/inbox.php', + 'detail': 'https://www.torrentbytes.net/details.php?id=%s', + 'search': 'https://www.torrentbytes.net/browse.php?search=%s&cat=%d', + 'download': 'https://www.torrentbytes.net/download.php?id=%s&name=%s', } cat_ids = [ - ([5], ['720p', '1080p']), + ([5], ['720p', '1080p', 'bd50']), ([19], ['cam']), ([19], ['ts', 'tc']), ([19], ['r5', 'scr']), ([19], ['dvdrip']), - ([5], ['brrip']), + ([19], ['brrip']), ([20], ['dvdr']), ] @@ -35,7 +35,7 @@ class TorrentBytes(TorrentProvider): def _searchOnTitle(self, title, movie, quality, results): url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0]) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) @@ -69,11 +69,11 @@ class TorrentBytes(TorrentProvider): log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'login': 'submit', - }) + } def loginSuccess(self, output): return 'logout.php' in output.lower() or 'Welcome' in output.lower() diff --git a/couchpotato/core/providers/torrent/torrentday/__init__.py b/couchpotato/core/providers/torrent/torrentday/__init__.py index d98bb917..133ec914 100644 --- a/couchpotato/core/providers/torrent/torrentday/__init__.py +++ b/couchpotato/core/providers/torrent/torrentday/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentDay + def start(): return TorrentDay() diff --git a/couchpotato/core/providers/torrent/torrentday/main.py b/couchpotato/core/providers/torrent/torrentday/main.py index 71812bbf..6d343234 100644 --- a/couchpotato/core/providers/torrent/torrentday/main.py +++ b/couchpotato/core/providers/torrent/torrentday/main.py @@ -1,4 +1,3 @@ -from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider @@ -24,13 +23,13 @@ class TorrentDay(TorrentProvider): ([5], ['bd50']), ] - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds def _searchOnTitle(self, title, movie, quality, results): q = '"%s %s"' % (title, movie['library']['year']) - params = { + data = { '/browse.php?': None, 'cata': 'yes', 'jxt': 8, @@ -38,7 +37,7 @@ class TorrentDay(TorrentProvider): 'search': q, } - data = self.getJsonData(self.urls['search'], params = params, opener = self.login_opener) + data = self.getJsonData(self.urls['search'], data = data) try: torrents = data.get('Fs', [])[0].get('Cn', {}).get('torrents', []) except: return @@ -54,11 +53,13 @@ class TorrentDay(TorrentProvider): }) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), + 'submit.x': 18, + 'submit.y': 11, 'submit': 'submit', - }) + } def loginSuccess(self, output): return 'Password not correct' not in output diff --git a/couchpotato/core/providers/torrent/torrentleech/__init__.py b/couchpotato/core/providers/torrent/torrentleech/__init__.py index c788477f..e64d4baa 100644 --- a/couchpotato/core/providers/torrent/torrentleech/__init__.py +++ b/couchpotato/core/providers/torrent/torrentleech/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentLeech + def start(): return TorrentLeech() diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 93b10ee3..ea6158df 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -12,12 +12,12 @@ log = CPLog(__name__) class TorrentLeech(TorrentProvider): urls = { - 'test' : 'http://www.torrentleech.org/', - 'login' : 'http://www.torrentleech.org/user/account/login/', + 'test': 'http://www.torrentleech.org/', + 'login': 'http://www.torrentleech.org/user/account/login/', 'login_check': 'http://torrentleech.org/user/messages', - 'detail' : 'http://www.torrentleech.org/torrent/%s', - 'search' : 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', - 'download' : 'http://www.torrentleech.org%s', + 'detail': 'http://www.torrentleech.org/torrent/%s', + 'search': 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d', + 'download': 'http://www.torrentleech.org%s', } cat_ids = [ @@ -36,7 +36,7 @@ class TorrentLeech(TorrentProvider): def _searchOnTitle(self, title, movie, quality, results): url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0]) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) @@ -68,12 +68,12 @@ class TorrentLeech(TorrentProvider): log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'remember_me': 'on', 'login': 'submit', - }) + } def loginSuccess(self, output): return '/user/account/logout' in output.lower() or 'welcome back' in output.lower() diff --git a/couchpotato/core/providers/torrent/torrentpotato/__init__.py b/couchpotato/core/providers/torrent/torrentpotato/__init__.py new file mode 100644 index 00000000..03795ffb --- /dev/null +++ b/couchpotato/core/providers/torrent/torrentpotato/__init__.py @@ -0,0 +1,67 @@ +from .main import TorrentPotato + + +def start(): + return TorrentPotato() + +config = [{ + 'name': 'torrentpotato', + 'groups': [ + { + 'tab': 'searcher', + 'list': 'torrent_providers', + 'name': 'TorrentPotato', + 'order': 10, + 'description': 'CouchPotato torrent provider. Checkout the wiki page about this provider for more info.', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': False, + }, + { + 'name': 'use', + 'default': '' + }, + { + 'name': 'host', + 'default': '', + 'description': 'The url path of your TorrentPotato provider.', + }, + { + 'name': 'extra_score', + 'advanced': True, + 'label': 'Extra Score', + 'default': '0', + 'description': 'Starting score for each release found via this provider.', + }, + { + 'name': 'name', + 'label': 'Username', + 'default': '', + }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'default': '1', + 'description': 'Will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'default': '40', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', + }, + { + 'name': 'pass_key', + 'default': ',', + 'label': 'Pass Key', + 'description': 'Can be found on your profile page', + 'type': 'combined', + 'combine': ['use', 'host', 'pass_key', 'name', 'seed_ratio', 'seed_time', 'extra_score'], + }, + ], + }, + ], +}] diff --git a/couchpotato/core/providers/torrent/torrentpotato/main.py b/couchpotato/core/providers/torrent/torrentpotato/main.py new file mode 100644 index 00000000..eaaf8d2c --- /dev/null +++ b/couchpotato/core/providers/torrent/torrentpotato/main.py @@ -0,0 +1,129 @@ +from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode +from couchpotato.core.helpers.variable import splitString, tryInt, tryFloat +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList +from couchpotato.core.providers.torrent.base import TorrentProvider +from urlparse import urlparse +import re +import traceback + +log = CPLog(__name__) + + +class TorrentPotato(TorrentProvider): + + urls = {} + limits_reached = {} + + http_time_between_calls = 1 # Seconds + + def search(self, movie, quality): + hosts = self.getHosts() + + results = ResultList(self, movie, quality, imdb_results = True) + + for host in hosts: + if self.isDisabled(host): + continue + + self._searchOnHost(host, movie, quality, results) + + return results + + def _searchOnHost(self, host, movie, quality, results): + + arguments = tryUrlencode({ + 'user': host['name'], + 'passkey': host['pass_key'], + 'imdbid': movie['library']['identifier'] + }) + url = '%s?%s' % (host['host'], arguments) + + torrents = self.getJsonData(url, cache_timeout = 1800) + + if torrents: + try: + if torrents.get('error'): + log.error('%s: %s', (torrents.get('error'), host['host'])) + elif torrents.get('results'): + for torrent in torrents.get('results', []): + results.append({ + 'id': torrent.get('torrent_id'), + 'protocol': 'torrent' if re.match('^(http|https|ftp)://.*$', torrent.get('download_url')) else 'torrent_magnet', + 'provider_extra': urlparse(host['host']).hostname or host['host'], + 'name': toUnicode(torrent.get('release_name')), + 'url': torrent.get('download_url'), + 'detail_url': torrent.get('details_url'), + 'size': torrent.get('size'), + 'score': host['extra_score'], + 'seeders': torrent.get('seeders'), + 'leechers': torrent.get('leechers'), + 'seed_ratio': host['seed_ratio'], + 'seed_time': host['seed_time'], + }) + + except: + log.error('Failed getting results from %s: %s', (host['host'], traceback.format_exc())) + + def getHosts(self): + + uses = splitString(str(self.conf('use')), clean = False) + hosts = splitString(self.conf('host'), clean = False) + names = splitString(self.conf('name'), clean = False) + seed_times = splitString(self.conf('seed_time'), clean = False) + seed_ratios = splitString(self.conf('seed_ratio'), clean = False) + pass_keys = splitString(self.conf('pass_key'), clean = False) + extra_score = splitString(self.conf('extra_score'), clean = False) + + list = [] + for nr in range(len(hosts)): + + try: key = pass_keys[nr] + except: key = '' + + try: host = hosts[nr] + except: host = '' + + try: name = names[nr] + except: name = '' + + try: ratio = seed_ratios[nr] + except: ratio = '' + + try: seed_time = seed_times[nr] + except: seed_time = '' + + list.append({ + 'use': uses[nr], + 'host': host, + 'name': name, + 'seed_ratio': tryFloat(ratio), + 'seed_time': tryInt(seed_time), + 'pass_key': key, + 'extra_score': tryInt(extra_score[nr]) if len(extra_score) > nr else 0 + }) + + return list + + def belongsTo(self, url, provider = None, host = None): + + hosts = self.getHosts() + + for host in hosts: + result = super(TorrentPotato, self).belongsTo(url, host = host['host'], provider = provider) + if result: + return result + + def isDisabled(self, host = None): + return not self.isEnabled(host) + + def isEnabled(self, host = None): + + # Return true if at least one is enabled and no host is given + if host is None: + for host in self.getHosts(): + if self.isEnabled(host): + return True + return False + + return TorrentProvider.isEnabled(self) and host['host'] and host['pass_key'] and int(host['use']) diff --git a/couchpotato/core/providers/torrent/torrentshack/__init__.py b/couchpotato/core/providers/torrent/torrentshack/__init__.py index 4171fc49..058236e4 100644 --- a/couchpotato/core/providers/torrent/torrentshack/__init__.py +++ b/couchpotato/core/providers/torrent/torrentshack/__init__.py @@ -1,5 +1,6 @@ from .main import TorrentShack + def start(): return TorrentShack() @@ -10,7 +11,7 @@ config = [{ 'tab': 'searcher', 'list': 'torrent_providers', 'name': 'TorrentShack', - 'description': 'See TorrentShack', + 'description': 'See TorrentShack', 'options': [ { 'name': 'enabled', diff --git a/couchpotato/core/providers/torrent/torrentshack/main.py b/couchpotato/core/providers/torrent/torrentshack/main.py index 6b3b5548..f0cd5997 100644 --- a/couchpotato/core/providers/torrent/torrentshack/main.py +++ b/couchpotato/core/providers/torrent/torrentshack/main.py @@ -4,6 +4,7 @@ from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.torrent.base import TorrentProvider import traceback +import six log = CPLog(__name__) @@ -11,12 +12,12 @@ log = CPLog(__name__) class TorrentShack(TorrentProvider): urls = { - 'test' : 'https://torrentshack.net/', - 'login' : 'https://torrentshack.net/login.php', + 'test': 'https://torrentshack.net/', + 'login': 'https://torrentshack.net/login.php', 'login_check': 'https://torrentshack.net/inbox.php', - 'detail' : 'https://torrentshack.net/torrent/%s', - 'search' : 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', - 'download' : 'https://torrentshack.net/%s', + 'detail': 'https://torrentshack.net/torrent/%s', + 'search': 'https://torrentshack.net/torrents.php?action=advanced&searchstr=%s&scene=%s&filter_cat[%d]=1', + 'download': 'https://torrentshack.net/%s', } cat_ids = [ @@ -34,7 +35,7 @@ class TorrentShack(TorrentProvider): scene_only = '1' if self.conf('scene_only') else '' url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), scene_only, self.getCatId(quality['identifier'])[0]) - data = self.getHTMLData(url, opener = self.login_opener) + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) @@ -53,7 +54,7 @@ class TorrentShack(TorrentProvider): results.append({ 'id': link['href'].replace('torrents.php?torrentid=', ''), - 'name': unicode(link.span.string).translate({ord(u'\xad'): None}), + 'name': six.text_type(link.span.string).translate({ord(six.u('\xad')): None}), 'url': self.urls['download'] % url['href'], 'detail_url': self.urls['download'] % link['href'], 'size': self.parseSize(result.find_all('td')[4].string), @@ -65,12 +66,12 @@ class TorrentShack(TorrentProvider): log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) def getLoginParams(self): - return tryUrlencode({ + return { 'username': self.conf('username'), 'password': self.conf('password'), 'keeplogged': '1', 'login': 'Login', - }) + } def loginSuccess(self, output): return 'logout.php' in output.lower() diff --git a/couchpotato/core/providers/torrent/yify/__init__.py b/couchpotato/core/providers/torrent/yify/__init__.py index 775ecdbe..3a359608 100644 --- a/couchpotato/core/providers/torrent/yify/__init__.py +++ b/couchpotato/core/providers/torrent/yify/__init__.py @@ -1,4 +1,5 @@ -from main import Yify +from .main import Yify + def start(): return Yify() @@ -18,6 +19,12 @@ config = [{ 'type': 'enabler', 'default': 0 }, + { + 'name': 'domain', + 'advanced': True, + 'label': 'Proxy server', + 'description': 'Domain for requests, keep empty to let CouchPotato pick.', + }, { 'name': 'seed_ratio', 'label': 'Seed ratio', diff --git a/couchpotato/core/providers/torrent/yify/main.py b/couchpotato/core/providers/torrent/yify/main.py index 60b2f9b1..fe1b8204 100644 --- a/couchpotato/core/providers/torrent/yify/main.py +++ b/couchpotato/core/providers/torrent/yify/main.py @@ -1,20 +1,28 @@ from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog -from couchpotato.core.providers.torrent.base import TorrentProvider +from couchpotato.core.providers.torrent.base import TorrentMagnetProvider import traceback log = CPLog(__name__) -class Yify(TorrentProvider): +class Yify(TorrentMagnetProvider): urls = { - 'test' : 'https://yify-torrents.com/api', - 'search' : 'https://yify-torrents.com/api/list.json?keywords=%s&quality=%s', - 'detail': 'https://yify-torrents.com/api/movie.json?id=%s' + 'test': '%s/api', + 'search': '%s/api/list.json?keywords=%s&quality=%s', + 'detail': '%s/api/movie.json?id=%s' } - http_time_between_calls = 1 #seconds + http_time_between_calls = 1 #seconds + + proxy_list = [ + 'http://yify.unlocktorrent.com', + 'http://yify-torrents.com.come.in', + 'http://yts.re', + 'http://yts.im' + 'https://yify-torrents.im', + ] def search(self, movie, quality): @@ -25,7 +33,9 @@ class Yify(TorrentProvider): def _search(self, movie, quality, results): - data = self.getJsonData(self.urls['search'] % (movie['library']['identifier'], quality['identifier'])) + search_url = self.urls['search'] % (self.getDomain(), movie['library']['identifier'], quality['identifier']) + + data = self.getJsonData(search_url) if data and data.get('MovieList'): try: @@ -41,8 +51,8 @@ class Yify(TorrentProvider): results.append({ 'id': result['MovieID'], 'name': title, - 'url': result['TorrentUrl'], - 'detail_url': self.urls['detail'] % result['MovieID'], + 'url': result['TorrentMagnetUrl'], + 'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']), 'size': self.parseSize(result['Size']), 'seeders': tryInt(result['TorrentSeeds']), 'leechers': tryInt(result['TorrentPeers']) @@ -51,3 +61,6 @@ class Yify(TorrentProvider): except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc())) + def correctProxy(self, data): + data = data.lower() + return 'yify' in data and 'yts' in data diff --git a/couchpotato/core/providers/trailer/hdtrailers/__init__.py b/couchpotato/core/providers/trailer/hdtrailers/__init__.py index 016db7a2..83b93004 100644 --- a/couchpotato/core/providers/trailer/hdtrailers/__init__.py +++ b/couchpotato/core/providers/trailer/hdtrailers/__init__.py @@ -1,5 +1,6 @@ from .main import HDTrailers + def start(): return HDTrailers() diff --git a/couchpotato/core/providers/trailer/hdtrailers/main.py b/couchpotato/core/providers/trailer/hdtrailers/main.py index abb91658..cba7609f 100644 --- a/couchpotato/core/providers/trailer/hdtrailers/main.py +++ b/couchpotato/core/providers/trailer/hdtrailers/main.py @@ -29,7 +29,7 @@ class HDTrailers(TrailerProvider): log.debug('No page found for: %s', movie_name) data = None - result_data = {'480p':[], '720p':[], '1080p':[]} + result_data = {'480p': [], '720p': [], '1080p': []} if not data: return result_data @@ -100,7 +100,7 @@ class HDTrailers(TrailerProvider): continue resolutions = tr.find_all('td', attrs = {'class':'bottomTableResolution'}) for res in resolutions: - if res.a: + if res.a and str(res.a.contents[0]) in results: results[str(res.a.contents[0])].insert(0, res.a['href']) except AttributeError: diff --git a/couchpotato/core/providers/userscript/allocine/__init__.py b/couchpotato/core/providers/userscript/allocine/__init__.py index e451996f..cb2ba992 100644 --- a/couchpotato/core/providers/userscript/allocine/__init__.py +++ b/couchpotato/core/providers/userscript/allocine/__init__.py @@ -1,5 +1,6 @@ from .main import AlloCine + def start(): return AlloCine() diff --git a/couchpotato/core/providers/userscript/appletrailers/__init__.py b/couchpotato/core/providers/userscript/appletrailers/__init__.py index e8078f47..075217a8 100644 --- a/couchpotato/core/providers/userscript/appletrailers/__init__.py +++ b/couchpotato/core/providers/userscript/appletrailers/__init__.py @@ -1,5 +1,6 @@ from .main import AppleTrailers + def start(): return AppleTrailers() diff --git a/couchpotato/core/providers/userscript/base.py b/couchpotato/core/providers/userscript/base.py index 571b76c0..531510b0 100644 --- a/couchpotato/core/providers/userscript/base.py +++ b/couchpotato/core/providers/userscript/base.py @@ -25,7 +25,7 @@ class UserscriptBase(Plugin): result = fireEvent('movie.search', q = '%s %s' % (name, year), limit = 1, merge = True) if len(result) > 0: - movie = fireEvent('movie.info', identifier = result[0].get('imdb'), merge = True) + movie = fireEvent('movie.info', identifier = result[0].get('imdb'), extended = False, merge = True) return movie else: return None @@ -54,7 +54,7 @@ class UserscriptBase(Plugin): return self.getInfo(getImdb(data)) def getInfo(self, identifier): - return fireEvent('movie.info', identifier = identifier, merge = True) + return fireEvent('movie.info', identifier = identifier, extended = False, merge = True) def getInclude(self): return self.includes diff --git a/couchpotato/core/providers/userscript/criticker/__init__.py b/couchpotato/core/providers/userscript/criticker/__init__.py index 129d878f..ae24aa1e 100644 --- a/couchpotato/core/providers/userscript/criticker/__init__.py +++ b/couchpotato/core/providers/userscript/criticker/__init__.py @@ -1,5 +1,6 @@ from .main import Criticker + def start(): return Criticker() diff --git a/couchpotato/core/providers/userscript/filmweb/__init__.py b/couchpotato/core/providers/userscript/filmweb/__init__.py index 8ead54d6..3098610c 100644 --- a/couchpotato/core/providers/userscript/filmweb/__init__.py +++ b/couchpotato/core/providers/userscript/filmweb/__init__.py @@ -1,5 +1,6 @@ from .main import Filmweb + def start(): return Filmweb() diff --git a/couchpotato/core/providers/userscript/flickchart/__init__.py b/couchpotato/core/providers/userscript/flickchart/__init__.py index 89d45d9c..18a88ffe 100644 --- a/couchpotato/core/providers/userscript/flickchart/__init__.py +++ b/couchpotato/core/providers/userscript/flickchart/__init__.py @@ -1,5 +1,6 @@ from .main import Flickchart + def start(): return Flickchart() diff --git a/couchpotato/core/providers/userscript/imdb/__init__.py b/couchpotato/core/providers/userscript/imdb/__init__.py index f10505da..c25319b7 100644 --- a/couchpotato/core/providers/userscript/imdb/__init__.py +++ b/couchpotato/core/providers/userscript/imdb/__init__.py @@ -1,5 +1,6 @@ from .main import IMDB + def start(): return IMDB() diff --git a/couchpotato/core/providers/userscript/imdb/main.py b/couchpotato/core/providers/userscript/imdb/main.py index 24278b1a..2a6efd6b 100644 --- a/couchpotato/core/providers/userscript/imdb/main.py +++ b/couchpotato/core/providers/userscript/imdb/main.py @@ -8,4 +8,4 @@ class IMDB(UserscriptBase): includes = ['*://*.imdb.com/title/tt*', '*://imdb.com/title/tt*'] def getMovie(self, url): - return fireEvent('movie.info', identifier = getImdb(url), merge = True) + return self.getInfo(getImdb(url)) diff --git a/couchpotato/core/providers/userscript/letterboxd/__init__.py b/couchpotato/core/providers/userscript/letterboxd/__init__.py index c8c17977..2fd89000 100644 --- a/couchpotato/core/providers/userscript/letterboxd/__init__.py +++ b/couchpotato/core/providers/userscript/letterboxd/__init__.py @@ -1,5 +1,6 @@ from .main import Letterboxd + def start(): return Letterboxd() diff --git a/couchpotato/core/providers/userscript/moviemeter/__init__.py b/couchpotato/core/providers/userscript/moviemeter/__init__.py index 5e3813c4..7a05a75a 100644 --- a/couchpotato/core/providers/userscript/moviemeter/__init__.py +++ b/couchpotato/core/providers/userscript/moviemeter/__init__.py @@ -1,5 +1,6 @@ from .main import MovieMeter + def start(): return MovieMeter() diff --git a/couchpotato/core/providers/userscript/moviesio/__init__.py b/couchpotato/core/providers/userscript/moviesio/__init__.py index 473f847d..e29e8d08 100644 --- a/couchpotato/core/providers/userscript/moviesio/__init__.py +++ b/couchpotato/core/providers/userscript/moviesio/__init__.py @@ -1,5 +1,6 @@ from .main import MoviesIO + def start(): return MoviesIO() diff --git a/couchpotato/core/providers/userscript/reddit/__init__.py b/couchpotato/core/providers/userscript/reddit/__init__.py new file mode 100644 index 00000000..a74bbf0d --- /dev/null +++ b/couchpotato/core/providers/userscript/reddit/__init__.py @@ -0,0 +1,7 @@ +from .main import Reddit + + +def start(): + return Reddit() + +config = [] diff --git a/couchpotato/core/providers/userscript/reddit/main.py b/couchpotato/core/providers/userscript/reddit/main.py new file mode 100644 index 00000000..9790f6e2 --- /dev/null +++ b/couchpotato/core/providers/userscript/reddit/main.py @@ -0,0 +1,17 @@ +from couchpotato import fireEvent +from couchpotato.core.helpers.variable import splitString +from couchpotato.core.providers.userscript.base import UserscriptBase + + +class Reddit(UserscriptBase): + + includes = ['*://www.reddit.com/r/Ijustwatched/comments/*'] + + def getMovie(self, url): + name = splitString(url, '/')[-1] + if name.startswith('ijw_'): + name = name[4:] + + year_name = fireEvent('scanner.name_year', name, single = True) + + return self.search(year_name.get('name'), year_name.get('year')) diff --git a/couchpotato/core/providers/userscript/rottentomatoes/__init__.py b/couchpotato/core/providers/userscript/rottentomatoes/__init__.py index ee8266eb..363f103e 100644 --- a/couchpotato/core/providers/userscript/rottentomatoes/__init__.py +++ b/couchpotato/core/providers/userscript/rottentomatoes/__init__.py @@ -1,5 +1,6 @@ from .main import RottenTomatoes + def start(): return RottenTomatoes() diff --git a/couchpotato/core/providers/userscript/sharethe/__init__.py b/couchpotato/core/providers/userscript/sharethe/__init__.py index 7661f761..3cf393af 100644 --- a/couchpotato/core/providers/userscript/sharethe/__init__.py +++ b/couchpotato/core/providers/userscript/sharethe/__init__.py @@ -1,5 +1,6 @@ from .main import ShareThe + def start(): return ShareThe() diff --git a/couchpotato/core/providers/userscript/tmdb/__init__.py b/couchpotato/core/providers/userscript/tmdb/__init__.py index be33372c..c77330c3 100644 --- a/couchpotato/core/providers/userscript/tmdb/__init__.py +++ b/couchpotato/core/providers/userscript/tmdb/__init__.py @@ -1,5 +1,6 @@ from .main import TMDB + def start(): return TMDB() diff --git a/couchpotato/core/providers/userscript/tmdb/main.py b/couchpotato/core/providers/userscript/tmdb/main.py index cab38fc6..b718fc3b 100644 --- a/couchpotato/core/providers/userscript/tmdb/main.py +++ b/couchpotato/core/providers/userscript/tmdb/main.py @@ -9,7 +9,7 @@ class TMDB(UserscriptBase): def getMovie(self, url): match = re.search('(?P\d+)', url) - movie = fireEvent('movie.info_by_tmdb', identifier = match.group('id'), merge = True) + movie = fireEvent('movie.info_by_tmdb', identifier = match.group('id'), extended = False, merge = True) if movie['imdb']: return self.getInfo(movie['imdb']) diff --git a/couchpotato/core/providers/userscript/trakt/__init__.py b/couchpotato/core/providers/userscript/trakt/__init__.py index ff67c1ec..39c17c32 100644 --- a/couchpotato/core/providers/userscript/trakt/__init__.py +++ b/couchpotato/core/providers/userscript/trakt/__init__.py @@ -1,5 +1,6 @@ from .main import Trakt + def start(): return Trakt() diff --git a/couchpotato/core/providers/userscript/whiwa/__init__.py b/couchpotato/core/providers/userscript/whiwa/__init__.py index 6577ae33..c8fd3c9d 100644 --- a/couchpotato/core/providers/userscript/whiwa/__init__.py +++ b/couchpotato/core/providers/userscript/whiwa/__init__.py @@ -1,5 +1,6 @@ from .main import WHiWA + def start(): return WHiWA() diff --git a/couchpotato/core/providers/userscript/youteather/__init__.py b/couchpotato/core/providers/userscript/youteather/__init__.py index a07bf56b..f31e911e 100644 --- a/couchpotato/core/providers/userscript/youteather/__init__.py +++ b/couchpotato/core/providers/userscript/youteather/__init__.py @@ -1,5 +1,6 @@ from .main import YouTheater + def start(): return YouTheater() diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index 61d982f2..0e65c778 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -1,4 +1,5 @@ from __future__ import with_statement +import traceback from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode @@ -77,7 +78,7 @@ class Settings(object): self.addSection(section_name) - for option_name, option in options.iteritems(): + for option_name, option in options.items(): self.setDefault(section_name, option_name, option.get('default', '')) # Migrate old settings from old location to the new location @@ -110,6 +111,10 @@ class Settings(object): except: return default + def delete(self, option = '', section = 'core'): + self.p.remove_option(section, option) + self.save() + def getEnabler(self, section, option): return self.getBool(section, option) @@ -195,6 +200,7 @@ class Settings(object): # After save (for re-interval etc) fireEvent('setting.save.%s.%s.after' % (section, option), single = True) + fireEvent('setting.save.%s.*.after' % section, single = True) return { 'success': True, @@ -216,14 +222,20 @@ class Settings(object): def setProperty(self, identifier, value = ''): from couchpotato import get_session - db = get_session() + try: + db = get_session() - p = db.query(Properties).filter_by(identifier = identifier).first() - if not p: - p = Properties() - db.add(p) + p = db.query(Properties).filter_by(identifier = identifier).first() + if not p: + p = Properties() + db.add(p) - p.identifier = identifier - p.value = toUnicode(value) + p.identifier = identifier + p.value = toUnicode(value) - db.commit() + db.commit() + except: + self.log.error('Failed: %s', traceback.format_exc()) + db.rollback() + finally: + db.close() diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index 8601c2b4..ef6e8a5c 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -18,6 +18,7 @@ options_defaults["shortnames"] = True # http://elixir.ematia.de/trac/wiki/Recipes/MultipleDatabasesOneMetadata __session__ = None + class SetEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, set): @@ -40,6 +41,7 @@ class JsonType(TypeDecorator): def process_result_value(self, value, dialect): return json.loads(value if value else '{}') + class MutableDict(Mutable, dict): @classmethod @@ -78,7 +80,7 @@ class Movie(Entity): such as trailers, nfo, thumbnails""" last_edit = Field(Integer, default = lambda: int(time.time()), index = True) - type = 'movie' # Compat tv branch + type = 'movie' # Compat tv branch library = ManyToOne('Library', cascade = 'delete, delete-orphan', single_parent = True) status = ManyToOne('Status') @@ -87,7 +89,8 @@ class Movie(Entity): releases = OneToMany('Release', cascade = 'all, delete-orphan') files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) -Media = Movie # Compat tv branch +Media = Movie # Compat tv branch + class Library(Entity): """""" @@ -215,6 +218,7 @@ class Profile(Entity): return orig_dict + class Category(Entity): """""" using_options(order_by = 'order') diff --git a/couchpotato/environment.py b/couchpotato/environment.py index 0f04d838..1c5863d1 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -2,10 +2,10 @@ from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.loader import Loader from couchpotato.core.settings import Settings from sqlalchemy.engine import create_engine -from sqlalchemy.orm import scoped_session from sqlalchemy.orm.session import sessionmaker import os + class Env(object): _appname = 'CouchPotato' @@ -23,7 +23,7 @@ class Env(object): _quiet = False _daemonized = False _desktop = None - _session = None + _engine = None ''' Data paths and directories ''' _app_dir = "" @@ -53,20 +53,20 @@ class Env(object): return setattr(Env, '_' + attr, value) @staticmethod - def getSession(engine = None): - existing_session = Env.get('session') - if existing_session: - return existing_session - - engine = Env.getEngine() - session = scoped_session(sessionmaker(bind = engine)) - Env.set('session', session) - - return session + def getSession(): + session = sessionmaker(bind = Env.getEngine()) + return session() @staticmethod def getEngine(): - return create_engine(Env.get('db_path'), echo = False, pool_recycle = 30) + existing_engine = Env.get('engine') + if existing_engine: + return existing_engine + + engine = create_engine(Env.get('db_path'), echo = False) + Env.set('engine', engine) + + return engine @staticmethod def setting(attr, section = 'core', value = None, default = '', type = None): @@ -78,6 +78,7 @@ class Env(object): return s.get(attr, default = default, section = section, type = type) # Set setting + s.addSection(section) s.set(section, attr, value) s.save() diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 571023ea..5c175201 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -8,6 +8,7 @@ from couchpotato.core.helpers.variable import getDataDir, tryInt from logging import handlers from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler +from uuid import uuid4 import locale import logging import os.path @@ -17,6 +18,7 @@ import time import traceback import warnings + def getOptions(base_path, args): # Options @@ -51,6 +53,7 @@ def getOptions(base_path, args): return options + # Tornado monkey patch logging.. def _log(status_code, request): @@ -120,7 +123,6 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En os.rmdir(backup) total_backups -= 1 - # Register environment settings Env.set('app_dir', toUnicode(base_path)) Env.set('data_dir', toUnicode(data_dir)) @@ -144,7 +146,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En Env.set('dev', development) # Disable logging for some modules - for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: + for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: @@ -167,7 +169,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En logger.addHandler(hdlr) # To file - hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) + hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding')) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) @@ -215,6 +217,10 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En Env.set('web_base', web_base) api_key = Env.setting('api_key') + if not api_key: + api_key = uuid4().hex + Env.setting('api_key', value = api_key) + api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) @@ -229,10 +235,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En 'ssl_key': Env.setting('ssl_key', default = None), } - # Load the app application = Application([], - log_function = lambda x : None, + log_function = lambda x: None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, @@ -245,9 +250,9 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers - (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler - (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key - (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs + (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler + (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key + (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), @@ -262,37 +267,32 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ - ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) + ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) - # Load configs & plugins loader = Env.get('loader') loader.preload(root = toUnicode(base_path)) loader.run() - # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order = True) - # Go go go! from tornado.ioloop import IOLoop loop = IOLoop.current() - # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') - if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { - "certfile": config['ssl_cert'], - "keyfile": config['ssl_key'], + 'certfile': config['ssl_cert'], + 'keyfile': config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive = True) @@ -304,7 +304,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En try: server.listen(config['port'], config['host']) loop.start() - except Exception, e: + except Exception as e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 59fac34b..03332281 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -11,6 +11,12 @@ pages: [], block: [], + initialize: function(){ + var self = this; + + self.global_events = {}; + }, + setup: function(options) { var self = this; self.setOptions(options); @@ -30,7 +36,7 @@ History.addEvent('change', self.openPage.bind(self)); self.c.addEvent('click:relay(a[href^=/]:not([target]))', self.pushState.bind(self)); self.c.addEvent('click:relay(a[href^=http])', self.openDerefered.bind(self)); - + // Check if device is touchenabled self.touch_device = 'ontouchstart' in window || navigator.msMaxTouchPoints; if(self.touch_device) @@ -55,7 +61,7 @@ History.push(url); } }, - + isMac: function(){ return Browser.Platform.mac }, @@ -111,7 +117,7 @@ } }) ]; - + setting_links.each(function(a){ self.block.more.addLink(a) }); @@ -336,6 +342,66 @@ }) ) ); + }, + + /* + * Global events + */ + on: function(name, handle){ + var self = this; + + if(!self.global_events[name]) + self.global_events[name] = []; + + self.global_events[name].push(handle); + + }, + + trigger: function(name, args, on_complete){ + var self = this; + + if(!self.global_events[name]){ return; } + + if(!on_complete && typeOf(args) == 'function'){ + on_complete = args; + args = []; + } + + // Create parallel callback + var callbacks = []; + self.global_events[name].each(function(handle, nr){ + + callbacks.push(function(callback){ + var results = handle.apply(handle, args || []); + callback(null, results || null); + }); + + }); + + // Fire events + async.parallel(callbacks, function(err, results){ + if(err) p(err); + + if(on_complete) + on_complete(results); + }); + + }, + + off: function(name, handle){ + var self = this; + + if(!self.global_events[name]) return; + + // Remove single + if(handle){ + self.global_events[name] = self.global_events[name].erase(handle); + } + // Reset full event + else { + self.global_events[name] = []; + } + } }); @@ -503,7 +569,7 @@ function randomString(length, extra) { case "string": saveKeyPath(argument.match(/[+-]|[^.]+/g)); break; } }); - return this.sort(comparer); + return this.stableSort(comparer); } }); @@ -527,4 +593,4 @@ var createSpinner = function(target, options){ }, options); return new Spinner(opts).spin(target); -}; \ No newline at end of file +}; diff --git a/couchpotato/static/scripts/library/Array.stableSort.js b/couchpotato/static/scripts/library/Array.stableSort.js new file mode 100644 index 00000000..062c7566 --- /dev/null +++ b/couchpotato/static/scripts/library/Array.stableSort.js @@ -0,0 +1,56 @@ +/* +--- + +script: Array.stableSort.js + +description: Add a stable sort algorithm for all browsers + +license: MIT-style license. + +authors: + - Yorick Sijsling + +requires: + core/1.3: '*' + +provides: + - [Array.stableSort, Array.mergeSort] + +... +*/ + +(function() { + + var defaultSortFunction = function(a, b) { + return a > b ? 1 : (a < b ? -1 : 0); + } + + Array.implement({ + + stableSort: function(compare) { + // I would love some real feature recognition. Problem is that an unstable algorithm sometimes/often gives the same result as an unstable algorithm. + return (Browser.chrome || Browser.firefox2 || Browser.opera9) ? this.mergeSort(compare) : this.sort(compare); + }, + + mergeSort: function(compare, token) { + compare = compare || defaultSortFunction; + if (this.length > 1) { + // Split and sort both parts + var right = this.splice(Math.floor(this.length / 2)).mergeSort(compare); + var left = this.splice(0).mergeSort(compare); // 'this' is now empty. + + // Merge parts together + while (left.length > 0 || right.length > 0) { + this.push( + right.length === 0 ? left.shift() + : left.length === 0 ? right.shift() + : compare(left[0], right[0]) > 0 ? right.shift() + : left.shift()); + } + } + return this; + } + + }); +})(); + diff --git a/couchpotato/static/scripts/library/async.js b/couchpotato/static/scripts/library/async.js new file mode 100644 index 00000000..cb6320d6 --- /dev/null +++ b/couchpotato/static/scripts/library/async.js @@ -0,0 +1,955 @@ +/*global setImmediate: false, setTimeout: false, console: false */ +(function () { + + var async = {}; + + // global on the server, window in the browser + var root, previous_async; + + root = this; + if (root != null) { + previous_async = root.async; + } + + async.noConflict = function () { + root.async = previous_async; + return async; + }; + + function only_once(fn) { + var called = false; + return function() { + if (called) throw new Error("Callback was already called."); + called = true; + fn.apply(root, arguments); + } + } + + //// cross-browser compatiblity functions //// + + var _each = function (arr, iterator) { + if (arr.forEach) { + return arr.forEach(iterator); + } + for (var i = 0; i < arr.length; i += 1) { + iterator(arr[i], i, arr); + } + }; + + var _map = function (arr, iterator) { + if (arr.map) { + return arr.map(iterator); + } + var results = []; + _each(arr, function (x, i, a) { + results.push(iterator(x, i, a)); + }); + return results; + }; + + var _reduce = function (arr, iterator, memo) { + if (arr.reduce) { + return arr.reduce(iterator, memo); + } + _each(arr, function (x, i, a) { + memo = iterator(memo, x, i, a); + }); + return memo; + }; + + var _keys = function (obj) { + if (Object.keys) { + return Object.keys(obj); + } + var keys = []; + for (var k in obj) { + if (obj.hasOwnProperty(k)) { + keys.push(k); + } + } + return keys; + }; + + //// exported async module functions //// + + //// nextTick implementation with browser-compatible fallback //// + if (typeof process === 'undefined' || !(process.nextTick)) { + if (typeof setImmediate === 'function') { + async.nextTick = function (fn) { + // not a direct alias for IE10 compatibility + setImmediate(fn); + }; + async.setImmediate = async.nextTick; + } + else { + async.nextTick = function (fn) { + setTimeout(fn, 0); + }; + async.setImmediate = async.nextTick; + } + } + else { + async.nextTick = process.nextTick; + if (typeof setImmediate !== 'undefined') { + async.setImmediate = setImmediate; + } + else { + async.setImmediate = async.nextTick; + } + } + + async.each = function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length) { + return callback(); + } + var completed = 0; + _each(arr, function (x) { + iterator(x, only_once(function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= arr.length) { + callback(null); + } + } + })); + }); + }; + async.forEach = async.each; + + async.eachSeries = function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length) { + return callback(); + } + var completed = 0; + var iterate = function () { + iterator(arr[completed], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= arr.length) { + callback(null); + } + else { + iterate(); + } + } + }); + }; + iterate(); + }; + async.forEachSeries = async.eachSeries; + + async.eachLimit = function (arr, limit, iterator, callback) { + var fn = _eachLimit(limit); + fn.apply(null, [arr, iterator, callback]); + }; + async.forEachLimit = async.eachLimit; + + var _eachLimit = function (limit) { + + return function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length || limit <= 0) { + return callback(); + } + var completed = 0; + var started = 0; + var running = 0; + + (function replenish () { + if (completed >= arr.length) { + return callback(); + } + + while (running < limit && started < arr.length) { + started += 1; + running += 1; + iterator(arr[started - 1], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + running -= 1; + if (completed >= arr.length) { + callback(); + } + else { + replenish(); + } + } + }); + } + })(); + }; + }; + + + var doParallel = function (fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [async.each].concat(args)); + }; + }; + var doParallelLimit = function(limit, fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [_eachLimit(limit)].concat(args)); + }; + }; + var doSeries = function (fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [async.eachSeries].concat(args)); + }; + }; + + + var _asyncMap = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (err, v) { + results[x.index] = v; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + }; + async.map = doParallel(_asyncMap); + async.mapSeries = doSeries(_asyncMap); + async.mapLimit = function (arr, limit, iterator, callback) { + return _mapLimit(limit)(arr, iterator, callback); + }; + + var _mapLimit = function(limit) { + return doParallelLimit(limit, _asyncMap); + }; + + // reduce only has a series version, as doing reduce in parallel won't + // work in many situations. + async.reduce = function (arr, memo, iterator, callback) { + async.eachSeries(arr, function (x, callback) { + iterator(memo, x, function (err, v) { + memo = v; + callback(err); + }); + }, function (err) { + callback(err, memo); + }); + }; + // inject alias + async.inject = async.reduce; + // foldl alias + async.foldl = async.reduce; + + async.reduceRight = function (arr, memo, iterator, callback) { + var reversed = _map(arr, function (x) { + return x; + }).reverse(); + async.reduce(reversed, memo, iterator, callback); + }; + // foldr alias + async.foldr = async.reduceRight; + + var _filter = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (v) { + if (v) { + results.push(x); + } + callback(); + }); + }, function (err) { + callback(_map(results.sort(function (a, b) { + return a.index - b.index; + }), function (x) { + return x.value; + })); + }); + }; + async.filter = doParallel(_filter); + async.filterSeries = doSeries(_filter); + // select alias + async.select = async.filter; + async.selectSeries = async.filterSeries; + + var _reject = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (v) { + if (!v) { + results.push(x); + } + callback(); + }); + }, function (err) { + callback(_map(results.sort(function (a, b) { + return a.index - b.index; + }), function (x) { + return x.value; + })); + }); + }; + async.reject = doParallel(_reject); + async.rejectSeries = doSeries(_reject); + + var _detect = function (eachfn, arr, iterator, main_callback) { + eachfn(arr, function (x, callback) { + iterator(x, function (result) { + if (result) { + main_callback(x); + main_callback = function () {}; + } + else { + callback(); + } + }); + }, function (err) { + main_callback(); + }); + }; + async.detect = doParallel(_detect); + async.detectSeries = doSeries(_detect); + + async.some = function (arr, iterator, main_callback) { + async.each(arr, function (x, callback) { + iterator(x, function (v) { + if (v) { + main_callback(true); + main_callback = function () {}; + } + callback(); + }); + }, function (err) { + main_callback(false); + }); + }; + // any alias + async.any = async.some; + + async.every = function (arr, iterator, main_callback) { + async.each(arr, function (x, callback) { + iterator(x, function (v) { + if (!v) { + main_callback(false); + main_callback = function () {}; + } + callback(); + }); + }, function (err) { + main_callback(true); + }); + }; + // all alias + async.all = async.every; + + async.sortBy = function (arr, iterator, callback) { + async.map(arr, function (x, callback) { + iterator(x, function (err, criteria) { + if (err) { + callback(err); + } + else { + callback(null, {value: x, criteria: criteria}); + } + }); + }, function (err, results) { + if (err) { + return callback(err); + } + else { + var fn = function (left, right) { + var a = left.criteria, b = right.criteria; + return a < b ? -1 : a > b ? 1 : 0; + }; + callback(null, _map(results.sort(fn), function (x) { + return x.value; + })); + } + }); + }; + + async.auto = function (tasks, callback) { + callback = callback || function () {}; + var keys = _keys(tasks); + if (!keys.length) { + return callback(null); + } + + var results = {}; + + var listeners = []; + var addListener = function (fn) { + listeners.unshift(fn); + }; + var removeListener = function (fn) { + for (var i = 0; i < listeners.length; i += 1) { + if (listeners[i] === fn) { + listeners.splice(i, 1); + return; + } + } + }; + var taskComplete = function () { + _each(listeners.slice(0), function (fn) { + fn(); + }); + }; + + addListener(function () { + if (_keys(results).length === keys.length) { + callback(null, results); + callback = function () {}; + } + }); + + _each(keys, function (k) { + var task = (tasks[k] instanceof Function) ? [tasks[k]]: tasks[k]; + var taskCallback = function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + if (err) { + var safeResults = {}; + _each(_keys(results), function(rkey) { + safeResults[rkey] = results[rkey]; + }); + safeResults[k] = args; + callback(err, safeResults); + // stop subsequent errors hitting callback multiple times + callback = function () {}; + } + else { + results[k] = args; + async.setImmediate(taskComplete); + } + }; + var requires = task.slice(0, Math.abs(task.length - 1)) || []; + var ready = function () { + return _reduce(requires, function (a, x) { + return (a && results.hasOwnProperty(x)); + }, true) && !results.hasOwnProperty(k); + }; + if (ready()) { + task[task.length - 1](taskCallback, results); + } + else { + var listener = function () { + if (ready()) { + removeListener(listener); + task[task.length - 1](taskCallback, results); + } + }; + addListener(listener); + } + }); + }; + + async.waterfall = function (tasks, callback) { + callback = callback || function () {}; + if (tasks.constructor !== Array) { + var err = new Error('First argument to waterfall must be an array of functions'); + return callback(err); + } + if (!tasks.length) { + return callback(); + } + var wrapIterator = function (iterator) { + return function (err) { + if (err) { + callback.apply(null, arguments); + callback = function () {}; + } + else { + var args = Array.prototype.slice.call(arguments, 1); + var next = iterator.next(); + if (next) { + args.push(wrapIterator(next)); + } + else { + args.push(callback); + } + async.setImmediate(function () { + iterator.apply(null, args); + }); + } + }; + }; + wrapIterator(async.iterator(tasks))(); + }; + + var _parallel = function(eachfn, tasks, callback) { + callback = callback || function () {}; + if (tasks.constructor === Array) { + eachfn.map(tasks, function (fn, callback) { + if (fn) { + fn(function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + callback.call(null, err, args); + }); + } + }, callback); + } + else { + var results = {}; + eachfn.each(_keys(tasks), function (k, callback) { + tasks[k](function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + results[k] = args; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + + async.parallel = function (tasks, callback) { + _parallel({ map: async.map, each: async.each }, tasks, callback); + }; + + async.parallelLimit = function(tasks, limit, callback) { + _parallel({ map: _mapLimit(limit), each: _eachLimit(limit) }, tasks, callback); + }; + + async.series = function (tasks, callback) { + callback = callback || function () {}; + if (tasks.constructor === Array) { + async.mapSeries(tasks, function (fn, callback) { + if (fn) { + fn(function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + callback.call(null, err, args); + }); + } + }, callback); + } + else { + var results = {}; + async.eachSeries(_keys(tasks), function (k, callback) { + tasks[k](function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + results[k] = args; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + + async.iterator = function (tasks) { + var makeCallback = function (index) { + var fn = function () { + if (tasks.length) { + tasks[index].apply(null, arguments); + } + return fn.next(); + }; + fn.next = function () { + return (index < tasks.length - 1) ? makeCallback(index + 1): null; + }; + return fn; + }; + return makeCallback(0); + }; + + async.apply = function (fn) { + var args = Array.prototype.slice.call(arguments, 1); + return function () { + return fn.apply( + null, args.concat(Array.prototype.slice.call(arguments)) + ); + }; + }; + + var _concat = function (eachfn, arr, fn, callback) { + var r = []; + eachfn(arr, function (x, cb) { + fn(x, function (err, y) { + r = r.concat(y || []); + cb(err); + }); + }, function (err) { + callback(err, r); + }); + }; + async.concat = doParallel(_concat); + async.concatSeries = doSeries(_concat); + + async.whilst = function (test, iterator, callback) { + if (test()) { + iterator(function (err) { + if (err) { + return callback(err); + } + async.whilst(test, iterator, callback); + }); + } + else { + callback(); + } + }; + + async.doWhilst = function (iterator, test, callback) { + iterator(function (err) { + if (err) { + return callback(err); + } + if (test()) { + async.doWhilst(iterator, test, callback); + } + else { + callback(); + } + }); + }; + + async.until = function (test, iterator, callback) { + if (!test()) { + iterator(function (err) { + if (err) { + return callback(err); + } + async.until(test, iterator, callback); + }); + } + else { + callback(); + } + }; + + async.doUntil = function (iterator, test, callback) { + iterator(function (err) { + if (err) { + return callback(err); + } + if (!test()) { + async.doUntil(iterator, test, callback); + } + else { + callback(); + } + }); + }; + + async.queue = function (worker, concurrency) { + if (concurrency === undefined) { + concurrency = 1; + } + function _insert(q, data, pos, callback) { + if(data.constructor !== Array) { + data = [data]; + } + _each(data, function(task) { + var item = { + data: task, + callback: typeof callback === 'function' ? callback : null + }; + + if (pos) { + q.tasks.unshift(item); + } else { + q.tasks.push(item); + } + + if (q.saturated && q.tasks.length === concurrency) { + q.saturated(); + } + async.setImmediate(q.process); + }); + } + + var workers = 0; + var q = { + tasks: [], + concurrency: concurrency, + saturated: null, + empty: null, + drain: null, + push: function (data, callback) { + _insert(q, data, false, callback); + }, + unshift: function (data, callback) { + _insert(q, data, true, callback); + }, + process: function () { + if (workers < q.concurrency && q.tasks.length) { + var task = q.tasks.shift(); + if (q.empty && q.tasks.length === 0) { + q.empty(); + } + workers += 1; + var next = function () { + workers -= 1; + if (task.callback) { + task.callback.apply(task, arguments); + } + if (q.drain && q.tasks.length + workers === 0) { + q.drain(); + } + q.process(); + }; + var cb = only_once(next); + worker(task.data, cb); + } + }, + length: function () { + return q.tasks.length; + }, + running: function () { + return workers; + } + }; + return q; + }; + + async.cargo = function (worker, payload) { + var working = false, + tasks = []; + + var cargo = { + tasks: tasks, + payload: payload, + saturated: null, + empty: null, + drain: null, + push: function (data, callback) { + if(data.constructor !== Array) { + data = [data]; + } + _each(data, function(task) { + tasks.push({ + data: task, + callback: typeof callback === 'function' ? callback : null + }); + if (cargo.saturated && tasks.length === payload) { + cargo.saturated(); + } + }); + async.setImmediate(cargo.process); + }, + process: function process() { + if (working) return; + if (tasks.length === 0) { + if(cargo.drain) cargo.drain(); + return; + } + + var ts = typeof payload === 'number' + ? tasks.splice(0, payload) + : tasks.splice(0); + + var ds = _map(ts, function (task) { + return task.data; + }); + + if(cargo.empty) cargo.empty(); + working = true; + worker(ds, function () { + working = false; + + var args = arguments; + _each(ts, function (data) { + if (data.callback) { + data.callback.apply(null, args); + } + }); + + process(); + }); + }, + length: function () { + return tasks.length; + }, + running: function () { + return working; + } + }; + return cargo; + }; + + var _console_fn = function (name) { + return function (fn) { + var args = Array.prototype.slice.call(arguments, 1); + fn.apply(null, args.concat([function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (typeof console !== 'undefined') { + if (err) { + if (console.error) { + console.error(err); + } + } + else if (console[name]) { + _each(args, function (x) { + console[name](x); + }); + } + } + }])); + }; + }; + async.log = _console_fn('log'); + async.dir = _console_fn('dir'); + /*async.info = _console_fn('info'); + async.warn = _console_fn('warn'); + async.error = _console_fn('error');*/ + + async.memoize = function (fn, hasher) { + var memo = {}; + var queues = {}; + hasher = hasher || function (x) { + return x; + }; + var memoized = function () { + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + var key = hasher.apply(null, args); + if (key in memo) { + callback.apply(null, memo[key]); + } + else if (key in queues) { + queues[key].push(callback); + } + else { + queues[key] = [callback]; + fn.apply(null, args.concat([function () { + memo[key] = arguments; + var q = queues[key]; + delete queues[key]; + for (var i = 0, l = q.length; i < l; i++) { + q[i].apply(null, arguments); + } + }])); + } + }; + memoized.memo = memo; + memoized.unmemoized = fn; + return memoized; + }; + + async.unmemoize = function (fn) { + return function () { + return (fn.unmemoized || fn).apply(null, arguments); + }; + }; + + async.times = function (count, iterator, callback) { + var counter = []; + for (var i = 0; i < count; i++) { + counter.push(i); + } + return async.map(counter, iterator, callback); + }; + + async.timesSeries = function (count, iterator, callback) { + var counter = []; + for (var i = 0; i < count; i++) { + counter.push(i); + } + return async.mapSeries(counter, iterator, callback); + }; + + async.compose = function (/* functions... */) { + var fns = Array.prototype.reverse.call(arguments); + return function () { + var that = this; + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + async.reduce(fns, args, function (newargs, fn, cb) { + fn.apply(that, newargs.concat([function () { + var err = arguments[0]; + var nextargs = Array.prototype.slice.call(arguments, 1); + cb(err, nextargs); + }])) + }, + function (err, results) { + callback.apply(that, [err].concat(results)); + }); + }; + }; + + var _applyEach = function (eachfn, fns /*args...*/) { + var go = function () { + var that = this; + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + return eachfn(fns, function (fn, cb) { + fn.apply(that, args.concat([cb])); + }, + callback); + }; + if (arguments.length > 2) { + var args = Array.prototype.slice.call(arguments, 2); + return go.apply(this, args); + } + else { + return go; + } + }; + async.applyEach = doParallel(_applyEach); + async.applyEachSeries = doSeries(_applyEach); + + async.forever = function (fn, callback) { + function next(err) { + if (err) { + if (callback) { + return callback(err); + } + throw err; + } + fn(next); + } + next(); + }; + + // AMD / RequireJS + if (typeof define !== 'undefined' && define.amd) { + define([], function () { + return async; + }); + } + // Node.js + else if (typeof module !== 'undefined' && module.exports) { + module.exports = async; + } + // included directly via +""" + soup = BeautifulSoup(doc, "xml") + # lxml would have stripped this while parsing, but we can add + # it later. + soup.script.string = 'console.log("< < hey > > ");' + encoded = soup.encode() + self.assertTrue(b"< < hey > >" in encoded) + + def test_can_parse_unicode_document(self): + markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!' + soup = self.soup(markup) + self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string) + + def test_popping_namespaced_tag(self): + markup = 'b2012-07-02T20:33:42Zcd' + soup = self.soup(markup) + self.assertEqual( + unicode(soup.rss), markup) def test_docstring_includes_correct_encoding(self): soup = self.soup("") @@ -472,6 +529,20 @@ class XMLTreeBuilderSmokeTest(object): self.assertEqual("http://example.com/", root['xmlns:a']) self.assertEqual("http://example.net/", root['xmlns:b']) + def test_closing_namespaced_tag(self): + markup = '

20010504

' + soup = self.soup(markup) + self.assertEqual(unicode(soup.p), markup) + + def test_namespaced_attributes(self): + markup = '' + soup = self.soup(markup) + self.assertEqual(unicode(soup.foo), markup) + + def test_namespaced_attributes_xml_namespace(self): + markup = 'bar' + soup = self.soup(markup) + self.assertEqual(unicode(soup.foo), markup) class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest): """Smoke test for a tree builder that supports HTML5.""" @@ -501,6 +572,12 @@ class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest): self.assertEqual(namespace, soup.math.namespace) self.assertEqual(namespace, soup.msqrt.namespace) + def test_xml_declaration_becomes_comment(self): + markup = '' + soup = self.soup(markup) + self.assertTrue(isinstance(soup.contents[0], Comment)) + self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?') + self.assertEqual("html", soup.contents[0].next_element.name) def skipIf(condition, reason): def nothing(test, *args, **kwargs): diff --git a/libs/gntp/__init__.py b/libs/gntp/__init__.py index eabbfa47..e69de29b 100755 --- a/libs/gntp/__init__.py +++ b/libs/gntp/__init__.py @@ -1,509 +0,0 @@ -import re -import hashlib -import time -import StringIO - -__version__ = '0.8' - -#GNTP/ [:][ :.] -GNTP_INFO_LINE = re.compile( - 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)' + - ' (?P[A-Z0-9]+(:(?P[A-F0-9]+))?) ?' + - '((?P[A-Z0-9]+):(?P[A-F0-9]+).(?P[A-F0-9]+))?\r\n', - re.IGNORECASE -) - -GNTP_INFO_LINE_SHORT = re.compile( - 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)', - re.IGNORECASE -) - -GNTP_HEADER = re.compile('([\w-]+):(.+)') - -GNTP_EOL = '\r\n' - - -class BaseError(Exception): - def gntp_error(self): - error = GNTPError(self.errorcode, self.errordesc) - return error.encode() - - -class ParseError(BaseError): - errorcode = 500 - errordesc = 'Error parsing the message' - - -class AuthError(BaseError): - errorcode = 400 - errordesc = 'Error with authorization' - - -class UnsupportedError(BaseError): - errorcode = 500 - errordesc = 'Currently unsupported by gntp.py' - - -class _GNTPBuffer(StringIO.StringIO): - """GNTP Buffer class""" - def writefmt(self, message = "", *args): - """Shortcut function for writing GNTP Headers""" - self.write((message % args).encode('utf8', 'replace')) - self.write(GNTP_EOL) - - -class _GNTPBase(object): - """Base initilization - - :param string messagetype: GNTP Message type - :param string version: GNTP Protocol version - :param string encription: Encryption protocol - """ - def __init__(self, messagetype = None, version = '1.0', encryption = None): - self.info = { - 'version': version, - 'messagetype': messagetype, - 'encryptionAlgorithmID': encryption - } - self.headers = {} - self.resources = {} - - def __str__(self): - return self.encode() - - def _parse_info(self, data): - """Parse the first line of a GNTP message to get security and other info values - - :param string data: GNTP Message - :return dict: Parsed GNTP Info line - """ - - match = GNTP_INFO_LINE.match(data) - - if not match: - raise ParseError('ERROR_PARSING_INFO_LINE') - - info = match.groupdict() - if info['encryptionAlgorithmID'] == 'NONE': - info['encryptionAlgorithmID'] = None - - return info - - def set_password(self, password, encryptAlgo = 'MD5'): - """Set a password for a GNTP Message - - :param string password: Null to clear password - :param string encryptAlgo: Supports MD5, SHA1, SHA256, SHA512 - """ - hash = { - 'MD5': hashlib.md5, - 'SHA1': hashlib.sha1, - 'SHA256': hashlib.sha256, - 'SHA512': hashlib.sha512, - } - - self.password = password - self.encryptAlgo = encryptAlgo.upper() - if not password: - self.info['encryptionAlgorithmID'] = None - self.info['keyHashAlgorithm'] = None - return - if not self.encryptAlgo in hash.keys(): - raise UnsupportedError('INVALID HASH "%s"' % self.encryptAlgo) - - hashfunction = hash.get(self.encryptAlgo) - - password = password.encode('utf8') - seed = time.ctime() - salt = hashfunction(seed).hexdigest() - saltHash = hashfunction(seed).digest() - keyBasis = password + saltHash - key = hashfunction(keyBasis).digest() - keyHash = hashfunction(key).hexdigest() - - self.info['keyHashAlgorithmID'] = self.encryptAlgo - self.info['keyHash'] = keyHash.upper() - self.info['salt'] = salt.upper() - - def _decode_hex(self, value): - """Helper function to decode hex string to `proper` hex string - - :param string value: Human readable hex string - :return string: Hex string - """ - result = '' - for i in range(0, len(value), 2): - tmp = int(value[i:i + 2], 16) - result += chr(tmp) - return result - - def _decode_binary(self, rawIdentifier, identifier): - rawIdentifier += '\r\n\r\n' - dataLength = int(identifier['Length']) - pointerStart = self.raw.find(rawIdentifier) + len(rawIdentifier) - pointerEnd = pointerStart + dataLength - data = self.raw[pointerStart:pointerEnd] - if not len(data) == dataLength: - raise ParseError('INVALID_DATA_LENGTH Expected: %s Recieved %s' % (dataLength, len(data))) - return data - - def _validate_password(self, password): - """Validate GNTP Message against stored password""" - self.password = password - if password == None: - raise AuthError('Missing password') - keyHash = self.info.get('keyHash', None) - if keyHash is None and self.password is None: - return True - if keyHash is None: - raise AuthError('Invalid keyHash') - if self.password is None: - raise AuthError('Missing password') - - password = self.password.encode('utf8') - saltHash = self._decode_hex(self.info['salt']) - - keyBasis = password + saltHash - key = hashlib.md5(keyBasis).digest() - keyHash = hashlib.md5(key).hexdigest() - - if not keyHash.upper() == self.info['keyHash'].upper(): - raise AuthError('Invalid Hash') - return True - - def validate(self): - """Verify required headers""" - for header in self._requiredHeaders: - if not self.headers.get(header, False): - raise ParseError('Missing Notification Header: ' + header) - - def _format_info(self): - """Generate info line for GNTP Message - - :return string: - """ - info = u'GNTP/%s %s' % ( - self.info.get('version'), - self.info.get('messagetype'), - ) - if self.info.get('encryptionAlgorithmID', None): - info += ' %s:%s' % ( - self.info.get('encryptionAlgorithmID'), - self.info.get('ivValue'), - ) - else: - info += ' NONE' - - if self.info.get('keyHashAlgorithmID', None): - info += ' %s:%s.%s' % ( - self.info.get('keyHashAlgorithmID'), - self.info.get('keyHash'), - self.info.get('salt') - ) - - return info - - def _parse_dict(self, data): - """Helper function to parse blocks of GNTP headers into a dictionary - - :param string data: - :return dict: - """ - dict = {} - for line in data.split('\r\n'): - match = GNTP_HEADER.match(line) - if not match: - continue - - key = unicode(match.group(1).strip(), 'utf8', 'replace') - val = unicode(match.group(2).strip(), 'utf8', 'replace') - dict[key] = val - return dict - - def add_header(self, key, value): - if isinstance(value, unicode): - self.headers[key] = value - else: - self.headers[key] = unicode('%s' % value, 'utf8', 'replace') - - def add_resource(self, data): - """Add binary resource - - :param string data: Binary Data - """ - identifier = hashlib.md5(data).hexdigest() - self.resources[identifier] = data - return 'x-growl-resource://%s' % identifier - - def decode(self, data, password = None): - """Decode GNTP Message - - :param string data: - """ - self.password = password - self.raw = data - parts = self.raw.split('\r\n\r\n') - self.info = self._parse_info(data) - self.headers = self._parse_dict(parts[0]) - - def encode(self): - """Encode a generic GNTP Message - - :return string: GNTP Message ready to be sent - """ - - buffer = _GNTPBuffer() - - buffer.writefmt(self._format_info()) - - #Headers - for k, v in self.headers.iteritems(): - buffer.writefmt('%s: %s', k, v) - buffer.writefmt() - - #Resources - for resource, data in self.resources.iteritems(): - buffer.writefmt('Identifier: %s', resource) - buffer.writefmt('Length: %d', len(data)) - buffer.writefmt() - buffer.write(data) - buffer.writefmt() - buffer.writefmt() - - return buffer.getvalue() - - -class GNTPRegister(_GNTPBase): - """Represents a GNTP Registration Command - - :param string data: (Optional) See decode() - :param string password: (Optional) Password to use while encoding/decoding messages - """ - _requiredHeaders = [ - 'Application-Name', - 'Notifications-Count' - ] - _requiredNotificationHeaders = ['Notification-Name'] - - def __init__(self, data = None, password = None): - _GNTPBase.__init__(self, 'REGISTER') - self.notifications = [] - - if data: - self.decode(data, password) - else: - self.set_password(password) - self.add_header('Application-Name', 'pygntp') - self.add_header('Notifications-Count', 0) - - def validate(self): - '''Validate required headers and validate notification headers''' - for header in self._requiredHeaders: - if not self.headers.get(header, False): - raise ParseError('Missing Registration Header: ' + header) - for notice in self.notifications: - for header in self._requiredNotificationHeaders: - if not notice.get(header, False): - raise ParseError('Missing Notification Header: ' + header) - - def decode(self, data, password): - """Decode existing GNTP Registration message - - :param string data: Message to decode - """ - self.raw = data - parts = self.raw.split('\r\n\r\n') - self.info = self._parse_info(data) - self._validate_password(password) - self.headers = self._parse_dict(parts[0]) - - for i, part in enumerate(parts): - if i == 0: - continue # Skip Header - if part.strip() == '': - continue - notice = self._parse_dict(part) - if notice.get('Notification-Name', False): - self.notifications.append(notice) - elif notice.get('Identifier', False): - notice['Data'] = self._decode_binary(part, notice) - #open('register.png','wblol').write(notice['Data']) - self.resources[notice.get('Identifier')] = notice - - def add_notification(self, name, enabled = True): - """Add new Notification to Registration message - - :param string name: Notification Name - :param boolean enabled: Enable this notification by default - """ - notice = {} - notice['Notification-Name'] = u'%s' % name - notice['Notification-Enabled'] = u'%s' % enabled - - self.notifications.append(notice) - self.add_header('Notifications-Count', len(self.notifications)) - - def encode(self): - """Encode a GNTP Registration Message - - :return string: Encoded GNTP Registration message - """ - - buffer = _GNTPBuffer() - - buffer.writefmt(self._format_info()) - - #Headers - for k, v in self.headers.iteritems(): - buffer.writefmt('%s: %s', k, v) - buffer.writefmt() - - #Notifications - if len(self.notifications) > 0: - for notice in self.notifications: - for k, v in notice.iteritems(): - buffer.writefmt('%s: %s', k, v) - buffer.writefmt() - - #Resources - for resource, data in self.resources.iteritems(): - buffer.writefmt('Identifier: %s', resource) - buffer.writefmt('Length: %d', len(data)) - buffer.writefmt() - buffer.write(data) - buffer.writefmt() - buffer.writefmt() - - return buffer.getvalue() - - -class GNTPNotice(_GNTPBase): - """Represents a GNTP Notification Command - - :param string data: (Optional) See decode() - :param string app: (Optional) Set Application-Name - :param string name: (Optional) Set Notification-Name - :param string title: (Optional) Set Notification Title - :param string password: (Optional) Password to use while encoding/decoding messages - """ - _requiredHeaders = [ - 'Application-Name', - 'Notification-Name', - 'Notification-Title' - ] - - def __init__(self, data = None, app = None, name = None, title = None, password = None): - _GNTPBase.__init__(self, 'NOTIFY') - - if data: - self.decode(data, password) - else: - self.set_password(password) - if app: - self.add_header('Application-Name', app) - if name: - self.add_header('Notification-Name', name) - if title: - self.add_header('Notification-Title', title) - - def decode(self, data, password): - """Decode existing GNTP Notification message - - :param string data: Message to decode. - """ - self.raw = data - parts = self.raw.split('\r\n\r\n') - self.info = self._parse_info(data) - self._validate_password(password) - self.headers = self._parse_dict(parts[0]) - - for i, part in enumerate(parts): - if i == 0: - continue # Skip Header - if part.strip() == '': - continue - notice = self._parse_dict(part) - if notice.get('Identifier', False): - notice['Data'] = self._decode_binary(part, notice) - #open('notice.png','wblol').write(notice['Data']) - self.resources[notice.get('Identifier')] = notice - - -class GNTPSubscribe(_GNTPBase): - """Represents a GNTP Subscribe Command - - :param string data: (Optional) See decode() - :param string password: (Optional) Password to use while encoding/decoding messages - """ - _requiredHeaders = [ - 'Subscriber-ID', - 'Subscriber-Name', - ] - - def __init__(self, data = None, password = None): - _GNTPBase.__init__(self, 'SUBSCRIBE') - if data: - self.decode(data, password) - else: - self.set_password(password) - - -class GNTPOK(_GNTPBase): - """Represents a GNTP OK Response - - :param string data: (Optional) See _GNTPResponse.decode() - :param string action: (Optional) Set type of action the OK Response is for - """ - _requiredHeaders = ['Response-Action'] - - def __init__(self, data = None, action = None): - _GNTPBase.__init__(self, '-OK') - if data: - self.decode(data) - if action: - self.add_header('Response-Action', action) - - -class GNTPError(_GNTPBase): - """Represents a GNTP Error response - - :param string data: (Optional) See _GNTPResponse.decode() - :param string errorcode: (Optional) Error code - :param string errordesc: (Optional) Error Description - """ - _requiredHeaders = ['Error-Code', 'Error-Description'] - - def __init__(self, data = None, errorcode = None, errordesc = None): - _GNTPBase.__init__(self, '-ERROR') - if data: - self.decode(data) - if errorcode: - self.add_header('Error-Code', errorcode) - self.add_header('Error-Description', errordesc) - - def error(self): - return (self.headers.get('Error-Code', None), - self.headers.get('Error-Description', None)) - - -def parse_gntp(data, password = None): - """Attempt to parse a message as a GNTP message - - :param string data: Message to be parsed - :param string password: Optional password to be used to verify the message - """ - match = GNTP_INFO_LINE_SHORT.match(data) - if not match: - raise ParseError('INVALID_GNTP_INFO') - info = match.groupdict() - if info['messagetype'] == 'REGISTER': - return GNTPRegister(data, password = password) - elif info['messagetype'] == 'NOTIFY': - return GNTPNotice(data, password = password) - elif info['messagetype'] == 'SUBSCRIBE': - return GNTPSubscribe(data, password = password) - elif info['messagetype'] == '-OK': - return GNTPOK(data) - elif info['messagetype'] == '-ERROR': - return GNTPError(data) - raise ParseError('INVALID_GNTP_MESSAGE') diff --git a/libs/gntp/cli.py b/libs/gntp/cli.py new file mode 100644 index 00000000..bc083062 --- /dev/null +++ b/libs/gntp/cli.py @@ -0,0 +1,141 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +import logging +import os +import sys +from optparse import OptionParser, OptionGroup + +from gntp.notifier import GrowlNotifier +from gntp.shim import RawConfigParser +from gntp.version import __version__ + +DEFAULT_CONFIG = os.path.expanduser('~/.gntp') + +config = RawConfigParser({ + 'hostname': 'localhost', + 'password': None, + 'port': 23053, +}) +config.read([DEFAULT_CONFIG]) +if not config.has_section('gntp'): + config.add_section('gntp') + + +class ClientParser(OptionParser): + def __init__(self): + OptionParser.__init__(self, version="%%prog %s" % __version__) + + group = OptionGroup(self, "Network Options") + group.add_option("-H", "--host", + dest="host", default=config.get('gntp', 'hostname'), + help="Specify a hostname to which to send a remote notification. [%default]") + group.add_option("--port", + dest="port", default=config.getint('gntp', 'port'), type="int", + help="port to listen on [%default]") + group.add_option("-P", "--password", + dest='password', default=config.get('gntp', 'password'), + help="Network password") + self.add_option_group(group) + + group = OptionGroup(self, "Notification Options") + group.add_option("-n", "--name", + dest="app", default='Python GNTP Test Client', + help="Set the name of the application [%default]") + group.add_option("-s", "--sticky", + dest='sticky', default=False, action="store_true", + help="Make the notification sticky [%default]") + group.add_option("--image", + dest="icon", default=None, + help="Icon for notification (URL or /path/to/file)") + group.add_option("-m", "--message", + dest="message", default=None, + help="Sets the message instead of using stdin") + group.add_option("-p", "--priority", + dest="priority", default=0, type="int", + help="-2 to 2 [%default]") + group.add_option("-d", "--identifier", + dest="identifier", + help="Identifier for coalescing") + group.add_option("-t", "--title", + dest="title", default=None, + help="Set the title of the notification [%default]") + group.add_option("-N", "--notification", + dest="name", default='Notification', + help="Set the notification name [%default]") + group.add_option("--callback", + dest="callback", + help="URL callback") + self.add_option_group(group) + + # Extra Options + self.add_option('-v', '--verbose', + dest='verbose', default=0, action='count', + help="Verbosity levels") + + def parse_args(self, args=None, values=None): + values, args = OptionParser.parse_args(self, args, values) + + if values.message is None: + print('Enter a message followed by Ctrl-D') + try: + message = sys.stdin.read() + except KeyboardInterrupt: + exit() + else: + message = values.message + + if values.title is None: + values.title = ' '.join(args) + + # If we still have an empty title, use the + # first bit of the message as the title + if values.title == '': + values.title = message[:20] + + values.verbose = logging.WARNING - values.verbose * 10 + + return values, message + + +def main(): + (options, message) = ClientParser().parse_args() + logging.basicConfig(level=options.verbose) + if not os.path.exists(DEFAULT_CONFIG): + logging.info('No config read found at %s', DEFAULT_CONFIG) + + growl = GrowlNotifier( + applicationName=options.app, + notifications=[options.name], + defaultNotifications=[options.name], + hostname=options.host, + password=options.password, + port=options.port, + ) + result = growl.register() + if result is not True: + exit(result) + + # This would likely be better placed within the growl notifier + # class but until I make _checkIcon smarter this is "easier" + if options.icon is not None and not options.icon.startswith('http'): + logging.info('Loading image %s', options.icon) + f = open(options.icon) + options.icon = f.read() + f.close() + + result = growl.notify( + noteType=options.name, + title=options.title, + description=message, + icon=options.icon, + sticky=options.sticky, + priority=options.priority, + callback=options.callback, + identifier=options.identifier, + ) + if result is not True: + exit(result) + +if __name__ == "__main__": + main() diff --git a/libs/gntp/config.py b/libs/gntp/config.py new file mode 100644 index 00000000..7536bd14 --- /dev/null +++ b/libs/gntp/config.py @@ -0,0 +1,77 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +""" +The gntp.config module is provided as an extended GrowlNotifier object that takes +advantage of the ConfigParser module to allow us to setup some default values +(such as hostname, password, and port) in a more global way to be shared among +programs using gntp +""" +import logging +import os + +import gntp.notifier +import gntp.shim + +__all__ = [ + 'mini', + 'GrowlNotifier' +] + +logger = logging.getLogger(__name__) + + +class GrowlNotifier(gntp.notifier.GrowlNotifier): + """ + ConfigParser enhanced GrowlNotifier object + + For right now, we are only interested in letting users overide certain + values from ~/.gntp + + :: + + [gntp] + hostname = ? + password = ? + port = ? + """ + def __init__(self, *args, **kwargs): + config = gntp.shim.RawConfigParser({ + 'hostname': kwargs.get('hostname', 'localhost'), + 'password': kwargs.get('password'), + 'port': kwargs.get('port', 23053), + }) + + config.read([os.path.expanduser('~/.gntp')]) + + # If the file does not exist, then there will be no gntp section defined + # and the config.get() lines below will get confused. Since we are not + # saving the config, it should be safe to just add it here so the + # code below doesn't complain + if not config.has_section('gntp'): + logger.info('Error reading ~/.gntp config file') + config.add_section('gntp') + + kwargs['password'] = config.get('gntp', 'password') + kwargs['hostname'] = config.get('gntp', 'hostname') + kwargs['port'] = config.getint('gntp', 'port') + + super(GrowlNotifier, self).__init__(*args, **kwargs) + + +def mini(description, **kwargs): + """Single notification function + + Simple notification function in one line. Has only one required parameter + and attempts to use reasonable defaults for everything else + :param string description: Notification message + """ + kwargs['notifierFactory'] = GrowlNotifier + gntp.notifier.mini(description, **kwargs) + + +if __name__ == '__main__': + # If we're running this module directly we're likely running it as a test + # so extra debugging is useful + logging.basicConfig(level=logging.INFO) + mini('Testing mini notification') diff --git a/libs/gntp/core.py b/libs/gntp/core.py new file mode 100644 index 00000000..ee544d3d --- /dev/null +++ b/libs/gntp/core.py @@ -0,0 +1,511 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +import hashlib +import re +import time + +import gntp.shim +import gntp.errors as errors + +__all__ = [ + 'GNTPRegister', + 'GNTPNotice', + 'GNTPSubscribe', + 'GNTPOK', + 'GNTPError', + 'parse_gntp', +] + +#GNTP/ [:][ :.] +GNTP_INFO_LINE = re.compile( + 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)' + + ' (?P[A-Z0-9]+(:(?P[A-F0-9]+))?) ?' + + '((?P[A-Z0-9]+):(?P[A-F0-9]+).(?P[A-F0-9]+))?\r\n', + re.IGNORECASE +) + +GNTP_INFO_LINE_SHORT = re.compile( + 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)', + re.IGNORECASE +) + +GNTP_HEADER = re.compile('([\w-]+):(.+)') + +GNTP_EOL = gntp.shim.b('\r\n') +GNTP_SEP = gntp.shim.b(': ') + + +class _GNTPBuffer(gntp.shim.StringIO): + """GNTP Buffer class""" + def writeln(self, value=None): + if value: + self.write(gntp.shim.b(value)) + self.write(GNTP_EOL) + + def writeheader(self, key, value): + if not isinstance(value, str): + value = str(value) + self.write(gntp.shim.b(key)) + self.write(GNTP_SEP) + self.write(gntp.shim.b(value)) + self.write(GNTP_EOL) + + +class _GNTPBase(object): + """Base initilization + + :param string messagetype: GNTP Message type + :param string version: GNTP Protocol version + :param string encription: Encryption protocol + """ + def __init__(self, messagetype=None, version='1.0', encryption=None): + self.info = { + 'version': version, + 'messagetype': messagetype, + 'encryptionAlgorithmID': encryption + } + self.hash_algo = { + 'MD5': hashlib.md5, + 'SHA1': hashlib.sha1, + 'SHA256': hashlib.sha256, + 'SHA512': hashlib.sha512, + } + self.headers = {} + self.resources = {} + + def __str__(self): + return self.encode() + + def _parse_info(self, data): + """Parse the first line of a GNTP message to get security and other info values + + :param string data: GNTP Message + :return dict: Parsed GNTP Info line + """ + + match = GNTP_INFO_LINE.match(data) + + if not match: + raise errors.ParseError('ERROR_PARSING_INFO_LINE') + + info = match.groupdict() + if info['encryptionAlgorithmID'] == 'NONE': + info['encryptionAlgorithmID'] = None + + return info + + def set_password(self, password, encryptAlgo='MD5'): + """Set a password for a GNTP Message + + :param string password: Null to clear password + :param string encryptAlgo: Supports MD5, SHA1, SHA256, SHA512 + """ + if not password: + self.info['encryptionAlgorithmID'] = None + self.info['keyHashAlgorithm'] = None + return + + self.password = gntp.shim.b(password) + self.encryptAlgo = encryptAlgo.upper() + + if not self.encryptAlgo in self.hash_algo: + raise errors.UnsupportedError('INVALID HASH "%s"' % self.encryptAlgo) + + hashfunction = self.hash_algo.get(self.encryptAlgo) + + password = password.encode('utf8') + seed = time.ctime().encode('utf8') + salt = hashfunction(seed).hexdigest() + saltHash = hashfunction(seed).digest() + keyBasis = password + saltHash + key = hashfunction(keyBasis).digest() + keyHash = hashfunction(key).hexdigest() + + self.info['keyHashAlgorithmID'] = self.encryptAlgo + self.info['keyHash'] = keyHash.upper() + self.info['salt'] = salt.upper() + + def _decode_hex(self, value): + """Helper function to decode hex string to `proper` hex string + + :param string value: Human readable hex string + :return string: Hex string + """ + result = '' + for i in range(0, len(value), 2): + tmp = int(value[i:i + 2], 16) + result += chr(tmp) + return result + + def _decode_binary(self, rawIdentifier, identifier): + rawIdentifier += '\r\n\r\n' + dataLength = int(identifier['Length']) + pointerStart = self.raw.find(rawIdentifier) + len(rawIdentifier) + pointerEnd = pointerStart + dataLength + data = self.raw[pointerStart:pointerEnd] + if not len(data) == dataLength: + raise errors.ParseError('INVALID_DATA_LENGTH Expected: %s Recieved %s' % (dataLength, len(data))) + return data + + def _validate_password(self, password): + """Validate GNTP Message against stored password""" + self.password = password + if password is None: + raise errors.AuthError('Missing password') + keyHash = self.info.get('keyHash', None) + if keyHash is None and self.password is None: + return True + if keyHash is None: + raise errors.AuthError('Invalid keyHash') + if self.password is None: + raise errors.AuthError('Missing password') + + keyHashAlgorithmID = self.info.get('keyHashAlgorithmID','MD5') + + password = self.password.encode('utf8') + saltHash = self._decode_hex(self.info['salt']) + + keyBasis = password + saltHash + self.key = self.hash_algo[keyHashAlgorithmID](keyBasis).digest() + keyHash = self.hash_algo[keyHashAlgorithmID](self.key).hexdigest() + + if not keyHash.upper() == self.info['keyHash'].upper(): + raise errors.AuthError('Invalid Hash') + return True + + def validate(self): + """Verify required headers""" + for header in self._requiredHeaders: + if not self.headers.get(header, False): + raise errors.ParseError('Missing Notification Header: ' + header) + + def _format_info(self): + """Generate info line for GNTP Message + + :return string: + """ + info = 'GNTP/%s %s' % ( + self.info.get('version'), + self.info.get('messagetype'), + ) + if self.info.get('encryptionAlgorithmID', None): + info += ' %s:%s' % ( + self.info.get('encryptionAlgorithmID'), + self.info.get('ivValue'), + ) + else: + info += ' NONE' + + if self.info.get('keyHashAlgorithmID', None): + info += ' %s:%s.%s' % ( + self.info.get('keyHashAlgorithmID'), + self.info.get('keyHash'), + self.info.get('salt') + ) + + return info + + def _parse_dict(self, data): + """Helper function to parse blocks of GNTP headers into a dictionary + + :param string data: + :return dict: Dictionary of parsed GNTP Headers + """ + d = {} + for line in data.split('\r\n'): + match = GNTP_HEADER.match(line) + if not match: + continue + + key = match.group(1).strip() + val = match.group(2).strip() + d[key] = val + return d + + def add_header(self, key, value): + self.headers[key] = value + + def add_resource(self, data): + """Add binary resource + + :param string data: Binary Data + """ + data = gntp.shim.b(data) + identifier = hashlib.md5(data).hexdigest() + self.resources[identifier] = data + return 'x-growl-resource://%s' % identifier + + def decode(self, data, password=None): + """Decode GNTP Message + + :param string data: + """ + self.password = password + self.raw = gntp.shim.u(data) + parts = self.raw.split('\r\n\r\n') + self.info = self._parse_info(self.raw) + self.headers = self._parse_dict(parts[0]) + + def encode(self): + """Encode a generic GNTP Message + + :return string: GNTP Message ready to be sent. Returned as a byte string + """ + + buff = _GNTPBuffer() + + buff.writeln(self._format_info()) + + #Headers + for k, v in self.headers.items(): + buff.writeheader(k, v) + buff.writeln() + + #Resources + for resource, data in self.resources.items(): + buff.writeheader('Identifier', resource) + buff.writeheader('Length', len(data)) + buff.writeln() + buff.write(data) + buff.writeln() + buff.writeln() + + return buff.getvalue() + + +class GNTPRegister(_GNTPBase): + """Represents a GNTP Registration Command + + :param string data: (Optional) See decode() + :param string password: (Optional) Password to use while encoding/decoding messages + """ + _requiredHeaders = [ + 'Application-Name', + 'Notifications-Count' + ] + _requiredNotificationHeaders = ['Notification-Name'] + + def __init__(self, data=None, password=None): + _GNTPBase.__init__(self, 'REGISTER') + self.notifications = [] + + if data: + self.decode(data, password) + else: + self.set_password(password) + self.add_header('Application-Name', 'pygntp') + self.add_header('Notifications-Count', 0) + + def validate(self): + '''Validate required headers and validate notification headers''' + for header in self._requiredHeaders: + if not self.headers.get(header, False): + raise errors.ParseError('Missing Registration Header: ' + header) + for notice in self.notifications: + for header in self._requiredNotificationHeaders: + if not notice.get(header, False): + raise errors.ParseError('Missing Notification Header: ' + header) + + def decode(self, data, password): + """Decode existing GNTP Registration message + + :param string data: Message to decode + """ + self.raw = gntp.shim.u(data) + parts = self.raw.split('\r\n\r\n') + self.info = self._parse_info(self.raw) + self._validate_password(password) + self.headers = self._parse_dict(parts[0]) + + for i, part in enumerate(parts): + if i == 0: + continue # Skip Header + if part.strip() == '': + continue + notice = self._parse_dict(part) + if notice.get('Notification-Name', False): + self.notifications.append(notice) + elif notice.get('Identifier', False): + notice['Data'] = self._decode_binary(part, notice) + #open('register.png','wblol').write(notice['Data']) + self.resources[notice.get('Identifier')] = notice + + def add_notification(self, name, enabled=True): + """Add new Notification to Registration message + + :param string name: Notification Name + :param boolean enabled: Enable this notification by default + """ + notice = {} + notice['Notification-Name'] = name + notice['Notification-Enabled'] = enabled + + self.notifications.append(notice) + self.add_header('Notifications-Count', len(self.notifications)) + + def encode(self): + """Encode a GNTP Registration Message + + :return string: Encoded GNTP Registration message. Returned as a byte string + """ + + buff = _GNTPBuffer() + + buff.writeln(self._format_info()) + + #Headers + for k, v in self.headers.items(): + buff.writeheader(k, v) + buff.writeln() + + #Notifications + if len(self.notifications) > 0: + for notice in self.notifications: + for k, v in notice.items(): + buff.writeheader(k, v) + buff.writeln() + + #Resources + for resource, data in self.resources.items(): + buff.writeheader('Identifier', resource) + buff.writeheader('Length', len(data)) + buff.writeln() + buff.write(data) + buff.writeln() + buff.writeln() + + return buff.getvalue() + + +class GNTPNotice(_GNTPBase): + """Represents a GNTP Notification Command + + :param string data: (Optional) See decode() + :param string app: (Optional) Set Application-Name + :param string name: (Optional) Set Notification-Name + :param string title: (Optional) Set Notification Title + :param string password: (Optional) Password to use while encoding/decoding messages + """ + _requiredHeaders = [ + 'Application-Name', + 'Notification-Name', + 'Notification-Title' + ] + + def __init__(self, data=None, app=None, name=None, title=None, password=None): + _GNTPBase.__init__(self, 'NOTIFY') + + if data: + self.decode(data, password) + else: + self.set_password(password) + if app: + self.add_header('Application-Name', app) + if name: + self.add_header('Notification-Name', name) + if title: + self.add_header('Notification-Title', title) + + def decode(self, data, password): + """Decode existing GNTP Notification message + + :param string data: Message to decode. + """ + self.raw = gntp.shim.u(data) + parts = self.raw.split('\r\n\r\n') + self.info = self._parse_info(self.raw) + self._validate_password(password) + self.headers = self._parse_dict(parts[0]) + + for i, part in enumerate(parts): + if i == 0: + continue # Skip Header + if part.strip() == '': + continue + notice = self._parse_dict(part) + if notice.get('Identifier', False): + notice['Data'] = self._decode_binary(part, notice) + #open('notice.png','wblol').write(notice['Data']) + self.resources[notice.get('Identifier')] = notice + + +class GNTPSubscribe(_GNTPBase): + """Represents a GNTP Subscribe Command + + :param string data: (Optional) See decode() + :param string password: (Optional) Password to use while encoding/decoding messages + """ + _requiredHeaders = [ + 'Subscriber-ID', + 'Subscriber-Name', + ] + + def __init__(self, data=None, password=None): + _GNTPBase.__init__(self, 'SUBSCRIBE') + if data: + self.decode(data, password) + else: + self.set_password(password) + + +class GNTPOK(_GNTPBase): + """Represents a GNTP OK Response + + :param string data: (Optional) See _GNTPResponse.decode() + :param string action: (Optional) Set type of action the OK Response is for + """ + _requiredHeaders = ['Response-Action'] + + def __init__(self, data=None, action=None): + _GNTPBase.__init__(self, '-OK') + if data: + self.decode(data) + if action: + self.add_header('Response-Action', action) + + +class GNTPError(_GNTPBase): + """Represents a GNTP Error response + + :param string data: (Optional) See _GNTPResponse.decode() + :param string errorcode: (Optional) Error code + :param string errordesc: (Optional) Error Description + """ + _requiredHeaders = ['Error-Code', 'Error-Description'] + + def __init__(self, data=None, errorcode=None, errordesc=None): + _GNTPBase.__init__(self, '-ERROR') + if data: + self.decode(data) + if errorcode: + self.add_header('Error-Code', errorcode) + self.add_header('Error-Description', errordesc) + + def error(self): + return (self.headers.get('Error-Code', None), + self.headers.get('Error-Description', None)) + + +def parse_gntp(data, password=None): + """Attempt to parse a message as a GNTP message + + :param string data: Message to be parsed + :param string password: Optional password to be used to verify the message + """ + data = gntp.shim.u(data) + match = GNTP_INFO_LINE_SHORT.match(data) + if not match: + raise errors.ParseError('INVALID_GNTP_INFO') + info = match.groupdict() + if info['messagetype'] == 'REGISTER': + return GNTPRegister(data, password=password) + elif info['messagetype'] == 'NOTIFY': + return GNTPNotice(data, password=password) + elif info['messagetype'] == 'SUBSCRIBE': + return GNTPSubscribe(data, password=password) + elif info['messagetype'] == '-OK': + return GNTPOK(data) + elif info['messagetype'] == '-ERROR': + return GNTPError(data) + raise errors.ParseError('INVALID_GNTP_MESSAGE') diff --git a/libs/gntp/errors.py b/libs/gntp/errors.py new file mode 100644 index 00000000..c006fd68 --- /dev/null +++ b/libs/gntp/errors.py @@ -0,0 +1,25 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +class BaseError(Exception): + pass + + +class ParseError(BaseError): + errorcode = 500 + errordesc = 'Error parsing the message' + + +class AuthError(BaseError): + errorcode = 400 + errordesc = 'Error with authorization' + + +class UnsupportedError(BaseError): + errorcode = 500 + errordesc = 'Currently unsupported by gntp.py' + + +class NetworkError(BaseError): + errorcode = 500 + errordesc = "Error connecting to growl server" diff --git a/libs/gntp/notifier.py b/libs/gntp/notifier.py index 539dae2a..1719ecdf 100755 --- a/libs/gntp/notifier.py +++ b/libs/gntp/notifier.py @@ -1,3 +1,6 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + """ The gntp.notifier module is provided as a simple way to send notifications using GNTP @@ -9,10 +12,15 @@ using GNTP `Original Python bindings `_ """ -import gntp -import socket import logging import platform +import socket +import sys + +from gntp.version import __version__ +import gntp.core +import gntp.errors as errors +import gntp.shim __all__ = [ 'mini', @@ -37,9 +45,9 @@ class GrowlNotifier(object): passwordHash = 'MD5' socketTimeout = 3 - def __init__(self, applicationName = 'Python GNTP', notifications = [], - defaultNotifications = None, applicationIcon = None, hostname = 'localhost', - password = None, port = 23053): + def __init__(self, applicationName='Python GNTP', notifications=[], + defaultNotifications=None, applicationIcon=None, hostname='localhost', + password=None, port=23053): self.applicationName = applicationName self.notifications = list(notifications) @@ -61,7 +69,7 @@ class GrowlNotifier(object): then we return False ''' logger.info('Checking icon') - return data.startswith('http') + return gntp.shim.u(data).startswith('http') def register(self): """Send GNTP Registration @@ -71,7 +79,7 @@ class GrowlNotifier(object): sent a registration message at least once """ logger.info('Sending registration to %s:%s', self.hostname, self.port) - register = gntp.GNTPRegister() + register = gntp.core.GNTPRegister() register.add_header('Application-Name', self.applicationName) for notification in self.notifications: enabled = notification in self.defaultNotifications @@ -80,16 +88,16 @@ class GrowlNotifier(object): if self._checkIcon(self.applicationIcon): register.add_header('Application-Icon', self.applicationIcon) else: - id = register.add_resource(self.applicationIcon) - register.add_header('Application-Icon', id) + resource = register.add_resource(self.applicationIcon) + register.add_header('Application-Icon', resource) if self.password: register.set_password(self.password, self.passwordHash) self.add_origin_info(register) self.register_hook(register) return self._send('register', register) - def notify(self, noteType, title, description, icon = None, sticky = False, - priority = None, callback = None, identifier = None): + def notify(self, noteType, title, description, icon=None, sticky=False, + priority=None, callback=None, identifier=None, custom={}): """Send a GNTP notifications .. warning:: @@ -102,6 +110,8 @@ class GrowlNotifier(object): :param boolean sticky: Sticky notification :param integer priority: Message priority level from -2 to 2 :param string callback: URL callback + :param dict custom: Custom attributes. Key names should be prefixed with X- + according to the spec but this is not enforced by this class .. warning:: For now, only URL callbacks are supported. In the future, the @@ -109,7 +119,7 @@ class GrowlNotifier(object): """ logger.info('Sending notification [%s] to %s:%s', noteType, self.hostname, self.port) assert noteType in self.notifications - notice = gntp.GNTPNotice() + notice = gntp.core.GNTPNotice() notice.add_header('Application-Name', self.applicationName) notice.add_header('Notification-Name', noteType) notice.add_header('Notification-Title', title) @@ -123,8 +133,8 @@ class GrowlNotifier(object): if self._checkIcon(icon): notice.add_header('Notification-Icon', icon) else: - id = notice.add_resource(icon) - notice.add_header('Notification-Icon', id) + resource = notice.add_resource(icon) + notice.add_header('Notification-Icon', resource) if description: notice.add_header('Notification-Text', description) @@ -133,6 +143,9 @@ class GrowlNotifier(object): if identifier: notice.add_header('Notification-Coalescing-ID', identifier) + for key in custom: + notice.add_header(key, custom[key]) + self.add_origin_info(notice) self.notify_hook(notice) @@ -140,7 +153,7 @@ class GrowlNotifier(object): def subscribe(self, id, name, port): """Send a Subscribe request to a remote machine""" - sub = gntp.GNTPSubscribe() + sub = gntp.core.GNTPSubscribe() sub.add_header('Subscriber-ID', id) sub.add_header('Subscriber-Name', name) sub.add_header('Subscriber-Port', port) @@ -156,7 +169,7 @@ class GrowlNotifier(object): """Add optional Origin headers to message""" packet.add_header('Origin-Machine-Name', platform.node()) packet.add_header('Origin-Software-Name', 'gntp.py') - packet.add_header('Origin-Software-Version', gntp.__version__) + packet.add_header('Origin-Software-Version', __version__) packet.add_header('Origin-Platform-Name', platform.system()) packet.add_header('Origin-Platform-Version', platform.platform()) @@ -179,27 +192,33 @@ class GrowlNotifier(object): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(self.socketTimeout) - s.connect((self.hostname, self.port)) - s.send(data) - recv_data = s.recv(1024) - while not recv_data.endswith("\r\n\r\n"): - recv_data += s.recv(1024) - response = gntp.parse_gntp(recv_data) + try: + s.connect((self.hostname, self.port)) + s.send(data) + recv_data = s.recv(1024) + while not recv_data.endswith(gntp.shim.b("\r\n\r\n")): + recv_data += s.recv(1024) + except socket.error: + # Python2.5 and Python3 compatibile exception + exc = sys.exc_info()[1] + raise errors.NetworkError(exc) + + response = gntp.core.parse_gntp(recv_data) s.close() logger.debug('From : %s:%s <%s>\n%s', self.hostname, self.port, response.__class__, response) - if type(response) == gntp.GNTPOK: + if type(response) == gntp.core.GNTPOK: return True logger.error('Invalid response: %s', response.error()) return response.error() -def mini(description, applicationName = 'PythonMini', noteType = "Message", - title = "Mini Message", applicationIcon = None, hostname = 'localhost', - password = None, port = 23053, sticky = False, priority = None, - callback = None, notificationIcon = None, identifier = None, - notifierFactory = GrowlNotifier): +def mini(description, applicationName='PythonMini', noteType="Message", + title="Mini Message", applicationIcon=None, hostname='localhost', + password=None, port=23053, sticky=False, priority=None, + callback=None, notificationIcon=None, identifier=None, + notifierFactory=GrowlNotifier): """Single notification function Simple notification function in one line. Has only one required parameter @@ -210,32 +229,37 @@ def mini(description, applicationName = 'PythonMini', noteType = "Message", For now, only URL callbacks are supported. In the future, the callback argument will also support a function """ - growl = notifierFactory( - applicationName = applicationName, - notifications = [noteType], - defaultNotifications = [noteType], - applicationIcon = applicationIcon, - hostname = hostname, - password = password, - port = port, - ) - result = growl.register() - if result is not True: - return result + try: + growl = notifierFactory( + applicationName=applicationName, + notifications=[noteType], + defaultNotifications=[noteType], + applicationIcon=applicationIcon, + hostname=hostname, + password=password, + port=port, + ) + result = growl.register() + if result is not True: + return result - return growl.notify( - noteType = noteType, - title = title, - description = description, - icon = notificationIcon, - sticky = sticky, - priority = priority, - callback = callback, - identifier = identifier, - ) + return growl.notify( + noteType=noteType, + title=title, + description=description, + icon=notificationIcon, + sticky=sticky, + priority=priority, + callback=callback, + identifier=identifier, + ) + except Exception: + # We want the "mini" function to be simple and swallow Exceptions + # in order to be less invasive + logger.exception("Growl error") if __name__ == '__main__': # If we're running this module directly we're likely running it as a test # so extra debugging is useful - logging.basicConfig(level = logging.INFO) + logging.basicConfig(level=logging.INFO) mini('Testing mini notification') diff --git a/libs/gntp/shim.py b/libs/gntp/shim.py new file mode 100644 index 00000000..3a387828 --- /dev/null +++ b/libs/gntp/shim.py @@ -0,0 +1,45 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +""" +Python2.5 and Python3.3 compatibility shim + +Heavily inspirted by the "six" library. +https://pypi.python.org/pypi/six +""" + +import sys + +PY3 = sys.version_info[0] == 3 + +if PY3: + def b(s): + if isinstance(s, bytes): + return s + return s.encode('utf8', 'replace') + + def u(s): + if isinstance(s, bytes): + return s.decode('utf8', 'replace') + return s + + from io import BytesIO as StringIO + from configparser import RawConfigParser +else: + def b(s): + if isinstance(s, unicode): + return s.encode('utf8', 'replace') + return s + + def u(s): + if isinstance(s, unicode): + return s + if isinstance(s, int): + s = str(s) + return unicode(s, "utf8", "replace") + + from StringIO import StringIO + from ConfigParser import RawConfigParser + +b.__doc__ = "Ensure we have a byte string" +u.__doc__ = "Ensure we have a unicode string" diff --git a/libs/gntp/version.py b/libs/gntp/version.py new file mode 100644 index 00000000..2166aaca --- /dev/null +++ b/libs/gntp/version.py @@ -0,0 +1,4 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +__version__ = '1.0.2' diff --git a/libs/guessit/__init__.py b/libs/guessit/__init__.py index ce140248..e6cfa276 100755 --- a/libs/guessit/__init__.py +++ b/libs/guessit/__init__.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals -__version__ = '0.7-dev' +__version__ = '0.6.2' __all__ = ['Guess', 'Language', 'guess_file_info', 'guess_video_info', 'guess_movie_info', 'guess_episode_info'] @@ -76,6 +76,7 @@ from guessit.language import Language from guessit.matcher import IterativeMatcher from guessit.textutils import clean_string import logging +import json log = logging.getLogger(__name__) @@ -105,17 +106,74 @@ def _guess_filename(filename, filetype): mtree = IterativeMatcher(filename, filetype=filetype) + m = mtree.matched() + + second_pass_opts = [] + second_pass_transfo_opts = {} + # if there are multiple possible years found, we assume the first one is # part of the title, reparse the tree taking this into account years = set(n.value for n in find_nodes(mtree.match_tree, 'year')) if len(years) >= 2: - mtree = IterativeMatcher(filename, filetype=filetype, - opts=['skip_first_year']) + second_pass_opts.append('skip_first_year') + to_skip_language_nodes = [] + + title_nodes = set(n for n in find_nodes(mtree.match_tree, ['title', 'series'])) + title_spans = {} + for title_node in title_nodes: + title_spans[title_node.span[0]] = title_node + title_spans[title_node.span[1]] = title_node + + for lang_key in ('language', 'subtitleLanguage'): + langs = {} + lang_nodes = set(n for n in find_nodes(mtree.match_tree, lang_key)) + + for lang_node in lang_nodes: + lang = lang_node.guess.get(lang_key, None) + if len(lang_node.value) > 3 and (lang_node.span[0] in title_spans.keys() or lang_node.span[1] in title_spans.keys()): + # Language is next or before title, and is not a language code. Add to skip for 2nd pass. + + # if filetype is subtitle and the language appears last, just before + # the extension, then it is likely a subtitle language + parts = clean_string(lang_node.root.value).split() + if m['type'] in ['moviesubtitle', 'episodesubtitle'] and (parts.index(lang_node.value) == len(parts) - 2): + continue + + to_skip_language_nodes.append(lang_node) + elif not lang in langs: + langs[lang] = lang_node + else: + # The same language was found. Keep the more confident one, and add others to skip for 2nd pass. + existing_lang_node = langs[lang] + to_skip = None + if existing_lang_node.guess.confidence('language') >= lang_node.guess.confidence('language'): + # lang_node is to remove + to_skip = lang_node + else: + # existing_lang_node is to remove + langs[lang] = lang_node + to_skip = existing_lang_node + to_skip_language_nodes.append(to_skip) + + + if to_skip_language_nodes: + second_pass_transfo_opts['guess_language'] = ( + ((), { 'skip': [ { 'node_idx': node.parent.node_idx, + 'span': node.span } + for node in to_skip_language_nodes ] })) + + if second_pass_opts or second_pass_transfo_opts: + # 2nd pass is needed + log.info("Running 2nd pass with options: %s" % second_pass_opts) + log.info("Transfo options: %s" % second_pass_transfo_opts) + mtree = IterativeMatcher(filename, filetype=filetype, + opts=second_pass_opts, + transfo_opts=second_pass_transfo_opts) m = mtree.matched() - if 'language' not in m and 'subtitleLanguage' not in m: + if 'language' not in m and 'subtitleLanguage' not in m or 'title' not in m: return m # if we found some language, make sure we didn't cut a title or sth... @@ -123,51 +181,10 @@ def _guess_filename(filename, filetype): opts=['nolanguage', 'nocountry']) m2 = mtree2.matched() - - if m.get('title') is None: - return m - if m.get('title') != m2.get('title'): title = next(find_nodes(mtree.match_tree, 'title')) title2 = next(find_nodes(mtree2.match_tree, 'title')) - langs = list(find_nodes(mtree.match_tree, ['language', 'subtitleLanguage'])) - if not langs: - return warning('A weird error happened with language detection') - - # find the language that is likely more relevant - for lng in langs: - if lng.value in title2.value: - # if the language was detected as part of a potential title, - # look at this one in particular - lang = lng - break - else: - # pick the first one if we don't have a better choice - lang = langs[0] - - - # language code are rarely part of a title, and those - # should be handled by the Language exceptions anyway - if len(lang.value) <= 3: - return m - - - # if filetype is subtitle and the language appears last, just before - # the extension, then it is likely a subtitle language - parts = clean_string(title.root.value).split() - if (m['type'] in ['moviesubtitle', 'episodesubtitle'] and - parts.index(lang.value) == len(parts) - 2): - return m - - # if the language was in the middle of the other potential title, - # keep the other title (eg: The Italian Job), except if it is at the - # very beginning, in which case we consider it an error - if m2['title'].startswith(lang.value): - return m - elif lang.value in title2.value: - return m2 - # if a node is in an explicit group, then the correct title is probably # the other one if title.root.node_at(title.node_idx[:2]).is_explicit(): @@ -175,9 +192,6 @@ def _guess_filename(filename, filetype): elif title2.root.node_at(title2.node_idx[:2]).is_explicit(): return m - return warning('Not sure of the title because of the language position') - - return m diff --git a/libs/guessit/__main__.py b/libs/guessit/__main__.py index 957ec9da..ccfa3af6 100755 --- a/libs/guessit/__main__.py +++ b/libs/guessit/__main__.py @@ -24,16 +24,19 @@ from guessit import u from guessit import slogging, guess_file_info from optparse import OptionParser import logging +import sys +import os +import locale -def detect_filename(filename, filetype, info=['filename']): +def detect_filename(filename, filetype, info=['filename'], advanced = False): filename = u(filename) print('For:', filename) - print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string()) + print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string(advanced)) -def run_demo(episodes=True, movies=True): +def run_demo(episodes=True, movies=True, advanced=False): # NOTE: tests should not be added here but rather in the tests/ folder # this is just intended as a quick example if episodes: @@ -50,7 +53,7 @@ def run_demo(episodes=True, movies=True): for f in testeps: print('-'*80) - detect_filename(f, filetype='episode') + detect_filename(f, filetype='episode', advanced=advanced) if movies: @@ -77,12 +80,17 @@ def run_demo(episodes=True, movies=True): for f in testmovies: print('-'*80) - detect_filename(f, filetype = 'movie') + detect_filename(f, filetype = 'movie', advanced = advanced) def main(): slogging.setupLogging() + # see http://bugs.python.org/issue2128 + if sys.version_info.major < 3 and os.name == 'nt': + for i, a in enumerate(sys.argv): + sys.argv[i] = a.decode(locale.getpreferredencoding()) + parser = OptionParser(usage = 'usage: %prog [options] file1 [file2...]') parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False, help = 'display debug output') @@ -92,6 +100,8 @@ def main(): 'them, comma-separated') parser.add_option('-t', '--type', dest = 'filetype', default = 'autodetect', help = 'the suggested file type: movie, episode or autodetect') + parser.add_option('-a', '--advanced', dest = 'advanced', action='store_true', default = False, + help = 'display advanced information for filename guesses, as json output') parser.add_option('-d', '--demo', action='store_true', dest='demo', default=False, help = 'run a few builtin tests instead of analyzing a file') @@ -100,13 +110,14 @@ def main(): logging.getLogger('guessit').setLevel(logging.DEBUG) if options.demo: - run_demo(episodes=True, movies=True) + run_demo(episodes=True, movies=True, advanced=options.advanced) else: if args: for filename in args: detect_filename(filename, filetype = options.filetype, - info = options.info.split(',')) + info = options.info.split(','), + advanced = options.advanced) else: parser.print_help() diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py index dc077e64..9531f82a 100755 --- a/libs/guessit/fileutils.py +++ b/libs/guessit/fileutils.py @@ -44,13 +44,14 @@ def split_path(path): result = [] while True: head, tail = os.path.split(path) + headlen = len(head) # on Unix systems, the root folder is '/' - if head == '/' and tail == '': + if head and head == '/'*headlen and tail == '': return ['/'] + result # on Windows, the root folder is a drive letter (eg: 'C:\') or for shares \\ - if ((len(head) == 3 and head[1:] == ':\\') or (len(head) == 2 and head == '\\\\')) and tail == '': + if ((headlen == 3 and head[1:] == ':\\') or (headlen == 2 and head == '\\\\')) and tail == '': return [head] + result if head == '' and tail == '': @@ -61,6 +62,7 @@ def split_path(path): path = head continue + # otherwise, add the last path fragment and keep splitting result = [tail] + result path = head diff --git a/libs/guessit/guess.py b/libs/guessit/guess.py index 33d36517..73babceb 100755 --- a/libs/guessit/guess.py +++ b/libs/guessit/guess.py @@ -41,15 +41,21 @@ class Guess(UnicodeMixin, dict): confidence = kwargs.pop('confidence') except KeyError: confidence = 0 + + try: + raw = kwargs.pop('raw') + except KeyError: + raw = None dict.__init__(self, *args, **kwargs) self._confidence = {} + self._raw = {} for prop in self: self._confidence[prop] = confidence - - - def to_dict(self): + self._raw[prop] = raw + + def to_dict(self, advanced=False): data = dict(self) for prop, value in data.items(): if isinstance(value, datetime.date): @@ -58,46 +64,65 @@ class Guess(UnicodeMixin, dict): data[prop] = u(value) elif isinstance(value, list): data[prop] = [u(x) for x in value] + if advanced: + data[prop] = {"value": data[prop], "raw": self.raw(prop), "confidence": self.confidence(prop)} return data - def nice_string(self): - data = self.to_dict() - - parts = json.dumps(data, indent=4).split('\n') - for i, p in enumerate(parts): - if p[:5] != ' "': - continue - - prop = p.split('"')[1] - parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:] - - return '\n'.join(parts) + def nice_string(self, advanced=False): + if advanced: + data = self.to_dict(advanced) + return json.dumps(data, indent=4) + else: + data = self.to_dict() + + parts = json.dumps(data, indent=4).split('\n') + for i, p in enumerate(parts): + if p[:5] != ' "': + continue + + prop = p.split('"')[1] + parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:] + + return '\n'.join(parts) def __unicode__(self): return u(self.to_dict()) def confidence(self, prop): return self._confidence.get(prop, -1) + + def raw(self, prop): + return self._raw.get(prop, None) - def set(self, prop, value, confidence=None): + def set(self, prop, value, confidence=None, raw=None): self[prop] = value if confidence is not None: self._confidence[prop] = confidence + if raw is not None: + self._raw[prop] = raw def set_confidence(self, prop, value): self._confidence[prop] = value + + def set_raw(self, prop, value): + self._raw[prop] = value - def update(self, other, confidence=None): + def update(self, other, confidence=None, raw=None): dict.update(self, other) if isinstance(other, Guess): for prop in other: self._confidence[prop] = other.confidence(prop) + self._raw[prop] = other.raw(prop) if confidence is not None: for prop in other: self._confidence[prop] = confidence + if raw is not None: + for prop in other: + self._raw[prop] = raw + def update_highest_confidence(self, other): """Update this guess with the values from the given one. In case there is property present in both, only the one with the highest one @@ -110,6 +135,7 @@ class Guess(UnicodeMixin, dict): continue self[prop] = other[prop] self._confidence[prop] = other.confidence(prop) + self._raw[prop] = other.raw(prop) def choose_int(g1, g2): @@ -181,7 +207,7 @@ def choose_string(g1, g2): elif v1l in v2l: return (v1, combined_prob) - # in case of conflict, return the one with highest priority + # in case of conflict, return the one with highest confidence else: if c1 > c2: return (v1, c1 - c2) @@ -288,7 +314,8 @@ def merge_all(guesses, append=None): result.set(prop, result.get(prop, []) + [g[prop]], # TODO: what to do with confidence here? maybe an # arithmetic mean... - confidence=g.confidence(prop)) + confidence=g.confidence(prop), + raw=g.raw(prop)) del g[prop] diff --git a/libs/guessit/language.py b/libs/guessit/language.py index 2714c6e0..4d22cf05 100755 --- a/libs/guessit/language.py +++ b/libs/guessit/language.py @@ -296,7 +296,7 @@ UNDETERMINED = Language('und') ALL_LANGUAGES = frozenset(Language(lng) for lng in lng_all_names) - frozenset([UNDETERMINED]) ALL_LANGUAGES_NAMES = lng_all_names -def search_language(string, lang_filter=None): +def search_language(string, lang_filter=None, skip=None): """Looks for language patterns, and if found return the language object, its group span and an associated confidence. @@ -345,6 +345,16 @@ def search_language(string, lang_filter=None): if pos != -1: end = pos + len(lang) + + # skip if span in in skip list + while skip and (pos - 1, end - 1) in skip: + pos = slow.find(lang, end) + if pos == -1: + continue + end = pos + len(lang) + if pos == -1: + continue + # make sure our word is always surrounded by separators if slow[pos - 1] not in sep or slow[end] not in sep: continue diff --git a/libs/guessit/matcher.py b/libs/guessit/matcher.py index 43378192..1984c01c 100755 --- a/libs/guessit/matcher.py +++ b/libs/guessit/matcher.py @@ -21,14 +21,14 @@ from __future__ import unicode_literals from guessit import PY3, u, base_text_type from guessit.matchtree import MatchTree -from guessit.textutils import normalize_unicode +from guessit.textutils import normalize_unicode, clean_string import logging log = logging.getLogger(__name__) class IterativeMatcher(object): - def __init__(self, filename, filetype='autodetect', opts=None): + def __init__(self, filename, filetype='autodetect', opts=None, transfo_opts=None): """An iterative matcher tries to match different patterns that appear in the filename. @@ -38,7 +38,8 @@ class IterativeMatcher(object): a movie. The recognized 'filetype' values are: - [ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ] + [ autodetect, subtitle, info, movie, moviesubtitle, movieinfo, episode, + episodesubtitle, episodeinfo ] The IterativeMatcher works mainly in 2 steps: @@ -61,15 +62,20 @@ class IterativeMatcher(object): it corresponds to a video codec, denoted by the letter'v' in the 4th line. (for more info, see guess.matchtree.to_string) + Second, it tries to merge all this information into a single object + containing all the found properties, and does some (basic) conflict + resolution when they arise. - Second, it tries to merge all this information into a single object - containing all the found properties, and does some (basic) conflict - resolution when they arise. + + When you create the Matcher, you can pass it: + - a list 'opts' of option names, that act as global flags + - a dict 'transfo_opts' of { transfo_name: (transfo_args, transfo_kwargs) } + with which to call the transfo.process() function. """ - valid_filetypes = ('autodetect', 'subtitle', 'video', - 'movie', 'moviesubtitle', - 'episode', 'episodesubtitle') + valid_filetypes = ('autodetect', 'subtitle', 'info', 'video', + 'movie', 'moviesubtitle', 'movieinfo', + 'episode', 'episodesubtitle', 'episodeinfo') if filetype not in valid_filetypes: raise ValueError("filetype needs to be one of %s" % valid_filetypes) if not PY3 and not isinstance(filename, unicode): @@ -80,10 +86,22 @@ class IterativeMatcher(object): if opts is None: opts = [] - elif isinstance(opts, base_text_type): - opts = opts.split() + if not isinstance(opts, list): + raise ValueError('opts must be a list of option names! Received: type=%s val=%s', + type(opts), opts) + + if transfo_opts is None: + transfo_opts = {} + if not isinstance(transfo_opts, dict): + raise ValueError('transfo_opts must be a dict of { transfo_name: (args, kwargs) }. '+ + 'Received: type=%s val=%s', type(transfo_opts), transfo_opts) self.match_tree = MatchTree(filename) + + # sanity check: make sure we don't process a (mostly) empty string + if clean_string(filename) == '': + return + mtree = self.match_tree mtree.guess.set('type', filetype, confidence=1.0) @@ -91,7 +109,11 @@ class IterativeMatcher(object): transfo = __import__('guessit.transfo.' + transfo_name, globals=globals(), locals=locals(), fromlist=['process'], level=0) - transfo.process(mtree, *args, **kwargs) + default_args, default_kwargs = transfo_opts.get(transfo_name, ((), {})) + all_args = args or default_args + all_kwargs = dict(default_kwargs) + all_kwargs.update(kwargs) # keep all kwargs merged together + transfo.process(mtree, *all_args, **all_kwargs) # 1- first split our path into dirs + basename + ext apply_transfo('split_path_components') @@ -111,7 +133,7 @@ class IterativeMatcher(object): # - language before episodes_rexps # - properties before language (eg: he-aac vs hebrew) # - release_group before properties (eg: XviD-?? vs xvid) - if mtree.guess['type'] in ('episode', 'episodesubtitle'): + if mtree.guess['type'] in ('episode', 'episodesubtitle', 'episodeinfo'): strategy = [ 'guess_date', 'guess_website', 'guess_release_group', 'guess_properties', 'guess_language', 'guess_video_rexps', @@ -124,6 +146,7 @@ class IterativeMatcher(object): if 'nolanguage' in opts: strategy.remove('guess_language') + for name in strategy: apply_transfo(name) @@ -143,7 +166,7 @@ class IterativeMatcher(object): # 5- try to identify the remaining unknown groups by looking at their # position relative to other known elements - if mtree.guess['type'] in ('episode', 'episodesubtitle'): + if mtree.guess['type'] in ('episode', 'episodesubtitle', 'episodeinfo'): apply_transfo('guess_episode_info_from_position') else: apply_transfo('guess_movie_title_from_position') diff --git a/libs/guessit/patterns.py b/libs/guessit/patterns.py index ed3982b9..f803a11c 100755 --- a/libs/guessit/patterns.py +++ b/libs/guessit/patterns.py @@ -25,6 +25,8 @@ import re subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa' ] +info_exts = [ 'nfo' ] + video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2', 'mka', 'mkv', 'mov', 'mp4', 'mp4a', 'mpeg', 'mpg', 'ogg', 'ogm', 'ogv', 'qt', 'ra', 'ram', 'rm', 'ts', 'wav', 'webm', 'wma', 'wmv'] @@ -32,7 +34,7 @@ video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2', group_delimiters = [ '()', '[]', '{}' ] # separator character regexp -sep = r'[][)(}{+ /\._-]' # regexp art, hehe :D +sep = r'[][,)(}{+ /\._-]' # regexp art, hehe :D # character used to represent a deleted char (when matching groups) deleted = '_' @@ -49,7 +51,7 @@ episode_rexps = [ # ... Season 2 ... #(r'[Ss](?P[0-9]{1,3})[^0-9]?(?P(?:-?[xX-][0-9]{1,3})+)[^0-9]', 1.0, (0, -1)), # ... 2x13 ... - (r'[^0-9](?P[0-9]{1,2})[^0-9]?(?P(?:-?[xX][0-9]{1,3})+)[^0-9]', 1.0, (1, -1)), + (r'[^0-9](?P[0-9]{1,2})[^0-9 .-]?(?P(?:-?[xX][0-9]{1,3})+)[^0-9]', 1.0, (1, -1)), # ... s02 ... #(sep + r's(?P[0-9]{1,2})' + sep, 0.6, (1, -1)), @@ -122,9 +124,12 @@ prop_multi = { 'format': { 'DVD': [ 'DVD', 'DVD-Rip', 'VIDEO-TS', 'DVDivX' ], 'VHS': [ 'VHS' ], 'WEB-DL': [ 'WEB-DL' ] }, + 'is3D': { True: [ '3D' ] }, + 'screenSize': { '480p': [ '480[pi]?' ], '720p': [ '720[pi]?' ], - '1080p': [ '1080[pi]?' ] }, + '1080i': [ '1080i' ], + '1080p': [ '1080p', '1080[^i]' ] }, 'videoCodec': { 'XviD': [ 'Xvid' ], 'DivX': [ 'DVDivX', 'DivX' ], @@ -140,7 +145,7 @@ prop_multi = { 'format': { 'DVD': [ 'DVD', 'DVD-Rip', 'VIDEO-TS', 'DVDivX' ], 'DTS': [ 'DTS' ], 'AAC': [ 'He-AAC', 'AAC-He', 'AAC' ] }, - 'audioChannels': { '5.1': [ r'5\.1', 'DD5[\._ ]1', '5ch' ] }, + 'audioChannels': { '5.1': [ r'5\.1', 'DD5[._ ]1', '5ch' ] }, 'episodeFormat': { 'Minisode': [ 'Minisodes?' ] } @@ -170,7 +175,7 @@ prop_single = { 'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', r'\[XCT\]', 'iNT', 'P } _dash = '-' -_psep = '[-\. _]?' +_psep = '[-. _]?' def _to_rexp(prop): return re.compile(prop.replace(_dash, _psep), re.IGNORECASE) @@ -237,8 +242,9 @@ def canonical_form(string): def compute_canonical_form(property_name, value): """Return the canonical form of a property given its type if it is a valid one, None otherwise.""" - for canonical_form, rexps in properties_rexps[property_name].items(): - for rexp in rexps: - if rexp.match(value): - return canonical_form + if isinstance(value, basestring): + for canonical_form, rexps in properties_rexps[property_name].items(): + for rexp in rexps: + if rexp.match(value): + return canonical_form return None diff --git a/libs/guessit/slogging.py b/libs/guessit/slogging.py index 75e261cf..39591a20 100755 --- a/libs/guessit/slogging.py +++ b/libs/guessit/slogging.py @@ -31,14 +31,15 @@ RED_FONT = "\x1B[0;31m" RESET_FONT = "\x1B[0m" -def setupLogging(colored=True, with_time=False, with_thread=False, filename=None): +def setupLogging(colored=True, with_time=False, with_thread=False, filename=None, with_lineno=False): """Set up a nice colored logger as the main application logger.""" class SimpleFormatter(logging.Formatter): def __init__(self, with_time, with_thread): self.fmt = (('%(asctime)s ' if with_time else '') + '%(levelname)-8s ' + - '[%(name)s:%(funcName)s]' + + '[%(name)s:%(funcName)s' + + (':%(lineno)s' if with_lineno else '') + ']' + ('[%(threadName)s]' if with_thread else '') + ' -- %(message)s') logging.Formatter.__init__(self, self.fmt) @@ -47,7 +48,8 @@ def setupLogging(colored=True, with_time=False, with_thread=False, filename=None def __init__(self, with_time, with_thread): self.fmt = (('%(asctime)s ' if with_time else '') + '-CC-%(levelname)-8s ' + - BLUE_FONT + '[%(name)s:%(funcName)s]' + + BLUE_FONT + '[%(name)s:%(funcName)s' + + (':%(lineno)s' if with_lineno else '') + ']' + RESET_FONT + ('[%(threadName)s]' if with_thread else '') + ' -- %(message)s') diff --git a/libs/guessit/textutils.py b/libs/guessit/textutils.py index f195e2b7..ae9d28c3 100755 --- a/libs/guessit/textutils.py +++ b/libs/guessit/textutils.py @@ -43,10 +43,13 @@ def strip_brackets(s): return s -def clean_string(s): - for c in sep[:-2]: # do not remove dashes ('-') - s = s.replace(c, ' ') - parts = s.split() +def clean_string(st): + for c in sep: + # do not remove certain chars + if c in ['-', ',']: + continue + st = st.replace(c, ' ') + parts = st.split() result = ' '.join(p for p in parts if p != '') # now also remove dashes on the outer part of the string diff --git a/libs/guessit/transfo/__init__.py b/libs/guessit/transfo/__init__.py index 820690a7..a28aa988 100755 --- a/libs/guessit/transfo/__init__.py +++ b/libs/guessit/transfo/__init__.py @@ -28,7 +28,7 @@ log = logging.getLogger(__name__) def found_property(node, name, confidence): - node.guess = Guess({name: node.clean_value}, confidence=confidence) + node.guess = Guess({name: node.clean_value}, confidence=confidence, raw=node.value) log.debug('Found with confidence %.2f: %s' % (confidence, node.guess)) @@ -52,11 +52,17 @@ def format_guess(guess): def find_and_split_node(node, strategy, logger): string = ' %s ' % node.value # add sentinels - for matcher, confidence in strategy: + for matcher, confidence, args, kwargs in strategy: + all_args = [string] if getattr(matcher, 'use_node', False): - result, span = matcher(string, node) + all_args.append(node) + if args: + all_args.append(args) + + if kwargs: + result, span = matcher(*all_args, **kwargs) else: - result, span = matcher(string) + result, span = matcher(*all_args) if result: # readjust span to compensate for sentinels @@ -69,7 +75,7 @@ def find_and_split_node(node, strategy, logger): if confidence is None: confidence = 1.0 - guess = format_guess(Guess(result, confidence=confidence)) + guess = format_guess(Guess(result, confidence=confidence, raw=string[span[0] + 1:span[1] + 1])) msg = 'Found with confidence %.2f: %s' % (confidence, guess) (logger or log).debug(msg) @@ -84,10 +90,12 @@ def find_and_split_node(node, strategy, logger): class SingleNodeGuesser(object): - def __init__(self, guess_func, confidence, logger=None): + def __init__(self, guess_func, confidence, logger, *args, **kwargs): self.guess_func = guess_func self.confidence = confidence self.logger = logger + self.args = args + self.kwargs = kwargs def process(self, mtree): # strategy is a list of pairs (guesser, confidence) @@ -95,7 +103,7 @@ class SingleNodeGuesser(object): # it will override it, otherwise it will leave the guess confidence # - if the guesser returns a simple dict as a guess and confidence is # specified, it will use it, or 1.0 otherwise - strategy = [ (self.guess_func, self.confidence) ] + strategy = [ (self.guess_func, self.confidence, self.args, self.kwargs) ] for node in mtree.unidentified_leaves(): find_and_split_node(node, strategy, self.logger) diff --git a/libs/guessit/transfo/guess_country.py b/libs/guessit/transfo/guess_country.py index 1d690698..aadb84f7 100755 --- a/libs/guessit/transfo/guess_country.py +++ b/libs/guessit/transfo/guess_country.py @@ -45,4 +45,4 @@ def process(mtree): except ValueError: continue - node.guess = Guess(country=country, confidence=1.0) + node.guess = Guess(country=country, confidence=1.0, raw=c) diff --git a/libs/guessit/transfo/guess_episodes_rexps.py b/libs/guessit/transfo/guess_episodes_rexps.py index 29562be2..30c2ca2f 100755 --- a/libs/guessit/transfo/guess_episodes_rexps.py +++ b/libs/guessit/transfo/guess_episodes_rexps.py @@ -40,27 +40,22 @@ def guess_episodes_rexps(string): for rexp, confidence, span_adjust in episode_rexps: match = re.search(rexp, string, re.IGNORECASE) if match: - guess = Guess(match.groupdict(), confidence=confidence) - span = (match.start() + span_adjust[0], + span = (match.start() + span_adjust[0], match.end() + span_adjust[1]) - - # episodes which have a season > 30 are most likely errors - # (Simpsons is at 24!) - if int(guess.get('season', 0)) > 30: - continue + guess = Guess(match.groupdict(), confidence=confidence, raw=string[span[0]:span[1]]) # decide whether we have only a single episode number or an # episode list if guess.get('episodeNumber'): eplist = number_list(guess['episodeNumber']) - guess.set('episodeNumber', eplist[0], confidence=confidence) + guess.set('episodeNumber', eplist[0], confidence=confidence, raw=string[span[0]:span[1]]) if len(eplist) > 1: - guess.set('episodeList', eplist, confidence=confidence) + guess.set('episodeList', eplist, confidence=confidence, raw=string[span[0]:span[1]]) if guess.get('bonusNumber'): eplist = number_list(guess['bonusNumber']) - guess.set('bonusNumber', eplist[0], confidence=confidence) + guess.set('bonusNumber', eplist[0], confidence=confidence, raw=string[span[0]:span[1]]) return guess, span diff --git a/libs/guessit/transfo/guess_filetype.py b/libs/guessit/transfo/guess_filetype.py index 4d98d016..4279c0b0 100755 --- a/libs/guessit/transfo/guess_filetype.py +++ b/libs/guessit/transfo/guess_filetype.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals from guessit import Guess -from guessit.patterns import (subtitle_exts, video_exts, episode_rexps, +from guessit.patterns import (subtitle_exts, info_exts, video_exts, episode_rexps, find_properties, compute_canonical_form) from guessit.date import valid_year from guessit.textutils import clean_string @@ -53,12 +53,16 @@ def guess_filetype(mtree, filetype): filetype_container[0] = 'episode' elif filetype_container[0] == 'subtitle': filetype_container[0] = 'episodesubtitle' + elif filetype_container[0] == 'info': + filetype_container[0] = 'episodeinfo' def upgrade_movie(): if filetype_container[0] == 'video': filetype_container[0] = 'movie' elif filetype_container[0] == 'subtitle': filetype_container[0] = 'moviesubtitle' + elif filetype_container[0] == 'info': + filetype_container[0] = 'movieinfo' def upgrade_subtitle(): if 'movie' in filetype_container[0]: @@ -68,6 +72,14 @@ def guess_filetype(mtree, filetype): else: filetype_container[0] = 'subtitle' + def upgrade_info(): + if 'movie' in filetype_container[0]: + filetype_container[0] = 'movieinfo' + elif 'episode' in filetype_container[0]: + filetype_container[0] = 'episodeinfo' + else: + filetype_container[0] = 'info' + def upgrade(type='unknown'): if filetype_container[0] == 'autodetect': filetype_container[0] = type @@ -78,6 +90,9 @@ def guess_filetype(mtree, filetype): if fileext in subtitle_exts: upgrade_subtitle() other = { 'container': fileext } + elif fileext in info_exts: + upgrade_info() + other = { 'container': fileext } elif fileext in video_exts: upgrade(type='video') other = { 'container': fileext } @@ -104,17 +119,20 @@ def guess_filetype(mtree, filetype): fname = clean_string(filename).lower() for m in MOVIES: if m in fname: + log.debug('Found in exception list of movies -> type = movie') upgrade_movie() for s in SERIES: if s in fname: + log.debug('Found in exception list of series -> type = episode') upgrade_episode() # now look whether there are some specific hints for episode vs movie - if filetype_container[0] in ('video', 'subtitle'): + if filetype_container[0] in ('video', 'subtitle', 'info'): # if we have an episode_rexp (eg: s02e13), it is an episode for rexp, _, _ in episode_rexps: match = re.search(rexp, filename, re.IGNORECASE) if match: + log.debug('Found matching regexp: "%s" (string = "%s") -> type = episode', rexp, match.group()) upgrade_episode() break @@ -133,24 +151,29 @@ def guess_filetype(mtree, filetype): possible = False if possible: + log.debug('Found possible episode number: %s (from string "%s") -> type = episode', epnumber, match.group()) upgrade_episode() # if we have certain properties characteristic of episodes, it is an ep for prop, value, _, _ in find_properties(filename): log.debug('prop: %s = %s' % (prop, value)) if prop == 'episodeFormat': + log.debug('Found characteristic property of episodes: %s = "%s"', prop, value) upgrade_episode() break elif compute_canonical_form('format', value) == 'DVB': + log.debug('Found characteristic property of episodes: %s = "%s"', prop, value) upgrade_episode() break # origin-specific type if 'tvu.org.ru' in filename: + log.debug('Found characteristic property of episodes: %s = "%s"', prop, value) upgrade_episode() # if no episode info found, assume it's a movie + log.debug('Nothing characteristic found, assuming type = movie') upgrade_movie() filetype = filetype_container[0] diff --git a/libs/guessit/transfo/guess_language.py b/libs/guessit/transfo/guess_language.py index 86c1cf55..648a06b1 100755 --- a/libs/guessit/transfo/guess_language.py +++ b/libs/guessit/transfo/guess_language.py @@ -22,22 +22,34 @@ from __future__ import unicode_literals from guessit import Guess from guessit.transfo import SingleNodeGuesser from guessit.language import search_language -from guessit.textutils import clean_string, find_words import logging log = logging.getLogger(__name__) -def guess_language(string): - language, span, confidence = search_language(string) +def guess_language(string, node, skip=None): + if skip: + relative_skip = [] + for entry in skip: + node_idx = entry['node_idx'] + span = entry['span'] + if node_idx == node.node_idx[:len(node_idx)]: + relative_span = (span[0] - node.offset + 1, span[1] - node.offset + 1) + relative_skip.append(relative_span) + skip = relative_skip + + language, span, confidence = search_language(string, skip=skip) if language: return (Guess({'language': language}, - confidence=confidence), + confidence=confidence, + raw= string[span[0]:span[1]]), span) return None, None +guess_language.use_node = True -def process(mtree): - SingleNodeGuesser(guess_language, None, log).process(mtree) + +def process(mtree, *args, **kwargs): + SingleNodeGuesser(guess_language, None, log, *args, **kwargs).process(mtree) # Note: 'language' is promoted to 'subtitleLanguage' in the post_process transfo diff --git a/libs/guessit/transfo/guess_movie_title_from_position.py b/libs/guessit/transfo/guess_movie_title_from_position.py index d2e2deb2..bcb42b45 100755 --- a/libs/guessit/transfo/guess_movie_title_from_position.py +++ b/libs/guessit/transfo/guess_movie_title_from_position.py @@ -29,7 +29,8 @@ log = logging.getLogger(__name__) def process(mtree): def found_property(node, name, value, confidence): node.guess = Guess({ name: value }, - confidence=confidence) + confidence=confidence, + raw=value) log.debug('Found with confidence %.2f: %s' % (confidence, node.guess)) def found_title(node, confidence): diff --git a/libs/guessit/transfo/guess_video_rexps.py b/libs/guessit/transfo/guess_video_rexps.py index 8ae9e6c6..1b511f15 100755 --- a/libs/guessit/transfo/guess_video_rexps.py +++ b/libs/guessit/transfo/guess_video_rexps.py @@ -38,9 +38,10 @@ def guess_video_rexps(string): # the soonest that we can catch it) if metadata.get('cdNumberTotal', -1) is None: del metadata['cdNumberTotal'] - return (Guess(metadata, confidence=confidence), - (match.start() + span_adjust[0], - match.end() + span_adjust[1] - 2)) + span = (match.start() + span_adjust[0], + match.end() + span_adjust[1] - 2) + return (Guess(metadata, confidence=confidence, raw=string[span[0]:span[1]]), + span) return None, None diff --git a/libs/guessit/transfo/guess_weak_episodes_rexps.py b/libs/guessit/transfo/guess_weak_episodes_rexps.py index 8436ade8..18306b43 100755 --- a/libs/guessit/transfo/guess_weak_episodes_rexps.py +++ b/libs/guessit/transfo/guess_weak_episodes_rexps.py @@ -48,9 +48,9 @@ def guess_weak_episodes_rexps(string, node): continue return Guess({ 'season': season, 'episodeNumber': epnum }, - confidence=0.6), span + confidence=0.6, raw=string[span[0]:span[1]]), span else: - return Guess(metadata, confidence=0.3), span + return Guess(metadata, confidence=0.3, raw=string[span[0]:span[1]]), span return None, None diff --git a/libs/html5lib/__init__.py b/libs/html5lib/__init__.py index 16537aad..19a4b7d6 100644 --- a/libs/html5lib/__init__.py +++ b/libs/html5lib/__init__.py @@ -1,4 +1,4 @@ -""" +""" HTML parsing library based on the WHATWG "HTML5" specification. The parser is designed to be compatible with existing HTML found in the wild and implements well-defined error recovery that @@ -8,10 +8,16 @@ Example usage: import html5lib f = open("my_document.html") -tree = html5lib.parse(f) +tree = html5lib.parse(f) """ -__version__ = "0.95-dev" -from html5parser import HTMLParser, parse, parseFragment -from treebuilders import getTreeBuilder -from treewalkers import getTreeWalker -from serializer import serialize + +from __future__ import absolute_import, division, unicode_literals + +from .html5parser import HTMLParser, parse, parseFragment +from .treebuilders import getTreeBuilder +from .treewalkers import getTreeWalker +from .serializer import serialize + +__all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder", + "getTreeWalker", "serialize"] +__version__ = "0.999" diff --git a/libs/html5lib/constants.py b/libs/html5lib/constants.py index b533018e..e7089846 100644 --- a/libs/html5lib/constants.py +++ b/libs/html5lib/constants.py @@ -1,302 +1,301 @@ -import string, gettext -_ = gettext.gettext +from __future__ import absolute_import, division, unicode_literals -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset +import string +import gettext +_ = gettext.gettext EOF = None E = { - "null-character": - _(u"Null character in input stream, replaced with U+FFFD."), - "invalid-codepoint": - _(u"Invalid codepoint in stream."), + "null-character": + _("Null character in input stream, replaced with U+FFFD."), + "invalid-codepoint": + _("Invalid codepoint in stream."), "incorrectly-placed-solidus": - _(u"Solidus (/) incorrectly placed in tag."), + _("Solidus (/) incorrectly placed in tag."), "incorrect-cr-newline-entity": - _(u"Incorrect CR newline entity, replaced with LF."), + _("Incorrect CR newline entity, replaced with LF."), "illegal-windows-1252-entity": - _(u"Entity used with illegal number (windows-1252 reference)."), + _("Entity used with illegal number (windows-1252 reference)."), "cant-convert-numeric-entity": - _(u"Numeric entity couldn't be converted to character " - u"(codepoint U+%(charAsInt)08x)."), + _("Numeric entity couldn't be converted to character " + "(codepoint U+%(charAsInt)08x)."), "illegal-codepoint-for-numeric-entity": - _(u"Numeric entity represents an illegal codepoint: " - u"U+%(charAsInt)08x."), + _("Numeric entity represents an illegal codepoint: " + "U+%(charAsInt)08x."), "numeric-entity-without-semicolon": - _(u"Numeric entity didn't end with ';'."), + _("Numeric entity didn't end with ';'."), "expected-numeric-entity-but-got-eof": - _(u"Numeric entity expected. Got end of file instead."), + _("Numeric entity expected. Got end of file instead."), "expected-numeric-entity": - _(u"Numeric entity expected but none found."), + _("Numeric entity expected but none found."), "named-entity-without-semicolon": - _(u"Named entity didn't end with ';'."), + _("Named entity didn't end with ';'."), "expected-named-entity": - _(u"Named entity expected. Got none."), + _("Named entity expected. Got none."), "attributes-in-end-tag": - _(u"End tag contains unexpected attributes."), + _("End tag contains unexpected attributes."), 'self-closing-flag-on-end-tag': - _(u"End tag contains unexpected self-closing flag."), + _("End tag contains unexpected self-closing flag."), "expected-tag-name-but-got-right-bracket": - _(u"Expected tag name. Got '>' instead."), + _("Expected tag name. Got '>' instead."), "expected-tag-name-but-got-question-mark": - _(u"Expected tag name. Got '?' instead. (HTML doesn't " - u"support processing instructions.)"), + _("Expected tag name. Got '?' instead. (HTML doesn't " + "support processing instructions.)"), "expected-tag-name": - _(u"Expected tag name. Got something else instead"), + _("Expected tag name. Got something else instead"), "expected-closing-tag-but-got-right-bracket": - _(u"Expected closing tag. Got '>' instead. Ignoring ''."), + _("Expected closing tag. Got '>' instead. Ignoring ''."), "expected-closing-tag-but-got-eof": - _(u"Expected closing tag. Unexpected end of file."), + _("Expected closing tag. Unexpected end of file."), "expected-closing-tag-but-got-char": - _(u"Expected closing tag. Unexpected character '%(data)s' found."), + _("Expected closing tag. Unexpected character '%(data)s' found."), "eof-in-tag-name": - _(u"Unexpected end of file in the tag name."), + _("Unexpected end of file in the tag name."), "expected-attribute-name-but-got-eof": - _(u"Unexpected end of file. Expected attribute name instead."), + _("Unexpected end of file. Expected attribute name instead."), "eof-in-attribute-name": - _(u"Unexpected end of file in attribute name."), + _("Unexpected end of file in attribute name."), "invalid-character-in-attribute-name": - _(u"Invalid chracter in attribute name"), + _("Invalid character in attribute name"), "duplicate-attribute": - _(u"Dropped duplicate attribute on tag."), + _("Dropped duplicate attribute on tag."), "expected-end-of-tag-name-but-got-eof": - _(u"Unexpected end of file. Expected = or end of tag."), + _("Unexpected end of file. Expected = or end of tag."), "expected-attribute-value-but-got-eof": - _(u"Unexpected end of file. Expected attribute value."), + _("Unexpected end of file. Expected attribute value."), "expected-attribute-value-but-got-right-bracket": - _(u"Expected attribute value. Got '>' instead."), + _("Expected attribute value. Got '>' instead."), 'equals-in-unquoted-attribute-value': - _(u"Unexpected = in unquoted attribute"), + _("Unexpected = in unquoted attribute"), 'unexpected-character-in-unquoted-attribute-value': - _(u"Unexpected character in unquoted attribute"), + _("Unexpected character in unquoted attribute"), "invalid-character-after-attribute-name": - _(u"Unexpected character after attribute name."), + _("Unexpected character after attribute name."), "unexpected-character-after-attribute-value": - _(u"Unexpected character after attribute value."), + _("Unexpected character after attribute value."), "eof-in-attribute-value-double-quote": - _(u"Unexpected end of file in attribute value (\")."), + _("Unexpected end of file in attribute value (\")."), "eof-in-attribute-value-single-quote": - _(u"Unexpected end of file in attribute value (')."), + _("Unexpected end of file in attribute value (')."), "eof-in-attribute-value-no-quotes": - _(u"Unexpected end of file in attribute value."), + _("Unexpected end of file in attribute value."), "unexpected-EOF-after-solidus-in-tag": - _(u"Unexpected end of file in tag. Expected >"), - "unexpected-character-after-soldius-in-tag": - _(u"Unexpected character after / in tag. Expected >"), + _("Unexpected end of file in tag. Expected >"), + "unexpected-character-after-solidus-in-tag": + _("Unexpected character after / in tag. Expected >"), "expected-dashes-or-doctype": - _(u"Expected '--' or 'DOCTYPE'. Not found."), + _("Expected '--' or 'DOCTYPE'. Not found."), "unexpected-bang-after-double-dash-in-comment": - _(u"Unexpected ! after -- in comment"), + _("Unexpected ! after -- in comment"), "unexpected-space-after-double-dash-in-comment": - _(u"Unexpected space after -- in comment"), + _("Unexpected space after -- in comment"), "incorrect-comment": - _(u"Incorrect comment."), + _("Incorrect comment."), "eof-in-comment": - _(u"Unexpected end of file in comment."), + _("Unexpected end of file in comment."), "eof-in-comment-end-dash": - _(u"Unexpected end of file in comment (-)"), + _("Unexpected end of file in comment (-)"), "unexpected-dash-after-double-dash-in-comment": - _(u"Unexpected '-' after '--' found in comment."), + _("Unexpected '-' after '--' found in comment."), "eof-in-comment-double-dash": - _(u"Unexpected end of file in comment (--)."), + _("Unexpected end of file in comment (--)."), "eof-in-comment-end-space-state": - _(u"Unexpected end of file in comment."), + _("Unexpected end of file in comment."), "eof-in-comment-end-bang-state": - _(u"Unexpected end of file in comment."), + _("Unexpected end of file in comment."), "unexpected-char-in-comment": - _(u"Unexpected character in comment found."), + _("Unexpected character in comment found."), "need-space-after-doctype": - _(u"No space after literal string 'DOCTYPE'."), + _("No space after literal string 'DOCTYPE'."), "expected-doctype-name-but-got-right-bracket": - _(u"Unexpected > character. Expected DOCTYPE name."), + _("Unexpected > character. Expected DOCTYPE name."), "expected-doctype-name-but-got-eof": - _(u"Unexpected end of file. Expected DOCTYPE name."), + _("Unexpected end of file. Expected DOCTYPE name."), "eof-in-doctype-name": - _(u"Unexpected end of file in DOCTYPE name."), + _("Unexpected end of file in DOCTYPE name."), "eof-in-doctype": - _(u"Unexpected end of file in DOCTYPE."), + _("Unexpected end of file in DOCTYPE."), "expected-space-or-right-bracket-in-doctype": - _(u"Expected space or '>'. Got '%(data)s'"), + _("Expected space or '>'. Got '%(data)s'"), "unexpected-end-of-doctype": - _(u"Unexpected end of DOCTYPE."), + _("Unexpected end of DOCTYPE."), "unexpected-char-in-doctype": - _(u"Unexpected character in DOCTYPE."), + _("Unexpected character in DOCTYPE."), "eof-in-innerhtml": - _(u"XXX innerHTML EOF"), + _("XXX innerHTML EOF"), "unexpected-doctype": - _(u"Unexpected DOCTYPE. Ignored."), + _("Unexpected DOCTYPE. Ignored."), "non-html-root": - _(u"html needs to be the first start tag."), + _("html needs to be the first start tag."), "expected-doctype-but-got-eof": - _(u"Unexpected End of file. Expected DOCTYPE."), + _("Unexpected End of file. Expected DOCTYPE."), "unknown-doctype": - _(u"Erroneous DOCTYPE."), + _("Erroneous DOCTYPE."), "expected-doctype-but-got-chars": - _(u"Unexpected non-space characters. Expected DOCTYPE."), + _("Unexpected non-space characters. Expected DOCTYPE."), "expected-doctype-but-got-start-tag": - _(u"Unexpected start tag (%(name)s). Expected DOCTYPE."), + _("Unexpected start tag (%(name)s). Expected DOCTYPE."), "expected-doctype-but-got-end-tag": - _(u"Unexpected end tag (%(name)s). Expected DOCTYPE."), + _("Unexpected end tag (%(name)s). Expected DOCTYPE."), "end-tag-after-implied-root": - _(u"Unexpected end tag (%(name)s) after the (implied) root element."), + _("Unexpected end tag (%(name)s) after the (implied) root element."), "expected-named-closing-tag-but-got-eof": - _(u"Unexpected end of file. Expected end tag (%(name)s)."), + _("Unexpected end of file. Expected end tag (%(name)s)."), "two-heads-are-not-better-than-one": - _(u"Unexpected start tag head in existing head. Ignored."), + _("Unexpected start tag head in existing head. Ignored."), "unexpected-end-tag": - _(u"Unexpected end tag (%(name)s). Ignored."), + _("Unexpected end tag (%(name)s). Ignored."), "unexpected-start-tag-out-of-my-head": - _(u"Unexpected start tag (%(name)s) that can be in head. Moved."), + _("Unexpected start tag (%(name)s) that can be in head. Moved."), "unexpected-start-tag": - _(u"Unexpected start tag (%(name)s)."), + _("Unexpected start tag (%(name)s)."), "missing-end-tag": - _(u"Missing end tag (%(name)s)."), + _("Missing end tag (%(name)s)."), "missing-end-tags": - _(u"Missing end tags (%(name)s)."), + _("Missing end tags (%(name)s)."), "unexpected-start-tag-implies-end-tag": - _(u"Unexpected start tag (%(startName)s) " - u"implies end tag (%(endName)s)."), + _("Unexpected start tag (%(startName)s) " + "implies end tag (%(endName)s)."), "unexpected-start-tag-treated-as": - _(u"Unexpected start tag (%(originalName)s). Treated as %(newName)s."), + _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."), "deprecated-tag": - _(u"Unexpected start tag %(name)s. Don't use it!"), + _("Unexpected start tag %(name)s. Don't use it!"), "unexpected-start-tag-ignored": - _(u"Unexpected start tag %(name)s. Ignored."), + _("Unexpected start tag %(name)s. Ignored."), "expected-one-end-tag-but-got-another": - _(u"Unexpected end tag (%(gotName)s). " - u"Missing end tag (%(expectedName)s)."), + _("Unexpected end tag (%(gotName)s). " + "Missing end tag (%(expectedName)s)."), "end-tag-too-early": - _(u"End tag (%(name)s) seen too early. Expected other end tag."), + _("End tag (%(name)s) seen too early. Expected other end tag."), "end-tag-too-early-named": - _(u"Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."), + _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."), "end-tag-too-early-ignored": - _(u"End tag (%(name)s) seen too early. Ignored."), + _("End tag (%(name)s) seen too early. Ignored."), "adoption-agency-1.1": - _(u"End tag (%(name)s) violates step 1, " - u"paragraph 1 of the adoption agency algorithm."), + _("End tag (%(name)s) violates step 1, " + "paragraph 1 of the adoption agency algorithm."), "adoption-agency-1.2": - _(u"End tag (%(name)s) violates step 1, " - u"paragraph 2 of the adoption agency algorithm."), + _("End tag (%(name)s) violates step 1, " + "paragraph 2 of the adoption agency algorithm."), "adoption-agency-1.3": - _(u"End tag (%(name)s) violates step 1, " - u"paragraph 3 of the adoption agency algorithm."), + _("End tag (%(name)s) violates step 1, " + "paragraph 3 of the adoption agency algorithm."), + "adoption-agency-4.4": + _("End tag (%(name)s) violates step 4, " + "paragraph 4 of the adoption agency algorithm."), "unexpected-end-tag-treated-as": - _(u"Unexpected end tag (%(originalName)s). Treated as %(newName)s."), + _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."), "no-end-tag": - _(u"This element (%(name)s) has no end tag."), + _("This element (%(name)s) has no end tag."), "unexpected-implied-end-tag-in-table": - _(u"Unexpected implied end tag (%(name)s) in the table phase."), + _("Unexpected implied end tag (%(name)s) in the table phase."), "unexpected-implied-end-tag-in-table-body": - _(u"Unexpected implied end tag (%(name)s) in the table body phase."), + _("Unexpected implied end tag (%(name)s) in the table body phase."), "unexpected-char-implies-table-voodoo": - _(u"Unexpected non-space characters in " - u"table context caused voodoo mode."), + _("Unexpected non-space characters in " + "table context caused voodoo mode."), "unexpected-hidden-input-in-table": - _(u"Unexpected input with type hidden in table context."), + _("Unexpected input with type hidden in table context."), "unexpected-form-in-table": - _(u"Unexpected form in table context."), + _("Unexpected form in table context."), "unexpected-start-tag-implies-table-voodoo": - _(u"Unexpected start tag (%(name)s) in " - u"table context caused voodoo mode."), + _("Unexpected start tag (%(name)s) in " + "table context caused voodoo mode."), "unexpected-end-tag-implies-table-voodoo": - _(u"Unexpected end tag (%(name)s) in " - u"table context caused voodoo mode."), + _("Unexpected end tag (%(name)s) in " + "table context caused voodoo mode."), "unexpected-cell-in-table-body": - _(u"Unexpected table cell start tag (%(name)s) " - u"in the table body phase."), + _("Unexpected table cell start tag (%(name)s) " + "in the table body phase."), "unexpected-cell-end-tag": - _(u"Got table cell end tag (%(name)s) " - u"while required end tags are missing."), + _("Got table cell end tag (%(name)s) " + "while required end tags are missing."), "unexpected-end-tag-in-table-body": - _(u"Unexpected end tag (%(name)s) in the table body phase. Ignored."), + _("Unexpected end tag (%(name)s) in the table body phase. Ignored."), "unexpected-implied-end-tag-in-table-row": - _(u"Unexpected implied end tag (%(name)s) in the table row phase."), + _("Unexpected implied end tag (%(name)s) in the table row phase."), "unexpected-end-tag-in-table-row": - _(u"Unexpected end tag (%(name)s) in the table row phase. Ignored."), + _("Unexpected end tag (%(name)s) in the table row phase. Ignored."), "unexpected-select-in-select": - _(u"Unexpected select start tag in the select phase " - u"treated as select end tag."), + _("Unexpected select start tag in the select phase " + "treated as select end tag."), "unexpected-input-in-select": - _(u"Unexpected input start tag in the select phase."), + _("Unexpected input start tag in the select phase."), "unexpected-start-tag-in-select": - _(u"Unexpected start tag token (%(name)s in the select phase. " - u"Ignored."), + _("Unexpected start tag token (%(name)s in the select phase. " + "Ignored."), "unexpected-end-tag-in-select": - _(u"Unexpected end tag (%(name)s) in the select phase. Ignored."), + _("Unexpected end tag (%(name)s) in the select phase. Ignored."), "unexpected-table-element-start-tag-in-select-in-table": - _(u"Unexpected table element start tag (%(name)s) in the select in table phase."), + _("Unexpected table element start tag (%(name)s) in the select in table phase."), "unexpected-table-element-end-tag-in-select-in-table": - _(u"Unexpected table element end tag (%(name)s) in the select in table phase."), + _("Unexpected table element end tag (%(name)s) in the select in table phase."), "unexpected-char-after-body": - _(u"Unexpected non-space characters in the after body phase."), + _("Unexpected non-space characters in the after body phase."), "unexpected-start-tag-after-body": - _(u"Unexpected start tag token (%(name)s)" - u" in the after body phase."), + _("Unexpected start tag token (%(name)s)" + " in the after body phase."), "unexpected-end-tag-after-body": - _(u"Unexpected end tag token (%(name)s)" - u" in the after body phase."), + _("Unexpected end tag token (%(name)s)" + " in the after body phase."), "unexpected-char-in-frameset": - _(u"Unepxected characters in the frameset phase. Characters ignored."), + _("Unexpected characters in the frameset phase. Characters ignored."), "unexpected-start-tag-in-frameset": - _(u"Unexpected start tag token (%(name)s)" - u" in the frameset phase. Ignored."), + _("Unexpected start tag token (%(name)s)" + " in the frameset phase. Ignored."), "unexpected-frameset-in-frameset-innerhtml": - _(u"Unexpected end tag token (frameset) " - u"in the frameset phase (innerHTML)."), + _("Unexpected end tag token (frameset) " + "in the frameset phase (innerHTML)."), "unexpected-end-tag-in-frameset": - _(u"Unexpected end tag token (%(name)s)" - u" in the frameset phase. Ignored."), + _("Unexpected end tag token (%(name)s)" + " in the frameset phase. Ignored."), "unexpected-char-after-frameset": - _(u"Unexpected non-space characters in the " - u"after frameset phase. Ignored."), + _("Unexpected non-space characters in the " + "after frameset phase. Ignored."), "unexpected-start-tag-after-frameset": - _(u"Unexpected start tag (%(name)s)" - u" in the after frameset phase. Ignored."), + _("Unexpected start tag (%(name)s)" + " in the after frameset phase. Ignored."), "unexpected-end-tag-after-frameset": - _(u"Unexpected end tag (%(name)s)" - u" in the after frameset phase. Ignored."), + _("Unexpected end tag (%(name)s)" + " in the after frameset phase. Ignored."), "unexpected-end-tag-after-body-innerhtml": - _(u"Unexpected end tag after body(innerHtml)"), + _("Unexpected end tag after body(innerHtml)"), "expected-eof-but-got-char": - _(u"Unexpected non-space characters. Expected end of file."), + _("Unexpected non-space characters. Expected end of file."), "expected-eof-but-got-start-tag": - _(u"Unexpected start tag (%(name)s)" - u". Expected end of file."), + _("Unexpected start tag (%(name)s)" + ". Expected end of file."), "expected-eof-but-got-end-tag": - _(u"Unexpected end tag (%(name)s)" - u". Expected end of file."), + _("Unexpected end tag (%(name)s)" + ". Expected end of file."), "eof-in-table": - _(u"Unexpected end of file. Expected table content."), + _("Unexpected end of file. Expected table content."), "eof-in-select": - _(u"Unexpected end of file. Expected select content."), + _("Unexpected end of file. Expected select content."), "eof-in-frameset": - _(u"Unexpected end of file. Expected frameset content."), + _("Unexpected end of file. Expected frameset content."), "eof-in-script-in-script": - _(u"Unexpected end of file. Expected script content."), + _("Unexpected end of file. Expected script content."), "eof-in-foreign-lands": - _(u"Unexpected end of file. Expected foreign content"), + _("Unexpected end of file. Expected foreign content"), "non-void-element-with-trailing-solidus": - _(u"Trailing solidus not allowed on element %(name)s"), + _("Trailing solidus not allowed on element %(name)s"), "unexpected-html-element-in-foreign-content": - _(u"Element %(name)s not allowed in a non-html context"), + _("Element %(name)s not allowed in a non-html context"), "unexpected-end-tag-before-html": - _(u"Unexpected end tag (%(name)s) before html."), + _("Unexpected end tag (%(name)s) before html."), "XXX-undefined-error": - (u"Undefined error (this sucks and should be fixed)"), + _("Undefined error (this sucks and should be fixed)"), } namespaces = { - "html":"http://www.w3.org/1999/xhtml", - "mathml":"http://www.w3.org/1998/Math/MathML", - "svg":"http://www.w3.org/2000/svg", - "xlink":"http://www.w3.org/1999/xlink", - "xml":"http://www.w3.org/XML/1998/namespace", - "xmlns":"http://www.w3.org/2000/xmlns/" + "html": "http://www.w3.org/1999/xhtml", + "mathml": "http://www.w3.org/1998/Math/MathML", + "svg": "http://www.w3.org/2000/svg", + "xlink": "http://www.w3.org/1999/xlink", + "xml": "http://www.w3.org/XML/1998/namespace", + "xmlns": "http://www.w3.org/2000/xmlns/" } scopingElements = frozenset(( @@ -380,7 +379,7 @@ specialElements = frozenset(( (namespaces["html"], "iframe"), # Note that image is commented out in the spec as "this isn't an # element that can end up on the stack, so it doesn't matter," - (namespaces["html"], "image"), + (namespaces["html"], "image"), (namespaces["html"], "img"), (namespaces["html"], "input"), (namespaces["html"], "isindex"), @@ -434,12 +433,30 @@ mathmlTextIntegrationPointElements = frozenset(( (namespaces["mathml"], "mtext") )) +adjustForeignAttributes = { + "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]), + "xlink:arcrole": ("xlink", "arcrole", namespaces["xlink"]), + "xlink:href": ("xlink", "href", namespaces["xlink"]), + "xlink:role": ("xlink", "role", namespaces["xlink"]), + "xlink:show": ("xlink", "show", namespaces["xlink"]), + "xlink:title": ("xlink", "title", namespaces["xlink"]), + "xlink:type": ("xlink", "type", namespaces["xlink"]), + "xml:base": ("xml", "base", namespaces["xml"]), + "xml:lang": ("xml", "lang", namespaces["xml"]), + "xml:space": ("xml", "space", namespaces["xml"]), + "xmlns": (None, "xmlns", namespaces["xmlns"]), + "xmlns:xlink": ("xmlns", "xlink", namespaces["xmlns"]) +} + +unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in + adjustForeignAttributes.items()]) + spaceCharacters = frozenset(( - u"\t", - u"\n", - u"\u000C", - u" ", - u"\r" + "\t", + "\n", + "\u000C", + " ", + "\r" )) tableInsertModeElements = frozenset(( @@ -456,8 +473,8 @@ asciiLetters = frozenset(string.ascii_letters) digits = frozenset(string.digits) hexDigits = frozenset(string.hexdigits) -asciiUpper2Lower = dict([(ord(c),ord(c.lower())) - for c in string.ascii_uppercase]) +asciiUpper2Lower = dict([(ord(c), ord(c.lower())) + for c in string.ascii_uppercase]) # Heading elements need to be ordered headingElements = ( @@ -503,8 +520,8 @@ booleanAttributes = { "": frozenset(("irrelevant",)), "style": frozenset(("scoped",)), "img": frozenset(("ismap",)), - "audio": frozenset(("autoplay","controls")), - "video": frozenset(("autoplay","controls")), + "audio": frozenset(("autoplay", "controls")), + "video": frozenset(("autoplay", "controls")), "script": frozenset(("defer", "async")), "details": frozenset(("open",)), "datagrid": frozenset(("multiple", "disabled")), @@ -523,2312 +540,2312 @@ booleanAttributes = { # entitiesWindows1252 has to be _ordered_ and needs to have an index. It # therefore can't be a frozenset. entitiesWindows1252 = ( - 8364, # 0x80 0x20AC EURO SIGN - 65533, # 0x81 UNDEFINED - 8218, # 0x82 0x201A SINGLE LOW-9 QUOTATION MARK - 402, # 0x83 0x0192 LATIN SMALL LETTER F WITH HOOK - 8222, # 0x84 0x201E DOUBLE LOW-9 QUOTATION MARK - 8230, # 0x85 0x2026 HORIZONTAL ELLIPSIS - 8224, # 0x86 0x2020 DAGGER - 8225, # 0x87 0x2021 DOUBLE DAGGER - 710, # 0x88 0x02C6 MODIFIER LETTER CIRCUMFLEX ACCENT - 8240, # 0x89 0x2030 PER MILLE SIGN - 352, # 0x8A 0x0160 LATIN CAPITAL LETTER S WITH CARON - 8249, # 0x8B 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK - 338, # 0x8C 0x0152 LATIN CAPITAL LIGATURE OE - 65533, # 0x8D UNDEFINED - 381, # 0x8E 0x017D LATIN CAPITAL LETTER Z WITH CARON - 65533, # 0x8F UNDEFINED - 65533, # 0x90 UNDEFINED - 8216, # 0x91 0x2018 LEFT SINGLE QUOTATION MARK - 8217, # 0x92 0x2019 RIGHT SINGLE QUOTATION MARK - 8220, # 0x93 0x201C LEFT DOUBLE QUOTATION MARK - 8221, # 0x94 0x201D RIGHT DOUBLE QUOTATION MARK - 8226, # 0x95 0x2022 BULLET - 8211, # 0x96 0x2013 EN DASH - 8212, # 0x97 0x2014 EM DASH - 732, # 0x98 0x02DC SMALL TILDE - 8482, # 0x99 0x2122 TRADE MARK SIGN - 353, # 0x9A 0x0161 LATIN SMALL LETTER S WITH CARON - 8250, # 0x9B 0x203A SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - 339, # 0x9C 0x0153 LATIN SMALL LIGATURE OE - 65533, # 0x9D UNDEFINED - 382, # 0x9E 0x017E LATIN SMALL LETTER Z WITH CARON - 376 # 0x9F 0x0178 LATIN CAPITAL LETTER Y WITH DIAERESIS + 8364, # 0x80 0x20AC EURO SIGN + 65533, # 0x81 UNDEFINED + 8218, # 0x82 0x201A SINGLE LOW-9 QUOTATION MARK + 402, # 0x83 0x0192 LATIN SMALL LETTER F WITH HOOK + 8222, # 0x84 0x201E DOUBLE LOW-9 QUOTATION MARK + 8230, # 0x85 0x2026 HORIZONTAL ELLIPSIS + 8224, # 0x86 0x2020 DAGGER + 8225, # 0x87 0x2021 DOUBLE DAGGER + 710, # 0x88 0x02C6 MODIFIER LETTER CIRCUMFLEX ACCENT + 8240, # 0x89 0x2030 PER MILLE SIGN + 352, # 0x8A 0x0160 LATIN CAPITAL LETTER S WITH CARON + 8249, # 0x8B 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 338, # 0x8C 0x0152 LATIN CAPITAL LIGATURE OE + 65533, # 0x8D UNDEFINED + 381, # 0x8E 0x017D LATIN CAPITAL LETTER Z WITH CARON + 65533, # 0x8F UNDEFINED + 65533, # 0x90 UNDEFINED + 8216, # 0x91 0x2018 LEFT SINGLE QUOTATION MARK + 8217, # 0x92 0x2019 RIGHT SINGLE QUOTATION MARK + 8220, # 0x93 0x201C LEFT DOUBLE QUOTATION MARK + 8221, # 0x94 0x201D RIGHT DOUBLE QUOTATION MARK + 8226, # 0x95 0x2022 BULLET + 8211, # 0x96 0x2013 EN DASH + 8212, # 0x97 0x2014 EM DASH + 732, # 0x98 0x02DC SMALL TILDE + 8482, # 0x99 0x2122 TRADE MARK SIGN + 353, # 0x9A 0x0161 LATIN SMALL LETTER S WITH CARON + 8250, # 0x9B 0x203A SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 339, # 0x9C 0x0153 LATIN SMALL LIGATURE OE + 65533, # 0x9D UNDEFINED + 382, # 0x9E 0x017E LATIN SMALL LETTER Z WITH CARON + 376 # 0x9F 0x0178 LATIN CAPITAL LETTER Y WITH DIAERESIS ) xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;')) entities = { - "AElig": u"\xc6", - "AElig;": u"\xc6", - "AMP": u"&", - "AMP;": u"&", - "Aacute": u"\xc1", - "Aacute;": u"\xc1", - "Abreve;": u"\u0102", - "Acirc": u"\xc2", - "Acirc;": u"\xc2", - "Acy;": u"\u0410", - "Afr;": u"\U0001d504", - "Agrave": u"\xc0", - "Agrave;": u"\xc0", - "Alpha;": u"\u0391", - "Amacr;": u"\u0100", - "And;": u"\u2a53", - "Aogon;": u"\u0104", - "Aopf;": u"\U0001d538", - "ApplyFunction;": u"\u2061", - "Aring": u"\xc5", - "Aring;": u"\xc5", - "Ascr;": u"\U0001d49c", - "Assign;": u"\u2254", - "Atilde": u"\xc3", - "Atilde;": u"\xc3", - "Auml": u"\xc4", - "Auml;": u"\xc4", - "Backslash;": u"\u2216", - "Barv;": u"\u2ae7", - "Barwed;": u"\u2306", - "Bcy;": u"\u0411", - "Because;": u"\u2235", - "Bernoullis;": u"\u212c", - "Beta;": u"\u0392", - "Bfr;": u"\U0001d505", - "Bopf;": u"\U0001d539", - "Breve;": u"\u02d8", - "Bscr;": u"\u212c", - "Bumpeq;": u"\u224e", - "CHcy;": u"\u0427", - "COPY": u"\xa9", - "COPY;": u"\xa9", - "Cacute;": u"\u0106", - "Cap;": u"\u22d2", - "CapitalDifferentialD;": u"\u2145", - "Cayleys;": u"\u212d", - "Ccaron;": u"\u010c", - "Ccedil": u"\xc7", - "Ccedil;": u"\xc7", - "Ccirc;": u"\u0108", - "Cconint;": u"\u2230", - "Cdot;": u"\u010a", - "Cedilla;": u"\xb8", - "CenterDot;": u"\xb7", - "Cfr;": u"\u212d", - "Chi;": u"\u03a7", - "CircleDot;": u"\u2299", - "CircleMinus;": u"\u2296", - "CirclePlus;": u"\u2295", - "CircleTimes;": u"\u2297", - "ClockwiseContourIntegral;": u"\u2232", - "CloseCurlyDoubleQuote;": u"\u201d", - "CloseCurlyQuote;": u"\u2019", - "Colon;": u"\u2237", - "Colone;": u"\u2a74", - "Congruent;": u"\u2261", - "Conint;": u"\u222f", - "ContourIntegral;": u"\u222e", - "Copf;": u"\u2102", - "Coproduct;": u"\u2210", - "CounterClockwiseContourIntegral;": u"\u2233", - "Cross;": u"\u2a2f", - "Cscr;": u"\U0001d49e", - "Cup;": u"\u22d3", - "CupCap;": u"\u224d", - "DD;": u"\u2145", - "DDotrahd;": u"\u2911", - "DJcy;": u"\u0402", - "DScy;": u"\u0405", - "DZcy;": u"\u040f", - "Dagger;": u"\u2021", - "Darr;": u"\u21a1", - "Dashv;": u"\u2ae4", - "Dcaron;": u"\u010e", - "Dcy;": u"\u0414", - "Del;": u"\u2207", - "Delta;": u"\u0394", - "Dfr;": u"\U0001d507", - "DiacriticalAcute;": u"\xb4", - "DiacriticalDot;": u"\u02d9", - "DiacriticalDoubleAcute;": u"\u02dd", - "DiacriticalGrave;": u"`", - "DiacriticalTilde;": u"\u02dc", - "Diamond;": u"\u22c4", - "DifferentialD;": u"\u2146", - "Dopf;": u"\U0001d53b", - "Dot;": u"\xa8", - "DotDot;": u"\u20dc", - "DotEqual;": u"\u2250", - "DoubleContourIntegral;": u"\u222f", - "DoubleDot;": u"\xa8", - "DoubleDownArrow;": u"\u21d3", - "DoubleLeftArrow;": u"\u21d0", - "DoubleLeftRightArrow;": u"\u21d4", - "DoubleLeftTee;": u"\u2ae4", - "DoubleLongLeftArrow;": u"\u27f8", - "DoubleLongLeftRightArrow;": u"\u27fa", - "DoubleLongRightArrow;": u"\u27f9", - "DoubleRightArrow;": u"\u21d2", - "DoubleRightTee;": u"\u22a8", - "DoubleUpArrow;": u"\u21d1", - "DoubleUpDownArrow;": u"\u21d5", - "DoubleVerticalBar;": u"\u2225", - "DownArrow;": u"\u2193", - "DownArrowBar;": u"\u2913", - "DownArrowUpArrow;": u"\u21f5", - "DownBreve;": u"\u0311", - "DownLeftRightVector;": u"\u2950", - "DownLeftTeeVector;": u"\u295e", - "DownLeftVector;": u"\u21bd", - "DownLeftVectorBar;": u"\u2956", - "DownRightTeeVector;": u"\u295f", - "DownRightVector;": u"\u21c1", - "DownRightVectorBar;": u"\u2957", - "DownTee;": u"\u22a4", - "DownTeeArrow;": u"\u21a7", - "Downarrow;": u"\u21d3", - "Dscr;": u"\U0001d49f", - "Dstrok;": u"\u0110", - "ENG;": u"\u014a", - "ETH": u"\xd0", - "ETH;": u"\xd0", - "Eacute": u"\xc9", - "Eacute;": u"\xc9", - "Ecaron;": u"\u011a", - "Ecirc": u"\xca", - "Ecirc;": u"\xca", - "Ecy;": u"\u042d", - "Edot;": u"\u0116", - "Efr;": u"\U0001d508", - "Egrave": u"\xc8", - "Egrave;": u"\xc8", - "Element;": u"\u2208", - "Emacr;": u"\u0112", - "EmptySmallSquare;": u"\u25fb", - "EmptyVerySmallSquare;": u"\u25ab", - "Eogon;": u"\u0118", - "Eopf;": u"\U0001d53c", - "Epsilon;": u"\u0395", - "Equal;": u"\u2a75", - "EqualTilde;": u"\u2242", - "Equilibrium;": u"\u21cc", - "Escr;": u"\u2130", - "Esim;": u"\u2a73", - "Eta;": u"\u0397", - "Euml": u"\xcb", - "Euml;": u"\xcb", - "Exists;": u"\u2203", - "ExponentialE;": u"\u2147", - "Fcy;": u"\u0424", - "Ffr;": u"\U0001d509", - "FilledSmallSquare;": u"\u25fc", - "FilledVerySmallSquare;": u"\u25aa", - "Fopf;": u"\U0001d53d", - "ForAll;": u"\u2200", - "Fouriertrf;": u"\u2131", - "Fscr;": u"\u2131", - "GJcy;": u"\u0403", - "GT": u">", - "GT;": u">", - "Gamma;": u"\u0393", - "Gammad;": u"\u03dc", - "Gbreve;": u"\u011e", - "Gcedil;": u"\u0122", - "Gcirc;": u"\u011c", - "Gcy;": u"\u0413", - "Gdot;": u"\u0120", - "Gfr;": u"\U0001d50a", - "Gg;": u"\u22d9", - "Gopf;": u"\U0001d53e", - "GreaterEqual;": u"\u2265", - "GreaterEqualLess;": u"\u22db", - "GreaterFullEqual;": u"\u2267", - "GreaterGreater;": u"\u2aa2", - "GreaterLess;": u"\u2277", - "GreaterSlantEqual;": u"\u2a7e", - "GreaterTilde;": u"\u2273", - "Gscr;": u"\U0001d4a2", - "Gt;": u"\u226b", - "HARDcy;": u"\u042a", - "Hacek;": u"\u02c7", - "Hat;": u"^", - "Hcirc;": u"\u0124", - "Hfr;": u"\u210c", - "HilbertSpace;": u"\u210b", - "Hopf;": u"\u210d", - "HorizontalLine;": u"\u2500", - "Hscr;": u"\u210b", - "Hstrok;": u"\u0126", - "HumpDownHump;": u"\u224e", - "HumpEqual;": u"\u224f", - "IEcy;": u"\u0415", - "IJlig;": u"\u0132", - "IOcy;": u"\u0401", - "Iacute": u"\xcd", - "Iacute;": u"\xcd", - "Icirc": u"\xce", - "Icirc;": u"\xce", - "Icy;": u"\u0418", - "Idot;": u"\u0130", - "Ifr;": u"\u2111", - "Igrave": u"\xcc", - "Igrave;": u"\xcc", - "Im;": u"\u2111", - "Imacr;": u"\u012a", - "ImaginaryI;": u"\u2148", - "Implies;": u"\u21d2", - "Int;": u"\u222c", - "Integral;": u"\u222b", - "Intersection;": u"\u22c2", - "InvisibleComma;": u"\u2063", - "InvisibleTimes;": u"\u2062", - "Iogon;": u"\u012e", - "Iopf;": u"\U0001d540", - "Iota;": u"\u0399", - "Iscr;": u"\u2110", - "Itilde;": u"\u0128", - "Iukcy;": u"\u0406", - "Iuml": u"\xcf", - "Iuml;": u"\xcf", - "Jcirc;": u"\u0134", - "Jcy;": u"\u0419", - "Jfr;": u"\U0001d50d", - "Jopf;": u"\U0001d541", - "Jscr;": u"\U0001d4a5", - "Jsercy;": u"\u0408", - "Jukcy;": u"\u0404", - "KHcy;": u"\u0425", - "KJcy;": u"\u040c", - "Kappa;": u"\u039a", - "Kcedil;": u"\u0136", - "Kcy;": u"\u041a", - "Kfr;": u"\U0001d50e", - "Kopf;": u"\U0001d542", - "Kscr;": u"\U0001d4a6", - "LJcy;": u"\u0409", - "LT": u"<", - "LT;": u"<", - "Lacute;": u"\u0139", - "Lambda;": u"\u039b", - "Lang;": u"\u27ea", - "Laplacetrf;": u"\u2112", - "Larr;": u"\u219e", - "Lcaron;": u"\u013d", - "Lcedil;": u"\u013b", - "Lcy;": u"\u041b", - "LeftAngleBracket;": u"\u27e8", - "LeftArrow;": u"\u2190", - "LeftArrowBar;": u"\u21e4", - "LeftArrowRightArrow;": u"\u21c6", - "LeftCeiling;": u"\u2308", - "LeftDoubleBracket;": u"\u27e6", - "LeftDownTeeVector;": u"\u2961", - "LeftDownVector;": u"\u21c3", - "LeftDownVectorBar;": u"\u2959", - "LeftFloor;": u"\u230a", - "LeftRightArrow;": u"\u2194", - "LeftRightVector;": u"\u294e", - "LeftTee;": u"\u22a3", - "LeftTeeArrow;": u"\u21a4", - "LeftTeeVector;": u"\u295a", - "LeftTriangle;": u"\u22b2", - "LeftTriangleBar;": u"\u29cf", - "LeftTriangleEqual;": u"\u22b4", - "LeftUpDownVector;": u"\u2951", - "LeftUpTeeVector;": u"\u2960", - "LeftUpVector;": u"\u21bf", - "LeftUpVectorBar;": u"\u2958", - "LeftVector;": u"\u21bc", - "LeftVectorBar;": u"\u2952", - "Leftarrow;": u"\u21d0", - "Leftrightarrow;": u"\u21d4", - "LessEqualGreater;": u"\u22da", - "LessFullEqual;": u"\u2266", - "LessGreater;": u"\u2276", - "LessLess;": u"\u2aa1", - "LessSlantEqual;": u"\u2a7d", - "LessTilde;": u"\u2272", - "Lfr;": u"\U0001d50f", - "Ll;": u"\u22d8", - "Lleftarrow;": u"\u21da", - "Lmidot;": u"\u013f", - "LongLeftArrow;": u"\u27f5", - "LongLeftRightArrow;": u"\u27f7", - "LongRightArrow;": u"\u27f6", - "Longleftarrow;": u"\u27f8", - "Longleftrightarrow;": u"\u27fa", - "Longrightarrow;": u"\u27f9", - "Lopf;": u"\U0001d543", - "LowerLeftArrow;": u"\u2199", - "LowerRightArrow;": u"\u2198", - "Lscr;": u"\u2112", - "Lsh;": u"\u21b0", - "Lstrok;": u"\u0141", - "Lt;": u"\u226a", - "Map;": u"\u2905", - "Mcy;": u"\u041c", - "MediumSpace;": u"\u205f", - "Mellintrf;": u"\u2133", - "Mfr;": u"\U0001d510", - "MinusPlus;": u"\u2213", - "Mopf;": u"\U0001d544", - "Mscr;": u"\u2133", - "Mu;": u"\u039c", - "NJcy;": u"\u040a", - "Nacute;": u"\u0143", - "Ncaron;": u"\u0147", - "Ncedil;": u"\u0145", - "Ncy;": u"\u041d", - "NegativeMediumSpace;": u"\u200b", - "NegativeThickSpace;": u"\u200b", - "NegativeThinSpace;": u"\u200b", - "NegativeVeryThinSpace;": u"\u200b", - "NestedGreaterGreater;": u"\u226b", - "NestedLessLess;": u"\u226a", - "NewLine;": u"\n", - "Nfr;": u"\U0001d511", - "NoBreak;": u"\u2060", - "NonBreakingSpace;": u"\xa0", - "Nopf;": u"\u2115", - "Not;": u"\u2aec", - "NotCongruent;": u"\u2262", - "NotCupCap;": u"\u226d", - "NotDoubleVerticalBar;": u"\u2226", - "NotElement;": u"\u2209", - "NotEqual;": u"\u2260", - "NotEqualTilde;": u"\u2242\u0338", - "NotExists;": u"\u2204", - "NotGreater;": u"\u226f", - "NotGreaterEqual;": u"\u2271", - "NotGreaterFullEqual;": u"\u2267\u0338", - "NotGreaterGreater;": u"\u226b\u0338", - "NotGreaterLess;": u"\u2279", - "NotGreaterSlantEqual;": u"\u2a7e\u0338", - "NotGreaterTilde;": u"\u2275", - "NotHumpDownHump;": u"\u224e\u0338", - "NotHumpEqual;": u"\u224f\u0338", - "NotLeftTriangle;": u"\u22ea", - "NotLeftTriangleBar;": u"\u29cf\u0338", - "NotLeftTriangleEqual;": u"\u22ec", - "NotLess;": u"\u226e", - "NotLessEqual;": u"\u2270", - "NotLessGreater;": u"\u2278", - "NotLessLess;": u"\u226a\u0338", - "NotLessSlantEqual;": u"\u2a7d\u0338", - "NotLessTilde;": u"\u2274", - "NotNestedGreaterGreater;": u"\u2aa2\u0338", - "NotNestedLessLess;": u"\u2aa1\u0338", - "NotPrecedes;": u"\u2280", - "NotPrecedesEqual;": u"\u2aaf\u0338", - "NotPrecedesSlantEqual;": u"\u22e0", - "NotReverseElement;": u"\u220c", - "NotRightTriangle;": u"\u22eb", - "NotRightTriangleBar;": u"\u29d0\u0338", - "NotRightTriangleEqual;": u"\u22ed", - "NotSquareSubset;": u"\u228f\u0338", - "NotSquareSubsetEqual;": u"\u22e2", - "NotSquareSuperset;": u"\u2290\u0338", - "NotSquareSupersetEqual;": u"\u22e3", - "NotSubset;": u"\u2282\u20d2", - "NotSubsetEqual;": u"\u2288", - "NotSucceeds;": u"\u2281", - "NotSucceedsEqual;": u"\u2ab0\u0338", - "NotSucceedsSlantEqual;": u"\u22e1", - "NotSucceedsTilde;": u"\u227f\u0338", - "NotSuperset;": u"\u2283\u20d2", - "NotSupersetEqual;": u"\u2289", - "NotTilde;": u"\u2241", - "NotTildeEqual;": u"\u2244", - "NotTildeFullEqual;": u"\u2247", - "NotTildeTilde;": u"\u2249", - "NotVerticalBar;": u"\u2224", - "Nscr;": u"\U0001d4a9", - "Ntilde": u"\xd1", - "Ntilde;": u"\xd1", - "Nu;": u"\u039d", - "OElig;": u"\u0152", - "Oacute": u"\xd3", - "Oacute;": u"\xd3", - "Ocirc": u"\xd4", - "Ocirc;": u"\xd4", - "Ocy;": u"\u041e", - "Odblac;": u"\u0150", - "Ofr;": u"\U0001d512", - "Ograve": u"\xd2", - "Ograve;": u"\xd2", - "Omacr;": u"\u014c", - "Omega;": u"\u03a9", - "Omicron;": u"\u039f", - "Oopf;": u"\U0001d546", - "OpenCurlyDoubleQuote;": u"\u201c", - "OpenCurlyQuote;": u"\u2018", - "Or;": u"\u2a54", - "Oscr;": u"\U0001d4aa", - "Oslash": u"\xd8", - "Oslash;": u"\xd8", - "Otilde": u"\xd5", - "Otilde;": u"\xd5", - "Otimes;": u"\u2a37", - "Ouml": u"\xd6", - "Ouml;": u"\xd6", - "OverBar;": u"\u203e", - "OverBrace;": u"\u23de", - "OverBracket;": u"\u23b4", - "OverParenthesis;": u"\u23dc", - "PartialD;": u"\u2202", - "Pcy;": u"\u041f", - "Pfr;": u"\U0001d513", - "Phi;": u"\u03a6", - "Pi;": u"\u03a0", - "PlusMinus;": u"\xb1", - "Poincareplane;": u"\u210c", - "Popf;": u"\u2119", - "Pr;": u"\u2abb", - "Precedes;": u"\u227a", - "PrecedesEqual;": u"\u2aaf", - "PrecedesSlantEqual;": u"\u227c", - "PrecedesTilde;": u"\u227e", - "Prime;": u"\u2033", - "Product;": u"\u220f", - "Proportion;": u"\u2237", - "Proportional;": u"\u221d", - "Pscr;": u"\U0001d4ab", - "Psi;": u"\u03a8", - "QUOT": u"\"", - "QUOT;": u"\"", - "Qfr;": u"\U0001d514", - "Qopf;": u"\u211a", - "Qscr;": u"\U0001d4ac", - "RBarr;": u"\u2910", - "REG": u"\xae", - "REG;": u"\xae", - "Racute;": u"\u0154", - "Rang;": u"\u27eb", - "Rarr;": u"\u21a0", - "Rarrtl;": u"\u2916", - "Rcaron;": u"\u0158", - "Rcedil;": u"\u0156", - "Rcy;": u"\u0420", - "Re;": u"\u211c", - "ReverseElement;": u"\u220b", - "ReverseEquilibrium;": u"\u21cb", - "ReverseUpEquilibrium;": u"\u296f", - "Rfr;": u"\u211c", - "Rho;": u"\u03a1", - "RightAngleBracket;": u"\u27e9", - "RightArrow;": u"\u2192", - "RightArrowBar;": u"\u21e5", - "RightArrowLeftArrow;": u"\u21c4", - "RightCeiling;": u"\u2309", - "RightDoubleBracket;": u"\u27e7", - "RightDownTeeVector;": u"\u295d", - "RightDownVector;": u"\u21c2", - "RightDownVectorBar;": u"\u2955", - "RightFloor;": u"\u230b", - "RightTee;": u"\u22a2", - "RightTeeArrow;": u"\u21a6", - "RightTeeVector;": u"\u295b", - "RightTriangle;": u"\u22b3", - "RightTriangleBar;": u"\u29d0", - "RightTriangleEqual;": u"\u22b5", - "RightUpDownVector;": u"\u294f", - "RightUpTeeVector;": u"\u295c", - "RightUpVector;": u"\u21be", - "RightUpVectorBar;": u"\u2954", - "RightVector;": u"\u21c0", - "RightVectorBar;": u"\u2953", - "Rightarrow;": u"\u21d2", - "Ropf;": u"\u211d", - "RoundImplies;": u"\u2970", - "Rrightarrow;": u"\u21db", - "Rscr;": u"\u211b", - "Rsh;": u"\u21b1", - "RuleDelayed;": u"\u29f4", - "SHCHcy;": u"\u0429", - "SHcy;": u"\u0428", - "SOFTcy;": u"\u042c", - "Sacute;": u"\u015a", - "Sc;": u"\u2abc", - "Scaron;": u"\u0160", - "Scedil;": u"\u015e", - "Scirc;": u"\u015c", - "Scy;": u"\u0421", - "Sfr;": u"\U0001d516", - "ShortDownArrow;": u"\u2193", - "ShortLeftArrow;": u"\u2190", - "ShortRightArrow;": u"\u2192", - "ShortUpArrow;": u"\u2191", - "Sigma;": u"\u03a3", - "SmallCircle;": u"\u2218", - "Sopf;": u"\U0001d54a", - "Sqrt;": u"\u221a", - "Square;": u"\u25a1", - "SquareIntersection;": u"\u2293", - "SquareSubset;": u"\u228f", - "SquareSubsetEqual;": u"\u2291", - "SquareSuperset;": u"\u2290", - "SquareSupersetEqual;": u"\u2292", - "SquareUnion;": u"\u2294", - "Sscr;": u"\U0001d4ae", - "Star;": u"\u22c6", - "Sub;": u"\u22d0", - "Subset;": u"\u22d0", - "SubsetEqual;": u"\u2286", - "Succeeds;": u"\u227b", - "SucceedsEqual;": u"\u2ab0", - "SucceedsSlantEqual;": u"\u227d", - "SucceedsTilde;": u"\u227f", - "SuchThat;": u"\u220b", - "Sum;": u"\u2211", - "Sup;": u"\u22d1", - "Superset;": u"\u2283", - "SupersetEqual;": u"\u2287", - "Supset;": u"\u22d1", - "THORN": u"\xde", - "THORN;": u"\xde", - "TRADE;": u"\u2122", - "TSHcy;": u"\u040b", - "TScy;": u"\u0426", - "Tab;": u"\t", - "Tau;": u"\u03a4", - "Tcaron;": u"\u0164", - "Tcedil;": u"\u0162", - "Tcy;": u"\u0422", - "Tfr;": u"\U0001d517", - "Therefore;": u"\u2234", - "Theta;": u"\u0398", - "ThickSpace;": u"\u205f\u200a", - "ThinSpace;": u"\u2009", - "Tilde;": u"\u223c", - "TildeEqual;": u"\u2243", - "TildeFullEqual;": u"\u2245", - "TildeTilde;": u"\u2248", - "Topf;": u"\U0001d54b", - "TripleDot;": u"\u20db", - "Tscr;": u"\U0001d4af", - "Tstrok;": u"\u0166", - "Uacute": u"\xda", - "Uacute;": u"\xda", - "Uarr;": u"\u219f", - "Uarrocir;": u"\u2949", - "Ubrcy;": u"\u040e", - "Ubreve;": u"\u016c", - "Ucirc": u"\xdb", - "Ucirc;": u"\xdb", - "Ucy;": u"\u0423", - "Udblac;": u"\u0170", - "Ufr;": u"\U0001d518", - "Ugrave": u"\xd9", - "Ugrave;": u"\xd9", - "Umacr;": u"\u016a", - "UnderBar;": u"_", - "UnderBrace;": u"\u23df", - "UnderBracket;": u"\u23b5", - "UnderParenthesis;": u"\u23dd", - "Union;": u"\u22c3", - "UnionPlus;": u"\u228e", - "Uogon;": u"\u0172", - "Uopf;": u"\U0001d54c", - "UpArrow;": u"\u2191", - "UpArrowBar;": u"\u2912", - "UpArrowDownArrow;": u"\u21c5", - "UpDownArrow;": u"\u2195", - "UpEquilibrium;": u"\u296e", - "UpTee;": u"\u22a5", - "UpTeeArrow;": u"\u21a5", - "Uparrow;": u"\u21d1", - "Updownarrow;": u"\u21d5", - "UpperLeftArrow;": u"\u2196", - "UpperRightArrow;": u"\u2197", - "Upsi;": u"\u03d2", - "Upsilon;": u"\u03a5", - "Uring;": u"\u016e", - "Uscr;": u"\U0001d4b0", - "Utilde;": u"\u0168", - "Uuml": u"\xdc", - "Uuml;": u"\xdc", - "VDash;": u"\u22ab", - "Vbar;": u"\u2aeb", - "Vcy;": u"\u0412", - "Vdash;": u"\u22a9", - "Vdashl;": u"\u2ae6", - "Vee;": u"\u22c1", - "Verbar;": u"\u2016", - "Vert;": u"\u2016", - "VerticalBar;": u"\u2223", - "VerticalLine;": u"|", - "VerticalSeparator;": u"\u2758", - "VerticalTilde;": u"\u2240", - "VeryThinSpace;": u"\u200a", - "Vfr;": u"\U0001d519", - "Vopf;": u"\U0001d54d", - "Vscr;": u"\U0001d4b1", - "Vvdash;": u"\u22aa", - "Wcirc;": u"\u0174", - "Wedge;": u"\u22c0", - "Wfr;": u"\U0001d51a", - "Wopf;": u"\U0001d54e", - "Wscr;": u"\U0001d4b2", - "Xfr;": u"\U0001d51b", - "Xi;": u"\u039e", - "Xopf;": u"\U0001d54f", - "Xscr;": u"\U0001d4b3", - "YAcy;": u"\u042f", - "YIcy;": u"\u0407", - "YUcy;": u"\u042e", - "Yacute": u"\xdd", - "Yacute;": u"\xdd", - "Ycirc;": u"\u0176", - "Ycy;": u"\u042b", - "Yfr;": u"\U0001d51c", - "Yopf;": u"\U0001d550", - "Yscr;": u"\U0001d4b4", - "Yuml;": u"\u0178", - "ZHcy;": u"\u0416", - "Zacute;": u"\u0179", - "Zcaron;": u"\u017d", - "Zcy;": u"\u0417", - "Zdot;": u"\u017b", - "ZeroWidthSpace;": u"\u200b", - "Zeta;": u"\u0396", - "Zfr;": u"\u2128", - "Zopf;": u"\u2124", - "Zscr;": u"\U0001d4b5", - "aacute": u"\xe1", - "aacute;": u"\xe1", - "abreve;": u"\u0103", - "ac;": u"\u223e", - "acE;": u"\u223e\u0333", - "acd;": u"\u223f", - "acirc": u"\xe2", - "acirc;": u"\xe2", - "acute": u"\xb4", - "acute;": u"\xb4", - "acy;": u"\u0430", - "aelig": u"\xe6", - "aelig;": u"\xe6", - "af;": u"\u2061", - "afr;": u"\U0001d51e", - "agrave": u"\xe0", - "agrave;": u"\xe0", - "alefsym;": u"\u2135", - "aleph;": u"\u2135", - "alpha;": u"\u03b1", - "amacr;": u"\u0101", - "amalg;": u"\u2a3f", - "amp": u"&", - "amp;": u"&", - "and;": u"\u2227", - "andand;": u"\u2a55", - "andd;": u"\u2a5c", - "andslope;": u"\u2a58", - "andv;": u"\u2a5a", - "ang;": u"\u2220", - "ange;": u"\u29a4", - "angle;": u"\u2220", - "angmsd;": u"\u2221", - "angmsdaa;": u"\u29a8", - "angmsdab;": u"\u29a9", - "angmsdac;": u"\u29aa", - "angmsdad;": u"\u29ab", - "angmsdae;": u"\u29ac", - "angmsdaf;": u"\u29ad", - "angmsdag;": u"\u29ae", - "angmsdah;": u"\u29af", - "angrt;": u"\u221f", - "angrtvb;": u"\u22be", - "angrtvbd;": u"\u299d", - "angsph;": u"\u2222", - "angst;": u"\xc5", - "angzarr;": u"\u237c", - "aogon;": u"\u0105", - "aopf;": u"\U0001d552", - "ap;": u"\u2248", - "apE;": u"\u2a70", - "apacir;": u"\u2a6f", - "ape;": u"\u224a", - "apid;": u"\u224b", - "apos;": u"'", - "approx;": u"\u2248", - "approxeq;": u"\u224a", - "aring": u"\xe5", - "aring;": u"\xe5", - "ascr;": u"\U0001d4b6", - "ast;": u"*", - "asymp;": u"\u2248", - "asympeq;": u"\u224d", - "atilde": u"\xe3", - "atilde;": u"\xe3", - "auml": u"\xe4", - "auml;": u"\xe4", - "awconint;": u"\u2233", - "awint;": u"\u2a11", - "bNot;": u"\u2aed", - "backcong;": u"\u224c", - "backepsilon;": u"\u03f6", - "backprime;": u"\u2035", - "backsim;": u"\u223d", - "backsimeq;": u"\u22cd", - "barvee;": u"\u22bd", - "barwed;": u"\u2305", - "barwedge;": u"\u2305", - "bbrk;": u"\u23b5", - "bbrktbrk;": u"\u23b6", - "bcong;": u"\u224c", - "bcy;": u"\u0431", - "bdquo;": u"\u201e", - "becaus;": u"\u2235", - "because;": u"\u2235", - "bemptyv;": u"\u29b0", - "bepsi;": u"\u03f6", - "bernou;": u"\u212c", - "beta;": u"\u03b2", - "beth;": u"\u2136", - "between;": u"\u226c", - "bfr;": u"\U0001d51f", - "bigcap;": u"\u22c2", - "bigcirc;": u"\u25ef", - "bigcup;": u"\u22c3", - "bigodot;": u"\u2a00", - "bigoplus;": u"\u2a01", - "bigotimes;": u"\u2a02", - "bigsqcup;": u"\u2a06", - "bigstar;": u"\u2605", - "bigtriangledown;": u"\u25bd", - "bigtriangleup;": u"\u25b3", - "biguplus;": u"\u2a04", - "bigvee;": u"\u22c1", - "bigwedge;": u"\u22c0", - "bkarow;": u"\u290d", - "blacklozenge;": u"\u29eb", - "blacksquare;": u"\u25aa", - "blacktriangle;": u"\u25b4", - "blacktriangledown;": u"\u25be", - "blacktriangleleft;": u"\u25c2", - "blacktriangleright;": u"\u25b8", - "blank;": u"\u2423", - "blk12;": u"\u2592", - "blk14;": u"\u2591", - "blk34;": u"\u2593", - "block;": u"\u2588", - "bne;": u"=\u20e5", - "bnequiv;": u"\u2261\u20e5", - "bnot;": u"\u2310", - "bopf;": u"\U0001d553", - "bot;": u"\u22a5", - "bottom;": u"\u22a5", - "bowtie;": u"\u22c8", - "boxDL;": u"\u2557", - "boxDR;": u"\u2554", - "boxDl;": u"\u2556", - "boxDr;": u"\u2553", - "boxH;": u"\u2550", - "boxHD;": u"\u2566", - "boxHU;": u"\u2569", - "boxHd;": u"\u2564", - "boxHu;": u"\u2567", - "boxUL;": u"\u255d", - "boxUR;": u"\u255a", - "boxUl;": u"\u255c", - "boxUr;": u"\u2559", - "boxV;": u"\u2551", - "boxVH;": u"\u256c", - "boxVL;": u"\u2563", - "boxVR;": u"\u2560", - "boxVh;": u"\u256b", - "boxVl;": u"\u2562", - "boxVr;": u"\u255f", - "boxbox;": u"\u29c9", - "boxdL;": u"\u2555", - "boxdR;": u"\u2552", - "boxdl;": u"\u2510", - "boxdr;": u"\u250c", - "boxh;": u"\u2500", - "boxhD;": u"\u2565", - "boxhU;": u"\u2568", - "boxhd;": u"\u252c", - "boxhu;": u"\u2534", - "boxminus;": u"\u229f", - "boxplus;": u"\u229e", - "boxtimes;": u"\u22a0", - "boxuL;": u"\u255b", - "boxuR;": u"\u2558", - "boxul;": u"\u2518", - "boxur;": u"\u2514", - "boxv;": u"\u2502", - "boxvH;": u"\u256a", - "boxvL;": u"\u2561", - "boxvR;": u"\u255e", - "boxvh;": u"\u253c", - "boxvl;": u"\u2524", - "boxvr;": u"\u251c", - "bprime;": u"\u2035", - "breve;": u"\u02d8", - "brvbar": u"\xa6", - "brvbar;": u"\xa6", - "bscr;": u"\U0001d4b7", - "bsemi;": u"\u204f", - "bsim;": u"\u223d", - "bsime;": u"\u22cd", - "bsol;": u"\\", - "bsolb;": u"\u29c5", - "bsolhsub;": u"\u27c8", - "bull;": u"\u2022", - "bullet;": u"\u2022", - "bump;": u"\u224e", - "bumpE;": u"\u2aae", - "bumpe;": u"\u224f", - "bumpeq;": u"\u224f", - "cacute;": u"\u0107", - "cap;": u"\u2229", - "capand;": u"\u2a44", - "capbrcup;": u"\u2a49", - "capcap;": u"\u2a4b", - "capcup;": u"\u2a47", - "capdot;": u"\u2a40", - "caps;": u"\u2229\ufe00", - "caret;": u"\u2041", - "caron;": u"\u02c7", - "ccaps;": u"\u2a4d", - "ccaron;": u"\u010d", - "ccedil": u"\xe7", - "ccedil;": u"\xe7", - "ccirc;": u"\u0109", - "ccups;": u"\u2a4c", - "ccupssm;": u"\u2a50", - "cdot;": u"\u010b", - "cedil": u"\xb8", - "cedil;": u"\xb8", - "cemptyv;": u"\u29b2", - "cent": u"\xa2", - "cent;": u"\xa2", - "centerdot;": u"\xb7", - "cfr;": u"\U0001d520", - "chcy;": u"\u0447", - "check;": u"\u2713", - "checkmark;": u"\u2713", - "chi;": u"\u03c7", - "cir;": u"\u25cb", - "cirE;": u"\u29c3", - "circ;": u"\u02c6", - "circeq;": u"\u2257", - "circlearrowleft;": u"\u21ba", - "circlearrowright;": u"\u21bb", - "circledR;": u"\xae", - "circledS;": u"\u24c8", - "circledast;": u"\u229b", - "circledcirc;": u"\u229a", - "circleddash;": u"\u229d", - "cire;": u"\u2257", - "cirfnint;": u"\u2a10", - "cirmid;": u"\u2aef", - "cirscir;": u"\u29c2", - "clubs;": u"\u2663", - "clubsuit;": u"\u2663", - "colon;": u":", - "colone;": u"\u2254", - "coloneq;": u"\u2254", - "comma;": u",", - "commat;": u"@", - "comp;": u"\u2201", - "compfn;": u"\u2218", - "complement;": u"\u2201", - "complexes;": u"\u2102", - "cong;": u"\u2245", - "congdot;": u"\u2a6d", - "conint;": u"\u222e", - "copf;": u"\U0001d554", - "coprod;": u"\u2210", - "copy": u"\xa9", - "copy;": u"\xa9", - "copysr;": u"\u2117", - "crarr;": u"\u21b5", - "cross;": u"\u2717", - "cscr;": u"\U0001d4b8", - "csub;": u"\u2acf", - "csube;": u"\u2ad1", - "csup;": u"\u2ad0", - "csupe;": u"\u2ad2", - "ctdot;": u"\u22ef", - "cudarrl;": u"\u2938", - "cudarrr;": u"\u2935", - "cuepr;": u"\u22de", - "cuesc;": u"\u22df", - "cularr;": u"\u21b6", - "cularrp;": u"\u293d", - "cup;": u"\u222a", - "cupbrcap;": u"\u2a48", - "cupcap;": u"\u2a46", - "cupcup;": u"\u2a4a", - "cupdot;": u"\u228d", - "cupor;": u"\u2a45", - "cups;": u"\u222a\ufe00", - "curarr;": u"\u21b7", - "curarrm;": u"\u293c", - "curlyeqprec;": u"\u22de", - "curlyeqsucc;": u"\u22df", - "curlyvee;": u"\u22ce", - "curlywedge;": u"\u22cf", - "curren": u"\xa4", - "curren;": u"\xa4", - "curvearrowleft;": u"\u21b6", - "curvearrowright;": u"\u21b7", - "cuvee;": u"\u22ce", - "cuwed;": u"\u22cf", - "cwconint;": u"\u2232", - "cwint;": u"\u2231", - "cylcty;": u"\u232d", - "dArr;": u"\u21d3", - "dHar;": u"\u2965", - "dagger;": u"\u2020", - "daleth;": u"\u2138", - "darr;": u"\u2193", - "dash;": u"\u2010", - "dashv;": u"\u22a3", - "dbkarow;": u"\u290f", - "dblac;": u"\u02dd", - "dcaron;": u"\u010f", - "dcy;": u"\u0434", - "dd;": u"\u2146", - "ddagger;": u"\u2021", - "ddarr;": u"\u21ca", - "ddotseq;": u"\u2a77", - "deg": u"\xb0", - "deg;": u"\xb0", - "delta;": u"\u03b4", - "demptyv;": u"\u29b1", - "dfisht;": u"\u297f", - "dfr;": u"\U0001d521", - "dharl;": u"\u21c3", - "dharr;": u"\u21c2", - "diam;": u"\u22c4", - "diamond;": u"\u22c4", - "diamondsuit;": u"\u2666", - "diams;": u"\u2666", - "die;": u"\xa8", - "digamma;": u"\u03dd", - "disin;": u"\u22f2", - "div;": u"\xf7", - "divide": u"\xf7", - "divide;": u"\xf7", - "divideontimes;": u"\u22c7", - "divonx;": u"\u22c7", - "djcy;": u"\u0452", - "dlcorn;": u"\u231e", - "dlcrop;": u"\u230d", - "dollar;": u"$", - "dopf;": u"\U0001d555", - "dot;": u"\u02d9", - "doteq;": u"\u2250", - "doteqdot;": u"\u2251", - "dotminus;": u"\u2238", - "dotplus;": u"\u2214", - "dotsquare;": u"\u22a1", - "doublebarwedge;": u"\u2306", - "downarrow;": u"\u2193", - "downdownarrows;": u"\u21ca", - "downharpoonleft;": u"\u21c3", - "downharpoonright;": u"\u21c2", - "drbkarow;": u"\u2910", - "drcorn;": u"\u231f", - "drcrop;": u"\u230c", - "dscr;": u"\U0001d4b9", - "dscy;": u"\u0455", - "dsol;": u"\u29f6", - "dstrok;": u"\u0111", - "dtdot;": u"\u22f1", - "dtri;": u"\u25bf", - "dtrif;": u"\u25be", - "duarr;": u"\u21f5", - "duhar;": u"\u296f", - "dwangle;": u"\u29a6", - "dzcy;": u"\u045f", - "dzigrarr;": u"\u27ff", - "eDDot;": u"\u2a77", - "eDot;": u"\u2251", - "eacute": u"\xe9", - "eacute;": u"\xe9", - "easter;": u"\u2a6e", - "ecaron;": u"\u011b", - "ecir;": u"\u2256", - "ecirc": u"\xea", - "ecirc;": u"\xea", - "ecolon;": u"\u2255", - "ecy;": u"\u044d", - "edot;": u"\u0117", - "ee;": u"\u2147", - "efDot;": u"\u2252", - "efr;": u"\U0001d522", - "eg;": u"\u2a9a", - "egrave": u"\xe8", - "egrave;": u"\xe8", - "egs;": u"\u2a96", - "egsdot;": u"\u2a98", - "el;": u"\u2a99", - "elinters;": u"\u23e7", - "ell;": u"\u2113", - "els;": u"\u2a95", - "elsdot;": u"\u2a97", - "emacr;": u"\u0113", - "empty;": u"\u2205", - "emptyset;": u"\u2205", - "emptyv;": u"\u2205", - "emsp13;": u"\u2004", - "emsp14;": u"\u2005", - "emsp;": u"\u2003", - "eng;": u"\u014b", - "ensp;": u"\u2002", - "eogon;": u"\u0119", - "eopf;": u"\U0001d556", - "epar;": u"\u22d5", - "eparsl;": u"\u29e3", - "eplus;": u"\u2a71", - "epsi;": u"\u03b5", - "epsilon;": u"\u03b5", - "epsiv;": u"\u03f5", - "eqcirc;": u"\u2256", - "eqcolon;": u"\u2255", - "eqsim;": u"\u2242", - "eqslantgtr;": u"\u2a96", - "eqslantless;": u"\u2a95", - "equals;": u"=", - "equest;": u"\u225f", - "equiv;": u"\u2261", - "equivDD;": u"\u2a78", - "eqvparsl;": u"\u29e5", - "erDot;": u"\u2253", - "erarr;": u"\u2971", - "escr;": u"\u212f", - "esdot;": u"\u2250", - "esim;": u"\u2242", - "eta;": u"\u03b7", - "eth": u"\xf0", - "eth;": u"\xf0", - "euml": u"\xeb", - "euml;": u"\xeb", - "euro;": u"\u20ac", - "excl;": u"!", - "exist;": u"\u2203", - "expectation;": u"\u2130", - "exponentiale;": u"\u2147", - "fallingdotseq;": u"\u2252", - "fcy;": u"\u0444", - "female;": u"\u2640", - "ffilig;": u"\ufb03", - "fflig;": u"\ufb00", - "ffllig;": u"\ufb04", - "ffr;": u"\U0001d523", - "filig;": u"\ufb01", - "fjlig;": u"fj", - "flat;": u"\u266d", - "fllig;": u"\ufb02", - "fltns;": u"\u25b1", - "fnof;": u"\u0192", - "fopf;": u"\U0001d557", - "forall;": u"\u2200", - "fork;": u"\u22d4", - "forkv;": u"\u2ad9", - "fpartint;": u"\u2a0d", - "frac12": u"\xbd", - "frac12;": u"\xbd", - "frac13;": u"\u2153", - "frac14": u"\xbc", - "frac14;": u"\xbc", - "frac15;": u"\u2155", - "frac16;": u"\u2159", - "frac18;": u"\u215b", - "frac23;": u"\u2154", - "frac25;": u"\u2156", - "frac34": u"\xbe", - "frac34;": u"\xbe", - "frac35;": u"\u2157", - "frac38;": u"\u215c", - "frac45;": u"\u2158", - "frac56;": u"\u215a", - "frac58;": u"\u215d", - "frac78;": u"\u215e", - "frasl;": u"\u2044", - "frown;": u"\u2322", - "fscr;": u"\U0001d4bb", - "gE;": u"\u2267", - "gEl;": u"\u2a8c", - "gacute;": u"\u01f5", - "gamma;": u"\u03b3", - "gammad;": u"\u03dd", - "gap;": u"\u2a86", - "gbreve;": u"\u011f", - "gcirc;": u"\u011d", - "gcy;": u"\u0433", - "gdot;": u"\u0121", - "ge;": u"\u2265", - "gel;": u"\u22db", - "geq;": u"\u2265", - "geqq;": u"\u2267", - "geqslant;": u"\u2a7e", - "ges;": u"\u2a7e", - "gescc;": u"\u2aa9", - "gesdot;": u"\u2a80", - "gesdoto;": u"\u2a82", - "gesdotol;": u"\u2a84", - "gesl;": u"\u22db\ufe00", - "gesles;": u"\u2a94", - "gfr;": u"\U0001d524", - "gg;": u"\u226b", - "ggg;": u"\u22d9", - "gimel;": u"\u2137", - "gjcy;": u"\u0453", - "gl;": u"\u2277", - "glE;": u"\u2a92", - "gla;": u"\u2aa5", - "glj;": u"\u2aa4", - "gnE;": u"\u2269", - "gnap;": u"\u2a8a", - "gnapprox;": u"\u2a8a", - "gne;": u"\u2a88", - "gneq;": u"\u2a88", - "gneqq;": u"\u2269", - "gnsim;": u"\u22e7", - "gopf;": u"\U0001d558", - "grave;": u"`", - "gscr;": u"\u210a", - "gsim;": u"\u2273", - "gsime;": u"\u2a8e", - "gsiml;": u"\u2a90", - "gt": u">", - "gt;": u">", - "gtcc;": u"\u2aa7", - "gtcir;": u"\u2a7a", - "gtdot;": u"\u22d7", - "gtlPar;": u"\u2995", - "gtquest;": u"\u2a7c", - "gtrapprox;": u"\u2a86", - "gtrarr;": u"\u2978", - "gtrdot;": u"\u22d7", - "gtreqless;": u"\u22db", - "gtreqqless;": u"\u2a8c", - "gtrless;": u"\u2277", - "gtrsim;": u"\u2273", - "gvertneqq;": u"\u2269\ufe00", - "gvnE;": u"\u2269\ufe00", - "hArr;": u"\u21d4", - "hairsp;": u"\u200a", - "half;": u"\xbd", - "hamilt;": u"\u210b", - "hardcy;": u"\u044a", - "harr;": u"\u2194", - "harrcir;": u"\u2948", - "harrw;": u"\u21ad", - "hbar;": u"\u210f", - "hcirc;": u"\u0125", - "hearts;": u"\u2665", - "heartsuit;": u"\u2665", - "hellip;": u"\u2026", - "hercon;": u"\u22b9", - "hfr;": u"\U0001d525", - "hksearow;": u"\u2925", - "hkswarow;": u"\u2926", - "hoarr;": u"\u21ff", - "homtht;": u"\u223b", - "hookleftarrow;": u"\u21a9", - "hookrightarrow;": u"\u21aa", - "hopf;": u"\U0001d559", - "horbar;": u"\u2015", - "hscr;": u"\U0001d4bd", - "hslash;": u"\u210f", - "hstrok;": u"\u0127", - "hybull;": u"\u2043", - "hyphen;": u"\u2010", - "iacute": u"\xed", - "iacute;": u"\xed", - "ic;": u"\u2063", - "icirc": u"\xee", - "icirc;": u"\xee", - "icy;": u"\u0438", - "iecy;": u"\u0435", - "iexcl": u"\xa1", - "iexcl;": u"\xa1", - "iff;": u"\u21d4", - "ifr;": u"\U0001d526", - "igrave": u"\xec", - "igrave;": u"\xec", - "ii;": u"\u2148", - "iiiint;": u"\u2a0c", - "iiint;": u"\u222d", - "iinfin;": u"\u29dc", - "iiota;": u"\u2129", - "ijlig;": u"\u0133", - "imacr;": u"\u012b", - "image;": u"\u2111", - "imagline;": u"\u2110", - "imagpart;": u"\u2111", - "imath;": u"\u0131", - "imof;": u"\u22b7", - "imped;": u"\u01b5", - "in;": u"\u2208", - "incare;": u"\u2105", - "infin;": u"\u221e", - "infintie;": u"\u29dd", - "inodot;": u"\u0131", - "int;": u"\u222b", - "intcal;": u"\u22ba", - "integers;": u"\u2124", - "intercal;": u"\u22ba", - "intlarhk;": u"\u2a17", - "intprod;": u"\u2a3c", - "iocy;": u"\u0451", - "iogon;": u"\u012f", - "iopf;": u"\U0001d55a", - "iota;": u"\u03b9", - "iprod;": u"\u2a3c", - "iquest": u"\xbf", - "iquest;": u"\xbf", - "iscr;": u"\U0001d4be", - "isin;": u"\u2208", - "isinE;": u"\u22f9", - "isindot;": u"\u22f5", - "isins;": u"\u22f4", - "isinsv;": u"\u22f3", - "isinv;": u"\u2208", - "it;": u"\u2062", - "itilde;": u"\u0129", - "iukcy;": u"\u0456", - "iuml": u"\xef", - "iuml;": u"\xef", - "jcirc;": u"\u0135", - "jcy;": u"\u0439", - "jfr;": u"\U0001d527", - "jmath;": u"\u0237", - "jopf;": u"\U0001d55b", - "jscr;": u"\U0001d4bf", - "jsercy;": u"\u0458", - "jukcy;": u"\u0454", - "kappa;": u"\u03ba", - "kappav;": u"\u03f0", - "kcedil;": u"\u0137", - "kcy;": u"\u043a", - "kfr;": u"\U0001d528", - "kgreen;": u"\u0138", - "khcy;": u"\u0445", - "kjcy;": u"\u045c", - "kopf;": u"\U0001d55c", - "kscr;": u"\U0001d4c0", - "lAarr;": u"\u21da", - "lArr;": u"\u21d0", - "lAtail;": u"\u291b", - "lBarr;": u"\u290e", - "lE;": u"\u2266", - "lEg;": u"\u2a8b", - "lHar;": u"\u2962", - "lacute;": u"\u013a", - "laemptyv;": u"\u29b4", - "lagran;": u"\u2112", - "lambda;": u"\u03bb", - "lang;": u"\u27e8", - "langd;": u"\u2991", - "langle;": u"\u27e8", - "lap;": u"\u2a85", - "laquo": u"\xab", - "laquo;": u"\xab", - "larr;": u"\u2190", - "larrb;": u"\u21e4", - "larrbfs;": u"\u291f", - "larrfs;": u"\u291d", - "larrhk;": u"\u21a9", - "larrlp;": u"\u21ab", - "larrpl;": u"\u2939", - "larrsim;": u"\u2973", - "larrtl;": u"\u21a2", - "lat;": u"\u2aab", - "latail;": u"\u2919", - "late;": u"\u2aad", - "lates;": u"\u2aad\ufe00", - "lbarr;": u"\u290c", - "lbbrk;": u"\u2772", - "lbrace;": u"{", - "lbrack;": u"[", - "lbrke;": u"\u298b", - "lbrksld;": u"\u298f", - "lbrkslu;": u"\u298d", - "lcaron;": u"\u013e", - "lcedil;": u"\u013c", - "lceil;": u"\u2308", - "lcub;": u"{", - "lcy;": u"\u043b", - "ldca;": u"\u2936", - "ldquo;": u"\u201c", - "ldquor;": u"\u201e", - "ldrdhar;": u"\u2967", - "ldrushar;": u"\u294b", - "ldsh;": u"\u21b2", - "le;": u"\u2264", - "leftarrow;": u"\u2190", - "leftarrowtail;": u"\u21a2", - "leftharpoondown;": u"\u21bd", - "leftharpoonup;": u"\u21bc", - "leftleftarrows;": u"\u21c7", - "leftrightarrow;": u"\u2194", - "leftrightarrows;": u"\u21c6", - "leftrightharpoons;": u"\u21cb", - "leftrightsquigarrow;": u"\u21ad", - "leftthreetimes;": u"\u22cb", - "leg;": u"\u22da", - "leq;": u"\u2264", - "leqq;": u"\u2266", - "leqslant;": u"\u2a7d", - "les;": u"\u2a7d", - "lescc;": u"\u2aa8", - "lesdot;": u"\u2a7f", - "lesdoto;": u"\u2a81", - "lesdotor;": u"\u2a83", - "lesg;": u"\u22da\ufe00", - "lesges;": u"\u2a93", - "lessapprox;": u"\u2a85", - "lessdot;": u"\u22d6", - "lesseqgtr;": u"\u22da", - "lesseqqgtr;": u"\u2a8b", - "lessgtr;": u"\u2276", - "lesssim;": u"\u2272", - "lfisht;": u"\u297c", - "lfloor;": u"\u230a", - "lfr;": u"\U0001d529", - "lg;": u"\u2276", - "lgE;": u"\u2a91", - "lhard;": u"\u21bd", - "lharu;": u"\u21bc", - "lharul;": u"\u296a", - "lhblk;": u"\u2584", - "ljcy;": u"\u0459", - "ll;": u"\u226a", - "llarr;": u"\u21c7", - "llcorner;": u"\u231e", - "llhard;": u"\u296b", - "lltri;": u"\u25fa", - "lmidot;": u"\u0140", - "lmoust;": u"\u23b0", - "lmoustache;": u"\u23b0", - "lnE;": u"\u2268", - "lnap;": u"\u2a89", - "lnapprox;": u"\u2a89", - "lne;": u"\u2a87", - "lneq;": u"\u2a87", - "lneqq;": u"\u2268", - "lnsim;": u"\u22e6", - "loang;": u"\u27ec", - "loarr;": u"\u21fd", - "lobrk;": u"\u27e6", - "longleftarrow;": u"\u27f5", - "longleftrightarrow;": u"\u27f7", - "longmapsto;": u"\u27fc", - "longrightarrow;": u"\u27f6", - "looparrowleft;": u"\u21ab", - "looparrowright;": u"\u21ac", - "lopar;": u"\u2985", - "lopf;": u"\U0001d55d", - "loplus;": u"\u2a2d", - "lotimes;": u"\u2a34", - "lowast;": u"\u2217", - "lowbar;": u"_", - "loz;": u"\u25ca", - "lozenge;": u"\u25ca", - "lozf;": u"\u29eb", - "lpar;": u"(", - "lparlt;": u"\u2993", - "lrarr;": u"\u21c6", - "lrcorner;": u"\u231f", - "lrhar;": u"\u21cb", - "lrhard;": u"\u296d", - "lrm;": u"\u200e", - "lrtri;": u"\u22bf", - "lsaquo;": u"\u2039", - "lscr;": u"\U0001d4c1", - "lsh;": u"\u21b0", - "lsim;": u"\u2272", - "lsime;": u"\u2a8d", - "lsimg;": u"\u2a8f", - "lsqb;": u"[", - "lsquo;": u"\u2018", - "lsquor;": u"\u201a", - "lstrok;": u"\u0142", - "lt": u"<", - "lt;": u"<", - "ltcc;": u"\u2aa6", - "ltcir;": u"\u2a79", - "ltdot;": u"\u22d6", - "lthree;": u"\u22cb", - "ltimes;": u"\u22c9", - "ltlarr;": u"\u2976", - "ltquest;": u"\u2a7b", - "ltrPar;": u"\u2996", - "ltri;": u"\u25c3", - "ltrie;": u"\u22b4", - "ltrif;": u"\u25c2", - "lurdshar;": u"\u294a", - "luruhar;": u"\u2966", - "lvertneqq;": u"\u2268\ufe00", - "lvnE;": u"\u2268\ufe00", - "mDDot;": u"\u223a", - "macr": u"\xaf", - "macr;": u"\xaf", - "male;": u"\u2642", - "malt;": u"\u2720", - "maltese;": u"\u2720", - "map;": u"\u21a6", - "mapsto;": u"\u21a6", - "mapstodown;": u"\u21a7", - "mapstoleft;": u"\u21a4", - "mapstoup;": u"\u21a5", - "marker;": u"\u25ae", - "mcomma;": u"\u2a29", - "mcy;": u"\u043c", - "mdash;": u"\u2014", - "measuredangle;": u"\u2221", - "mfr;": u"\U0001d52a", - "mho;": u"\u2127", - "micro": u"\xb5", - "micro;": u"\xb5", - "mid;": u"\u2223", - "midast;": u"*", - "midcir;": u"\u2af0", - "middot": u"\xb7", - "middot;": u"\xb7", - "minus;": u"\u2212", - "minusb;": u"\u229f", - "minusd;": u"\u2238", - "minusdu;": u"\u2a2a", - "mlcp;": u"\u2adb", - "mldr;": u"\u2026", - "mnplus;": u"\u2213", - "models;": u"\u22a7", - "mopf;": u"\U0001d55e", - "mp;": u"\u2213", - "mscr;": u"\U0001d4c2", - "mstpos;": u"\u223e", - "mu;": u"\u03bc", - "multimap;": u"\u22b8", - "mumap;": u"\u22b8", - "nGg;": u"\u22d9\u0338", - "nGt;": u"\u226b\u20d2", - "nGtv;": u"\u226b\u0338", - "nLeftarrow;": u"\u21cd", - "nLeftrightarrow;": u"\u21ce", - "nLl;": u"\u22d8\u0338", - "nLt;": u"\u226a\u20d2", - "nLtv;": u"\u226a\u0338", - "nRightarrow;": u"\u21cf", - "nVDash;": u"\u22af", - "nVdash;": u"\u22ae", - "nabla;": u"\u2207", - "nacute;": u"\u0144", - "nang;": u"\u2220\u20d2", - "nap;": u"\u2249", - "napE;": u"\u2a70\u0338", - "napid;": u"\u224b\u0338", - "napos;": u"\u0149", - "napprox;": u"\u2249", - "natur;": u"\u266e", - "natural;": u"\u266e", - "naturals;": u"\u2115", - "nbsp": u"\xa0", - "nbsp;": u"\xa0", - "nbump;": u"\u224e\u0338", - "nbumpe;": u"\u224f\u0338", - "ncap;": u"\u2a43", - "ncaron;": u"\u0148", - "ncedil;": u"\u0146", - "ncong;": u"\u2247", - "ncongdot;": u"\u2a6d\u0338", - "ncup;": u"\u2a42", - "ncy;": u"\u043d", - "ndash;": u"\u2013", - "ne;": u"\u2260", - "neArr;": u"\u21d7", - "nearhk;": u"\u2924", - "nearr;": u"\u2197", - "nearrow;": u"\u2197", - "nedot;": u"\u2250\u0338", - "nequiv;": u"\u2262", - "nesear;": u"\u2928", - "nesim;": u"\u2242\u0338", - "nexist;": u"\u2204", - "nexists;": u"\u2204", - "nfr;": u"\U0001d52b", - "ngE;": u"\u2267\u0338", - "nge;": u"\u2271", - "ngeq;": u"\u2271", - "ngeqq;": u"\u2267\u0338", - "ngeqslant;": u"\u2a7e\u0338", - "nges;": u"\u2a7e\u0338", - "ngsim;": u"\u2275", - "ngt;": u"\u226f", - "ngtr;": u"\u226f", - "nhArr;": u"\u21ce", - "nharr;": u"\u21ae", - "nhpar;": u"\u2af2", - "ni;": u"\u220b", - "nis;": u"\u22fc", - "nisd;": u"\u22fa", - "niv;": u"\u220b", - "njcy;": u"\u045a", - "nlArr;": u"\u21cd", - "nlE;": u"\u2266\u0338", - "nlarr;": u"\u219a", - "nldr;": u"\u2025", - "nle;": u"\u2270", - "nleftarrow;": u"\u219a", - "nleftrightarrow;": u"\u21ae", - "nleq;": u"\u2270", - "nleqq;": u"\u2266\u0338", - "nleqslant;": u"\u2a7d\u0338", - "nles;": u"\u2a7d\u0338", - "nless;": u"\u226e", - "nlsim;": u"\u2274", - "nlt;": u"\u226e", - "nltri;": u"\u22ea", - "nltrie;": u"\u22ec", - "nmid;": u"\u2224", - "nopf;": u"\U0001d55f", - "not": u"\xac", - "not;": u"\xac", - "notin;": u"\u2209", - "notinE;": u"\u22f9\u0338", - "notindot;": u"\u22f5\u0338", - "notinva;": u"\u2209", - "notinvb;": u"\u22f7", - "notinvc;": u"\u22f6", - "notni;": u"\u220c", - "notniva;": u"\u220c", - "notnivb;": u"\u22fe", - "notnivc;": u"\u22fd", - "npar;": u"\u2226", - "nparallel;": u"\u2226", - "nparsl;": u"\u2afd\u20e5", - "npart;": u"\u2202\u0338", - "npolint;": u"\u2a14", - "npr;": u"\u2280", - "nprcue;": u"\u22e0", - "npre;": u"\u2aaf\u0338", - "nprec;": u"\u2280", - "npreceq;": u"\u2aaf\u0338", - "nrArr;": u"\u21cf", - "nrarr;": u"\u219b", - "nrarrc;": u"\u2933\u0338", - "nrarrw;": u"\u219d\u0338", - "nrightarrow;": u"\u219b", - "nrtri;": u"\u22eb", - "nrtrie;": u"\u22ed", - "nsc;": u"\u2281", - "nsccue;": u"\u22e1", - "nsce;": u"\u2ab0\u0338", - "nscr;": u"\U0001d4c3", - "nshortmid;": u"\u2224", - "nshortparallel;": u"\u2226", - "nsim;": u"\u2241", - "nsime;": u"\u2244", - "nsimeq;": u"\u2244", - "nsmid;": u"\u2224", - "nspar;": u"\u2226", - "nsqsube;": u"\u22e2", - "nsqsupe;": u"\u22e3", - "nsub;": u"\u2284", - "nsubE;": u"\u2ac5\u0338", - "nsube;": u"\u2288", - "nsubset;": u"\u2282\u20d2", - "nsubseteq;": u"\u2288", - "nsubseteqq;": u"\u2ac5\u0338", - "nsucc;": u"\u2281", - "nsucceq;": u"\u2ab0\u0338", - "nsup;": u"\u2285", - "nsupE;": u"\u2ac6\u0338", - "nsupe;": u"\u2289", - "nsupset;": u"\u2283\u20d2", - "nsupseteq;": u"\u2289", - "nsupseteqq;": u"\u2ac6\u0338", - "ntgl;": u"\u2279", - "ntilde": u"\xf1", - "ntilde;": u"\xf1", - "ntlg;": u"\u2278", - "ntriangleleft;": u"\u22ea", - "ntrianglelefteq;": u"\u22ec", - "ntriangleright;": u"\u22eb", - "ntrianglerighteq;": u"\u22ed", - "nu;": u"\u03bd", - "num;": u"#", - "numero;": u"\u2116", - "numsp;": u"\u2007", - "nvDash;": u"\u22ad", - "nvHarr;": u"\u2904", - "nvap;": u"\u224d\u20d2", - "nvdash;": u"\u22ac", - "nvge;": u"\u2265\u20d2", - "nvgt;": u">\u20d2", - "nvinfin;": u"\u29de", - "nvlArr;": u"\u2902", - "nvle;": u"\u2264\u20d2", - "nvlt;": u"<\u20d2", - "nvltrie;": u"\u22b4\u20d2", - "nvrArr;": u"\u2903", - "nvrtrie;": u"\u22b5\u20d2", - "nvsim;": u"\u223c\u20d2", - "nwArr;": u"\u21d6", - "nwarhk;": u"\u2923", - "nwarr;": u"\u2196", - "nwarrow;": u"\u2196", - "nwnear;": u"\u2927", - "oS;": u"\u24c8", - "oacute": u"\xf3", - "oacute;": u"\xf3", - "oast;": u"\u229b", - "ocir;": u"\u229a", - "ocirc": u"\xf4", - "ocirc;": u"\xf4", - "ocy;": u"\u043e", - "odash;": u"\u229d", - "odblac;": u"\u0151", - "odiv;": u"\u2a38", - "odot;": u"\u2299", - "odsold;": u"\u29bc", - "oelig;": u"\u0153", - "ofcir;": u"\u29bf", - "ofr;": u"\U0001d52c", - "ogon;": u"\u02db", - "ograve": u"\xf2", - "ograve;": u"\xf2", - "ogt;": u"\u29c1", - "ohbar;": u"\u29b5", - "ohm;": u"\u03a9", - "oint;": u"\u222e", - "olarr;": u"\u21ba", - "olcir;": u"\u29be", - "olcross;": u"\u29bb", - "oline;": u"\u203e", - "olt;": u"\u29c0", - "omacr;": u"\u014d", - "omega;": u"\u03c9", - "omicron;": u"\u03bf", - "omid;": u"\u29b6", - "ominus;": u"\u2296", - "oopf;": u"\U0001d560", - "opar;": u"\u29b7", - "operp;": u"\u29b9", - "oplus;": u"\u2295", - "or;": u"\u2228", - "orarr;": u"\u21bb", - "ord;": u"\u2a5d", - "order;": u"\u2134", - "orderof;": u"\u2134", - "ordf": u"\xaa", - "ordf;": u"\xaa", - "ordm": u"\xba", - "ordm;": u"\xba", - "origof;": u"\u22b6", - "oror;": u"\u2a56", - "orslope;": u"\u2a57", - "orv;": u"\u2a5b", - "oscr;": u"\u2134", - "oslash": u"\xf8", - "oslash;": u"\xf8", - "osol;": u"\u2298", - "otilde": u"\xf5", - "otilde;": u"\xf5", - "otimes;": u"\u2297", - "otimesas;": u"\u2a36", - "ouml": u"\xf6", - "ouml;": u"\xf6", - "ovbar;": u"\u233d", - "par;": u"\u2225", - "para": u"\xb6", - "para;": u"\xb6", - "parallel;": u"\u2225", - "parsim;": u"\u2af3", - "parsl;": u"\u2afd", - "part;": u"\u2202", - "pcy;": u"\u043f", - "percnt;": u"%", - "period;": u".", - "permil;": u"\u2030", - "perp;": u"\u22a5", - "pertenk;": u"\u2031", - "pfr;": u"\U0001d52d", - "phi;": u"\u03c6", - "phiv;": u"\u03d5", - "phmmat;": u"\u2133", - "phone;": u"\u260e", - "pi;": u"\u03c0", - "pitchfork;": u"\u22d4", - "piv;": u"\u03d6", - "planck;": u"\u210f", - "planckh;": u"\u210e", - "plankv;": u"\u210f", - "plus;": u"+", - "plusacir;": u"\u2a23", - "plusb;": u"\u229e", - "pluscir;": u"\u2a22", - "plusdo;": u"\u2214", - "plusdu;": u"\u2a25", - "pluse;": u"\u2a72", - "plusmn": u"\xb1", - "plusmn;": u"\xb1", - "plussim;": u"\u2a26", - "plustwo;": u"\u2a27", - "pm;": u"\xb1", - "pointint;": u"\u2a15", - "popf;": u"\U0001d561", - "pound": u"\xa3", - "pound;": u"\xa3", - "pr;": u"\u227a", - "prE;": u"\u2ab3", - "prap;": u"\u2ab7", - "prcue;": u"\u227c", - "pre;": u"\u2aaf", - "prec;": u"\u227a", - "precapprox;": u"\u2ab7", - "preccurlyeq;": u"\u227c", - "preceq;": u"\u2aaf", - "precnapprox;": u"\u2ab9", - "precneqq;": u"\u2ab5", - "precnsim;": u"\u22e8", - "precsim;": u"\u227e", - "prime;": u"\u2032", - "primes;": u"\u2119", - "prnE;": u"\u2ab5", - "prnap;": u"\u2ab9", - "prnsim;": u"\u22e8", - "prod;": u"\u220f", - "profalar;": u"\u232e", - "profline;": u"\u2312", - "profsurf;": u"\u2313", - "prop;": u"\u221d", - "propto;": u"\u221d", - "prsim;": u"\u227e", - "prurel;": u"\u22b0", - "pscr;": u"\U0001d4c5", - "psi;": u"\u03c8", - "puncsp;": u"\u2008", - "qfr;": u"\U0001d52e", - "qint;": u"\u2a0c", - "qopf;": u"\U0001d562", - "qprime;": u"\u2057", - "qscr;": u"\U0001d4c6", - "quaternions;": u"\u210d", - "quatint;": u"\u2a16", - "quest;": u"?", - "questeq;": u"\u225f", - "quot": u"\"", - "quot;": u"\"", - "rAarr;": u"\u21db", - "rArr;": u"\u21d2", - "rAtail;": u"\u291c", - "rBarr;": u"\u290f", - "rHar;": u"\u2964", - "race;": u"\u223d\u0331", - "racute;": u"\u0155", - "radic;": u"\u221a", - "raemptyv;": u"\u29b3", - "rang;": u"\u27e9", - "rangd;": u"\u2992", - "range;": u"\u29a5", - "rangle;": u"\u27e9", - "raquo": u"\xbb", - "raquo;": u"\xbb", - "rarr;": u"\u2192", - "rarrap;": u"\u2975", - "rarrb;": u"\u21e5", - "rarrbfs;": u"\u2920", - "rarrc;": u"\u2933", - "rarrfs;": u"\u291e", - "rarrhk;": u"\u21aa", - "rarrlp;": u"\u21ac", - "rarrpl;": u"\u2945", - "rarrsim;": u"\u2974", - "rarrtl;": u"\u21a3", - "rarrw;": u"\u219d", - "ratail;": u"\u291a", - "ratio;": u"\u2236", - "rationals;": u"\u211a", - "rbarr;": u"\u290d", - "rbbrk;": u"\u2773", - "rbrace;": u"}", - "rbrack;": u"]", - "rbrke;": u"\u298c", - "rbrksld;": u"\u298e", - "rbrkslu;": u"\u2990", - "rcaron;": u"\u0159", - "rcedil;": u"\u0157", - "rceil;": u"\u2309", - "rcub;": u"}", - "rcy;": u"\u0440", - "rdca;": u"\u2937", - "rdldhar;": u"\u2969", - "rdquo;": u"\u201d", - "rdquor;": u"\u201d", - "rdsh;": u"\u21b3", - "real;": u"\u211c", - "realine;": u"\u211b", - "realpart;": u"\u211c", - "reals;": u"\u211d", - "rect;": u"\u25ad", - "reg": u"\xae", - "reg;": u"\xae", - "rfisht;": u"\u297d", - "rfloor;": u"\u230b", - "rfr;": u"\U0001d52f", - "rhard;": u"\u21c1", - "rharu;": u"\u21c0", - "rharul;": u"\u296c", - "rho;": u"\u03c1", - "rhov;": u"\u03f1", - "rightarrow;": u"\u2192", - "rightarrowtail;": u"\u21a3", - "rightharpoondown;": u"\u21c1", - "rightharpoonup;": u"\u21c0", - "rightleftarrows;": u"\u21c4", - "rightleftharpoons;": u"\u21cc", - "rightrightarrows;": u"\u21c9", - "rightsquigarrow;": u"\u219d", - "rightthreetimes;": u"\u22cc", - "ring;": u"\u02da", - "risingdotseq;": u"\u2253", - "rlarr;": u"\u21c4", - "rlhar;": u"\u21cc", - "rlm;": u"\u200f", - "rmoust;": u"\u23b1", - "rmoustache;": u"\u23b1", - "rnmid;": u"\u2aee", - "roang;": u"\u27ed", - "roarr;": u"\u21fe", - "robrk;": u"\u27e7", - "ropar;": u"\u2986", - "ropf;": u"\U0001d563", - "roplus;": u"\u2a2e", - "rotimes;": u"\u2a35", - "rpar;": u")", - "rpargt;": u"\u2994", - "rppolint;": u"\u2a12", - "rrarr;": u"\u21c9", - "rsaquo;": u"\u203a", - "rscr;": u"\U0001d4c7", - "rsh;": u"\u21b1", - "rsqb;": u"]", - "rsquo;": u"\u2019", - "rsquor;": u"\u2019", - "rthree;": u"\u22cc", - "rtimes;": u"\u22ca", - "rtri;": u"\u25b9", - "rtrie;": u"\u22b5", - "rtrif;": u"\u25b8", - "rtriltri;": u"\u29ce", - "ruluhar;": u"\u2968", - "rx;": u"\u211e", - "sacute;": u"\u015b", - "sbquo;": u"\u201a", - "sc;": u"\u227b", - "scE;": u"\u2ab4", - "scap;": u"\u2ab8", - "scaron;": u"\u0161", - "sccue;": u"\u227d", - "sce;": u"\u2ab0", - "scedil;": u"\u015f", - "scirc;": u"\u015d", - "scnE;": u"\u2ab6", - "scnap;": u"\u2aba", - "scnsim;": u"\u22e9", - "scpolint;": u"\u2a13", - "scsim;": u"\u227f", - "scy;": u"\u0441", - "sdot;": u"\u22c5", - "sdotb;": u"\u22a1", - "sdote;": u"\u2a66", - "seArr;": u"\u21d8", - "searhk;": u"\u2925", - "searr;": u"\u2198", - "searrow;": u"\u2198", - "sect": u"\xa7", - "sect;": u"\xa7", - "semi;": u";", - "seswar;": u"\u2929", - "setminus;": u"\u2216", - "setmn;": u"\u2216", - "sext;": u"\u2736", - "sfr;": u"\U0001d530", - "sfrown;": u"\u2322", - "sharp;": u"\u266f", - "shchcy;": u"\u0449", - "shcy;": u"\u0448", - "shortmid;": u"\u2223", - "shortparallel;": u"\u2225", - "shy": u"\xad", - "shy;": u"\xad", - "sigma;": u"\u03c3", - "sigmaf;": u"\u03c2", - "sigmav;": u"\u03c2", - "sim;": u"\u223c", - "simdot;": u"\u2a6a", - "sime;": u"\u2243", - "simeq;": u"\u2243", - "simg;": u"\u2a9e", - "simgE;": u"\u2aa0", - "siml;": u"\u2a9d", - "simlE;": u"\u2a9f", - "simne;": u"\u2246", - "simplus;": u"\u2a24", - "simrarr;": u"\u2972", - "slarr;": u"\u2190", - "smallsetminus;": u"\u2216", - "smashp;": u"\u2a33", - "smeparsl;": u"\u29e4", - "smid;": u"\u2223", - "smile;": u"\u2323", - "smt;": u"\u2aaa", - "smte;": u"\u2aac", - "smtes;": u"\u2aac\ufe00", - "softcy;": u"\u044c", - "sol;": u"/", - "solb;": u"\u29c4", - "solbar;": u"\u233f", - "sopf;": u"\U0001d564", - "spades;": u"\u2660", - "spadesuit;": u"\u2660", - "spar;": u"\u2225", - "sqcap;": u"\u2293", - "sqcaps;": u"\u2293\ufe00", - "sqcup;": u"\u2294", - "sqcups;": u"\u2294\ufe00", - "sqsub;": u"\u228f", - "sqsube;": u"\u2291", - "sqsubset;": u"\u228f", - "sqsubseteq;": u"\u2291", - "sqsup;": u"\u2290", - "sqsupe;": u"\u2292", - "sqsupset;": u"\u2290", - "sqsupseteq;": u"\u2292", - "squ;": u"\u25a1", - "square;": u"\u25a1", - "squarf;": u"\u25aa", - "squf;": u"\u25aa", - "srarr;": u"\u2192", - "sscr;": u"\U0001d4c8", - "ssetmn;": u"\u2216", - "ssmile;": u"\u2323", - "sstarf;": u"\u22c6", - "star;": u"\u2606", - "starf;": u"\u2605", - "straightepsilon;": u"\u03f5", - "straightphi;": u"\u03d5", - "strns;": u"\xaf", - "sub;": u"\u2282", - "subE;": u"\u2ac5", - "subdot;": u"\u2abd", - "sube;": u"\u2286", - "subedot;": u"\u2ac3", - "submult;": u"\u2ac1", - "subnE;": u"\u2acb", - "subne;": u"\u228a", - "subplus;": u"\u2abf", - "subrarr;": u"\u2979", - "subset;": u"\u2282", - "subseteq;": u"\u2286", - "subseteqq;": u"\u2ac5", - "subsetneq;": u"\u228a", - "subsetneqq;": u"\u2acb", - "subsim;": u"\u2ac7", - "subsub;": u"\u2ad5", - "subsup;": u"\u2ad3", - "succ;": u"\u227b", - "succapprox;": u"\u2ab8", - "succcurlyeq;": u"\u227d", - "succeq;": u"\u2ab0", - "succnapprox;": u"\u2aba", - "succneqq;": u"\u2ab6", - "succnsim;": u"\u22e9", - "succsim;": u"\u227f", - "sum;": u"\u2211", - "sung;": u"\u266a", - "sup1": u"\xb9", - "sup1;": u"\xb9", - "sup2": u"\xb2", - "sup2;": u"\xb2", - "sup3": u"\xb3", - "sup3;": u"\xb3", - "sup;": u"\u2283", - "supE;": u"\u2ac6", - "supdot;": u"\u2abe", - "supdsub;": u"\u2ad8", - "supe;": u"\u2287", - "supedot;": u"\u2ac4", - "suphsol;": u"\u27c9", - "suphsub;": u"\u2ad7", - "suplarr;": u"\u297b", - "supmult;": u"\u2ac2", - "supnE;": u"\u2acc", - "supne;": u"\u228b", - "supplus;": u"\u2ac0", - "supset;": u"\u2283", - "supseteq;": u"\u2287", - "supseteqq;": u"\u2ac6", - "supsetneq;": u"\u228b", - "supsetneqq;": u"\u2acc", - "supsim;": u"\u2ac8", - "supsub;": u"\u2ad4", - "supsup;": u"\u2ad6", - "swArr;": u"\u21d9", - "swarhk;": u"\u2926", - "swarr;": u"\u2199", - "swarrow;": u"\u2199", - "swnwar;": u"\u292a", - "szlig": u"\xdf", - "szlig;": u"\xdf", - "target;": u"\u2316", - "tau;": u"\u03c4", - "tbrk;": u"\u23b4", - "tcaron;": u"\u0165", - "tcedil;": u"\u0163", - "tcy;": u"\u0442", - "tdot;": u"\u20db", - "telrec;": u"\u2315", - "tfr;": u"\U0001d531", - "there4;": u"\u2234", - "therefore;": u"\u2234", - "theta;": u"\u03b8", - "thetasym;": u"\u03d1", - "thetav;": u"\u03d1", - "thickapprox;": u"\u2248", - "thicksim;": u"\u223c", - "thinsp;": u"\u2009", - "thkap;": u"\u2248", - "thksim;": u"\u223c", - "thorn": u"\xfe", - "thorn;": u"\xfe", - "tilde;": u"\u02dc", - "times": u"\xd7", - "times;": u"\xd7", - "timesb;": u"\u22a0", - "timesbar;": u"\u2a31", - "timesd;": u"\u2a30", - "tint;": u"\u222d", - "toea;": u"\u2928", - "top;": u"\u22a4", - "topbot;": u"\u2336", - "topcir;": u"\u2af1", - "topf;": u"\U0001d565", - "topfork;": u"\u2ada", - "tosa;": u"\u2929", - "tprime;": u"\u2034", - "trade;": u"\u2122", - "triangle;": u"\u25b5", - "triangledown;": u"\u25bf", - "triangleleft;": u"\u25c3", - "trianglelefteq;": u"\u22b4", - "triangleq;": u"\u225c", - "triangleright;": u"\u25b9", - "trianglerighteq;": u"\u22b5", - "tridot;": u"\u25ec", - "trie;": u"\u225c", - "triminus;": u"\u2a3a", - "triplus;": u"\u2a39", - "trisb;": u"\u29cd", - "tritime;": u"\u2a3b", - "trpezium;": u"\u23e2", - "tscr;": u"\U0001d4c9", - "tscy;": u"\u0446", - "tshcy;": u"\u045b", - "tstrok;": u"\u0167", - "twixt;": u"\u226c", - "twoheadleftarrow;": u"\u219e", - "twoheadrightarrow;": u"\u21a0", - "uArr;": u"\u21d1", - "uHar;": u"\u2963", - "uacute": u"\xfa", - "uacute;": u"\xfa", - "uarr;": u"\u2191", - "ubrcy;": u"\u045e", - "ubreve;": u"\u016d", - "ucirc": u"\xfb", - "ucirc;": u"\xfb", - "ucy;": u"\u0443", - "udarr;": u"\u21c5", - "udblac;": u"\u0171", - "udhar;": u"\u296e", - "ufisht;": u"\u297e", - "ufr;": u"\U0001d532", - "ugrave": u"\xf9", - "ugrave;": u"\xf9", - "uharl;": u"\u21bf", - "uharr;": u"\u21be", - "uhblk;": u"\u2580", - "ulcorn;": u"\u231c", - "ulcorner;": u"\u231c", - "ulcrop;": u"\u230f", - "ultri;": u"\u25f8", - "umacr;": u"\u016b", - "uml": u"\xa8", - "uml;": u"\xa8", - "uogon;": u"\u0173", - "uopf;": u"\U0001d566", - "uparrow;": u"\u2191", - "updownarrow;": u"\u2195", - "upharpoonleft;": u"\u21bf", - "upharpoonright;": u"\u21be", - "uplus;": u"\u228e", - "upsi;": u"\u03c5", - "upsih;": u"\u03d2", - "upsilon;": u"\u03c5", - "upuparrows;": u"\u21c8", - "urcorn;": u"\u231d", - "urcorner;": u"\u231d", - "urcrop;": u"\u230e", - "uring;": u"\u016f", - "urtri;": u"\u25f9", - "uscr;": u"\U0001d4ca", - "utdot;": u"\u22f0", - "utilde;": u"\u0169", - "utri;": u"\u25b5", - "utrif;": u"\u25b4", - "uuarr;": u"\u21c8", - "uuml": u"\xfc", - "uuml;": u"\xfc", - "uwangle;": u"\u29a7", - "vArr;": u"\u21d5", - "vBar;": u"\u2ae8", - "vBarv;": u"\u2ae9", - "vDash;": u"\u22a8", - "vangrt;": u"\u299c", - "varepsilon;": u"\u03f5", - "varkappa;": u"\u03f0", - "varnothing;": u"\u2205", - "varphi;": u"\u03d5", - "varpi;": u"\u03d6", - "varpropto;": u"\u221d", - "varr;": u"\u2195", - "varrho;": u"\u03f1", - "varsigma;": u"\u03c2", - "varsubsetneq;": u"\u228a\ufe00", - "varsubsetneqq;": u"\u2acb\ufe00", - "varsupsetneq;": u"\u228b\ufe00", - "varsupsetneqq;": u"\u2acc\ufe00", - "vartheta;": u"\u03d1", - "vartriangleleft;": u"\u22b2", - "vartriangleright;": u"\u22b3", - "vcy;": u"\u0432", - "vdash;": u"\u22a2", - "vee;": u"\u2228", - "veebar;": u"\u22bb", - "veeeq;": u"\u225a", - "vellip;": u"\u22ee", - "verbar;": u"|", - "vert;": u"|", - "vfr;": u"\U0001d533", - "vltri;": u"\u22b2", - "vnsub;": u"\u2282\u20d2", - "vnsup;": u"\u2283\u20d2", - "vopf;": u"\U0001d567", - "vprop;": u"\u221d", - "vrtri;": u"\u22b3", - "vscr;": u"\U0001d4cb", - "vsubnE;": u"\u2acb\ufe00", - "vsubne;": u"\u228a\ufe00", - "vsupnE;": u"\u2acc\ufe00", - "vsupne;": u"\u228b\ufe00", - "vzigzag;": u"\u299a", - "wcirc;": u"\u0175", - "wedbar;": u"\u2a5f", - "wedge;": u"\u2227", - "wedgeq;": u"\u2259", - "weierp;": u"\u2118", - "wfr;": u"\U0001d534", - "wopf;": u"\U0001d568", - "wp;": u"\u2118", - "wr;": u"\u2240", - "wreath;": u"\u2240", - "wscr;": u"\U0001d4cc", - "xcap;": u"\u22c2", - "xcirc;": u"\u25ef", - "xcup;": u"\u22c3", - "xdtri;": u"\u25bd", - "xfr;": u"\U0001d535", - "xhArr;": u"\u27fa", - "xharr;": u"\u27f7", - "xi;": u"\u03be", - "xlArr;": u"\u27f8", - "xlarr;": u"\u27f5", - "xmap;": u"\u27fc", - "xnis;": u"\u22fb", - "xodot;": u"\u2a00", - "xopf;": u"\U0001d569", - "xoplus;": u"\u2a01", - "xotime;": u"\u2a02", - "xrArr;": u"\u27f9", - "xrarr;": u"\u27f6", - "xscr;": u"\U0001d4cd", - "xsqcup;": u"\u2a06", - "xuplus;": u"\u2a04", - "xutri;": u"\u25b3", - "xvee;": u"\u22c1", - "xwedge;": u"\u22c0", - "yacute": u"\xfd", - "yacute;": u"\xfd", - "yacy;": u"\u044f", - "ycirc;": u"\u0177", - "ycy;": u"\u044b", - "yen": u"\xa5", - "yen;": u"\xa5", - "yfr;": u"\U0001d536", - "yicy;": u"\u0457", - "yopf;": u"\U0001d56a", - "yscr;": u"\U0001d4ce", - "yucy;": u"\u044e", - "yuml": u"\xff", - "yuml;": u"\xff", - "zacute;": u"\u017a", - "zcaron;": u"\u017e", - "zcy;": u"\u0437", - "zdot;": u"\u017c", - "zeetrf;": u"\u2128", - "zeta;": u"\u03b6", - "zfr;": u"\U0001d537", - "zhcy;": u"\u0436", - "zigrarr;": u"\u21dd", - "zopf;": u"\U0001d56b", - "zscr;": u"\U0001d4cf", - "zwj;": u"\u200d", - "zwnj;": u"\u200c", + "AElig": "\xc6", + "AElig;": "\xc6", + "AMP": "&", + "AMP;": "&", + "Aacute": "\xc1", + "Aacute;": "\xc1", + "Abreve;": "\u0102", + "Acirc": "\xc2", + "Acirc;": "\xc2", + "Acy;": "\u0410", + "Afr;": "\U0001d504", + "Agrave": "\xc0", + "Agrave;": "\xc0", + "Alpha;": "\u0391", + "Amacr;": "\u0100", + "And;": "\u2a53", + "Aogon;": "\u0104", + "Aopf;": "\U0001d538", + "ApplyFunction;": "\u2061", + "Aring": "\xc5", + "Aring;": "\xc5", + "Ascr;": "\U0001d49c", + "Assign;": "\u2254", + "Atilde": "\xc3", + "Atilde;": "\xc3", + "Auml": "\xc4", + "Auml;": "\xc4", + "Backslash;": "\u2216", + "Barv;": "\u2ae7", + "Barwed;": "\u2306", + "Bcy;": "\u0411", + "Because;": "\u2235", + "Bernoullis;": "\u212c", + "Beta;": "\u0392", + "Bfr;": "\U0001d505", + "Bopf;": "\U0001d539", + "Breve;": "\u02d8", + "Bscr;": "\u212c", + "Bumpeq;": "\u224e", + "CHcy;": "\u0427", + "COPY": "\xa9", + "COPY;": "\xa9", + "Cacute;": "\u0106", + "Cap;": "\u22d2", + "CapitalDifferentialD;": "\u2145", + "Cayleys;": "\u212d", + "Ccaron;": "\u010c", + "Ccedil": "\xc7", + "Ccedil;": "\xc7", + "Ccirc;": "\u0108", + "Cconint;": "\u2230", + "Cdot;": "\u010a", + "Cedilla;": "\xb8", + "CenterDot;": "\xb7", + "Cfr;": "\u212d", + "Chi;": "\u03a7", + "CircleDot;": "\u2299", + "CircleMinus;": "\u2296", + "CirclePlus;": "\u2295", + "CircleTimes;": "\u2297", + "ClockwiseContourIntegral;": "\u2232", + "CloseCurlyDoubleQuote;": "\u201d", + "CloseCurlyQuote;": "\u2019", + "Colon;": "\u2237", + "Colone;": "\u2a74", + "Congruent;": "\u2261", + "Conint;": "\u222f", + "ContourIntegral;": "\u222e", + "Copf;": "\u2102", + "Coproduct;": "\u2210", + "CounterClockwiseContourIntegral;": "\u2233", + "Cross;": "\u2a2f", + "Cscr;": "\U0001d49e", + "Cup;": "\u22d3", + "CupCap;": "\u224d", + "DD;": "\u2145", + "DDotrahd;": "\u2911", + "DJcy;": "\u0402", + "DScy;": "\u0405", + "DZcy;": "\u040f", + "Dagger;": "\u2021", + "Darr;": "\u21a1", + "Dashv;": "\u2ae4", + "Dcaron;": "\u010e", + "Dcy;": "\u0414", + "Del;": "\u2207", + "Delta;": "\u0394", + "Dfr;": "\U0001d507", + "DiacriticalAcute;": "\xb4", + "DiacriticalDot;": "\u02d9", + "DiacriticalDoubleAcute;": "\u02dd", + "DiacriticalGrave;": "`", + "DiacriticalTilde;": "\u02dc", + "Diamond;": "\u22c4", + "DifferentialD;": "\u2146", + "Dopf;": "\U0001d53b", + "Dot;": "\xa8", + "DotDot;": "\u20dc", + "DotEqual;": "\u2250", + "DoubleContourIntegral;": "\u222f", + "DoubleDot;": "\xa8", + "DoubleDownArrow;": "\u21d3", + "DoubleLeftArrow;": "\u21d0", + "DoubleLeftRightArrow;": "\u21d4", + "DoubleLeftTee;": "\u2ae4", + "DoubleLongLeftArrow;": "\u27f8", + "DoubleLongLeftRightArrow;": "\u27fa", + "DoubleLongRightArrow;": "\u27f9", + "DoubleRightArrow;": "\u21d2", + "DoubleRightTee;": "\u22a8", + "DoubleUpArrow;": "\u21d1", + "DoubleUpDownArrow;": "\u21d5", + "DoubleVerticalBar;": "\u2225", + "DownArrow;": "\u2193", + "DownArrowBar;": "\u2913", + "DownArrowUpArrow;": "\u21f5", + "DownBreve;": "\u0311", + "DownLeftRightVector;": "\u2950", + "DownLeftTeeVector;": "\u295e", + "DownLeftVector;": "\u21bd", + "DownLeftVectorBar;": "\u2956", + "DownRightTeeVector;": "\u295f", + "DownRightVector;": "\u21c1", + "DownRightVectorBar;": "\u2957", + "DownTee;": "\u22a4", + "DownTeeArrow;": "\u21a7", + "Downarrow;": "\u21d3", + "Dscr;": "\U0001d49f", + "Dstrok;": "\u0110", + "ENG;": "\u014a", + "ETH": "\xd0", + "ETH;": "\xd0", + "Eacute": "\xc9", + "Eacute;": "\xc9", + "Ecaron;": "\u011a", + "Ecirc": "\xca", + "Ecirc;": "\xca", + "Ecy;": "\u042d", + "Edot;": "\u0116", + "Efr;": "\U0001d508", + "Egrave": "\xc8", + "Egrave;": "\xc8", + "Element;": "\u2208", + "Emacr;": "\u0112", + "EmptySmallSquare;": "\u25fb", + "EmptyVerySmallSquare;": "\u25ab", + "Eogon;": "\u0118", + "Eopf;": "\U0001d53c", + "Epsilon;": "\u0395", + "Equal;": "\u2a75", + "EqualTilde;": "\u2242", + "Equilibrium;": "\u21cc", + "Escr;": "\u2130", + "Esim;": "\u2a73", + "Eta;": "\u0397", + "Euml": "\xcb", + "Euml;": "\xcb", + "Exists;": "\u2203", + "ExponentialE;": "\u2147", + "Fcy;": "\u0424", + "Ffr;": "\U0001d509", + "FilledSmallSquare;": "\u25fc", + "FilledVerySmallSquare;": "\u25aa", + "Fopf;": "\U0001d53d", + "ForAll;": "\u2200", + "Fouriertrf;": "\u2131", + "Fscr;": "\u2131", + "GJcy;": "\u0403", + "GT": ">", + "GT;": ">", + "Gamma;": "\u0393", + "Gammad;": "\u03dc", + "Gbreve;": "\u011e", + "Gcedil;": "\u0122", + "Gcirc;": "\u011c", + "Gcy;": "\u0413", + "Gdot;": "\u0120", + "Gfr;": "\U0001d50a", + "Gg;": "\u22d9", + "Gopf;": "\U0001d53e", + "GreaterEqual;": "\u2265", + "GreaterEqualLess;": "\u22db", + "GreaterFullEqual;": "\u2267", + "GreaterGreater;": "\u2aa2", + "GreaterLess;": "\u2277", + "GreaterSlantEqual;": "\u2a7e", + "GreaterTilde;": "\u2273", + "Gscr;": "\U0001d4a2", + "Gt;": "\u226b", + "HARDcy;": "\u042a", + "Hacek;": "\u02c7", + "Hat;": "^", + "Hcirc;": "\u0124", + "Hfr;": "\u210c", + "HilbertSpace;": "\u210b", + "Hopf;": "\u210d", + "HorizontalLine;": "\u2500", + "Hscr;": "\u210b", + "Hstrok;": "\u0126", + "HumpDownHump;": "\u224e", + "HumpEqual;": "\u224f", + "IEcy;": "\u0415", + "IJlig;": "\u0132", + "IOcy;": "\u0401", + "Iacute": "\xcd", + "Iacute;": "\xcd", + "Icirc": "\xce", + "Icirc;": "\xce", + "Icy;": "\u0418", + "Idot;": "\u0130", + "Ifr;": "\u2111", + "Igrave": "\xcc", + "Igrave;": "\xcc", + "Im;": "\u2111", + "Imacr;": "\u012a", + "ImaginaryI;": "\u2148", + "Implies;": "\u21d2", + "Int;": "\u222c", + "Integral;": "\u222b", + "Intersection;": "\u22c2", + "InvisibleComma;": "\u2063", + "InvisibleTimes;": "\u2062", + "Iogon;": "\u012e", + "Iopf;": "\U0001d540", + "Iota;": "\u0399", + "Iscr;": "\u2110", + "Itilde;": "\u0128", + "Iukcy;": "\u0406", + "Iuml": "\xcf", + "Iuml;": "\xcf", + "Jcirc;": "\u0134", + "Jcy;": "\u0419", + "Jfr;": "\U0001d50d", + "Jopf;": "\U0001d541", + "Jscr;": "\U0001d4a5", + "Jsercy;": "\u0408", + "Jukcy;": "\u0404", + "KHcy;": "\u0425", + "KJcy;": "\u040c", + "Kappa;": "\u039a", + "Kcedil;": "\u0136", + "Kcy;": "\u041a", + "Kfr;": "\U0001d50e", + "Kopf;": "\U0001d542", + "Kscr;": "\U0001d4a6", + "LJcy;": "\u0409", + "LT": "<", + "LT;": "<", + "Lacute;": "\u0139", + "Lambda;": "\u039b", + "Lang;": "\u27ea", + "Laplacetrf;": "\u2112", + "Larr;": "\u219e", + "Lcaron;": "\u013d", + "Lcedil;": "\u013b", + "Lcy;": "\u041b", + "LeftAngleBracket;": "\u27e8", + "LeftArrow;": "\u2190", + "LeftArrowBar;": "\u21e4", + "LeftArrowRightArrow;": "\u21c6", + "LeftCeiling;": "\u2308", + "LeftDoubleBracket;": "\u27e6", + "LeftDownTeeVector;": "\u2961", + "LeftDownVector;": "\u21c3", + "LeftDownVectorBar;": "\u2959", + "LeftFloor;": "\u230a", + "LeftRightArrow;": "\u2194", + "LeftRightVector;": "\u294e", + "LeftTee;": "\u22a3", + "LeftTeeArrow;": "\u21a4", + "LeftTeeVector;": "\u295a", + "LeftTriangle;": "\u22b2", + "LeftTriangleBar;": "\u29cf", + "LeftTriangleEqual;": "\u22b4", + "LeftUpDownVector;": "\u2951", + "LeftUpTeeVector;": "\u2960", + "LeftUpVector;": "\u21bf", + "LeftUpVectorBar;": "\u2958", + "LeftVector;": "\u21bc", + "LeftVectorBar;": "\u2952", + "Leftarrow;": "\u21d0", + "Leftrightarrow;": "\u21d4", + "LessEqualGreater;": "\u22da", + "LessFullEqual;": "\u2266", + "LessGreater;": "\u2276", + "LessLess;": "\u2aa1", + "LessSlantEqual;": "\u2a7d", + "LessTilde;": "\u2272", + "Lfr;": "\U0001d50f", + "Ll;": "\u22d8", + "Lleftarrow;": "\u21da", + "Lmidot;": "\u013f", + "LongLeftArrow;": "\u27f5", + "LongLeftRightArrow;": "\u27f7", + "LongRightArrow;": "\u27f6", + "Longleftarrow;": "\u27f8", + "Longleftrightarrow;": "\u27fa", + "Longrightarrow;": "\u27f9", + "Lopf;": "\U0001d543", + "LowerLeftArrow;": "\u2199", + "LowerRightArrow;": "\u2198", + "Lscr;": "\u2112", + "Lsh;": "\u21b0", + "Lstrok;": "\u0141", + "Lt;": "\u226a", + "Map;": "\u2905", + "Mcy;": "\u041c", + "MediumSpace;": "\u205f", + "Mellintrf;": "\u2133", + "Mfr;": "\U0001d510", + "MinusPlus;": "\u2213", + "Mopf;": "\U0001d544", + "Mscr;": "\u2133", + "Mu;": "\u039c", + "NJcy;": "\u040a", + "Nacute;": "\u0143", + "Ncaron;": "\u0147", + "Ncedil;": "\u0145", + "Ncy;": "\u041d", + "NegativeMediumSpace;": "\u200b", + "NegativeThickSpace;": "\u200b", + "NegativeThinSpace;": "\u200b", + "NegativeVeryThinSpace;": "\u200b", + "NestedGreaterGreater;": "\u226b", + "NestedLessLess;": "\u226a", + "NewLine;": "\n", + "Nfr;": "\U0001d511", + "NoBreak;": "\u2060", + "NonBreakingSpace;": "\xa0", + "Nopf;": "\u2115", + "Not;": "\u2aec", + "NotCongruent;": "\u2262", + "NotCupCap;": "\u226d", + "NotDoubleVerticalBar;": "\u2226", + "NotElement;": "\u2209", + "NotEqual;": "\u2260", + "NotEqualTilde;": "\u2242\u0338", + "NotExists;": "\u2204", + "NotGreater;": "\u226f", + "NotGreaterEqual;": "\u2271", + "NotGreaterFullEqual;": "\u2267\u0338", + "NotGreaterGreater;": "\u226b\u0338", + "NotGreaterLess;": "\u2279", + "NotGreaterSlantEqual;": "\u2a7e\u0338", + "NotGreaterTilde;": "\u2275", + "NotHumpDownHump;": "\u224e\u0338", + "NotHumpEqual;": "\u224f\u0338", + "NotLeftTriangle;": "\u22ea", + "NotLeftTriangleBar;": "\u29cf\u0338", + "NotLeftTriangleEqual;": "\u22ec", + "NotLess;": "\u226e", + "NotLessEqual;": "\u2270", + "NotLessGreater;": "\u2278", + "NotLessLess;": "\u226a\u0338", + "NotLessSlantEqual;": "\u2a7d\u0338", + "NotLessTilde;": "\u2274", + "NotNestedGreaterGreater;": "\u2aa2\u0338", + "NotNestedLessLess;": "\u2aa1\u0338", + "NotPrecedes;": "\u2280", + "NotPrecedesEqual;": "\u2aaf\u0338", + "NotPrecedesSlantEqual;": "\u22e0", + "NotReverseElement;": "\u220c", + "NotRightTriangle;": "\u22eb", + "NotRightTriangleBar;": "\u29d0\u0338", + "NotRightTriangleEqual;": "\u22ed", + "NotSquareSubset;": "\u228f\u0338", + "NotSquareSubsetEqual;": "\u22e2", + "NotSquareSuperset;": "\u2290\u0338", + "NotSquareSupersetEqual;": "\u22e3", + "NotSubset;": "\u2282\u20d2", + "NotSubsetEqual;": "\u2288", + "NotSucceeds;": "\u2281", + "NotSucceedsEqual;": "\u2ab0\u0338", + "NotSucceedsSlantEqual;": "\u22e1", + "NotSucceedsTilde;": "\u227f\u0338", + "NotSuperset;": "\u2283\u20d2", + "NotSupersetEqual;": "\u2289", + "NotTilde;": "\u2241", + "NotTildeEqual;": "\u2244", + "NotTildeFullEqual;": "\u2247", + "NotTildeTilde;": "\u2249", + "NotVerticalBar;": "\u2224", + "Nscr;": "\U0001d4a9", + "Ntilde": "\xd1", + "Ntilde;": "\xd1", + "Nu;": "\u039d", + "OElig;": "\u0152", + "Oacute": "\xd3", + "Oacute;": "\xd3", + "Ocirc": "\xd4", + "Ocirc;": "\xd4", + "Ocy;": "\u041e", + "Odblac;": "\u0150", + "Ofr;": "\U0001d512", + "Ograve": "\xd2", + "Ograve;": "\xd2", + "Omacr;": "\u014c", + "Omega;": "\u03a9", + "Omicron;": "\u039f", + "Oopf;": "\U0001d546", + "OpenCurlyDoubleQuote;": "\u201c", + "OpenCurlyQuote;": "\u2018", + "Or;": "\u2a54", + "Oscr;": "\U0001d4aa", + "Oslash": "\xd8", + "Oslash;": "\xd8", + "Otilde": "\xd5", + "Otilde;": "\xd5", + "Otimes;": "\u2a37", + "Ouml": "\xd6", + "Ouml;": "\xd6", + "OverBar;": "\u203e", + "OverBrace;": "\u23de", + "OverBracket;": "\u23b4", + "OverParenthesis;": "\u23dc", + "PartialD;": "\u2202", + "Pcy;": "\u041f", + "Pfr;": "\U0001d513", + "Phi;": "\u03a6", + "Pi;": "\u03a0", + "PlusMinus;": "\xb1", + "Poincareplane;": "\u210c", + "Popf;": "\u2119", + "Pr;": "\u2abb", + "Precedes;": "\u227a", + "PrecedesEqual;": "\u2aaf", + "PrecedesSlantEqual;": "\u227c", + "PrecedesTilde;": "\u227e", + "Prime;": "\u2033", + "Product;": "\u220f", + "Proportion;": "\u2237", + "Proportional;": "\u221d", + "Pscr;": "\U0001d4ab", + "Psi;": "\u03a8", + "QUOT": "\"", + "QUOT;": "\"", + "Qfr;": "\U0001d514", + "Qopf;": "\u211a", + "Qscr;": "\U0001d4ac", + "RBarr;": "\u2910", + "REG": "\xae", + "REG;": "\xae", + "Racute;": "\u0154", + "Rang;": "\u27eb", + "Rarr;": "\u21a0", + "Rarrtl;": "\u2916", + "Rcaron;": "\u0158", + "Rcedil;": "\u0156", + "Rcy;": "\u0420", + "Re;": "\u211c", + "ReverseElement;": "\u220b", + "ReverseEquilibrium;": "\u21cb", + "ReverseUpEquilibrium;": "\u296f", + "Rfr;": "\u211c", + "Rho;": "\u03a1", + "RightAngleBracket;": "\u27e9", + "RightArrow;": "\u2192", + "RightArrowBar;": "\u21e5", + "RightArrowLeftArrow;": "\u21c4", + "RightCeiling;": "\u2309", + "RightDoubleBracket;": "\u27e7", + "RightDownTeeVector;": "\u295d", + "RightDownVector;": "\u21c2", + "RightDownVectorBar;": "\u2955", + "RightFloor;": "\u230b", + "RightTee;": "\u22a2", + "RightTeeArrow;": "\u21a6", + "RightTeeVector;": "\u295b", + "RightTriangle;": "\u22b3", + "RightTriangleBar;": "\u29d0", + "RightTriangleEqual;": "\u22b5", + "RightUpDownVector;": "\u294f", + "RightUpTeeVector;": "\u295c", + "RightUpVector;": "\u21be", + "RightUpVectorBar;": "\u2954", + "RightVector;": "\u21c0", + "RightVectorBar;": "\u2953", + "Rightarrow;": "\u21d2", + "Ropf;": "\u211d", + "RoundImplies;": "\u2970", + "Rrightarrow;": "\u21db", + "Rscr;": "\u211b", + "Rsh;": "\u21b1", + "RuleDelayed;": "\u29f4", + "SHCHcy;": "\u0429", + "SHcy;": "\u0428", + "SOFTcy;": "\u042c", + "Sacute;": "\u015a", + "Sc;": "\u2abc", + "Scaron;": "\u0160", + "Scedil;": "\u015e", + "Scirc;": "\u015c", + "Scy;": "\u0421", + "Sfr;": "\U0001d516", + "ShortDownArrow;": "\u2193", + "ShortLeftArrow;": "\u2190", + "ShortRightArrow;": "\u2192", + "ShortUpArrow;": "\u2191", + "Sigma;": "\u03a3", + "SmallCircle;": "\u2218", + "Sopf;": "\U0001d54a", + "Sqrt;": "\u221a", + "Square;": "\u25a1", + "SquareIntersection;": "\u2293", + "SquareSubset;": "\u228f", + "SquareSubsetEqual;": "\u2291", + "SquareSuperset;": "\u2290", + "SquareSupersetEqual;": "\u2292", + "SquareUnion;": "\u2294", + "Sscr;": "\U0001d4ae", + "Star;": "\u22c6", + "Sub;": "\u22d0", + "Subset;": "\u22d0", + "SubsetEqual;": "\u2286", + "Succeeds;": "\u227b", + "SucceedsEqual;": "\u2ab0", + "SucceedsSlantEqual;": "\u227d", + "SucceedsTilde;": "\u227f", + "SuchThat;": "\u220b", + "Sum;": "\u2211", + "Sup;": "\u22d1", + "Superset;": "\u2283", + "SupersetEqual;": "\u2287", + "Supset;": "\u22d1", + "THORN": "\xde", + "THORN;": "\xde", + "TRADE;": "\u2122", + "TSHcy;": "\u040b", + "TScy;": "\u0426", + "Tab;": "\t", + "Tau;": "\u03a4", + "Tcaron;": "\u0164", + "Tcedil;": "\u0162", + "Tcy;": "\u0422", + "Tfr;": "\U0001d517", + "Therefore;": "\u2234", + "Theta;": "\u0398", + "ThickSpace;": "\u205f\u200a", + "ThinSpace;": "\u2009", + "Tilde;": "\u223c", + "TildeEqual;": "\u2243", + "TildeFullEqual;": "\u2245", + "TildeTilde;": "\u2248", + "Topf;": "\U0001d54b", + "TripleDot;": "\u20db", + "Tscr;": "\U0001d4af", + "Tstrok;": "\u0166", + "Uacute": "\xda", + "Uacute;": "\xda", + "Uarr;": "\u219f", + "Uarrocir;": "\u2949", + "Ubrcy;": "\u040e", + "Ubreve;": "\u016c", + "Ucirc": "\xdb", + "Ucirc;": "\xdb", + "Ucy;": "\u0423", + "Udblac;": "\u0170", + "Ufr;": "\U0001d518", + "Ugrave": "\xd9", + "Ugrave;": "\xd9", + "Umacr;": "\u016a", + "UnderBar;": "_", + "UnderBrace;": "\u23df", + "UnderBracket;": "\u23b5", + "UnderParenthesis;": "\u23dd", + "Union;": "\u22c3", + "UnionPlus;": "\u228e", + "Uogon;": "\u0172", + "Uopf;": "\U0001d54c", + "UpArrow;": "\u2191", + "UpArrowBar;": "\u2912", + "UpArrowDownArrow;": "\u21c5", + "UpDownArrow;": "\u2195", + "UpEquilibrium;": "\u296e", + "UpTee;": "\u22a5", + "UpTeeArrow;": "\u21a5", + "Uparrow;": "\u21d1", + "Updownarrow;": "\u21d5", + "UpperLeftArrow;": "\u2196", + "UpperRightArrow;": "\u2197", + "Upsi;": "\u03d2", + "Upsilon;": "\u03a5", + "Uring;": "\u016e", + "Uscr;": "\U0001d4b0", + "Utilde;": "\u0168", + "Uuml": "\xdc", + "Uuml;": "\xdc", + "VDash;": "\u22ab", + "Vbar;": "\u2aeb", + "Vcy;": "\u0412", + "Vdash;": "\u22a9", + "Vdashl;": "\u2ae6", + "Vee;": "\u22c1", + "Verbar;": "\u2016", + "Vert;": "\u2016", + "VerticalBar;": "\u2223", + "VerticalLine;": "|", + "VerticalSeparator;": "\u2758", + "VerticalTilde;": "\u2240", + "VeryThinSpace;": "\u200a", + "Vfr;": "\U0001d519", + "Vopf;": "\U0001d54d", + "Vscr;": "\U0001d4b1", + "Vvdash;": "\u22aa", + "Wcirc;": "\u0174", + "Wedge;": "\u22c0", + "Wfr;": "\U0001d51a", + "Wopf;": "\U0001d54e", + "Wscr;": "\U0001d4b2", + "Xfr;": "\U0001d51b", + "Xi;": "\u039e", + "Xopf;": "\U0001d54f", + "Xscr;": "\U0001d4b3", + "YAcy;": "\u042f", + "YIcy;": "\u0407", + "YUcy;": "\u042e", + "Yacute": "\xdd", + "Yacute;": "\xdd", + "Ycirc;": "\u0176", + "Ycy;": "\u042b", + "Yfr;": "\U0001d51c", + "Yopf;": "\U0001d550", + "Yscr;": "\U0001d4b4", + "Yuml;": "\u0178", + "ZHcy;": "\u0416", + "Zacute;": "\u0179", + "Zcaron;": "\u017d", + "Zcy;": "\u0417", + "Zdot;": "\u017b", + "ZeroWidthSpace;": "\u200b", + "Zeta;": "\u0396", + "Zfr;": "\u2128", + "Zopf;": "\u2124", + "Zscr;": "\U0001d4b5", + "aacute": "\xe1", + "aacute;": "\xe1", + "abreve;": "\u0103", + "ac;": "\u223e", + "acE;": "\u223e\u0333", + "acd;": "\u223f", + "acirc": "\xe2", + "acirc;": "\xe2", + "acute": "\xb4", + "acute;": "\xb4", + "acy;": "\u0430", + "aelig": "\xe6", + "aelig;": "\xe6", + "af;": "\u2061", + "afr;": "\U0001d51e", + "agrave": "\xe0", + "agrave;": "\xe0", + "alefsym;": "\u2135", + "aleph;": "\u2135", + "alpha;": "\u03b1", + "amacr;": "\u0101", + "amalg;": "\u2a3f", + "amp": "&", + "amp;": "&", + "and;": "\u2227", + "andand;": "\u2a55", + "andd;": "\u2a5c", + "andslope;": "\u2a58", + "andv;": "\u2a5a", + "ang;": "\u2220", + "ange;": "\u29a4", + "angle;": "\u2220", + "angmsd;": "\u2221", + "angmsdaa;": "\u29a8", + "angmsdab;": "\u29a9", + "angmsdac;": "\u29aa", + "angmsdad;": "\u29ab", + "angmsdae;": "\u29ac", + "angmsdaf;": "\u29ad", + "angmsdag;": "\u29ae", + "angmsdah;": "\u29af", + "angrt;": "\u221f", + "angrtvb;": "\u22be", + "angrtvbd;": "\u299d", + "angsph;": "\u2222", + "angst;": "\xc5", + "angzarr;": "\u237c", + "aogon;": "\u0105", + "aopf;": "\U0001d552", + "ap;": "\u2248", + "apE;": "\u2a70", + "apacir;": "\u2a6f", + "ape;": "\u224a", + "apid;": "\u224b", + "apos;": "'", + "approx;": "\u2248", + "approxeq;": "\u224a", + "aring": "\xe5", + "aring;": "\xe5", + "ascr;": "\U0001d4b6", + "ast;": "*", + "asymp;": "\u2248", + "asympeq;": "\u224d", + "atilde": "\xe3", + "atilde;": "\xe3", + "auml": "\xe4", + "auml;": "\xe4", + "awconint;": "\u2233", + "awint;": "\u2a11", + "bNot;": "\u2aed", + "backcong;": "\u224c", + "backepsilon;": "\u03f6", + "backprime;": "\u2035", + "backsim;": "\u223d", + "backsimeq;": "\u22cd", + "barvee;": "\u22bd", + "barwed;": "\u2305", + "barwedge;": "\u2305", + "bbrk;": "\u23b5", + "bbrktbrk;": "\u23b6", + "bcong;": "\u224c", + "bcy;": "\u0431", + "bdquo;": "\u201e", + "becaus;": "\u2235", + "because;": "\u2235", + "bemptyv;": "\u29b0", + "bepsi;": "\u03f6", + "bernou;": "\u212c", + "beta;": "\u03b2", + "beth;": "\u2136", + "between;": "\u226c", + "bfr;": "\U0001d51f", + "bigcap;": "\u22c2", + "bigcirc;": "\u25ef", + "bigcup;": "\u22c3", + "bigodot;": "\u2a00", + "bigoplus;": "\u2a01", + "bigotimes;": "\u2a02", + "bigsqcup;": "\u2a06", + "bigstar;": "\u2605", + "bigtriangledown;": "\u25bd", + "bigtriangleup;": "\u25b3", + "biguplus;": "\u2a04", + "bigvee;": "\u22c1", + "bigwedge;": "\u22c0", + "bkarow;": "\u290d", + "blacklozenge;": "\u29eb", + "blacksquare;": "\u25aa", + "blacktriangle;": "\u25b4", + "blacktriangledown;": "\u25be", + "blacktriangleleft;": "\u25c2", + "blacktriangleright;": "\u25b8", + "blank;": "\u2423", + "blk12;": "\u2592", + "blk14;": "\u2591", + "blk34;": "\u2593", + "block;": "\u2588", + "bne;": "=\u20e5", + "bnequiv;": "\u2261\u20e5", + "bnot;": "\u2310", + "bopf;": "\U0001d553", + "bot;": "\u22a5", + "bottom;": "\u22a5", + "bowtie;": "\u22c8", + "boxDL;": "\u2557", + "boxDR;": "\u2554", + "boxDl;": "\u2556", + "boxDr;": "\u2553", + "boxH;": "\u2550", + "boxHD;": "\u2566", + "boxHU;": "\u2569", + "boxHd;": "\u2564", + "boxHu;": "\u2567", + "boxUL;": "\u255d", + "boxUR;": "\u255a", + "boxUl;": "\u255c", + "boxUr;": "\u2559", + "boxV;": "\u2551", + "boxVH;": "\u256c", + "boxVL;": "\u2563", + "boxVR;": "\u2560", + "boxVh;": "\u256b", + "boxVl;": "\u2562", + "boxVr;": "\u255f", + "boxbox;": "\u29c9", + "boxdL;": "\u2555", + "boxdR;": "\u2552", + "boxdl;": "\u2510", + "boxdr;": "\u250c", + "boxh;": "\u2500", + "boxhD;": "\u2565", + "boxhU;": "\u2568", + "boxhd;": "\u252c", + "boxhu;": "\u2534", + "boxminus;": "\u229f", + "boxplus;": "\u229e", + "boxtimes;": "\u22a0", + "boxuL;": "\u255b", + "boxuR;": "\u2558", + "boxul;": "\u2518", + "boxur;": "\u2514", + "boxv;": "\u2502", + "boxvH;": "\u256a", + "boxvL;": "\u2561", + "boxvR;": "\u255e", + "boxvh;": "\u253c", + "boxvl;": "\u2524", + "boxvr;": "\u251c", + "bprime;": "\u2035", + "breve;": "\u02d8", + "brvbar": "\xa6", + "brvbar;": "\xa6", + "bscr;": "\U0001d4b7", + "bsemi;": "\u204f", + "bsim;": "\u223d", + "bsime;": "\u22cd", + "bsol;": "\\", + "bsolb;": "\u29c5", + "bsolhsub;": "\u27c8", + "bull;": "\u2022", + "bullet;": "\u2022", + "bump;": "\u224e", + "bumpE;": "\u2aae", + "bumpe;": "\u224f", + "bumpeq;": "\u224f", + "cacute;": "\u0107", + "cap;": "\u2229", + "capand;": "\u2a44", + "capbrcup;": "\u2a49", + "capcap;": "\u2a4b", + "capcup;": "\u2a47", + "capdot;": "\u2a40", + "caps;": "\u2229\ufe00", + "caret;": "\u2041", + "caron;": "\u02c7", + "ccaps;": "\u2a4d", + "ccaron;": "\u010d", + "ccedil": "\xe7", + "ccedil;": "\xe7", + "ccirc;": "\u0109", + "ccups;": "\u2a4c", + "ccupssm;": "\u2a50", + "cdot;": "\u010b", + "cedil": "\xb8", + "cedil;": "\xb8", + "cemptyv;": "\u29b2", + "cent": "\xa2", + "cent;": "\xa2", + "centerdot;": "\xb7", + "cfr;": "\U0001d520", + "chcy;": "\u0447", + "check;": "\u2713", + "checkmark;": "\u2713", + "chi;": "\u03c7", + "cir;": "\u25cb", + "cirE;": "\u29c3", + "circ;": "\u02c6", + "circeq;": "\u2257", + "circlearrowleft;": "\u21ba", + "circlearrowright;": "\u21bb", + "circledR;": "\xae", + "circledS;": "\u24c8", + "circledast;": "\u229b", + "circledcirc;": "\u229a", + "circleddash;": "\u229d", + "cire;": "\u2257", + "cirfnint;": "\u2a10", + "cirmid;": "\u2aef", + "cirscir;": "\u29c2", + "clubs;": "\u2663", + "clubsuit;": "\u2663", + "colon;": ":", + "colone;": "\u2254", + "coloneq;": "\u2254", + "comma;": ",", + "commat;": "@", + "comp;": "\u2201", + "compfn;": "\u2218", + "complement;": "\u2201", + "complexes;": "\u2102", + "cong;": "\u2245", + "congdot;": "\u2a6d", + "conint;": "\u222e", + "copf;": "\U0001d554", + "coprod;": "\u2210", + "copy": "\xa9", + "copy;": "\xa9", + "copysr;": "\u2117", + "crarr;": "\u21b5", + "cross;": "\u2717", + "cscr;": "\U0001d4b8", + "csub;": "\u2acf", + "csube;": "\u2ad1", + "csup;": "\u2ad0", + "csupe;": "\u2ad2", + "ctdot;": "\u22ef", + "cudarrl;": "\u2938", + "cudarrr;": "\u2935", + "cuepr;": "\u22de", + "cuesc;": "\u22df", + "cularr;": "\u21b6", + "cularrp;": "\u293d", + "cup;": "\u222a", + "cupbrcap;": "\u2a48", + "cupcap;": "\u2a46", + "cupcup;": "\u2a4a", + "cupdot;": "\u228d", + "cupor;": "\u2a45", + "cups;": "\u222a\ufe00", + "curarr;": "\u21b7", + "curarrm;": "\u293c", + "curlyeqprec;": "\u22de", + "curlyeqsucc;": "\u22df", + "curlyvee;": "\u22ce", + "curlywedge;": "\u22cf", + "curren": "\xa4", + "curren;": "\xa4", + "curvearrowleft;": "\u21b6", + "curvearrowright;": "\u21b7", + "cuvee;": "\u22ce", + "cuwed;": "\u22cf", + "cwconint;": "\u2232", + "cwint;": "\u2231", + "cylcty;": "\u232d", + "dArr;": "\u21d3", + "dHar;": "\u2965", + "dagger;": "\u2020", + "daleth;": "\u2138", + "darr;": "\u2193", + "dash;": "\u2010", + "dashv;": "\u22a3", + "dbkarow;": "\u290f", + "dblac;": "\u02dd", + "dcaron;": "\u010f", + "dcy;": "\u0434", + "dd;": "\u2146", + "ddagger;": "\u2021", + "ddarr;": "\u21ca", + "ddotseq;": "\u2a77", + "deg": "\xb0", + "deg;": "\xb0", + "delta;": "\u03b4", + "demptyv;": "\u29b1", + "dfisht;": "\u297f", + "dfr;": "\U0001d521", + "dharl;": "\u21c3", + "dharr;": "\u21c2", + "diam;": "\u22c4", + "diamond;": "\u22c4", + "diamondsuit;": "\u2666", + "diams;": "\u2666", + "die;": "\xa8", + "digamma;": "\u03dd", + "disin;": "\u22f2", + "div;": "\xf7", + "divide": "\xf7", + "divide;": "\xf7", + "divideontimes;": "\u22c7", + "divonx;": "\u22c7", + "djcy;": "\u0452", + "dlcorn;": "\u231e", + "dlcrop;": "\u230d", + "dollar;": "$", + "dopf;": "\U0001d555", + "dot;": "\u02d9", + "doteq;": "\u2250", + "doteqdot;": "\u2251", + "dotminus;": "\u2238", + "dotplus;": "\u2214", + "dotsquare;": "\u22a1", + "doublebarwedge;": "\u2306", + "downarrow;": "\u2193", + "downdownarrows;": "\u21ca", + "downharpoonleft;": "\u21c3", + "downharpoonright;": "\u21c2", + "drbkarow;": "\u2910", + "drcorn;": "\u231f", + "drcrop;": "\u230c", + "dscr;": "\U0001d4b9", + "dscy;": "\u0455", + "dsol;": "\u29f6", + "dstrok;": "\u0111", + "dtdot;": "\u22f1", + "dtri;": "\u25bf", + "dtrif;": "\u25be", + "duarr;": "\u21f5", + "duhar;": "\u296f", + "dwangle;": "\u29a6", + "dzcy;": "\u045f", + "dzigrarr;": "\u27ff", + "eDDot;": "\u2a77", + "eDot;": "\u2251", + "eacute": "\xe9", + "eacute;": "\xe9", + "easter;": "\u2a6e", + "ecaron;": "\u011b", + "ecir;": "\u2256", + "ecirc": "\xea", + "ecirc;": "\xea", + "ecolon;": "\u2255", + "ecy;": "\u044d", + "edot;": "\u0117", + "ee;": "\u2147", + "efDot;": "\u2252", + "efr;": "\U0001d522", + "eg;": "\u2a9a", + "egrave": "\xe8", + "egrave;": "\xe8", + "egs;": "\u2a96", + "egsdot;": "\u2a98", + "el;": "\u2a99", + "elinters;": "\u23e7", + "ell;": "\u2113", + "els;": "\u2a95", + "elsdot;": "\u2a97", + "emacr;": "\u0113", + "empty;": "\u2205", + "emptyset;": "\u2205", + "emptyv;": "\u2205", + "emsp13;": "\u2004", + "emsp14;": "\u2005", + "emsp;": "\u2003", + "eng;": "\u014b", + "ensp;": "\u2002", + "eogon;": "\u0119", + "eopf;": "\U0001d556", + "epar;": "\u22d5", + "eparsl;": "\u29e3", + "eplus;": "\u2a71", + "epsi;": "\u03b5", + "epsilon;": "\u03b5", + "epsiv;": "\u03f5", + "eqcirc;": "\u2256", + "eqcolon;": "\u2255", + "eqsim;": "\u2242", + "eqslantgtr;": "\u2a96", + "eqslantless;": "\u2a95", + "equals;": "=", + "equest;": "\u225f", + "equiv;": "\u2261", + "equivDD;": "\u2a78", + "eqvparsl;": "\u29e5", + "erDot;": "\u2253", + "erarr;": "\u2971", + "escr;": "\u212f", + "esdot;": "\u2250", + "esim;": "\u2242", + "eta;": "\u03b7", + "eth": "\xf0", + "eth;": "\xf0", + "euml": "\xeb", + "euml;": "\xeb", + "euro;": "\u20ac", + "excl;": "!", + "exist;": "\u2203", + "expectation;": "\u2130", + "exponentiale;": "\u2147", + "fallingdotseq;": "\u2252", + "fcy;": "\u0444", + "female;": "\u2640", + "ffilig;": "\ufb03", + "fflig;": "\ufb00", + "ffllig;": "\ufb04", + "ffr;": "\U0001d523", + "filig;": "\ufb01", + "fjlig;": "fj", + "flat;": "\u266d", + "fllig;": "\ufb02", + "fltns;": "\u25b1", + "fnof;": "\u0192", + "fopf;": "\U0001d557", + "forall;": "\u2200", + "fork;": "\u22d4", + "forkv;": "\u2ad9", + "fpartint;": "\u2a0d", + "frac12": "\xbd", + "frac12;": "\xbd", + "frac13;": "\u2153", + "frac14": "\xbc", + "frac14;": "\xbc", + "frac15;": "\u2155", + "frac16;": "\u2159", + "frac18;": "\u215b", + "frac23;": "\u2154", + "frac25;": "\u2156", + "frac34": "\xbe", + "frac34;": "\xbe", + "frac35;": "\u2157", + "frac38;": "\u215c", + "frac45;": "\u2158", + "frac56;": "\u215a", + "frac58;": "\u215d", + "frac78;": "\u215e", + "frasl;": "\u2044", + "frown;": "\u2322", + "fscr;": "\U0001d4bb", + "gE;": "\u2267", + "gEl;": "\u2a8c", + "gacute;": "\u01f5", + "gamma;": "\u03b3", + "gammad;": "\u03dd", + "gap;": "\u2a86", + "gbreve;": "\u011f", + "gcirc;": "\u011d", + "gcy;": "\u0433", + "gdot;": "\u0121", + "ge;": "\u2265", + "gel;": "\u22db", + "geq;": "\u2265", + "geqq;": "\u2267", + "geqslant;": "\u2a7e", + "ges;": "\u2a7e", + "gescc;": "\u2aa9", + "gesdot;": "\u2a80", + "gesdoto;": "\u2a82", + "gesdotol;": "\u2a84", + "gesl;": "\u22db\ufe00", + "gesles;": "\u2a94", + "gfr;": "\U0001d524", + "gg;": "\u226b", + "ggg;": "\u22d9", + "gimel;": "\u2137", + "gjcy;": "\u0453", + "gl;": "\u2277", + "glE;": "\u2a92", + "gla;": "\u2aa5", + "glj;": "\u2aa4", + "gnE;": "\u2269", + "gnap;": "\u2a8a", + "gnapprox;": "\u2a8a", + "gne;": "\u2a88", + "gneq;": "\u2a88", + "gneqq;": "\u2269", + "gnsim;": "\u22e7", + "gopf;": "\U0001d558", + "grave;": "`", + "gscr;": "\u210a", + "gsim;": "\u2273", + "gsime;": "\u2a8e", + "gsiml;": "\u2a90", + "gt": ">", + "gt;": ">", + "gtcc;": "\u2aa7", + "gtcir;": "\u2a7a", + "gtdot;": "\u22d7", + "gtlPar;": "\u2995", + "gtquest;": "\u2a7c", + "gtrapprox;": "\u2a86", + "gtrarr;": "\u2978", + "gtrdot;": "\u22d7", + "gtreqless;": "\u22db", + "gtreqqless;": "\u2a8c", + "gtrless;": "\u2277", + "gtrsim;": "\u2273", + "gvertneqq;": "\u2269\ufe00", + "gvnE;": "\u2269\ufe00", + "hArr;": "\u21d4", + "hairsp;": "\u200a", + "half;": "\xbd", + "hamilt;": "\u210b", + "hardcy;": "\u044a", + "harr;": "\u2194", + "harrcir;": "\u2948", + "harrw;": "\u21ad", + "hbar;": "\u210f", + "hcirc;": "\u0125", + "hearts;": "\u2665", + "heartsuit;": "\u2665", + "hellip;": "\u2026", + "hercon;": "\u22b9", + "hfr;": "\U0001d525", + "hksearow;": "\u2925", + "hkswarow;": "\u2926", + "hoarr;": "\u21ff", + "homtht;": "\u223b", + "hookleftarrow;": "\u21a9", + "hookrightarrow;": "\u21aa", + "hopf;": "\U0001d559", + "horbar;": "\u2015", + "hscr;": "\U0001d4bd", + "hslash;": "\u210f", + "hstrok;": "\u0127", + "hybull;": "\u2043", + "hyphen;": "\u2010", + "iacute": "\xed", + "iacute;": "\xed", + "ic;": "\u2063", + "icirc": "\xee", + "icirc;": "\xee", + "icy;": "\u0438", + "iecy;": "\u0435", + "iexcl": "\xa1", + "iexcl;": "\xa1", + "iff;": "\u21d4", + "ifr;": "\U0001d526", + "igrave": "\xec", + "igrave;": "\xec", + "ii;": "\u2148", + "iiiint;": "\u2a0c", + "iiint;": "\u222d", + "iinfin;": "\u29dc", + "iiota;": "\u2129", + "ijlig;": "\u0133", + "imacr;": "\u012b", + "image;": "\u2111", + "imagline;": "\u2110", + "imagpart;": "\u2111", + "imath;": "\u0131", + "imof;": "\u22b7", + "imped;": "\u01b5", + "in;": "\u2208", + "incare;": "\u2105", + "infin;": "\u221e", + "infintie;": "\u29dd", + "inodot;": "\u0131", + "int;": "\u222b", + "intcal;": "\u22ba", + "integers;": "\u2124", + "intercal;": "\u22ba", + "intlarhk;": "\u2a17", + "intprod;": "\u2a3c", + "iocy;": "\u0451", + "iogon;": "\u012f", + "iopf;": "\U0001d55a", + "iota;": "\u03b9", + "iprod;": "\u2a3c", + "iquest": "\xbf", + "iquest;": "\xbf", + "iscr;": "\U0001d4be", + "isin;": "\u2208", + "isinE;": "\u22f9", + "isindot;": "\u22f5", + "isins;": "\u22f4", + "isinsv;": "\u22f3", + "isinv;": "\u2208", + "it;": "\u2062", + "itilde;": "\u0129", + "iukcy;": "\u0456", + "iuml": "\xef", + "iuml;": "\xef", + "jcirc;": "\u0135", + "jcy;": "\u0439", + "jfr;": "\U0001d527", + "jmath;": "\u0237", + "jopf;": "\U0001d55b", + "jscr;": "\U0001d4bf", + "jsercy;": "\u0458", + "jukcy;": "\u0454", + "kappa;": "\u03ba", + "kappav;": "\u03f0", + "kcedil;": "\u0137", + "kcy;": "\u043a", + "kfr;": "\U0001d528", + "kgreen;": "\u0138", + "khcy;": "\u0445", + "kjcy;": "\u045c", + "kopf;": "\U0001d55c", + "kscr;": "\U0001d4c0", + "lAarr;": "\u21da", + "lArr;": "\u21d0", + "lAtail;": "\u291b", + "lBarr;": "\u290e", + "lE;": "\u2266", + "lEg;": "\u2a8b", + "lHar;": "\u2962", + "lacute;": "\u013a", + "laemptyv;": "\u29b4", + "lagran;": "\u2112", + "lambda;": "\u03bb", + "lang;": "\u27e8", + "langd;": "\u2991", + "langle;": "\u27e8", + "lap;": "\u2a85", + "laquo": "\xab", + "laquo;": "\xab", + "larr;": "\u2190", + "larrb;": "\u21e4", + "larrbfs;": "\u291f", + "larrfs;": "\u291d", + "larrhk;": "\u21a9", + "larrlp;": "\u21ab", + "larrpl;": "\u2939", + "larrsim;": "\u2973", + "larrtl;": "\u21a2", + "lat;": "\u2aab", + "latail;": "\u2919", + "late;": "\u2aad", + "lates;": "\u2aad\ufe00", + "lbarr;": "\u290c", + "lbbrk;": "\u2772", + "lbrace;": "{", + "lbrack;": "[", + "lbrke;": "\u298b", + "lbrksld;": "\u298f", + "lbrkslu;": "\u298d", + "lcaron;": "\u013e", + "lcedil;": "\u013c", + "lceil;": "\u2308", + "lcub;": "{", + "lcy;": "\u043b", + "ldca;": "\u2936", + "ldquo;": "\u201c", + "ldquor;": "\u201e", + "ldrdhar;": "\u2967", + "ldrushar;": "\u294b", + "ldsh;": "\u21b2", + "le;": "\u2264", + "leftarrow;": "\u2190", + "leftarrowtail;": "\u21a2", + "leftharpoondown;": "\u21bd", + "leftharpoonup;": "\u21bc", + "leftleftarrows;": "\u21c7", + "leftrightarrow;": "\u2194", + "leftrightarrows;": "\u21c6", + "leftrightharpoons;": "\u21cb", + "leftrightsquigarrow;": "\u21ad", + "leftthreetimes;": "\u22cb", + "leg;": "\u22da", + "leq;": "\u2264", + "leqq;": "\u2266", + "leqslant;": "\u2a7d", + "les;": "\u2a7d", + "lescc;": "\u2aa8", + "lesdot;": "\u2a7f", + "lesdoto;": "\u2a81", + "lesdotor;": "\u2a83", + "lesg;": "\u22da\ufe00", + "lesges;": "\u2a93", + "lessapprox;": "\u2a85", + "lessdot;": "\u22d6", + "lesseqgtr;": "\u22da", + "lesseqqgtr;": "\u2a8b", + "lessgtr;": "\u2276", + "lesssim;": "\u2272", + "lfisht;": "\u297c", + "lfloor;": "\u230a", + "lfr;": "\U0001d529", + "lg;": "\u2276", + "lgE;": "\u2a91", + "lhard;": "\u21bd", + "lharu;": "\u21bc", + "lharul;": "\u296a", + "lhblk;": "\u2584", + "ljcy;": "\u0459", + "ll;": "\u226a", + "llarr;": "\u21c7", + "llcorner;": "\u231e", + "llhard;": "\u296b", + "lltri;": "\u25fa", + "lmidot;": "\u0140", + "lmoust;": "\u23b0", + "lmoustache;": "\u23b0", + "lnE;": "\u2268", + "lnap;": "\u2a89", + "lnapprox;": "\u2a89", + "lne;": "\u2a87", + "lneq;": "\u2a87", + "lneqq;": "\u2268", + "lnsim;": "\u22e6", + "loang;": "\u27ec", + "loarr;": "\u21fd", + "lobrk;": "\u27e6", + "longleftarrow;": "\u27f5", + "longleftrightarrow;": "\u27f7", + "longmapsto;": "\u27fc", + "longrightarrow;": "\u27f6", + "looparrowleft;": "\u21ab", + "looparrowright;": "\u21ac", + "lopar;": "\u2985", + "lopf;": "\U0001d55d", + "loplus;": "\u2a2d", + "lotimes;": "\u2a34", + "lowast;": "\u2217", + "lowbar;": "_", + "loz;": "\u25ca", + "lozenge;": "\u25ca", + "lozf;": "\u29eb", + "lpar;": "(", + "lparlt;": "\u2993", + "lrarr;": "\u21c6", + "lrcorner;": "\u231f", + "lrhar;": "\u21cb", + "lrhard;": "\u296d", + "lrm;": "\u200e", + "lrtri;": "\u22bf", + "lsaquo;": "\u2039", + "lscr;": "\U0001d4c1", + "lsh;": "\u21b0", + "lsim;": "\u2272", + "lsime;": "\u2a8d", + "lsimg;": "\u2a8f", + "lsqb;": "[", + "lsquo;": "\u2018", + "lsquor;": "\u201a", + "lstrok;": "\u0142", + "lt": "<", + "lt;": "<", + "ltcc;": "\u2aa6", + "ltcir;": "\u2a79", + "ltdot;": "\u22d6", + "lthree;": "\u22cb", + "ltimes;": "\u22c9", + "ltlarr;": "\u2976", + "ltquest;": "\u2a7b", + "ltrPar;": "\u2996", + "ltri;": "\u25c3", + "ltrie;": "\u22b4", + "ltrif;": "\u25c2", + "lurdshar;": "\u294a", + "luruhar;": "\u2966", + "lvertneqq;": "\u2268\ufe00", + "lvnE;": "\u2268\ufe00", + "mDDot;": "\u223a", + "macr": "\xaf", + "macr;": "\xaf", + "male;": "\u2642", + "malt;": "\u2720", + "maltese;": "\u2720", + "map;": "\u21a6", + "mapsto;": "\u21a6", + "mapstodown;": "\u21a7", + "mapstoleft;": "\u21a4", + "mapstoup;": "\u21a5", + "marker;": "\u25ae", + "mcomma;": "\u2a29", + "mcy;": "\u043c", + "mdash;": "\u2014", + "measuredangle;": "\u2221", + "mfr;": "\U0001d52a", + "mho;": "\u2127", + "micro": "\xb5", + "micro;": "\xb5", + "mid;": "\u2223", + "midast;": "*", + "midcir;": "\u2af0", + "middot": "\xb7", + "middot;": "\xb7", + "minus;": "\u2212", + "minusb;": "\u229f", + "minusd;": "\u2238", + "minusdu;": "\u2a2a", + "mlcp;": "\u2adb", + "mldr;": "\u2026", + "mnplus;": "\u2213", + "models;": "\u22a7", + "mopf;": "\U0001d55e", + "mp;": "\u2213", + "mscr;": "\U0001d4c2", + "mstpos;": "\u223e", + "mu;": "\u03bc", + "multimap;": "\u22b8", + "mumap;": "\u22b8", + "nGg;": "\u22d9\u0338", + "nGt;": "\u226b\u20d2", + "nGtv;": "\u226b\u0338", + "nLeftarrow;": "\u21cd", + "nLeftrightarrow;": "\u21ce", + "nLl;": "\u22d8\u0338", + "nLt;": "\u226a\u20d2", + "nLtv;": "\u226a\u0338", + "nRightarrow;": "\u21cf", + "nVDash;": "\u22af", + "nVdash;": "\u22ae", + "nabla;": "\u2207", + "nacute;": "\u0144", + "nang;": "\u2220\u20d2", + "nap;": "\u2249", + "napE;": "\u2a70\u0338", + "napid;": "\u224b\u0338", + "napos;": "\u0149", + "napprox;": "\u2249", + "natur;": "\u266e", + "natural;": "\u266e", + "naturals;": "\u2115", + "nbsp": "\xa0", + "nbsp;": "\xa0", + "nbump;": "\u224e\u0338", + "nbumpe;": "\u224f\u0338", + "ncap;": "\u2a43", + "ncaron;": "\u0148", + "ncedil;": "\u0146", + "ncong;": "\u2247", + "ncongdot;": "\u2a6d\u0338", + "ncup;": "\u2a42", + "ncy;": "\u043d", + "ndash;": "\u2013", + "ne;": "\u2260", + "neArr;": "\u21d7", + "nearhk;": "\u2924", + "nearr;": "\u2197", + "nearrow;": "\u2197", + "nedot;": "\u2250\u0338", + "nequiv;": "\u2262", + "nesear;": "\u2928", + "nesim;": "\u2242\u0338", + "nexist;": "\u2204", + "nexists;": "\u2204", + "nfr;": "\U0001d52b", + "ngE;": "\u2267\u0338", + "nge;": "\u2271", + "ngeq;": "\u2271", + "ngeqq;": "\u2267\u0338", + "ngeqslant;": "\u2a7e\u0338", + "nges;": "\u2a7e\u0338", + "ngsim;": "\u2275", + "ngt;": "\u226f", + "ngtr;": "\u226f", + "nhArr;": "\u21ce", + "nharr;": "\u21ae", + "nhpar;": "\u2af2", + "ni;": "\u220b", + "nis;": "\u22fc", + "nisd;": "\u22fa", + "niv;": "\u220b", + "njcy;": "\u045a", + "nlArr;": "\u21cd", + "nlE;": "\u2266\u0338", + "nlarr;": "\u219a", + "nldr;": "\u2025", + "nle;": "\u2270", + "nleftarrow;": "\u219a", + "nleftrightarrow;": "\u21ae", + "nleq;": "\u2270", + "nleqq;": "\u2266\u0338", + "nleqslant;": "\u2a7d\u0338", + "nles;": "\u2a7d\u0338", + "nless;": "\u226e", + "nlsim;": "\u2274", + "nlt;": "\u226e", + "nltri;": "\u22ea", + "nltrie;": "\u22ec", + "nmid;": "\u2224", + "nopf;": "\U0001d55f", + "not": "\xac", + "not;": "\xac", + "notin;": "\u2209", + "notinE;": "\u22f9\u0338", + "notindot;": "\u22f5\u0338", + "notinva;": "\u2209", + "notinvb;": "\u22f7", + "notinvc;": "\u22f6", + "notni;": "\u220c", + "notniva;": "\u220c", + "notnivb;": "\u22fe", + "notnivc;": "\u22fd", + "npar;": "\u2226", + "nparallel;": "\u2226", + "nparsl;": "\u2afd\u20e5", + "npart;": "\u2202\u0338", + "npolint;": "\u2a14", + "npr;": "\u2280", + "nprcue;": "\u22e0", + "npre;": "\u2aaf\u0338", + "nprec;": "\u2280", + "npreceq;": "\u2aaf\u0338", + "nrArr;": "\u21cf", + "nrarr;": "\u219b", + "nrarrc;": "\u2933\u0338", + "nrarrw;": "\u219d\u0338", + "nrightarrow;": "\u219b", + "nrtri;": "\u22eb", + "nrtrie;": "\u22ed", + "nsc;": "\u2281", + "nsccue;": "\u22e1", + "nsce;": "\u2ab0\u0338", + "nscr;": "\U0001d4c3", + "nshortmid;": "\u2224", + "nshortparallel;": "\u2226", + "nsim;": "\u2241", + "nsime;": "\u2244", + "nsimeq;": "\u2244", + "nsmid;": "\u2224", + "nspar;": "\u2226", + "nsqsube;": "\u22e2", + "nsqsupe;": "\u22e3", + "nsub;": "\u2284", + "nsubE;": "\u2ac5\u0338", + "nsube;": "\u2288", + "nsubset;": "\u2282\u20d2", + "nsubseteq;": "\u2288", + "nsubseteqq;": "\u2ac5\u0338", + "nsucc;": "\u2281", + "nsucceq;": "\u2ab0\u0338", + "nsup;": "\u2285", + "nsupE;": "\u2ac6\u0338", + "nsupe;": "\u2289", + "nsupset;": "\u2283\u20d2", + "nsupseteq;": "\u2289", + "nsupseteqq;": "\u2ac6\u0338", + "ntgl;": "\u2279", + "ntilde": "\xf1", + "ntilde;": "\xf1", + "ntlg;": "\u2278", + "ntriangleleft;": "\u22ea", + "ntrianglelefteq;": "\u22ec", + "ntriangleright;": "\u22eb", + "ntrianglerighteq;": "\u22ed", + "nu;": "\u03bd", + "num;": "#", + "numero;": "\u2116", + "numsp;": "\u2007", + "nvDash;": "\u22ad", + "nvHarr;": "\u2904", + "nvap;": "\u224d\u20d2", + "nvdash;": "\u22ac", + "nvge;": "\u2265\u20d2", + "nvgt;": ">\u20d2", + "nvinfin;": "\u29de", + "nvlArr;": "\u2902", + "nvle;": "\u2264\u20d2", + "nvlt;": "<\u20d2", + "nvltrie;": "\u22b4\u20d2", + "nvrArr;": "\u2903", + "nvrtrie;": "\u22b5\u20d2", + "nvsim;": "\u223c\u20d2", + "nwArr;": "\u21d6", + "nwarhk;": "\u2923", + "nwarr;": "\u2196", + "nwarrow;": "\u2196", + "nwnear;": "\u2927", + "oS;": "\u24c8", + "oacute": "\xf3", + "oacute;": "\xf3", + "oast;": "\u229b", + "ocir;": "\u229a", + "ocirc": "\xf4", + "ocirc;": "\xf4", + "ocy;": "\u043e", + "odash;": "\u229d", + "odblac;": "\u0151", + "odiv;": "\u2a38", + "odot;": "\u2299", + "odsold;": "\u29bc", + "oelig;": "\u0153", + "ofcir;": "\u29bf", + "ofr;": "\U0001d52c", + "ogon;": "\u02db", + "ograve": "\xf2", + "ograve;": "\xf2", + "ogt;": "\u29c1", + "ohbar;": "\u29b5", + "ohm;": "\u03a9", + "oint;": "\u222e", + "olarr;": "\u21ba", + "olcir;": "\u29be", + "olcross;": "\u29bb", + "oline;": "\u203e", + "olt;": "\u29c0", + "omacr;": "\u014d", + "omega;": "\u03c9", + "omicron;": "\u03bf", + "omid;": "\u29b6", + "ominus;": "\u2296", + "oopf;": "\U0001d560", + "opar;": "\u29b7", + "operp;": "\u29b9", + "oplus;": "\u2295", + "or;": "\u2228", + "orarr;": "\u21bb", + "ord;": "\u2a5d", + "order;": "\u2134", + "orderof;": "\u2134", + "ordf": "\xaa", + "ordf;": "\xaa", + "ordm": "\xba", + "ordm;": "\xba", + "origof;": "\u22b6", + "oror;": "\u2a56", + "orslope;": "\u2a57", + "orv;": "\u2a5b", + "oscr;": "\u2134", + "oslash": "\xf8", + "oslash;": "\xf8", + "osol;": "\u2298", + "otilde": "\xf5", + "otilde;": "\xf5", + "otimes;": "\u2297", + "otimesas;": "\u2a36", + "ouml": "\xf6", + "ouml;": "\xf6", + "ovbar;": "\u233d", + "par;": "\u2225", + "para": "\xb6", + "para;": "\xb6", + "parallel;": "\u2225", + "parsim;": "\u2af3", + "parsl;": "\u2afd", + "part;": "\u2202", + "pcy;": "\u043f", + "percnt;": "%", + "period;": ".", + "permil;": "\u2030", + "perp;": "\u22a5", + "pertenk;": "\u2031", + "pfr;": "\U0001d52d", + "phi;": "\u03c6", + "phiv;": "\u03d5", + "phmmat;": "\u2133", + "phone;": "\u260e", + "pi;": "\u03c0", + "pitchfork;": "\u22d4", + "piv;": "\u03d6", + "planck;": "\u210f", + "planckh;": "\u210e", + "plankv;": "\u210f", + "plus;": "+", + "plusacir;": "\u2a23", + "plusb;": "\u229e", + "pluscir;": "\u2a22", + "plusdo;": "\u2214", + "plusdu;": "\u2a25", + "pluse;": "\u2a72", + "plusmn": "\xb1", + "plusmn;": "\xb1", + "plussim;": "\u2a26", + "plustwo;": "\u2a27", + "pm;": "\xb1", + "pointint;": "\u2a15", + "popf;": "\U0001d561", + "pound": "\xa3", + "pound;": "\xa3", + "pr;": "\u227a", + "prE;": "\u2ab3", + "prap;": "\u2ab7", + "prcue;": "\u227c", + "pre;": "\u2aaf", + "prec;": "\u227a", + "precapprox;": "\u2ab7", + "preccurlyeq;": "\u227c", + "preceq;": "\u2aaf", + "precnapprox;": "\u2ab9", + "precneqq;": "\u2ab5", + "precnsim;": "\u22e8", + "precsim;": "\u227e", + "prime;": "\u2032", + "primes;": "\u2119", + "prnE;": "\u2ab5", + "prnap;": "\u2ab9", + "prnsim;": "\u22e8", + "prod;": "\u220f", + "profalar;": "\u232e", + "profline;": "\u2312", + "profsurf;": "\u2313", + "prop;": "\u221d", + "propto;": "\u221d", + "prsim;": "\u227e", + "prurel;": "\u22b0", + "pscr;": "\U0001d4c5", + "psi;": "\u03c8", + "puncsp;": "\u2008", + "qfr;": "\U0001d52e", + "qint;": "\u2a0c", + "qopf;": "\U0001d562", + "qprime;": "\u2057", + "qscr;": "\U0001d4c6", + "quaternions;": "\u210d", + "quatint;": "\u2a16", + "quest;": "?", + "questeq;": "\u225f", + "quot": "\"", + "quot;": "\"", + "rAarr;": "\u21db", + "rArr;": "\u21d2", + "rAtail;": "\u291c", + "rBarr;": "\u290f", + "rHar;": "\u2964", + "race;": "\u223d\u0331", + "racute;": "\u0155", + "radic;": "\u221a", + "raemptyv;": "\u29b3", + "rang;": "\u27e9", + "rangd;": "\u2992", + "range;": "\u29a5", + "rangle;": "\u27e9", + "raquo": "\xbb", + "raquo;": "\xbb", + "rarr;": "\u2192", + "rarrap;": "\u2975", + "rarrb;": "\u21e5", + "rarrbfs;": "\u2920", + "rarrc;": "\u2933", + "rarrfs;": "\u291e", + "rarrhk;": "\u21aa", + "rarrlp;": "\u21ac", + "rarrpl;": "\u2945", + "rarrsim;": "\u2974", + "rarrtl;": "\u21a3", + "rarrw;": "\u219d", + "ratail;": "\u291a", + "ratio;": "\u2236", + "rationals;": "\u211a", + "rbarr;": "\u290d", + "rbbrk;": "\u2773", + "rbrace;": "}", + "rbrack;": "]", + "rbrke;": "\u298c", + "rbrksld;": "\u298e", + "rbrkslu;": "\u2990", + "rcaron;": "\u0159", + "rcedil;": "\u0157", + "rceil;": "\u2309", + "rcub;": "}", + "rcy;": "\u0440", + "rdca;": "\u2937", + "rdldhar;": "\u2969", + "rdquo;": "\u201d", + "rdquor;": "\u201d", + "rdsh;": "\u21b3", + "real;": "\u211c", + "realine;": "\u211b", + "realpart;": "\u211c", + "reals;": "\u211d", + "rect;": "\u25ad", + "reg": "\xae", + "reg;": "\xae", + "rfisht;": "\u297d", + "rfloor;": "\u230b", + "rfr;": "\U0001d52f", + "rhard;": "\u21c1", + "rharu;": "\u21c0", + "rharul;": "\u296c", + "rho;": "\u03c1", + "rhov;": "\u03f1", + "rightarrow;": "\u2192", + "rightarrowtail;": "\u21a3", + "rightharpoondown;": "\u21c1", + "rightharpoonup;": "\u21c0", + "rightleftarrows;": "\u21c4", + "rightleftharpoons;": "\u21cc", + "rightrightarrows;": "\u21c9", + "rightsquigarrow;": "\u219d", + "rightthreetimes;": "\u22cc", + "ring;": "\u02da", + "risingdotseq;": "\u2253", + "rlarr;": "\u21c4", + "rlhar;": "\u21cc", + "rlm;": "\u200f", + "rmoust;": "\u23b1", + "rmoustache;": "\u23b1", + "rnmid;": "\u2aee", + "roang;": "\u27ed", + "roarr;": "\u21fe", + "robrk;": "\u27e7", + "ropar;": "\u2986", + "ropf;": "\U0001d563", + "roplus;": "\u2a2e", + "rotimes;": "\u2a35", + "rpar;": ")", + "rpargt;": "\u2994", + "rppolint;": "\u2a12", + "rrarr;": "\u21c9", + "rsaquo;": "\u203a", + "rscr;": "\U0001d4c7", + "rsh;": "\u21b1", + "rsqb;": "]", + "rsquo;": "\u2019", + "rsquor;": "\u2019", + "rthree;": "\u22cc", + "rtimes;": "\u22ca", + "rtri;": "\u25b9", + "rtrie;": "\u22b5", + "rtrif;": "\u25b8", + "rtriltri;": "\u29ce", + "ruluhar;": "\u2968", + "rx;": "\u211e", + "sacute;": "\u015b", + "sbquo;": "\u201a", + "sc;": "\u227b", + "scE;": "\u2ab4", + "scap;": "\u2ab8", + "scaron;": "\u0161", + "sccue;": "\u227d", + "sce;": "\u2ab0", + "scedil;": "\u015f", + "scirc;": "\u015d", + "scnE;": "\u2ab6", + "scnap;": "\u2aba", + "scnsim;": "\u22e9", + "scpolint;": "\u2a13", + "scsim;": "\u227f", + "scy;": "\u0441", + "sdot;": "\u22c5", + "sdotb;": "\u22a1", + "sdote;": "\u2a66", + "seArr;": "\u21d8", + "searhk;": "\u2925", + "searr;": "\u2198", + "searrow;": "\u2198", + "sect": "\xa7", + "sect;": "\xa7", + "semi;": ";", + "seswar;": "\u2929", + "setminus;": "\u2216", + "setmn;": "\u2216", + "sext;": "\u2736", + "sfr;": "\U0001d530", + "sfrown;": "\u2322", + "sharp;": "\u266f", + "shchcy;": "\u0449", + "shcy;": "\u0448", + "shortmid;": "\u2223", + "shortparallel;": "\u2225", + "shy": "\xad", + "shy;": "\xad", + "sigma;": "\u03c3", + "sigmaf;": "\u03c2", + "sigmav;": "\u03c2", + "sim;": "\u223c", + "simdot;": "\u2a6a", + "sime;": "\u2243", + "simeq;": "\u2243", + "simg;": "\u2a9e", + "simgE;": "\u2aa0", + "siml;": "\u2a9d", + "simlE;": "\u2a9f", + "simne;": "\u2246", + "simplus;": "\u2a24", + "simrarr;": "\u2972", + "slarr;": "\u2190", + "smallsetminus;": "\u2216", + "smashp;": "\u2a33", + "smeparsl;": "\u29e4", + "smid;": "\u2223", + "smile;": "\u2323", + "smt;": "\u2aaa", + "smte;": "\u2aac", + "smtes;": "\u2aac\ufe00", + "softcy;": "\u044c", + "sol;": "/", + "solb;": "\u29c4", + "solbar;": "\u233f", + "sopf;": "\U0001d564", + "spades;": "\u2660", + "spadesuit;": "\u2660", + "spar;": "\u2225", + "sqcap;": "\u2293", + "sqcaps;": "\u2293\ufe00", + "sqcup;": "\u2294", + "sqcups;": "\u2294\ufe00", + "sqsub;": "\u228f", + "sqsube;": "\u2291", + "sqsubset;": "\u228f", + "sqsubseteq;": "\u2291", + "sqsup;": "\u2290", + "sqsupe;": "\u2292", + "sqsupset;": "\u2290", + "sqsupseteq;": "\u2292", + "squ;": "\u25a1", + "square;": "\u25a1", + "squarf;": "\u25aa", + "squf;": "\u25aa", + "srarr;": "\u2192", + "sscr;": "\U0001d4c8", + "ssetmn;": "\u2216", + "ssmile;": "\u2323", + "sstarf;": "\u22c6", + "star;": "\u2606", + "starf;": "\u2605", + "straightepsilon;": "\u03f5", + "straightphi;": "\u03d5", + "strns;": "\xaf", + "sub;": "\u2282", + "subE;": "\u2ac5", + "subdot;": "\u2abd", + "sube;": "\u2286", + "subedot;": "\u2ac3", + "submult;": "\u2ac1", + "subnE;": "\u2acb", + "subne;": "\u228a", + "subplus;": "\u2abf", + "subrarr;": "\u2979", + "subset;": "\u2282", + "subseteq;": "\u2286", + "subseteqq;": "\u2ac5", + "subsetneq;": "\u228a", + "subsetneqq;": "\u2acb", + "subsim;": "\u2ac7", + "subsub;": "\u2ad5", + "subsup;": "\u2ad3", + "succ;": "\u227b", + "succapprox;": "\u2ab8", + "succcurlyeq;": "\u227d", + "succeq;": "\u2ab0", + "succnapprox;": "\u2aba", + "succneqq;": "\u2ab6", + "succnsim;": "\u22e9", + "succsim;": "\u227f", + "sum;": "\u2211", + "sung;": "\u266a", + "sup1": "\xb9", + "sup1;": "\xb9", + "sup2": "\xb2", + "sup2;": "\xb2", + "sup3": "\xb3", + "sup3;": "\xb3", + "sup;": "\u2283", + "supE;": "\u2ac6", + "supdot;": "\u2abe", + "supdsub;": "\u2ad8", + "supe;": "\u2287", + "supedot;": "\u2ac4", + "suphsol;": "\u27c9", + "suphsub;": "\u2ad7", + "suplarr;": "\u297b", + "supmult;": "\u2ac2", + "supnE;": "\u2acc", + "supne;": "\u228b", + "supplus;": "\u2ac0", + "supset;": "\u2283", + "supseteq;": "\u2287", + "supseteqq;": "\u2ac6", + "supsetneq;": "\u228b", + "supsetneqq;": "\u2acc", + "supsim;": "\u2ac8", + "supsub;": "\u2ad4", + "supsup;": "\u2ad6", + "swArr;": "\u21d9", + "swarhk;": "\u2926", + "swarr;": "\u2199", + "swarrow;": "\u2199", + "swnwar;": "\u292a", + "szlig": "\xdf", + "szlig;": "\xdf", + "target;": "\u2316", + "tau;": "\u03c4", + "tbrk;": "\u23b4", + "tcaron;": "\u0165", + "tcedil;": "\u0163", + "tcy;": "\u0442", + "tdot;": "\u20db", + "telrec;": "\u2315", + "tfr;": "\U0001d531", + "there4;": "\u2234", + "therefore;": "\u2234", + "theta;": "\u03b8", + "thetasym;": "\u03d1", + "thetav;": "\u03d1", + "thickapprox;": "\u2248", + "thicksim;": "\u223c", + "thinsp;": "\u2009", + "thkap;": "\u2248", + "thksim;": "\u223c", + "thorn": "\xfe", + "thorn;": "\xfe", + "tilde;": "\u02dc", + "times": "\xd7", + "times;": "\xd7", + "timesb;": "\u22a0", + "timesbar;": "\u2a31", + "timesd;": "\u2a30", + "tint;": "\u222d", + "toea;": "\u2928", + "top;": "\u22a4", + "topbot;": "\u2336", + "topcir;": "\u2af1", + "topf;": "\U0001d565", + "topfork;": "\u2ada", + "tosa;": "\u2929", + "tprime;": "\u2034", + "trade;": "\u2122", + "triangle;": "\u25b5", + "triangledown;": "\u25bf", + "triangleleft;": "\u25c3", + "trianglelefteq;": "\u22b4", + "triangleq;": "\u225c", + "triangleright;": "\u25b9", + "trianglerighteq;": "\u22b5", + "tridot;": "\u25ec", + "trie;": "\u225c", + "triminus;": "\u2a3a", + "triplus;": "\u2a39", + "trisb;": "\u29cd", + "tritime;": "\u2a3b", + "trpezium;": "\u23e2", + "tscr;": "\U0001d4c9", + "tscy;": "\u0446", + "tshcy;": "\u045b", + "tstrok;": "\u0167", + "twixt;": "\u226c", + "twoheadleftarrow;": "\u219e", + "twoheadrightarrow;": "\u21a0", + "uArr;": "\u21d1", + "uHar;": "\u2963", + "uacute": "\xfa", + "uacute;": "\xfa", + "uarr;": "\u2191", + "ubrcy;": "\u045e", + "ubreve;": "\u016d", + "ucirc": "\xfb", + "ucirc;": "\xfb", + "ucy;": "\u0443", + "udarr;": "\u21c5", + "udblac;": "\u0171", + "udhar;": "\u296e", + "ufisht;": "\u297e", + "ufr;": "\U0001d532", + "ugrave": "\xf9", + "ugrave;": "\xf9", + "uharl;": "\u21bf", + "uharr;": "\u21be", + "uhblk;": "\u2580", + "ulcorn;": "\u231c", + "ulcorner;": "\u231c", + "ulcrop;": "\u230f", + "ultri;": "\u25f8", + "umacr;": "\u016b", + "uml": "\xa8", + "uml;": "\xa8", + "uogon;": "\u0173", + "uopf;": "\U0001d566", + "uparrow;": "\u2191", + "updownarrow;": "\u2195", + "upharpoonleft;": "\u21bf", + "upharpoonright;": "\u21be", + "uplus;": "\u228e", + "upsi;": "\u03c5", + "upsih;": "\u03d2", + "upsilon;": "\u03c5", + "upuparrows;": "\u21c8", + "urcorn;": "\u231d", + "urcorner;": "\u231d", + "urcrop;": "\u230e", + "uring;": "\u016f", + "urtri;": "\u25f9", + "uscr;": "\U0001d4ca", + "utdot;": "\u22f0", + "utilde;": "\u0169", + "utri;": "\u25b5", + "utrif;": "\u25b4", + "uuarr;": "\u21c8", + "uuml": "\xfc", + "uuml;": "\xfc", + "uwangle;": "\u29a7", + "vArr;": "\u21d5", + "vBar;": "\u2ae8", + "vBarv;": "\u2ae9", + "vDash;": "\u22a8", + "vangrt;": "\u299c", + "varepsilon;": "\u03f5", + "varkappa;": "\u03f0", + "varnothing;": "\u2205", + "varphi;": "\u03d5", + "varpi;": "\u03d6", + "varpropto;": "\u221d", + "varr;": "\u2195", + "varrho;": "\u03f1", + "varsigma;": "\u03c2", + "varsubsetneq;": "\u228a\ufe00", + "varsubsetneqq;": "\u2acb\ufe00", + "varsupsetneq;": "\u228b\ufe00", + "varsupsetneqq;": "\u2acc\ufe00", + "vartheta;": "\u03d1", + "vartriangleleft;": "\u22b2", + "vartriangleright;": "\u22b3", + "vcy;": "\u0432", + "vdash;": "\u22a2", + "vee;": "\u2228", + "veebar;": "\u22bb", + "veeeq;": "\u225a", + "vellip;": "\u22ee", + "verbar;": "|", + "vert;": "|", + "vfr;": "\U0001d533", + "vltri;": "\u22b2", + "vnsub;": "\u2282\u20d2", + "vnsup;": "\u2283\u20d2", + "vopf;": "\U0001d567", + "vprop;": "\u221d", + "vrtri;": "\u22b3", + "vscr;": "\U0001d4cb", + "vsubnE;": "\u2acb\ufe00", + "vsubne;": "\u228a\ufe00", + "vsupnE;": "\u2acc\ufe00", + "vsupne;": "\u228b\ufe00", + "vzigzag;": "\u299a", + "wcirc;": "\u0175", + "wedbar;": "\u2a5f", + "wedge;": "\u2227", + "wedgeq;": "\u2259", + "weierp;": "\u2118", + "wfr;": "\U0001d534", + "wopf;": "\U0001d568", + "wp;": "\u2118", + "wr;": "\u2240", + "wreath;": "\u2240", + "wscr;": "\U0001d4cc", + "xcap;": "\u22c2", + "xcirc;": "\u25ef", + "xcup;": "\u22c3", + "xdtri;": "\u25bd", + "xfr;": "\U0001d535", + "xhArr;": "\u27fa", + "xharr;": "\u27f7", + "xi;": "\u03be", + "xlArr;": "\u27f8", + "xlarr;": "\u27f5", + "xmap;": "\u27fc", + "xnis;": "\u22fb", + "xodot;": "\u2a00", + "xopf;": "\U0001d569", + "xoplus;": "\u2a01", + "xotime;": "\u2a02", + "xrArr;": "\u27f9", + "xrarr;": "\u27f6", + "xscr;": "\U0001d4cd", + "xsqcup;": "\u2a06", + "xuplus;": "\u2a04", + "xutri;": "\u25b3", + "xvee;": "\u22c1", + "xwedge;": "\u22c0", + "yacute": "\xfd", + "yacute;": "\xfd", + "yacy;": "\u044f", + "ycirc;": "\u0177", + "ycy;": "\u044b", + "yen": "\xa5", + "yen;": "\xa5", + "yfr;": "\U0001d536", + "yicy;": "\u0457", + "yopf;": "\U0001d56a", + "yscr;": "\U0001d4ce", + "yucy;": "\u044e", + "yuml": "\xff", + "yuml;": "\xff", + "zacute;": "\u017a", + "zcaron;": "\u017e", + "zcy;": "\u0437", + "zdot;": "\u017c", + "zeetrf;": "\u2128", + "zeta;": "\u03b6", + "zfr;": "\U0001d537", + "zhcy;": "\u0436", + "zigrarr;": "\u21dd", + "zopf;": "\U0001d56b", + "zscr;": "\U0001d4cf", + "zwj;": "\u200d", + "zwnj;": "\u200c", } replacementCharacters = { - 0x0:u"\uFFFD", - 0x0d:u"\u000D", - 0x80:u"\u20AC", - 0x81:u"\u0081", - 0x81:u"\u0081", - 0x82:u"\u201A", - 0x83:u"\u0192", - 0x84:u"\u201E", - 0x85:u"\u2026", - 0x86:u"\u2020", - 0x87:u"\u2021", - 0x88:u"\u02C6", - 0x89:u"\u2030", - 0x8A:u"\u0160", - 0x8B:u"\u2039", - 0x8C:u"\u0152", - 0x8D:u"\u008D", - 0x8E:u"\u017D", - 0x8F:u"\u008F", - 0x90:u"\u0090", - 0x91:u"\u2018", - 0x92:u"\u2019", - 0x93:u"\u201C", - 0x94:u"\u201D", - 0x95:u"\u2022", - 0x96:u"\u2013", - 0x97:u"\u2014", - 0x98:u"\u02DC", - 0x99:u"\u2122", - 0x9A:u"\u0161", - 0x9B:u"\u203A", - 0x9C:u"\u0153", - 0x9D:u"\u009D", - 0x9E:u"\u017E", - 0x9F:u"\u0178", + 0x0: "\uFFFD", + 0x0d: "\u000D", + 0x80: "\u20AC", + 0x81: "\u0081", + 0x81: "\u0081", + 0x82: "\u201A", + 0x83: "\u0192", + 0x84: "\u201E", + 0x85: "\u2026", + 0x86: "\u2020", + 0x87: "\u2021", + 0x88: "\u02C6", + 0x89: "\u2030", + 0x8A: "\u0160", + 0x8B: "\u2039", + 0x8C: "\u0152", + 0x8D: "\u008D", + 0x8E: "\u017D", + 0x8F: "\u008F", + 0x90: "\u0090", + 0x91: "\u2018", + 0x92: "\u2019", + 0x93: "\u201C", + 0x94: "\u201D", + 0x95: "\u2022", + 0x96: "\u2013", + 0x97: "\u2014", + 0x98: "\u02DC", + 0x99: "\u2122", + 0x9A: "\u0161", + 0x9B: "\u203A", + 0x9C: "\u0153", + 0x9D: "\u009D", + 0x9E: "\u017E", + 0x9F: "\u0178", } encodings = { @@ -3061,25 +3078,27 @@ encodings = { 'x-x-big5': 'big5'} tokenTypes = { - "Doctype":0, - "Characters":1, - "SpaceCharacters":2, - "StartTag":3, - "EndTag":4, - "EmptyTag":5, - "Comment":6, - "ParseError":7 + "Doctype": 0, + "Characters": 1, + "SpaceCharacters": 2, + "StartTag": 3, + "EndTag": 4, + "EmptyTag": 5, + "Comment": 6, + "ParseError": 7 } -tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"], +tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"], tokenTypes["EmptyTag"])) -prefixes = dict([(v,k) for k,v in namespaces.iteritems()]) +prefixes = dict([(v, k) for k, v in namespaces.items()]) prefixes["http://www.w3.org/1998/Math/MathML"] = "math" + class DataLossWarning(UserWarning): pass + class ReparseException(Exception): pass diff --git a/libs/html5lib/filters/_base.py b/libs/html5lib/filters/_base.py index bca94ada..c7dbaed0 100644 --- a/libs/html5lib/filters/_base.py +++ b/libs/html5lib/filters/_base.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, unicode_literals + class Filter(object): def __init__(self, source): diff --git a/libs/html5lib/filters/alphabeticalattributes.py b/libs/html5lib/filters/alphabeticalattributes.py new file mode 100644 index 00000000..fed6996c --- /dev/null +++ b/libs/html5lib/filters/alphabeticalattributes.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, division, unicode_literals + +from . import _base + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + + +class Filter(_base.Filter): + def __iter__(self): + for token in _base.Filter.__iter__(self): + if token["type"] in ("StartTag", "EmptyTag"): + attrs = OrderedDict() + for name, value in sorted(token["data"].items(), + key=lambda x: x[0]): + attrs[name] = value + token["data"] = attrs + yield token diff --git a/libs/html5lib/filters/formfiller.py b/libs/html5lib/filters/formfiller.py deleted file mode 100644 index 94001714..00000000 --- a/libs/html5lib/filters/formfiller.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# The goal is to finally have a form filler where you pass data for -# each form, using the algorithm for "Seeding a form with initial values" -# See http://www.whatwg.org/specs/web-forms/current-work/#seeding -# - -import _base - -from html5lib.constants import spaceCharacters -spaceCharacters = u"".join(spaceCharacters) - -class SimpleFilter(_base.Filter): - def __init__(self, source, fieldStorage): - _base.Filter.__init__(self, source) - self.fieldStorage = fieldStorage - - def __iter__(self): - field_indices = {} - state = None - field_name = None - for token in _base.Filter.__iter__(self): - type = token["type"] - if type in ("StartTag", "EmptyTag"): - name = token["name"].lower() - if name == "input": - field_name = None - field_type = None - input_value_index = -1 - input_checked_index = -1 - for i,(n,v) in enumerate(token["data"]): - n = n.lower() - if n == u"name": - field_name = v.strip(spaceCharacters) - elif n == u"type": - field_type = v.strip(spaceCharacters) - elif n == u"checked": - input_checked_index = i - elif n == u"value": - input_value_index = i - - value_list = self.fieldStorage.getlist(field_name) - field_index = field_indices.setdefault(field_name, 0) - if field_index < len(value_list): - value = value_list[field_index] - else: - value = "" - - if field_type in (u"checkbox", u"radio"): - if value_list: - if token["data"][input_value_index][1] == value: - if input_checked_index < 0: - token["data"].append((u"checked", u"")) - field_indices[field_name] = field_index + 1 - elif input_checked_index >= 0: - del token["data"][input_checked_index] - - elif field_type not in (u"button", u"submit", u"reset"): - if input_value_index >= 0: - token["data"][input_value_index] = (u"value", value) - else: - token["data"].append((u"value", value)) - field_indices[field_name] = field_index + 1 - - field_type = None - field_name = None - - elif name == "textarea": - field_type = "textarea" - field_name = dict((token["data"])[::-1])["name"] - - elif name == "select": - field_type = "select" - attributes = dict(token["data"][::-1]) - field_name = attributes.get("name") - is_select_multiple = "multiple" in attributes - is_selected_option_found = False - - elif field_type == "select" and field_name and name == "option": - option_selected_index = -1 - option_value = None - for i,(n,v) in enumerate(token["data"]): - n = n.lower() - if n == "selected": - option_selected_index = i - elif n == "value": - option_value = v.strip(spaceCharacters) - if option_value is None: - raise NotImplementedError("