diff --git a/contributing.md b/contributing.md index ef8546f0..d5db0b42 100644 --- a/contributing.md +++ b/contributing.md @@ -1,15 +1,25 @@ -#So you feel like posting a bug, sending me a pull request or just telling me how awesome I am. No problem! +## Got a issue/feature request or submitting a pull request? -##Just make sure you think of the following things: +Make sure you think of the following things: - * Search through the existing (and closed) issues first. See if you can get your answer there. +## Issue + * Search through the existing (and closed) issues first, see if you can get your answer there. * Double check the result manually, because it could be an external issue. * Post logs! Without seeing what is going on, I can't reproduce the error. - * What is the movie + quality you are searching for. - * What are you settings for the specific problem. - * What providers are you using. (While your logs include these, scanning through hundred of lines of log isn't my hobby). - * Give me a short step by step of how to reproduce. + * Also check the logs before submitting, obvious errors like permission or http errors are often not related to CP. + * What is the movie + quality you are searching for? + * What are you're settings for the specific problem? + * What providers are you using? (While you're logs include these, scanning through hundred of lines of log isn't our hobby) + * Post the logs from config directory, please do not copy paste the UI. Use pastebin to store these logs! + * Give a short step by step of how to reproduce the error. * What hardware / OS are you using and what are the limits? NAS can be slow and maybe have a different python installed then when you use CP on OSX or Windows for example. - * I will mark issues with the "can't reproduce" tag. Don't go asking me "why closed" if it clearly says the issue in the tag ;) + * I will mark issues with the "can't reproduce" tag. Don't go asking "why closed" if it clearly says the issue in the tag ;) + * If you're running on a NAS (QNAP, Austor etc..) with pre-made packages, make sure these are setup to use our source repo (RuudBurger/CouchPotatoServer) and nothing else!! -**If I don't get enough info, the chance of the issue getting closed is a lot bigger ;)** +## Pull Request + * Make sure you're pull request is made for develop branch (or relevant feature branch) + * Have you tested your PR? If not, why? + * Are there any limitations of your PR we should know of? + * Make sure to keep you're PR up-to-date with the branch you're trying to push into. + +**If we don't get enough info, the chance of the issue getting closed is a lot bigger ;)** diff --git a/couchpotato/core/_base/clientscript/main.py b/couchpotato/core/_base/clientscript/main.py index 1b7f1636..b80ddcc1 100644 --- a/couchpotato/core/_base/clientscript/main.py +++ b/couchpotato/core/_base/clientscript/main.py @@ -34,6 +34,8 @@ class ClientScript(Plugin): 'scripts/library/question.js', 'scripts/library/scrollspy.js', 'scripts/library/spin.js', + 'scripts/library/Array.stableSort.js', + 'scripts/library/async.js', 'scripts/couchpotato.js', 'scripts/api.js', 'scripts/library/history.js', diff --git a/couchpotato/core/_base/scheduler/main.py b/couchpotato/core/_base/scheduler/main.py index 2c97e1b4..773213be 100644 --- a/couchpotato/core/_base/scheduler/main.py +++ b/couchpotato/core/_base/scheduler/main.py @@ -31,13 +31,13 @@ class Scheduler(Plugin): pass def doShutdown(self): - super(Scheduler, self).doShutdown() self.stop() + return super(Scheduler, self).doShutdown() def stop(self): if self.started: log.debug('Stopping scheduler') - self.sched.shutdown() + self.sched.shutdown(wait = False) log.debug('Scheduler stopped') self.started = False diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index aecf0c4f..648c2c4c 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -183,9 +183,6 @@ class GitUpdater(BaseUpdater): def doUpdate(self): try: - log.debug('Stashing local changes') - self.repo.saveStash() - log.info('Updating to latest version') self.repo.pull() diff --git a/couchpotato/core/_base/updater/static/updater.js b/couchpotato/core/_base/updater/static/updater.js index 0577c783..860ad514 100644 --- a/couchpotato/core/_base/updater/static/updater.js +++ b/couchpotato/core/_base/updater/static/updater.js @@ -24,7 +24,7 @@ var UpdaterBase = new Class({ self.doUpdate(); else { App.unBlockPage(); - App.fireEvent('message', 'No updates available'); + App.on('message', 'No updates available'); } } }) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 9e24d914..b6894edf 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -49,21 +49,26 @@ class Downloader(Provider): return [] - def _download(self, data = None, movie = None, manual = False, filedata = None): - if not movie: movie = {} + def _download(self, data = None, media = None, manual = False, filedata = None): + if not media: media = {} if not data: data = {} if self.isDisabled(manual, data): return - return self.download(data = data, movie = movie, filedata = filedata) + return self.download(data = data, media = media, filedata = filedata) - def _getAllDownloadStatus(self): + def _getAllDownloadStatus(self, download_ids): if self.isDisabled(manual = True, data = {}): return - return self.getAllDownloadStatus() + ids = [download_id['id'] for download_id in download_ids if download_id['downloader'] == self.getName()] - def getAllDownloadStatus(self): + if ids: + return self.getAllDownloadStatus(ids) + else: + return + + def getAllDownloadStatus(self, ids): return def _removeFailed(self, release_download): diff --git a/couchpotato/core/downloaders/blackhole/__init__.py b/couchpotato/core/downloaders/blackhole/__init__.py index 6b5279a1..91164d66 100644 --- a/couchpotato/core/downloaders/blackhole/__init__.py +++ b/couchpotato/core/downloaders/blackhole/__init__.py @@ -13,7 +13,7 @@ config = [{ 'list': 'download_providers', 'name': 'blackhole', 'label': 'Black hole', - 'description': 'Download the NZB/Torrent to a specific folder.', + 'description': 'Download the NZB/Torrent to a specific folder. Note: Seeding and copying/linking features do not work with Black hole.', 'wizard': True, 'options': [ { diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index 854860cd..5216370f 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -12,8 +12,8 @@ class Blackhole(Downloader): protocol = ['nzb', 'torrent', 'torrent_magnet'] - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} directory = self.conf('directory') @@ -33,7 +33,7 @@ class Blackhole(Downloader): log.error('No nzb/torrent available: %s', data.get('url')) return False - file_name = self.createFileName(data, filedata, movie) + file_name = self.createFileName(data, filedata, media) full_path = os.path.join(directory, file_name) if self.conf('create_subdir'): @@ -51,10 +51,10 @@ class Blackhole(Downloader): with open(full_path, 'wb') as f: f.write(filedata) os.chmod(full_path, Env.getPermission('file')) - return True + return self.downloadReturnId('') else: log.info('File %s already exists.', full_path) - return True + return self.downloadReturnId('') except: log.error('Failed to download to blackhole %s', traceback.format_exc()) diff --git a/couchpotato/core/downloaders/deluge/main.py b/couchpotato/core/downloaders/deluge/main.py index f65b74ae..57c19580 100644 --- a/couchpotato/core/downloaders/deluge/main.py +++ b/couchpotato/core/downloaders/deluge/main.py @@ -32,7 +32,10 @@ class Deluge(Downloader): return self.drpc - def download(self, data, movie, filedata = None): + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol'))) if not self.connect(): @@ -73,7 +76,7 @@ class Deluge(Downloader): if data.get('protocol') == 'torrent_magnet': remote_torrent = self.drpc.add_torrent_magnet(data.get('url'), options) else: - filename = self.createFileName(data, filedata, movie) + filename = self.createFileName(data, filedata, media) remote_torrent = self.drpc.add_torrent_file(filename, filedata, options) if not remote_torrent: @@ -83,7 +86,7 @@ class Deluge(Downloader): log.info('Torrent sent to Deluge successfully.') return self.downloadReturnId(remote_torrent) - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): log.debug('Checking Deluge download status.') @@ -100,38 +103,39 @@ class Deluge(Downloader): for torrent_id in queue: torrent = queue[torrent_id] - log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) - - # Deluge has no easy way to work out if a torrent is stalled or failing. - #status = 'failed' - status = 'busy' - if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio']): - # We have torrent['seeding_time'] to work out what the seeding time is, but we do not - # have access to the downloader seed_time, as with deluge we have no way to pass it - # when the torrent is added. So Deluge will only look at the ratio. - # See above comment in download(). - status = 'seeding' - elif torrent['is_seed'] and torrent['is_finished'] and torrent['paused'] and torrent['state'] == 'Paused': - status = 'completed' - - download_dir = sp(torrent['save_path']) - if torrent['move_on_completed']: - download_dir = torrent['move_completed_path'] - - torrent_files = [] - for file_item in torrent['files']: - torrent_files.append(os.path.join(download_dir), sp(file_item['path'])) - - release_downloads.append({ - 'id': torrent['hash'], - 'name': torrent['name'], - 'status': status, - 'original_status': torrent['state'], - 'seed_ratio': torrent['ratio'], - 'timeleft': str(timedelta(seconds = torrent['eta'])), - 'folder': sp(download_dir if len(torrent_files) == 1 else os.path.join(download_dir, torrent['name'])), - 'files': '|'.join(torrent_files), - }) + if torrent['hash'] in ids: + log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) + + # Deluge has no easy way to work out if a torrent is stalled or failing. + #status = 'failed' + status = 'busy' + if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio']): + # We have torrent['seeding_time'] to work out what the seeding time is, but we do not + # have access to the downloader seed_time, as with deluge we have no way to pass it + # when the torrent is added. So Deluge will only look at the ratio. + # See above comment in download(). + status = 'seeding' + elif torrent['is_seed'] and torrent['is_finished'] and torrent['paused'] and torrent['state'] == 'Paused': + status = 'completed' + + download_dir = sp(torrent['save_path']) + if torrent['move_on_completed']: + download_dir = torrent['move_completed_path'] + + torrent_files = [] + for file_item in torrent['files']: + torrent_files.append(sp(os.path.join(download_dir, file_item['path']))) + + release_downloads.append({ + 'id': torrent['hash'], + 'name': torrent['name'], + 'status': status, + 'original_status': torrent['state'], + 'seed_ratio': torrent['ratio'], + 'timeleft': str(timedelta(seconds = torrent['eta'])), + 'folder': sp(download_dir if len(torrent_files) == 1 else os.path.join(download_dir, torrent['name'])), + 'files': '|'.join(torrent_files), + }) return release_downloads diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index f8506134..482da2ca 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -19,8 +19,8 @@ class NZBGet(Downloader): url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc' - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} if not filedata: @@ -30,7 +30,7 @@ class NZBGet(Downloader): log.info('Sending "%s" to NZBGet.', data.get('name')) url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} - nzb_name = ss('%s.nzb' % self.createNzbName(data, movie)) + nzb_name = ss('%s.nzb' % self.createNzbName(data, media)) rpc = xmlrpclib.ServerProxy(url) try: @@ -67,7 +67,7 @@ class NZBGet(Downloader): log.error('NZBGet could not add %s to the queue.', nzb_name) return False - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): log.debug('Checking NZBGet download status.') @@ -102,51 +102,54 @@ class NZBGet(Downloader): release_downloads = ReleaseDownloadList(self) for nzb in groups: - log.debug('Found %s in NZBGet download queue', nzb['NZBFilename']) try: nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0] except: nzb_id = nzb['NZBID'] - - timeleft = -1 - try: - if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']): - timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) - except: - pass - - release_downloads.append({ - 'id': nzb_id, - 'name': nzb['NZBFilename'], - 'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED', - # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item - 'timeleft': timeleft, - }) + if nzb_id in ids: + log.debug('Found %s in NZBGet download queue', nzb['NZBFilename']) + timeleft = -1 + try: + if nzb['ActiveDownloads'] > 0 and nzb['DownloadRate'] > 0 and not (status['DownloadPaused'] or status['Download2Paused']): + timeleft = str(timedelta(seconds = nzb['RemainingSizeMB'] / status['DownloadRate'] * 2 ^ 20)) + except: + pass + + release_downloads.append({ + 'id': nzb_id, + 'name': nzb['NZBFilename'], + 'original_status': 'DOWNLOADING' if nzb['ActiveDownloads'] > 0 else 'QUEUED', + # Seems to have no native API function for time left. This will return the time left after NZBGet started downloading this item + 'timeleft': timeleft, + }) for nzb in queue: # 'Parameters' is not passed in rpc.postqueue - log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename']) - release_downloads.append({ - 'id': nzb['NZBID'], - 'name': nzb['NZBFilename'], - 'original_status': nzb['Stage'], - 'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1, - }) + if nzb['NZBID'] in ids: + log.debug('Found %s in NZBGet postprocessing queue', nzb['NZBFilename']) + release_downloads.append({ + 'id': nzb['NZBID'], + 'name': nzb['NZBFilename'], + 'original_status': nzb['Stage'], + 'timeleft': str(timedelta(seconds = 0)) if not status['PostPaused'] else -1, + }) for nzb in history: - log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log'])) try: nzb_id = [param['Value'] for param in nzb['Parameters'] if param['Name'] == 'couchpotato'][0] except: nzb_id = nzb['NZBID'] - release_downloads.append({ - 'id': nzb_id, - 'name': nzb['NZBFilename'], - 'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed', - 'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'], - 'timeleft': str(timedelta(seconds = 0)), - 'folder': sp(nzb['DestDir']) - }) + + if nzb_id in ids: + log.debug('Found %s in NZBGet history. ParStatus: %s, ScriptStatus: %s, Log: %s', (nzb['NZBFilename'] , nzb['ParStatus'], nzb['ScriptStatus'] , nzb['Log'])) + release_downloads.append({ + 'id': nzb_id, + 'name': nzb['NZBFilename'], + 'status': 'completed' if nzb['ParStatus'] in ['SUCCESS', 'NONE'] and nzb['ScriptStatus'] in ['SUCCESS', 'NONE'] else 'failed', + 'original_status': nzb['ParStatus'] + ', ' + nzb['ScriptStatus'], + 'timeleft': str(timedelta(seconds = 0)), + 'folder': sp(nzb['DestDir']) + }) return release_downloads diff --git a/couchpotato/core/downloaders/nzbvortex/main.py b/couchpotato/core/downloaders/nzbvortex/main.py index f4e233be..bcf8cae7 100644 --- a/couchpotato/core/downloaders/nzbvortex/main.py +++ b/couchpotato/core/downloaders/nzbvortex/main.py @@ -8,9 +8,11 @@ from uuid import uuid4 import hashlib import httplib import json +import os import socket import ssl import sys +import time import traceback import urllib2 @@ -23,44 +25,46 @@ class NZBVortex(Downloader): api_level = None session_id = None - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} # Send the nzb try: - nzb_filename = self.createFileName(data, filedata, movie) + nzb_filename = self.createFileName(data, filedata, media) self.call('nzb/add', params = {'file': (nzb_filename, filedata)}, multipart = True) + time.sleep(10) raw_statuses = self.call('nzb') - nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if nzb['name'] == nzb_filename][0] + nzb_id = [nzb['id'] for nzb in raw_statuses.get('nzbs', []) if os.path.basename(item['nzbFileName']) == nzb_filename][0] return self.downloadReturnId(nzb_id) except: log.error('Something went wrong sending the NZB file: %s', traceback.format_exc()) return False - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): raw_statuses = self.call('nzb') release_downloads = ReleaseDownloadList(self) for nzb in raw_statuses.get('nzbs', []): + if nzb['id'] in ids: - # Check status - status = 'busy' - if nzb['state'] == 20: - status = 'completed' - elif nzb['state'] in [21, 22, 24]: - status = 'failed' - - release_downloads.append({ - 'id': nzb['id'], - 'name': nzb['uiTitle'], - 'status': status, - 'original_status': nzb['state'], - 'timeleft':-1, - 'folder': sp(nzb['destinationPath']), - }) + # Check status + status = 'busy' + if nzb['state'] == 20: + status = 'completed' + elif nzb['state'] in [21, 22, 24]: + status = 'failed' + + release_downloads.append({ + 'id': nzb['id'], + 'name': nzb['uiTitle'], + 'status': status, + 'original_status': nzb['state'], + 'timeleft':-1, + 'folder': sp(nzb['destinationPath']), + }) return release_downloads diff --git a/couchpotato/core/downloaders/pneumatic/main.py b/couchpotato/core/downloaders/pneumatic/main.py index 643350e1..863cdb07 100644 --- a/couchpotato/core/downloaders/pneumatic/main.py +++ b/couchpotato/core/downloaders/pneumatic/main.py @@ -12,8 +12,8 @@ class Pneumatic(Downloader): protocol = ['nzb'] strm_syntax = 'plugin://plugin.program.pneumatic/?mode=strm&type=add_file&nzb=%s&nzbname=%s' - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} directory = self.conf('directory') @@ -25,7 +25,7 @@ class Pneumatic(Downloader): log.error('No nzb available!') return False - fullPath = os.path.join(directory, self.createFileName(data, filedata, movie)) + fullPath = os.path.join(directory, self.createFileName(data, filedata, media)) try: if not os.path.isfile(fullPath): @@ -33,7 +33,7 @@ class Pneumatic(Downloader): with open(fullPath, 'wb') as f: f.write(filedata) - nzb_name = self.createNzbName(data, movie) + nzb_name = self.createNzbName(data, media) strm_path = os.path.join(directory, nzb_name) strm_file = open(strm_path + '.strm', 'wb') @@ -41,11 +41,11 @@ class Pneumatic(Downloader): strm_file.write(strmContent) strm_file.close() - return True + return self.downloadReturnId('') else: log.info('File %s already exists.', fullPath) - return True + return self.downloadReturnId('') except: log.error('Failed to download .strm: %s', traceback.format_exc()) diff --git a/couchpotato/core/downloaders/rtorrent/__init__.py b/couchpotato/core/downloaders/rtorrent/__init__.py index 026a56c6..684ea45e 100755 --- a/couchpotato/core/downloaders/rtorrent/__init__.py +++ b/couchpotato/core/downloaders/rtorrent/__init__.py @@ -58,14 +58,6 @@ config = [{ 'advanced': True, 'description': 'Also remove the leftover files.', }, - { - 'name': 'append_label', - 'label': 'Append Label', - 'default': False, - 'advanced': True, - 'type': 'bool', - 'description': 'Append label to download location. Requires you to set the download location above.', - }, { 'name': 'paused', 'type': 'bool', diff --git a/couchpotato/core/downloaders/rtorrent/main.py b/couchpotato/core/downloaders/rtorrent/main.py index d7ae589f..2cbfd902 100755 --- a/couchpotato/core/downloaders/rtorrent/main.py +++ b/couchpotato/core/downloaders/rtorrent/main.py @@ -77,7 +77,10 @@ class rTorrent(Downloader): return True - def download(self, data, movie, filedata = None): + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} + log.debug('Sending "%s" to rTorrent.', (data.get('name'))) if not self.connect(): @@ -125,9 +128,7 @@ class rTorrent(Downloader): if self.conf('label'): torrent.set_custom(1, self.conf('label')) - if self.conf('directory') and self.conf('append_label'): - torrent.set_directory(os.path.join(self.conf('directory'), self.conf('label'))) - elif self.conf('directory'): + if self.conf('directory'): torrent.set_directory(self.conf('directory')) # Set Ratio Group @@ -142,7 +143,7 @@ class rTorrent(Downloader): log.error('Failed to send torrent to rTorrent: %s', err) return False - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): log.debug('Checking rTorrent download status.') if not self.connect(): @@ -154,27 +155,28 @@ class rTorrent(Downloader): release_downloads = ReleaseDownloadList(self) for torrent in torrents: - torrent_files = [] - for file_item in torrent.get_files(): - torrent_files.append(sp(os.path.join(torrent.directory, file_item.path))) - - status = 'busy' - if torrent.complete: - if torrent.active: - status = 'seeding' - else: - status = 'completed' - - release_downloads.append({ - 'id': torrent.info_hash, - 'name': torrent.name, - 'status': status, - 'seed_ratio': torrent.ratio, - 'original_status': torrent.state, - 'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1, - 'folder': sp(torrent.directory), - 'files': '|'.join(torrent_files) - }) + if torrent.info_hash in ids: + torrent_files = [] + for file_item in torrent.get_files(): + torrent_files.append(sp(os.path.join(torrent.directory, file_item.path))) + + status = 'busy' + if torrent.complete: + if torrent.active: + status = 'seeding' + else: + status = 'completed' + + release_downloads.append({ + 'id': torrent.info_hash, + 'name': torrent.name, + 'status': status, + 'seed_ratio': torrent.ratio, + 'original_status': torrent.state, + 'timeleft': str(timedelta(seconds = float(torrent.left_bytes) / torrent.down_rate)) if torrent.down_rate > 0 else -1, + 'folder': sp(torrent.directory), + 'files': '|'.join(torrent_files) + }) return release_downloads diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index aba21231..808ceab7 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -16,8 +16,8 @@ class Sabnzbd(Downloader): protocol = ['nzb'] - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} log.info('Sending "%s" to SABnzbd.', data.get('name')) @@ -25,7 +25,7 @@ class Sabnzbd(Downloader): req_params = { 'cat': self.conf('category'), 'mode': 'addurl', - 'nzbname': self.createNzbName(data, movie), + 'nzbname': self.createNzbName(data, media), 'priority': self.conf('priority'), } @@ -36,7 +36,7 @@ class Sabnzbd(Downloader): return False # If it's a .rar, it adds the .rar extension, otherwise it stays .nzb - nzb_filename = self.createFileName(data, filedata, movie) + nzb_filename = self.createFileName(data, filedata, media) req_params['mode'] = 'addfile' else: req_params['name'] = data.get('url') @@ -64,7 +64,7 @@ class Sabnzbd(Downloader): log.error('Error getting data from SABNZBd: %s', sab_data) return False - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): log.debug('Checking SABnzbd download status.') @@ -91,35 +91,36 @@ class Sabnzbd(Downloader): # Get busy releases for nzb in queue.get('slots', []): - status = 'busy' - if 'ENCRYPTED / ' in nzb['filename']: - status = 'failed' - - release_downloads.append({ - 'id': nzb['nzo_id'], - 'name': nzb['filename'], - 'status': status, - 'original_status': nzb['status'], - 'timeleft': nzb['timeleft'] if not queue['paused'] else -1, - }) + if nzb['nzo_id'] in ids: + status = 'busy' + if 'ENCRYPTED / ' in nzb['filename']: + status = 'failed' + + release_downloads.append({ + 'id': nzb['nzo_id'], + 'name': nzb['filename'], + 'status': status, + 'original_status': nzb['status'], + 'timeleft': nzb['timeleft'] if not queue['paused'] else -1, + }) # Get old releases for nzb in history.get('slots', []): - - status = 'busy' - if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()): - status = 'failed' - elif nzb['status'] == 'Completed': - status = 'completed' - - release_downloads.append({ - 'id': nzb['nzo_id'], - 'name': nzb['name'], - 'status': status, - 'original_status': nzb['status'], - 'timeleft': str(timedelta(seconds = 0)), - 'folder': sp(os.path.dirname(nzb['storage']) if os.path.isfile(nzb['storage']) else nzb['storage']), - }) + if nzb['nzo_id'] in ids: + status = 'busy' + if nzb['status'] == 'Failed' or (nzb['status'] == 'Completed' and nzb['fail_message'].strip()): + status = 'failed' + elif nzb['status'] == 'Completed': + status = 'completed' + + release_downloads.append({ + 'id': nzb['nzo_id'], + 'name': nzb['name'], + 'status': status, + 'original_status': nzb['status'], + 'timeleft': str(timedelta(seconds = 0)), + 'folder': sp(os.path.dirname(nzb['storage']) if os.path.isfile(nzb['storage']) else nzb['storage']), + }) return release_downloads diff --git a/couchpotato/core/downloaders/synology/main.py b/couchpotato/core/downloaders/synology/main.py index d5082c77..5d192fcf 100644 --- a/couchpotato/core/downloaders/synology/main.py +++ b/couchpotato/core/downloaders/synology/main.py @@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog import json import requests +import traceback log = CPLog(__name__) @@ -12,8 +13,8 @@ class Synology(Downloader): protocol = ['nzb', 'torrent', 'torrent_magnet'] log = CPLog(__name__) - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} response = False @@ -34,14 +35,14 @@ class Synology(Downloader): elif data['protocol'] in ['nzb', 'torrent']: log.info('Adding %s' % data['protocol']) if not filedata: - log.error('No %s data found' % data['protocol']) + log.error('No %s data found', data['protocol']) else: filename = data['name'] + '.' + data['protocol'] response = srpc.create_task(filename = filename, filedata = filedata) - except Exception, err: - log.error('Exception while adding torrent: %s', err) + except: + log.error('Exception while adding torrent: %s', traceback.format_exc()) finally: - return response + return self.downloadReturnId('') if response else False def getEnabledProtocol(self): if self.conf('use_for') == 'both': diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 2eabb2e8..46b5f3b4 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -31,7 +31,9 @@ class Transmission(Downloader): return self.trpc - def download(self, data, movie, filedata = None): + def download(self, data = None, media = None, filedata = None): + if not media: media = {} + if not data: data = {} log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('protocol'))) @@ -81,7 +83,7 @@ class Transmission(Downloader): log.info('Torrent sent to Transmission successfully.') return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): log.debug('Checking Transmission download status.') @@ -100,31 +102,32 @@ class Transmission(Downloader): return False for torrent in queue['torrents']: - log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', - (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent['eta'], torrent['uploadRatio'], torrent['isFinished'])) - - torrent_files = [] - for file_item in torrent['files']: - torrent_files.append(sp(os.path.join(torrent['downloadDir'], file_item['name']))) - - status = 'busy' - if torrent.get('isStalled') and self.conf('stalled_as_failed'): - status = 'failed' - elif torrent['status'] == 0 and torrent['percentDone'] == 1: - status = 'completed' - elif torrent['status'] in [5, 6]: - status = 'seeding' - - release_downloads.append({ - 'id': torrent['hashString'], - 'name': torrent['name'], - 'status': status, - 'original_status': torrent['status'], - 'seed_ratio': torrent['uploadRatio'], - 'timeleft': str(timedelta(seconds = torrent['eta'])), - 'folder': sp(torrent['downloadDir'] if len(torrent_files) == 1 else os.path.join(torrent['downloadDir'], torrent['name'])), - 'files': '|'.join(torrent_files) - }) + if torrent['hashString'] in ids: + log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / eta=%s / uploadRatio=%s / isFinished=%s', + (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent['eta'], torrent['uploadRatio'], torrent['isFinished'])) + + torrent_files = [] + for file_item in torrent['files']: + torrent_files.append(sp(os.path.join(torrent['downloadDir'], file_item['name']))) + + status = 'busy' + if torrent.get('isStalled') and self.conf('stalled_as_failed'): + status = 'failed' + elif torrent['status'] == 0 and torrent['percentDone'] == 1: + status = 'completed' + elif torrent['status'] in [5, 6]: + status = 'seeding' + + release_downloads.append({ + 'id': torrent['hashString'], + 'name': torrent['name'], + 'status': status, + 'original_status': torrent['status'], + 'seed_ratio': torrent['uploadRatio'], + 'timeleft': str(timedelta(seconds = torrent['eta'])), + 'folder': sp(torrent['downloadDir'] if len(torrent_files) == 1 else os.path.join(torrent['downloadDir'], torrent['name'])), + 'files': '|'.join(torrent_files) + }) return release_downloads diff --git a/couchpotato/core/downloaders/utorrent/main.py b/couchpotato/core/downloaders/utorrent/main.py index 1db1b8a3..056cd15d 100644 --- a/couchpotato/core/downloaders/utorrent/main.py +++ b/couchpotato/core/downloaders/utorrent/main.py @@ -36,8 +36,8 @@ class uTorrent(Downloader): return self.utorrent_api - def download(self, data = None, movie = None, filedata = None): - if not movie: movie = {} + def download(self, data = None, media = None, filedata = None): + if not media: media = {} if not data: data = {} log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('protocol'))) @@ -77,7 +77,8 @@ class uTorrent(Downloader): else: info = bdecode(filedata)["info"] torrent_hash = sha1(benc(info)).hexdigest().upper() - torrent_filename = self.createFileName(data, filedata, movie) + + torrent_filename = self.createFileName(data, filedata, media) if data.get('seed_ratio'): torrent_params['seed_override'] = 1 @@ -104,7 +105,7 @@ class uTorrent(Downloader): return self.downloadReturnId(torrent_hash) - def getAllDownloadStatus(self): + def getAllDownloadStatus(self, ids): log.debug('Checking uTorrent download status.') @@ -129,47 +130,48 @@ class uTorrent(Downloader): # Get torrents for torrent in queue['torrents']: + if torrent[0] in ids: - #Get files of the torrent - torrent_files = [] - try: - torrent_files = json.loads(self.utorrent_api.get_files(torrent[0])) - torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]] - except: - log.debug('Failed getting files from torrent: %s', torrent[2]) - - status_flags = { - "STARTED" : 1, - "CHECKING" : 2, - "CHECK-START" : 4, - "CHECKED" : 8, - "ERROR" : 16, - "PAUSED" : 32, - "QUEUED" : 64, - "LOADED" : 128 - } - - status = 'busy' - if (torrent[1] & status_flags["STARTED"] or torrent[1] & status_flags["QUEUED"]) and torrent[4] == 1000: - status = 'seeding' - elif (torrent[1] & status_flags["ERROR"]): - status = 'failed' - elif torrent[4] == 1000: - status = 'completed' - - if not status == 'busy': - self.removeReadOnly(torrent_files) - - release_downloads.append({ - 'id': torrent[0], - 'name': torrent[2], - 'status': status, - 'seed_ratio': float(torrent[7]) / 1000, - 'original_status': torrent[1], - 'timeleft': str(timedelta(seconds = torrent[10])), - 'folder': sp(torrent[26]), - 'files': '|'.join(torrent_files) - }) + #Get files of the torrent + torrent_files = [] + try: + torrent_files = json.loads(self.utorrent_api.get_files(torrent[0])) + torrent_files = [sp(os.path.join(torrent[26], torrent_file[0])) for torrent_file in torrent_files['files'][1]] + except: + log.debug('Failed getting files from torrent: %s', torrent[2]) + + status_flags = { + "STARTED" : 1, + "CHECKING" : 2, + "CHECK-START" : 4, + "CHECKED" : 8, + "ERROR" : 16, + "PAUSED" : 32, + "QUEUED" : 64, + "LOADED" : 128 + } + + status = 'busy' + if (torrent[1] & status_flags["STARTED"] or torrent[1] & status_flags["QUEUED"]) and torrent[4] == 1000: + status = 'seeding' + elif (torrent[1] & status_flags["ERROR"]): + status = 'failed' + elif torrent[4] == 1000: + status = 'completed' + + if not status == 'busy': + self.removeReadOnly(torrent_files) + + release_downloads.append({ + 'id': torrent[0], + 'name': torrent[2], + 'status': status, + 'seed_ratio': float(torrent[7]) / 1000, + 'original_status': torrent[1], + 'timeleft': str(timedelta(seconds = torrent[10])), + 'folder': sp(torrent[26]), + 'files': '|'.join(torrent_files) + }) return release_downloads diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index bfebcc1f..bcc698d7 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -49,8 +49,21 @@ def ss(original, *args): return u_original.encode('UTF-8') def sp(path, *args): + # Standardise encoding, normalise case, path and strip trailing '/' or '\' - return os.path.normcase(os.path.normpath(ss(path, *args))).rstrip(os.path.sep) + if not path or len(path) == 0: + return path + + # convert windows path (from remote box) to *nix path + if os.path.sep == '/' and '\\' in path: + path = '/' + path.replace(':', '').replace('\\', '/') + + path = os.path.normcase(os.path.normpath(ss(path, *args))) + + if path != os.path.sep: + path = path.rstrip(os.path.sep) + + return path def ek(original, *args): if isinstance(original, (str, unicode)): diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 7d35b997..1cf1f484 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -11,6 +11,9 @@ import sys log = CPLog(__name__) +def fnEscape(pattern): + return pattern.replace('[','[[').replace(']','[]]').replace('[[','[[]') + def link(src, dst): if os.name == 'nt': import ctypes diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index e6a249d5..1ba83863 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -38,7 +38,7 @@ class MediaBase(Plugin): def notifyFront(): db = get_session() media = db.query(Media).filter_by(id = media_id).first() - fireEvent('notify.frontend', type = '%s.update.%s' % (media.type, media.id), data = media.to_dict(self.default_dict)) + fireEvent('notify.frontend', type = '%s.update' % media.type, data = media.to_dict(self.default_dict)) db.expire_all() return notifyFront diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 87afb82a..86d5a4bc 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -1,10 +1,15 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent -from couchpotato.core.helpers.variable import splitString +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import mergeDicts, splitString, getImdb from couchpotato.core.logger import CPLog from couchpotato.core.media import MediaBase -from couchpotato.core.settings.model import Media +from couchpotato.core.settings.model import Library, LibraryTitle, Release, \ + Media +from sqlalchemy.orm import joinedload_all +from sqlalchemy.sql.expression import or_, asc, not_, desc +from string import ascii_lowercase log = CPLog(__name__) @@ -20,7 +25,49 @@ class MediaPlugin(MediaBase): } }) - addEvent('app.load', self.addSingleRefresh) + addApiView('media.list', self.listView, docs = { + 'desc': 'List media', + 'params': { + 'type': {'type': 'string', 'desc': 'Media type to filter on.'}, + 'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'}, + 'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'}, + 'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'}, + 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'}, + 'search': {'desc': 'Search movie title'}, + }, + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'empty': bool, any movies returned or not, + 'media': array, media found, +}"""} + }) + + addApiView('media.get', self.getView, docs = { + 'desc': 'Get media by id', + 'params': { + 'id': {'desc': 'The id of the media'}, + } + }) + + addApiView('media.delete', self.deleteView, docs = { + 'desc': 'Delete a media from the wanted list', + 'params': { + 'id': {'desc': 'Media ID(s) you want to delete.', 'type': 'int (comma separated)'}, + 'delete_from': {'desc': 'Delete media from this page', 'type': 'string: all (default), wanted, manage'}, + } + }) + + addApiView('media.available_chars', self.charView) + + addEvent('app.load', self.addSingleRefreshView) + addEvent('app.load', self.addSingleListView) + addEvent('app.load', self.addSingleCharView) + addEvent('app.load', self.addSingleDeleteView) + + addEvent('media.get', self.get) + addEvent('media.list', self.list) + addEvent('media.delete', self.delete) + addEvent('media.restatus', self.restatus) def refresh(self, id = '', **kwargs): db = get_session() @@ -34,7 +81,7 @@ class MediaPlugin(MediaBase): for title in media.library.titles: if title.default: default_title = title.title - fireEvent('notify.frontend', type = '%s.busy.%s' % (media.type, x), data = True) + fireEvent('notify.frontend', type = '%s.busy' % media.type, data = {'id': x}) fireEventAsync('library.update.%s' % media.type, identifier = media.library.identifier, default_title = default_title, force = True, on_complete = self.createOnComplete(x)) db.expire_all() @@ -43,7 +90,369 @@ class MediaPlugin(MediaBase): 'success': True, } - def addSingleRefresh(self): + def addSingleRefreshView(self): for media_type in fireEvent('media.types', merge = True): addApiView('%s.refresh' % media_type, self.refresh) + + def get(self, media_id): + + db = get_session() + + imdb_id = getImdb(str(media_id)) + + if imdb_id: + m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first() + else: + m = db.query(Media).filter_by(id = media_id).first() + + results = None + if m: + results = m.to_dict(self.default_dict) + + db.expire_all() + return results + + def getView(self, id = None, **kwargs): + + media = self.get(id) if id else None + + return { + 'success': media is not None, + 'media': media, + } + + def list(self, types = None, status = None, release_status = None, limit_offset = None, starts_with = None, search = None, order = None): + + db = get_session() + + # Make a list from string + if status and not isinstance(status, (list, tuple)): + status = [status] + if release_status and not isinstance(release_status, (list, tuple)): + release_status = [release_status] + if types and not isinstance(types, (list, tuple)): + types = [types] + + # query movie ids + q = db.query(Media) \ + .with_entities(Media.id) \ + .group_by(Media.id) + + # Filter on movie status + if status and len(status) > 0: + statuses = fireEvent('status.get', status, single = len(status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.filter(Media.status_id.in_(statuses)) + + # Filter on release status + if release_status and len(release_status) > 0: + q = q.join(Media.releases) + + statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.filter(Release.status_id.in_(statuses)) + + # Filter on type + if types and len(types) > 0: + try: q = q.filter(Media.type.in_(types)) + except: pass + + # Only join when searching / ordering + if starts_with or search or order != 'release_order': + q = q.join(Media.library, Library.titles) \ + .filter(LibraryTitle.default == True) + + # Add search filters + filter_or = [] + if starts_with: + starts_with = toUnicode(starts_with.lower()) + if starts_with in ascii_lowercase: + filter_or.append(LibraryTitle.simple_title.startswith(starts_with)) + else: + ignore = [] + for letter in ascii_lowercase: + ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter))) + filter_or.append(not_(or_(*ignore))) + + if search: + filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%')) + + if len(filter_or) > 0: + q = q.filter(or_(*filter_or)) + + total_count = q.count() + if total_count == 0: + return 0, [] + + if order == 'release_order': + q = q.order_by(desc(Release.last_edit)) + else: + q = q.order_by(asc(LibraryTitle.simple_title)) + + if limit_offset: + splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset + limit = splt[0] + offset = 0 if len(splt) is 1 else splt[1] + q = q.limit(limit).offset(offset) + + # Get all media_ids in sorted order + media_ids = [m.id for m in q.all()] + + # List release statuses + releases = db.query(Release) \ + .filter(Release.movie_id.in_(media_ids)) \ + .all() + + release_statuses = dict((m, set()) for m in media_ids) + releases_count = dict((m, 0) for m in media_ids) + for release in releases: + release_statuses[release.movie_id].add('%d,%d' % (release.status_id, release.quality_id)) + releases_count[release.movie_id] += 1 + + # Get main movie data + q2 = db.query(Media) \ + .options(joinedload_all('library.titles')) \ + .options(joinedload_all('library.files')) \ + .options(joinedload_all('status')) \ + .options(joinedload_all('files')) + + q2 = q2.filter(Media.id.in_(media_ids)) + + results = q2.all() + + # Create dict by movie id + movie_dict = {} + for movie in results: + movie_dict[movie.id] = movie + + # List movies based on media_ids order + movies = [] + for media_id in media_ids: + + releases = [] + for r in release_statuses.get(media_id): + x = splitString(r) + releases.append({'status_id': x[0], 'quality_id': x[1]}) + + # Merge releases with movie dict + movies.append(mergeDicts(movie_dict[media_id].to_dict({ + 'library': {'titles': {}, 'files':{}}, + 'files': {}, + }), { + 'releases': releases, + 'releases_count': releases_count.get(media_id), + })) + + db.expire_all() + return total_count, movies + + def listView(self, **kwargs): + + types = splitString(kwargs.get('types')) + status = splitString(kwargs.get('status')) + release_status = splitString(kwargs.get('release_status')) + limit_offset = kwargs.get('limit_offset') + starts_with = kwargs.get('starts_with') + search = kwargs.get('search') + order = kwargs.get('order') + + total_movies, movies = self.list( + types = types, + status = status, + release_status = release_status, + limit_offset = limit_offset, + starts_with = starts_with, + search = search, + order = order + ) + + return { + 'success': True, + 'empty': len(movies) == 0, + 'total': total_movies, + 'movies': movies, + } + + def addSingleListView(self): + + for media_type in fireEvent('media.types', merge = True): + def tempList(*args, **kwargs): + return self.listView(types = media_type, *args, **kwargs) + addApiView('%s.list' % media_type, tempList) + + def availableChars(self, types = None, status = None, release_status = None): + + types = types or [] + status = status or [] + release_status = release_status or [] + + db = get_session() + + # Make a list from string + if not isinstance(status, (list, tuple)): + status = [status] + if release_status and not isinstance(release_status, (list, tuple)): + release_status = [release_status] + if types and not isinstance(types, (list, tuple)): + types = [types] + + q = db.query(Media) + + # Filter on movie status + if status and len(status) > 0: + statuses = fireEvent('status.get', status, single = len(release_status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.filter(Media.status_id.in_(statuses)) + + # Filter on release status + if release_status and len(release_status) > 0: + + statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) + statuses = [s.get('id') for s in statuses] + + q = q.join(Media.releases) \ + .filter(Release.status_id.in_(statuses)) + + # Filter on type + if types and len(types) > 0: + try: q = q.filter(Media.type.in_(types)) + except: pass + + q = q.join(Library, LibraryTitle) \ + .with_entities(LibraryTitle.simple_title) \ + .filter(LibraryTitle.default == True) + + titles = q.all() + + chars = set() + for title in titles: + try: + char = title[0][0] + char = char if char in ascii_lowercase else '#' + chars.add(str(char)) + except: + log.error('Failed getting title for %s', title.libraries_id) + + if len(chars) == 25: + break + + db.expire_all() + return ''.join(sorted(chars)) + + def charView(self, **kwargs): + + type = splitString(kwargs.get('type', 'movie')) + status = splitString(kwargs.get('status', None)) + release_status = splitString(kwargs.get('release_status', None)) + chars = self.availableChars(type, status, release_status) + + return { + 'success': True, + 'empty': len(chars) == 0, + 'chars': chars, + } + + def addSingleCharView(self): + + for media_type in fireEvent('media.types', merge = True): + def tempChar(*args, **kwargs): + return self.charView(types = media_type, *args, **kwargs) + addApiView('%s.available_chars' % media_type, tempChar) + + def delete(self, media_id, delete_from = None): + + db = get_session() + + media = db.query(Media).filter_by(id = media_id).first() + if media: + deleted = False + if delete_from == 'all': + db.delete(media) + db.commit() + deleted = True + else: + done_status = fireEvent('status.get', 'done', single = True) + + total_releases = len(media.releases) + total_deleted = 0 + new_movie_status = None + for release in media.releases: + if delete_from in ['wanted', 'snatched', 'late']: + if release.status_id != done_status.get('id'): + db.delete(release) + total_deleted += 1 + new_movie_status = 'done' + elif delete_from == 'manage': + if release.status_id == done_status.get('id'): + db.delete(release) + total_deleted += 1 + new_movie_status = 'active' + db.commit() + + if total_releases == total_deleted: + db.delete(media) + db.commit() + deleted = True + elif new_movie_status: + new_status = fireEvent('status.get', new_movie_status, single = True) + media.profile_id = None + media.status_id = new_status.get('id') + db.commit() + else: + fireEvent('media.restatus', media.id, single = True) + + if deleted: + fireEvent('notify.frontend', type = 'movie.deleted', data = media.to_dict()) + + db.expire_all() + return True + + def deleteView(self, id = '', **kwargs): + + ids = splitString(id) + for media_id in ids: + self.delete(media_id, delete_from = kwargs.get('delete_from', 'all')) + + return { + 'success': True, + } + + def addSingleDeleteView(self): + + for media_type in fireEvent('media.types', merge = True): + def tempDelete(*args, **kwargs): + return self.deleteView(types = media_type, *args, **kwargs) + addApiView('%s.delete' % media_type, tempDelete) + + def restatus(self, media_id): + + active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True) + + db = get_session() + + m = db.query(Media).filter_by(id = media_id).first() + if not m or len(m.library.titles) == 0: + log.debug('Can\'t restatus movie, doesn\'t seem to exist.') + return False + + log.debug('Changing status for %s', m.library.titles[0].title) + if not m.profile: + m.status_id = done_status.get('id') + else: + move_to_wanted = True + + for t in m.profile.types: + for release in m.releases: + if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): + move_to_wanted = False + + m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') + + db.commit() + + return True + diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index fdf09309..3c73eb27 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -147,7 +147,7 @@ class Searcher(SearcherBase): except: pass # Match longest name between [] - try: check_names.append(max(check_name.split('['), key = len)) + try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', check_name), key = len).strip()) except: pass for check_name in list(set(check_names)): diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index 817b0a38..20663b59 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -2,15 +2,10 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import getImdb, splitString, tryInt, \ - mergeDicts +from couchpotato.core.helpers.variable import splitString, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media.movie import MovieTypeBase -from couchpotato.core.settings.model import Library, LibraryTitle, Media, \ - Release -from sqlalchemy.orm import joinedload_all -from sqlalchemy.sql.expression import or_, asc, not_, desc -from string import ascii_lowercase +from couchpotato.core.settings.model import Media import time log = CPLog(__name__) @@ -26,33 +21,12 @@ class MovieBase(MovieTypeBase): super(MovieBase, self).__init__() self.initType() - addApiView('movie.list', self.listView, docs = { - 'desc': 'List movies in wanted list', - 'params': { - 'status': {'type': 'array or csv', 'desc': 'Filter movie by status. Example:"active,done"'}, - 'release_status': {'type': 'array or csv', 'desc': 'Filter movie by status of its releases. Example:"snatched,available"'}, - 'limit_offset': {'desc': 'Limit and offset the movie list. Examples: "50" or "50,30"'}, - 'starts_with': {'desc': 'Starts with these characters. Example: "a" returns all movies starting with the letter "a"'}, - 'search': {'desc': 'Search movie title'}, - }, - 'return': {'type': 'object', 'example': """{ - 'success': True, - 'empty': bool, any movies returned or not, - 'movies': array, movies found, -}"""} - }) - addApiView('movie.get', self.getView, docs = { - 'desc': 'Get a movie by id', - 'params': { - 'id': {'desc': 'The id of the movie'}, - } - }) - addApiView('movie.available_chars', self.charView) addApiView('movie.add', self.addView, docs = { 'desc': 'Add new movie to the wanted list', 'params': { 'identifier': {'desc': 'IMDB id of the movie your want to add.'}, 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'}, + 'category_id': {'desc': 'ID of category you want the add the movie in. If empty will use no category.'}, 'title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'}, } }) @@ -64,255 +38,8 @@ class MovieBase(MovieTypeBase): 'default_title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'}, } }) - addApiView('movie.delete', self.deleteView, docs = { - 'desc': 'Delete a movie from the wanted list', - 'params': { - 'id': {'desc': 'Movie ID(s) you want to delete.', 'type': 'int (comma separated)'}, - 'delete_from': {'desc': 'Delete movie from this page', 'type': 'string: all (default), wanted, manage'}, - } - }) addEvent('movie.add', self.add) - addEvent('movie.delete', self.delete) - addEvent('movie.get', self.get) - addEvent('movie.list', self.list) - addEvent('movie.restatus', self.restatus) - - def getView(self, id = None, **kwargs): - - movie = self.get(id) if id else None - - return { - 'success': movie is not None, - 'movie': movie, - } - - def get(self, movie_id): - - db = get_session() - - imdb_id = getImdb(str(movie_id)) - - if imdb_id: - m = db.query(Media).filter(Media.library.has(identifier = imdb_id)).first() - else: - m = db.query(Media).filter_by(id = movie_id).first() - - results = None - if m: - results = m.to_dict(self.default_dict) - - db.expire_all() - return results - - def list(self, status = None, release_status = None, limit_offset = None, starts_with = None, search = None, order = None): - - db = get_session() - - # Make a list from string - if status and not isinstance(status, (list, tuple)): - status = [status] - if release_status and not isinstance(release_status, (list, tuple)): - release_status = [release_status] - - # query movie ids - q = db.query(Media) \ - .with_entities(Media.id) \ - .group_by(Media.id) - - # Filter on movie status - if status and len(status) > 0: - statuses = fireEvent('status.get', status, single = len(status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.filter(Media.status_id.in_(statuses)) - - # Filter on release status - if release_status and len(release_status) > 0: - q = q.join(Media.releases) - - statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.filter(Release.status_id.in_(statuses)) - - # Only join when searching / ordering - if starts_with or search or order != 'release_order': - q = q.join(Media.library, Library.titles) \ - .filter(LibraryTitle.default == True) - - # Add search filters - filter_or = [] - if starts_with: - starts_with = toUnicode(starts_with.lower()) - if starts_with in ascii_lowercase: - filter_or.append(LibraryTitle.simple_title.startswith(starts_with)) - else: - ignore = [] - for letter in ascii_lowercase: - ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter))) - filter_or.append(not_(or_(*ignore))) - - if search: - filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%')) - - if len(filter_or) > 0: - q = q.filter(or_(*filter_or)) - - total_count = q.count() - if total_count == 0: - return 0, [] - - if order == 'release_order': - q = q.order_by(desc(Release.last_edit)) - else: - q = q.order_by(asc(LibraryTitle.simple_title)) - - if limit_offset: - splt = splitString(limit_offset) if isinstance(limit_offset, (str, unicode)) else limit_offset - limit = splt[0] - offset = 0 if len(splt) is 1 else splt[1] - q = q.limit(limit).offset(offset) - - # Get all movie_ids in sorted order - movie_ids = [m.id for m in q.all()] - - # List release statuses - releases = db.query(Release) \ - .filter(Release.movie_id.in_(movie_ids)) \ - .all() - - release_statuses = dict((m, set()) for m in movie_ids) - releases_count = dict((m, 0) for m in movie_ids) - for release in releases: - release_statuses[release.movie_id].add('%d,%d' % (release.status_id, release.quality_id)) - releases_count[release.movie_id] += 1 - - # Get main movie data - q2 = db.query(Media) \ - .options(joinedload_all('library.titles')) \ - .options(joinedload_all('library.files')) \ - .options(joinedload_all('status')) \ - .options(joinedload_all('files')) - - q2 = q2.filter(Media.id.in_(movie_ids)) - - results = q2.all() - - # Create dict by movie id - movie_dict = {} - for movie in results: - movie_dict[movie.id] = movie - - # List movies based on movie_ids order - movies = [] - for movie_id in movie_ids: - - releases = [] - for r in release_statuses.get(movie_id): - x = splitString(r) - releases.append({'status_id': x[0], 'quality_id': x[1]}) - - # Merge releases with movie dict - movies.append(mergeDicts(movie_dict[movie_id].to_dict({ - 'library': {'titles': {}, 'files':{}}, - 'files': {}, - }), { - 'releases': releases, - 'releases_count': releases_count.get(movie_id), - })) - - db.expire_all() - return total_count, movies - - def availableChars(self, status = None, release_status = None): - - status = status or [] - release_status = release_status or [] - - db = get_session() - - # Make a list from string - if not isinstance(status, (list, tuple)): - status = [status] - if release_status and not isinstance(release_status, (list, tuple)): - release_status = [release_status] - - q = db.query(Media) - - # Filter on movie status - if status and len(status) > 0: - statuses = fireEvent('status.get', status, single = len(release_status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.filter(Media.status_id.in_(statuses)) - - # Filter on release status - if release_status and len(release_status) > 0: - - statuses = fireEvent('status.get', release_status, single = len(release_status) > 1) - statuses = [s.get('id') for s in statuses] - - q = q.join(Media.releases) \ - .filter(Release.status_id.in_(statuses)) - - q = q.join(Library, LibraryTitle) \ - .with_entities(LibraryTitle.simple_title) \ - .filter(LibraryTitle.default == True) - - titles = q.all() - - chars = set() - for title in titles: - try: - char = title[0][0] - char = char if char in ascii_lowercase else '#' - chars.add(str(char)) - except: - log.error('Failed getting title for %s', title.libraries_id) - - if len(chars) == 25: - break - - db.expire_all() - return ''.join(sorted(chars)) - - def listView(self, **kwargs): - - status = splitString(kwargs.get('status')) - release_status = splitString(kwargs.get('release_status')) - limit_offset = kwargs.get('limit_offset') - starts_with = kwargs.get('starts_with') - search = kwargs.get('search') - order = kwargs.get('order') - - total_movies, movies = self.list( - status = status, - release_status = release_status, - limit_offset = limit_offset, - starts_with = starts_with, - search = search, - order = order - ) - - return { - 'success': True, - 'empty': len(movies) == 0, - 'total': total_movies, - 'movies': movies, - } - - def charView(self, **kwargs): - - status = splitString(kwargs.get('status', None)) - release_status = splitString(kwargs.get('release_status', None)) - chars = self.availableChars(status, release_status) - - return { - 'success': True, - 'empty': len(chars) == 0, - 'chars': chars, - } def add(self, params = None, force_readd = True, search_after = True, update_library = False, status_id = None): if not params: params = {} @@ -421,9 +148,9 @@ class MovieBase(MovieTypeBase): available_status = fireEvent('status.get', 'available', single = True) ids = splitString(id) - for movie_id in ids: + for media_id in ids: - m = db.query(Media).filter_by(id = movie_id).first() + m = db.query(Media).filter_by(id = media_id).first() if not m: continue @@ -446,98 +173,12 @@ class MovieBase(MovieTypeBase): db.commit() - fireEvent('movie.restatus', m.id) + fireEvent('media.restatus', m.id) movie_dict = m.to_dict(self.default_dict) - fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id)) + fireEventAsync('movie.searcher.single', movie_dict, on_complete = self.createNotifyFront(media_id)) db.expire_all() return { 'success': True, } - - def deleteView(self, id = '', **kwargs): - - ids = splitString(id) - for movie_id in ids: - self.delete(movie_id, delete_from = kwargs.get('delete_from', 'all')) - - return { - 'success': True, - } - - def delete(self, movie_id, delete_from = None): - - db = get_session() - - movie = db.query(Media).filter_by(id = movie_id).first() - if movie: - deleted = False - if delete_from == 'all': - db.delete(movie) - db.commit() - deleted = True - else: - done_status = fireEvent('status.get', 'done', single = True) - - total_releases = len(movie.releases) - total_deleted = 0 - new_movie_status = None - for release in movie.releases: - if delete_from in ['wanted', 'snatched', 'late']: - if release.status_id != done_status.get('id'): - db.delete(release) - total_deleted += 1 - new_movie_status = 'done' - elif delete_from == 'manage': - if release.status_id == done_status.get('id'): - db.delete(release) - total_deleted += 1 - new_movie_status = 'active' - db.commit() - - if total_releases == total_deleted: - db.delete(movie) - db.commit() - deleted = True - elif new_movie_status: - new_status = fireEvent('status.get', new_movie_status, single = True) - movie.profile_id = None - movie.status_id = new_status.get('id') - db.commit() - else: - fireEvent('movie.restatus', movie.id, single = True) - - if deleted: - fireEvent('notify.frontend', type = 'movie.deleted', data = movie.to_dict()) - - db.expire_all() - return True - - def restatus(self, movie_id): - - active_status, done_status = fireEvent('status.get', ['active', 'done'], single = True) - - db = get_session() - - m = db.query(Media).filter_by(id = movie_id).first() - if not m or len(m.library.titles) == 0: - log.debug('Can\'t restatus movie, doesn\'t seem to exist.') - return False - - log.debug('Changing status for %s', m.library.titles[0].title) - if not m.profile: - m.status_id = done_status.get('id') - else: - move_to_wanted = True - - for t in m.profile.types: - for release in m.releases: - if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): - move_to_wanted = False - - m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') - - db.commit() - - return True diff --git a/couchpotato/core/media/movie/_base/static/list.js b/couchpotato/core/media/movie/_base/static/list.js index aaa8be12..85dee2e5 100644 --- a/couchpotato/core/media/movie/_base/static/list.js +++ b/couchpotato/core/media/movie/_base/static/list.js @@ -52,8 +52,8 @@ var MovieList = new Class({ self.getMovies(); - App.addEvent('movie.added', self.movieAdded.bind(self)) - App.addEvent('movie.deleted', self.movieDeleted.bind(self)) + App.on('movie.added', self.movieAdded.bind(self)) + App.on('movie.deleted', self.movieDeleted.bind(self)) }, movieDeleted: function(notification){ @@ -65,6 +65,7 @@ var MovieList = new Class({ movie.destroy(); delete self.movies_added[notification.data.id]; self.setCounter(self.counter_count-1); + self.total_movies--; } }) } @@ -75,6 +76,7 @@ var MovieList = new Class({ movieAdded: function(notification){ var self = this; + self.fireEvent('movieAdded', notification); if(self.options.add_new && !self.movies_added[notification.data.id] && notification.data.status.identifier == self.options.status){ window.scroll(0,0); self.createMovie(notification.data, 'top'); @@ -279,7 +281,7 @@ var MovieList = new Class({ // Get available chars and highlight if(!available_chars && (self.navigation.isDisplayed() || self.navigation.isVisible())) - Api.request('movie.available_chars', { + Api.request('media.available_chars', { 'data': Object.merge({ 'status': self.options.status }, self.filter), @@ -370,7 +372,7 @@ var MovieList = new Class({ 'click': function(e){ (e).preventDefault(); this.set('text', 'Deleting..') - Api.request('movie.delete', { + Api.request('media.delete', { 'data': { 'id': ids.join(','), 'delete_from': self.options.identifier @@ -390,6 +392,7 @@ var MovieList = new Class({ self.movies.erase(movie); movie.destroy(); self.setCounter(self.counter_count-1); + self.total_movies--; }); self.calculateSelected(); @@ -547,8 +550,9 @@ var MovieList = new Class({ } - Api.request(self.options.api_call || 'movie.list', { + Api.request(self.options.api_call || 'media.list', { 'data': Object.merge({ + 'type': 'movie', 'status': self.options.status, 'limit_offset': self.options.limit ? self.options.limit + ',' + self.offset : null }, self.filter), diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index f6e0f542..66c84c68 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -126,7 +126,9 @@ MA.Release = new Class({ else self.showHelper(); - App.addEvent('movie.searcher.ended.'+self.movie.data.id, function(notification){ + App.on('movie.searcher.ended', function(notification){ + if(self.movie.data.id != notification.data.id) return; + self.releases = null; if(self.options_container){ self.options_container.destroy(); @@ -250,12 +252,14 @@ MA.Release = new Class({ else if(!self.next_release && status.identifier == 'available'){ self.next_release = release; } - + var update_handle = function(notification) { - var q = self.movie.quality.getElement('.q_id' + release.quality_id), + if(notification.data.id != release.id) return; + + var q = self.movie.quality.getElement('.q_id' + release.quality_id), status = Status.get(release.status_id), - new_status = Status.get(notification.data); - + new_status = Status.get(notification.data.status_id); + release.status_id = new_status.id release.el.set('class', 'item ' + new_status.identifier); @@ -272,7 +276,7 @@ MA.Release = new Class({ } } - App.addEvent('release.update_status.' + release.id, update_handle); + App.on('release.update_status', update_handle); }); @@ -285,7 +289,7 @@ MA.Release = new Class({ if(self.next_release || (self.last_release && ['ignored', 'failed'].indexOf(self.last_release.status.identifier) === false)){ self.trynext_container = new Element('div.buttons.try_container').inject(self.release_container, 'top'); - + var nr = self.next_release, lr = self.last_release; @@ -427,7 +431,7 @@ MA.Release = new Class({ markMovieDone: function(){ var self = this; - Api.request('movie.delete', { + Api.request('media.delete', { 'data': { 'id': self.movie.get('id'), 'delete_from': 'wanted' @@ -446,7 +450,7 @@ MA.Release = new Class({ }, - tryNextRelease: function(movie_id){ + tryNextRelease: function(){ var self = this; Api.request('movie.searcher.try_next', { @@ -817,7 +821,7 @@ MA.Delete = new Class({ self.callChain(); }, function(){ - Api.request('movie.delete', { + Api.request('media.delete', { 'data': { 'id': self.movie.get('id'), 'delete_from': self.movie.list.options.identifier diff --git a/couchpotato/core/media/movie/_base/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css index c013bd80..a88a2077 100644 --- a/couchpotato/core/media/movie/_base/static/movie.css +++ b/couchpotato/core/media/movie/_base/static/movie.css @@ -1036,7 +1036,7 @@ text-overflow: ellipsis; overflow: hidden; width: 85%; - direction: rtl; + direction: ltr; vertical-align: middle; } diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index a865325b..9c0ae015 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -23,23 +23,49 @@ var Movie = new Class({ addEvents: function(){ var self = this; - App.addEvent('movie.update.'+self.data.id, function(notification){ + self.global_events = {} + + // Do refresh with new data + self.global_events['movie.update'] = function(notification){ + if(self.data.id != notification.data.id) return; + self.busy(false); self.removeView(); self.update.delay(2000, self, notification); - }); + } + App.on('movie.update', self.global_events['movie.update']); + // Add spinner on load / search ['movie.busy', 'movie.searcher.started'].each(function(listener){ - App.addEvent(listener+'.'+self.data.id, function(notification){ - if(notification.data) + self.global_events[listener] = function(notification){ + if(notification.data && self.data.id == notification.data.id) self.busy(true) - }); + } + App.on(listener, self.global_events[listener]); }) - App.addEvent('movie.searcher.ended.'+self.data.id, function(notification){ - if(notification.data) + // Remove spinner + self.global_events['movie.searcher.ended'] = function(notification){ + if(notification.data && self.data.id == notification.data.id) self.busy(false) - }); + } + App.on('movie.searcher.ended', self.global_events['movie.searcher.ended']); + + // Reload when releases have updated + self.global_events['release.update_status'] = function(notification){ + var data = notification.data + if(data && self.data.id == data.movie_id){ + + if(!self.data.releases) + self.data.releases = []; + + self.data.releases.push({'quality_id': data.quality_id, 'status_id': data.status_id}); + self.updateReleases(); + } + } + + App.on('release.update_status', self.global_events['release.update_status']); + }, destroy: function(){ @@ -52,10 +78,9 @@ var Movie = new Class({ self.list.checkIfEmpty(); // Remove events - App.removeEvents('movie.update.'+self.data.id); - ['movie.busy', 'movie.searcher.started'].each(function(listener){ - App.removeEvents(listener+'.'+self.data.id); - }) + Object.each(self.global_events, function(handle, listener){ + App.off(listener, handle); + }); }, busy: function(set_busy, timeout){ @@ -179,21 +204,7 @@ var Movie = new Class({ }); // Add releases - if(self.data.releases) - self.data.releases.each(function(release){ - - var q = self.quality.getElement('.q_id'+ release.quality_id), - status = Status.get(release.status_id); - - if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done')) - var q = self.addQuality(release.quality_id) - - if (status && q && !q.hasClass(status.identifier)){ - q.addClass(status.identifier); - q.set('title', (q.get('title') ? q.get('title') : '') + ' status: '+ status.label) - } - - }); + self.updateReleases(); Object.each(self.options.actions, function(action, key){ self.action[key.toLowerCase()] = action = new self.options.actions[key](self) @@ -203,6 +214,26 @@ var Movie = new Class({ }, + updateReleases: function(){ + var self = this; + if(!self.data.releases || self.data.releases.length == 0) return; + + self.data.releases.each(function(release){ + + var q = self.quality.getElement('.q_id'+ release.quality_id), + status = Status.get(release.status_id); + + if(!q && (status.identifier == 'snatched' || status.identifier == 'seeding' || status.identifier == 'done')) + var q = self.addQuality(release.quality_id) + + if (status && q && !q.hasClass(status.identifier)){ + q.addClass(status.identifier); + q.set('title', (q.get('title') ? q.get('title') : '') + ' status: '+ status.label) + } + + }); + }, + addQuality: function(quality_id){ var self = this; diff --git a/couchpotato/core/media/movie/library/movie/main.py b/couchpotato/core/media/movie/library/movie/main.py index 718e7390..b0d05202 100644 --- a/couchpotato/core/media/movie/library/movie/main.py +++ b/couchpotato/core/media/movie/library/movie/main.py @@ -151,7 +151,7 @@ class MovieLibraryPlugin(LibraryBase): else: dates = library.info.get('release_date') - if dates and dates.get('expires', 0) < time.time() or not dates: + if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: dates = fireEvent('movie.release_date', identifier = identifier, merge = True) library.info.update({'release_date': dates }) db.commit() diff --git a/couchpotato/core/media/movie/searcher/main.py b/couchpotato/core/media/movie/searcher/main.py index 93441c59..d90f9598 100644 --- a/couchpotato/core/media/movie/searcher/main.py +++ b/couchpotato/core/media/movie/searcher/main.py @@ -145,10 +145,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): default_title = getTitle(movie['library']) if not default_title: log.error('No proper info found for movie, removing it from library to cause it from having more issues.') - fireEvent('movie.delete', movie['id'], single = True) + fireEvent('media.delete', movie['id'], single = True) return - fireEvent('notify.frontend', type = 'movie.searcher.started.%s' % movie['id'], data = True, message = 'Searching for "%s"' % default_title) + fireEvent('notify.frontend', type = 'movie.searcher.started', data = {'id': movie['id']}, message = 'Searching for "%s"' % default_title) ret = False @@ -192,7 +192,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): else: log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title)) - fireEvent('movie.restatus', movie['id']) + fireEvent('media.restatus', movie['id']) break # Break if CP wants to shut down @@ -202,7 +202,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): if len(too_early_to_search) > 0: log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) - fireEvent('notify.frontend', type = 'movie.searcher.ended.%s' % movie['id'], data = True) + fireEvent('notify.frontend', type = 'movie.searcher.ended', data = {'id': movie['id']}) return ret @@ -318,14 +318,14 @@ class MovieSearcher(SearcherBase, MovieTypeBase): 'success': trynext } - def tryNextRelease(self, movie_id, manual = False): + def tryNextRelease(self, media_id, manual = False): snatched_status, done_status, ignored_status = fireEvent('status.get', ['snatched', 'done', 'ignored'], single = True) try: db = get_session() rels = db.query(Release) \ - .filter_by(movie_id = movie_id) \ + .filter_by(movie_id = media_id) \ .filter(Release.status_id.in_([snatched_status.get('id'), done_status.get('id')])) \ .all() @@ -333,7 +333,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): rel.status_id = ignored_status.get('id') db.commit() - movie_dict = fireEvent('movie.get', movie_id, single = True) + movie_dict = fireEvent('media.get', media_id = media_id, single = True) log.info('Trying next release for: %s', getTitle(movie_dict['library'])) fireEvent('movie.searcher.single', movie_dict, manual = manual) diff --git a/couchpotato/core/notifications/base.py b/couchpotato/core/notifications/base.py index 4c0d0992..63d2075e 100644 --- a/couchpotato/core/notifications/base.py +++ b/couchpotato/core/notifications/base.py @@ -17,7 +17,7 @@ class Notification(Provider): listen_to = [ 'renamer.after', 'movie.snatched', 'updater.available', 'updater.updated', - 'core.message', + 'core.message.important', ] dont_listen_to = [] diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index 04acf284..cd63c2cb 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -21,6 +21,12 @@ class CoreNotifier(Notification): m_lock = None + listen_to = [ + 'renamer.after', 'movie.snatched', + 'updater.available', 'updater.updated', + 'core.message', 'core.message.important', + ] + def __init__(self): super(CoreNotifier, self).__init__() @@ -121,7 +127,10 @@ class CoreNotifier(Notification): for message in messages: if message.get('time') > last_check: - fireEvent('core.message', message = message.get('message'), data = message) + message['sticky'] = True # Always sticky core messages + + message_type = 'core.message.important' if message.get('important') else 'core.message' + fireEvent(message_type, message = message.get('message'), data = message) if last_check < message.get('time'): last_check = message.get('time') diff --git a/couchpotato/core/notifications/core/static/notification.js b/couchpotato/core/notifications/core/static/notification.js index e485976e..a0c3b15c 100644 --- a/couchpotato/core/notifications/core/static/notification.js +++ b/couchpotato/core/notifications/core/static/notification.js @@ -10,8 +10,8 @@ var NotificationBase = new Class({ // Listener App.addEvent('unload', self.stopPoll.bind(self)); App.addEvent('reload', self.startInterval.bind(self, [true])); - App.addEvent('notification', self.notify.bind(self)); - App.addEvent('message', self.showMessage.bind(self)); + App.on('notification', self.notify.bind(self)); + App.on('message', self.showMessage.bind(self)); // Add test buttons to settings page App.addEvent('load', self.addTestButtons.bind(self)); @@ -50,9 +50,9 @@ var NotificationBase = new Class({ , 'top'); self.notifications.include(result); - if(result.data.important !== undefined && !result.read){ + if((result.data.important !== undefined || result.data.sticky !== undefined) && !result.read){ var sticky = true - App.fireEvent('message', [result.message, sticky, result]) + App.trigger('message', [result.message, sticky, result]) } else if(!result.read){ self.setBadge(self.notifications.filter(function(n){ return !n.read}).length) @@ -147,7 +147,7 @@ var NotificationBase = new Class({ // Process data if(json){ Array.each(json.result, function(result){ - App.fireEvent(result.type, result); + App.trigger(result.type, result); if(result.message && result.read === undefined) self.showMessage(result.message); }) diff --git a/couchpotato/core/notifications/email/main.py b/couchpotato/core/notifications/email/main.py index c67ac97d..41a4323b 100644 --- a/couchpotato/core/notifications/email/main.py +++ b/couchpotato/core/notifications/email/main.py @@ -4,6 +4,7 @@ from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from couchpotato.environment import Env from email.mime.text import MIMEText +from email.utils import formatdate, make_msgid import smtplib import traceback @@ -30,6 +31,8 @@ class Email(Notification): message['Subject'] = self.default_title message['From'] = from_address message['To'] = to_address + message['Date'] = formatdate(localtime = 1) + message['Message-ID'] = make_msgid() try: # Open the SMTP connection, via SSL if requested diff --git a/couchpotato/core/notifications/pushbullet/__init__.py b/couchpotato/core/notifications/pushbullet/__init__.py new file mode 100644 index 00000000..e61a44e3 --- /dev/null +++ b/couchpotato/core/notifications/pushbullet/__init__.py @@ -0,0 +1,39 @@ +from .main import Pushbullet + +def start(): + return Pushbullet() + +config = [{ + 'name': 'pushbullet', + 'groups': [ + { + 'tab': 'notifications', + 'list': 'notification_providers', + 'name': 'pushbullet', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'api_key', + 'label': 'User API Key' + }, + { + 'name': 'devices', + 'default': '', + 'advanced': True, + 'description': 'IDs of devices to send notifications to, empty = all devices' + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/notifications/pushbullet/main.py b/couchpotato/core/notifications/pushbullet/main.py new file mode 100644 index 00000000..2e6db29d --- /dev/null +++ b/couchpotato/core/notifications/pushbullet/main.py @@ -0,0 +1,86 @@ +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification +import base64 +import json + +log = CPLog(__name__) + + +class Pushbullet(Notification): + + url = 'https://api.pushbullet.com/api/%s' + + def notify(self, message = '', data = None, listener = None): + if not data: data = {} + + devices = self.getDevices() + if devices is None: + return False + + # Get all the device IDs linked to this user + if not len(devices): + response = self.request('devices') + if not response: + return False + + devices += [device.get('id') for device in response['devices']] + + successful = 0 + for device in devices: + response = self.request( + 'pushes', + cache = False, + device_id = device, + type = 'note', + title = self.default_title, + body = toUnicode(message) + ) + + if response: + successful += 1 + else: + log.error('Unable to push notification to Pushbullet device with ID %s' % device) + + return successful == len(devices) + + def getDevices(self): + devices = [d.strip() for d in self.conf('devices').split(',')] + + # Remove empty items + devices = [d for d in devices if len(d)] + + # Break on any ids that aren't integers + valid_devices = [] + + for device_id in devices: + d = tryInt(device_id, None) + + if not d: + log.error('Device ID "%s" is not valid', device_id) + return None + + valid_devices.append(d) + + return valid_devices + + def request(self, method, cache = True, **kwargs): + try: + base64string = base64.encodestring('%s:' % self.conf('api_key'))[:-1] + + headers = { + "Authorization": "Basic %s" % base64string + } + + if cache: + return self.getJsonData(self.url % method, headers = headers, params = kwargs) + else: + data = self.urlopen(self.url % method, headers = headers, params = kwargs) + return json.loads(data) + + except Exception, ex: + log.error('Pushbullet request failed') + log.debug(ex) + + return None diff --git a/couchpotato/core/notifications/xbmc/__init__.py b/couchpotato/core/notifications/xbmc/__init__.py index dafa0f63..04662e27 100644 --- a/couchpotato/core/notifications/xbmc/__init__.py +++ b/couchpotato/core/notifications/xbmc/__init__.py @@ -46,6 +46,14 @@ config = [{ 'advanced': True, 'description': 'Only scan new movie folder at remote XBMC servers. Works if movie location is the same.', }, + { + 'name': 'force_full_scan', + 'label': 'Always do a full scan', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Do a full scan instead of only the new movie. Useful if the XBMC path is different from the path CPS uses.', + }, { 'name': 'on_snatch', 'default': 0, diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index dc185c41..9dac4979 100755 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -36,7 +36,7 @@ class XBMC(Notification): if data and data.get('destination_dir') and (not self.conf('only_first') or hosts.index(host) == 0): param = {} - if self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0]): + if not self.conf('force_full_scan') and (self.conf('remote_dir_scan') or socket.getfqdn('localhost') == socket.getfqdn(host.split(':')[0])): param = {'directory': data['destination_dir']} calls.append(('VideoLibrary.Scan', param)) diff --git a/couchpotato/core/plugins/automation/__init__.py b/couchpotato/core/plugins/automation/__init__.py index 440232b2..a81719c4 100644 --- a/couchpotato/core/plugins/automation/__init__.py +++ b/couchpotato/core/plugins/automation/__init__.py @@ -41,7 +41,7 @@ config = [{ 'label': 'Required Genres', 'default': '', 'placeholder': 'Example: Action, Crime & Drama', - 'description': 'Ignore movies that don\'t contain at least one set of genres. Sets are separated by "," and each word within a set must be separated with "&"' + 'description': ('Ignore movies that don\'t contain at least one set of genres.', 'Sets are separated by "," and each word within a set must be separated with "&"') }, { 'name': 'ignored_genres', diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 92547cb0..2edcd3be 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -43,7 +43,7 @@ class Automation(Plugin): if self.shuttingDown(): break - movie_dict = fireEvent('movie.get', movie_id, single = True) + movie_dict = fireEvent('media.get', movie_id, single = True) fireEvent('movie.searcher.single', movie_dict) - return True \ No newline at end of file + return True diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 649e359d..2bc8fb9e 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -289,19 +289,19 @@ class Plugin(object): Env.get('cache').set(cache_key_md5, value, timeout) return value - def createNzbName(self, data, movie): - tag = self.cpTag(movie) + def createNzbName(self, data, media): + tag = self.cpTag(media) return '%s%s' % (toSafeString(toUnicode(data.get('name'))[:127 - len(tag)]), tag) - def createFileName(self, data, filedata, movie): - name = sp(os.path.join(self.createNzbName(data, movie))) + def createFileName(self, data, filedata, media): + name = sp(os.path.join(self.createNzbName(data, media))) if data.get('protocol') == 'nzb' and 'DOCTYPE nzb' not in filedata and '' not in filedata: return '%s.%s' % (name, 'rar') return '%s.%s' % (name, data.get('protocol')) - def cpTag(self, movie): + def cpTag(self, media): if Env.setting('enabled', 'renamer'): - return '.cp(' + movie['library'].get('identifier') + ')' if movie['library'].get('identifier') else '' + return '.cp(' + media['library'].get('identifier') + ')' if media['library'].get('identifier') else '' return '' diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index e8ccaf7e..a764f317 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -1,6 +1,6 @@ from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent, fireEventAsync -from couchpotato.core.helpers.encoding import ss +from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import splitString, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -79,6 +79,7 @@ class Manage(Plugin): try: directories = self.directories() + directories.sort() added_identifiers = [] # Add some progress @@ -111,22 +112,20 @@ class Manage(Plugin): if self.conf('cleanup') and full and not self.shuttingDown(): # Get movies with done status - total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True) + total_movies, done_movies = fireEvent('media.list', types = 'movie', status = 'done', single = True) for done_movie in done_movies: if done_movie['library']['identifier'] not in added_identifiers: - fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all') + fireEvent('media.delete', media_id = done_movie['id'], delete_from = 'all') else: releases = fireEvent('release.for_movie', id = done_movie.get('id'), single = True) for release in releases: - if len(release.get('files', [])) == 0: - fireEvent('release.delete', release['id']) - else: + if len(release.get('files', [])) > 0: for release_file in release.get('files', []): # Remove release not available anymore - if not os.path.isfile(ss(release_file['path'])): + if not os.path.isfile(sp(release_file['path'])): fireEvent('release.clean', release['id']) break @@ -201,7 +200,7 @@ class Manage(Plugin): self.in_progress[folder]['to_go'] -= 1 total = self.in_progress[folder]['total'] - movie_dict = fireEvent('movie.get', identifier, single = True) + movie_dict = fireEvent('media.get', identifier, single = True) fireEvent('notify.frontend', type = 'movie.added', data = movie_dict, message = None if total > 5 else 'Added "%s" to manage.' % getTitle(movie_dict['library'])) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 7992965f..3aaa77ea 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -2,7 +2,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent from couchpotato.core.helpers.encoding import toUnicode, ss -from couchpotato.core.helpers.variable import mergeDicts, md5, getExt +from couchpotato.core.helpers.variable import mergeDicts, getExt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Quality, Profile, ProfileType @@ -19,14 +19,14 @@ class QualityPlugin(Plugin): {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, {'identifier': '1080p', 'hd': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':['avi'], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r')]}, - {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg'], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, - {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p'], 'ext':['avi', 'mpg', 'mpeg'], 'tags': ['webrip', ('web', 'rip')]}, - {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, - {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']} + {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, + {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, + {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, + {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':[]}, + {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync', 'hdts'], 'allow': [], 'ext':[]}, + {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':[]} ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] @@ -50,6 +50,8 @@ class QualityPlugin(Plugin): addEvent('app.initialize', self.fill, priority = 10) + addEvent('app.test', self.doTest) + def preReleases(self): return self.pre_releases @@ -165,9 +167,10 @@ class QualityPlugin(Plugin): if not extra: extra = {} # Create hash for cache - cache_key = md5(str([f.replace('.' + getExt(f), '') for f in files])) + cache_key = str([f.replace('.' + getExt(f), '') if len(getExt(f)) < 4 else f for f in files]) cached = self.getCache(cache_key) - if cached and len(extra) == 0: return cached + if cached and len(extra) == 0: + return cached qualities = self.all() @@ -228,11 +231,6 @@ class QualityPlugin(Plugin): if len(set(words) & set(alt)) == len(alt): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) score += points.get(tag_type) - elif len(set(words) & set(alt)) > 0: - partial = list(set(words) & set(alt))[0] - if len(partial) > 2: - log.debug('Found %s via partial %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) - score += points.get(tag_type) / 3 if (isinstance(alt, (str, unicode)) and ss(alt.lower()) in cur_file.lower()): log.debug('Found %s via %s %s in %s', (quality['identifier'], tag_type, quality.get(tag_type), cur_file)) @@ -285,3 +283,33 @@ class QualityPlugin(Plugin): if add_score != 0: for allow in quality.get('allow', []): score[allow] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 + + def doTest(self): + + tests = { + 'Movie Name (1999)-DVD-Rip.avi': 'dvdrip', + 'Movie Name 1999 720p Bluray.mkv': '720p', + 'Movie Name 1999 BR-Rip 720p.avi': 'brrip', + 'Movie Name 1999 720p Web Rip.avi': 'scr', + 'Movie Name 1999 Web DL.avi': 'brrip', + 'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr', + 'Movie.Name.1999.DVDRip-Group': 'dvdrip', + 'Movie.Name.1999.DVD-Rip-Group': 'dvdrip', + 'Movie.Name.1999.DVD-R-Group': 'dvdr', + } + + correct = 0 + for name in tests: + success = self.guess([name]).get('identifier') == tests[name] + if not success: + log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier'))) + + correct += success + + if correct == len(tests): + log.info('Quality test successful') + return True + else: + log.error('Quality test failed: %s out of %s succeeded', (correct, len(tests))) + + diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 009a60e9..11779803 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -100,14 +100,14 @@ class Release(Plugin): done_status, snatched_status = fireEvent('status.get', ['done', 'snatched'], single = True) # Add movie - movie = db.query(Media).filter_by(library_id = group['library'].get('id')).first() - if not movie: - movie = Media( + media = db.query(Media).filter_by(library_id = group['library'].get('id')).first() + if not media: + media = Media( library_id = group['library'].get('id'), profile_id = 0, status_id = done_status.get('id') ) - db.add(movie) + db.add(media) db.commit() # Add Release @@ -120,7 +120,7 @@ class Release(Plugin): if not rel: rel = Relea( identifier = identifier, - movie = movie, + movie = media, quality_id = group['meta_data']['quality'].get('id'), status_id = done_status.get('id') ) @@ -142,7 +142,7 @@ class Release(Plugin): except: log.debug('Failed to attach "%s" to release: %s', (added_files, traceback.format_exc())) - fireEvent('movie.restatus', movie.id) + fireEvent('media.restatus', media.id) return True @@ -269,7 +269,7 @@ class Release(Plugin): if filedata == 'try_next': return filedata - download_result = fireEvent('download', data = data, movie = media, manual = manual, filedata = filedata, single = True) + download_result = fireEvent('download', data = data, media = media, manual = manual, filedata = filedata, single = True) log.debug('Downloader result: %s', download_result) if download_result: @@ -288,7 +288,7 @@ class Release(Plugin): value = toUnicode(download_result.get(key)) ) rls.info.append(rls_info) - db.commit() + db.commit() log_movie = '%s (%s) in %s' % (getTitle(media['library']), media['library']['year'], rls.quality.label) snatch_message = 'Snatched "%s": %s' % (data.get('name'), log_movie) @@ -446,6 +446,6 @@ class Release(Plugin): db.commit() #Update all movie info as there is no release update function - fireEvent('notify.frontend', type = 'release.update_status.%s' % rel.id, data = status.get('id')) + fireEvent('notify.frontend', type = 'release.update_status', data = rel.to_dict()) return True diff --git a/couchpotato/core/plugins/renamer/__init__.py b/couchpotato/core/plugins/renamer/__init__.py index c8f6b37f..8b602cbd 100755 --- a/couchpotato/core/plugins/renamer/__init__.py +++ b/couchpotato/core/plugins/renamer/__init__.py @@ -93,7 +93,7 @@ config = [{ 'default': 1, 'type': 'int', 'unit': 'min(s)', - 'description': 'Detect movie status every X minutes. Will start the renamer if movie is completed or handle failed download if these options are enabled', + 'description': ('Detect movie status every X minutes.', 'Will start the renamer if movie is completed or handle failed download if these options are enabled'), }, { 'advanced': True, @@ -122,13 +122,13 @@ config = [{ 'advanced': True, 'name': 'separator', 'label': 'File-Separator', - 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', + 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), }, { 'advanced': True, 'name': 'foldersep', 'label': 'Folder-Separator', - 'description': 'Replace all the spaces with a character. Example: ".", "-" (without quotes). Leave empty to use spaces.', + 'description': ('Replace all the spaces with a character.', 'Example: ".", "-" (without quotes). Leave empty to use spaces.'), }, { 'name': 'file_action', @@ -136,7 +136,7 @@ config = [{ 'default': 'link', 'type': 'dropdown', 'values': [('Link', 'link'), ('Copy', 'copy'), ('Move', 'move')], - 'description': 'Link or Copy after downloading completed (and allow for seeding), or Move after seeding completed. Link first tries hard link, then sym link and falls back to Copy.', + 'description': ('Link, Copy or Move after download completed.', 'Link first tries hard link, then sym link and falls back to Copy. It is perfered to use link when downloading torrents as it will save you space, while still beeing able to seed.'), 'advanced': True, }, { diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index ba2cf245..b4f668ff 100755 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -3,7 +3,7 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss, sp from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ - getImdb, link, symlink, tryInt, splitString + getImdb, link, symlink, tryInt, splitString, fnEscape from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ @@ -30,10 +30,10 @@ class Renamer(Plugin): 'desc': 'For the renamer to check for new files to rename in a folder', 'params': { 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'}, - 'base_folder': {'desc': 'Optional: The folder to find releases in. Leave empty for default folder.'}, - 'movie_folder': {'desc': 'Optional: The folder of a specific release to scan. Don\'t use in combination with base_folder.'}, + 'media_folder': {'desc': 'Optional: The folder of the media to scan. Keep empty for default renamer folder.'}, + 'files': {'desc': 'Optional: Provide the release files if more releases are in the same media_folder, delimited with a \'|\'. Note that no dedicated release folder is expected for releases with one file.'}, 'downloader' : {'desc': 'Optional: The downloader the release has been downloaded with. \'download_id\' is required with this option.'}, - 'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in movie_folder. \'downloader\' is required with this option.'}, + 'download_id': {'desc': 'Optional: The nzb/torrent ID of the release in media_folder. \'downloader\' is required with this option.'}, 'status': {'desc': 'Optional: The status of the release: \'completed\' (default) or \'seeding\''}, }, }) @@ -64,26 +64,31 @@ class Renamer(Plugin): def scanView(self, **kwargs): async = tryInt(kwargs.get('async', 0)) - movie_folder = sp(kwargs.get('movie_folder')) - base_folder = sp(kwargs.get('base_folder')) + media_folder = sp(kwargs.get('media_folder')) + + # Backwards compatibility, to be removed after a few versions :) + if not media_folder: + media_folder = sp(kwargs.get('movie_folder')) + downloader = kwargs.get('downloader') download_id = kwargs.get('download_id') + files = '|'.join([sp(filename) for filename in splitString(kwargs.get('files'), '|')]) status = kwargs.get('status', 'completed') - release_download = None - if not base_folder and movie_folder: - release_download = {'folder': movie_folder} - release_download.update({'id': download_id, 'downloader': downloader, 'status': status} if download_id else {}) + release_download = {'folder': media_folder} if media_folder else None + if release_download: + release_download.update({'id': download_id, 'downloader': downloader, 'status': status, 'files': files} if download_id else {}) fire_handle = fireEvent if not async else fireEventAsync - fire_handle('renamer.scan', base_folder = base_folder, release_download = release_download) + fire_handle('renamer.scan', release_download) return { 'success': True } - def scan(self, base_folder = None, release_download = None): + def scan(self, release_download = None): + if not release_download: release_download = {} if self.isDisabled(): return @@ -92,11 +97,11 @@ class Renamer(Plugin): log.info('Renamer is already running, if you see this often, check the logs above for errors.') return - from_folder = sp(self.conf('from') if not base_folder else base_folder) + from_folder = sp(self.conf('from')) to_folder = sp(self.conf('to')) - # Get movie folder to process - movie_folder = sp(release_download and release_download.get('folder')) + # Get media folder to process + media_folder = release_download.get('folder') # Get all folders that should not be processed no_process = [to_folder] @@ -104,7 +109,7 @@ class Renamer(Plugin): no_process.extend([item['destination'] for item in cat_list]) try: if Env.setting('library', section = 'manage').strip(): - no_process.extend(splitString(Env.setting('library', section = 'manage'), '::')) + no_process.extend([sp(manage_folder) for manage_folder in splitString(Env.setting('library', section = 'manage'), '::')]) except: pass @@ -114,40 +119,40 @@ class Renamer(Plugin): return else: for item in no_process: - if from_folder in item: - log.error('To protect your data, the movie libraries can\'t be inside of or the same as the "from" folder.') + if '%s%s' % (from_folder, os.path.sep) in item: + log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.') return - # Check to see if the no_process folders are inside the provided movie_folder - if movie_folder and not os.path.isdir(movie_folder): - log.debug('The provided movie folder %s does not exist. Trying to find it in the \'from\' folder.', movie_folder) + # Check to see if the no_process folders are inside the provided media_folder + if media_folder and not os.path.isdir(media_folder): + log.debug('The provided media folder %s does not exist. Trying to find it in the \'from\' folder.', media_folder) # Update to the from folder - if len(release_download.get('files')) == 1: - new_movie_folder = from_folder + if len(splitString(release_download.get('files'), '|')) == 1: + new_media_folder = from_folder else: - new_movie_folder = sp(os.path.join(from_folder, os.path.basename(movie_folder))) + new_media_folder = os.path.join(from_folder, os.path.basename(media_folder)) - if not os.path.isdir(new_movie_folder): - log.error('The provided movie folder %s does not exist and could also not be found in the \'from\' folder.', movie_folder) + if not os.path.isdir(new_media_folder): + log.error('The provided media folder %s does not exist and could also not be found in the \'from\' folder.', media_folder) return # Update the files - new_files = [os.path.join(new_movie_folder, os.path.relpath(filename, movie_folder)) for filename in splitString(release_download.get('files'), '|')] + new_files = [os.path.join(new_media_folder, os.path.relpath(filename, media_folder)) for filename in splitString(release_download.get('files'), '|')] if new_files and not os.path.isfile(new_files[0]): - log.error('The provided movie folder %s does not exist and its files could also not be found in the \'from\' folder.', movie_folder) + log.error('The provided media folder %s does not exist and its files could also not be found in the \'from\' folder.', media_folder) return # Update release_download info to the from folder - log.debug('Release %s found in the \'from\' folder.', movie_folder) - release_download['folder'] = new_movie_folder + log.debug('Release %s found in the \'from\' folder.', media_folder) + release_download['folder'] = new_media_folder release_download['files'] = '|'.join(new_files) - movie_folder = new_movie_folder + media_folder = new_media_folder - if movie_folder: + if media_folder: for item in no_process: - if movie_folder in item: - log.error('To protect your data, the movie libraries can\'t be inside of or the same as the provided movie folder.') + if '%s%s' % (media_folder, os.path.sep) in item: + log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.') return # Make sure a checkSnatched marked all downloads/seeds as such @@ -156,26 +161,26 @@ class Renamer(Plugin): self.renaming_started = True - # make sure the movie folder name is included in the search + # make sure the media folder name is included in the search folder = None files = [] - if movie_folder: - log.info('Scanning movie folder %s...', movie_folder) - folder = os.path.dirname(movie_folder) + if media_folder: + log.info('Scanning media folder %s...', media_folder) + folder = os.path.dirname(media_folder) if release_download.get('files', ''): files = splitString(release_download['files'], '|') # If there is only one file in the torrent, the downloader did not create a subfolder if len(files) == 1: - folder = movie_folder + folder = media_folder else: # Get all files from the specified folder try: - for root, folders, names in os.walk(movie_folder): - files.extend([os.path.join(root, name) for name in names]) + for root, folders, names in os.walk(media_folder): + files.extend([sp(os.path.join(root, name)) for name in names]) except: - log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc())) + log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc())) db = get_session() @@ -185,7 +190,7 @@ class Renamer(Plugin): # Unpack any archives extr_files = None if self.conf('unrar'): - folder, movie_folder, files, extr_files = self.extractFiles(folder = folder, movie_folder = movie_folder, files = files, + folder, media_folder, files, extr_files = self.extractFiles(folder = folder, media_folder = media_folder, files = files, cleanup = self.conf('cleanup') and not self.downloadIsTorrent(release_download)) groups = fireEvent('scanner.scan', folder = folder if folder else from_folder, @@ -496,8 +501,8 @@ class Renamer(Plugin): if os.path.isfile(src): os.remove(src) - parent_dir = sp(os.path.dirname(src)) - if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not parent_dir in [destination, movie_folder] and not from_folder in parent_dir: + parent_dir = os.path.dirname(src) + if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not '%s%s' % (parent_dir, os.path.sep) in [destination, media_folder] and not '%s%s' % (from_folder, os.path.sep) in parent_dir: delete_folders.append(parent_dir) except: @@ -529,7 +534,7 @@ class Renamer(Plugin): self.tagRelease(group = group, tag = 'failed_rename') # Tag folder if it is in the 'from' folder and it will not be removed because it is a torrent - if self.movieInFromFolder(movie_folder) and self.downloadIsTorrent(release_download): + if self.movieInFromFolder(media_folder) and self.downloadIsTorrent(release_download): self.tagRelease(group = group, tag = 'renamed_already') # Remove matching releases @@ -541,12 +546,12 @@ class Renamer(Plugin): log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) if group['dirname'] and group['parentdir'] and not self.downloadIsTorrent(release_download): - if movie_folder: + if media_folder: # Delete the movie folder - group_folder = movie_folder + group_folder = media_folder else: # Delete the first empty subfolder in the tree relative to the 'from' folder - group_folder = sp(os.path.join(from_folder, os.path.relpath(group['parentdir'], from_folder)).split(os.path.sep)[0]) + group_folder = sp(os.path.join(from_folder, os.path.relpath(group['parentdir'], from_folder).split(os.path.sep)[0])) try: log.info('Deleting folder: %s', group_folder) @@ -606,7 +611,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) elif isinstance(release_download, dict): # Tag download_files if they are known if release_download['files']: - tag_files = release_download['files'].split('|') + tag_files = splitString(release_download['files'], '|') # Tag all files in release folder else: @@ -614,6 +619,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) tag_files.extend([os.path.join(root, name) for name in names]) for filename in tag_files: + + # Dont tag .ignore files + if os.path.splitext(filename)[1] == '.ignore': + continue + tag_filename = '%s.%s.ignore' % (os.path.splitext(filename)[0], tag) if not os.path.isfile(tag_filename): self.createFile(tag_filename, text) @@ -630,21 +640,21 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Untag download_files if they are known if release_download['files']: - tag_files = release_download['files'].split('|') + tag_files = splitString(release_download['files'], '|') # Untag all files in release folder else: for root, folders, names in os.walk(release_download['folder']): - tag_files.extend([os.path.join(root, name) for name in names if not os.path.splitext(name)[1] == '.ignore']) + tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore']) # Find all .ignore files in folder ignore_files = [] for root, dirnames, filenames in os.walk(folder): - ignore_files.extend(fnmatch.filter([os.path.join(root, filename) for filename in filenames], '*%s.ignore' % tag)) + ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag)) # Match all found ignore files with the tag_files and delete if found for tag_file in tag_files: - ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (re.escape(os.path.splitext(tag_file)[0]), tag if tag else '*')) + ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*'))) for filename in ignore_file: try: os.remove(filename) @@ -664,20 +674,20 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Find tag on download_files if they are known if release_download['files']: - tag_files = release_download['files'].split('|') + tag_files = splitString(release_download['files'], '|') # Find tag on all files in release folder else: for root, folders, names in os.walk(release_download['folder']): - tag_files.extend([os.path.join(root, name) for name in names if not os.path.splitext(name)[1] == '.ignore']) + tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore']) # Find all .ignore files in folder for root, dirnames, filenames in os.walk(folder): - ignore_files.extend(fnmatch.filter([os.path.join(root, filename) for filename in filenames], '*%s.ignore' % tag)) + ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag)) # Match all found ignore files with the tag_files and return True found for tag_file in tag_files: - ignore_file = fnmatch.filter(ignore_files, '%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*')) + ignore_file = fnmatch.filter(ignore_files, fnEscape('%s.%s.ignore' % (os.path.splitext(tag_file)[0], tag if tag else '*'))) if ignore_file: return True @@ -772,7 +782,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) return string def deleteEmptyFolder(self, folder, show_error = True): - folder = ss(folder) + folder = sp(folder) loge = log.error if show_error else log.debug for root, dirs, files in os.walk(folder): @@ -806,126 +816,159 @@ Remove it if you want it to be renamed (again, or at least let it try again) Release.status_id.in_([snatched_status.get('id'), seeding_status.get('id'), missing_status.get('id')]) ).all() + if not rels: + #No releases found that need status checking + self.checking_snatched = False + return True + + # Collect all download information with the download IDs from the releases + download_ids = [] + try: + for rel in rels: + rel_dict = rel.to_dict({'info': {}}) + if rel_dict['info'].get('download_id') and rel_dict['info'].get('download_downloader'): + download_ids.append({'id': rel_dict['info']['download_id'], 'downloader': rel_dict['info']['download_downloader']}) + except: + log.error('Error getting download IDs from database') + self.checking_snatched = False + return False + + release_downloads = fireEvent('download.status', download_ids, merge = True) + if not release_downloads: + log.debug('Download status functionality is not implemented for any active downloaders.') + fireEvent('renamer.scan') + + self.checking_snatched = False + return True + scan_releases = [] scan_required = False - if rels: - log.debug('Checking status snatched releases...') + log.debug('Checking status snatched releases...') - release_downloads = fireEvent('download.status', merge = True) - if not release_downloads: - log.debug('Download status functionality is not implemented for active downloaders.') - scan_required = True - else: - try: - for rel in rels: - rel_dict = rel.to_dict({'info': {}}) - movie_dict = fireEvent('movie.get', rel.movie_id, single = True) + try: + for rel in rels: + rel_dict = rel.to_dict({'info': {}}) + movie_dict = fireEvent('media.get', media_id = rel.movie_id, single = True) - if not isinstance(rel_dict['info'], (dict)): - log.error('Faulty release found without any info, ignoring.') + if not isinstance(rel_dict['info'], (dict)): + log.error('Faulty release found without any info, ignoring.') + fireEvent('release.update_status', rel.id, status = ignored_status, single = True) + continue + + # Check if download ID is available + if not rel_dict['info'].get('download_id') or not rel_dict['info'].get('download_downloader'): + log.debug('Download status functionality is not implemented for downloader (%s) of release %s.', (rel_dict['info'].get('download_downloader', 'unknown'), rel_dict['info']['name'])) + scan_required = True + + # Continue with next release + continue + + # Find release in downloaders + nzbname = self.createNzbName(rel_dict['info'], movie_dict) + + for release_download in release_downloads: + found_release = False + if rel_dict['info'].get('download_id'): + if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']: + log.debug('Found release by id: %s', release_download['id']) + found_release = True + break + else: + if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']: + log.debug('Found release by release name or imdb ID: %s', release_download['name']) + found_release = True + break + + if not found_release: + log.info('%s not found in downloaders', nzbname) + + #Check status if already missing and for how long, if > 1 week, set to ignored else to missing + if rel.status_id == missing_status.get('id'): + if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60: fireEvent('release.update_status', rel.id, status = ignored_status, single = True) - continue + else: + # Set the release to missing + fireEvent('release.update_status', rel.id, status = missing_status, single = True) - # check status - nzbname = self.createNzbName(rel_dict['info'], movie_dict) + # Continue with next release + continue - found = False - for release_download in release_downloads: - found_release = False - if rel_dict['info'].get('download_id'): - if release_download['id'] == rel_dict['info']['download_id'] and release_download['downloader'] == rel_dict['info']['download_downloader']: - log.debug('Found release by id: %s', release_download['id']) - found_release = True + # Log that we found the release + timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft'] + log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft)) + + # Check status of release + if release_download['status'] == 'busy': + # Set the release to snatched if it was missing before + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) + + # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading + if self.movieInFromFolder(release_download['folder']): + self.tagRelease(release_download = release_download, tag = 'downloading') + + elif release_download['status'] == 'seeding': + #If linking setting is enabled, process release + if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download): + log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio'])) + + # Remove the downloading tag + self.untagRelease(release_download = release_download, tag = 'downloading') + + # Scan and set the torrent to paused if required + release_download.update({'pause': True, 'scan': True, 'process_complete': False}) + scan_releases.append(release_download) + else: + #let it seed + log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio'])) + + # Set the release to seeding + fireEvent('release.update_status', rel.id, status = seeding_status, single = True) + + elif release_download['status'] == 'failed': + # Set the release to failed + fireEvent('release.update_status', rel.id, status = failed_status, single = True) + + fireEvent('download.remove_failed', release_download, single = True) + + if self.conf('next_on_failed'): + fireEvent('movie.searcher.try_next_release', media_id = rel.movie_id) + + elif release_download['status'] == 'completed': + log.info('Download of %s completed!', release_download['name']) + + #Make sure the downloader sent over a path to look in + if self.statusInfoComplete(release_download): + + # If the release has been seeding, process now the seeding is done + if rel.status_id == seeding_status.get('id'): + if self.conf('file_action') != 'move': + # Set the release to done as the movie has already been renamed + fireEvent('release.update_status', rel.id, status = downloaded_status, single = True) + + # Allow the downloader to clean-up + release_download.update({'pause': False, 'scan': False, 'process_complete': True}) + scan_releases.append(release_download) else: - if release_download['name'] == nzbname or rel_dict['info']['name'] in release_download['name'] or getImdb(release_download['name']) == movie_dict['library']['identifier']: - found_release = True + # Scan and Allow the downloader to clean-up + release_download.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_releases.append(release_download) - if found_release: - timeleft = 'N/A' if release_download['timeleft'] == -1 else release_download['timeleft'] - log.debug('Found %s: %s, time to go: %s', (release_download['name'], release_download['status'].upper(), timeleft)) + else: + # Set the release to snatched if it was missing before + fireEvent('release.update_status', rel.id, status = snatched_status, single = True) - if release_download['status'] == 'busy': - # Set the release to snatched if it was missing before - fireEvent('release.update_status', rel.id, status = snatched_status, single = True) + # Remove the downloading tag + self.untagRelease(release_download = release_download, tag = 'downloading') - # Tag folder if it is in the 'from' folder and it will not be processed because it is still downloading - if self.movieInFromFolder(release_download['folder']): - self.tagRelease(release_download = release_download, tag = 'downloading') + # Scan and Allow the downloader to clean-up + release_download.update({'pause': False, 'scan': True, 'process_complete': True}) + scan_releases.append(release_download) + else: + scan_required = True - elif release_download['status'] == 'seeding': - #If linking setting is enabled, process release - if self.conf('file_action') != 'move' and not rel.status_id == seeding_status.get('id') and self.statusInfoComplete(release_download): - log.info('Download of %s completed! It is now being processed while leaving the original files alone for seeding. Current ratio: %s.', (release_download['name'], release_download['seed_ratio'])) - - # Remove the downloading tag - self.untagRelease(release_download = release_download, tag = 'downloading') - - # Scan and set the torrent to paused if required - release_download.update({'pause': True, 'scan': True, 'process_complete': False}) - scan_releases.append(release_download) - else: - #let it seed - log.debug('%s is seeding with ratio: %s', (release_download['name'], release_download['seed_ratio'])) - - # Set the release to seeding - fireEvent('release.update_status', rel.id, status = seeding_status, single = True) - - elif release_download['status'] == 'failed': - # Set the release to failed - fireEvent('release.update_status', rel.id, status = failed_status, single = True) - - fireEvent('download.remove_failed', release_download, single = True) - - if self.conf('next_on_failed'): - fireEvent('movie.searcher.try_next_release', movie_id = rel.movie_id) - elif release_download['status'] == 'completed': - log.info('Download of %s completed!', release_download['name']) - if self.statusInfoComplete(release_download): - - # If the release has been seeding, process now the seeding is done - if rel.status_id == seeding_status.get('id'): - if self.conf('file_action') != 'move': - # Set the release to done as the movie has already been renamed - fireEvent('release.update_status', rel.id, status = downloaded_status, single = True) - - # Allow the downloader to clean-up - release_download.update({'pause': False, 'scan': False, 'process_complete': True}) - scan_releases.append(release_download) - else: - # Scan and Allow the downloader to clean-up - release_download.update({'pause': False, 'scan': True, 'process_complete': True}) - scan_releases.append(release_download) - - else: - # Set the release to snatched if it was missing before - fireEvent('release.update_status', rel.id, status = snatched_status, single = True) - - # Remove the downloading tag - self.untagRelease(release_download = release_download, tag = 'downloading') - - # Scan and Allow the downloader to clean-up - release_download.update({'pause': False, 'scan': True, 'process_complete': True}) - scan_releases.append(release_download) - else: - scan_required = True - - found = True - break - - if not found: - log.info('%s not found in downloaders', nzbname) - - #Check status if already missing and for how long, if > 1 week, set to ignored else to missing - if rel.status_id == missing_status.get('id'): - if rel.last_edit < int(time.time()) - 7 * 24 * 60 * 60: - fireEvent('release.update_status', rel.id, status = ignored_status, single = True) - else: - # Set the release to missing - fireEvent('release.update_status', rel.id, status = missing_status, single = True) - - except: - log.error('Failed checking for release in downloader: %s', traceback.format_exc()) + except: + log.error('Failed checking for release in downloader: %s', traceback.format_exc()) # The following can either be done here, or inside the scanner if we pass it scan_items in one go for release_download in scan_releases: @@ -948,7 +991,6 @@ Remove it if you want it to be renamed (again, or at least let it try again) fireEvent('renamer.scan') self.checking_snatched = False - return True def extendReleaseDownload(self, release_download): @@ -995,10 +1037,10 @@ Remove it if you want it to be renamed (again, or at least let it try again) def statusInfoComplete(self, release_download): return release_download['id'] and release_download['downloader'] and release_download['folder'] - def movieInFromFolder(self, movie_folder): - return (movie_folder and sp(self.conf('from')) in movie_folder) or not movie_folder + def movieInFromFolder(self, media_folder): + return media_folder and '%s%s' % (sp(self.conf('from')), os.path.sep) in sp(media_folder) or not media_folder - def extractFiles(self, folder = None, movie_folder = None, files = None, cleanup = False): + def extractFiles(self, folder = None, media_folder = None, files = None, cleanup = False): if not files: files = [] # RegEx for finding rar files @@ -1013,12 +1055,12 @@ Remove it if you want it to be renamed (again, or at least let it try again) folder = from_folder check_file_date = True - if movie_folder: + if media_folder: check_file_date = False if not files: for root, folders, names in os.walk(folder): - files.extend([os.path.join(root, name) for name in names]) + files.extend([sp(os.path.join(root, name)) for name in names]) # Find all archive files archives = [re.search(archive_regex, name).groupdict() for name in files if re.search(archive_regex, name)] @@ -1064,13 +1106,13 @@ Remove it if you want it to be renamed (again, or at least let it try again) log.info('Archive %s found. Extracting...', os.path.basename(archive['file'])) try: rar_handle = RarFile(archive['file']) - extr_path = sp(os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder))) + extr_path = os.path.join(from_folder, os.path.relpath(os.path.dirname(archive['file']), folder)) self.makeDir(extr_path) for packedinfo in rar_handle.infolist(): - if not packedinfo.isdir and not os.path.isfile(os.path.join(extr_path, os.path.basename(packedinfo.filename))): + if not packedinfo.isdir and not os.path.isfile(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))): log.debug('Extracting %s...', packedinfo.filename) rar_handle.extract(condition = [packedinfo.index], path = extr_path, withSubpath = False, overwrite = False) - extr_files.append(os.path.join(extr_path, os.path.basename(packedinfo.filename))) + extr_files.append(sp(os.path.join(extr_path, os.path.basename(packedinfo.filename)))) del rar_handle except Exception, e: log.error('Failed to extract %s: %s %s', (archive['file'], e, traceback.format_exc())) @@ -1087,9 +1129,9 @@ Remove it if you want it to be renamed (again, or at least let it try again) files.remove(filename) # Move the rest of the files and folders if any files are extracted to the from folder (only if folder was provided) - if extr_files and sp(folder) != from_folder: + if extr_files and folder != from_folder: for leftoverfile in list(files): - move_to = sp(os.path.join(from_folder, os.path.relpath(leftoverfile, folder))) + move_to = os.path.join(from_folder, os.path.relpath(leftoverfile, folder)) try: self.makeDir(os.path.dirname(move_to)) @@ -1109,18 +1151,18 @@ Remove it if you want it to be renamed (again, or at least let it try again) if cleanup: # Remove all left over folders - log.debug('Removing old movie folder %s...', movie_folder) - self.deleteEmptyFolder(movie_folder) + log.debug('Removing old movie folder %s...', media_folder) + self.deleteEmptyFolder(media_folder) - movie_folder = sp(os.path.join(from_folder, os.path.relpath(movie_folder, folder))) + media_folder = os.path.join(from_folder, os.path.relpath(media_folder, folder)) folder = from_folder if extr_files: files.extend(extr_files) - # Cleanup files and folder if movie_folder was not provided - if not movie_folder: + # Cleanup files and folder if media_folder was not provided + if not media_folder: files = [] folder = None - return folder, movie_folder, files, extr_files + return folder, media_folder, files, extr_files diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index e77e62ad..24eb7ac7 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -124,7 +124,7 @@ class Scanner(Plugin): try: files = [] for root, dirs, walk_files in os.walk(folder): - files.extend([os.path.join(root, filename) for filename in walk_files]) + files.extend([sp(os.path.join(root, filename)) for filename in walk_files]) # Break if CP wants to shut down if self.shuttingDown(): @@ -134,7 +134,7 @@ class Scanner(Plugin): log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) else: check_file_date = False - files = [ss(x) for x in files] + files = [sp(x) for x in files] for file_path in files: @@ -454,7 +454,7 @@ class Scanner(Plugin): data['resolution_width'] = meta.get('resolution_width', 720) data['resolution_height'] = meta.get('resolution_height', 480) data['audio_channels'] = meta.get('audio_channels', 2.0) - data['aspect'] = meta.get('resolution_width', 720) / meta.get('resolution_height', 480) + data['aspect'] = round(float(meta.get('resolution_width', 720)) / meta.get('resolution_height', 480), 2) except: log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc())) pass diff --git a/couchpotato/core/plugins/score/main.py b/couchpotato/core/plugins/score/main.py index 5f9da1a1..54b6ca31 100644 --- a/couchpotato/core/plugins/score/main.py +++ b/couchpotato/core/plugins/score/main.py @@ -1,11 +1,11 @@ -from couchpotato.core.event import addEvent +from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import getTitle, splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \ sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \ - halfMultipartScore + halfMultipartScore, sceneScore from couchpotato.environment import Env log = CPLog(__name__) @@ -62,4 +62,7 @@ class Score(Plugin): if extra_score: score += extra_score(nzb) + # Scene / Nuke scoring + score += sceneScore(nzb['name']) + return score diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index 6aa0b465..895f5fc0 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -1,8 +1,13 @@ from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import simplifyString from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.logger import CPLog from couchpotato.environment import Env import re +import traceback + +log = CPLog(__name__) + name_scores = [ # Tags @@ -160,3 +165,38 @@ def halfMultipartScore(nzb_name): return -30 return 0 + + +def sceneScore(nzb_name): + + check_names = [nzb_name] + + # Match names between " + try: check_names.append(re.search(r'([\'"])[^\1]*\1', nzb_name).group(0)) + except: pass + + # Match longest name between [] + try: check_names.append(max(re.findall(r'[^[]*\[([^]]*)\]', nzb_name), key = len).strip()) + except: pass + + for name in check_names: + + # Strip twice, remove possible file extensions + name = name.lower().strip(' "\'\.-_\[\]') + name = re.sub('\.([a-z0-9]{0,4})$', '', name) + name = name.strip(' "\'\.-_\[\]') + + # Make sure year and groupname is in there + year = re.findall('(?P19[0-9]{2}|20[0-9]{2})', name) + group = re.findall('\-([a-z0-9]+)$', name) + + if len(year) > 0 and len(group) > 0: + try: + validate = fireEvent('release.validate', name, single = True) + if validate and tryInt(validate.get('score')) != 0: + log.debug('Release "%s" scored %s, reason: %s', (nzb_name, validate['score'], validate['reasons'])) + return tryInt(validate.get('score')) + except: + log.error('Failed scoring scene: %s', traceback.format_exc()) + + return 0 diff --git a/couchpotato/core/plugins/subtitle/__init__.py b/couchpotato/core/plugins/subtitle/__init__.py index bbd40853..fcff4cdf 100644 --- a/couchpotato/core/plugins/subtitle/__init__.py +++ b/couchpotato/core/plugins/subtitle/__init__.py @@ -20,7 +20,7 @@ config = [{ }, { 'name': 'languages', - 'description': 'Comma separated, 2 letter country code. Example: en, nl. See the codes at on Wikipedia', + 'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at on Wikipedia'), }, # { # 'name': 'automatic', diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index e447dacd..7504d6a9 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -1,6 +1,6 @@ from couchpotato import get_session from couchpotato.core.event import addEvent, fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, sp from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -58,9 +58,9 @@ class Subtitle(Plugin): for d_sub in downloaded: log.info('Found subtitle (%s): %s', (d_sub.language.alpha2, files)) - group['files']['subtitle'].append(d_sub.path) - group['before_rename'].append(d_sub.path) - group['subtitle_language'][d_sub.path] = [d_sub.language.alpha2] + group['files']['subtitle'].append(sp(d_sub.path)) + group['before_rename'].append(sp(d_sub.path)) + group['subtitle_language'][sp(d_sub.path)] = [d_sub.language.alpha2] return True diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index da27d853..41b1e62f 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -264,14 +264,14 @@ class ResultList(list): result_ids = None provider = None - movie = None + media = None quality = None - def __init__(self, provider, movie, quality, **kwargs): + def __init__(self, provider, media, quality, **kwargs): self.result_ids = [] self.provider = provider - self.movie = movie + self.media = media self.quality = quality self.kwargs = kwargs @@ -285,13 +285,13 @@ class ResultList(list): new_result = self.fillResult(result) - is_correct = fireEvent('searcher.correct_release', new_result, self.movie, self.quality, + is_correct = fireEvent('searcher.correct_release', new_result, self.media, self.quality, imdb_results = self.kwargs.get('imdb_results', False), single = True) if is_correct and new_result['id'] not in self.result_ids: is_correct_weight = float(is_correct) - new_result['score'] += fireEvent('score.calculate', new_result, self.movie, single = True) + new_result['score'] += fireEvent('score.calculate', new_result, self.media, single = True) old_score = new_result['score'] new_result['score'] = int(old_score * is_correct_weight) diff --git a/couchpotato/core/providers/info/_modifier/main.py b/couchpotato/core/providers/info/_modifier/main.py index 0bb2e6a4..daf2a306 100644 --- a/couchpotato/core/providers/info/_modifier/main.py +++ b/couchpotato/core/providers/info/_modifier/main.py @@ -93,11 +93,11 @@ class MovieResultModifier(Plugin): for movie in l.movies: if movie.status_id == active_status['id']: - temp['in_wanted'] = fireEvent('movie.get', movie.id, single = True) + temp['in_wanted'] = fireEvent('media.get', movie.id, single = True) for release in movie.releases: if release.status_id == done_status['id']: - temp['in_library'] = fireEvent('movie.get', movie.id, single = True) + temp['in_library'] = fireEvent('media.get', movie.id, single = True) except: log.error('Tried getting more info on searched movies: %s', traceback.format_exc()) diff --git a/couchpotato/core/providers/info/couchpotatoapi/main.py b/couchpotato/core/providers/info/couchpotatoapi/main.py index 89eddc3c..4dd942e0 100644 --- a/couchpotato/core/providers/info/couchpotatoapi/main.py +++ b/couchpotato/core/providers/info/couchpotatoapi/main.py @@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.logger import CPLog from couchpotato.core.providers.info.base import MovieProvider from couchpotato.environment import Env +import base64 import time log = CPLog(__name__) @@ -11,6 +12,7 @@ log = CPLog(__name__) class CouchPotatoApi(MovieProvider): urls = { + 'validate': 'https://api.couchpota.to/validate/%s/', 'search': 'https://api.couchpota.to/search/%s/', 'info': 'https://api.couchpota.to/info/%s/', 'is_movie': 'https://api.couchpota.to/ismovie/%s/', @@ -30,6 +32,8 @@ class CouchPotatoApi(MovieProvider): addEvent('movie.suggest', self.getSuggestions) addEvent('movie.is_movie', self.isMovie) + addEvent('release.validate', self.validate) + addEvent('cp.source_url', self.getSourceUrl) addEvent('cp.messages', self.getMessages) @@ -51,6 +55,14 @@ class CouchPotatoApi(MovieProvider): def search(self, q, limit = 5): return self.getJsonData(self.urls['search'] % tryUrlencode(q) + ('?limit=%s' % limit), headers = self.getRequestHeaders()) + def validate(self, name = None): + + if not name: + return + + name_enc = base64.b64encode(name) + return self.getJsonData(self.urls['validate'] % name_enc, headers = self.getRequestHeaders()) + def isMovie(self, identifier = None): if not identifier: diff --git a/couchpotato/core/providers/info/omdbapi/main.py b/couchpotato/core/providers/info/omdbapi/main.py index 47374f47..605dadf1 100755 --- a/couchpotato/core/providers/info/omdbapi/main.py +++ b/couchpotato/core/providers/info/omdbapi/main.py @@ -84,6 +84,10 @@ class OMDBAPI(MovieProvider): year = tryInt(movie.get('Year', '')) + actors = {} + for actor in splitString(movie.get('Actors', '')): + actors[actor] = '' #omdb does not return actor roles + movie_data = { 'type': 'movie', 'via_imdb': True, @@ -105,7 +109,7 @@ class OMDBAPI(MovieProvider): 'genres': splitString(movie.get('Genre', '')), 'directors': splitString(movie.get('Director', '')), 'writers': splitString(movie.get('Writer', '')), - 'actors': splitString(movie.get('Actors', '')), + 'actor_roles': actors, } movie_data = dict((k, v) for k, v in movie_data.iteritems() if v) except: diff --git a/couchpotato/core/providers/info/themoviedb/main.py b/couchpotato/core/providers/info/themoviedb/main.py index a7901351..cd957927 100644 --- a/couchpotato/core/providers/info/themoviedb/main.py +++ b/couchpotato/core/providers/info/themoviedb/main.py @@ -92,6 +92,13 @@ class TheMovieDb(MovieProvider): poster_original = self.getImage(movie, type = 'poster', size = 'original') backdrop_original = self.getImage(movie, type = 'backdrop', size = 'original') + images = { + 'poster': [poster] if poster else [], + #'backdrop': [backdrop] if backdrop else [], + 'poster_original': [poster_original] if poster_original else [], + 'backdrop_original': [backdrop_original] if backdrop_original else [], + } + # Genres try: genres = [genre.name for genre in movie.genres] @@ -103,18 +110,22 @@ class TheMovieDb(MovieProvider): if not movie.releasedate or year == '1900' or year.lower() == 'none': year = None + # Gather actors data + actors = {} + for cast_item in movie.cast: + try: + actors[toUnicode(cast_item.name)] = toUnicode(cast_item.character) + images['actor %s' % toUnicode(cast_item.name)] = self.getImage(cast_item, type = 'profile', size = 'original') + except: + log.debug('Error getting cast info for %s: %s', (cast_item, traceback.format_exc())) + movie_data = { 'type': 'movie', 'via_tmdb': True, 'tmdb_id': movie.id, 'titles': [toUnicode(movie.title)], 'original_title': movie.originaltitle, - 'images': { - 'poster': [poster] if poster else [], - #'backdrop': [backdrop] if backdrop else [], - 'poster_original': [poster_original] if poster_original else [], - 'backdrop_original': [backdrop_original] if backdrop_original else [], - }, + 'images': images, 'imdb': movie.imdb, 'runtime': movie.runtime, 'released': str(movie.releasedate), @@ -122,6 +133,7 @@ class TheMovieDb(MovieProvider): 'plot': movie.overview, 'genres': genres, 'collection': getattr(movie.collection, 'name', None), + 'actor_roles': actors } movie_data = dict((k, v) for k, v in movie_data.iteritems() if v) @@ -145,7 +157,7 @@ class TheMovieDb(MovieProvider): try: image_url = getattr(movie, type).geturl(size = 'original') except: - log.debug('Failed getting %s.%s for "%s"', (type, size, movie.title)) + log.debug('Failed getting %s.%s for "%s"', (type, size, movie)) return image_url diff --git a/couchpotato/core/providers/metadata/xbmc/main.py b/couchpotato/core/providers/metadata/xbmc/main.py index 7073363d..4c547c35 100644 --- a/couchpotato/core/providers/metadata/xbmc/main.py +++ b/couchpotato/core/providers/metadata/xbmc/main.py @@ -65,7 +65,7 @@ class XBMC(MetaDataBase): name = type try: - if data['library'].get(type): + if movie_info.get(type): el = SubElement(nfoxml, name) el.text = toUnicode(movie_info.get(type, '')) except: @@ -89,10 +89,18 @@ class XBMC(MetaDataBase): genres.text = toUnicode(genre) # Actors - for actor in movie_info.get('actors', []): - actors = SubElement(nfoxml, 'actor') - name = SubElement(actors, 'name') - name.text = toUnicode(actor) + for actor_name in movie_info.get('actor_roles', {}): + role_name = movie_info['actor_roles'][actor_name] + + actor = SubElement(nfoxml, 'actor') + name = SubElement(actor, 'name') + name.text = toUnicode(actor_name) + if role_name: + role = SubElement(actor, 'role') + role.text = toUnicode(role_name) + if movie_info['images'].get('actor %s' % actor_name, ''): + thumb = SubElement(actor, 'thumb') + thumb.text = toUnicode(movie_info['images'].get('actor %s' % actor_name)) # Directors for director_name in movie_info.get('directors', []): @@ -112,6 +120,51 @@ class XBMC(MetaDataBase): sorttitle = SubElement(nfoxml, 'sorttitle') sorttitle.text = '%s %s' % (toUnicode(collection_name), movie_info.get('year')) + # Images + for image_url in movie_info['images']['poster_original']: + image = SubElement(nfoxml, 'thumb') + image.text = toUnicode(image_url) + fanart = SubElement(nfoxml, 'fanart') + for image_url in movie_info['images']['backdrop_original']: + image = SubElement(fanart, 'thumb') + image.text = toUnicode(image_url) + + # Add trailer if found + trailer_found = False + if data.get('renamed_files'): + for filename in data.get('renamed_files'): + if 'trailer' in filename: + trailer = SubElement(nfoxml, 'trailer') + trailer.text = toUnicode(filename) + trailer_found = True + if not trailer_found and data['files'].get('trailer'): + trailer = SubElement(nfoxml, 'trailer') + trailer.text = toUnicode(data['files']['trailer'][0]) + + # Add file metadata + fileinfo = SubElement(nfoxml, 'fileinfo') + streamdetails = SubElement(fileinfo, 'streamdetails') + + # Video data + if data['meta_data'].get('video'): + video = SubElement(streamdetails, 'video') + codec = SubElement(video, 'codec') + codec.text = toUnicode(data['meta_data']['video']) + aspect = SubElement(video, 'aspect') + aspect.text = str(data['meta_data']['aspect']) + width = SubElement(video, 'width') + width.text = str(data['meta_data']['resolution_width']) + height = SubElement(video, 'height') + height.text = str(data['meta_data']['resolution_height']) + + # Audio data + if data['meta_data'].get('audio'): + audio = SubElement(streamdetails, 'audio') + codec = SubElement(audio, 'codec') + codec.text = toUnicode(data['meta_data'].get('audio')) + channels = SubElement(audio, 'channels') + channels.text = toUnicode(data['meta_data'].get('audio_channels')) + # Clean up the xml and return it nfoxml = xml.dom.minidom.parseString(tostring(nfoxml)) xml_string = nfoxml.toprettyxml(indent = ' ') diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index bd1b6c32..c1eed852 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -10,6 +10,7 @@ from urllib2 import HTTPError from urlparse import urlparse import time import traceback +import urllib2 log = CPLog(__name__) @@ -159,7 +160,15 @@ class Newznab(NZBProvider, RSS): return 'try_next' try: - data = self.urlopen(url, show_error = False) + # Get final redirected url + log.debug('Checking %s for redirects.', url) + req = urllib2.Request(url) + res = urllib2.urlopen(req) + finalurl = res.geturl() + if finalurl != url: + log.debug('Redirect url used: %s', finalurl) + + data = self.urlopen(finalurl, show_error = False) self.limits_reached[host] = False return data except HTTPError, e: diff --git a/couchpotato/core/providers/torrent/bithdtv/__init__.py b/couchpotato/core/providers/torrent/bithdtv/__init__.py index 2c7af031..8c6f97a0 100644 --- a/couchpotato/core/providers/torrent/bithdtv/__init__.py +++ b/couchpotato/core/providers/torrent/bithdtv/__init__.py @@ -4,7 +4,7 @@ def start(): return BiTHDTV() config = [{ - 'name': 'BiT-HDTV', + 'name': 'bithdtv', 'groups': [ { 'tab': 'searcher', diff --git a/couchpotato/core/providers/torrent/torrentpotato/__init__.py b/couchpotato/core/providers/torrent/torrentpotato/__init__.py new file mode 100644 index 00000000..5054f98c --- /dev/null +++ b/couchpotato/core/providers/torrent/torrentpotato/__init__.py @@ -0,0 +1,66 @@ +from .main import TorrentPotato + +def start(): + return TorrentPotato() + +config = [{ + 'name': 'torrentpotato', + 'groups': [ + { + 'tab': 'searcher', + 'list': 'torrent_providers', + 'name': 'TorrentPotato', + 'order': 10, + 'description': 'CouchPotato torrent provider. Checkout the wiki page about this provider for more info.', + 'wizard': True, + 'options': [ + { + 'name': 'enabled', + 'type': 'enabler', + 'default': False, + }, + { + 'name': 'use', + 'default': '' + }, + { + 'name': 'host', + 'default': '', + 'description': 'The url path of your TorrentPotato provider.', + }, + { + 'name': 'extra_score', + 'advanced': True, + 'label': 'Extra Score', + 'default': '0', + 'description': 'Starting score for each release found via this provider.', + }, + { + 'name': 'name', + 'label': 'Username', + 'default': '', + }, + { + 'name': 'seed_ratio', + 'label': 'Seed ratio', + 'default': '1', + 'description': 'Will not be (re)moved until this seed ratio is met.', + }, + { + 'name': 'seed_time', + 'label': 'Seed time', + 'default': '40', + 'description': 'Will not be (re)moved until this seed time (in hours) is met.', + }, + { + 'name': 'pass_key', + 'default': ',', + 'label': 'Pass Key', + 'description': 'Can be found on your profile page', + 'type': 'combined', + 'combine': ['use', 'host', 'pass_key', 'name', 'seed_ratio', 'seed_time', 'extra_score'], + }, + ], + }, + ], +}] diff --git a/couchpotato/core/providers/torrent/torrentpotato/main.py b/couchpotato/core/providers/torrent/torrentpotato/main.py new file mode 100644 index 00000000..a76c0c8f --- /dev/null +++ b/couchpotato/core/providers/torrent/torrentpotato/main.py @@ -0,0 +1,129 @@ +from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode +from couchpotato.core.helpers.variable import splitString, tryInt, tryFloat +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.base import ResultList +from couchpotato.core.providers.torrent.base import TorrentProvider +from urlparse import urlparse +import re +import traceback + +log = CPLog(__name__) + + +class TorrentPotato(TorrentProvider): + + urls = {} + limits_reached = {} + + http_time_between_calls = 1 # Seconds + + def search(self, movie, quality): + hosts = self.getHosts() + + results = ResultList(self, movie, quality, imdb_results = True) + + for host in hosts: + if self.isDisabled(host): + continue + + self._searchOnHost(host, movie, quality, results) + + return results + + def _searchOnHost(self, host, movie, quality, results): + + arguments = tryUrlencode({ + 'user': host['name'], + 'passkey': host['pass_key'], + 'imdbid': movie['library']['identifier'] + }) + url = '%s?%s' % (host['host'], arguments) + + torrents = self.getJsonData(url, cache_timeout = 1800) + + if torrents: + try: + if torrents.get('error'): + log.error('%s: %s', (torrents.get('error'), host['host'])) + elif torrents.get('results'): + for torrent in torrents.get('results', []): + results.append({ + 'id': torrent.get('torrent_id'), + 'protocol': 'torrent' if re.match('^(http|https|ftp)://.*$', torrent.get('download_url')) else 'torrent_magnet', + 'provider_extra': urlparse(host['host']).hostname or host['host'], + 'name': toUnicode(torrent.get('release_name')), + 'url': torrent.get('download_url'), + 'detail_url': torrent.get('details_url'), + 'size': torrent.get('size'), + 'score': host['extra_score'], + 'seeders': torrent.get('seeders'), + 'leechers': torrent.get('leechers'), + 'seed_ratio': host['seed_ratio'], + 'seed_time': host['seed_time'], + }) + + except: + log.error('Failed getting results from %s: %s', (host['host'], traceback.format_exc())) + + def getHosts(self): + + uses = splitString(str(self.conf('use')), clean = False) + hosts = splitString(self.conf('host'), clean = False) + names = splitString(self.conf('name'), clean = False) + seed_times = splitString(self.conf('seed_time'), clean = False) + seed_ratios = splitString(self.conf('seed_ratio'), clean = False) + pass_keys = splitString(self.conf('pass_key'), clean = False) + extra_score = splitString(self.conf('extra_score'), clean = False) + + list = [] + for nr in range(len(hosts)): + + try: key = pass_keys[nr] + except: key = '' + + try: host = hosts[nr] + except: host = '' + + try: name = names[nr] + except: name = '' + + try: ratio = seed_ratios[nr] + except: ratio = '' + + try: seed_time = seed_times[nr] + except: seed_time = '' + + list.append({ + 'use': uses[nr], + 'host': host, + 'name': name, + 'seed_ratio': tryFloat(ratio), + 'seed_time': tryInt(seed_time), + 'pass_key': key, + 'extra_score': tryInt(extra_score[nr]) if len(extra_score) > nr else 0 + }) + + return list + + def belongsTo(self, url, provider = None, host = None): + + hosts = self.getHosts() + + for host in hosts: + result = super(TorrentPotato, self).belongsTo(url, host = host['host'], provider = provider) + if result: + return result + + def isDisabled(self, host = None): + return not self.isEnabled(host) + + def isEnabled(self, host = None): + + # Return true if at least one is enabled and no host is given + if host is None: + for host in self.getHosts(): + if self.isEnabled(host): + return True + return False + + return TorrentProvider.isEnabled(self) and host['host'] and host['pass_key'] and int(host['use']) diff --git a/couchpotato/environment.py b/couchpotato/environment.py index 0f04d838..b393ef94 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -78,6 +78,7 @@ class Env(object): return s.get(attr, default = default, section = section, type = type) # Set setting + s.addSection(section) s.set(section, attr, value) s.save() diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 571023ea..452f0055 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -8,6 +8,7 @@ from couchpotato.core.helpers.variable import getDataDir, tryInt from logging import handlers from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler +from uuid import uuid4 import locale import logging import os.path @@ -144,7 +145,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En Env.set('dev', development) # Disable logging for some modules - for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: + for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: @@ -215,6 +216,10 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En Env.set('web_base', web_base) api_key = Env.setting('api_key') + if not api_key: + api_key = uuid4().hex + Env.setting('api_key', value = api_key) + api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 59fac34b..eae865f4 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -11,6 +11,12 @@ pages: [], block: [], + initialize: function(){ + var self = this; + + self.global_events = {}; + }, + setup: function(options) { var self = this; self.setOptions(options); @@ -30,7 +36,7 @@ History.addEvent('change', self.openPage.bind(self)); self.c.addEvent('click:relay(a[href^=/]:not([target]))', self.pushState.bind(self)); self.c.addEvent('click:relay(a[href^=http])', self.openDerefered.bind(self)); - + // Check if device is touchenabled self.touch_device = 'ontouchstart' in window || navigator.msMaxTouchPoints; if(self.touch_device) @@ -55,7 +61,7 @@ History.push(url); } }, - + isMac: function(){ return Browser.Platform.mac }, @@ -111,7 +117,7 @@ } }) ]; - + setting_links.each(function(a){ self.block.more.addLink(a) }); @@ -336,6 +342,66 @@ }) ) ); + }, + + /* + * Global events + */ + on: function(name, handle){ + var self = this; + + if(!self.global_events[name]) + self.global_events[name] = []; + + self.global_events[name].push(handle); + + }, + + trigger: function(name, args, on_complete){ + var self = this; + + if(!self.global_events[name]){ return; } + + if(!on_complete && typeOf(args) == 'function'){ + on_complete = args; + args = {}; + } + + // Create parallel callback + var callbacks = []; + self.global_events[name].each(function(handle, nr){ + + callbacks.push(function(callback){ + var results = handle(args || {}); + callback(null, results || null); + }); + + }); + + // Fire events + async.parallel(callbacks, function(err, results){ + if(err) p(err); + + if(on_complete) + on_complete(results); + }); + + }, + + off: function(name, handle){ + var self = this; + + if(!self.global_events[name]) return; + + // Remove single + if(handle){ + self.global_events[name] = self.global_events[name].erase(handle); + } + // Reset full event + else { + self.global_events[name] = []; + } + } }); @@ -503,7 +569,7 @@ function randomString(length, extra) { case "string": saveKeyPath(argument.match(/[+-]|[^.]+/g)); break; } }); - return this.sort(comparer); + return this.stableSort(comparer); } }); diff --git a/couchpotato/static/scripts/library/Array.stableSort.js b/couchpotato/static/scripts/library/Array.stableSort.js new file mode 100644 index 00000000..062c7566 --- /dev/null +++ b/couchpotato/static/scripts/library/Array.stableSort.js @@ -0,0 +1,56 @@ +/* +--- + +script: Array.stableSort.js + +description: Add a stable sort algorithm for all browsers + +license: MIT-style license. + +authors: + - Yorick Sijsling + +requires: + core/1.3: '*' + +provides: + - [Array.stableSort, Array.mergeSort] + +... +*/ + +(function() { + + var defaultSortFunction = function(a, b) { + return a > b ? 1 : (a < b ? -1 : 0); + } + + Array.implement({ + + stableSort: function(compare) { + // I would love some real feature recognition. Problem is that an unstable algorithm sometimes/often gives the same result as an unstable algorithm. + return (Browser.chrome || Browser.firefox2 || Browser.opera9) ? this.mergeSort(compare) : this.sort(compare); + }, + + mergeSort: function(compare, token) { + compare = compare || defaultSortFunction; + if (this.length > 1) { + // Split and sort both parts + var right = this.splice(Math.floor(this.length / 2)).mergeSort(compare); + var left = this.splice(0).mergeSort(compare); // 'this' is now empty. + + // Merge parts together + while (left.length > 0 || right.length > 0) { + this.push( + right.length === 0 ? left.shift() + : left.length === 0 ? right.shift() + : compare(left[0], right[0]) > 0 ? right.shift() + : left.shift()); + } + } + return this; + } + + }); +})(); + diff --git a/couchpotato/static/scripts/library/async.js b/couchpotato/static/scripts/library/async.js new file mode 100644 index 00000000..cb6320d6 --- /dev/null +++ b/couchpotato/static/scripts/library/async.js @@ -0,0 +1,955 @@ +/*global setImmediate: false, setTimeout: false, console: false */ +(function () { + + var async = {}; + + // global on the server, window in the browser + var root, previous_async; + + root = this; + if (root != null) { + previous_async = root.async; + } + + async.noConflict = function () { + root.async = previous_async; + return async; + }; + + function only_once(fn) { + var called = false; + return function() { + if (called) throw new Error("Callback was already called."); + called = true; + fn.apply(root, arguments); + } + } + + //// cross-browser compatiblity functions //// + + var _each = function (arr, iterator) { + if (arr.forEach) { + return arr.forEach(iterator); + } + for (var i = 0; i < arr.length; i += 1) { + iterator(arr[i], i, arr); + } + }; + + var _map = function (arr, iterator) { + if (arr.map) { + return arr.map(iterator); + } + var results = []; + _each(arr, function (x, i, a) { + results.push(iterator(x, i, a)); + }); + return results; + }; + + var _reduce = function (arr, iterator, memo) { + if (arr.reduce) { + return arr.reduce(iterator, memo); + } + _each(arr, function (x, i, a) { + memo = iterator(memo, x, i, a); + }); + return memo; + }; + + var _keys = function (obj) { + if (Object.keys) { + return Object.keys(obj); + } + var keys = []; + for (var k in obj) { + if (obj.hasOwnProperty(k)) { + keys.push(k); + } + } + return keys; + }; + + //// exported async module functions //// + + //// nextTick implementation with browser-compatible fallback //// + if (typeof process === 'undefined' || !(process.nextTick)) { + if (typeof setImmediate === 'function') { + async.nextTick = function (fn) { + // not a direct alias for IE10 compatibility + setImmediate(fn); + }; + async.setImmediate = async.nextTick; + } + else { + async.nextTick = function (fn) { + setTimeout(fn, 0); + }; + async.setImmediate = async.nextTick; + } + } + else { + async.nextTick = process.nextTick; + if (typeof setImmediate !== 'undefined') { + async.setImmediate = setImmediate; + } + else { + async.setImmediate = async.nextTick; + } + } + + async.each = function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length) { + return callback(); + } + var completed = 0; + _each(arr, function (x) { + iterator(x, only_once(function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= arr.length) { + callback(null); + } + } + })); + }); + }; + async.forEach = async.each; + + async.eachSeries = function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length) { + return callback(); + } + var completed = 0; + var iterate = function () { + iterator(arr[completed], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= arr.length) { + callback(null); + } + else { + iterate(); + } + } + }); + }; + iterate(); + }; + async.forEachSeries = async.eachSeries; + + async.eachLimit = function (arr, limit, iterator, callback) { + var fn = _eachLimit(limit); + fn.apply(null, [arr, iterator, callback]); + }; + async.forEachLimit = async.eachLimit; + + var _eachLimit = function (limit) { + + return function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length || limit <= 0) { + return callback(); + } + var completed = 0; + var started = 0; + var running = 0; + + (function replenish () { + if (completed >= arr.length) { + return callback(); + } + + while (running < limit && started < arr.length) { + started += 1; + running += 1; + iterator(arr[started - 1], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + running -= 1; + if (completed >= arr.length) { + callback(); + } + else { + replenish(); + } + } + }); + } + })(); + }; + }; + + + var doParallel = function (fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [async.each].concat(args)); + }; + }; + var doParallelLimit = function(limit, fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [_eachLimit(limit)].concat(args)); + }; + }; + var doSeries = function (fn) { + return function () { + var args = Array.prototype.slice.call(arguments); + return fn.apply(null, [async.eachSeries].concat(args)); + }; + }; + + + var _asyncMap = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (err, v) { + results[x.index] = v; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + }; + async.map = doParallel(_asyncMap); + async.mapSeries = doSeries(_asyncMap); + async.mapLimit = function (arr, limit, iterator, callback) { + return _mapLimit(limit)(arr, iterator, callback); + }; + + var _mapLimit = function(limit) { + return doParallelLimit(limit, _asyncMap); + }; + + // reduce only has a series version, as doing reduce in parallel won't + // work in many situations. + async.reduce = function (arr, memo, iterator, callback) { + async.eachSeries(arr, function (x, callback) { + iterator(memo, x, function (err, v) { + memo = v; + callback(err); + }); + }, function (err) { + callback(err, memo); + }); + }; + // inject alias + async.inject = async.reduce; + // foldl alias + async.foldl = async.reduce; + + async.reduceRight = function (arr, memo, iterator, callback) { + var reversed = _map(arr, function (x) { + return x; + }).reverse(); + async.reduce(reversed, memo, iterator, callback); + }; + // foldr alias + async.foldr = async.reduceRight; + + var _filter = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (v) { + if (v) { + results.push(x); + } + callback(); + }); + }, function (err) { + callback(_map(results.sort(function (a, b) { + return a.index - b.index; + }), function (x) { + return x.value; + })); + }); + }; + async.filter = doParallel(_filter); + async.filterSeries = doSeries(_filter); + // select alias + async.select = async.filter; + async.selectSeries = async.filterSeries; + + var _reject = function (eachfn, arr, iterator, callback) { + var results = []; + arr = _map(arr, function (x, i) { + return {index: i, value: x}; + }); + eachfn(arr, function (x, callback) { + iterator(x.value, function (v) { + if (!v) { + results.push(x); + } + callback(); + }); + }, function (err) { + callback(_map(results.sort(function (a, b) { + return a.index - b.index; + }), function (x) { + return x.value; + })); + }); + }; + async.reject = doParallel(_reject); + async.rejectSeries = doSeries(_reject); + + var _detect = function (eachfn, arr, iterator, main_callback) { + eachfn(arr, function (x, callback) { + iterator(x, function (result) { + if (result) { + main_callback(x); + main_callback = function () {}; + } + else { + callback(); + } + }); + }, function (err) { + main_callback(); + }); + }; + async.detect = doParallel(_detect); + async.detectSeries = doSeries(_detect); + + async.some = function (arr, iterator, main_callback) { + async.each(arr, function (x, callback) { + iterator(x, function (v) { + if (v) { + main_callback(true); + main_callback = function () {}; + } + callback(); + }); + }, function (err) { + main_callback(false); + }); + }; + // any alias + async.any = async.some; + + async.every = function (arr, iterator, main_callback) { + async.each(arr, function (x, callback) { + iterator(x, function (v) { + if (!v) { + main_callback(false); + main_callback = function () {}; + } + callback(); + }); + }, function (err) { + main_callback(true); + }); + }; + // all alias + async.all = async.every; + + async.sortBy = function (arr, iterator, callback) { + async.map(arr, function (x, callback) { + iterator(x, function (err, criteria) { + if (err) { + callback(err); + } + else { + callback(null, {value: x, criteria: criteria}); + } + }); + }, function (err, results) { + if (err) { + return callback(err); + } + else { + var fn = function (left, right) { + var a = left.criteria, b = right.criteria; + return a < b ? -1 : a > b ? 1 : 0; + }; + callback(null, _map(results.sort(fn), function (x) { + return x.value; + })); + } + }); + }; + + async.auto = function (tasks, callback) { + callback = callback || function () {}; + var keys = _keys(tasks); + if (!keys.length) { + return callback(null); + } + + var results = {}; + + var listeners = []; + var addListener = function (fn) { + listeners.unshift(fn); + }; + var removeListener = function (fn) { + for (var i = 0; i < listeners.length; i += 1) { + if (listeners[i] === fn) { + listeners.splice(i, 1); + return; + } + } + }; + var taskComplete = function () { + _each(listeners.slice(0), function (fn) { + fn(); + }); + }; + + addListener(function () { + if (_keys(results).length === keys.length) { + callback(null, results); + callback = function () {}; + } + }); + + _each(keys, function (k) { + var task = (tasks[k] instanceof Function) ? [tasks[k]]: tasks[k]; + var taskCallback = function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + if (err) { + var safeResults = {}; + _each(_keys(results), function(rkey) { + safeResults[rkey] = results[rkey]; + }); + safeResults[k] = args; + callback(err, safeResults); + // stop subsequent errors hitting callback multiple times + callback = function () {}; + } + else { + results[k] = args; + async.setImmediate(taskComplete); + } + }; + var requires = task.slice(0, Math.abs(task.length - 1)) || []; + var ready = function () { + return _reduce(requires, function (a, x) { + return (a && results.hasOwnProperty(x)); + }, true) && !results.hasOwnProperty(k); + }; + if (ready()) { + task[task.length - 1](taskCallback, results); + } + else { + var listener = function () { + if (ready()) { + removeListener(listener); + task[task.length - 1](taskCallback, results); + } + }; + addListener(listener); + } + }); + }; + + async.waterfall = function (tasks, callback) { + callback = callback || function () {}; + if (tasks.constructor !== Array) { + var err = new Error('First argument to waterfall must be an array of functions'); + return callback(err); + } + if (!tasks.length) { + return callback(); + } + var wrapIterator = function (iterator) { + return function (err) { + if (err) { + callback.apply(null, arguments); + callback = function () {}; + } + else { + var args = Array.prototype.slice.call(arguments, 1); + var next = iterator.next(); + if (next) { + args.push(wrapIterator(next)); + } + else { + args.push(callback); + } + async.setImmediate(function () { + iterator.apply(null, args); + }); + } + }; + }; + wrapIterator(async.iterator(tasks))(); + }; + + var _parallel = function(eachfn, tasks, callback) { + callback = callback || function () {}; + if (tasks.constructor === Array) { + eachfn.map(tasks, function (fn, callback) { + if (fn) { + fn(function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + callback.call(null, err, args); + }); + } + }, callback); + } + else { + var results = {}; + eachfn.each(_keys(tasks), function (k, callback) { + tasks[k](function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + results[k] = args; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + + async.parallel = function (tasks, callback) { + _parallel({ map: async.map, each: async.each }, tasks, callback); + }; + + async.parallelLimit = function(tasks, limit, callback) { + _parallel({ map: _mapLimit(limit), each: _eachLimit(limit) }, tasks, callback); + }; + + async.series = function (tasks, callback) { + callback = callback || function () {}; + if (tasks.constructor === Array) { + async.mapSeries(tasks, function (fn, callback) { + if (fn) { + fn(function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + callback.call(null, err, args); + }); + } + }, callback); + } + else { + var results = {}; + async.eachSeries(_keys(tasks), function (k, callback) { + tasks[k](function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (args.length <= 1) { + args = args[0]; + } + results[k] = args; + callback(err); + }); + }, function (err) { + callback(err, results); + }); + } + }; + + async.iterator = function (tasks) { + var makeCallback = function (index) { + var fn = function () { + if (tasks.length) { + tasks[index].apply(null, arguments); + } + return fn.next(); + }; + fn.next = function () { + return (index < tasks.length - 1) ? makeCallback(index + 1): null; + }; + return fn; + }; + return makeCallback(0); + }; + + async.apply = function (fn) { + var args = Array.prototype.slice.call(arguments, 1); + return function () { + return fn.apply( + null, args.concat(Array.prototype.slice.call(arguments)) + ); + }; + }; + + var _concat = function (eachfn, arr, fn, callback) { + var r = []; + eachfn(arr, function (x, cb) { + fn(x, function (err, y) { + r = r.concat(y || []); + cb(err); + }); + }, function (err) { + callback(err, r); + }); + }; + async.concat = doParallel(_concat); + async.concatSeries = doSeries(_concat); + + async.whilst = function (test, iterator, callback) { + if (test()) { + iterator(function (err) { + if (err) { + return callback(err); + } + async.whilst(test, iterator, callback); + }); + } + else { + callback(); + } + }; + + async.doWhilst = function (iterator, test, callback) { + iterator(function (err) { + if (err) { + return callback(err); + } + if (test()) { + async.doWhilst(iterator, test, callback); + } + else { + callback(); + } + }); + }; + + async.until = function (test, iterator, callback) { + if (!test()) { + iterator(function (err) { + if (err) { + return callback(err); + } + async.until(test, iterator, callback); + }); + } + else { + callback(); + } + }; + + async.doUntil = function (iterator, test, callback) { + iterator(function (err) { + if (err) { + return callback(err); + } + if (!test()) { + async.doUntil(iterator, test, callback); + } + else { + callback(); + } + }); + }; + + async.queue = function (worker, concurrency) { + if (concurrency === undefined) { + concurrency = 1; + } + function _insert(q, data, pos, callback) { + if(data.constructor !== Array) { + data = [data]; + } + _each(data, function(task) { + var item = { + data: task, + callback: typeof callback === 'function' ? callback : null + }; + + if (pos) { + q.tasks.unshift(item); + } else { + q.tasks.push(item); + } + + if (q.saturated && q.tasks.length === concurrency) { + q.saturated(); + } + async.setImmediate(q.process); + }); + } + + var workers = 0; + var q = { + tasks: [], + concurrency: concurrency, + saturated: null, + empty: null, + drain: null, + push: function (data, callback) { + _insert(q, data, false, callback); + }, + unshift: function (data, callback) { + _insert(q, data, true, callback); + }, + process: function () { + if (workers < q.concurrency && q.tasks.length) { + var task = q.tasks.shift(); + if (q.empty && q.tasks.length === 0) { + q.empty(); + } + workers += 1; + var next = function () { + workers -= 1; + if (task.callback) { + task.callback.apply(task, arguments); + } + if (q.drain && q.tasks.length + workers === 0) { + q.drain(); + } + q.process(); + }; + var cb = only_once(next); + worker(task.data, cb); + } + }, + length: function () { + return q.tasks.length; + }, + running: function () { + return workers; + } + }; + return q; + }; + + async.cargo = function (worker, payload) { + var working = false, + tasks = []; + + var cargo = { + tasks: tasks, + payload: payload, + saturated: null, + empty: null, + drain: null, + push: function (data, callback) { + if(data.constructor !== Array) { + data = [data]; + } + _each(data, function(task) { + tasks.push({ + data: task, + callback: typeof callback === 'function' ? callback : null + }); + if (cargo.saturated && tasks.length === payload) { + cargo.saturated(); + } + }); + async.setImmediate(cargo.process); + }, + process: function process() { + if (working) return; + if (tasks.length === 0) { + if(cargo.drain) cargo.drain(); + return; + } + + var ts = typeof payload === 'number' + ? tasks.splice(0, payload) + : tasks.splice(0); + + var ds = _map(ts, function (task) { + return task.data; + }); + + if(cargo.empty) cargo.empty(); + working = true; + worker(ds, function () { + working = false; + + var args = arguments; + _each(ts, function (data) { + if (data.callback) { + data.callback.apply(null, args); + } + }); + + process(); + }); + }, + length: function () { + return tasks.length; + }, + running: function () { + return working; + } + }; + return cargo; + }; + + var _console_fn = function (name) { + return function (fn) { + var args = Array.prototype.slice.call(arguments, 1); + fn.apply(null, args.concat([function (err) { + var args = Array.prototype.slice.call(arguments, 1); + if (typeof console !== 'undefined') { + if (err) { + if (console.error) { + console.error(err); + } + } + else if (console[name]) { + _each(args, function (x) { + console[name](x); + }); + } + } + }])); + }; + }; + async.log = _console_fn('log'); + async.dir = _console_fn('dir'); + /*async.info = _console_fn('info'); + async.warn = _console_fn('warn'); + async.error = _console_fn('error');*/ + + async.memoize = function (fn, hasher) { + var memo = {}; + var queues = {}; + hasher = hasher || function (x) { + return x; + }; + var memoized = function () { + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + var key = hasher.apply(null, args); + if (key in memo) { + callback.apply(null, memo[key]); + } + else if (key in queues) { + queues[key].push(callback); + } + else { + queues[key] = [callback]; + fn.apply(null, args.concat([function () { + memo[key] = arguments; + var q = queues[key]; + delete queues[key]; + for (var i = 0, l = q.length; i < l; i++) { + q[i].apply(null, arguments); + } + }])); + } + }; + memoized.memo = memo; + memoized.unmemoized = fn; + return memoized; + }; + + async.unmemoize = function (fn) { + return function () { + return (fn.unmemoized || fn).apply(null, arguments); + }; + }; + + async.times = function (count, iterator, callback) { + var counter = []; + for (var i = 0; i < count; i++) { + counter.push(i); + } + return async.map(counter, iterator, callback); + }; + + async.timesSeries = function (count, iterator, callback) { + var counter = []; + for (var i = 0; i < count; i++) { + counter.push(i); + } + return async.mapSeries(counter, iterator, callback); + }; + + async.compose = function (/* functions... */) { + var fns = Array.prototype.reverse.call(arguments); + return function () { + var that = this; + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + async.reduce(fns, args, function (newargs, fn, cb) { + fn.apply(that, newargs.concat([function () { + var err = arguments[0]; + var nextargs = Array.prototype.slice.call(arguments, 1); + cb(err, nextargs); + }])) + }, + function (err, results) { + callback.apply(that, [err].concat(results)); + }); + }; + }; + + var _applyEach = function (eachfn, fns /*args...*/) { + var go = function () { + var that = this; + var args = Array.prototype.slice.call(arguments); + var callback = args.pop(); + return eachfn(fns, function (fn, cb) { + fn.apply(that, args.concat([cb])); + }, + callback); + }; + if (arguments.length > 2) { + var args = Array.prototype.slice.call(arguments, 2); + return go.apply(this, args); + } + else { + return go; + } + }; + async.applyEach = doParallel(_applyEach); + async.applyEachSeries = doSeries(_applyEach); + + async.forever = function (fn, callback) { + function next(err) { + if (err) { + if (callback) { + return callback(err); + } + throw err; + } + fn(next); + } + next(); + }; + + // AMD / RequireJS + if (typeof define !== 'undefined' && define.amd) { + define([], function () { + return async; + }); + } + // Node.js + else if (typeof module !== 'undefined' && module.exports) { + module.exports = async; + } + // included directly via +""" + soup = BeautifulSoup(doc, "xml") + # lxml would have stripped this while parsing, but we can add + # it later. + soup.script.string = 'console.log("< < hey > > ");' + encoded = soup.encode() + self.assertTrue(b"< < hey > >" in encoded) + + def test_can_parse_unicode_document(self): + markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!' + soup = self.soup(markup) + self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string) + + def test_popping_namespaced_tag(self): + markup = 'b2012-07-02T20:33:42Zcd' + soup = self.soup(markup) + self.assertEqual( + unicode(soup.rss), markup) def test_docstring_includes_correct_encoding(self): soup = self.soup("") @@ -472,6 +529,20 @@ class XMLTreeBuilderSmokeTest(object): self.assertEqual("http://example.com/", root['xmlns:a']) self.assertEqual("http://example.net/", root['xmlns:b']) + def test_closing_namespaced_tag(self): + markup = '

20010504

' + soup = self.soup(markup) + self.assertEqual(unicode(soup.p), markup) + + def test_namespaced_attributes(self): + markup = '' + soup = self.soup(markup) + self.assertEqual(unicode(soup.foo), markup) + + def test_namespaced_attributes_xml_namespace(self): + markup = 'bar' + soup = self.soup(markup) + self.assertEqual(unicode(soup.foo), markup) class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest): """Smoke test for a tree builder that supports HTML5.""" @@ -501,6 +572,12 @@ class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest): self.assertEqual(namespace, soup.math.namespace) self.assertEqual(namespace, soup.msqrt.namespace) + def test_xml_declaration_becomes_comment(self): + markup = '' + soup = self.soup(markup) + self.assertTrue(isinstance(soup.contents[0], Comment)) + self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?') + self.assertEqual("html", soup.contents[0].next_element.name) def skipIf(condition, reason): def nothing(test, *args, **kwargs): diff --git a/libs/gntp/__init__.py b/libs/gntp/__init__.py index eabbfa47..e69de29b 100755 --- a/libs/gntp/__init__.py +++ b/libs/gntp/__init__.py @@ -1,509 +0,0 @@ -import re -import hashlib -import time -import StringIO - -__version__ = '0.8' - -#GNTP/ [:][ :.] -GNTP_INFO_LINE = re.compile( - 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)' + - ' (?P[A-Z0-9]+(:(?P[A-F0-9]+))?) ?' + - '((?P[A-Z0-9]+):(?P[A-F0-9]+).(?P[A-F0-9]+))?\r\n', - re.IGNORECASE -) - -GNTP_INFO_LINE_SHORT = re.compile( - 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)', - re.IGNORECASE -) - -GNTP_HEADER = re.compile('([\w-]+):(.+)') - -GNTP_EOL = '\r\n' - - -class BaseError(Exception): - def gntp_error(self): - error = GNTPError(self.errorcode, self.errordesc) - return error.encode() - - -class ParseError(BaseError): - errorcode = 500 - errordesc = 'Error parsing the message' - - -class AuthError(BaseError): - errorcode = 400 - errordesc = 'Error with authorization' - - -class UnsupportedError(BaseError): - errorcode = 500 - errordesc = 'Currently unsupported by gntp.py' - - -class _GNTPBuffer(StringIO.StringIO): - """GNTP Buffer class""" - def writefmt(self, message = "", *args): - """Shortcut function for writing GNTP Headers""" - self.write((message % args).encode('utf8', 'replace')) - self.write(GNTP_EOL) - - -class _GNTPBase(object): - """Base initilization - - :param string messagetype: GNTP Message type - :param string version: GNTP Protocol version - :param string encription: Encryption protocol - """ - def __init__(self, messagetype = None, version = '1.0', encryption = None): - self.info = { - 'version': version, - 'messagetype': messagetype, - 'encryptionAlgorithmID': encryption - } - self.headers = {} - self.resources = {} - - def __str__(self): - return self.encode() - - def _parse_info(self, data): - """Parse the first line of a GNTP message to get security and other info values - - :param string data: GNTP Message - :return dict: Parsed GNTP Info line - """ - - match = GNTP_INFO_LINE.match(data) - - if not match: - raise ParseError('ERROR_PARSING_INFO_LINE') - - info = match.groupdict() - if info['encryptionAlgorithmID'] == 'NONE': - info['encryptionAlgorithmID'] = None - - return info - - def set_password(self, password, encryptAlgo = 'MD5'): - """Set a password for a GNTP Message - - :param string password: Null to clear password - :param string encryptAlgo: Supports MD5, SHA1, SHA256, SHA512 - """ - hash = { - 'MD5': hashlib.md5, - 'SHA1': hashlib.sha1, - 'SHA256': hashlib.sha256, - 'SHA512': hashlib.sha512, - } - - self.password = password - self.encryptAlgo = encryptAlgo.upper() - if not password: - self.info['encryptionAlgorithmID'] = None - self.info['keyHashAlgorithm'] = None - return - if not self.encryptAlgo in hash.keys(): - raise UnsupportedError('INVALID HASH "%s"' % self.encryptAlgo) - - hashfunction = hash.get(self.encryptAlgo) - - password = password.encode('utf8') - seed = time.ctime() - salt = hashfunction(seed).hexdigest() - saltHash = hashfunction(seed).digest() - keyBasis = password + saltHash - key = hashfunction(keyBasis).digest() - keyHash = hashfunction(key).hexdigest() - - self.info['keyHashAlgorithmID'] = self.encryptAlgo - self.info['keyHash'] = keyHash.upper() - self.info['salt'] = salt.upper() - - def _decode_hex(self, value): - """Helper function to decode hex string to `proper` hex string - - :param string value: Human readable hex string - :return string: Hex string - """ - result = '' - for i in range(0, len(value), 2): - tmp = int(value[i:i + 2], 16) - result += chr(tmp) - return result - - def _decode_binary(self, rawIdentifier, identifier): - rawIdentifier += '\r\n\r\n' - dataLength = int(identifier['Length']) - pointerStart = self.raw.find(rawIdentifier) + len(rawIdentifier) - pointerEnd = pointerStart + dataLength - data = self.raw[pointerStart:pointerEnd] - if not len(data) == dataLength: - raise ParseError('INVALID_DATA_LENGTH Expected: %s Recieved %s' % (dataLength, len(data))) - return data - - def _validate_password(self, password): - """Validate GNTP Message against stored password""" - self.password = password - if password == None: - raise AuthError('Missing password') - keyHash = self.info.get('keyHash', None) - if keyHash is None and self.password is None: - return True - if keyHash is None: - raise AuthError('Invalid keyHash') - if self.password is None: - raise AuthError('Missing password') - - password = self.password.encode('utf8') - saltHash = self._decode_hex(self.info['salt']) - - keyBasis = password + saltHash - key = hashlib.md5(keyBasis).digest() - keyHash = hashlib.md5(key).hexdigest() - - if not keyHash.upper() == self.info['keyHash'].upper(): - raise AuthError('Invalid Hash') - return True - - def validate(self): - """Verify required headers""" - for header in self._requiredHeaders: - if not self.headers.get(header, False): - raise ParseError('Missing Notification Header: ' + header) - - def _format_info(self): - """Generate info line for GNTP Message - - :return string: - """ - info = u'GNTP/%s %s' % ( - self.info.get('version'), - self.info.get('messagetype'), - ) - if self.info.get('encryptionAlgorithmID', None): - info += ' %s:%s' % ( - self.info.get('encryptionAlgorithmID'), - self.info.get('ivValue'), - ) - else: - info += ' NONE' - - if self.info.get('keyHashAlgorithmID', None): - info += ' %s:%s.%s' % ( - self.info.get('keyHashAlgorithmID'), - self.info.get('keyHash'), - self.info.get('salt') - ) - - return info - - def _parse_dict(self, data): - """Helper function to parse blocks of GNTP headers into a dictionary - - :param string data: - :return dict: - """ - dict = {} - for line in data.split('\r\n'): - match = GNTP_HEADER.match(line) - if not match: - continue - - key = unicode(match.group(1).strip(), 'utf8', 'replace') - val = unicode(match.group(2).strip(), 'utf8', 'replace') - dict[key] = val - return dict - - def add_header(self, key, value): - if isinstance(value, unicode): - self.headers[key] = value - else: - self.headers[key] = unicode('%s' % value, 'utf8', 'replace') - - def add_resource(self, data): - """Add binary resource - - :param string data: Binary Data - """ - identifier = hashlib.md5(data).hexdigest() - self.resources[identifier] = data - return 'x-growl-resource://%s' % identifier - - def decode(self, data, password = None): - """Decode GNTP Message - - :param string data: - """ - self.password = password - self.raw = data - parts = self.raw.split('\r\n\r\n') - self.info = self._parse_info(data) - self.headers = self._parse_dict(parts[0]) - - def encode(self): - """Encode a generic GNTP Message - - :return string: GNTP Message ready to be sent - """ - - buffer = _GNTPBuffer() - - buffer.writefmt(self._format_info()) - - #Headers - for k, v in self.headers.iteritems(): - buffer.writefmt('%s: %s', k, v) - buffer.writefmt() - - #Resources - for resource, data in self.resources.iteritems(): - buffer.writefmt('Identifier: %s', resource) - buffer.writefmt('Length: %d', len(data)) - buffer.writefmt() - buffer.write(data) - buffer.writefmt() - buffer.writefmt() - - return buffer.getvalue() - - -class GNTPRegister(_GNTPBase): - """Represents a GNTP Registration Command - - :param string data: (Optional) See decode() - :param string password: (Optional) Password to use while encoding/decoding messages - """ - _requiredHeaders = [ - 'Application-Name', - 'Notifications-Count' - ] - _requiredNotificationHeaders = ['Notification-Name'] - - def __init__(self, data = None, password = None): - _GNTPBase.__init__(self, 'REGISTER') - self.notifications = [] - - if data: - self.decode(data, password) - else: - self.set_password(password) - self.add_header('Application-Name', 'pygntp') - self.add_header('Notifications-Count', 0) - - def validate(self): - '''Validate required headers and validate notification headers''' - for header in self._requiredHeaders: - if not self.headers.get(header, False): - raise ParseError('Missing Registration Header: ' + header) - for notice in self.notifications: - for header in self._requiredNotificationHeaders: - if not notice.get(header, False): - raise ParseError('Missing Notification Header: ' + header) - - def decode(self, data, password): - """Decode existing GNTP Registration message - - :param string data: Message to decode - """ - self.raw = data - parts = self.raw.split('\r\n\r\n') - self.info = self._parse_info(data) - self._validate_password(password) - self.headers = self._parse_dict(parts[0]) - - for i, part in enumerate(parts): - if i == 0: - continue # Skip Header - if part.strip() == '': - continue - notice = self._parse_dict(part) - if notice.get('Notification-Name', False): - self.notifications.append(notice) - elif notice.get('Identifier', False): - notice['Data'] = self._decode_binary(part, notice) - #open('register.png','wblol').write(notice['Data']) - self.resources[notice.get('Identifier')] = notice - - def add_notification(self, name, enabled = True): - """Add new Notification to Registration message - - :param string name: Notification Name - :param boolean enabled: Enable this notification by default - """ - notice = {} - notice['Notification-Name'] = u'%s' % name - notice['Notification-Enabled'] = u'%s' % enabled - - self.notifications.append(notice) - self.add_header('Notifications-Count', len(self.notifications)) - - def encode(self): - """Encode a GNTP Registration Message - - :return string: Encoded GNTP Registration message - """ - - buffer = _GNTPBuffer() - - buffer.writefmt(self._format_info()) - - #Headers - for k, v in self.headers.iteritems(): - buffer.writefmt('%s: %s', k, v) - buffer.writefmt() - - #Notifications - if len(self.notifications) > 0: - for notice in self.notifications: - for k, v in notice.iteritems(): - buffer.writefmt('%s: %s', k, v) - buffer.writefmt() - - #Resources - for resource, data in self.resources.iteritems(): - buffer.writefmt('Identifier: %s', resource) - buffer.writefmt('Length: %d', len(data)) - buffer.writefmt() - buffer.write(data) - buffer.writefmt() - buffer.writefmt() - - return buffer.getvalue() - - -class GNTPNotice(_GNTPBase): - """Represents a GNTP Notification Command - - :param string data: (Optional) See decode() - :param string app: (Optional) Set Application-Name - :param string name: (Optional) Set Notification-Name - :param string title: (Optional) Set Notification Title - :param string password: (Optional) Password to use while encoding/decoding messages - """ - _requiredHeaders = [ - 'Application-Name', - 'Notification-Name', - 'Notification-Title' - ] - - def __init__(self, data = None, app = None, name = None, title = None, password = None): - _GNTPBase.__init__(self, 'NOTIFY') - - if data: - self.decode(data, password) - else: - self.set_password(password) - if app: - self.add_header('Application-Name', app) - if name: - self.add_header('Notification-Name', name) - if title: - self.add_header('Notification-Title', title) - - def decode(self, data, password): - """Decode existing GNTP Notification message - - :param string data: Message to decode. - """ - self.raw = data - parts = self.raw.split('\r\n\r\n') - self.info = self._parse_info(data) - self._validate_password(password) - self.headers = self._parse_dict(parts[0]) - - for i, part in enumerate(parts): - if i == 0: - continue # Skip Header - if part.strip() == '': - continue - notice = self._parse_dict(part) - if notice.get('Identifier', False): - notice['Data'] = self._decode_binary(part, notice) - #open('notice.png','wblol').write(notice['Data']) - self.resources[notice.get('Identifier')] = notice - - -class GNTPSubscribe(_GNTPBase): - """Represents a GNTP Subscribe Command - - :param string data: (Optional) See decode() - :param string password: (Optional) Password to use while encoding/decoding messages - """ - _requiredHeaders = [ - 'Subscriber-ID', - 'Subscriber-Name', - ] - - def __init__(self, data = None, password = None): - _GNTPBase.__init__(self, 'SUBSCRIBE') - if data: - self.decode(data, password) - else: - self.set_password(password) - - -class GNTPOK(_GNTPBase): - """Represents a GNTP OK Response - - :param string data: (Optional) See _GNTPResponse.decode() - :param string action: (Optional) Set type of action the OK Response is for - """ - _requiredHeaders = ['Response-Action'] - - def __init__(self, data = None, action = None): - _GNTPBase.__init__(self, '-OK') - if data: - self.decode(data) - if action: - self.add_header('Response-Action', action) - - -class GNTPError(_GNTPBase): - """Represents a GNTP Error response - - :param string data: (Optional) See _GNTPResponse.decode() - :param string errorcode: (Optional) Error code - :param string errordesc: (Optional) Error Description - """ - _requiredHeaders = ['Error-Code', 'Error-Description'] - - def __init__(self, data = None, errorcode = None, errordesc = None): - _GNTPBase.__init__(self, '-ERROR') - if data: - self.decode(data) - if errorcode: - self.add_header('Error-Code', errorcode) - self.add_header('Error-Description', errordesc) - - def error(self): - return (self.headers.get('Error-Code', None), - self.headers.get('Error-Description', None)) - - -def parse_gntp(data, password = None): - """Attempt to parse a message as a GNTP message - - :param string data: Message to be parsed - :param string password: Optional password to be used to verify the message - """ - match = GNTP_INFO_LINE_SHORT.match(data) - if not match: - raise ParseError('INVALID_GNTP_INFO') - info = match.groupdict() - if info['messagetype'] == 'REGISTER': - return GNTPRegister(data, password = password) - elif info['messagetype'] == 'NOTIFY': - return GNTPNotice(data, password = password) - elif info['messagetype'] == 'SUBSCRIBE': - return GNTPSubscribe(data, password = password) - elif info['messagetype'] == '-OK': - return GNTPOK(data) - elif info['messagetype'] == '-ERROR': - return GNTPError(data) - raise ParseError('INVALID_GNTP_MESSAGE') diff --git a/libs/gntp/cli.py b/libs/gntp/cli.py new file mode 100644 index 00000000..bc083062 --- /dev/null +++ b/libs/gntp/cli.py @@ -0,0 +1,141 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +import logging +import os +import sys +from optparse import OptionParser, OptionGroup + +from gntp.notifier import GrowlNotifier +from gntp.shim import RawConfigParser +from gntp.version import __version__ + +DEFAULT_CONFIG = os.path.expanduser('~/.gntp') + +config = RawConfigParser({ + 'hostname': 'localhost', + 'password': None, + 'port': 23053, +}) +config.read([DEFAULT_CONFIG]) +if not config.has_section('gntp'): + config.add_section('gntp') + + +class ClientParser(OptionParser): + def __init__(self): + OptionParser.__init__(self, version="%%prog %s" % __version__) + + group = OptionGroup(self, "Network Options") + group.add_option("-H", "--host", + dest="host", default=config.get('gntp', 'hostname'), + help="Specify a hostname to which to send a remote notification. [%default]") + group.add_option("--port", + dest="port", default=config.getint('gntp', 'port'), type="int", + help="port to listen on [%default]") + group.add_option("-P", "--password", + dest='password', default=config.get('gntp', 'password'), + help="Network password") + self.add_option_group(group) + + group = OptionGroup(self, "Notification Options") + group.add_option("-n", "--name", + dest="app", default='Python GNTP Test Client', + help="Set the name of the application [%default]") + group.add_option("-s", "--sticky", + dest='sticky', default=False, action="store_true", + help="Make the notification sticky [%default]") + group.add_option("--image", + dest="icon", default=None, + help="Icon for notification (URL or /path/to/file)") + group.add_option("-m", "--message", + dest="message", default=None, + help="Sets the message instead of using stdin") + group.add_option("-p", "--priority", + dest="priority", default=0, type="int", + help="-2 to 2 [%default]") + group.add_option("-d", "--identifier", + dest="identifier", + help="Identifier for coalescing") + group.add_option("-t", "--title", + dest="title", default=None, + help="Set the title of the notification [%default]") + group.add_option("-N", "--notification", + dest="name", default='Notification', + help="Set the notification name [%default]") + group.add_option("--callback", + dest="callback", + help="URL callback") + self.add_option_group(group) + + # Extra Options + self.add_option('-v', '--verbose', + dest='verbose', default=0, action='count', + help="Verbosity levels") + + def parse_args(self, args=None, values=None): + values, args = OptionParser.parse_args(self, args, values) + + if values.message is None: + print('Enter a message followed by Ctrl-D') + try: + message = sys.stdin.read() + except KeyboardInterrupt: + exit() + else: + message = values.message + + if values.title is None: + values.title = ' '.join(args) + + # If we still have an empty title, use the + # first bit of the message as the title + if values.title == '': + values.title = message[:20] + + values.verbose = logging.WARNING - values.verbose * 10 + + return values, message + + +def main(): + (options, message) = ClientParser().parse_args() + logging.basicConfig(level=options.verbose) + if not os.path.exists(DEFAULT_CONFIG): + logging.info('No config read found at %s', DEFAULT_CONFIG) + + growl = GrowlNotifier( + applicationName=options.app, + notifications=[options.name], + defaultNotifications=[options.name], + hostname=options.host, + password=options.password, + port=options.port, + ) + result = growl.register() + if result is not True: + exit(result) + + # This would likely be better placed within the growl notifier + # class but until I make _checkIcon smarter this is "easier" + if options.icon is not None and not options.icon.startswith('http'): + logging.info('Loading image %s', options.icon) + f = open(options.icon) + options.icon = f.read() + f.close() + + result = growl.notify( + noteType=options.name, + title=options.title, + description=message, + icon=options.icon, + sticky=options.sticky, + priority=options.priority, + callback=options.callback, + identifier=options.identifier, + ) + if result is not True: + exit(result) + +if __name__ == "__main__": + main() diff --git a/libs/gntp/config.py b/libs/gntp/config.py new file mode 100644 index 00000000..7536bd14 --- /dev/null +++ b/libs/gntp/config.py @@ -0,0 +1,77 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +""" +The gntp.config module is provided as an extended GrowlNotifier object that takes +advantage of the ConfigParser module to allow us to setup some default values +(such as hostname, password, and port) in a more global way to be shared among +programs using gntp +""" +import logging +import os + +import gntp.notifier +import gntp.shim + +__all__ = [ + 'mini', + 'GrowlNotifier' +] + +logger = logging.getLogger(__name__) + + +class GrowlNotifier(gntp.notifier.GrowlNotifier): + """ + ConfigParser enhanced GrowlNotifier object + + For right now, we are only interested in letting users overide certain + values from ~/.gntp + + :: + + [gntp] + hostname = ? + password = ? + port = ? + """ + def __init__(self, *args, **kwargs): + config = gntp.shim.RawConfigParser({ + 'hostname': kwargs.get('hostname', 'localhost'), + 'password': kwargs.get('password'), + 'port': kwargs.get('port', 23053), + }) + + config.read([os.path.expanduser('~/.gntp')]) + + # If the file does not exist, then there will be no gntp section defined + # and the config.get() lines below will get confused. Since we are not + # saving the config, it should be safe to just add it here so the + # code below doesn't complain + if not config.has_section('gntp'): + logger.info('Error reading ~/.gntp config file') + config.add_section('gntp') + + kwargs['password'] = config.get('gntp', 'password') + kwargs['hostname'] = config.get('gntp', 'hostname') + kwargs['port'] = config.getint('gntp', 'port') + + super(GrowlNotifier, self).__init__(*args, **kwargs) + + +def mini(description, **kwargs): + """Single notification function + + Simple notification function in one line. Has only one required parameter + and attempts to use reasonable defaults for everything else + :param string description: Notification message + """ + kwargs['notifierFactory'] = GrowlNotifier + gntp.notifier.mini(description, **kwargs) + + +if __name__ == '__main__': + # If we're running this module directly we're likely running it as a test + # so extra debugging is useful + logging.basicConfig(level=logging.INFO) + mini('Testing mini notification') diff --git a/libs/gntp/core.py b/libs/gntp/core.py new file mode 100644 index 00000000..ee544d3d --- /dev/null +++ b/libs/gntp/core.py @@ -0,0 +1,511 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +import hashlib +import re +import time + +import gntp.shim +import gntp.errors as errors + +__all__ = [ + 'GNTPRegister', + 'GNTPNotice', + 'GNTPSubscribe', + 'GNTPOK', + 'GNTPError', + 'parse_gntp', +] + +#GNTP/ [:][ :.] +GNTP_INFO_LINE = re.compile( + 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)' + + ' (?P[A-Z0-9]+(:(?P[A-F0-9]+))?) ?' + + '((?P[A-Z0-9]+):(?P[A-F0-9]+).(?P[A-F0-9]+))?\r\n', + re.IGNORECASE +) + +GNTP_INFO_LINE_SHORT = re.compile( + 'GNTP/(?P\d+\.\d+) (?PREGISTER|NOTIFY|SUBSCRIBE|\-OK|\-ERROR)', + re.IGNORECASE +) + +GNTP_HEADER = re.compile('([\w-]+):(.+)') + +GNTP_EOL = gntp.shim.b('\r\n') +GNTP_SEP = gntp.shim.b(': ') + + +class _GNTPBuffer(gntp.shim.StringIO): + """GNTP Buffer class""" + def writeln(self, value=None): + if value: + self.write(gntp.shim.b(value)) + self.write(GNTP_EOL) + + def writeheader(self, key, value): + if not isinstance(value, str): + value = str(value) + self.write(gntp.shim.b(key)) + self.write(GNTP_SEP) + self.write(gntp.shim.b(value)) + self.write(GNTP_EOL) + + +class _GNTPBase(object): + """Base initilization + + :param string messagetype: GNTP Message type + :param string version: GNTP Protocol version + :param string encription: Encryption protocol + """ + def __init__(self, messagetype=None, version='1.0', encryption=None): + self.info = { + 'version': version, + 'messagetype': messagetype, + 'encryptionAlgorithmID': encryption + } + self.hash_algo = { + 'MD5': hashlib.md5, + 'SHA1': hashlib.sha1, + 'SHA256': hashlib.sha256, + 'SHA512': hashlib.sha512, + } + self.headers = {} + self.resources = {} + + def __str__(self): + return self.encode() + + def _parse_info(self, data): + """Parse the first line of a GNTP message to get security and other info values + + :param string data: GNTP Message + :return dict: Parsed GNTP Info line + """ + + match = GNTP_INFO_LINE.match(data) + + if not match: + raise errors.ParseError('ERROR_PARSING_INFO_LINE') + + info = match.groupdict() + if info['encryptionAlgorithmID'] == 'NONE': + info['encryptionAlgorithmID'] = None + + return info + + def set_password(self, password, encryptAlgo='MD5'): + """Set a password for a GNTP Message + + :param string password: Null to clear password + :param string encryptAlgo: Supports MD5, SHA1, SHA256, SHA512 + """ + if not password: + self.info['encryptionAlgorithmID'] = None + self.info['keyHashAlgorithm'] = None + return + + self.password = gntp.shim.b(password) + self.encryptAlgo = encryptAlgo.upper() + + if not self.encryptAlgo in self.hash_algo: + raise errors.UnsupportedError('INVALID HASH "%s"' % self.encryptAlgo) + + hashfunction = self.hash_algo.get(self.encryptAlgo) + + password = password.encode('utf8') + seed = time.ctime().encode('utf8') + salt = hashfunction(seed).hexdigest() + saltHash = hashfunction(seed).digest() + keyBasis = password + saltHash + key = hashfunction(keyBasis).digest() + keyHash = hashfunction(key).hexdigest() + + self.info['keyHashAlgorithmID'] = self.encryptAlgo + self.info['keyHash'] = keyHash.upper() + self.info['salt'] = salt.upper() + + def _decode_hex(self, value): + """Helper function to decode hex string to `proper` hex string + + :param string value: Human readable hex string + :return string: Hex string + """ + result = '' + for i in range(0, len(value), 2): + tmp = int(value[i:i + 2], 16) + result += chr(tmp) + return result + + def _decode_binary(self, rawIdentifier, identifier): + rawIdentifier += '\r\n\r\n' + dataLength = int(identifier['Length']) + pointerStart = self.raw.find(rawIdentifier) + len(rawIdentifier) + pointerEnd = pointerStart + dataLength + data = self.raw[pointerStart:pointerEnd] + if not len(data) == dataLength: + raise errors.ParseError('INVALID_DATA_LENGTH Expected: %s Recieved %s' % (dataLength, len(data))) + return data + + def _validate_password(self, password): + """Validate GNTP Message against stored password""" + self.password = password + if password is None: + raise errors.AuthError('Missing password') + keyHash = self.info.get('keyHash', None) + if keyHash is None and self.password is None: + return True + if keyHash is None: + raise errors.AuthError('Invalid keyHash') + if self.password is None: + raise errors.AuthError('Missing password') + + keyHashAlgorithmID = self.info.get('keyHashAlgorithmID','MD5') + + password = self.password.encode('utf8') + saltHash = self._decode_hex(self.info['salt']) + + keyBasis = password + saltHash + self.key = self.hash_algo[keyHashAlgorithmID](keyBasis).digest() + keyHash = self.hash_algo[keyHashAlgorithmID](self.key).hexdigest() + + if not keyHash.upper() == self.info['keyHash'].upper(): + raise errors.AuthError('Invalid Hash') + return True + + def validate(self): + """Verify required headers""" + for header in self._requiredHeaders: + if not self.headers.get(header, False): + raise errors.ParseError('Missing Notification Header: ' + header) + + def _format_info(self): + """Generate info line for GNTP Message + + :return string: + """ + info = 'GNTP/%s %s' % ( + self.info.get('version'), + self.info.get('messagetype'), + ) + if self.info.get('encryptionAlgorithmID', None): + info += ' %s:%s' % ( + self.info.get('encryptionAlgorithmID'), + self.info.get('ivValue'), + ) + else: + info += ' NONE' + + if self.info.get('keyHashAlgorithmID', None): + info += ' %s:%s.%s' % ( + self.info.get('keyHashAlgorithmID'), + self.info.get('keyHash'), + self.info.get('salt') + ) + + return info + + def _parse_dict(self, data): + """Helper function to parse blocks of GNTP headers into a dictionary + + :param string data: + :return dict: Dictionary of parsed GNTP Headers + """ + d = {} + for line in data.split('\r\n'): + match = GNTP_HEADER.match(line) + if not match: + continue + + key = match.group(1).strip() + val = match.group(2).strip() + d[key] = val + return d + + def add_header(self, key, value): + self.headers[key] = value + + def add_resource(self, data): + """Add binary resource + + :param string data: Binary Data + """ + data = gntp.shim.b(data) + identifier = hashlib.md5(data).hexdigest() + self.resources[identifier] = data + return 'x-growl-resource://%s' % identifier + + def decode(self, data, password=None): + """Decode GNTP Message + + :param string data: + """ + self.password = password + self.raw = gntp.shim.u(data) + parts = self.raw.split('\r\n\r\n') + self.info = self._parse_info(self.raw) + self.headers = self._parse_dict(parts[0]) + + def encode(self): + """Encode a generic GNTP Message + + :return string: GNTP Message ready to be sent. Returned as a byte string + """ + + buff = _GNTPBuffer() + + buff.writeln(self._format_info()) + + #Headers + for k, v in self.headers.items(): + buff.writeheader(k, v) + buff.writeln() + + #Resources + for resource, data in self.resources.items(): + buff.writeheader('Identifier', resource) + buff.writeheader('Length', len(data)) + buff.writeln() + buff.write(data) + buff.writeln() + buff.writeln() + + return buff.getvalue() + + +class GNTPRegister(_GNTPBase): + """Represents a GNTP Registration Command + + :param string data: (Optional) See decode() + :param string password: (Optional) Password to use while encoding/decoding messages + """ + _requiredHeaders = [ + 'Application-Name', + 'Notifications-Count' + ] + _requiredNotificationHeaders = ['Notification-Name'] + + def __init__(self, data=None, password=None): + _GNTPBase.__init__(self, 'REGISTER') + self.notifications = [] + + if data: + self.decode(data, password) + else: + self.set_password(password) + self.add_header('Application-Name', 'pygntp') + self.add_header('Notifications-Count', 0) + + def validate(self): + '''Validate required headers and validate notification headers''' + for header in self._requiredHeaders: + if not self.headers.get(header, False): + raise errors.ParseError('Missing Registration Header: ' + header) + for notice in self.notifications: + for header in self._requiredNotificationHeaders: + if not notice.get(header, False): + raise errors.ParseError('Missing Notification Header: ' + header) + + def decode(self, data, password): + """Decode existing GNTP Registration message + + :param string data: Message to decode + """ + self.raw = gntp.shim.u(data) + parts = self.raw.split('\r\n\r\n') + self.info = self._parse_info(self.raw) + self._validate_password(password) + self.headers = self._parse_dict(parts[0]) + + for i, part in enumerate(parts): + if i == 0: + continue # Skip Header + if part.strip() == '': + continue + notice = self._parse_dict(part) + if notice.get('Notification-Name', False): + self.notifications.append(notice) + elif notice.get('Identifier', False): + notice['Data'] = self._decode_binary(part, notice) + #open('register.png','wblol').write(notice['Data']) + self.resources[notice.get('Identifier')] = notice + + def add_notification(self, name, enabled=True): + """Add new Notification to Registration message + + :param string name: Notification Name + :param boolean enabled: Enable this notification by default + """ + notice = {} + notice['Notification-Name'] = name + notice['Notification-Enabled'] = enabled + + self.notifications.append(notice) + self.add_header('Notifications-Count', len(self.notifications)) + + def encode(self): + """Encode a GNTP Registration Message + + :return string: Encoded GNTP Registration message. Returned as a byte string + """ + + buff = _GNTPBuffer() + + buff.writeln(self._format_info()) + + #Headers + for k, v in self.headers.items(): + buff.writeheader(k, v) + buff.writeln() + + #Notifications + if len(self.notifications) > 0: + for notice in self.notifications: + for k, v in notice.items(): + buff.writeheader(k, v) + buff.writeln() + + #Resources + for resource, data in self.resources.items(): + buff.writeheader('Identifier', resource) + buff.writeheader('Length', len(data)) + buff.writeln() + buff.write(data) + buff.writeln() + buff.writeln() + + return buff.getvalue() + + +class GNTPNotice(_GNTPBase): + """Represents a GNTP Notification Command + + :param string data: (Optional) See decode() + :param string app: (Optional) Set Application-Name + :param string name: (Optional) Set Notification-Name + :param string title: (Optional) Set Notification Title + :param string password: (Optional) Password to use while encoding/decoding messages + """ + _requiredHeaders = [ + 'Application-Name', + 'Notification-Name', + 'Notification-Title' + ] + + def __init__(self, data=None, app=None, name=None, title=None, password=None): + _GNTPBase.__init__(self, 'NOTIFY') + + if data: + self.decode(data, password) + else: + self.set_password(password) + if app: + self.add_header('Application-Name', app) + if name: + self.add_header('Notification-Name', name) + if title: + self.add_header('Notification-Title', title) + + def decode(self, data, password): + """Decode existing GNTP Notification message + + :param string data: Message to decode. + """ + self.raw = gntp.shim.u(data) + parts = self.raw.split('\r\n\r\n') + self.info = self._parse_info(self.raw) + self._validate_password(password) + self.headers = self._parse_dict(parts[0]) + + for i, part in enumerate(parts): + if i == 0: + continue # Skip Header + if part.strip() == '': + continue + notice = self._parse_dict(part) + if notice.get('Identifier', False): + notice['Data'] = self._decode_binary(part, notice) + #open('notice.png','wblol').write(notice['Data']) + self.resources[notice.get('Identifier')] = notice + + +class GNTPSubscribe(_GNTPBase): + """Represents a GNTP Subscribe Command + + :param string data: (Optional) See decode() + :param string password: (Optional) Password to use while encoding/decoding messages + """ + _requiredHeaders = [ + 'Subscriber-ID', + 'Subscriber-Name', + ] + + def __init__(self, data=None, password=None): + _GNTPBase.__init__(self, 'SUBSCRIBE') + if data: + self.decode(data, password) + else: + self.set_password(password) + + +class GNTPOK(_GNTPBase): + """Represents a GNTP OK Response + + :param string data: (Optional) See _GNTPResponse.decode() + :param string action: (Optional) Set type of action the OK Response is for + """ + _requiredHeaders = ['Response-Action'] + + def __init__(self, data=None, action=None): + _GNTPBase.__init__(self, '-OK') + if data: + self.decode(data) + if action: + self.add_header('Response-Action', action) + + +class GNTPError(_GNTPBase): + """Represents a GNTP Error response + + :param string data: (Optional) See _GNTPResponse.decode() + :param string errorcode: (Optional) Error code + :param string errordesc: (Optional) Error Description + """ + _requiredHeaders = ['Error-Code', 'Error-Description'] + + def __init__(self, data=None, errorcode=None, errordesc=None): + _GNTPBase.__init__(self, '-ERROR') + if data: + self.decode(data) + if errorcode: + self.add_header('Error-Code', errorcode) + self.add_header('Error-Description', errordesc) + + def error(self): + return (self.headers.get('Error-Code', None), + self.headers.get('Error-Description', None)) + + +def parse_gntp(data, password=None): + """Attempt to parse a message as a GNTP message + + :param string data: Message to be parsed + :param string password: Optional password to be used to verify the message + """ + data = gntp.shim.u(data) + match = GNTP_INFO_LINE_SHORT.match(data) + if not match: + raise errors.ParseError('INVALID_GNTP_INFO') + info = match.groupdict() + if info['messagetype'] == 'REGISTER': + return GNTPRegister(data, password=password) + elif info['messagetype'] == 'NOTIFY': + return GNTPNotice(data, password=password) + elif info['messagetype'] == 'SUBSCRIBE': + return GNTPSubscribe(data, password=password) + elif info['messagetype'] == '-OK': + return GNTPOK(data) + elif info['messagetype'] == '-ERROR': + return GNTPError(data) + raise errors.ParseError('INVALID_GNTP_MESSAGE') diff --git a/libs/gntp/errors.py b/libs/gntp/errors.py new file mode 100644 index 00000000..c006fd68 --- /dev/null +++ b/libs/gntp/errors.py @@ -0,0 +1,25 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +class BaseError(Exception): + pass + + +class ParseError(BaseError): + errorcode = 500 + errordesc = 'Error parsing the message' + + +class AuthError(BaseError): + errorcode = 400 + errordesc = 'Error with authorization' + + +class UnsupportedError(BaseError): + errorcode = 500 + errordesc = 'Currently unsupported by gntp.py' + + +class NetworkError(BaseError): + errorcode = 500 + errordesc = "Error connecting to growl server" diff --git a/libs/gntp/notifier.py b/libs/gntp/notifier.py index 539dae2a..1719ecdf 100755 --- a/libs/gntp/notifier.py +++ b/libs/gntp/notifier.py @@ -1,3 +1,6 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + """ The gntp.notifier module is provided as a simple way to send notifications using GNTP @@ -9,10 +12,15 @@ using GNTP `Original Python bindings `_ """ -import gntp -import socket import logging import platform +import socket +import sys + +from gntp.version import __version__ +import gntp.core +import gntp.errors as errors +import gntp.shim __all__ = [ 'mini', @@ -37,9 +45,9 @@ class GrowlNotifier(object): passwordHash = 'MD5' socketTimeout = 3 - def __init__(self, applicationName = 'Python GNTP', notifications = [], - defaultNotifications = None, applicationIcon = None, hostname = 'localhost', - password = None, port = 23053): + def __init__(self, applicationName='Python GNTP', notifications=[], + defaultNotifications=None, applicationIcon=None, hostname='localhost', + password=None, port=23053): self.applicationName = applicationName self.notifications = list(notifications) @@ -61,7 +69,7 @@ class GrowlNotifier(object): then we return False ''' logger.info('Checking icon') - return data.startswith('http') + return gntp.shim.u(data).startswith('http') def register(self): """Send GNTP Registration @@ -71,7 +79,7 @@ class GrowlNotifier(object): sent a registration message at least once """ logger.info('Sending registration to %s:%s', self.hostname, self.port) - register = gntp.GNTPRegister() + register = gntp.core.GNTPRegister() register.add_header('Application-Name', self.applicationName) for notification in self.notifications: enabled = notification in self.defaultNotifications @@ -80,16 +88,16 @@ class GrowlNotifier(object): if self._checkIcon(self.applicationIcon): register.add_header('Application-Icon', self.applicationIcon) else: - id = register.add_resource(self.applicationIcon) - register.add_header('Application-Icon', id) + resource = register.add_resource(self.applicationIcon) + register.add_header('Application-Icon', resource) if self.password: register.set_password(self.password, self.passwordHash) self.add_origin_info(register) self.register_hook(register) return self._send('register', register) - def notify(self, noteType, title, description, icon = None, sticky = False, - priority = None, callback = None, identifier = None): + def notify(self, noteType, title, description, icon=None, sticky=False, + priority=None, callback=None, identifier=None, custom={}): """Send a GNTP notifications .. warning:: @@ -102,6 +110,8 @@ class GrowlNotifier(object): :param boolean sticky: Sticky notification :param integer priority: Message priority level from -2 to 2 :param string callback: URL callback + :param dict custom: Custom attributes. Key names should be prefixed with X- + according to the spec but this is not enforced by this class .. warning:: For now, only URL callbacks are supported. In the future, the @@ -109,7 +119,7 @@ class GrowlNotifier(object): """ logger.info('Sending notification [%s] to %s:%s', noteType, self.hostname, self.port) assert noteType in self.notifications - notice = gntp.GNTPNotice() + notice = gntp.core.GNTPNotice() notice.add_header('Application-Name', self.applicationName) notice.add_header('Notification-Name', noteType) notice.add_header('Notification-Title', title) @@ -123,8 +133,8 @@ class GrowlNotifier(object): if self._checkIcon(icon): notice.add_header('Notification-Icon', icon) else: - id = notice.add_resource(icon) - notice.add_header('Notification-Icon', id) + resource = notice.add_resource(icon) + notice.add_header('Notification-Icon', resource) if description: notice.add_header('Notification-Text', description) @@ -133,6 +143,9 @@ class GrowlNotifier(object): if identifier: notice.add_header('Notification-Coalescing-ID', identifier) + for key in custom: + notice.add_header(key, custom[key]) + self.add_origin_info(notice) self.notify_hook(notice) @@ -140,7 +153,7 @@ class GrowlNotifier(object): def subscribe(self, id, name, port): """Send a Subscribe request to a remote machine""" - sub = gntp.GNTPSubscribe() + sub = gntp.core.GNTPSubscribe() sub.add_header('Subscriber-ID', id) sub.add_header('Subscriber-Name', name) sub.add_header('Subscriber-Port', port) @@ -156,7 +169,7 @@ class GrowlNotifier(object): """Add optional Origin headers to message""" packet.add_header('Origin-Machine-Name', platform.node()) packet.add_header('Origin-Software-Name', 'gntp.py') - packet.add_header('Origin-Software-Version', gntp.__version__) + packet.add_header('Origin-Software-Version', __version__) packet.add_header('Origin-Platform-Name', platform.system()) packet.add_header('Origin-Platform-Version', platform.platform()) @@ -179,27 +192,33 @@ class GrowlNotifier(object): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(self.socketTimeout) - s.connect((self.hostname, self.port)) - s.send(data) - recv_data = s.recv(1024) - while not recv_data.endswith("\r\n\r\n"): - recv_data += s.recv(1024) - response = gntp.parse_gntp(recv_data) + try: + s.connect((self.hostname, self.port)) + s.send(data) + recv_data = s.recv(1024) + while not recv_data.endswith(gntp.shim.b("\r\n\r\n")): + recv_data += s.recv(1024) + except socket.error: + # Python2.5 and Python3 compatibile exception + exc = sys.exc_info()[1] + raise errors.NetworkError(exc) + + response = gntp.core.parse_gntp(recv_data) s.close() logger.debug('From : %s:%s <%s>\n%s', self.hostname, self.port, response.__class__, response) - if type(response) == gntp.GNTPOK: + if type(response) == gntp.core.GNTPOK: return True logger.error('Invalid response: %s', response.error()) return response.error() -def mini(description, applicationName = 'PythonMini', noteType = "Message", - title = "Mini Message", applicationIcon = None, hostname = 'localhost', - password = None, port = 23053, sticky = False, priority = None, - callback = None, notificationIcon = None, identifier = None, - notifierFactory = GrowlNotifier): +def mini(description, applicationName='PythonMini', noteType="Message", + title="Mini Message", applicationIcon=None, hostname='localhost', + password=None, port=23053, sticky=False, priority=None, + callback=None, notificationIcon=None, identifier=None, + notifierFactory=GrowlNotifier): """Single notification function Simple notification function in one line. Has only one required parameter @@ -210,32 +229,37 @@ def mini(description, applicationName = 'PythonMini', noteType = "Message", For now, only URL callbacks are supported. In the future, the callback argument will also support a function """ - growl = notifierFactory( - applicationName = applicationName, - notifications = [noteType], - defaultNotifications = [noteType], - applicationIcon = applicationIcon, - hostname = hostname, - password = password, - port = port, - ) - result = growl.register() - if result is not True: - return result + try: + growl = notifierFactory( + applicationName=applicationName, + notifications=[noteType], + defaultNotifications=[noteType], + applicationIcon=applicationIcon, + hostname=hostname, + password=password, + port=port, + ) + result = growl.register() + if result is not True: + return result - return growl.notify( - noteType = noteType, - title = title, - description = description, - icon = notificationIcon, - sticky = sticky, - priority = priority, - callback = callback, - identifier = identifier, - ) + return growl.notify( + noteType=noteType, + title=title, + description=description, + icon=notificationIcon, + sticky=sticky, + priority=priority, + callback=callback, + identifier=identifier, + ) + except Exception: + # We want the "mini" function to be simple and swallow Exceptions + # in order to be less invasive + logger.exception("Growl error") if __name__ == '__main__': # If we're running this module directly we're likely running it as a test # so extra debugging is useful - logging.basicConfig(level = logging.INFO) + logging.basicConfig(level=logging.INFO) mini('Testing mini notification') diff --git a/libs/gntp/shim.py b/libs/gntp/shim.py new file mode 100644 index 00000000..3a387828 --- /dev/null +++ b/libs/gntp/shim.py @@ -0,0 +1,45 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +""" +Python2.5 and Python3.3 compatibility shim + +Heavily inspirted by the "six" library. +https://pypi.python.org/pypi/six +""" + +import sys + +PY3 = sys.version_info[0] == 3 + +if PY3: + def b(s): + if isinstance(s, bytes): + return s + return s.encode('utf8', 'replace') + + def u(s): + if isinstance(s, bytes): + return s.decode('utf8', 'replace') + return s + + from io import BytesIO as StringIO + from configparser import RawConfigParser +else: + def b(s): + if isinstance(s, unicode): + return s.encode('utf8', 'replace') + return s + + def u(s): + if isinstance(s, unicode): + return s + if isinstance(s, int): + s = str(s) + return unicode(s, "utf8", "replace") + + from StringIO import StringIO + from ConfigParser import RawConfigParser + +b.__doc__ = "Ensure we have a byte string" +u.__doc__ = "Ensure we have a unicode string" diff --git a/libs/gntp/version.py b/libs/gntp/version.py new file mode 100644 index 00000000..2166aaca --- /dev/null +++ b/libs/gntp/version.py @@ -0,0 +1,4 @@ +# Copyright: 2013 Paul Traylor +# These sources are released under the terms of the MIT license: see LICENSE + +__version__ = '1.0.2' diff --git a/libs/guessit/__init__.py b/libs/guessit/__init__.py index ce140248..e6cfa276 100755 --- a/libs/guessit/__init__.py +++ b/libs/guessit/__init__.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals -__version__ = '0.7-dev' +__version__ = '0.6.2' __all__ = ['Guess', 'Language', 'guess_file_info', 'guess_video_info', 'guess_movie_info', 'guess_episode_info'] @@ -76,6 +76,7 @@ from guessit.language import Language from guessit.matcher import IterativeMatcher from guessit.textutils import clean_string import logging +import json log = logging.getLogger(__name__) @@ -105,17 +106,74 @@ def _guess_filename(filename, filetype): mtree = IterativeMatcher(filename, filetype=filetype) + m = mtree.matched() + + second_pass_opts = [] + second_pass_transfo_opts = {} + # if there are multiple possible years found, we assume the first one is # part of the title, reparse the tree taking this into account years = set(n.value for n in find_nodes(mtree.match_tree, 'year')) if len(years) >= 2: - mtree = IterativeMatcher(filename, filetype=filetype, - opts=['skip_first_year']) + second_pass_opts.append('skip_first_year') + to_skip_language_nodes = [] + + title_nodes = set(n for n in find_nodes(mtree.match_tree, ['title', 'series'])) + title_spans = {} + for title_node in title_nodes: + title_spans[title_node.span[0]] = title_node + title_spans[title_node.span[1]] = title_node + + for lang_key in ('language', 'subtitleLanguage'): + langs = {} + lang_nodes = set(n for n in find_nodes(mtree.match_tree, lang_key)) + + for lang_node in lang_nodes: + lang = lang_node.guess.get(lang_key, None) + if len(lang_node.value) > 3 and (lang_node.span[0] in title_spans.keys() or lang_node.span[1] in title_spans.keys()): + # Language is next or before title, and is not a language code. Add to skip for 2nd pass. + + # if filetype is subtitle and the language appears last, just before + # the extension, then it is likely a subtitle language + parts = clean_string(lang_node.root.value).split() + if m['type'] in ['moviesubtitle', 'episodesubtitle'] and (parts.index(lang_node.value) == len(parts) - 2): + continue + + to_skip_language_nodes.append(lang_node) + elif not lang in langs: + langs[lang] = lang_node + else: + # The same language was found. Keep the more confident one, and add others to skip for 2nd pass. + existing_lang_node = langs[lang] + to_skip = None + if existing_lang_node.guess.confidence('language') >= lang_node.guess.confidence('language'): + # lang_node is to remove + to_skip = lang_node + else: + # existing_lang_node is to remove + langs[lang] = lang_node + to_skip = existing_lang_node + to_skip_language_nodes.append(to_skip) + + + if to_skip_language_nodes: + second_pass_transfo_opts['guess_language'] = ( + ((), { 'skip': [ { 'node_idx': node.parent.node_idx, + 'span': node.span } + for node in to_skip_language_nodes ] })) + + if second_pass_opts or second_pass_transfo_opts: + # 2nd pass is needed + log.info("Running 2nd pass with options: %s" % second_pass_opts) + log.info("Transfo options: %s" % second_pass_transfo_opts) + mtree = IterativeMatcher(filename, filetype=filetype, + opts=second_pass_opts, + transfo_opts=second_pass_transfo_opts) m = mtree.matched() - if 'language' not in m and 'subtitleLanguage' not in m: + if 'language' not in m and 'subtitleLanguage' not in m or 'title' not in m: return m # if we found some language, make sure we didn't cut a title or sth... @@ -123,51 +181,10 @@ def _guess_filename(filename, filetype): opts=['nolanguage', 'nocountry']) m2 = mtree2.matched() - - if m.get('title') is None: - return m - if m.get('title') != m2.get('title'): title = next(find_nodes(mtree.match_tree, 'title')) title2 = next(find_nodes(mtree2.match_tree, 'title')) - langs = list(find_nodes(mtree.match_tree, ['language', 'subtitleLanguage'])) - if not langs: - return warning('A weird error happened with language detection') - - # find the language that is likely more relevant - for lng in langs: - if lng.value in title2.value: - # if the language was detected as part of a potential title, - # look at this one in particular - lang = lng - break - else: - # pick the first one if we don't have a better choice - lang = langs[0] - - - # language code are rarely part of a title, and those - # should be handled by the Language exceptions anyway - if len(lang.value) <= 3: - return m - - - # if filetype is subtitle and the language appears last, just before - # the extension, then it is likely a subtitle language - parts = clean_string(title.root.value).split() - if (m['type'] in ['moviesubtitle', 'episodesubtitle'] and - parts.index(lang.value) == len(parts) - 2): - return m - - # if the language was in the middle of the other potential title, - # keep the other title (eg: The Italian Job), except if it is at the - # very beginning, in which case we consider it an error - if m2['title'].startswith(lang.value): - return m - elif lang.value in title2.value: - return m2 - # if a node is in an explicit group, then the correct title is probably # the other one if title.root.node_at(title.node_idx[:2]).is_explicit(): @@ -175,9 +192,6 @@ def _guess_filename(filename, filetype): elif title2.root.node_at(title2.node_idx[:2]).is_explicit(): return m - return warning('Not sure of the title because of the language position') - - return m diff --git a/libs/guessit/__main__.py b/libs/guessit/__main__.py index 957ec9da..ccfa3af6 100755 --- a/libs/guessit/__main__.py +++ b/libs/guessit/__main__.py @@ -24,16 +24,19 @@ from guessit import u from guessit import slogging, guess_file_info from optparse import OptionParser import logging +import sys +import os +import locale -def detect_filename(filename, filetype, info=['filename']): +def detect_filename(filename, filetype, info=['filename'], advanced = False): filename = u(filename) print('For:', filename) - print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string()) + print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string(advanced)) -def run_demo(episodes=True, movies=True): +def run_demo(episodes=True, movies=True, advanced=False): # NOTE: tests should not be added here but rather in the tests/ folder # this is just intended as a quick example if episodes: @@ -50,7 +53,7 @@ def run_demo(episodes=True, movies=True): for f in testeps: print('-'*80) - detect_filename(f, filetype='episode') + detect_filename(f, filetype='episode', advanced=advanced) if movies: @@ -77,12 +80,17 @@ def run_demo(episodes=True, movies=True): for f in testmovies: print('-'*80) - detect_filename(f, filetype = 'movie') + detect_filename(f, filetype = 'movie', advanced = advanced) def main(): slogging.setupLogging() + # see http://bugs.python.org/issue2128 + if sys.version_info.major < 3 and os.name == 'nt': + for i, a in enumerate(sys.argv): + sys.argv[i] = a.decode(locale.getpreferredencoding()) + parser = OptionParser(usage = 'usage: %prog [options] file1 [file2...]') parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False, help = 'display debug output') @@ -92,6 +100,8 @@ def main(): 'them, comma-separated') parser.add_option('-t', '--type', dest = 'filetype', default = 'autodetect', help = 'the suggested file type: movie, episode or autodetect') + parser.add_option('-a', '--advanced', dest = 'advanced', action='store_true', default = False, + help = 'display advanced information for filename guesses, as json output') parser.add_option('-d', '--demo', action='store_true', dest='demo', default=False, help = 'run a few builtin tests instead of analyzing a file') @@ -100,13 +110,14 @@ def main(): logging.getLogger('guessit').setLevel(logging.DEBUG) if options.demo: - run_demo(episodes=True, movies=True) + run_demo(episodes=True, movies=True, advanced=options.advanced) else: if args: for filename in args: detect_filename(filename, filetype = options.filetype, - info = options.info.split(',')) + info = options.info.split(','), + advanced = options.advanced) else: parser.print_help() diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py index dc077e64..9531f82a 100755 --- a/libs/guessit/fileutils.py +++ b/libs/guessit/fileutils.py @@ -44,13 +44,14 @@ def split_path(path): result = [] while True: head, tail = os.path.split(path) + headlen = len(head) # on Unix systems, the root folder is '/' - if head == '/' and tail == '': + if head and head == '/'*headlen and tail == '': return ['/'] + result # on Windows, the root folder is a drive letter (eg: 'C:\') or for shares \\ - if ((len(head) == 3 and head[1:] == ':\\') or (len(head) == 2 and head == '\\\\')) and tail == '': + if ((headlen == 3 and head[1:] == ':\\') or (headlen == 2 and head == '\\\\')) and tail == '': return [head] + result if head == '' and tail == '': @@ -61,6 +62,7 @@ def split_path(path): path = head continue + # otherwise, add the last path fragment and keep splitting result = [tail] + result path = head diff --git a/libs/guessit/guess.py b/libs/guessit/guess.py index 33d36517..73babceb 100755 --- a/libs/guessit/guess.py +++ b/libs/guessit/guess.py @@ -41,15 +41,21 @@ class Guess(UnicodeMixin, dict): confidence = kwargs.pop('confidence') except KeyError: confidence = 0 + + try: + raw = kwargs.pop('raw') + except KeyError: + raw = None dict.__init__(self, *args, **kwargs) self._confidence = {} + self._raw = {} for prop in self: self._confidence[prop] = confidence - - - def to_dict(self): + self._raw[prop] = raw + + def to_dict(self, advanced=False): data = dict(self) for prop, value in data.items(): if isinstance(value, datetime.date): @@ -58,46 +64,65 @@ class Guess(UnicodeMixin, dict): data[prop] = u(value) elif isinstance(value, list): data[prop] = [u(x) for x in value] + if advanced: + data[prop] = {"value": data[prop], "raw": self.raw(prop), "confidence": self.confidence(prop)} return data - def nice_string(self): - data = self.to_dict() - - parts = json.dumps(data, indent=4).split('\n') - for i, p in enumerate(parts): - if p[:5] != ' "': - continue - - prop = p.split('"')[1] - parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:] - - return '\n'.join(parts) + def nice_string(self, advanced=False): + if advanced: + data = self.to_dict(advanced) + return json.dumps(data, indent=4) + else: + data = self.to_dict() + + parts = json.dumps(data, indent=4).split('\n') + for i, p in enumerate(parts): + if p[:5] != ' "': + continue + + prop = p.split('"')[1] + parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:] + + return '\n'.join(parts) def __unicode__(self): return u(self.to_dict()) def confidence(self, prop): return self._confidence.get(prop, -1) + + def raw(self, prop): + return self._raw.get(prop, None) - def set(self, prop, value, confidence=None): + def set(self, prop, value, confidence=None, raw=None): self[prop] = value if confidence is not None: self._confidence[prop] = confidence + if raw is not None: + self._raw[prop] = raw def set_confidence(self, prop, value): self._confidence[prop] = value + + def set_raw(self, prop, value): + self._raw[prop] = value - def update(self, other, confidence=None): + def update(self, other, confidence=None, raw=None): dict.update(self, other) if isinstance(other, Guess): for prop in other: self._confidence[prop] = other.confidence(prop) + self._raw[prop] = other.raw(prop) if confidence is not None: for prop in other: self._confidence[prop] = confidence + if raw is not None: + for prop in other: + self._raw[prop] = raw + def update_highest_confidence(self, other): """Update this guess with the values from the given one. In case there is property present in both, only the one with the highest one @@ -110,6 +135,7 @@ class Guess(UnicodeMixin, dict): continue self[prop] = other[prop] self._confidence[prop] = other.confidence(prop) + self._raw[prop] = other.raw(prop) def choose_int(g1, g2): @@ -181,7 +207,7 @@ def choose_string(g1, g2): elif v1l in v2l: return (v1, combined_prob) - # in case of conflict, return the one with highest priority + # in case of conflict, return the one with highest confidence else: if c1 > c2: return (v1, c1 - c2) @@ -288,7 +314,8 @@ def merge_all(guesses, append=None): result.set(prop, result.get(prop, []) + [g[prop]], # TODO: what to do with confidence here? maybe an # arithmetic mean... - confidence=g.confidence(prop)) + confidence=g.confidence(prop), + raw=g.raw(prop)) del g[prop] diff --git a/libs/guessit/language.py b/libs/guessit/language.py index 2714c6e0..4d22cf05 100755 --- a/libs/guessit/language.py +++ b/libs/guessit/language.py @@ -296,7 +296,7 @@ UNDETERMINED = Language('und') ALL_LANGUAGES = frozenset(Language(lng) for lng in lng_all_names) - frozenset([UNDETERMINED]) ALL_LANGUAGES_NAMES = lng_all_names -def search_language(string, lang_filter=None): +def search_language(string, lang_filter=None, skip=None): """Looks for language patterns, and if found return the language object, its group span and an associated confidence. @@ -345,6 +345,16 @@ def search_language(string, lang_filter=None): if pos != -1: end = pos + len(lang) + + # skip if span in in skip list + while skip and (pos - 1, end - 1) in skip: + pos = slow.find(lang, end) + if pos == -1: + continue + end = pos + len(lang) + if pos == -1: + continue + # make sure our word is always surrounded by separators if slow[pos - 1] not in sep or slow[end] not in sep: continue diff --git a/libs/guessit/matcher.py b/libs/guessit/matcher.py index 43378192..1984c01c 100755 --- a/libs/guessit/matcher.py +++ b/libs/guessit/matcher.py @@ -21,14 +21,14 @@ from __future__ import unicode_literals from guessit import PY3, u, base_text_type from guessit.matchtree import MatchTree -from guessit.textutils import normalize_unicode +from guessit.textutils import normalize_unicode, clean_string import logging log = logging.getLogger(__name__) class IterativeMatcher(object): - def __init__(self, filename, filetype='autodetect', opts=None): + def __init__(self, filename, filetype='autodetect', opts=None, transfo_opts=None): """An iterative matcher tries to match different patterns that appear in the filename. @@ -38,7 +38,8 @@ class IterativeMatcher(object): a movie. The recognized 'filetype' values are: - [ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ] + [ autodetect, subtitle, info, movie, moviesubtitle, movieinfo, episode, + episodesubtitle, episodeinfo ] The IterativeMatcher works mainly in 2 steps: @@ -61,15 +62,20 @@ class IterativeMatcher(object): it corresponds to a video codec, denoted by the letter'v' in the 4th line. (for more info, see guess.matchtree.to_string) + Second, it tries to merge all this information into a single object + containing all the found properties, and does some (basic) conflict + resolution when they arise. - Second, it tries to merge all this information into a single object - containing all the found properties, and does some (basic) conflict - resolution when they arise. + + When you create the Matcher, you can pass it: + - a list 'opts' of option names, that act as global flags + - a dict 'transfo_opts' of { transfo_name: (transfo_args, transfo_kwargs) } + with which to call the transfo.process() function. """ - valid_filetypes = ('autodetect', 'subtitle', 'video', - 'movie', 'moviesubtitle', - 'episode', 'episodesubtitle') + valid_filetypes = ('autodetect', 'subtitle', 'info', 'video', + 'movie', 'moviesubtitle', 'movieinfo', + 'episode', 'episodesubtitle', 'episodeinfo') if filetype not in valid_filetypes: raise ValueError("filetype needs to be one of %s" % valid_filetypes) if not PY3 and not isinstance(filename, unicode): @@ -80,10 +86,22 @@ class IterativeMatcher(object): if opts is None: opts = [] - elif isinstance(opts, base_text_type): - opts = opts.split() + if not isinstance(opts, list): + raise ValueError('opts must be a list of option names! Received: type=%s val=%s', + type(opts), opts) + + if transfo_opts is None: + transfo_opts = {} + if not isinstance(transfo_opts, dict): + raise ValueError('transfo_opts must be a dict of { transfo_name: (args, kwargs) }. '+ + 'Received: type=%s val=%s', type(transfo_opts), transfo_opts) self.match_tree = MatchTree(filename) + + # sanity check: make sure we don't process a (mostly) empty string + if clean_string(filename) == '': + return + mtree = self.match_tree mtree.guess.set('type', filetype, confidence=1.0) @@ -91,7 +109,11 @@ class IterativeMatcher(object): transfo = __import__('guessit.transfo.' + transfo_name, globals=globals(), locals=locals(), fromlist=['process'], level=0) - transfo.process(mtree, *args, **kwargs) + default_args, default_kwargs = transfo_opts.get(transfo_name, ((), {})) + all_args = args or default_args + all_kwargs = dict(default_kwargs) + all_kwargs.update(kwargs) # keep all kwargs merged together + transfo.process(mtree, *all_args, **all_kwargs) # 1- first split our path into dirs + basename + ext apply_transfo('split_path_components') @@ -111,7 +133,7 @@ class IterativeMatcher(object): # - language before episodes_rexps # - properties before language (eg: he-aac vs hebrew) # - release_group before properties (eg: XviD-?? vs xvid) - if mtree.guess['type'] in ('episode', 'episodesubtitle'): + if mtree.guess['type'] in ('episode', 'episodesubtitle', 'episodeinfo'): strategy = [ 'guess_date', 'guess_website', 'guess_release_group', 'guess_properties', 'guess_language', 'guess_video_rexps', @@ -124,6 +146,7 @@ class IterativeMatcher(object): if 'nolanguage' in opts: strategy.remove('guess_language') + for name in strategy: apply_transfo(name) @@ -143,7 +166,7 @@ class IterativeMatcher(object): # 5- try to identify the remaining unknown groups by looking at their # position relative to other known elements - if mtree.guess['type'] in ('episode', 'episodesubtitle'): + if mtree.guess['type'] in ('episode', 'episodesubtitle', 'episodeinfo'): apply_transfo('guess_episode_info_from_position') else: apply_transfo('guess_movie_title_from_position') diff --git a/libs/guessit/patterns.py b/libs/guessit/patterns.py index ed3982b9..f803a11c 100755 --- a/libs/guessit/patterns.py +++ b/libs/guessit/patterns.py @@ -25,6 +25,8 @@ import re subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa' ] +info_exts = [ 'nfo' ] + video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2', 'mka', 'mkv', 'mov', 'mp4', 'mp4a', 'mpeg', 'mpg', 'ogg', 'ogm', 'ogv', 'qt', 'ra', 'ram', 'rm', 'ts', 'wav', 'webm', 'wma', 'wmv'] @@ -32,7 +34,7 @@ video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2', group_delimiters = [ '()', '[]', '{}' ] # separator character regexp -sep = r'[][)(}{+ /\._-]' # regexp art, hehe :D +sep = r'[][,)(}{+ /\._-]' # regexp art, hehe :D # character used to represent a deleted char (when matching groups) deleted = '_' @@ -49,7 +51,7 @@ episode_rexps = [ # ... Season 2 ... #(r'[Ss](?P[0-9]{1,3})[^0-9]?(?P(?:-?[xX-][0-9]{1,3})+)[^0-9]', 1.0, (0, -1)), # ... 2x13 ... - (r'[^0-9](?P[0-9]{1,2})[^0-9]?(?P(?:-?[xX][0-9]{1,3})+)[^0-9]', 1.0, (1, -1)), + (r'[^0-9](?P[0-9]{1,2})[^0-9 .-]?(?P(?:-?[xX][0-9]{1,3})+)[^0-9]', 1.0, (1, -1)), # ... s02 ... #(sep + r's(?P[0-9]{1,2})' + sep, 0.6, (1, -1)), @@ -122,9 +124,12 @@ prop_multi = { 'format': { 'DVD': [ 'DVD', 'DVD-Rip', 'VIDEO-TS', 'DVDivX' ], 'VHS': [ 'VHS' ], 'WEB-DL': [ 'WEB-DL' ] }, + 'is3D': { True: [ '3D' ] }, + 'screenSize': { '480p': [ '480[pi]?' ], '720p': [ '720[pi]?' ], - '1080p': [ '1080[pi]?' ] }, + '1080i': [ '1080i' ], + '1080p': [ '1080p', '1080[^i]' ] }, 'videoCodec': { 'XviD': [ 'Xvid' ], 'DivX': [ 'DVDivX', 'DivX' ], @@ -140,7 +145,7 @@ prop_multi = { 'format': { 'DVD': [ 'DVD', 'DVD-Rip', 'VIDEO-TS', 'DVDivX' ], 'DTS': [ 'DTS' ], 'AAC': [ 'He-AAC', 'AAC-He', 'AAC' ] }, - 'audioChannels': { '5.1': [ r'5\.1', 'DD5[\._ ]1', '5ch' ] }, + 'audioChannels': { '5.1': [ r'5\.1', 'DD5[._ ]1', '5ch' ] }, 'episodeFormat': { 'Minisode': [ 'Minisodes?' ] } @@ -170,7 +175,7 @@ prop_single = { 'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', r'\[XCT\]', 'iNT', 'P } _dash = '-' -_psep = '[-\. _]?' +_psep = '[-. _]?' def _to_rexp(prop): return re.compile(prop.replace(_dash, _psep), re.IGNORECASE) @@ -237,8 +242,9 @@ def canonical_form(string): def compute_canonical_form(property_name, value): """Return the canonical form of a property given its type if it is a valid one, None otherwise.""" - for canonical_form, rexps in properties_rexps[property_name].items(): - for rexp in rexps: - if rexp.match(value): - return canonical_form + if isinstance(value, basestring): + for canonical_form, rexps in properties_rexps[property_name].items(): + for rexp in rexps: + if rexp.match(value): + return canonical_form return None diff --git a/libs/guessit/slogging.py b/libs/guessit/slogging.py index 75e261cf..39591a20 100755 --- a/libs/guessit/slogging.py +++ b/libs/guessit/slogging.py @@ -31,14 +31,15 @@ RED_FONT = "\x1B[0;31m" RESET_FONT = "\x1B[0m" -def setupLogging(colored=True, with_time=False, with_thread=False, filename=None): +def setupLogging(colored=True, with_time=False, with_thread=False, filename=None, with_lineno=False): """Set up a nice colored logger as the main application logger.""" class SimpleFormatter(logging.Formatter): def __init__(self, with_time, with_thread): self.fmt = (('%(asctime)s ' if with_time else '') + '%(levelname)-8s ' + - '[%(name)s:%(funcName)s]' + + '[%(name)s:%(funcName)s' + + (':%(lineno)s' if with_lineno else '') + ']' + ('[%(threadName)s]' if with_thread else '') + ' -- %(message)s') logging.Formatter.__init__(self, self.fmt) @@ -47,7 +48,8 @@ def setupLogging(colored=True, with_time=False, with_thread=False, filename=None def __init__(self, with_time, with_thread): self.fmt = (('%(asctime)s ' if with_time else '') + '-CC-%(levelname)-8s ' + - BLUE_FONT + '[%(name)s:%(funcName)s]' + + BLUE_FONT + '[%(name)s:%(funcName)s' + + (':%(lineno)s' if with_lineno else '') + ']' + RESET_FONT + ('[%(threadName)s]' if with_thread else '') + ' -- %(message)s') diff --git a/libs/guessit/textutils.py b/libs/guessit/textutils.py index f195e2b7..ae9d28c3 100755 --- a/libs/guessit/textutils.py +++ b/libs/guessit/textutils.py @@ -43,10 +43,13 @@ def strip_brackets(s): return s -def clean_string(s): - for c in sep[:-2]: # do not remove dashes ('-') - s = s.replace(c, ' ') - parts = s.split() +def clean_string(st): + for c in sep: + # do not remove certain chars + if c in ['-', ',']: + continue + st = st.replace(c, ' ') + parts = st.split() result = ' '.join(p for p in parts if p != '') # now also remove dashes on the outer part of the string diff --git a/libs/guessit/transfo/__init__.py b/libs/guessit/transfo/__init__.py index 820690a7..a28aa988 100755 --- a/libs/guessit/transfo/__init__.py +++ b/libs/guessit/transfo/__init__.py @@ -28,7 +28,7 @@ log = logging.getLogger(__name__) def found_property(node, name, confidence): - node.guess = Guess({name: node.clean_value}, confidence=confidence) + node.guess = Guess({name: node.clean_value}, confidence=confidence, raw=node.value) log.debug('Found with confidence %.2f: %s' % (confidence, node.guess)) @@ -52,11 +52,17 @@ def format_guess(guess): def find_and_split_node(node, strategy, logger): string = ' %s ' % node.value # add sentinels - for matcher, confidence in strategy: + for matcher, confidence, args, kwargs in strategy: + all_args = [string] if getattr(matcher, 'use_node', False): - result, span = matcher(string, node) + all_args.append(node) + if args: + all_args.append(args) + + if kwargs: + result, span = matcher(*all_args, **kwargs) else: - result, span = matcher(string) + result, span = matcher(*all_args) if result: # readjust span to compensate for sentinels @@ -69,7 +75,7 @@ def find_and_split_node(node, strategy, logger): if confidence is None: confidence = 1.0 - guess = format_guess(Guess(result, confidence=confidence)) + guess = format_guess(Guess(result, confidence=confidence, raw=string[span[0] + 1:span[1] + 1])) msg = 'Found with confidence %.2f: %s' % (confidence, guess) (logger or log).debug(msg) @@ -84,10 +90,12 @@ def find_and_split_node(node, strategy, logger): class SingleNodeGuesser(object): - def __init__(self, guess_func, confidence, logger=None): + def __init__(self, guess_func, confidence, logger, *args, **kwargs): self.guess_func = guess_func self.confidence = confidence self.logger = logger + self.args = args + self.kwargs = kwargs def process(self, mtree): # strategy is a list of pairs (guesser, confidence) @@ -95,7 +103,7 @@ class SingleNodeGuesser(object): # it will override it, otherwise it will leave the guess confidence # - if the guesser returns a simple dict as a guess and confidence is # specified, it will use it, or 1.0 otherwise - strategy = [ (self.guess_func, self.confidence) ] + strategy = [ (self.guess_func, self.confidence, self.args, self.kwargs) ] for node in mtree.unidentified_leaves(): find_and_split_node(node, strategy, self.logger) diff --git a/libs/guessit/transfo/guess_country.py b/libs/guessit/transfo/guess_country.py index 1d690698..aadb84f7 100755 --- a/libs/guessit/transfo/guess_country.py +++ b/libs/guessit/transfo/guess_country.py @@ -45,4 +45,4 @@ def process(mtree): except ValueError: continue - node.guess = Guess(country=country, confidence=1.0) + node.guess = Guess(country=country, confidence=1.0, raw=c) diff --git a/libs/guessit/transfo/guess_episodes_rexps.py b/libs/guessit/transfo/guess_episodes_rexps.py index 29562be2..30c2ca2f 100755 --- a/libs/guessit/transfo/guess_episodes_rexps.py +++ b/libs/guessit/transfo/guess_episodes_rexps.py @@ -40,27 +40,22 @@ def guess_episodes_rexps(string): for rexp, confidence, span_adjust in episode_rexps: match = re.search(rexp, string, re.IGNORECASE) if match: - guess = Guess(match.groupdict(), confidence=confidence) - span = (match.start() + span_adjust[0], + span = (match.start() + span_adjust[0], match.end() + span_adjust[1]) - - # episodes which have a season > 30 are most likely errors - # (Simpsons is at 24!) - if int(guess.get('season', 0)) > 30: - continue + guess = Guess(match.groupdict(), confidence=confidence, raw=string[span[0]:span[1]]) # decide whether we have only a single episode number or an # episode list if guess.get('episodeNumber'): eplist = number_list(guess['episodeNumber']) - guess.set('episodeNumber', eplist[0], confidence=confidence) + guess.set('episodeNumber', eplist[0], confidence=confidence, raw=string[span[0]:span[1]]) if len(eplist) > 1: - guess.set('episodeList', eplist, confidence=confidence) + guess.set('episodeList', eplist, confidence=confidence, raw=string[span[0]:span[1]]) if guess.get('bonusNumber'): eplist = number_list(guess['bonusNumber']) - guess.set('bonusNumber', eplist[0], confidence=confidence) + guess.set('bonusNumber', eplist[0], confidence=confidence, raw=string[span[0]:span[1]]) return guess, span diff --git a/libs/guessit/transfo/guess_filetype.py b/libs/guessit/transfo/guess_filetype.py index 4d98d016..4279c0b0 100755 --- a/libs/guessit/transfo/guess_filetype.py +++ b/libs/guessit/transfo/guess_filetype.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals from guessit import Guess -from guessit.patterns import (subtitle_exts, video_exts, episode_rexps, +from guessit.patterns import (subtitle_exts, info_exts, video_exts, episode_rexps, find_properties, compute_canonical_form) from guessit.date import valid_year from guessit.textutils import clean_string @@ -53,12 +53,16 @@ def guess_filetype(mtree, filetype): filetype_container[0] = 'episode' elif filetype_container[0] == 'subtitle': filetype_container[0] = 'episodesubtitle' + elif filetype_container[0] == 'info': + filetype_container[0] = 'episodeinfo' def upgrade_movie(): if filetype_container[0] == 'video': filetype_container[0] = 'movie' elif filetype_container[0] == 'subtitle': filetype_container[0] = 'moviesubtitle' + elif filetype_container[0] == 'info': + filetype_container[0] = 'movieinfo' def upgrade_subtitle(): if 'movie' in filetype_container[0]: @@ -68,6 +72,14 @@ def guess_filetype(mtree, filetype): else: filetype_container[0] = 'subtitle' + def upgrade_info(): + if 'movie' in filetype_container[0]: + filetype_container[0] = 'movieinfo' + elif 'episode' in filetype_container[0]: + filetype_container[0] = 'episodeinfo' + else: + filetype_container[0] = 'info' + def upgrade(type='unknown'): if filetype_container[0] == 'autodetect': filetype_container[0] = type @@ -78,6 +90,9 @@ def guess_filetype(mtree, filetype): if fileext in subtitle_exts: upgrade_subtitle() other = { 'container': fileext } + elif fileext in info_exts: + upgrade_info() + other = { 'container': fileext } elif fileext in video_exts: upgrade(type='video') other = { 'container': fileext } @@ -104,17 +119,20 @@ def guess_filetype(mtree, filetype): fname = clean_string(filename).lower() for m in MOVIES: if m in fname: + log.debug('Found in exception list of movies -> type = movie') upgrade_movie() for s in SERIES: if s in fname: + log.debug('Found in exception list of series -> type = episode') upgrade_episode() # now look whether there are some specific hints for episode vs movie - if filetype_container[0] in ('video', 'subtitle'): + if filetype_container[0] in ('video', 'subtitle', 'info'): # if we have an episode_rexp (eg: s02e13), it is an episode for rexp, _, _ in episode_rexps: match = re.search(rexp, filename, re.IGNORECASE) if match: + log.debug('Found matching regexp: "%s" (string = "%s") -> type = episode', rexp, match.group()) upgrade_episode() break @@ -133,24 +151,29 @@ def guess_filetype(mtree, filetype): possible = False if possible: + log.debug('Found possible episode number: %s (from string "%s") -> type = episode', epnumber, match.group()) upgrade_episode() # if we have certain properties characteristic of episodes, it is an ep for prop, value, _, _ in find_properties(filename): log.debug('prop: %s = %s' % (prop, value)) if prop == 'episodeFormat': + log.debug('Found characteristic property of episodes: %s = "%s"', prop, value) upgrade_episode() break elif compute_canonical_form('format', value) == 'DVB': + log.debug('Found characteristic property of episodes: %s = "%s"', prop, value) upgrade_episode() break # origin-specific type if 'tvu.org.ru' in filename: + log.debug('Found characteristic property of episodes: %s = "%s"', prop, value) upgrade_episode() # if no episode info found, assume it's a movie + log.debug('Nothing characteristic found, assuming type = movie') upgrade_movie() filetype = filetype_container[0] diff --git a/libs/guessit/transfo/guess_language.py b/libs/guessit/transfo/guess_language.py index 86c1cf55..648a06b1 100755 --- a/libs/guessit/transfo/guess_language.py +++ b/libs/guessit/transfo/guess_language.py @@ -22,22 +22,34 @@ from __future__ import unicode_literals from guessit import Guess from guessit.transfo import SingleNodeGuesser from guessit.language import search_language -from guessit.textutils import clean_string, find_words import logging log = logging.getLogger(__name__) -def guess_language(string): - language, span, confidence = search_language(string) +def guess_language(string, node, skip=None): + if skip: + relative_skip = [] + for entry in skip: + node_idx = entry['node_idx'] + span = entry['span'] + if node_idx == node.node_idx[:len(node_idx)]: + relative_span = (span[0] - node.offset + 1, span[1] - node.offset + 1) + relative_skip.append(relative_span) + skip = relative_skip + + language, span, confidence = search_language(string, skip=skip) if language: return (Guess({'language': language}, - confidence=confidence), + confidence=confidence, + raw= string[span[0]:span[1]]), span) return None, None +guess_language.use_node = True -def process(mtree): - SingleNodeGuesser(guess_language, None, log).process(mtree) + +def process(mtree, *args, **kwargs): + SingleNodeGuesser(guess_language, None, log, *args, **kwargs).process(mtree) # Note: 'language' is promoted to 'subtitleLanguage' in the post_process transfo diff --git a/libs/guessit/transfo/guess_movie_title_from_position.py b/libs/guessit/transfo/guess_movie_title_from_position.py index d2e2deb2..bcb42b45 100755 --- a/libs/guessit/transfo/guess_movie_title_from_position.py +++ b/libs/guessit/transfo/guess_movie_title_from_position.py @@ -29,7 +29,8 @@ log = logging.getLogger(__name__) def process(mtree): def found_property(node, name, value, confidence): node.guess = Guess({ name: value }, - confidence=confidence) + confidence=confidence, + raw=value) log.debug('Found with confidence %.2f: %s' % (confidence, node.guess)) def found_title(node, confidence): diff --git a/libs/guessit/transfo/guess_video_rexps.py b/libs/guessit/transfo/guess_video_rexps.py index 8ae9e6c6..1b511f15 100755 --- a/libs/guessit/transfo/guess_video_rexps.py +++ b/libs/guessit/transfo/guess_video_rexps.py @@ -38,9 +38,10 @@ def guess_video_rexps(string): # the soonest that we can catch it) if metadata.get('cdNumberTotal', -1) is None: del metadata['cdNumberTotal'] - return (Guess(metadata, confidence=confidence), - (match.start() + span_adjust[0], - match.end() + span_adjust[1] - 2)) + span = (match.start() + span_adjust[0], + match.end() + span_adjust[1] - 2) + return (Guess(metadata, confidence=confidence, raw=string[span[0]:span[1]]), + span) return None, None diff --git a/libs/guessit/transfo/guess_weak_episodes_rexps.py b/libs/guessit/transfo/guess_weak_episodes_rexps.py index 8436ade8..18306b43 100755 --- a/libs/guessit/transfo/guess_weak_episodes_rexps.py +++ b/libs/guessit/transfo/guess_weak_episodes_rexps.py @@ -48,9 +48,9 @@ def guess_weak_episodes_rexps(string, node): continue return Guess({ 'season': season, 'episodeNumber': epnum }, - confidence=0.6), span + confidence=0.6, raw=string[span[0]:span[1]]), span else: - return Guess(metadata, confidence=0.3), span + return Guess(metadata, confidence=0.3, raw=string[span[0]:span[1]]), span return None, None diff --git a/libs/html5lib/__init__.py b/libs/html5lib/__init__.py index 16537aad..66c1a8eb 100644 --- a/libs/html5lib/__init__.py +++ b/libs/html5lib/__init__.py @@ -1,4 +1,4 @@ -""" +""" HTML parsing library based on the WHATWG "HTML5" specification. The parser is designed to be compatible with existing HTML found in the wild and implements well-defined error recovery that @@ -8,10 +8,16 @@ Example usage: import html5lib f = open("my_document.html") -tree = html5lib.parse(f) +tree = html5lib.parse(f) """ -__version__ = "0.95-dev" -from html5parser import HTMLParser, parse, parseFragment -from treebuilders import getTreeBuilder -from treewalkers import getTreeWalker -from serializer import serialize + +from __future__ import absolute_import, division, unicode_literals + +from .html5parser import HTMLParser, parse, parseFragment +from .treebuilders import getTreeBuilder +from .treewalkers import getTreeWalker +from .serializer import serialize + +__all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder", + "getTreeWalker", "serialize"] +__version__ = "0.99" diff --git a/libs/html5lib/constants.py b/libs/html5lib/constants.py index b533018e..e7089846 100644 --- a/libs/html5lib/constants.py +++ b/libs/html5lib/constants.py @@ -1,302 +1,301 @@ -import string, gettext -_ = gettext.gettext +from __future__ import absolute_import, division, unicode_literals -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset +import string +import gettext +_ = gettext.gettext EOF = None E = { - "null-character": - _(u"Null character in input stream, replaced with U+FFFD."), - "invalid-codepoint": - _(u"Invalid codepoint in stream."), + "null-character": + _("Null character in input stream, replaced with U+FFFD."), + "invalid-codepoint": + _("Invalid codepoint in stream."), "incorrectly-placed-solidus": - _(u"Solidus (/) incorrectly placed in tag."), + _("Solidus (/) incorrectly placed in tag."), "incorrect-cr-newline-entity": - _(u"Incorrect CR newline entity, replaced with LF."), + _("Incorrect CR newline entity, replaced with LF."), "illegal-windows-1252-entity": - _(u"Entity used with illegal number (windows-1252 reference)."), + _("Entity used with illegal number (windows-1252 reference)."), "cant-convert-numeric-entity": - _(u"Numeric entity couldn't be converted to character " - u"(codepoint U+%(charAsInt)08x)."), + _("Numeric entity couldn't be converted to character " + "(codepoint U+%(charAsInt)08x)."), "illegal-codepoint-for-numeric-entity": - _(u"Numeric entity represents an illegal codepoint: " - u"U+%(charAsInt)08x."), + _("Numeric entity represents an illegal codepoint: " + "U+%(charAsInt)08x."), "numeric-entity-without-semicolon": - _(u"Numeric entity didn't end with ';'."), + _("Numeric entity didn't end with ';'."), "expected-numeric-entity-but-got-eof": - _(u"Numeric entity expected. Got end of file instead."), + _("Numeric entity expected. Got end of file instead."), "expected-numeric-entity": - _(u"Numeric entity expected but none found."), + _("Numeric entity expected but none found."), "named-entity-without-semicolon": - _(u"Named entity didn't end with ';'."), + _("Named entity didn't end with ';'."), "expected-named-entity": - _(u"Named entity expected. Got none."), + _("Named entity expected. Got none."), "attributes-in-end-tag": - _(u"End tag contains unexpected attributes."), + _("End tag contains unexpected attributes."), 'self-closing-flag-on-end-tag': - _(u"End tag contains unexpected self-closing flag."), + _("End tag contains unexpected self-closing flag."), "expected-tag-name-but-got-right-bracket": - _(u"Expected tag name. Got '>' instead."), + _("Expected tag name. Got '>' instead."), "expected-tag-name-but-got-question-mark": - _(u"Expected tag name. Got '?' instead. (HTML doesn't " - u"support processing instructions.)"), + _("Expected tag name. Got '?' instead. (HTML doesn't " + "support processing instructions.)"), "expected-tag-name": - _(u"Expected tag name. Got something else instead"), + _("Expected tag name. Got something else instead"), "expected-closing-tag-but-got-right-bracket": - _(u"Expected closing tag. Got '>' instead. Ignoring ''."), + _("Expected closing tag. Got '>' instead. Ignoring ''."), "expected-closing-tag-but-got-eof": - _(u"Expected closing tag. Unexpected end of file."), + _("Expected closing tag. Unexpected end of file."), "expected-closing-tag-but-got-char": - _(u"Expected closing tag. Unexpected character '%(data)s' found."), + _("Expected closing tag. Unexpected character '%(data)s' found."), "eof-in-tag-name": - _(u"Unexpected end of file in the tag name."), + _("Unexpected end of file in the tag name."), "expected-attribute-name-but-got-eof": - _(u"Unexpected end of file. Expected attribute name instead."), + _("Unexpected end of file. Expected attribute name instead."), "eof-in-attribute-name": - _(u"Unexpected end of file in attribute name."), + _("Unexpected end of file in attribute name."), "invalid-character-in-attribute-name": - _(u"Invalid chracter in attribute name"), + _("Invalid character in attribute name"), "duplicate-attribute": - _(u"Dropped duplicate attribute on tag."), + _("Dropped duplicate attribute on tag."), "expected-end-of-tag-name-but-got-eof": - _(u"Unexpected end of file. Expected = or end of tag."), + _("Unexpected end of file. Expected = or end of tag."), "expected-attribute-value-but-got-eof": - _(u"Unexpected end of file. Expected attribute value."), + _("Unexpected end of file. Expected attribute value."), "expected-attribute-value-but-got-right-bracket": - _(u"Expected attribute value. Got '>' instead."), + _("Expected attribute value. Got '>' instead."), 'equals-in-unquoted-attribute-value': - _(u"Unexpected = in unquoted attribute"), + _("Unexpected = in unquoted attribute"), 'unexpected-character-in-unquoted-attribute-value': - _(u"Unexpected character in unquoted attribute"), + _("Unexpected character in unquoted attribute"), "invalid-character-after-attribute-name": - _(u"Unexpected character after attribute name."), + _("Unexpected character after attribute name."), "unexpected-character-after-attribute-value": - _(u"Unexpected character after attribute value."), + _("Unexpected character after attribute value."), "eof-in-attribute-value-double-quote": - _(u"Unexpected end of file in attribute value (\")."), + _("Unexpected end of file in attribute value (\")."), "eof-in-attribute-value-single-quote": - _(u"Unexpected end of file in attribute value (')."), + _("Unexpected end of file in attribute value (')."), "eof-in-attribute-value-no-quotes": - _(u"Unexpected end of file in attribute value."), + _("Unexpected end of file in attribute value."), "unexpected-EOF-after-solidus-in-tag": - _(u"Unexpected end of file in tag. Expected >"), - "unexpected-character-after-soldius-in-tag": - _(u"Unexpected character after / in tag. Expected >"), + _("Unexpected end of file in tag. Expected >"), + "unexpected-character-after-solidus-in-tag": + _("Unexpected character after / in tag. Expected >"), "expected-dashes-or-doctype": - _(u"Expected '--' or 'DOCTYPE'. Not found."), + _("Expected '--' or 'DOCTYPE'. Not found."), "unexpected-bang-after-double-dash-in-comment": - _(u"Unexpected ! after -- in comment"), + _("Unexpected ! after -- in comment"), "unexpected-space-after-double-dash-in-comment": - _(u"Unexpected space after -- in comment"), + _("Unexpected space after -- in comment"), "incorrect-comment": - _(u"Incorrect comment."), + _("Incorrect comment."), "eof-in-comment": - _(u"Unexpected end of file in comment."), + _("Unexpected end of file in comment."), "eof-in-comment-end-dash": - _(u"Unexpected end of file in comment (-)"), + _("Unexpected end of file in comment (-)"), "unexpected-dash-after-double-dash-in-comment": - _(u"Unexpected '-' after '--' found in comment."), + _("Unexpected '-' after '--' found in comment."), "eof-in-comment-double-dash": - _(u"Unexpected end of file in comment (--)."), + _("Unexpected end of file in comment (--)."), "eof-in-comment-end-space-state": - _(u"Unexpected end of file in comment."), + _("Unexpected end of file in comment."), "eof-in-comment-end-bang-state": - _(u"Unexpected end of file in comment."), + _("Unexpected end of file in comment."), "unexpected-char-in-comment": - _(u"Unexpected character in comment found."), + _("Unexpected character in comment found."), "need-space-after-doctype": - _(u"No space after literal string 'DOCTYPE'."), + _("No space after literal string 'DOCTYPE'."), "expected-doctype-name-but-got-right-bracket": - _(u"Unexpected > character. Expected DOCTYPE name."), + _("Unexpected > character. Expected DOCTYPE name."), "expected-doctype-name-but-got-eof": - _(u"Unexpected end of file. Expected DOCTYPE name."), + _("Unexpected end of file. Expected DOCTYPE name."), "eof-in-doctype-name": - _(u"Unexpected end of file in DOCTYPE name."), + _("Unexpected end of file in DOCTYPE name."), "eof-in-doctype": - _(u"Unexpected end of file in DOCTYPE."), + _("Unexpected end of file in DOCTYPE."), "expected-space-or-right-bracket-in-doctype": - _(u"Expected space or '>'. Got '%(data)s'"), + _("Expected space or '>'. Got '%(data)s'"), "unexpected-end-of-doctype": - _(u"Unexpected end of DOCTYPE."), + _("Unexpected end of DOCTYPE."), "unexpected-char-in-doctype": - _(u"Unexpected character in DOCTYPE."), + _("Unexpected character in DOCTYPE."), "eof-in-innerhtml": - _(u"XXX innerHTML EOF"), + _("XXX innerHTML EOF"), "unexpected-doctype": - _(u"Unexpected DOCTYPE. Ignored."), + _("Unexpected DOCTYPE. Ignored."), "non-html-root": - _(u"html needs to be the first start tag."), + _("html needs to be the first start tag."), "expected-doctype-but-got-eof": - _(u"Unexpected End of file. Expected DOCTYPE."), + _("Unexpected End of file. Expected DOCTYPE."), "unknown-doctype": - _(u"Erroneous DOCTYPE."), + _("Erroneous DOCTYPE."), "expected-doctype-but-got-chars": - _(u"Unexpected non-space characters. Expected DOCTYPE."), + _("Unexpected non-space characters. Expected DOCTYPE."), "expected-doctype-but-got-start-tag": - _(u"Unexpected start tag (%(name)s). Expected DOCTYPE."), + _("Unexpected start tag (%(name)s). Expected DOCTYPE."), "expected-doctype-but-got-end-tag": - _(u"Unexpected end tag (%(name)s). Expected DOCTYPE."), + _("Unexpected end tag (%(name)s). Expected DOCTYPE."), "end-tag-after-implied-root": - _(u"Unexpected end tag (%(name)s) after the (implied) root element."), + _("Unexpected end tag (%(name)s) after the (implied) root element."), "expected-named-closing-tag-but-got-eof": - _(u"Unexpected end of file. Expected end tag (%(name)s)."), + _("Unexpected end of file. Expected end tag (%(name)s)."), "two-heads-are-not-better-than-one": - _(u"Unexpected start tag head in existing head. Ignored."), + _("Unexpected start tag head in existing head. Ignored."), "unexpected-end-tag": - _(u"Unexpected end tag (%(name)s). Ignored."), + _("Unexpected end tag (%(name)s). Ignored."), "unexpected-start-tag-out-of-my-head": - _(u"Unexpected start tag (%(name)s) that can be in head. Moved."), + _("Unexpected start tag (%(name)s) that can be in head. Moved."), "unexpected-start-tag": - _(u"Unexpected start tag (%(name)s)."), + _("Unexpected start tag (%(name)s)."), "missing-end-tag": - _(u"Missing end tag (%(name)s)."), + _("Missing end tag (%(name)s)."), "missing-end-tags": - _(u"Missing end tags (%(name)s)."), + _("Missing end tags (%(name)s)."), "unexpected-start-tag-implies-end-tag": - _(u"Unexpected start tag (%(startName)s) " - u"implies end tag (%(endName)s)."), + _("Unexpected start tag (%(startName)s) " + "implies end tag (%(endName)s)."), "unexpected-start-tag-treated-as": - _(u"Unexpected start tag (%(originalName)s). Treated as %(newName)s."), + _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."), "deprecated-tag": - _(u"Unexpected start tag %(name)s. Don't use it!"), + _("Unexpected start tag %(name)s. Don't use it!"), "unexpected-start-tag-ignored": - _(u"Unexpected start tag %(name)s. Ignored."), + _("Unexpected start tag %(name)s. Ignored."), "expected-one-end-tag-but-got-another": - _(u"Unexpected end tag (%(gotName)s). " - u"Missing end tag (%(expectedName)s)."), + _("Unexpected end tag (%(gotName)s). " + "Missing end tag (%(expectedName)s)."), "end-tag-too-early": - _(u"End tag (%(name)s) seen too early. Expected other end tag."), + _("End tag (%(name)s) seen too early. Expected other end tag."), "end-tag-too-early-named": - _(u"Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."), + _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."), "end-tag-too-early-ignored": - _(u"End tag (%(name)s) seen too early. Ignored."), + _("End tag (%(name)s) seen too early. Ignored."), "adoption-agency-1.1": - _(u"End tag (%(name)s) violates step 1, " - u"paragraph 1 of the adoption agency algorithm."), + _("End tag (%(name)s) violates step 1, " + "paragraph 1 of the adoption agency algorithm."), "adoption-agency-1.2": - _(u"End tag (%(name)s) violates step 1, " - u"paragraph 2 of the adoption agency algorithm."), + _("End tag (%(name)s) violates step 1, " + "paragraph 2 of the adoption agency algorithm."), "adoption-agency-1.3": - _(u"End tag (%(name)s) violates step 1, " - u"paragraph 3 of the adoption agency algorithm."), + _("End tag (%(name)s) violates step 1, " + "paragraph 3 of the adoption agency algorithm."), + "adoption-agency-4.4": + _("End tag (%(name)s) violates step 4, " + "paragraph 4 of the adoption agency algorithm."), "unexpected-end-tag-treated-as": - _(u"Unexpected end tag (%(originalName)s). Treated as %(newName)s."), + _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."), "no-end-tag": - _(u"This element (%(name)s) has no end tag."), + _("This element (%(name)s) has no end tag."), "unexpected-implied-end-tag-in-table": - _(u"Unexpected implied end tag (%(name)s) in the table phase."), + _("Unexpected implied end tag (%(name)s) in the table phase."), "unexpected-implied-end-tag-in-table-body": - _(u"Unexpected implied end tag (%(name)s) in the table body phase."), + _("Unexpected implied end tag (%(name)s) in the table body phase."), "unexpected-char-implies-table-voodoo": - _(u"Unexpected non-space characters in " - u"table context caused voodoo mode."), + _("Unexpected non-space characters in " + "table context caused voodoo mode."), "unexpected-hidden-input-in-table": - _(u"Unexpected input with type hidden in table context."), + _("Unexpected input with type hidden in table context."), "unexpected-form-in-table": - _(u"Unexpected form in table context."), + _("Unexpected form in table context."), "unexpected-start-tag-implies-table-voodoo": - _(u"Unexpected start tag (%(name)s) in " - u"table context caused voodoo mode."), + _("Unexpected start tag (%(name)s) in " + "table context caused voodoo mode."), "unexpected-end-tag-implies-table-voodoo": - _(u"Unexpected end tag (%(name)s) in " - u"table context caused voodoo mode."), + _("Unexpected end tag (%(name)s) in " + "table context caused voodoo mode."), "unexpected-cell-in-table-body": - _(u"Unexpected table cell start tag (%(name)s) " - u"in the table body phase."), + _("Unexpected table cell start tag (%(name)s) " + "in the table body phase."), "unexpected-cell-end-tag": - _(u"Got table cell end tag (%(name)s) " - u"while required end tags are missing."), + _("Got table cell end tag (%(name)s) " + "while required end tags are missing."), "unexpected-end-tag-in-table-body": - _(u"Unexpected end tag (%(name)s) in the table body phase. Ignored."), + _("Unexpected end tag (%(name)s) in the table body phase. Ignored."), "unexpected-implied-end-tag-in-table-row": - _(u"Unexpected implied end tag (%(name)s) in the table row phase."), + _("Unexpected implied end tag (%(name)s) in the table row phase."), "unexpected-end-tag-in-table-row": - _(u"Unexpected end tag (%(name)s) in the table row phase. Ignored."), + _("Unexpected end tag (%(name)s) in the table row phase. Ignored."), "unexpected-select-in-select": - _(u"Unexpected select start tag in the select phase " - u"treated as select end tag."), + _("Unexpected select start tag in the select phase " + "treated as select end tag."), "unexpected-input-in-select": - _(u"Unexpected input start tag in the select phase."), + _("Unexpected input start tag in the select phase."), "unexpected-start-tag-in-select": - _(u"Unexpected start tag token (%(name)s in the select phase. " - u"Ignored."), + _("Unexpected start tag token (%(name)s in the select phase. " + "Ignored."), "unexpected-end-tag-in-select": - _(u"Unexpected end tag (%(name)s) in the select phase. Ignored."), + _("Unexpected end tag (%(name)s) in the select phase. Ignored."), "unexpected-table-element-start-tag-in-select-in-table": - _(u"Unexpected table element start tag (%(name)s) in the select in table phase."), + _("Unexpected table element start tag (%(name)s) in the select in table phase."), "unexpected-table-element-end-tag-in-select-in-table": - _(u"Unexpected table element end tag (%(name)s) in the select in table phase."), + _("Unexpected table element end tag (%(name)s) in the select in table phase."), "unexpected-char-after-body": - _(u"Unexpected non-space characters in the after body phase."), + _("Unexpected non-space characters in the after body phase."), "unexpected-start-tag-after-body": - _(u"Unexpected start tag token (%(name)s)" - u" in the after body phase."), + _("Unexpected start tag token (%(name)s)" + " in the after body phase."), "unexpected-end-tag-after-body": - _(u"Unexpected end tag token (%(name)s)" - u" in the after body phase."), + _("Unexpected end tag token (%(name)s)" + " in the after body phase."), "unexpected-char-in-frameset": - _(u"Unepxected characters in the frameset phase. Characters ignored."), + _("Unexpected characters in the frameset phase. Characters ignored."), "unexpected-start-tag-in-frameset": - _(u"Unexpected start tag token (%(name)s)" - u" in the frameset phase. Ignored."), + _("Unexpected start tag token (%(name)s)" + " in the frameset phase. Ignored."), "unexpected-frameset-in-frameset-innerhtml": - _(u"Unexpected end tag token (frameset) " - u"in the frameset phase (innerHTML)."), + _("Unexpected end tag token (frameset) " + "in the frameset phase (innerHTML)."), "unexpected-end-tag-in-frameset": - _(u"Unexpected end tag token (%(name)s)" - u" in the frameset phase. Ignored."), + _("Unexpected end tag token (%(name)s)" + " in the frameset phase. Ignored."), "unexpected-char-after-frameset": - _(u"Unexpected non-space characters in the " - u"after frameset phase. Ignored."), + _("Unexpected non-space characters in the " + "after frameset phase. Ignored."), "unexpected-start-tag-after-frameset": - _(u"Unexpected start tag (%(name)s)" - u" in the after frameset phase. Ignored."), + _("Unexpected start tag (%(name)s)" + " in the after frameset phase. Ignored."), "unexpected-end-tag-after-frameset": - _(u"Unexpected end tag (%(name)s)" - u" in the after frameset phase. Ignored."), + _("Unexpected end tag (%(name)s)" + " in the after frameset phase. Ignored."), "unexpected-end-tag-after-body-innerhtml": - _(u"Unexpected end tag after body(innerHtml)"), + _("Unexpected end tag after body(innerHtml)"), "expected-eof-but-got-char": - _(u"Unexpected non-space characters. Expected end of file."), + _("Unexpected non-space characters. Expected end of file."), "expected-eof-but-got-start-tag": - _(u"Unexpected start tag (%(name)s)" - u". Expected end of file."), + _("Unexpected start tag (%(name)s)" + ". Expected end of file."), "expected-eof-but-got-end-tag": - _(u"Unexpected end tag (%(name)s)" - u". Expected end of file."), + _("Unexpected end tag (%(name)s)" + ". Expected end of file."), "eof-in-table": - _(u"Unexpected end of file. Expected table content."), + _("Unexpected end of file. Expected table content."), "eof-in-select": - _(u"Unexpected end of file. Expected select content."), + _("Unexpected end of file. Expected select content."), "eof-in-frameset": - _(u"Unexpected end of file. Expected frameset content."), + _("Unexpected end of file. Expected frameset content."), "eof-in-script-in-script": - _(u"Unexpected end of file. Expected script content."), + _("Unexpected end of file. Expected script content."), "eof-in-foreign-lands": - _(u"Unexpected end of file. Expected foreign content"), + _("Unexpected end of file. Expected foreign content"), "non-void-element-with-trailing-solidus": - _(u"Trailing solidus not allowed on element %(name)s"), + _("Trailing solidus not allowed on element %(name)s"), "unexpected-html-element-in-foreign-content": - _(u"Element %(name)s not allowed in a non-html context"), + _("Element %(name)s not allowed in a non-html context"), "unexpected-end-tag-before-html": - _(u"Unexpected end tag (%(name)s) before html."), + _("Unexpected end tag (%(name)s) before html."), "XXX-undefined-error": - (u"Undefined error (this sucks and should be fixed)"), + _("Undefined error (this sucks and should be fixed)"), } namespaces = { - "html":"http://www.w3.org/1999/xhtml", - "mathml":"http://www.w3.org/1998/Math/MathML", - "svg":"http://www.w3.org/2000/svg", - "xlink":"http://www.w3.org/1999/xlink", - "xml":"http://www.w3.org/XML/1998/namespace", - "xmlns":"http://www.w3.org/2000/xmlns/" + "html": "http://www.w3.org/1999/xhtml", + "mathml": "http://www.w3.org/1998/Math/MathML", + "svg": "http://www.w3.org/2000/svg", + "xlink": "http://www.w3.org/1999/xlink", + "xml": "http://www.w3.org/XML/1998/namespace", + "xmlns": "http://www.w3.org/2000/xmlns/" } scopingElements = frozenset(( @@ -380,7 +379,7 @@ specialElements = frozenset(( (namespaces["html"], "iframe"), # Note that image is commented out in the spec as "this isn't an # element that can end up on the stack, so it doesn't matter," - (namespaces["html"], "image"), + (namespaces["html"], "image"), (namespaces["html"], "img"), (namespaces["html"], "input"), (namespaces["html"], "isindex"), @@ -434,12 +433,30 @@ mathmlTextIntegrationPointElements = frozenset(( (namespaces["mathml"], "mtext") )) +adjustForeignAttributes = { + "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]), + "xlink:arcrole": ("xlink", "arcrole", namespaces["xlink"]), + "xlink:href": ("xlink", "href", namespaces["xlink"]), + "xlink:role": ("xlink", "role", namespaces["xlink"]), + "xlink:show": ("xlink", "show", namespaces["xlink"]), + "xlink:title": ("xlink", "title", namespaces["xlink"]), + "xlink:type": ("xlink", "type", namespaces["xlink"]), + "xml:base": ("xml", "base", namespaces["xml"]), + "xml:lang": ("xml", "lang", namespaces["xml"]), + "xml:space": ("xml", "space", namespaces["xml"]), + "xmlns": (None, "xmlns", namespaces["xmlns"]), + "xmlns:xlink": ("xmlns", "xlink", namespaces["xmlns"]) +} + +unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in + adjustForeignAttributes.items()]) + spaceCharacters = frozenset(( - u"\t", - u"\n", - u"\u000C", - u" ", - u"\r" + "\t", + "\n", + "\u000C", + " ", + "\r" )) tableInsertModeElements = frozenset(( @@ -456,8 +473,8 @@ asciiLetters = frozenset(string.ascii_letters) digits = frozenset(string.digits) hexDigits = frozenset(string.hexdigits) -asciiUpper2Lower = dict([(ord(c),ord(c.lower())) - for c in string.ascii_uppercase]) +asciiUpper2Lower = dict([(ord(c), ord(c.lower())) + for c in string.ascii_uppercase]) # Heading elements need to be ordered headingElements = ( @@ -503,8 +520,8 @@ booleanAttributes = { "": frozenset(("irrelevant",)), "style": frozenset(("scoped",)), "img": frozenset(("ismap",)), - "audio": frozenset(("autoplay","controls")), - "video": frozenset(("autoplay","controls")), + "audio": frozenset(("autoplay", "controls")), + "video": frozenset(("autoplay", "controls")), "script": frozenset(("defer", "async")), "details": frozenset(("open",)), "datagrid": frozenset(("multiple", "disabled")), @@ -523,2312 +540,2312 @@ booleanAttributes = { # entitiesWindows1252 has to be _ordered_ and needs to have an index. It # therefore can't be a frozenset. entitiesWindows1252 = ( - 8364, # 0x80 0x20AC EURO SIGN - 65533, # 0x81 UNDEFINED - 8218, # 0x82 0x201A SINGLE LOW-9 QUOTATION MARK - 402, # 0x83 0x0192 LATIN SMALL LETTER F WITH HOOK - 8222, # 0x84 0x201E DOUBLE LOW-9 QUOTATION MARK - 8230, # 0x85 0x2026 HORIZONTAL ELLIPSIS - 8224, # 0x86 0x2020 DAGGER - 8225, # 0x87 0x2021 DOUBLE DAGGER - 710, # 0x88 0x02C6 MODIFIER LETTER CIRCUMFLEX ACCENT - 8240, # 0x89 0x2030 PER MILLE SIGN - 352, # 0x8A 0x0160 LATIN CAPITAL LETTER S WITH CARON - 8249, # 0x8B 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK - 338, # 0x8C 0x0152 LATIN CAPITAL LIGATURE OE - 65533, # 0x8D UNDEFINED - 381, # 0x8E 0x017D LATIN CAPITAL LETTER Z WITH CARON - 65533, # 0x8F UNDEFINED - 65533, # 0x90 UNDEFINED - 8216, # 0x91 0x2018 LEFT SINGLE QUOTATION MARK - 8217, # 0x92 0x2019 RIGHT SINGLE QUOTATION MARK - 8220, # 0x93 0x201C LEFT DOUBLE QUOTATION MARK - 8221, # 0x94 0x201D RIGHT DOUBLE QUOTATION MARK - 8226, # 0x95 0x2022 BULLET - 8211, # 0x96 0x2013 EN DASH - 8212, # 0x97 0x2014 EM DASH - 732, # 0x98 0x02DC SMALL TILDE - 8482, # 0x99 0x2122 TRADE MARK SIGN - 353, # 0x9A 0x0161 LATIN SMALL LETTER S WITH CARON - 8250, # 0x9B 0x203A SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - 339, # 0x9C 0x0153 LATIN SMALL LIGATURE OE - 65533, # 0x9D UNDEFINED - 382, # 0x9E 0x017E LATIN SMALL LETTER Z WITH CARON - 376 # 0x9F 0x0178 LATIN CAPITAL LETTER Y WITH DIAERESIS + 8364, # 0x80 0x20AC EURO SIGN + 65533, # 0x81 UNDEFINED + 8218, # 0x82 0x201A SINGLE LOW-9 QUOTATION MARK + 402, # 0x83 0x0192 LATIN SMALL LETTER F WITH HOOK + 8222, # 0x84 0x201E DOUBLE LOW-9 QUOTATION MARK + 8230, # 0x85 0x2026 HORIZONTAL ELLIPSIS + 8224, # 0x86 0x2020 DAGGER + 8225, # 0x87 0x2021 DOUBLE DAGGER + 710, # 0x88 0x02C6 MODIFIER LETTER CIRCUMFLEX ACCENT + 8240, # 0x89 0x2030 PER MILLE SIGN + 352, # 0x8A 0x0160 LATIN CAPITAL LETTER S WITH CARON + 8249, # 0x8B 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 338, # 0x8C 0x0152 LATIN CAPITAL LIGATURE OE + 65533, # 0x8D UNDEFINED + 381, # 0x8E 0x017D LATIN CAPITAL LETTER Z WITH CARON + 65533, # 0x8F UNDEFINED + 65533, # 0x90 UNDEFINED + 8216, # 0x91 0x2018 LEFT SINGLE QUOTATION MARK + 8217, # 0x92 0x2019 RIGHT SINGLE QUOTATION MARK + 8220, # 0x93 0x201C LEFT DOUBLE QUOTATION MARK + 8221, # 0x94 0x201D RIGHT DOUBLE QUOTATION MARK + 8226, # 0x95 0x2022 BULLET + 8211, # 0x96 0x2013 EN DASH + 8212, # 0x97 0x2014 EM DASH + 732, # 0x98 0x02DC SMALL TILDE + 8482, # 0x99 0x2122 TRADE MARK SIGN + 353, # 0x9A 0x0161 LATIN SMALL LETTER S WITH CARON + 8250, # 0x9B 0x203A SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 339, # 0x9C 0x0153 LATIN SMALL LIGATURE OE + 65533, # 0x9D UNDEFINED + 382, # 0x9E 0x017E LATIN SMALL LETTER Z WITH CARON + 376 # 0x9F 0x0178 LATIN CAPITAL LETTER Y WITH DIAERESIS ) xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;')) entities = { - "AElig": u"\xc6", - "AElig;": u"\xc6", - "AMP": u"&", - "AMP;": u"&", - "Aacute": u"\xc1", - "Aacute;": u"\xc1", - "Abreve;": u"\u0102", - "Acirc": u"\xc2", - "Acirc;": u"\xc2", - "Acy;": u"\u0410", - "Afr;": u"\U0001d504", - "Agrave": u"\xc0", - "Agrave;": u"\xc0", - "Alpha;": u"\u0391", - "Amacr;": u"\u0100", - "And;": u"\u2a53", - "Aogon;": u"\u0104", - "Aopf;": u"\U0001d538", - "ApplyFunction;": u"\u2061", - "Aring": u"\xc5", - "Aring;": u"\xc5", - "Ascr;": u"\U0001d49c", - "Assign;": u"\u2254", - "Atilde": u"\xc3", - "Atilde;": u"\xc3", - "Auml": u"\xc4", - "Auml;": u"\xc4", - "Backslash;": u"\u2216", - "Barv;": u"\u2ae7", - "Barwed;": u"\u2306", - "Bcy;": u"\u0411", - "Because;": u"\u2235", - "Bernoullis;": u"\u212c", - "Beta;": u"\u0392", - "Bfr;": u"\U0001d505", - "Bopf;": u"\U0001d539", - "Breve;": u"\u02d8", - "Bscr;": u"\u212c", - "Bumpeq;": u"\u224e", - "CHcy;": u"\u0427", - "COPY": u"\xa9", - "COPY;": u"\xa9", - "Cacute;": u"\u0106", - "Cap;": u"\u22d2", - "CapitalDifferentialD;": u"\u2145", - "Cayleys;": u"\u212d", - "Ccaron;": u"\u010c", - "Ccedil": u"\xc7", - "Ccedil;": u"\xc7", - "Ccirc;": u"\u0108", - "Cconint;": u"\u2230", - "Cdot;": u"\u010a", - "Cedilla;": u"\xb8", - "CenterDot;": u"\xb7", - "Cfr;": u"\u212d", - "Chi;": u"\u03a7", - "CircleDot;": u"\u2299", - "CircleMinus;": u"\u2296", - "CirclePlus;": u"\u2295", - "CircleTimes;": u"\u2297", - "ClockwiseContourIntegral;": u"\u2232", - "CloseCurlyDoubleQuote;": u"\u201d", - "CloseCurlyQuote;": u"\u2019", - "Colon;": u"\u2237", - "Colone;": u"\u2a74", - "Congruent;": u"\u2261", - "Conint;": u"\u222f", - "ContourIntegral;": u"\u222e", - "Copf;": u"\u2102", - "Coproduct;": u"\u2210", - "CounterClockwiseContourIntegral;": u"\u2233", - "Cross;": u"\u2a2f", - "Cscr;": u"\U0001d49e", - "Cup;": u"\u22d3", - "CupCap;": u"\u224d", - "DD;": u"\u2145", - "DDotrahd;": u"\u2911", - "DJcy;": u"\u0402", - "DScy;": u"\u0405", - "DZcy;": u"\u040f", - "Dagger;": u"\u2021", - "Darr;": u"\u21a1", - "Dashv;": u"\u2ae4", - "Dcaron;": u"\u010e", - "Dcy;": u"\u0414", - "Del;": u"\u2207", - "Delta;": u"\u0394", - "Dfr;": u"\U0001d507", - "DiacriticalAcute;": u"\xb4", - "DiacriticalDot;": u"\u02d9", - "DiacriticalDoubleAcute;": u"\u02dd", - "DiacriticalGrave;": u"`", - "DiacriticalTilde;": u"\u02dc", - "Diamond;": u"\u22c4", - "DifferentialD;": u"\u2146", - "Dopf;": u"\U0001d53b", - "Dot;": u"\xa8", - "DotDot;": u"\u20dc", - "DotEqual;": u"\u2250", - "DoubleContourIntegral;": u"\u222f", - "DoubleDot;": u"\xa8", - "DoubleDownArrow;": u"\u21d3", - "DoubleLeftArrow;": u"\u21d0", - "DoubleLeftRightArrow;": u"\u21d4", - "DoubleLeftTee;": u"\u2ae4", - "DoubleLongLeftArrow;": u"\u27f8", - "DoubleLongLeftRightArrow;": u"\u27fa", - "DoubleLongRightArrow;": u"\u27f9", - "DoubleRightArrow;": u"\u21d2", - "DoubleRightTee;": u"\u22a8", - "DoubleUpArrow;": u"\u21d1", - "DoubleUpDownArrow;": u"\u21d5", - "DoubleVerticalBar;": u"\u2225", - "DownArrow;": u"\u2193", - "DownArrowBar;": u"\u2913", - "DownArrowUpArrow;": u"\u21f5", - "DownBreve;": u"\u0311", - "DownLeftRightVector;": u"\u2950", - "DownLeftTeeVector;": u"\u295e", - "DownLeftVector;": u"\u21bd", - "DownLeftVectorBar;": u"\u2956", - "DownRightTeeVector;": u"\u295f", - "DownRightVector;": u"\u21c1", - "DownRightVectorBar;": u"\u2957", - "DownTee;": u"\u22a4", - "DownTeeArrow;": u"\u21a7", - "Downarrow;": u"\u21d3", - "Dscr;": u"\U0001d49f", - "Dstrok;": u"\u0110", - "ENG;": u"\u014a", - "ETH": u"\xd0", - "ETH;": u"\xd0", - "Eacute": u"\xc9", - "Eacute;": u"\xc9", - "Ecaron;": u"\u011a", - "Ecirc": u"\xca", - "Ecirc;": u"\xca", - "Ecy;": u"\u042d", - "Edot;": u"\u0116", - "Efr;": u"\U0001d508", - "Egrave": u"\xc8", - "Egrave;": u"\xc8", - "Element;": u"\u2208", - "Emacr;": u"\u0112", - "EmptySmallSquare;": u"\u25fb", - "EmptyVerySmallSquare;": u"\u25ab", - "Eogon;": u"\u0118", - "Eopf;": u"\U0001d53c", - "Epsilon;": u"\u0395", - "Equal;": u"\u2a75", - "EqualTilde;": u"\u2242", - "Equilibrium;": u"\u21cc", - "Escr;": u"\u2130", - "Esim;": u"\u2a73", - "Eta;": u"\u0397", - "Euml": u"\xcb", - "Euml;": u"\xcb", - "Exists;": u"\u2203", - "ExponentialE;": u"\u2147", - "Fcy;": u"\u0424", - "Ffr;": u"\U0001d509", - "FilledSmallSquare;": u"\u25fc", - "FilledVerySmallSquare;": u"\u25aa", - "Fopf;": u"\U0001d53d", - "ForAll;": u"\u2200", - "Fouriertrf;": u"\u2131", - "Fscr;": u"\u2131", - "GJcy;": u"\u0403", - "GT": u">", - "GT;": u">", - "Gamma;": u"\u0393", - "Gammad;": u"\u03dc", - "Gbreve;": u"\u011e", - "Gcedil;": u"\u0122", - "Gcirc;": u"\u011c", - "Gcy;": u"\u0413", - "Gdot;": u"\u0120", - "Gfr;": u"\U0001d50a", - "Gg;": u"\u22d9", - "Gopf;": u"\U0001d53e", - "GreaterEqual;": u"\u2265", - "GreaterEqualLess;": u"\u22db", - "GreaterFullEqual;": u"\u2267", - "GreaterGreater;": u"\u2aa2", - "GreaterLess;": u"\u2277", - "GreaterSlantEqual;": u"\u2a7e", - "GreaterTilde;": u"\u2273", - "Gscr;": u"\U0001d4a2", - "Gt;": u"\u226b", - "HARDcy;": u"\u042a", - "Hacek;": u"\u02c7", - "Hat;": u"^", - "Hcirc;": u"\u0124", - "Hfr;": u"\u210c", - "HilbertSpace;": u"\u210b", - "Hopf;": u"\u210d", - "HorizontalLine;": u"\u2500", - "Hscr;": u"\u210b", - "Hstrok;": u"\u0126", - "HumpDownHump;": u"\u224e", - "HumpEqual;": u"\u224f", - "IEcy;": u"\u0415", - "IJlig;": u"\u0132", - "IOcy;": u"\u0401", - "Iacute": u"\xcd", - "Iacute;": u"\xcd", - "Icirc": u"\xce", - "Icirc;": u"\xce", - "Icy;": u"\u0418", - "Idot;": u"\u0130", - "Ifr;": u"\u2111", - "Igrave": u"\xcc", - "Igrave;": u"\xcc", - "Im;": u"\u2111", - "Imacr;": u"\u012a", - "ImaginaryI;": u"\u2148", - "Implies;": u"\u21d2", - "Int;": u"\u222c", - "Integral;": u"\u222b", - "Intersection;": u"\u22c2", - "InvisibleComma;": u"\u2063", - "InvisibleTimes;": u"\u2062", - "Iogon;": u"\u012e", - "Iopf;": u"\U0001d540", - "Iota;": u"\u0399", - "Iscr;": u"\u2110", - "Itilde;": u"\u0128", - "Iukcy;": u"\u0406", - "Iuml": u"\xcf", - "Iuml;": u"\xcf", - "Jcirc;": u"\u0134", - "Jcy;": u"\u0419", - "Jfr;": u"\U0001d50d", - "Jopf;": u"\U0001d541", - "Jscr;": u"\U0001d4a5", - "Jsercy;": u"\u0408", - "Jukcy;": u"\u0404", - "KHcy;": u"\u0425", - "KJcy;": u"\u040c", - "Kappa;": u"\u039a", - "Kcedil;": u"\u0136", - "Kcy;": u"\u041a", - "Kfr;": u"\U0001d50e", - "Kopf;": u"\U0001d542", - "Kscr;": u"\U0001d4a6", - "LJcy;": u"\u0409", - "LT": u"<", - "LT;": u"<", - "Lacute;": u"\u0139", - "Lambda;": u"\u039b", - "Lang;": u"\u27ea", - "Laplacetrf;": u"\u2112", - "Larr;": u"\u219e", - "Lcaron;": u"\u013d", - "Lcedil;": u"\u013b", - "Lcy;": u"\u041b", - "LeftAngleBracket;": u"\u27e8", - "LeftArrow;": u"\u2190", - "LeftArrowBar;": u"\u21e4", - "LeftArrowRightArrow;": u"\u21c6", - "LeftCeiling;": u"\u2308", - "LeftDoubleBracket;": u"\u27e6", - "LeftDownTeeVector;": u"\u2961", - "LeftDownVector;": u"\u21c3", - "LeftDownVectorBar;": u"\u2959", - "LeftFloor;": u"\u230a", - "LeftRightArrow;": u"\u2194", - "LeftRightVector;": u"\u294e", - "LeftTee;": u"\u22a3", - "LeftTeeArrow;": u"\u21a4", - "LeftTeeVector;": u"\u295a", - "LeftTriangle;": u"\u22b2", - "LeftTriangleBar;": u"\u29cf", - "LeftTriangleEqual;": u"\u22b4", - "LeftUpDownVector;": u"\u2951", - "LeftUpTeeVector;": u"\u2960", - "LeftUpVector;": u"\u21bf", - "LeftUpVectorBar;": u"\u2958", - "LeftVector;": u"\u21bc", - "LeftVectorBar;": u"\u2952", - "Leftarrow;": u"\u21d0", - "Leftrightarrow;": u"\u21d4", - "LessEqualGreater;": u"\u22da", - "LessFullEqual;": u"\u2266", - "LessGreater;": u"\u2276", - "LessLess;": u"\u2aa1", - "LessSlantEqual;": u"\u2a7d", - "LessTilde;": u"\u2272", - "Lfr;": u"\U0001d50f", - "Ll;": u"\u22d8", - "Lleftarrow;": u"\u21da", - "Lmidot;": u"\u013f", - "LongLeftArrow;": u"\u27f5", - "LongLeftRightArrow;": u"\u27f7", - "LongRightArrow;": u"\u27f6", - "Longleftarrow;": u"\u27f8", - "Longleftrightarrow;": u"\u27fa", - "Longrightarrow;": u"\u27f9", - "Lopf;": u"\U0001d543", - "LowerLeftArrow;": u"\u2199", - "LowerRightArrow;": u"\u2198", - "Lscr;": u"\u2112", - "Lsh;": u"\u21b0", - "Lstrok;": u"\u0141", - "Lt;": u"\u226a", - "Map;": u"\u2905", - "Mcy;": u"\u041c", - "MediumSpace;": u"\u205f", - "Mellintrf;": u"\u2133", - "Mfr;": u"\U0001d510", - "MinusPlus;": u"\u2213", - "Mopf;": u"\U0001d544", - "Mscr;": u"\u2133", - "Mu;": u"\u039c", - "NJcy;": u"\u040a", - "Nacute;": u"\u0143", - "Ncaron;": u"\u0147", - "Ncedil;": u"\u0145", - "Ncy;": u"\u041d", - "NegativeMediumSpace;": u"\u200b", - "NegativeThickSpace;": u"\u200b", - "NegativeThinSpace;": u"\u200b", - "NegativeVeryThinSpace;": u"\u200b", - "NestedGreaterGreater;": u"\u226b", - "NestedLessLess;": u"\u226a", - "NewLine;": u"\n", - "Nfr;": u"\U0001d511", - "NoBreak;": u"\u2060", - "NonBreakingSpace;": u"\xa0", - "Nopf;": u"\u2115", - "Not;": u"\u2aec", - "NotCongruent;": u"\u2262", - "NotCupCap;": u"\u226d", - "NotDoubleVerticalBar;": u"\u2226", - "NotElement;": u"\u2209", - "NotEqual;": u"\u2260", - "NotEqualTilde;": u"\u2242\u0338", - "NotExists;": u"\u2204", - "NotGreater;": u"\u226f", - "NotGreaterEqual;": u"\u2271", - "NotGreaterFullEqual;": u"\u2267\u0338", - "NotGreaterGreater;": u"\u226b\u0338", - "NotGreaterLess;": u"\u2279", - "NotGreaterSlantEqual;": u"\u2a7e\u0338", - "NotGreaterTilde;": u"\u2275", - "NotHumpDownHump;": u"\u224e\u0338", - "NotHumpEqual;": u"\u224f\u0338", - "NotLeftTriangle;": u"\u22ea", - "NotLeftTriangleBar;": u"\u29cf\u0338", - "NotLeftTriangleEqual;": u"\u22ec", - "NotLess;": u"\u226e", - "NotLessEqual;": u"\u2270", - "NotLessGreater;": u"\u2278", - "NotLessLess;": u"\u226a\u0338", - "NotLessSlantEqual;": u"\u2a7d\u0338", - "NotLessTilde;": u"\u2274", - "NotNestedGreaterGreater;": u"\u2aa2\u0338", - "NotNestedLessLess;": u"\u2aa1\u0338", - "NotPrecedes;": u"\u2280", - "NotPrecedesEqual;": u"\u2aaf\u0338", - "NotPrecedesSlantEqual;": u"\u22e0", - "NotReverseElement;": u"\u220c", - "NotRightTriangle;": u"\u22eb", - "NotRightTriangleBar;": u"\u29d0\u0338", - "NotRightTriangleEqual;": u"\u22ed", - "NotSquareSubset;": u"\u228f\u0338", - "NotSquareSubsetEqual;": u"\u22e2", - "NotSquareSuperset;": u"\u2290\u0338", - "NotSquareSupersetEqual;": u"\u22e3", - "NotSubset;": u"\u2282\u20d2", - "NotSubsetEqual;": u"\u2288", - "NotSucceeds;": u"\u2281", - "NotSucceedsEqual;": u"\u2ab0\u0338", - "NotSucceedsSlantEqual;": u"\u22e1", - "NotSucceedsTilde;": u"\u227f\u0338", - "NotSuperset;": u"\u2283\u20d2", - "NotSupersetEqual;": u"\u2289", - "NotTilde;": u"\u2241", - "NotTildeEqual;": u"\u2244", - "NotTildeFullEqual;": u"\u2247", - "NotTildeTilde;": u"\u2249", - "NotVerticalBar;": u"\u2224", - "Nscr;": u"\U0001d4a9", - "Ntilde": u"\xd1", - "Ntilde;": u"\xd1", - "Nu;": u"\u039d", - "OElig;": u"\u0152", - "Oacute": u"\xd3", - "Oacute;": u"\xd3", - "Ocirc": u"\xd4", - "Ocirc;": u"\xd4", - "Ocy;": u"\u041e", - "Odblac;": u"\u0150", - "Ofr;": u"\U0001d512", - "Ograve": u"\xd2", - "Ograve;": u"\xd2", - "Omacr;": u"\u014c", - "Omega;": u"\u03a9", - "Omicron;": u"\u039f", - "Oopf;": u"\U0001d546", - "OpenCurlyDoubleQuote;": u"\u201c", - "OpenCurlyQuote;": u"\u2018", - "Or;": u"\u2a54", - "Oscr;": u"\U0001d4aa", - "Oslash": u"\xd8", - "Oslash;": u"\xd8", - "Otilde": u"\xd5", - "Otilde;": u"\xd5", - "Otimes;": u"\u2a37", - "Ouml": u"\xd6", - "Ouml;": u"\xd6", - "OverBar;": u"\u203e", - "OverBrace;": u"\u23de", - "OverBracket;": u"\u23b4", - "OverParenthesis;": u"\u23dc", - "PartialD;": u"\u2202", - "Pcy;": u"\u041f", - "Pfr;": u"\U0001d513", - "Phi;": u"\u03a6", - "Pi;": u"\u03a0", - "PlusMinus;": u"\xb1", - "Poincareplane;": u"\u210c", - "Popf;": u"\u2119", - "Pr;": u"\u2abb", - "Precedes;": u"\u227a", - "PrecedesEqual;": u"\u2aaf", - "PrecedesSlantEqual;": u"\u227c", - "PrecedesTilde;": u"\u227e", - "Prime;": u"\u2033", - "Product;": u"\u220f", - "Proportion;": u"\u2237", - "Proportional;": u"\u221d", - "Pscr;": u"\U0001d4ab", - "Psi;": u"\u03a8", - "QUOT": u"\"", - "QUOT;": u"\"", - "Qfr;": u"\U0001d514", - "Qopf;": u"\u211a", - "Qscr;": u"\U0001d4ac", - "RBarr;": u"\u2910", - "REG": u"\xae", - "REG;": u"\xae", - "Racute;": u"\u0154", - "Rang;": u"\u27eb", - "Rarr;": u"\u21a0", - "Rarrtl;": u"\u2916", - "Rcaron;": u"\u0158", - "Rcedil;": u"\u0156", - "Rcy;": u"\u0420", - "Re;": u"\u211c", - "ReverseElement;": u"\u220b", - "ReverseEquilibrium;": u"\u21cb", - "ReverseUpEquilibrium;": u"\u296f", - "Rfr;": u"\u211c", - "Rho;": u"\u03a1", - "RightAngleBracket;": u"\u27e9", - "RightArrow;": u"\u2192", - "RightArrowBar;": u"\u21e5", - "RightArrowLeftArrow;": u"\u21c4", - "RightCeiling;": u"\u2309", - "RightDoubleBracket;": u"\u27e7", - "RightDownTeeVector;": u"\u295d", - "RightDownVector;": u"\u21c2", - "RightDownVectorBar;": u"\u2955", - "RightFloor;": u"\u230b", - "RightTee;": u"\u22a2", - "RightTeeArrow;": u"\u21a6", - "RightTeeVector;": u"\u295b", - "RightTriangle;": u"\u22b3", - "RightTriangleBar;": u"\u29d0", - "RightTriangleEqual;": u"\u22b5", - "RightUpDownVector;": u"\u294f", - "RightUpTeeVector;": u"\u295c", - "RightUpVector;": u"\u21be", - "RightUpVectorBar;": u"\u2954", - "RightVector;": u"\u21c0", - "RightVectorBar;": u"\u2953", - "Rightarrow;": u"\u21d2", - "Ropf;": u"\u211d", - "RoundImplies;": u"\u2970", - "Rrightarrow;": u"\u21db", - "Rscr;": u"\u211b", - "Rsh;": u"\u21b1", - "RuleDelayed;": u"\u29f4", - "SHCHcy;": u"\u0429", - "SHcy;": u"\u0428", - "SOFTcy;": u"\u042c", - "Sacute;": u"\u015a", - "Sc;": u"\u2abc", - "Scaron;": u"\u0160", - "Scedil;": u"\u015e", - "Scirc;": u"\u015c", - "Scy;": u"\u0421", - "Sfr;": u"\U0001d516", - "ShortDownArrow;": u"\u2193", - "ShortLeftArrow;": u"\u2190", - "ShortRightArrow;": u"\u2192", - "ShortUpArrow;": u"\u2191", - "Sigma;": u"\u03a3", - "SmallCircle;": u"\u2218", - "Sopf;": u"\U0001d54a", - "Sqrt;": u"\u221a", - "Square;": u"\u25a1", - "SquareIntersection;": u"\u2293", - "SquareSubset;": u"\u228f", - "SquareSubsetEqual;": u"\u2291", - "SquareSuperset;": u"\u2290", - "SquareSupersetEqual;": u"\u2292", - "SquareUnion;": u"\u2294", - "Sscr;": u"\U0001d4ae", - "Star;": u"\u22c6", - "Sub;": u"\u22d0", - "Subset;": u"\u22d0", - "SubsetEqual;": u"\u2286", - "Succeeds;": u"\u227b", - "SucceedsEqual;": u"\u2ab0", - "SucceedsSlantEqual;": u"\u227d", - "SucceedsTilde;": u"\u227f", - "SuchThat;": u"\u220b", - "Sum;": u"\u2211", - "Sup;": u"\u22d1", - "Superset;": u"\u2283", - "SupersetEqual;": u"\u2287", - "Supset;": u"\u22d1", - "THORN": u"\xde", - "THORN;": u"\xde", - "TRADE;": u"\u2122", - "TSHcy;": u"\u040b", - "TScy;": u"\u0426", - "Tab;": u"\t", - "Tau;": u"\u03a4", - "Tcaron;": u"\u0164", - "Tcedil;": u"\u0162", - "Tcy;": u"\u0422", - "Tfr;": u"\U0001d517", - "Therefore;": u"\u2234", - "Theta;": u"\u0398", - "ThickSpace;": u"\u205f\u200a", - "ThinSpace;": u"\u2009", - "Tilde;": u"\u223c", - "TildeEqual;": u"\u2243", - "TildeFullEqual;": u"\u2245", - "TildeTilde;": u"\u2248", - "Topf;": u"\U0001d54b", - "TripleDot;": u"\u20db", - "Tscr;": u"\U0001d4af", - "Tstrok;": u"\u0166", - "Uacute": u"\xda", - "Uacute;": u"\xda", - "Uarr;": u"\u219f", - "Uarrocir;": u"\u2949", - "Ubrcy;": u"\u040e", - "Ubreve;": u"\u016c", - "Ucirc": u"\xdb", - "Ucirc;": u"\xdb", - "Ucy;": u"\u0423", - "Udblac;": u"\u0170", - "Ufr;": u"\U0001d518", - "Ugrave": u"\xd9", - "Ugrave;": u"\xd9", - "Umacr;": u"\u016a", - "UnderBar;": u"_", - "UnderBrace;": u"\u23df", - "UnderBracket;": u"\u23b5", - "UnderParenthesis;": u"\u23dd", - "Union;": u"\u22c3", - "UnionPlus;": u"\u228e", - "Uogon;": u"\u0172", - "Uopf;": u"\U0001d54c", - "UpArrow;": u"\u2191", - "UpArrowBar;": u"\u2912", - "UpArrowDownArrow;": u"\u21c5", - "UpDownArrow;": u"\u2195", - "UpEquilibrium;": u"\u296e", - "UpTee;": u"\u22a5", - "UpTeeArrow;": u"\u21a5", - "Uparrow;": u"\u21d1", - "Updownarrow;": u"\u21d5", - "UpperLeftArrow;": u"\u2196", - "UpperRightArrow;": u"\u2197", - "Upsi;": u"\u03d2", - "Upsilon;": u"\u03a5", - "Uring;": u"\u016e", - "Uscr;": u"\U0001d4b0", - "Utilde;": u"\u0168", - "Uuml": u"\xdc", - "Uuml;": u"\xdc", - "VDash;": u"\u22ab", - "Vbar;": u"\u2aeb", - "Vcy;": u"\u0412", - "Vdash;": u"\u22a9", - "Vdashl;": u"\u2ae6", - "Vee;": u"\u22c1", - "Verbar;": u"\u2016", - "Vert;": u"\u2016", - "VerticalBar;": u"\u2223", - "VerticalLine;": u"|", - "VerticalSeparator;": u"\u2758", - "VerticalTilde;": u"\u2240", - "VeryThinSpace;": u"\u200a", - "Vfr;": u"\U0001d519", - "Vopf;": u"\U0001d54d", - "Vscr;": u"\U0001d4b1", - "Vvdash;": u"\u22aa", - "Wcirc;": u"\u0174", - "Wedge;": u"\u22c0", - "Wfr;": u"\U0001d51a", - "Wopf;": u"\U0001d54e", - "Wscr;": u"\U0001d4b2", - "Xfr;": u"\U0001d51b", - "Xi;": u"\u039e", - "Xopf;": u"\U0001d54f", - "Xscr;": u"\U0001d4b3", - "YAcy;": u"\u042f", - "YIcy;": u"\u0407", - "YUcy;": u"\u042e", - "Yacute": u"\xdd", - "Yacute;": u"\xdd", - "Ycirc;": u"\u0176", - "Ycy;": u"\u042b", - "Yfr;": u"\U0001d51c", - "Yopf;": u"\U0001d550", - "Yscr;": u"\U0001d4b4", - "Yuml;": u"\u0178", - "ZHcy;": u"\u0416", - "Zacute;": u"\u0179", - "Zcaron;": u"\u017d", - "Zcy;": u"\u0417", - "Zdot;": u"\u017b", - "ZeroWidthSpace;": u"\u200b", - "Zeta;": u"\u0396", - "Zfr;": u"\u2128", - "Zopf;": u"\u2124", - "Zscr;": u"\U0001d4b5", - "aacute": u"\xe1", - "aacute;": u"\xe1", - "abreve;": u"\u0103", - "ac;": u"\u223e", - "acE;": u"\u223e\u0333", - "acd;": u"\u223f", - "acirc": u"\xe2", - "acirc;": u"\xe2", - "acute": u"\xb4", - "acute;": u"\xb4", - "acy;": u"\u0430", - "aelig": u"\xe6", - "aelig;": u"\xe6", - "af;": u"\u2061", - "afr;": u"\U0001d51e", - "agrave": u"\xe0", - "agrave;": u"\xe0", - "alefsym;": u"\u2135", - "aleph;": u"\u2135", - "alpha;": u"\u03b1", - "amacr;": u"\u0101", - "amalg;": u"\u2a3f", - "amp": u"&", - "amp;": u"&", - "and;": u"\u2227", - "andand;": u"\u2a55", - "andd;": u"\u2a5c", - "andslope;": u"\u2a58", - "andv;": u"\u2a5a", - "ang;": u"\u2220", - "ange;": u"\u29a4", - "angle;": u"\u2220", - "angmsd;": u"\u2221", - "angmsdaa;": u"\u29a8", - "angmsdab;": u"\u29a9", - "angmsdac;": u"\u29aa", - "angmsdad;": u"\u29ab", - "angmsdae;": u"\u29ac", - "angmsdaf;": u"\u29ad", - "angmsdag;": u"\u29ae", - "angmsdah;": u"\u29af", - "angrt;": u"\u221f", - "angrtvb;": u"\u22be", - "angrtvbd;": u"\u299d", - "angsph;": u"\u2222", - "angst;": u"\xc5", - "angzarr;": u"\u237c", - "aogon;": u"\u0105", - "aopf;": u"\U0001d552", - "ap;": u"\u2248", - "apE;": u"\u2a70", - "apacir;": u"\u2a6f", - "ape;": u"\u224a", - "apid;": u"\u224b", - "apos;": u"'", - "approx;": u"\u2248", - "approxeq;": u"\u224a", - "aring": u"\xe5", - "aring;": u"\xe5", - "ascr;": u"\U0001d4b6", - "ast;": u"*", - "asymp;": u"\u2248", - "asympeq;": u"\u224d", - "atilde": u"\xe3", - "atilde;": u"\xe3", - "auml": u"\xe4", - "auml;": u"\xe4", - "awconint;": u"\u2233", - "awint;": u"\u2a11", - "bNot;": u"\u2aed", - "backcong;": u"\u224c", - "backepsilon;": u"\u03f6", - "backprime;": u"\u2035", - "backsim;": u"\u223d", - "backsimeq;": u"\u22cd", - "barvee;": u"\u22bd", - "barwed;": u"\u2305", - "barwedge;": u"\u2305", - "bbrk;": u"\u23b5", - "bbrktbrk;": u"\u23b6", - "bcong;": u"\u224c", - "bcy;": u"\u0431", - "bdquo;": u"\u201e", - "becaus;": u"\u2235", - "because;": u"\u2235", - "bemptyv;": u"\u29b0", - "bepsi;": u"\u03f6", - "bernou;": u"\u212c", - "beta;": u"\u03b2", - "beth;": u"\u2136", - "between;": u"\u226c", - "bfr;": u"\U0001d51f", - "bigcap;": u"\u22c2", - "bigcirc;": u"\u25ef", - "bigcup;": u"\u22c3", - "bigodot;": u"\u2a00", - "bigoplus;": u"\u2a01", - "bigotimes;": u"\u2a02", - "bigsqcup;": u"\u2a06", - "bigstar;": u"\u2605", - "bigtriangledown;": u"\u25bd", - "bigtriangleup;": u"\u25b3", - "biguplus;": u"\u2a04", - "bigvee;": u"\u22c1", - "bigwedge;": u"\u22c0", - "bkarow;": u"\u290d", - "blacklozenge;": u"\u29eb", - "blacksquare;": u"\u25aa", - "blacktriangle;": u"\u25b4", - "blacktriangledown;": u"\u25be", - "blacktriangleleft;": u"\u25c2", - "blacktriangleright;": u"\u25b8", - "blank;": u"\u2423", - "blk12;": u"\u2592", - "blk14;": u"\u2591", - "blk34;": u"\u2593", - "block;": u"\u2588", - "bne;": u"=\u20e5", - "bnequiv;": u"\u2261\u20e5", - "bnot;": u"\u2310", - "bopf;": u"\U0001d553", - "bot;": u"\u22a5", - "bottom;": u"\u22a5", - "bowtie;": u"\u22c8", - "boxDL;": u"\u2557", - "boxDR;": u"\u2554", - "boxDl;": u"\u2556", - "boxDr;": u"\u2553", - "boxH;": u"\u2550", - "boxHD;": u"\u2566", - "boxHU;": u"\u2569", - "boxHd;": u"\u2564", - "boxHu;": u"\u2567", - "boxUL;": u"\u255d", - "boxUR;": u"\u255a", - "boxUl;": u"\u255c", - "boxUr;": u"\u2559", - "boxV;": u"\u2551", - "boxVH;": u"\u256c", - "boxVL;": u"\u2563", - "boxVR;": u"\u2560", - "boxVh;": u"\u256b", - "boxVl;": u"\u2562", - "boxVr;": u"\u255f", - "boxbox;": u"\u29c9", - "boxdL;": u"\u2555", - "boxdR;": u"\u2552", - "boxdl;": u"\u2510", - "boxdr;": u"\u250c", - "boxh;": u"\u2500", - "boxhD;": u"\u2565", - "boxhU;": u"\u2568", - "boxhd;": u"\u252c", - "boxhu;": u"\u2534", - "boxminus;": u"\u229f", - "boxplus;": u"\u229e", - "boxtimes;": u"\u22a0", - "boxuL;": u"\u255b", - "boxuR;": u"\u2558", - "boxul;": u"\u2518", - "boxur;": u"\u2514", - "boxv;": u"\u2502", - "boxvH;": u"\u256a", - "boxvL;": u"\u2561", - "boxvR;": u"\u255e", - "boxvh;": u"\u253c", - "boxvl;": u"\u2524", - "boxvr;": u"\u251c", - "bprime;": u"\u2035", - "breve;": u"\u02d8", - "brvbar": u"\xa6", - "brvbar;": u"\xa6", - "bscr;": u"\U0001d4b7", - "bsemi;": u"\u204f", - "bsim;": u"\u223d", - "bsime;": u"\u22cd", - "bsol;": u"\\", - "bsolb;": u"\u29c5", - "bsolhsub;": u"\u27c8", - "bull;": u"\u2022", - "bullet;": u"\u2022", - "bump;": u"\u224e", - "bumpE;": u"\u2aae", - "bumpe;": u"\u224f", - "bumpeq;": u"\u224f", - "cacute;": u"\u0107", - "cap;": u"\u2229", - "capand;": u"\u2a44", - "capbrcup;": u"\u2a49", - "capcap;": u"\u2a4b", - "capcup;": u"\u2a47", - "capdot;": u"\u2a40", - "caps;": u"\u2229\ufe00", - "caret;": u"\u2041", - "caron;": u"\u02c7", - "ccaps;": u"\u2a4d", - "ccaron;": u"\u010d", - "ccedil": u"\xe7", - "ccedil;": u"\xe7", - "ccirc;": u"\u0109", - "ccups;": u"\u2a4c", - "ccupssm;": u"\u2a50", - "cdot;": u"\u010b", - "cedil": u"\xb8", - "cedil;": u"\xb8", - "cemptyv;": u"\u29b2", - "cent": u"\xa2", - "cent;": u"\xa2", - "centerdot;": u"\xb7", - "cfr;": u"\U0001d520", - "chcy;": u"\u0447", - "check;": u"\u2713", - "checkmark;": u"\u2713", - "chi;": u"\u03c7", - "cir;": u"\u25cb", - "cirE;": u"\u29c3", - "circ;": u"\u02c6", - "circeq;": u"\u2257", - "circlearrowleft;": u"\u21ba", - "circlearrowright;": u"\u21bb", - "circledR;": u"\xae", - "circledS;": u"\u24c8", - "circledast;": u"\u229b", - "circledcirc;": u"\u229a", - "circleddash;": u"\u229d", - "cire;": u"\u2257", - "cirfnint;": u"\u2a10", - "cirmid;": u"\u2aef", - "cirscir;": u"\u29c2", - "clubs;": u"\u2663", - "clubsuit;": u"\u2663", - "colon;": u":", - "colone;": u"\u2254", - "coloneq;": u"\u2254", - "comma;": u",", - "commat;": u"@", - "comp;": u"\u2201", - "compfn;": u"\u2218", - "complement;": u"\u2201", - "complexes;": u"\u2102", - "cong;": u"\u2245", - "congdot;": u"\u2a6d", - "conint;": u"\u222e", - "copf;": u"\U0001d554", - "coprod;": u"\u2210", - "copy": u"\xa9", - "copy;": u"\xa9", - "copysr;": u"\u2117", - "crarr;": u"\u21b5", - "cross;": u"\u2717", - "cscr;": u"\U0001d4b8", - "csub;": u"\u2acf", - "csube;": u"\u2ad1", - "csup;": u"\u2ad0", - "csupe;": u"\u2ad2", - "ctdot;": u"\u22ef", - "cudarrl;": u"\u2938", - "cudarrr;": u"\u2935", - "cuepr;": u"\u22de", - "cuesc;": u"\u22df", - "cularr;": u"\u21b6", - "cularrp;": u"\u293d", - "cup;": u"\u222a", - "cupbrcap;": u"\u2a48", - "cupcap;": u"\u2a46", - "cupcup;": u"\u2a4a", - "cupdot;": u"\u228d", - "cupor;": u"\u2a45", - "cups;": u"\u222a\ufe00", - "curarr;": u"\u21b7", - "curarrm;": u"\u293c", - "curlyeqprec;": u"\u22de", - "curlyeqsucc;": u"\u22df", - "curlyvee;": u"\u22ce", - "curlywedge;": u"\u22cf", - "curren": u"\xa4", - "curren;": u"\xa4", - "curvearrowleft;": u"\u21b6", - "curvearrowright;": u"\u21b7", - "cuvee;": u"\u22ce", - "cuwed;": u"\u22cf", - "cwconint;": u"\u2232", - "cwint;": u"\u2231", - "cylcty;": u"\u232d", - "dArr;": u"\u21d3", - "dHar;": u"\u2965", - "dagger;": u"\u2020", - "daleth;": u"\u2138", - "darr;": u"\u2193", - "dash;": u"\u2010", - "dashv;": u"\u22a3", - "dbkarow;": u"\u290f", - "dblac;": u"\u02dd", - "dcaron;": u"\u010f", - "dcy;": u"\u0434", - "dd;": u"\u2146", - "ddagger;": u"\u2021", - "ddarr;": u"\u21ca", - "ddotseq;": u"\u2a77", - "deg": u"\xb0", - "deg;": u"\xb0", - "delta;": u"\u03b4", - "demptyv;": u"\u29b1", - "dfisht;": u"\u297f", - "dfr;": u"\U0001d521", - "dharl;": u"\u21c3", - "dharr;": u"\u21c2", - "diam;": u"\u22c4", - "diamond;": u"\u22c4", - "diamondsuit;": u"\u2666", - "diams;": u"\u2666", - "die;": u"\xa8", - "digamma;": u"\u03dd", - "disin;": u"\u22f2", - "div;": u"\xf7", - "divide": u"\xf7", - "divide;": u"\xf7", - "divideontimes;": u"\u22c7", - "divonx;": u"\u22c7", - "djcy;": u"\u0452", - "dlcorn;": u"\u231e", - "dlcrop;": u"\u230d", - "dollar;": u"$", - "dopf;": u"\U0001d555", - "dot;": u"\u02d9", - "doteq;": u"\u2250", - "doteqdot;": u"\u2251", - "dotminus;": u"\u2238", - "dotplus;": u"\u2214", - "dotsquare;": u"\u22a1", - "doublebarwedge;": u"\u2306", - "downarrow;": u"\u2193", - "downdownarrows;": u"\u21ca", - "downharpoonleft;": u"\u21c3", - "downharpoonright;": u"\u21c2", - "drbkarow;": u"\u2910", - "drcorn;": u"\u231f", - "drcrop;": u"\u230c", - "dscr;": u"\U0001d4b9", - "dscy;": u"\u0455", - "dsol;": u"\u29f6", - "dstrok;": u"\u0111", - "dtdot;": u"\u22f1", - "dtri;": u"\u25bf", - "dtrif;": u"\u25be", - "duarr;": u"\u21f5", - "duhar;": u"\u296f", - "dwangle;": u"\u29a6", - "dzcy;": u"\u045f", - "dzigrarr;": u"\u27ff", - "eDDot;": u"\u2a77", - "eDot;": u"\u2251", - "eacute": u"\xe9", - "eacute;": u"\xe9", - "easter;": u"\u2a6e", - "ecaron;": u"\u011b", - "ecir;": u"\u2256", - "ecirc": u"\xea", - "ecirc;": u"\xea", - "ecolon;": u"\u2255", - "ecy;": u"\u044d", - "edot;": u"\u0117", - "ee;": u"\u2147", - "efDot;": u"\u2252", - "efr;": u"\U0001d522", - "eg;": u"\u2a9a", - "egrave": u"\xe8", - "egrave;": u"\xe8", - "egs;": u"\u2a96", - "egsdot;": u"\u2a98", - "el;": u"\u2a99", - "elinters;": u"\u23e7", - "ell;": u"\u2113", - "els;": u"\u2a95", - "elsdot;": u"\u2a97", - "emacr;": u"\u0113", - "empty;": u"\u2205", - "emptyset;": u"\u2205", - "emptyv;": u"\u2205", - "emsp13;": u"\u2004", - "emsp14;": u"\u2005", - "emsp;": u"\u2003", - "eng;": u"\u014b", - "ensp;": u"\u2002", - "eogon;": u"\u0119", - "eopf;": u"\U0001d556", - "epar;": u"\u22d5", - "eparsl;": u"\u29e3", - "eplus;": u"\u2a71", - "epsi;": u"\u03b5", - "epsilon;": u"\u03b5", - "epsiv;": u"\u03f5", - "eqcirc;": u"\u2256", - "eqcolon;": u"\u2255", - "eqsim;": u"\u2242", - "eqslantgtr;": u"\u2a96", - "eqslantless;": u"\u2a95", - "equals;": u"=", - "equest;": u"\u225f", - "equiv;": u"\u2261", - "equivDD;": u"\u2a78", - "eqvparsl;": u"\u29e5", - "erDot;": u"\u2253", - "erarr;": u"\u2971", - "escr;": u"\u212f", - "esdot;": u"\u2250", - "esim;": u"\u2242", - "eta;": u"\u03b7", - "eth": u"\xf0", - "eth;": u"\xf0", - "euml": u"\xeb", - "euml;": u"\xeb", - "euro;": u"\u20ac", - "excl;": u"!", - "exist;": u"\u2203", - "expectation;": u"\u2130", - "exponentiale;": u"\u2147", - "fallingdotseq;": u"\u2252", - "fcy;": u"\u0444", - "female;": u"\u2640", - "ffilig;": u"\ufb03", - "fflig;": u"\ufb00", - "ffllig;": u"\ufb04", - "ffr;": u"\U0001d523", - "filig;": u"\ufb01", - "fjlig;": u"fj", - "flat;": u"\u266d", - "fllig;": u"\ufb02", - "fltns;": u"\u25b1", - "fnof;": u"\u0192", - "fopf;": u"\U0001d557", - "forall;": u"\u2200", - "fork;": u"\u22d4", - "forkv;": u"\u2ad9", - "fpartint;": u"\u2a0d", - "frac12": u"\xbd", - "frac12;": u"\xbd", - "frac13;": u"\u2153", - "frac14": u"\xbc", - "frac14;": u"\xbc", - "frac15;": u"\u2155", - "frac16;": u"\u2159", - "frac18;": u"\u215b", - "frac23;": u"\u2154", - "frac25;": u"\u2156", - "frac34": u"\xbe", - "frac34;": u"\xbe", - "frac35;": u"\u2157", - "frac38;": u"\u215c", - "frac45;": u"\u2158", - "frac56;": u"\u215a", - "frac58;": u"\u215d", - "frac78;": u"\u215e", - "frasl;": u"\u2044", - "frown;": u"\u2322", - "fscr;": u"\U0001d4bb", - "gE;": u"\u2267", - "gEl;": u"\u2a8c", - "gacute;": u"\u01f5", - "gamma;": u"\u03b3", - "gammad;": u"\u03dd", - "gap;": u"\u2a86", - "gbreve;": u"\u011f", - "gcirc;": u"\u011d", - "gcy;": u"\u0433", - "gdot;": u"\u0121", - "ge;": u"\u2265", - "gel;": u"\u22db", - "geq;": u"\u2265", - "geqq;": u"\u2267", - "geqslant;": u"\u2a7e", - "ges;": u"\u2a7e", - "gescc;": u"\u2aa9", - "gesdot;": u"\u2a80", - "gesdoto;": u"\u2a82", - "gesdotol;": u"\u2a84", - "gesl;": u"\u22db\ufe00", - "gesles;": u"\u2a94", - "gfr;": u"\U0001d524", - "gg;": u"\u226b", - "ggg;": u"\u22d9", - "gimel;": u"\u2137", - "gjcy;": u"\u0453", - "gl;": u"\u2277", - "glE;": u"\u2a92", - "gla;": u"\u2aa5", - "glj;": u"\u2aa4", - "gnE;": u"\u2269", - "gnap;": u"\u2a8a", - "gnapprox;": u"\u2a8a", - "gne;": u"\u2a88", - "gneq;": u"\u2a88", - "gneqq;": u"\u2269", - "gnsim;": u"\u22e7", - "gopf;": u"\U0001d558", - "grave;": u"`", - "gscr;": u"\u210a", - "gsim;": u"\u2273", - "gsime;": u"\u2a8e", - "gsiml;": u"\u2a90", - "gt": u">", - "gt;": u">", - "gtcc;": u"\u2aa7", - "gtcir;": u"\u2a7a", - "gtdot;": u"\u22d7", - "gtlPar;": u"\u2995", - "gtquest;": u"\u2a7c", - "gtrapprox;": u"\u2a86", - "gtrarr;": u"\u2978", - "gtrdot;": u"\u22d7", - "gtreqless;": u"\u22db", - "gtreqqless;": u"\u2a8c", - "gtrless;": u"\u2277", - "gtrsim;": u"\u2273", - "gvertneqq;": u"\u2269\ufe00", - "gvnE;": u"\u2269\ufe00", - "hArr;": u"\u21d4", - "hairsp;": u"\u200a", - "half;": u"\xbd", - "hamilt;": u"\u210b", - "hardcy;": u"\u044a", - "harr;": u"\u2194", - "harrcir;": u"\u2948", - "harrw;": u"\u21ad", - "hbar;": u"\u210f", - "hcirc;": u"\u0125", - "hearts;": u"\u2665", - "heartsuit;": u"\u2665", - "hellip;": u"\u2026", - "hercon;": u"\u22b9", - "hfr;": u"\U0001d525", - "hksearow;": u"\u2925", - "hkswarow;": u"\u2926", - "hoarr;": u"\u21ff", - "homtht;": u"\u223b", - "hookleftarrow;": u"\u21a9", - "hookrightarrow;": u"\u21aa", - "hopf;": u"\U0001d559", - "horbar;": u"\u2015", - "hscr;": u"\U0001d4bd", - "hslash;": u"\u210f", - "hstrok;": u"\u0127", - "hybull;": u"\u2043", - "hyphen;": u"\u2010", - "iacute": u"\xed", - "iacute;": u"\xed", - "ic;": u"\u2063", - "icirc": u"\xee", - "icirc;": u"\xee", - "icy;": u"\u0438", - "iecy;": u"\u0435", - "iexcl": u"\xa1", - "iexcl;": u"\xa1", - "iff;": u"\u21d4", - "ifr;": u"\U0001d526", - "igrave": u"\xec", - "igrave;": u"\xec", - "ii;": u"\u2148", - "iiiint;": u"\u2a0c", - "iiint;": u"\u222d", - "iinfin;": u"\u29dc", - "iiota;": u"\u2129", - "ijlig;": u"\u0133", - "imacr;": u"\u012b", - "image;": u"\u2111", - "imagline;": u"\u2110", - "imagpart;": u"\u2111", - "imath;": u"\u0131", - "imof;": u"\u22b7", - "imped;": u"\u01b5", - "in;": u"\u2208", - "incare;": u"\u2105", - "infin;": u"\u221e", - "infintie;": u"\u29dd", - "inodot;": u"\u0131", - "int;": u"\u222b", - "intcal;": u"\u22ba", - "integers;": u"\u2124", - "intercal;": u"\u22ba", - "intlarhk;": u"\u2a17", - "intprod;": u"\u2a3c", - "iocy;": u"\u0451", - "iogon;": u"\u012f", - "iopf;": u"\U0001d55a", - "iota;": u"\u03b9", - "iprod;": u"\u2a3c", - "iquest": u"\xbf", - "iquest;": u"\xbf", - "iscr;": u"\U0001d4be", - "isin;": u"\u2208", - "isinE;": u"\u22f9", - "isindot;": u"\u22f5", - "isins;": u"\u22f4", - "isinsv;": u"\u22f3", - "isinv;": u"\u2208", - "it;": u"\u2062", - "itilde;": u"\u0129", - "iukcy;": u"\u0456", - "iuml": u"\xef", - "iuml;": u"\xef", - "jcirc;": u"\u0135", - "jcy;": u"\u0439", - "jfr;": u"\U0001d527", - "jmath;": u"\u0237", - "jopf;": u"\U0001d55b", - "jscr;": u"\U0001d4bf", - "jsercy;": u"\u0458", - "jukcy;": u"\u0454", - "kappa;": u"\u03ba", - "kappav;": u"\u03f0", - "kcedil;": u"\u0137", - "kcy;": u"\u043a", - "kfr;": u"\U0001d528", - "kgreen;": u"\u0138", - "khcy;": u"\u0445", - "kjcy;": u"\u045c", - "kopf;": u"\U0001d55c", - "kscr;": u"\U0001d4c0", - "lAarr;": u"\u21da", - "lArr;": u"\u21d0", - "lAtail;": u"\u291b", - "lBarr;": u"\u290e", - "lE;": u"\u2266", - "lEg;": u"\u2a8b", - "lHar;": u"\u2962", - "lacute;": u"\u013a", - "laemptyv;": u"\u29b4", - "lagran;": u"\u2112", - "lambda;": u"\u03bb", - "lang;": u"\u27e8", - "langd;": u"\u2991", - "langle;": u"\u27e8", - "lap;": u"\u2a85", - "laquo": u"\xab", - "laquo;": u"\xab", - "larr;": u"\u2190", - "larrb;": u"\u21e4", - "larrbfs;": u"\u291f", - "larrfs;": u"\u291d", - "larrhk;": u"\u21a9", - "larrlp;": u"\u21ab", - "larrpl;": u"\u2939", - "larrsim;": u"\u2973", - "larrtl;": u"\u21a2", - "lat;": u"\u2aab", - "latail;": u"\u2919", - "late;": u"\u2aad", - "lates;": u"\u2aad\ufe00", - "lbarr;": u"\u290c", - "lbbrk;": u"\u2772", - "lbrace;": u"{", - "lbrack;": u"[", - "lbrke;": u"\u298b", - "lbrksld;": u"\u298f", - "lbrkslu;": u"\u298d", - "lcaron;": u"\u013e", - "lcedil;": u"\u013c", - "lceil;": u"\u2308", - "lcub;": u"{", - "lcy;": u"\u043b", - "ldca;": u"\u2936", - "ldquo;": u"\u201c", - "ldquor;": u"\u201e", - "ldrdhar;": u"\u2967", - "ldrushar;": u"\u294b", - "ldsh;": u"\u21b2", - "le;": u"\u2264", - "leftarrow;": u"\u2190", - "leftarrowtail;": u"\u21a2", - "leftharpoondown;": u"\u21bd", - "leftharpoonup;": u"\u21bc", - "leftleftarrows;": u"\u21c7", - "leftrightarrow;": u"\u2194", - "leftrightarrows;": u"\u21c6", - "leftrightharpoons;": u"\u21cb", - "leftrightsquigarrow;": u"\u21ad", - "leftthreetimes;": u"\u22cb", - "leg;": u"\u22da", - "leq;": u"\u2264", - "leqq;": u"\u2266", - "leqslant;": u"\u2a7d", - "les;": u"\u2a7d", - "lescc;": u"\u2aa8", - "lesdot;": u"\u2a7f", - "lesdoto;": u"\u2a81", - "lesdotor;": u"\u2a83", - "lesg;": u"\u22da\ufe00", - "lesges;": u"\u2a93", - "lessapprox;": u"\u2a85", - "lessdot;": u"\u22d6", - "lesseqgtr;": u"\u22da", - "lesseqqgtr;": u"\u2a8b", - "lessgtr;": u"\u2276", - "lesssim;": u"\u2272", - "lfisht;": u"\u297c", - "lfloor;": u"\u230a", - "lfr;": u"\U0001d529", - "lg;": u"\u2276", - "lgE;": u"\u2a91", - "lhard;": u"\u21bd", - "lharu;": u"\u21bc", - "lharul;": u"\u296a", - "lhblk;": u"\u2584", - "ljcy;": u"\u0459", - "ll;": u"\u226a", - "llarr;": u"\u21c7", - "llcorner;": u"\u231e", - "llhard;": u"\u296b", - "lltri;": u"\u25fa", - "lmidot;": u"\u0140", - "lmoust;": u"\u23b0", - "lmoustache;": u"\u23b0", - "lnE;": u"\u2268", - "lnap;": u"\u2a89", - "lnapprox;": u"\u2a89", - "lne;": u"\u2a87", - "lneq;": u"\u2a87", - "lneqq;": u"\u2268", - "lnsim;": u"\u22e6", - "loang;": u"\u27ec", - "loarr;": u"\u21fd", - "lobrk;": u"\u27e6", - "longleftarrow;": u"\u27f5", - "longleftrightarrow;": u"\u27f7", - "longmapsto;": u"\u27fc", - "longrightarrow;": u"\u27f6", - "looparrowleft;": u"\u21ab", - "looparrowright;": u"\u21ac", - "lopar;": u"\u2985", - "lopf;": u"\U0001d55d", - "loplus;": u"\u2a2d", - "lotimes;": u"\u2a34", - "lowast;": u"\u2217", - "lowbar;": u"_", - "loz;": u"\u25ca", - "lozenge;": u"\u25ca", - "lozf;": u"\u29eb", - "lpar;": u"(", - "lparlt;": u"\u2993", - "lrarr;": u"\u21c6", - "lrcorner;": u"\u231f", - "lrhar;": u"\u21cb", - "lrhard;": u"\u296d", - "lrm;": u"\u200e", - "lrtri;": u"\u22bf", - "lsaquo;": u"\u2039", - "lscr;": u"\U0001d4c1", - "lsh;": u"\u21b0", - "lsim;": u"\u2272", - "lsime;": u"\u2a8d", - "lsimg;": u"\u2a8f", - "lsqb;": u"[", - "lsquo;": u"\u2018", - "lsquor;": u"\u201a", - "lstrok;": u"\u0142", - "lt": u"<", - "lt;": u"<", - "ltcc;": u"\u2aa6", - "ltcir;": u"\u2a79", - "ltdot;": u"\u22d6", - "lthree;": u"\u22cb", - "ltimes;": u"\u22c9", - "ltlarr;": u"\u2976", - "ltquest;": u"\u2a7b", - "ltrPar;": u"\u2996", - "ltri;": u"\u25c3", - "ltrie;": u"\u22b4", - "ltrif;": u"\u25c2", - "lurdshar;": u"\u294a", - "luruhar;": u"\u2966", - "lvertneqq;": u"\u2268\ufe00", - "lvnE;": u"\u2268\ufe00", - "mDDot;": u"\u223a", - "macr": u"\xaf", - "macr;": u"\xaf", - "male;": u"\u2642", - "malt;": u"\u2720", - "maltese;": u"\u2720", - "map;": u"\u21a6", - "mapsto;": u"\u21a6", - "mapstodown;": u"\u21a7", - "mapstoleft;": u"\u21a4", - "mapstoup;": u"\u21a5", - "marker;": u"\u25ae", - "mcomma;": u"\u2a29", - "mcy;": u"\u043c", - "mdash;": u"\u2014", - "measuredangle;": u"\u2221", - "mfr;": u"\U0001d52a", - "mho;": u"\u2127", - "micro": u"\xb5", - "micro;": u"\xb5", - "mid;": u"\u2223", - "midast;": u"*", - "midcir;": u"\u2af0", - "middot": u"\xb7", - "middot;": u"\xb7", - "minus;": u"\u2212", - "minusb;": u"\u229f", - "minusd;": u"\u2238", - "minusdu;": u"\u2a2a", - "mlcp;": u"\u2adb", - "mldr;": u"\u2026", - "mnplus;": u"\u2213", - "models;": u"\u22a7", - "mopf;": u"\U0001d55e", - "mp;": u"\u2213", - "mscr;": u"\U0001d4c2", - "mstpos;": u"\u223e", - "mu;": u"\u03bc", - "multimap;": u"\u22b8", - "mumap;": u"\u22b8", - "nGg;": u"\u22d9\u0338", - "nGt;": u"\u226b\u20d2", - "nGtv;": u"\u226b\u0338", - "nLeftarrow;": u"\u21cd", - "nLeftrightarrow;": u"\u21ce", - "nLl;": u"\u22d8\u0338", - "nLt;": u"\u226a\u20d2", - "nLtv;": u"\u226a\u0338", - "nRightarrow;": u"\u21cf", - "nVDash;": u"\u22af", - "nVdash;": u"\u22ae", - "nabla;": u"\u2207", - "nacute;": u"\u0144", - "nang;": u"\u2220\u20d2", - "nap;": u"\u2249", - "napE;": u"\u2a70\u0338", - "napid;": u"\u224b\u0338", - "napos;": u"\u0149", - "napprox;": u"\u2249", - "natur;": u"\u266e", - "natural;": u"\u266e", - "naturals;": u"\u2115", - "nbsp": u"\xa0", - "nbsp;": u"\xa0", - "nbump;": u"\u224e\u0338", - "nbumpe;": u"\u224f\u0338", - "ncap;": u"\u2a43", - "ncaron;": u"\u0148", - "ncedil;": u"\u0146", - "ncong;": u"\u2247", - "ncongdot;": u"\u2a6d\u0338", - "ncup;": u"\u2a42", - "ncy;": u"\u043d", - "ndash;": u"\u2013", - "ne;": u"\u2260", - "neArr;": u"\u21d7", - "nearhk;": u"\u2924", - "nearr;": u"\u2197", - "nearrow;": u"\u2197", - "nedot;": u"\u2250\u0338", - "nequiv;": u"\u2262", - "nesear;": u"\u2928", - "nesim;": u"\u2242\u0338", - "nexist;": u"\u2204", - "nexists;": u"\u2204", - "nfr;": u"\U0001d52b", - "ngE;": u"\u2267\u0338", - "nge;": u"\u2271", - "ngeq;": u"\u2271", - "ngeqq;": u"\u2267\u0338", - "ngeqslant;": u"\u2a7e\u0338", - "nges;": u"\u2a7e\u0338", - "ngsim;": u"\u2275", - "ngt;": u"\u226f", - "ngtr;": u"\u226f", - "nhArr;": u"\u21ce", - "nharr;": u"\u21ae", - "nhpar;": u"\u2af2", - "ni;": u"\u220b", - "nis;": u"\u22fc", - "nisd;": u"\u22fa", - "niv;": u"\u220b", - "njcy;": u"\u045a", - "nlArr;": u"\u21cd", - "nlE;": u"\u2266\u0338", - "nlarr;": u"\u219a", - "nldr;": u"\u2025", - "nle;": u"\u2270", - "nleftarrow;": u"\u219a", - "nleftrightarrow;": u"\u21ae", - "nleq;": u"\u2270", - "nleqq;": u"\u2266\u0338", - "nleqslant;": u"\u2a7d\u0338", - "nles;": u"\u2a7d\u0338", - "nless;": u"\u226e", - "nlsim;": u"\u2274", - "nlt;": u"\u226e", - "nltri;": u"\u22ea", - "nltrie;": u"\u22ec", - "nmid;": u"\u2224", - "nopf;": u"\U0001d55f", - "not": u"\xac", - "not;": u"\xac", - "notin;": u"\u2209", - "notinE;": u"\u22f9\u0338", - "notindot;": u"\u22f5\u0338", - "notinva;": u"\u2209", - "notinvb;": u"\u22f7", - "notinvc;": u"\u22f6", - "notni;": u"\u220c", - "notniva;": u"\u220c", - "notnivb;": u"\u22fe", - "notnivc;": u"\u22fd", - "npar;": u"\u2226", - "nparallel;": u"\u2226", - "nparsl;": u"\u2afd\u20e5", - "npart;": u"\u2202\u0338", - "npolint;": u"\u2a14", - "npr;": u"\u2280", - "nprcue;": u"\u22e0", - "npre;": u"\u2aaf\u0338", - "nprec;": u"\u2280", - "npreceq;": u"\u2aaf\u0338", - "nrArr;": u"\u21cf", - "nrarr;": u"\u219b", - "nrarrc;": u"\u2933\u0338", - "nrarrw;": u"\u219d\u0338", - "nrightarrow;": u"\u219b", - "nrtri;": u"\u22eb", - "nrtrie;": u"\u22ed", - "nsc;": u"\u2281", - "nsccue;": u"\u22e1", - "nsce;": u"\u2ab0\u0338", - "nscr;": u"\U0001d4c3", - "nshortmid;": u"\u2224", - "nshortparallel;": u"\u2226", - "nsim;": u"\u2241", - "nsime;": u"\u2244", - "nsimeq;": u"\u2244", - "nsmid;": u"\u2224", - "nspar;": u"\u2226", - "nsqsube;": u"\u22e2", - "nsqsupe;": u"\u22e3", - "nsub;": u"\u2284", - "nsubE;": u"\u2ac5\u0338", - "nsube;": u"\u2288", - "nsubset;": u"\u2282\u20d2", - "nsubseteq;": u"\u2288", - "nsubseteqq;": u"\u2ac5\u0338", - "nsucc;": u"\u2281", - "nsucceq;": u"\u2ab0\u0338", - "nsup;": u"\u2285", - "nsupE;": u"\u2ac6\u0338", - "nsupe;": u"\u2289", - "nsupset;": u"\u2283\u20d2", - "nsupseteq;": u"\u2289", - "nsupseteqq;": u"\u2ac6\u0338", - "ntgl;": u"\u2279", - "ntilde": u"\xf1", - "ntilde;": u"\xf1", - "ntlg;": u"\u2278", - "ntriangleleft;": u"\u22ea", - "ntrianglelefteq;": u"\u22ec", - "ntriangleright;": u"\u22eb", - "ntrianglerighteq;": u"\u22ed", - "nu;": u"\u03bd", - "num;": u"#", - "numero;": u"\u2116", - "numsp;": u"\u2007", - "nvDash;": u"\u22ad", - "nvHarr;": u"\u2904", - "nvap;": u"\u224d\u20d2", - "nvdash;": u"\u22ac", - "nvge;": u"\u2265\u20d2", - "nvgt;": u">\u20d2", - "nvinfin;": u"\u29de", - "nvlArr;": u"\u2902", - "nvle;": u"\u2264\u20d2", - "nvlt;": u"<\u20d2", - "nvltrie;": u"\u22b4\u20d2", - "nvrArr;": u"\u2903", - "nvrtrie;": u"\u22b5\u20d2", - "nvsim;": u"\u223c\u20d2", - "nwArr;": u"\u21d6", - "nwarhk;": u"\u2923", - "nwarr;": u"\u2196", - "nwarrow;": u"\u2196", - "nwnear;": u"\u2927", - "oS;": u"\u24c8", - "oacute": u"\xf3", - "oacute;": u"\xf3", - "oast;": u"\u229b", - "ocir;": u"\u229a", - "ocirc": u"\xf4", - "ocirc;": u"\xf4", - "ocy;": u"\u043e", - "odash;": u"\u229d", - "odblac;": u"\u0151", - "odiv;": u"\u2a38", - "odot;": u"\u2299", - "odsold;": u"\u29bc", - "oelig;": u"\u0153", - "ofcir;": u"\u29bf", - "ofr;": u"\U0001d52c", - "ogon;": u"\u02db", - "ograve": u"\xf2", - "ograve;": u"\xf2", - "ogt;": u"\u29c1", - "ohbar;": u"\u29b5", - "ohm;": u"\u03a9", - "oint;": u"\u222e", - "olarr;": u"\u21ba", - "olcir;": u"\u29be", - "olcross;": u"\u29bb", - "oline;": u"\u203e", - "olt;": u"\u29c0", - "omacr;": u"\u014d", - "omega;": u"\u03c9", - "omicron;": u"\u03bf", - "omid;": u"\u29b6", - "ominus;": u"\u2296", - "oopf;": u"\U0001d560", - "opar;": u"\u29b7", - "operp;": u"\u29b9", - "oplus;": u"\u2295", - "or;": u"\u2228", - "orarr;": u"\u21bb", - "ord;": u"\u2a5d", - "order;": u"\u2134", - "orderof;": u"\u2134", - "ordf": u"\xaa", - "ordf;": u"\xaa", - "ordm": u"\xba", - "ordm;": u"\xba", - "origof;": u"\u22b6", - "oror;": u"\u2a56", - "orslope;": u"\u2a57", - "orv;": u"\u2a5b", - "oscr;": u"\u2134", - "oslash": u"\xf8", - "oslash;": u"\xf8", - "osol;": u"\u2298", - "otilde": u"\xf5", - "otilde;": u"\xf5", - "otimes;": u"\u2297", - "otimesas;": u"\u2a36", - "ouml": u"\xf6", - "ouml;": u"\xf6", - "ovbar;": u"\u233d", - "par;": u"\u2225", - "para": u"\xb6", - "para;": u"\xb6", - "parallel;": u"\u2225", - "parsim;": u"\u2af3", - "parsl;": u"\u2afd", - "part;": u"\u2202", - "pcy;": u"\u043f", - "percnt;": u"%", - "period;": u".", - "permil;": u"\u2030", - "perp;": u"\u22a5", - "pertenk;": u"\u2031", - "pfr;": u"\U0001d52d", - "phi;": u"\u03c6", - "phiv;": u"\u03d5", - "phmmat;": u"\u2133", - "phone;": u"\u260e", - "pi;": u"\u03c0", - "pitchfork;": u"\u22d4", - "piv;": u"\u03d6", - "planck;": u"\u210f", - "planckh;": u"\u210e", - "plankv;": u"\u210f", - "plus;": u"+", - "plusacir;": u"\u2a23", - "plusb;": u"\u229e", - "pluscir;": u"\u2a22", - "plusdo;": u"\u2214", - "plusdu;": u"\u2a25", - "pluse;": u"\u2a72", - "plusmn": u"\xb1", - "plusmn;": u"\xb1", - "plussim;": u"\u2a26", - "plustwo;": u"\u2a27", - "pm;": u"\xb1", - "pointint;": u"\u2a15", - "popf;": u"\U0001d561", - "pound": u"\xa3", - "pound;": u"\xa3", - "pr;": u"\u227a", - "prE;": u"\u2ab3", - "prap;": u"\u2ab7", - "prcue;": u"\u227c", - "pre;": u"\u2aaf", - "prec;": u"\u227a", - "precapprox;": u"\u2ab7", - "preccurlyeq;": u"\u227c", - "preceq;": u"\u2aaf", - "precnapprox;": u"\u2ab9", - "precneqq;": u"\u2ab5", - "precnsim;": u"\u22e8", - "precsim;": u"\u227e", - "prime;": u"\u2032", - "primes;": u"\u2119", - "prnE;": u"\u2ab5", - "prnap;": u"\u2ab9", - "prnsim;": u"\u22e8", - "prod;": u"\u220f", - "profalar;": u"\u232e", - "profline;": u"\u2312", - "profsurf;": u"\u2313", - "prop;": u"\u221d", - "propto;": u"\u221d", - "prsim;": u"\u227e", - "prurel;": u"\u22b0", - "pscr;": u"\U0001d4c5", - "psi;": u"\u03c8", - "puncsp;": u"\u2008", - "qfr;": u"\U0001d52e", - "qint;": u"\u2a0c", - "qopf;": u"\U0001d562", - "qprime;": u"\u2057", - "qscr;": u"\U0001d4c6", - "quaternions;": u"\u210d", - "quatint;": u"\u2a16", - "quest;": u"?", - "questeq;": u"\u225f", - "quot": u"\"", - "quot;": u"\"", - "rAarr;": u"\u21db", - "rArr;": u"\u21d2", - "rAtail;": u"\u291c", - "rBarr;": u"\u290f", - "rHar;": u"\u2964", - "race;": u"\u223d\u0331", - "racute;": u"\u0155", - "radic;": u"\u221a", - "raemptyv;": u"\u29b3", - "rang;": u"\u27e9", - "rangd;": u"\u2992", - "range;": u"\u29a5", - "rangle;": u"\u27e9", - "raquo": u"\xbb", - "raquo;": u"\xbb", - "rarr;": u"\u2192", - "rarrap;": u"\u2975", - "rarrb;": u"\u21e5", - "rarrbfs;": u"\u2920", - "rarrc;": u"\u2933", - "rarrfs;": u"\u291e", - "rarrhk;": u"\u21aa", - "rarrlp;": u"\u21ac", - "rarrpl;": u"\u2945", - "rarrsim;": u"\u2974", - "rarrtl;": u"\u21a3", - "rarrw;": u"\u219d", - "ratail;": u"\u291a", - "ratio;": u"\u2236", - "rationals;": u"\u211a", - "rbarr;": u"\u290d", - "rbbrk;": u"\u2773", - "rbrace;": u"}", - "rbrack;": u"]", - "rbrke;": u"\u298c", - "rbrksld;": u"\u298e", - "rbrkslu;": u"\u2990", - "rcaron;": u"\u0159", - "rcedil;": u"\u0157", - "rceil;": u"\u2309", - "rcub;": u"}", - "rcy;": u"\u0440", - "rdca;": u"\u2937", - "rdldhar;": u"\u2969", - "rdquo;": u"\u201d", - "rdquor;": u"\u201d", - "rdsh;": u"\u21b3", - "real;": u"\u211c", - "realine;": u"\u211b", - "realpart;": u"\u211c", - "reals;": u"\u211d", - "rect;": u"\u25ad", - "reg": u"\xae", - "reg;": u"\xae", - "rfisht;": u"\u297d", - "rfloor;": u"\u230b", - "rfr;": u"\U0001d52f", - "rhard;": u"\u21c1", - "rharu;": u"\u21c0", - "rharul;": u"\u296c", - "rho;": u"\u03c1", - "rhov;": u"\u03f1", - "rightarrow;": u"\u2192", - "rightarrowtail;": u"\u21a3", - "rightharpoondown;": u"\u21c1", - "rightharpoonup;": u"\u21c0", - "rightleftarrows;": u"\u21c4", - "rightleftharpoons;": u"\u21cc", - "rightrightarrows;": u"\u21c9", - "rightsquigarrow;": u"\u219d", - "rightthreetimes;": u"\u22cc", - "ring;": u"\u02da", - "risingdotseq;": u"\u2253", - "rlarr;": u"\u21c4", - "rlhar;": u"\u21cc", - "rlm;": u"\u200f", - "rmoust;": u"\u23b1", - "rmoustache;": u"\u23b1", - "rnmid;": u"\u2aee", - "roang;": u"\u27ed", - "roarr;": u"\u21fe", - "robrk;": u"\u27e7", - "ropar;": u"\u2986", - "ropf;": u"\U0001d563", - "roplus;": u"\u2a2e", - "rotimes;": u"\u2a35", - "rpar;": u")", - "rpargt;": u"\u2994", - "rppolint;": u"\u2a12", - "rrarr;": u"\u21c9", - "rsaquo;": u"\u203a", - "rscr;": u"\U0001d4c7", - "rsh;": u"\u21b1", - "rsqb;": u"]", - "rsquo;": u"\u2019", - "rsquor;": u"\u2019", - "rthree;": u"\u22cc", - "rtimes;": u"\u22ca", - "rtri;": u"\u25b9", - "rtrie;": u"\u22b5", - "rtrif;": u"\u25b8", - "rtriltri;": u"\u29ce", - "ruluhar;": u"\u2968", - "rx;": u"\u211e", - "sacute;": u"\u015b", - "sbquo;": u"\u201a", - "sc;": u"\u227b", - "scE;": u"\u2ab4", - "scap;": u"\u2ab8", - "scaron;": u"\u0161", - "sccue;": u"\u227d", - "sce;": u"\u2ab0", - "scedil;": u"\u015f", - "scirc;": u"\u015d", - "scnE;": u"\u2ab6", - "scnap;": u"\u2aba", - "scnsim;": u"\u22e9", - "scpolint;": u"\u2a13", - "scsim;": u"\u227f", - "scy;": u"\u0441", - "sdot;": u"\u22c5", - "sdotb;": u"\u22a1", - "sdote;": u"\u2a66", - "seArr;": u"\u21d8", - "searhk;": u"\u2925", - "searr;": u"\u2198", - "searrow;": u"\u2198", - "sect": u"\xa7", - "sect;": u"\xa7", - "semi;": u";", - "seswar;": u"\u2929", - "setminus;": u"\u2216", - "setmn;": u"\u2216", - "sext;": u"\u2736", - "sfr;": u"\U0001d530", - "sfrown;": u"\u2322", - "sharp;": u"\u266f", - "shchcy;": u"\u0449", - "shcy;": u"\u0448", - "shortmid;": u"\u2223", - "shortparallel;": u"\u2225", - "shy": u"\xad", - "shy;": u"\xad", - "sigma;": u"\u03c3", - "sigmaf;": u"\u03c2", - "sigmav;": u"\u03c2", - "sim;": u"\u223c", - "simdot;": u"\u2a6a", - "sime;": u"\u2243", - "simeq;": u"\u2243", - "simg;": u"\u2a9e", - "simgE;": u"\u2aa0", - "siml;": u"\u2a9d", - "simlE;": u"\u2a9f", - "simne;": u"\u2246", - "simplus;": u"\u2a24", - "simrarr;": u"\u2972", - "slarr;": u"\u2190", - "smallsetminus;": u"\u2216", - "smashp;": u"\u2a33", - "smeparsl;": u"\u29e4", - "smid;": u"\u2223", - "smile;": u"\u2323", - "smt;": u"\u2aaa", - "smte;": u"\u2aac", - "smtes;": u"\u2aac\ufe00", - "softcy;": u"\u044c", - "sol;": u"/", - "solb;": u"\u29c4", - "solbar;": u"\u233f", - "sopf;": u"\U0001d564", - "spades;": u"\u2660", - "spadesuit;": u"\u2660", - "spar;": u"\u2225", - "sqcap;": u"\u2293", - "sqcaps;": u"\u2293\ufe00", - "sqcup;": u"\u2294", - "sqcups;": u"\u2294\ufe00", - "sqsub;": u"\u228f", - "sqsube;": u"\u2291", - "sqsubset;": u"\u228f", - "sqsubseteq;": u"\u2291", - "sqsup;": u"\u2290", - "sqsupe;": u"\u2292", - "sqsupset;": u"\u2290", - "sqsupseteq;": u"\u2292", - "squ;": u"\u25a1", - "square;": u"\u25a1", - "squarf;": u"\u25aa", - "squf;": u"\u25aa", - "srarr;": u"\u2192", - "sscr;": u"\U0001d4c8", - "ssetmn;": u"\u2216", - "ssmile;": u"\u2323", - "sstarf;": u"\u22c6", - "star;": u"\u2606", - "starf;": u"\u2605", - "straightepsilon;": u"\u03f5", - "straightphi;": u"\u03d5", - "strns;": u"\xaf", - "sub;": u"\u2282", - "subE;": u"\u2ac5", - "subdot;": u"\u2abd", - "sube;": u"\u2286", - "subedot;": u"\u2ac3", - "submult;": u"\u2ac1", - "subnE;": u"\u2acb", - "subne;": u"\u228a", - "subplus;": u"\u2abf", - "subrarr;": u"\u2979", - "subset;": u"\u2282", - "subseteq;": u"\u2286", - "subseteqq;": u"\u2ac5", - "subsetneq;": u"\u228a", - "subsetneqq;": u"\u2acb", - "subsim;": u"\u2ac7", - "subsub;": u"\u2ad5", - "subsup;": u"\u2ad3", - "succ;": u"\u227b", - "succapprox;": u"\u2ab8", - "succcurlyeq;": u"\u227d", - "succeq;": u"\u2ab0", - "succnapprox;": u"\u2aba", - "succneqq;": u"\u2ab6", - "succnsim;": u"\u22e9", - "succsim;": u"\u227f", - "sum;": u"\u2211", - "sung;": u"\u266a", - "sup1": u"\xb9", - "sup1;": u"\xb9", - "sup2": u"\xb2", - "sup2;": u"\xb2", - "sup3": u"\xb3", - "sup3;": u"\xb3", - "sup;": u"\u2283", - "supE;": u"\u2ac6", - "supdot;": u"\u2abe", - "supdsub;": u"\u2ad8", - "supe;": u"\u2287", - "supedot;": u"\u2ac4", - "suphsol;": u"\u27c9", - "suphsub;": u"\u2ad7", - "suplarr;": u"\u297b", - "supmult;": u"\u2ac2", - "supnE;": u"\u2acc", - "supne;": u"\u228b", - "supplus;": u"\u2ac0", - "supset;": u"\u2283", - "supseteq;": u"\u2287", - "supseteqq;": u"\u2ac6", - "supsetneq;": u"\u228b", - "supsetneqq;": u"\u2acc", - "supsim;": u"\u2ac8", - "supsub;": u"\u2ad4", - "supsup;": u"\u2ad6", - "swArr;": u"\u21d9", - "swarhk;": u"\u2926", - "swarr;": u"\u2199", - "swarrow;": u"\u2199", - "swnwar;": u"\u292a", - "szlig": u"\xdf", - "szlig;": u"\xdf", - "target;": u"\u2316", - "tau;": u"\u03c4", - "tbrk;": u"\u23b4", - "tcaron;": u"\u0165", - "tcedil;": u"\u0163", - "tcy;": u"\u0442", - "tdot;": u"\u20db", - "telrec;": u"\u2315", - "tfr;": u"\U0001d531", - "there4;": u"\u2234", - "therefore;": u"\u2234", - "theta;": u"\u03b8", - "thetasym;": u"\u03d1", - "thetav;": u"\u03d1", - "thickapprox;": u"\u2248", - "thicksim;": u"\u223c", - "thinsp;": u"\u2009", - "thkap;": u"\u2248", - "thksim;": u"\u223c", - "thorn": u"\xfe", - "thorn;": u"\xfe", - "tilde;": u"\u02dc", - "times": u"\xd7", - "times;": u"\xd7", - "timesb;": u"\u22a0", - "timesbar;": u"\u2a31", - "timesd;": u"\u2a30", - "tint;": u"\u222d", - "toea;": u"\u2928", - "top;": u"\u22a4", - "topbot;": u"\u2336", - "topcir;": u"\u2af1", - "topf;": u"\U0001d565", - "topfork;": u"\u2ada", - "tosa;": u"\u2929", - "tprime;": u"\u2034", - "trade;": u"\u2122", - "triangle;": u"\u25b5", - "triangledown;": u"\u25bf", - "triangleleft;": u"\u25c3", - "trianglelefteq;": u"\u22b4", - "triangleq;": u"\u225c", - "triangleright;": u"\u25b9", - "trianglerighteq;": u"\u22b5", - "tridot;": u"\u25ec", - "trie;": u"\u225c", - "triminus;": u"\u2a3a", - "triplus;": u"\u2a39", - "trisb;": u"\u29cd", - "tritime;": u"\u2a3b", - "trpezium;": u"\u23e2", - "tscr;": u"\U0001d4c9", - "tscy;": u"\u0446", - "tshcy;": u"\u045b", - "tstrok;": u"\u0167", - "twixt;": u"\u226c", - "twoheadleftarrow;": u"\u219e", - "twoheadrightarrow;": u"\u21a0", - "uArr;": u"\u21d1", - "uHar;": u"\u2963", - "uacute": u"\xfa", - "uacute;": u"\xfa", - "uarr;": u"\u2191", - "ubrcy;": u"\u045e", - "ubreve;": u"\u016d", - "ucirc": u"\xfb", - "ucirc;": u"\xfb", - "ucy;": u"\u0443", - "udarr;": u"\u21c5", - "udblac;": u"\u0171", - "udhar;": u"\u296e", - "ufisht;": u"\u297e", - "ufr;": u"\U0001d532", - "ugrave": u"\xf9", - "ugrave;": u"\xf9", - "uharl;": u"\u21bf", - "uharr;": u"\u21be", - "uhblk;": u"\u2580", - "ulcorn;": u"\u231c", - "ulcorner;": u"\u231c", - "ulcrop;": u"\u230f", - "ultri;": u"\u25f8", - "umacr;": u"\u016b", - "uml": u"\xa8", - "uml;": u"\xa8", - "uogon;": u"\u0173", - "uopf;": u"\U0001d566", - "uparrow;": u"\u2191", - "updownarrow;": u"\u2195", - "upharpoonleft;": u"\u21bf", - "upharpoonright;": u"\u21be", - "uplus;": u"\u228e", - "upsi;": u"\u03c5", - "upsih;": u"\u03d2", - "upsilon;": u"\u03c5", - "upuparrows;": u"\u21c8", - "urcorn;": u"\u231d", - "urcorner;": u"\u231d", - "urcrop;": u"\u230e", - "uring;": u"\u016f", - "urtri;": u"\u25f9", - "uscr;": u"\U0001d4ca", - "utdot;": u"\u22f0", - "utilde;": u"\u0169", - "utri;": u"\u25b5", - "utrif;": u"\u25b4", - "uuarr;": u"\u21c8", - "uuml": u"\xfc", - "uuml;": u"\xfc", - "uwangle;": u"\u29a7", - "vArr;": u"\u21d5", - "vBar;": u"\u2ae8", - "vBarv;": u"\u2ae9", - "vDash;": u"\u22a8", - "vangrt;": u"\u299c", - "varepsilon;": u"\u03f5", - "varkappa;": u"\u03f0", - "varnothing;": u"\u2205", - "varphi;": u"\u03d5", - "varpi;": u"\u03d6", - "varpropto;": u"\u221d", - "varr;": u"\u2195", - "varrho;": u"\u03f1", - "varsigma;": u"\u03c2", - "varsubsetneq;": u"\u228a\ufe00", - "varsubsetneqq;": u"\u2acb\ufe00", - "varsupsetneq;": u"\u228b\ufe00", - "varsupsetneqq;": u"\u2acc\ufe00", - "vartheta;": u"\u03d1", - "vartriangleleft;": u"\u22b2", - "vartriangleright;": u"\u22b3", - "vcy;": u"\u0432", - "vdash;": u"\u22a2", - "vee;": u"\u2228", - "veebar;": u"\u22bb", - "veeeq;": u"\u225a", - "vellip;": u"\u22ee", - "verbar;": u"|", - "vert;": u"|", - "vfr;": u"\U0001d533", - "vltri;": u"\u22b2", - "vnsub;": u"\u2282\u20d2", - "vnsup;": u"\u2283\u20d2", - "vopf;": u"\U0001d567", - "vprop;": u"\u221d", - "vrtri;": u"\u22b3", - "vscr;": u"\U0001d4cb", - "vsubnE;": u"\u2acb\ufe00", - "vsubne;": u"\u228a\ufe00", - "vsupnE;": u"\u2acc\ufe00", - "vsupne;": u"\u228b\ufe00", - "vzigzag;": u"\u299a", - "wcirc;": u"\u0175", - "wedbar;": u"\u2a5f", - "wedge;": u"\u2227", - "wedgeq;": u"\u2259", - "weierp;": u"\u2118", - "wfr;": u"\U0001d534", - "wopf;": u"\U0001d568", - "wp;": u"\u2118", - "wr;": u"\u2240", - "wreath;": u"\u2240", - "wscr;": u"\U0001d4cc", - "xcap;": u"\u22c2", - "xcirc;": u"\u25ef", - "xcup;": u"\u22c3", - "xdtri;": u"\u25bd", - "xfr;": u"\U0001d535", - "xhArr;": u"\u27fa", - "xharr;": u"\u27f7", - "xi;": u"\u03be", - "xlArr;": u"\u27f8", - "xlarr;": u"\u27f5", - "xmap;": u"\u27fc", - "xnis;": u"\u22fb", - "xodot;": u"\u2a00", - "xopf;": u"\U0001d569", - "xoplus;": u"\u2a01", - "xotime;": u"\u2a02", - "xrArr;": u"\u27f9", - "xrarr;": u"\u27f6", - "xscr;": u"\U0001d4cd", - "xsqcup;": u"\u2a06", - "xuplus;": u"\u2a04", - "xutri;": u"\u25b3", - "xvee;": u"\u22c1", - "xwedge;": u"\u22c0", - "yacute": u"\xfd", - "yacute;": u"\xfd", - "yacy;": u"\u044f", - "ycirc;": u"\u0177", - "ycy;": u"\u044b", - "yen": u"\xa5", - "yen;": u"\xa5", - "yfr;": u"\U0001d536", - "yicy;": u"\u0457", - "yopf;": u"\U0001d56a", - "yscr;": u"\U0001d4ce", - "yucy;": u"\u044e", - "yuml": u"\xff", - "yuml;": u"\xff", - "zacute;": u"\u017a", - "zcaron;": u"\u017e", - "zcy;": u"\u0437", - "zdot;": u"\u017c", - "zeetrf;": u"\u2128", - "zeta;": u"\u03b6", - "zfr;": u"\U0001d537", - "zhcy;": u"\u0436", - "zigrarr;": u"\u21dd", - "zopf;": u"\U0001d56b", - "zscr;": u"\U0001d4cf", - "zwj;": u"\u200d", - "zwnj;": u"\u200c", + "AElig": "\xc6", + "AElig;": "\xc6", + "AMP": "&", + "AMP;": "&", + "Aacute": "\xc1", + "Aacute;": "\xc1", + "Abreve;": "\u0102", + "Acirc": "\xc2", + "Acirc;": "\xc2", + "Acy;": "\u0410", + "Afr;": "\U0001d504", + "Agrave": "\xc0", + "Agrave;": "\xc0", + "Alpha;": "\u0391", + "Amacr;": "\u0100", + "And;": "\u2a53", + "Aogon;": "\u0104", + "Aopf;": "\U0001d538", + "ApplyFunction;": "\u2061", + "Aring": "\xc5", + "Aring;": "\xc5", + "Ascr;": "\U0001d49c", + "Assign;": "\u2254", + "Atilde": "\xc3", + "Atilde;": "\xc3", + "Auml": "\xc4", + "Auml;": "\xc4", + "Backslash;": "\u2216", + "Barv;": "\u2ae7", + "Barwed;": "\u2306", + "Bcy;": "\u0411", + "Because;": "\u2235", + "Bernoullis;": "\u212c", + "Beta;": "\u0392", + "Bfr;": "\U0001d505", + "Bopf;": "\U0001d539", + "Breve;": "\u02d8", + "Bscr;": "\u212c", + "Bumpeq;": "\u224e", + "CHcy;": "\u0427", + "COPY": "\xa9", + "COPY;": "\xa9", + "Cacute;": "\u0106", + "Cap;": "\u22d2", + "CapitalDifferentialD;": "\u2145", + "Cayleys;": "\u212d", + "Ccaron;": "\u010c", + "Ccedil": "\xc7", + "Ccedil;": "\xc7", + "Ccirc;": "\u0108", + "Cconint;": "\u2230", + "Cdot;": "\u010a", + "Cedilla;": "\xb8", + "CenterDot;": "\xb7", + "Cfr;": "\u212d", + "Chi;": "\u03a7", + "CircleDot;": "\u2299", + "CircleMinus;": "\u2296", + "CirclePlus;": "\u2295", + "CircleTimes;": "\u2297", + "ClockwiseContourIntegral;": "\u2232", + "CloseCurlyDoubleQuote;": "\u201d", + "CloseCurlyQuote;": "\u2019", + "Colon;": "\u2237", + "Colone;": "\u2a74", + "Congruent;": "\u2261", + "Conint;": "\u222f", + "ContourIntegral;": "\u222e", + "Copf;": "\u2102", + "Coproduct;": "\u2210", + "CounterClockwiseContourIntegral;": "\u2233", + "Cross;": "\u2a2f", + "Cscr;": "\U0001d49e", + "Cup;": "\u22d3", + "CupCap;": "\u224d", + "DD;": "\u2145", + "DDotrahd;": "\u2911", + "DJcy;": "\u0402", + "DScy;": "\u0405", + "DZcy;": "\u040f", + "Dagger;": "\u2021", + "Darr;": "\u21a1", + "Dashv;": "\u2ae4", + "Dcaron;": "\u010e", + "Dcy;": "\u0414", + "Del;": "\u2207", + "Delta;": "\u0394", + "Dfr;": "\U0001d507", + "DiacriticalAcute;": "\xb4", + "DiacriticalDot;": "\u02d9", + "DiacriticalDoubleAcute;": "\u02dd", + "DiacriticalGrave;": "`", + "DiacriticalTilde;": "\u02dc", + "Diamond;": "\u22c4", + "DifferentialD;": "\u2146", + "Dopf;": "\U0001d53b", + "Dot;": "\xa8", + "DotDot;": "\u20dc", + "DotEqual;": "\u2250", + "DoubleContourIntegral;": "\u222f", + "DoubleDot;": "\xa8", + "DoubleDownArrow;": "\u21d3", + "DoubleLeftArrow;": "\u21d0", + "DoubleLeftRightArrow;": "\u21d4", + "DoubleLeftTee;": "\u2ae4", + "DoubleLongLeftArrow;": "\u27f8", + "DoubleLongLeftRightArrow;": "\u27fa", + "DoubleLongRightArrow;": "\u27f9", + "DoubleRightArrow;": "\u21d2", + "DoubleRightTee;": "\u22a8", + "DoubleUpArrow;": "\u21d1", + "DoubleUpDownArrow;": "\u21d5", + "DoubleVerticalBar;": "\u2225", + "DownArrow;": "\u2193", + "DownArrowBar;": "\u2913", + "DownArrowUpArrow;": "\u21f5", + "DownBreve;": "\u0311", + "DownLeftRightVector;": "\u2950", + "DownLeftTeeVector;": "\u295e", + "DownLeftVector;": "\u21bd", + "DownLeftVectorBar;": "\u2956", + "DownRightTeeVector;": "\u295f", + "DownRightVector;": "\u21c1", + "DownRightVectorBar;": "\u2957", + "DownTee;": "\u22a4", + "DownTeeArrow;": "\u21a7", + "Downarrow;": "\u21d3", + "Dscr;": "\U0001d49f", + "Dstrok;": "\u0110", + "ENG;": "\u014a", + "ETH": "\xd0", + "ETH;": "\xd0", + "Eacute": "\xc9", + "Eacute;": "\xc9", + "Ecaron;": "\u011a", + "Ecirc": "\xca", + "Ecirc;": "\xca", + "Ecy;": "\u042d", + "Edot;": "\u0116", + "Efr;": "\U0001d508", + "Egrave": "\xc8", + "Egrave;": "\xc8", + "Element;": "\u2208", + "Emacr;": "\u0112", + "EmptySmallSquare;": "\u25fb", + "EmptyVerySmallSquare;": "\u25ab", + "Eogon;": "\u0118", + "Eopf;": "\U0001d53c", + "Epsilon;": "\u0395", + "Equal;": "\u2a75", + "EqualTilde;": "\u2242", + "Equilibrium;": "\u21cc", + "Escr;": "\u2130", + "Esim;": "\u2a73", + "Eta;": "\u0397", + "Euml": "\xcb", + "Euml;": "\xcb", + "Exists;": "\u2203", + "ExponentialE;": "\u2147", + "Fcy;": "\u0424", + "Ffr;": "\U0001d509", + "FilledSmallSquare;": "\u25fc", + "FilledVerySmallSquare;": "\u25aa", + "Fopf;": "\U0001d53d", + "ForAll;": "\u2200", + "Fouriertrf;": "\u2131", + "Fscr;": "\u2131", + "GJcy;": "\u0403", + "GT": ">", + "GT;": ">", + "Gamma;": "\u0393", + "Gammad;": "\u03dc", + "Gbreve;": "\u011e", + "Gcedil;": "\u0122", + "Gcirc;": "\u011c", + "Gcy;": "\u0413", + "Gdot;": "\u0120", + "Gfr;": "\U0001d50a", + "Gg;": "\u22d9", + "Gopf;": "\U0001d53e", + "GreaterEqual;": "\u2265", + "GreaterEqualLess;": "\u22db", + "GreaterFullEqual;": "\u2267", + "GreaterGreater;": "\u2aa2", + "GreaterLess;": "\u2277", + "GreaterSlantEqual;": "\u2a7e", + "GreaterTilde;": "\u2273", + "Gscr;": "\U0001d4a2", + "Gt;": "\u226b", + "HARDcy;": "\u042a", + "Hacek;": "\u02c7", + "Hat;": "^", + "Hcirc;": "\u0124", + "Hfr;": "\u210c", + "HilbertSpace;": "\u210b", + "Hopf;": "\u210d", + "HorizontalLine;": "\u2500", + "Hscr;": "\u210b", + "Hstrok;": "\u0126", + "HumpDownHump;": "\u224e", + "HumpEqual;": "\u224f", + "IEcy;": "\u0415", + "IJlig;": "\u0132", + "IOcy;": "\u0401", + "Iacute": "\xcd", + "Iacute;": "\xcd", + "Icirc": "\xce", + "Icirc;": "\xce", + "Icy;": "\u0418", + "Idot;": "\u0130", + "Ifr;": "\u2111", + "Igrave": "\xcc", + "Igrave;": "\xcc", + "Im;": "\u2111", + "Imacr;": "\u012a", + "ImaginaryI;": "\u2148", + "Implies;": "\u21d2", + "Int;": "\u222c", + "Integral;": "\u222b", + "Intersection;": "\u22c2", + "InvisibleComma;": "\u2063", + "InvisibleTimes;": "\u2062", + "Iogon;": "\u012e", + "Iopf;": "\U0001d540", + "Iota;": "\u0399", + "Iscr;": "\u2110", + "Itilde;": "\u0128", + "Iukcy;": "\u0406", + "Iuml": "\xcf", + "Iuml;": "\xcf", + "Jcirc;": "\u0134", + "Jcy;": "\u0419", + "Jfr;": "\U0001d50d", + "Jopf;": "\U0001d541", + "Jscr;": "\U0001d4a5", + "Jsercy;": "\u0408", + "Jukcy;": "\u0404", + "KHcy;": "\u0425", + "KJcy;": "\u040c", + "Kappa;": "\u039a", + "Kcedil;": "\u0136", + "Kcy;": "\u041a", + "Kfr;": "\U0001d50e", + "Kopf;": "\U0001d542", + "Kscr;": "\U0001d4a6", + "LJcy;": "\u0409", + "LT": "<", + "LT;": "<", + "Lacute;": "\u0139", + "Lambda;": "\u039b", + "Lang;": "\u27ea", + "Laplacetrf;": "\u2112", + "Larr;": "\u219e", + "Lcaron;": "\u013d", + "Lcedil;": "\u013b", + "Lcy;": "\u041b", + "LeftAngleBracket;": "\u27e8", + "LeftArrow;": "\u2190", + "LeftArrowBar;": "\u21e4", + "LeftArrowRightArrow;": "\u21c6", + "LeftCeiling;": "\u2308", + "LeftDoubleBracket;": "\u27e6", + "LeftDownTeeVector;": "\u2961", + "LeftDownVector;": "\u21c3", + "LeftDownVectorBar;": "\u2959", + "LeftFloor;": "\u230a", + "LeftRightArrow;": "\u2194", + "LeftRightVector;": "\u294e", + "LeftTee;": "\u22a3", + "LeftTeeArrow;": "\u21a4", + "LeftTeeVector;": "\u295a", + "LeftTriangle;": "\u22b2", + "LeftTriangleBar;": "\u29cf", + "LeftTriangleEqual;": "\u22b4", + "LeftUpDownVector;": "\u2951", + "LeftUpTeeVector;": "\u2960", + "LeftUpVector;": "\u21bf", + "LeftUpVectorBar;": "\u2958", + "LeftVector;": "\u21bc", + "LeftVectorBar;": "\u2952", + "Leftarrow;": "\u21d0", + "Leftrightarrow;": "\u21d4", + "LessEqualGreater;": "\u22da", + "LessFullEqual;": "\u2266", + "LessGreater;": "\u2276", + "LessLess;": "\u2aa1", + "LessSlantEqual;": "\u2a7d", + "LessTilde;": "\u2272", + "Lfr;": "\U0001d50f", + "Ll;": "\u22d8", + "Lleftarrow;": "\u21da", + "Lmidot;": "\u013f", + "LongLeftArrow;": "\u27f5", + "LongLeftRightArrow;": "\u27f7", + "LongRightArrow;": "\u27f6", + "Longleftarrow;": "\u27f8", + "Longleftrightarrow;": "\u27fa", + "Longrightarrow;": "\u27f9", + "Lopf;": "\U0001d543", + "LowerLeftArrow;": "\u2199", + "LowerRightArrow;": "\u2198", + "Lscr;": "\u2112", + "Lsh;": "\u21b0", + "Lstrok;": "\u0141", + "Lt;": "\u226a", + "Map;": "\u2905", + "Mcy;": "\u041c", + "MediumSpace;": "\u205f", + "Mellintrf;": "\u2133", + "Mfr;": "\U0001d510", + "MinusPlus;": "\u2213", + "Mopf;": "\U0001d544", + "Mscr;": "\u2133", + "Mu;": "\u039c", + "NJcy;": "\u040a", + "Nacute;": "\u0143", + "Ncaron;": "\u0147", + "Ncedil;": "\u0145", + "Ncy;": "\u041d", + "NegativeMediumSpace;": "\u200b", + "NegativeThickSpace;": "\u200b", + "NegativeThinSpace;": "\u200b", + "NegativeVeryThinSpace;": "\u200b", + "NestedGreaterGreater;": "\u226b", + "NestedLessLess;": "\u226a", + "NewLine;": "\n", + "Nfr;": "\U0001d511", + "NoBreak;": "\u2060", + "NonBreakingSpace;": "\xa0", + "Nopf;": "\u2115", + "Not;": "\u2aec", + "NotCongruent;": "\u2262", + "NotCupCap;": "\u226d", + "NotDoubleVerticalBar;": "\u2226", + "NotElement;": "\u2209", + "NotEqual;": "\u2260", + "NotEqualTilde;": "\u2242\u0338", + "NotExists;": "\u2204", + "NotGreater;": "\u226f", + "NotGreaterEqual;": "\u2271", + "NotGreaterFullEqual;": "\u2267\u0338", + "NotGreaterGreater;": "\u226b\u0338", + "NotGreaterLess;": "\u2279", + "NotGreaterSlantEqual;": "\u2a7e\u0338", + "NotGreaterTilde;": "\u2275", + "NotHumpDownHump;": "\u224e\u0338", + "NotHumpEqual;": "\u224f\u0338", + "NotLeftTriangle;": "\u22ea", + "NotLeftTriangleBar;": "\u29cf\u0338", + "NotLeftTriangleEqual;": "\u22ec", + "NotLess;": "\u226e", + "NotLessEqual;": "\u2270", + "NotLessGreater;": "\u2278", + "NotLessLess;": "\u226a\u0338", + "NotLessSlantEqual;": "\u2a7d\u0338", + "NotLessTilde;": "\u2274", + "NotNestedGreaterGreater;": "\u2aa2\u0338", + "NotNestedLessLess;": "\u2aa1\u0338", + "NotPrecedes;": "\u2280", + "NotPrecedesEqual;": "\u2aaf\u0338", + "NotPrecedesSlantEqual;": "\u22e0", + "NotReverseElement;": "\u220c", + "NotRightTriangle;": "\u22eb", + "NotRightTriangleBar;": "\u29d0\u0338", + "NotRightTriangleEqual;": "\u22ed", + "NotSquareSubset;": "\u228f\u0338", + "NotSquareSubsetEqual;": "\u22e2", + "NotSquareSuperset;": "\u2290\u0338", + "NotSquareSupersetEqual;": "\u22e3", + "NotSubset;": "\u2282\u20d2", + "NotSubsetEqual;": "\u2288", + "NotSucceeds;": "\u2281", + "NotSucceedsEqual;": "\u2ab0\u0338", + "NotSucceedsSlantEqual;": "\u22e1", + "NotSucceedsTilde;": "\u227f\u0338", + "NotSuperset;": "\u2283\u20d2", + "NotSupersetEqual;": "\u2289", + "NotTilde;": "\u2241", + "NotTildeEqual;": "\u2244", + "NotTildeFullEqual;": "\u2247", + "NotTildeTilde;": "\u2249", + "NotVerticalBar;": "\u2224", + "Nscr;": "\U0001d4a9", + "Ntilde": "\xd1", + "Ntilde;": "\xd1", + "Nu;": "\u039d", + "OElig;": "\u0152", + "Oacute": "\xd3", + "Oacute;": "\xd3", + "Ocirc": "\xd4", + "Ocirc;": "\xd4", + "Ocy;": "\u041e", + "Odblac;": "\u0150", + "Ofr;": "\U0001d512", + "Ograve": "\xd2", + "Ograve;": "\xd2", + "Omacr;": "\u014c", + "Omega;": "\u03a9", + "Omicron;": "\u039f", + "Oopf;": "\U0001d546", + "OpenCurlyDoubleQuote;": "\u201c", + "OpenCurlyQuote;": "\u2018", + "Or;": "\u2a54", + "Oscr;": "\U0001d4aa", + "Oslash": "\xd8", + "Oslash;": "\xd8", + "Otilde": "\xd5", + "Otilde;": "\xd5", + "Otimes;": "\u2a37", + "Ouml": "\xd6", + "Ouml;": "\xd6", + "OverBar;": "\u203e", + "OverBrace;": "\u23de", + "OverBracket;": "\u23b4", + "OverParenthesis;": "\u23dc", + "PartialD;": "\u2202", + "Pcy;": "\u041f", + "Pfr;": "\U0001d513", + "Phi;": "\u03a6", + "Pi;": "\u03a0", + "PlusMinus;": "\xb1", + "Poincareplane;": "\u210c", + "Popf;": "\u2119", + "Pr;": "\u2abb", + "Precedes;": "\u227a", + "PrecedesEqual;": "\u2aaf", + "PrecedesSlantEqual;": "\u227c", + "PrecedesTilde;": "\u227e", + "Prime;": "\u2033", + "Product;": "\u220f", + "Proportion;": "\u2237", + "Proportional;": "\u221d", + "Pscr;": "\U0001d4ab", + "Psi;": "\u03a8", + "QUOT": "\"", + "QUOT;": "\"", + "Qfr;": "\U0001d514", + "Qopf;": "\u211a", + "Qscr;": "\U0001d4ac", + "RBarr;": "\u2910", + "REG": "\xae", + "REG;": "\xae", + "Racute;": "\u0154", + "Rang;": "\u27eb", + "Rarr;": "\u21a0", + "Rarrtl;": "\u2916", + "Rcaron;": "\u0158", + "Rcedil;": "\u0156", + "Rcy;": "\u0420", + "Re;": "\u211c", + "ReverseElement;": "\u220b", + "ReverseEquilibrium;": "\u21cb", + "ReverseUpEquilibrium;": "\u296f", + "Rfr;": "\u211c", + "Rho;": "\u03a1", + "RightAngleBracket;": "\u27e9", + "RightArrow;": "\u2192", + "RightArrowBar;": "\u21e5", + "RightArrowLeftArrow;": "\u21c4", + "RightCeiling;": "\u2309", + "RightDoubleBracket;": "\u27e7", + "RightDownTeeVector;": "\u295d", + "RightDownVector;": "\u21c2", + "RightDownVectorBar;": "\u2955", + "RightFloor;": "\u230b", + "RightTee;": "\u22a2", + "RightTeeArrow;": "\u21a6", + "RightTeeVector;": "\u295b", + "RightTriangle;": "\u22b3", + "RightTriangleBar;": "\u29d0", + "RightTriangleEqual;": "\u22b5", + "RightUpDownVector;": "\u294f", + "RightUpTeeVector;": "\u295c", + "RightUpVector;": "\u21be", + "RightUpVectorBar;": "\u2954", + "RightVector;": "\u21c0", + "RightVectorBar;": "\u2953", + "Rightarrow;": "\u21d2", + "Ropf;": "\u211d", + "RoundImplies;": "\u2970", + "Rrightarrow;": "\u21db", + "Rscr;": "\u211b", + "Rsh;": "\u21b1", + "RuleDelayed;": "\u29f4", + "SHCHcy;": "\u0429", + "SHcy;": "\u0428", + "SOFTcy;": "\u042c", + "Sacute;": "\u015a", + "Sc;": "\u2abc", + "Scaron;": "\u0160", + "Scedil;": "\u015e", + "Scirc;": "\u015c", + "Scy;": "\u0421", + "Sfr;": "\U0001d516", + "ShortDownArrow;": "\u2193", + "ShortLeftArrow;": "\u2190", + "ShortRightArrow;": "\u2192", + "ShortUpArrow;": "\u2191", + "Sigma;": "\u03a3", + "SmallCircle;": "\u2218", + "Sopf;": "\U0001d54a", + "Sqrt;": "\u221a", + "Square;": "\u25a1", + "SquareIntersection;": "\u2293", + "SquareSubset;": "\u228f", + "SquareSubsetEqual;": "\u2291", + "SquareSuperset;": "\u2290", + "SquareSupersetEqual;": "\u2292", + "SquareUnion;": "\u2294", + "Sscr;": "\U0001d4ae", + "Star;": "\u22c6", + "Sub;": "\u22d0", + "Subset;": "\u22d0", + "SubsetEqual;": "\u2286", + "Succeeds;": "\u227b", + "SucceedsEqual;": "\u2ab0", + "SucceedsSlantEqual;": "\u227d", + "SucceedsTilde;": "\u227f", + "SuchThat;": "\u220b", + "Sum;": "\u2211", + "Sup;": "\u22d1", + "Superset;": "\u2283", + "SupersetEqual;": "\u2287", + "Supset;": "\u22d1", + "THORN": "\xde", + "THORN;": "\xde", + "TRADE;": "\u2122", + "TSHcy;": "\u040b", + "TScy;": "\u0426", + "Tab;": "\t", + "Tau;": "\u03a4", + "Tcaron;": "\u0164", + "Tcedil;": "\u0162", + "Tcy;": "\u0422", + "Tfr;": "\U0001d517", + "Therefore;": "\u2234", + "Theta;": "\u0398", + "ThickSpace;": "\u205f\u200a", + "ThinSpace;": "\u2009", + "Tilde;": "\u223c", + "TildeEqual;": "\u2243", + "TildeFullEqual;": "\u2245", + "TildeTilde;": "\u2248", + "Topf;": "\U0001d54b", + "TripleDot;": "\u20db", + "Tscr;": "\U0001d4af", + "Tstrok;": "\u0166", + "Uacute": "\xda", + "Uacute;": "\xda", + "Uarr;": "\u219f", + "Uarrocir;": "\u2949", + "Ubrcy;": "\u040e", + "Ubreve;": "\u016c", + "Ucirc": "\xdb", + "Ucirc;": "\xdb", + "Ucy;": "\u0423", + "Udblac;": "\u0170", + "Ufr;": "\U0001d518", + "Ugrave": "\xd9", + "Ugrave;": "\xd9", + "Umacr;": "\u016a", + "UnderBar;": "_", + "UnderBrace;": "\u23df", + "UnderBracket;": "\u23b5", + "UnderParenthesis;": "\u23dd", + "Union;": "\u22c3", + "UnionPlus;": "\u228e", + "Uogon;": "\u0172", + "Uopf;": "\U0001d54c", + "UpArrow;": "\u2191", + "UpArrowBar;": "\u2912", + "UpArrowDownArrow;": "\u21c5", + "UpDownArrow;": "\u2195", + "UpEquilibrium;": "\u296e", + "UpTee;": "\u22a5", + "UpTeeArrow;": "\u21a5", + "Uparrow;": "\u21d1", + "Updownarrow;": "\u21d5", + "UpperLeftArrow;": "\u2196", + "UpperRightArrow;": "\u2197", + "Upsi;": "\u03d2", + "Upsilon;": "\u03a5", + "Uring;": "\u016e", + "Uscr;": "\U0001d4b0", + "Utilde;": "\u0168", + "Uuml": "\xdc", + "Uuml;": "\xdc", + "VDash;": "\u22ab", + "Vbar;": "\u2aeb", + "Vcy;": "\u0412", + "Vdash;": "\u22a9", + "Vdashl;": "\u2ae6", + "Vee;": "\u22c1", + "Verbar;": "\u2016", + "Vert;": "\u2016", + "VerticalBar;": "\u2223", + "VerticalLine;": "|", + "VerticalSeparator;": "\u2758", + "VerticalTilde;": "\u2240", + "VeryThinSpace;": "\u200a", + "Vfr;": "\U0001d519", + "Vopf;": "\U0001d54d", + "Vscr;": "\U0001d4b1", + "Vvdash;": "\u22aa", + "Wcirc;": "\u0174", + "Wedge;": "\u22c0", + "Wfr;": "\U0001d51a", + "Wopf;": "\U0001d54e", + "Wscr;": "\U0001d4b2", + "Xfr;": "\U0001d51b", + "Xi;": "\u039e", + "Xopf;": "\U0001d54f", + "Xscr;": "\U0001d4b3", + "YAcy;": "\u042f", + "YIcy;": "\u0407", + "YUcy;": "\u042e", + "Yacute": "\xdd", + "Yacute;": "\xdd", + "Ycirc;": "\u0176", + "Ycy;": "\u042b", + "Yfr;": "\U0001d51c", + "Yopf;": "\U0001d550", + "Yscr;": "\U0001d4b4", + "Yuml;": "\u0178", + "ZHcy;": "\u0416", + "Zacute;": "\u0179", + "Zcaron;": "\u017d", + "Zcy;": "\u0417", + "Zdot;": "\u017b", + "ZeroWidthSpace;": "\u200b", + "Zeta;": "\u0396", + "Zfr;": "\u2128", + "Zopf;": "\u2124", + "Zscr;": "\U0001d4b5", + "aacute": "\xe1", + "aacute;": "\xe1", + "abreve;": "\u0103", + "ac;": "\u223e", + "acE;": "\u223e\u0333", + "acd;": "\u223f", + "acirc": "\xe2", + "acirc;": "\xe2", + "acute": "\xb4", + "acute;": "\xb4", + "acy;": "\u0430", + "aelig": "\xe6", + "aelig;": "\xe6", + "af;": "\u2061", + "afr;": "\U0001d51e", + "agrave": "\xe0", + "agrave;": "\xe0", + "alefsym;": "\u2135", + "aleph;": "\u2135", + "alpha;": "\u03b1", + "amacr;": "\u0101", + "amalg;": "\u2a3f", + "amp": "&", + "amp;": "&", + "and;": "\u2227", + "andand;": "\u2a55", + "andd;": "\u2a5c", + "andslope;": "\u2a58", + "andv;": "\u2a5a", + "ang;": "\u2220", + "ange;": "\u29a4", + "angle;": "\u2220", + "angmsd;": "\u2221", + "angmsdaa;": "\u29a8", + "angmsdab;": "\u29a9", + "angmsdac;": "\u29aa", + "angmsdad;": "\u29ab", + "angmsdae;": "\u29ac", + "angmsdaf;": "\u29ad", + "angmsdag;": "\u29ae", + "angmsdah;": "\u29af", + "angrt;": "\u221f", + "angrtvb;": "\u22be", + "angrtvbd;": "\u299d", + "angsph;": "\u2222", + "angst;": "\xc5", + "angzarr;": "\u237c", + "aogon;": "\u0105", + "aopf;": "\U0001d552", + "ap;": "\u2248", + "apE;": "\u2a70", + "apacir;": "\u2a6f", + "ape;": "\u224a", + "apid;": "\u224b", + "apos;": "'", + "approx;": "\u2248", + "approxeq;": "\u224a", + "aring": "\xe5", + "aring;": "\xe5", + "ascr;": "\U0001d4b6", + "ast;": "*", + "asymp;": "\u2248", + "asympeq;": "\u224d", + "atilde": "\xe3", + "atilde;": "\xe3", + "auml": "\xe4", + "auml;": "\xe4", + "awconint;": "\u2233", + "awint;": "\u2a11", + "bNot;": "\u2aed", + "backcong;": "\u224c", + "backepsilon;": "\u03f6", + "backprime;": "\u2035", + "backsim;": "\u223d", + "backsimeq;": "\u22cd", + "barvee;": "\u22bd", + "barwed;": "\u2305", + "barwedge;": "\u2305", + "bbrk;": "\u23b5", + "bbrktbrk;": "\u23b6", + "bcong;": "\u224c", + "bcy;": "\u0431", + "bdquo;": "\u201e", + "becaus;": "\u2235", + "because;": "\u2235", + "bemptyv;": "\u29b0", + "bepsi;": "\u03f6", + "bernou;": "\u212c", + "beta;": "\u03b2", + "beth;": "\u2136", + "between;": "\u226c", + "bfr;": "\U0001d51f", + "bigcap;": "\u22c2", + "bigcirc;": "\u25ef", + "bigcup;": "\u22c3", + "bigodot;": "\u2a00", + "bigoplus;": "\u2a01", + "bigotimes;": "\u2a02", + "bigsqcup;": "\u2a06", + "bigstar;": "\u2605", + "bigtriangledown;": "\u25bd", + "bigtriangleup;": "\u25b3", + "biguplus;": "\u2a04", + "bigvee;": "\u22c1", + "bigwedge;": "\u22c0", + "bkarow;": "\u290d", + "blacklozenge;": "\u29eb", + "blacksquare;": "\u25aa", + "blacktriangle;": "\u25b4", + "blacktriangledown;": "\u25be", + "blacktriangleleft;": "\u25c2", + "blacktriangleright;": "\u25b8", + "blank;": "\u2423", + "blk12;": "\u2592", + "blk14;": "\u2591", + "blk34;": "\u2593", + "block;": "\u2588", + "bne;": "=\u20e5", + "bnequiv;": "\u2261\u20e5", + "bnot;": "\u2310", + "bopf;": "\U0001d553", + "bot;": "\u22a5", + "bottom;": "\u22a5", + "bowtie;": "\u22c8", + "boxDL;": "\u2557", + "boxDR;": "\u2554", + "boxDl;": "\u2556", + "boxDr;": "\u2553", + "boxH;": "\u2550", + "boxHD;": "\u2566", + "boxHU;": "\u2569", + "boxHd;": "\u2564", + "boxHu;": "\u2567", + "boxUL;": "\u255d", + "boxUR;": "\u255a", + "boxUl;": "\u255c", + "boxUr;": "\u2559", + "boxV;": "\u2551", + "boxVH;": "\u256c", + "boxVL;": "\u2563", + "boxVR;": "\u2560", + "boxVh;": "\u256b", + "boxVl;": "\u2562", + "boxVr;": "\u255f", + "boxbox;": "\u29c9", + "boxdL;": "\u2555", + "boxdR;": "\u2552", + "boxdl;": "\u2510", + "boxdr;": "\u250c", + "boxh;": "\u2500", + "boxhD;": "\u2565", + "boxhU;": "\u2568", + "boxhd;": "\u252c", + "boxhu;": "\u2534", + "boxminus;": "\u229f", + "boxplus;": "\u229e", + "boxtimes;": "\u22a0", + "boxuL;": "\u255b", + "boxuR;": "\u2558", + "boxul;": "\u2518", + "boxur;": "\u2514", + "boxv;": "\u2502", + "boxvH;": "\u256a", + "boxvL;": "\u2561", + "boxvR;": "\u255e", + "boxvh;": "\u253c", + "boxvl;": "\u2524", + "boxvr;": "\u251c", + "bprime;": "\u2035", + "breve;": "\u02d8", + "brvbar": "\xa6", + "brvbar;": "\xa6", + "bscr;": "\U0001d4b7", + "bsemi;": "\u204f", + "bsim;": "\u223d", + "bsime;": "\u22cd", + "bsol;": "\\", + "bsolb;": "\u29c5", + "bsolhsub;": "\u27c8", + "bull;": "\u2022", + "bullet;": "\u2022", + "bump;": "\u224e", + "bumpE;": "\u2aae", + "bumpe;": "\u224f", + "bumpeq;": "\u224f", + "cacute;": "\u0107", + "cap;": "\u2229", + "capand;": "\u2a44", + "capbrcup;": "\u2a49", + "capcap;": "\u2a4b", + "capcup;": "\u2a47", + "capdot;": "\u2a40", + "caps;": "\u2229\ufe00", + "caret;": "\u2041", + "caron;": "\u02c7", + "ccaps;": "\u2a4d", + "ccaron;": "\u010d", + "ccedil": "\xe7", + "ccedil;": "\xe7", + "ccirc;": "\u0109", + "ccups;": "\u2a4c", + "ccupssm;": "\u2a50", + "cdot;": "\u010b", + "cedil": "\xb8", + "cedil;": "\xb8", + "cemptyv;": "\u29b2", + "cent": "\xa2", + "cent;": "\xa2", + "centerdot;": "\xb7", + "cfr;": "\U0001d520", + "chcy;": "\u0447", + "check;": "\u2713", + "checkmark;": "\u2713", + "chi;": "\u03c7", + "cir;": "\u25cb", + "cirE;": "\u29c3", + "circ;": "\u02c6", + "circeq;": "\u2257", + "circlearrowleft;": "\u21ba", + "circlearrowright;": "\u21bb", + "circledR;": "\xae", + "circledS;": "\u24c8", + "circledast;": "\u229b", + "circledcirc;": "\u229a", + "circleddash;": "\u229d", + "cire;": "\u2257", + "cirfnint;": "\u2a10", + "cirmid;": "\u2aef", + "cirscir;": "\u29c2", + "clubs;": "\u2663", + "clubsuit;": "\u2663", + "colon;": ":", + "colone;": "\u2254", + "coloneq;": "\u2254", + "comma;": ",", + "commat;": "@", + "comp;": "\u2201", + "compfn;": "\u2218", + "complement;": "\u2201", + "complexes;": "\u2102", + "cong;": "\u2245", + "congdot;": "\u2a6d", + "conint;": "\u222e", + "copf;": "\U0001d554", + "coprod;": "\u2210", + "copy": "\xa9", + "copy;": "\xa9", + "copysr;": "\u2117", + "crarr;": "\u21b5", + "cross;": "\u2717", + "cscr;": "\U0001d4b8", + "csub;": "\u2acf", + "csube;": "\u2ad1", + "csup;": "\u2ad0", + "csupe;": "\u2ad2", + "ctdot;": "\u22ef", + "cudarrl;": "\u2938", + "cudarrr;": "\u2935", + "cuepr;": "\u22de", + "cuesc;": "\u22df", + "cularr;": "\u21b6", + "cularrp;": "\u293d", + "cup;": "\u222a", + "cupbrcap;": "\u2a48", + "cupcap;": "\u2a46", + "cupcup;": "\u2a4a", + "cupdot;": "\u228d", + "cupor;": "\u2a45", + "cups;": "\u222a\ufe00", + "curarr;": "\u21b7", + "curarrm;": "\u293c", + "curlyeqprec;": "\u22de", + "curlyeqsucc;": "\u22df", + "curlyvee;": "\u22ce", + "curlywedge;": "\u22cf", + "curren": "\xa4", + "curren;": "\xa4", + "curvearrowleft;": "\u21b6", + "curvearrowright;": "\u21b7", + "cuvee;": "\u22ce", + "cuwed;": "\u22cf", + "cwconint;": "\u2232", + "cwint;": "\u2231", + "cylcty;": "\u232d", + "dArr;": "\u21d3", + "dHar;": "\u2965", + "dagger;": "\u2020", + "daleth;": "\u2138", + "darr;": "\u2193", + "dash;": "\u2010", + "dashv;": "\u22a3", + "dbkarow;": "\u290f", + "dblac;": "\u02dd", + "dcaron;": "\u010f", + "dcy;": "\u0434", + "dd;": "\u2146", + "ddagger;": "\u2021", + "ddarr;": "\u21ca", + "ddotseq;": "\u2a77", + "deg": "\xb0", + "deg;": "\xb0", + "delta;": "\u03b4", + "demptyv;": "\u29b1", + "dfisht;": "\u297f", + "dfr;": "\U0001d521", + "dharl;": "\u21c3", + "dharr;": "\u21c2", + "diam;": "\u22c4", + "diamond;": "\u22c4", + "diamondsuit;": "\u2666", + "diams;": "\u2666", + "die;": "\xa8", + "digamma;": "\u03dd", + "disin;": "\u22f2", + "div;": "\xf7", + "divide": "\xf7", + "divide;": "\xf7", + "divideontimes;": "\u22c7", + "divonx;": "\u22c7", + "djcy;": "\u0452", + "dlcorn;": "\u231e", + "dlcrop;": "\u230d", + "dollar;": "$", + "dopf;": "\U0001d555", + "dot;": "\u02d9", + "doteq;": "\u2250", + "doteqdot;": "\u2251", + "dotminus;": "\u2238", + "dotplus;": "\u2214", + "dotsquare;": "\u22a1", + "doublebarwedge;": "\u2306", + "downarrow;": "\u2193", + "downdownarrows;": "\u21ca", + "downharpoonleft;": "\u21c3", + "downharpoonright;": "\u21c2", + "drbkarow;": "\u2910", + "drcorn;": "\u231f", + "drcrop;": "\u230c", + "dscr;": "\U0001d4b9", + "dscy;": "\u0455", + "dsol;": "\u29f6", + "dstrok;": "\u0111", + "dtdot;": "\u22f1", + "dtri;": "\u25bf", + "dtrif;": "\u25be", + "duarr;": "\u21f5", + "duhar;": "\u296f", + "dwangle;": "\u29a6", + "dzcy;": "\u045f", + "dzigrarr;": "\u27ff", + "eDDot;": "\u2a77", + "eDot;": "\u2251", + "eacute": "\xe9", + "eacute;": "\xe9", + "easter;": "\u2a6e", + "ecaron;": "\u011b", + "ecir;": "\u2256", + "ecirc": "\xea", + "ecirc;": "\xea", + "ecolon;": "\u2255", + "ecy;": "\u044d", + "edot;": "\u0117", + "ee;": "\u2147", + "efDot;": "\u2252", + "efr;": "\U0001d522", + "eg;": "\u2a9a", + "egrave": "\xe8", + "egrave;": "\xe8", + "egs;": "\u2a96", + "egsdot;": "\u2a98", + "el;": "\u2a99", + "elinters;": "\u23e7", + "ell;": "\u2113", + "els;": "\u2a95", + "elsdot;": "\u2a97", + "emacr;": "\u0113", + "empty;": "\u2205", + "emptyset;": "\u2205", + "emptyv;": "\u2205", + "emsp13;": "\u2004", + "emsp14;": "\u2005", + "emsp;": "\u2003", + "eng;": "\u014b", + "ensp;": "\u2002", + "eogon;": "\u0119", + "eopf;": "\U0001d556", + "epar;": "\u22d5", + "eparsl;": "\u29e3", + "eplus;": "\u2a71", + "epsi;": "\u03b5", + "epsilon;": "\u03b5", + "epsiv;": "\u03f5", + "eqcirc;": "\u2256", + "eqcolon;": "\u2255", + "eqsim;": "\u2242", + "eqslantgtr;": "\u2a96", + "eqslantless;": "\u2a95", + "equals;": "=", + "equest;": "\u225f", + "equiv;": "\u2261", + "equivDD;": "\u2a78", + "eqvparsl;": "\u29e5", + "erDot;": "\u2253", + "erarr;": "\u2971", + "escr;": "\u212f", + "esdot;": "\u2250", + "esim;": "\u2242", + "eta;": "\u03b7", + "eth": "\xf0", + "eth;": "\xf0", + "euml": "\xeb", + "euml;": "\xeb", + "euro;": "\u20ac", + "excl;": "!", + "exist;": "\u2203", + "expectation;": "\u2130", + "exponentiale;": "\u2147", + "fallingdotseq;": "\u2252", + "fcy;": "\u0444", + "female;": "\u2640", + "ffilig;": "\ufb03", + "fflig;": "\ufb00", + "ffllig;": "\ufb04", + "ffr;": "\U0001d523", + "filig;": "\ufb01", + "fjlig;": "fj", + "flat;": "\u266d", + "fllig;": "\ufb02", + "fltns;": "\u25b1", + "fnof;": "\u0192", + "fopf;": "\U0001d557", + "forall;": "\u2200", + "fork;": "\u22d4", + "forkv;": "\u2ad9", + "fpartint;": "\u2a0d", + "frac12": "\xbd", + "frac12;": "\xbd", + "frac13;": "\u2153", + "frac14": "\xbc", + "frac14;": "\xbc", + "frac15;": "\u2155", + "frac16;": "\u2159", + "frac18;": "\u215b", + "frac23;": "\u2154", + "frac25;": "\u2156", + "frac34": "\xbe", + "frac34;": "\xbe", + "frac35;": "\u2157", + "frac38;": "\u215c", + "frac45;": "\u2158", + "frac56;": "\u215a", + "frac58;": "\u215d", + "frac78;": "\u215e", + "frasl;": "\u2044", + "frown;": "\u2322", + "fscr;": "\U0001d4bb", + "gE;": "\u2267", + "gEl;": "\u2a8c", + "gacute;": "\u01f5", + "gamma;": "\u03b3", + "gammad;": "\u03dd", + "gap;": "\u2a86", + "gbreve;": "\u011f", + "gcirc;": "\u011d", + "gcy;": "\u0433", + "gdot;": "\u0121", + "ge;": "\u2265", + "gel;": "\u22db", + "geq;": "\u2265", + "geqq;": "\u2267", + "geqslant;": "\u2a7e", + "ges;": "\u2a7e", + "gescc;": "\u2aa9", + "gesdot;": "\u2a80", + "gesdoto;": "\u2a82", + "gesdotol;": "\u2a84", + "gesl;": "\u22db\ufe00", + "gesles;": "\u2a94", + "gfr;": "\U0001d524", + "gg;": "\u226b", + "ggg;": "\u22d9", + "gimel;": "\u2137", + "gjcy;": "\u0453", + "gl;": "\u2277", + "glE;": "\u2a92", + "gla;": "\u2aa5", + "glj;": "\u2aa4", + "gnE;": "\u2269", + "gnap;": "\u2a8a", + "gnapprox;": "\u2a8a", + "gne;": "\u2a88", + "gneq;": "\u2a88", + "gneqq;": "\u2269", + "gnsim;": "\u22e7", + "gopf;": "\U0001d558", + "grave;": "`", + "gscr;": "\u210a", + "gsim;": "\u2273", + "gsime;": "\u2a8e", + "gsiml;": "\u2a90", + "gt": ">", + "gt;": ">", + "gtcc;": "\u2aa7", + "gtcir;": "\u2a7a", + "gtdot;": "\u22d7", + "gtlPar;": "\u2995", + "gtquest;": "\u2a7c", + "gtrapprox;": "\u2a86", + "gtrarr;": "\u2978", + "gtrdot;": "\u22d7", + "gtreqless;": "\u22db", + "gtreqqless;": "\u2a8c", + "gtrless;": "\u2277", + "gtrsim;": "\u2273", + "gvertneqq;": "\u2269\ufe00", + "gvnE;": "\u2269\ufe00", + "hArr;": "\u21d4", + "hairsp;": "\u200a", + "half;": "\xbd", + "hamilt;": "\u210b", + "hardcy;": "\u044a", + "harr;": "\u2194", + "harrcir;": "\u2948", + "harrw;": "\u21ad", + "hbar;": "\u210f", + "hcirc;": "\u0125", + "hearts;": "\u2665", + "heartsuit;": "\u2665", + "hellip;": "\u2026", + "hercon;": "\u22b9", + "hfr;": "\U0001d525", + "hksearow;": "\u2925", + "hkswarow;": "\u2926", + "hoarr;": "\u21ff", + "homtht;": "\u223b", + "hookleftarrow;": "\u21a9", + "hookrightarrow;": "\u21aa", + "hopf;": "\U0001d559", + "horbar;": "\u2015", + "hscr;": "\U0001d4bd", + "hslash;": "\u210f", + "hstrok;": "\u0127", + "hybull;": "\u2043", + "hyphen;": "\u2010", + "iacute": "\xed", + "iacute;": "\xed", + "ic;": "\u2063", + "icirc": "\xee", + "icirc;": "\xee", + "icy;": "\u0438", + "iecy;": "\u0435", + "iexcl": "\xa1", + "iexcl;": "\xa1", + "iff;": "\u21d4", + "ifr;": "\U0001d526", + "igrave": "\xec", + "igrave;": "\xec", + "ii;": "\u2148", + "iiiint;": "\u2a0c", + "iiint;": "\u222d", + "iinfin;": "\u29dc", + "iiota;": "\u2129", + "ijlig;": "\u0133", + "imacr;": "\u012b", + "image;": "\u2111", + "imagline;": "\u2110", + "imagpart;": "\u2111", + "imath;": "\u0131", + "imof;": "\u22b7", + "imped;": "\u01b5", + "in;": "\u2208", + "incare;": "\u2105", + "infin;": "\u221e", + "infintie;": "\u29dd", + "inodot;": "\u0131", + "int;": "\u222b", + "intcal;": "\u22ba", + "integers;": "\u2124", + "intercal;": "\u22ba", + "intlarhk;": "\u2a17", + "intprod;": "\u2a3c", + "iocy;": "\u0451", + "iogon;": "\u012f", + "iopf;": "\U0001d55a", + "iota;": "\u03b9", + "iprod;": "\u2a3c", + "iquest": "\xbf", + "iquest;": "\xbf", + "iscr;": "\U0001d4be", + "isin;": "\u2208", + "isinE;": "\u22f9", + "isindot;": "\u22f5", + "isins;": "\u22f4", + "isinsv;": "\u22f3", + "isinv;": "\u2208", + "it;": "\u2062", + "itilde;": "\u0129", + "iukcy;": "\u0456", + "iuml": "\xef", + "iuml;": "\xef", + "jcirc;": "\u0135", + "jcy;": "\u0439", + "jfr;": "\U0001d527", + "jmath;": "\u0237", + "jopf;": "\U0001d55b", + "jscr;": "\U0001d4bf", + "jsercy;": "\u0458", + "jukcy;": "\u0454", + "kappa;": "\u03ba", + "kappav;": "\u03f0", + "kcedil;": "\u0137", + "kcy;": "\u043a", + "kfr;": "\U0001d528", + "kgreen;": "\u0138", + "khcy;": "\u0445", + "kjcy;": "\u045c", + "kopf;": "\U0001d55c", + "kscr;": "\U0001d4c0", + "lAarr;": "\u21da", + "lArr;": "\u21d0", + "lAtail;": "\u291b", + "lBarr;": "\u290e", + "lE;": "\u2266", + "lEg;": "\u2a8b", + "lHar;": "\u2962", + "lacute;": "\u013a", + "laemptyv;": "\u29b4", + "lagran;": "\u2112", + "lambda;": "\u03bb", + "lang;": "\u27e8", + "langd;": "\u2991", + "langle;": "\u27e8", + "lap;": "\u2a85", + "laquo": "\xab", + "laquo;": "\xab", + "larr;": "\u2190", + "larrb;": "\u21e4", + "larrbfs;": "\u291f", + "larrfs;": "\u291d", + "larrhk;": "\u21a9", + "larrlp;": "\u21ab", + "larrpl;": "\u2939", + "larrsim;": "\u2973", + "larrtl;": "\u21a2", + "lat;": "\u2aab", + "latail;": "\u2919", + "late;": "\u2aad", + "lates;": "\u2aad\ufe00", + "lbarr;": "\u290c", + "lbbrk;": "\u2772", + "lbrace;": "{", + "lbrack;": "[", + "lbrke;": "\u298b", + "lbrksld;": "\u298f", + "lbrkslu;": "\u298d", + "lcaron;": "\u013e", + "lcedil;": "\u013c", + "lceil;": "\u2308", + "lcub;": "{", + "lcy;": "\u043b", + "ldca;": "\u2936", + "ldquo;": "\u201c", + "ldquor;": "\u201e", + "ldrdhar;": "\u2967", + "ldrushar;": "\u294b", + "ldsh;": "\u21b2", + "le;": "\u2264", + "leftarrow;": "\u2190", + "leftarrowtail;": "\u21a2", + "leftharpoondown;": "\u21bd", + "leftharpoonup;": "\u21bc", + "leftleftarrows;": "\u21c7", + "leftrightarrow;": "\u2194", + "leftrightarrows;": "\u21c6", + "leftrightharpoons;": "\u21cb", + "leftrightsquigarrow;": "\u21ad", + "leftthreetimes;": "\u22cb", + "leg;": "\u22da", + "leq;": "\u2264", + "leqq;": "\u2266", + "leqslant;": "\u2a7d", + "les;": "\u2a7d", + "lescc;": "\u2aa8", + "lesdot;": "\u2a7f", + "lesdoto;": "\u2a81", + "lesdotor;": "\u2a83", + "lesg;": "\u22da\ufe00", + "lesges;": "\u2a93", + "lessapprox;": "\u2a85", + "lessdot;": "\u22d6", + "lesseqgtr;": "\u22da", + "lesseqqgtr;": "\u2a8b", + "lessgtr;": "\u2276", + "lesssim;": "\u2272", + "lfisht;": "\u297c", + "lfloor;": "\u230a", + "lfr;": "\U0001d529", + "lg;": "\u2276", + "lgE;": "\u2a91", + "lhard;": "\u21bd", + "lharu;": "\u21bc", + "lharul;": "\u296a", + "lhblk;": "\u2584", + "ljcy;": "\u0459", + "ll;": "\u226a", + "llarr;": "\u21c7", + "llcorner;": "\u231e", + "llhard;": "\u296b", + "lltri;": "\u25fa", + "lmidot;": "\u0140", + "lmoust;": "\u23b0", + "lmoustache;": "\u23b0", + "lnE;": "\u2268", + "lnap;": "\u2a89", + "lnapprox;": "\u2a89", + "lne;": "\u2a87", + "lneq;": "\u2a87", + "lneqq;": "\u2268", + "lnsim;": "\u22e6", + "loang;": "\u27ec", + "loarr;": "\u21fd", + "lobrk;": "\u27e6", + "longleftarrow;": "\u27f5", + "longleftrightarrow;": "\u27f7", + "longmapsto;": "\u27fc", + "longrightarrow;": "\u27f6", + "looparrowleft;": "\u21ab", + "looparrowright;": "\u21ac", + "lopar;": "\u2985", + "lopf;": "\U0001d55d", + "loplus;": "\u2a2d", + "lotimes;": "\u2a34", + "lowast;": "\u2217", + "lowbar;": "_", + "loz;": "\u25ca", + "lozenge;": "\u25ca", + "lozf;": "\u29eb", + "lpar;": "(", + "lparlt;": "\u2993", + "lrarr;": "\u21c6", + "lrcorner;": "\u231f", + "lrhar;": "\u21cb", + "lrhard;": "\u296d", + "lrm;": "\u200e", + "lrtri;": "\u22bf", + "lsaquo;": "\u2039", + "lscr;": "\U0001d4c1", + "lsh;": "\u21b0", + "lsim;": "\u2272", + "lsime;": "\u2a8d", + "lsimg;": "\u2a8f", + "lsqb;": "[", + "lsquo;": "\u2018", + "lsquor;": "\u201a", + "lstrok;": "\u0142", + "lt": "<", + "lt;": "<", + "ltcc;": "\u2aa6", + "ltcir;": "\u2a79", + "ltdot;": "\u22d6", + "lthree;": "\u22cb", + "ltimes;": "\u22c9", + "ltlarr;": "\u2976", + "ltquest;": "\u2a7b", + "ltrPar;": "\u2996", + "ltri;": "\u25c3", + "ltrie;": "\u22b4", + "ltrif;": "\u25c2", + "lurdshar;": "\u294a", + "luruhar;": "\u2966", + "lvertneqq;": "\u2268\ufe00", + "lvnE;": "\u2268\ufe00", + "mDDot;": "\u223a", + "macr": "\xaf", + "macr;": "\xaf", + "male;": "\u2642", + "malt;": "\u2720", + "maltese;": "\u2720", + "map;": "\u21a6", + "mapsto;": "\u21a6", + "mapstodown;": "\u21a7", + "mapstoleft;": "\u21a4", + "mapstoup;": "\u21a5", + "marker;": "\u25ae", + "mcomma;": "\u2a29", + "mcy;": "\u043c", + "mdash;": "\u2014", + "measuredangle;": "\u2221", + "mfr;": "\U0001d52a", + "mho;": "\u2127", + "micro": "\xb5", + "micro;": "\xb5", + "mid;": "\u2223", + "midast;": "*", + "midcir;": "\u2af0", + "middot": "\xb7", + "middot;": "\xb7", + "minus;": "\u2212", + "minusb;": "\u229f", + "minusd;": "\u2238", + "minusdu;": "\u2a2a", + "mlcp;": "\u2adb", + "mldr;": "\u2026", + "mnplus;": "\u2213", + "models;": "\u22a7", + "mopf;": "\U0001d55e", + "mp;": "\u2213", + "mscr;": "\U0001d4c2", + "mstpos;": "\u223e", + "mu;": "\u03bc", + "multimap;": "\u22b8", + "mumap;": "\u22b8", + "nGg;": "\u22d9\u0338", + "nGt;": "\u226b\u20d2", + "nGtv;": "\u226b\u0338", + "nLeftarrow;": "\u21cd", + "nLeftrightarrow;": "\u21ce", + "nLl;": "\u22d8\u0338", + "nLt;": "\u226a\u20d2", + "nLtv;": "\u226a\u0338", + "nRightarrow;": "\u21cf", + "nVDash;": "\u22af", + "nVdash;": "\u22ae", + "nabla;": "\u2207", + "nacute;": "\u0144", + "nang;": "\u2220\u20d2", + "nap;": "\u2249", + "napE;": "\u2a70\u0338", + "napid;": "\u224b\u0338", + "napos;": "\u0149", + "napprox;": "\u2249", + "natur;": "\u266e", + "natural;": "\u266e", + "naturals;": "\u2115", + "nbsp": "\xa0", + "nbsp;": "\xa0", + "nbump;": "\u224e\u0338", + "nbumpe;": "\u224f\u0338", + "ncap;": "\u2a43", + "ncaron;": "\u0148", + "ncedil;": "\u0146", + "ncong;": "\u2247", + "ncongdot;": "\u2a6d\u0338", + "ncup;": "\u2a42", + "ncy;": "\u043d", + "ndash;": "\u2013", + "ne;": "\u2260", + "neArr;": "\u21d7", + "nearhk;": "\u2924", + "nearr;": "\u2197", + "nearrow;": "\u2197", + "nedot;": "\u2250\u0338", + "nequiv;": "\u2262", + "nesear;": "\u2928", + "nesim;": "\u2242\u0338", + "nexist;": "\u2204", + "nexists;": "\u2204", + "nfr;": "\U0001d52b", + "ngE;": "\u2267\u0338", + "nge;": "\u2271", + "ngeq;": "\u2271", + "ngeqq;": "\u2267\u0338", + "ngeqslant;": "\u2a7e\u0338", + "nges;": "\u2a7e\u0338", + "ngsim;": "\u2275", + "ngt;": "\u226f", + "ngtr;": "\u226f", + "nhArr;": "\u21ce", + "nharr;": "\u21ae", + "nhpar;": "\u2af2", + "ni;": "\u220b", + "nis;": "\u22fc", + "nisd;": "\u22fa", + "niv;": "\u220b", + "njcy;": "\u045a", + "nlArr;": "\u21cd", + "nlE;": "\u2266\u0338", + "nlarr;": "\u219a", + "nldr;": "\u2025", + "nle;": "\u2270", + "nleftarrow;": "\u219a", + "nleftrightarrow;": "\u21ae", + "nleq;": "\u2270", + "nleqq;": "\u2266\u0338", + "nleqslant;": "\u2a7d\u0338", + "nles;": "\u2a7d\u0338", + "nless;": "\u226e", + "nlsim;": "\u2274", + "nlt;": "\u226e", + "nltri;": "\u22ea", + "nltrie;": "\u22ec", + "nmid;": "\u2224", + "nopf;": "\U0001d55f", + "not": "\xac", + "not;": "\xac", + "notin;": "\u2209", + "notinE;": "\u22f9\u0338", + "notindot;": "\u22f5\u0338", + "notinva;": "\u2209", + "notinvb;": "\u22f7", + "notinvc;": "\u22f6", + "notni;": "\u220c", + "notniva;": "\u220c", + "notnivb;": "\u22fe", + "notnivc;": "\u22fd", + "npar;": "\u2226", + "nparallel;": "\u2226", + "nparsl;": "\u2afd\u20e5", + "npart;": "\u2202\u0338", + "npolint;": "\u2a14", + "npr;": "\u2280", + "nprcue;": "\u22e0", + "npre;": "\u2aaf\u0338", + "nprec;": "\u2280", + "npreceq;": "\u2aaf\u0338", + "nrArr;": "\u21cf", + "nrarr;": "\u219b", + "nrarrc;": "\u2933\u0338", + "nrarrw;": "\u219d\u0338", + "nrightarrow;": "\u219b", + "nrtri;": "\u22eb", + "nrtrie;": "\u22ed", + "nsc;": "\u2281", + "nsccue;": "\u22e1", + "nsce;": "\u2ab0\u0338", + "nscr;": "\U0001d4c3", + "nshortmid;": "\u2224", + "nshortparallel;": "\u2226", + "nsim;": "\u2241", + "nsime;": "\u2244", + "nsimeq;": "\u2244", + "nsmid;": "\u2224", + "nspar;": "\u2226", + "nsqsube;": "\u22e2", + "nsqsupe;": "\u22e3", + "nsub;": "\u2284", + "nsubE;": "\u2ac5\u0338", + "nsube;": "\u2288", + "nsubset;": "\u2282\u20d2", + "nsubseteq;": "\u2288", + "nsubseteqq;": "\u2ac5\u0338", + "nsucc;": "\u2281", + "nsucceq;": "\u2ab0\u0338", + "nsup;": "\u2285", + "nsupE;": "\u2ac6\u0338", + "nsupe;": "\u2289", + "nsupset;": "\u2283\u20d2", + "nsupseteq;": "\u2289", + "nsupseteqq;": "\u2ac6\u0338", + "ntgl;": "\u2279", + "ntilde": "\xf1", + "ntilde;": "\xf1", + "ntlg;": "\u2278", + "ntriangleleft;": "\u22ea", + "ntrianglelefteq;": "\u22ec", + "ntriangleright;": "\u22eb", + "ntrianglerighteq;": "\u22ed", + "nu;": "\u03bd", + "num;": "#", + "numero;": "\u2116", + "numsp;": "\u2007", + "nvDash;": "\u22ad", + "nvHarr;": "\u2904", + "nvap;": "\u224d\u20d2", + "nvdash;": "\u22ac", + "nvge;": "\u2265\u20d2", + "nvgt;": ">\u20d2", + "nvinfin;": "\u29de", + "nvlArr;": "\u2902", + "nvle;": "\u2264\u20d2", + "nvlt;": "<\u20d2", + "nvltrie;": "\u22b4\u20d2", + "nvrArr;": "\u2903", + "nvrtrie;": "\u22b5\u20d2", + "nvsim;": "\u223c\u20d2", + "nwArr;": "\u21d6", + "nwarhk;": "\u2923", + "nwarr;": "\u2196", + "nwarrow;": "\u2196", + "nwnear;": "\u2927", + "oS;": "\u24c8", + "oacute": "\xf3", + "oacute;": "\xf3", + "oast;": "\u229b", + "ocir;": "\u229a", + "ocirc": "\xf4", + "ocirc;": "\xf4", + "ocy;": "\u043e", + "odash;": "\u229d", + "odblac;": "\u0151", + "odiv;": "\u2a38", + "odot;": "\u2299", + "odsold;": "\u29bc", + "oelig;": "\u0153", + "ofcir;": "\u29bf", + "ofr;": "\U0001d52c", + "ogon;": "\u02db", + "ograve": "\xf2", + "ograve;": "\xf2", + "ogt;": "\u29c1", + "ohbar;": "\u29b5", + "ohm;": "\u03a9", + "oint;": "\u222e", + "olarr;": "\u21ba", + "olcir;": "\u29be", + "olcross;": "\u29bb", + "oline;": "\u203e", + "olt;": "\u29c0", + "omacr;": "\u014d", + "omega;": "\u03c9", + "omicron;": "\u03bf", + "omid;": "\u29b6", + "ominus;": "\u2296", + "oopf;": "\U0001d560", + "opar;": "\u29b7", + "operp;": "\u29b9", + "oplus;": "\u2295", + "or;": "\u2228", + "orarr;": "\u21bb", + "ord;": "\u2a5d", + "order;": "\u2134", + "orderof;": "\u2134", + "ordf": "\xaa", + "ordf;": "\xaa", + "ordm": "\xba", + "ordm;": "\xba", + "origof;": "\u22b6", + "oror;": "\u2a56", + "orslope;": "\u2a57", + "orv;": "\u2a5b", + "oscr;": "\u2134", + "oslash": "\xf8", + "oslash;": "\xf8", + "osol;": "\u2298", + "otilde": "\xf5", + "otilde;": "\xf5", + "otimes;": "\u2297", + "otimesas;": "\u2a36", + "ouml": "\xf6", + "ouml;": "\xf6", + "ovbar;": "\u233d", + "par;": "\u2225", + "para": "\xb6", + "para;": "\xb6", + "parallel;": "\u2225", + "parsim;": "\u2af3", + "parsl;": "\u2afd", + "part;": "\u2202", + "pcy;": "\u043f", + "percnt;": "%", + "period;": ".", + "permil;": "\u2030", + "perp;": "\u22a5", + "pertenk;": "\u2031", + "pfr;": "\U0001d52d", + "phi;": "\u03c6", + "phiv;": "\u03d5", + "phmmat;": "\u2133", + "phone;": "\u260e", + "pi;": "\u03c0", + "pitchfork;": "\u22d4", + "piv;": "\u03d6", + "planck;": "\u210f", + "planckh;": "\u210e", + "plankv;": "\u210f", + "plus;": "+", + "plusacir;": "\u2a23", + "plusb;": "\u229e", + "pluscir;": "\u2a22", + "plusdo;": "\u2214", + "plusdu;": "\u2a25", + "pluse;": "\u2a72", + "plusmn": "\xb1", + "plusmn;": "\xb1", + "plussim;": "\u2a26", + "plustwo;": "\u2a27", + "pm;": "\xb1", + "pointint;": "\u2a15", + "popf;": "\U0001d561", + "pound": "\xa3", + "pound;": "\xa3", + "pr;": "\u227a", + "prE;": "\u2ab3", + "prap;": "\u2ab7", + "prcue;": "\u227c", + "pre;": "\u2aaf", + "prec;": "\u227a", + "precapprox;": "\u2ab7", + "preccurlyeq;": "\u227c", + "preceq;": "\u2aaf", + "precnapprox;": "\u2ab9", + "precneqq;": "\u2ab5", + "precnsim;": "\u22e8", + "precsim;": "\u227e", + "prime;": "\u2032", + "primes;": "\u2119", + "prnE;": "\u2ab5", + "prnap;": "\u2ab9", + "prnsim;": "\u22e8", + "prod;": "\u220f", + "profalar;": "\u232e", + "profline;": "\u2312", + "profsurf;": "\u2313", + "prop;": "\u221d", + "propto;": "\u221d", + "prsim;": "\u227e", + "prurel;": "\u22b0", + "pscr;": "\U0001d4c5", + "psi;": "\u03c8", + "puncsp;": "\u2008", + "qfr;": "\U0001d52e", + "qint;": "\u2a0c", + "qopf;": "\U0001d562", + "qprime;": "\u2057", + "qscr;": "\U0001d4c6", + "quaternions;": "\u210d", + "quatint;": "\u2a16", + "quest;": "?", + "questeq;": "\u225f", + "quot": "\"", + "quot;": "\"", + "rAarr;": "\u21db", + "rArr;": "\u21d2", + "rAtail;": "\u291c", + "rBarr;": "\u290f", + "rHar;": "\u2964", + "race;": "\u223d\u0331", + "racute;": "\u0155", + "radic;": "\u221a", + "raemptyv;": "\u29b3", + "rang;": "\u27e9", + "rangd;": "\u2992", + "range;": "\u29a5", + "rangle;": "\u27e9", + "raquo": "\xbb", + "raquo;": "\xbb", + "rarr;": "\u2192", + "rarrap;": "\u2975", + "rarrb;": "\u21e5", + "rarrbfs;": "\u2920", + "rarrc;": "\u2933", + "rarrfs;": "\u291e", + "rarrhk;": "\u21aa", + "rarrlp;": "\u21ac", + "rarrpl;": "\u2945", + "rarrsim;": "\u2974", + "rarrtl;": "\u21a3", + "rarrw;": "\u219d", + "ratail;": "\u291a", + "ratio;": "\u2236", + "rationals;": "\u211a", + "rbarr;": "\u290d", + "rbbrk;": "\u2773", + "rbrace;": "}", + "rbrack;": "]", + "rbrke;": "\u298c", + "rbrksld;": "\u298e", + "rbrkslu;": "\u2990", + "rcaron;": "\u0159", + "rcedil;": "\u0157", + "rceil;": "\u2309", + "rcub;": "}", + "rcy;": "\u0440", + "rdca;": "\u2937", + "rdldhar;": "\u2969", + "rdquo;": "\u201d", + "rdquor;": "\u201d", + "rdsh;": "\u21b3", + "real;": "\u211c", + "realine;": "\u211b", + "realpart;": "\u211c", + "reals;": "\u211d", + "rect;": "\u25ad", + "reg": "\xae", + "reg;": "\xae", + "rfisht;": "\u297d", + "rfloor;": "\u230b", + "rfr;": "\U0001d52f", + "rhard;": "\u21c1", + "rharu;": "\u21c0", + "rharul;": "\u296c", + "rho;": "\u03c1", + "rhov;": "\u03f1", + "rightarrow;": "\u2192", + "rightarrowtail;": "\u21a3", + "rightharpoondown;": "\u21c1", + "rightharpoonup;": "\u21c0", + "rightleftarrows;": "\u21c4", + "rightleftharpoons;": "\u21cc", + "rightrightarrows;": "\u21c9", + "rightsquigarrow;": "\u219d", + "rightthreetimes;": "\u22cc", + "ring;": "\u02da", + "risingdotseq;": "\u2253", + "rlarr;": "\u21c4", + "rlhar;": "\u21cc", + "rlm;": "\u200f", + "rmoust;": "\u23b1", + "rmoustache;": "\u23b1", + "rnmid;": "\u2aee", + "roang;": "\u27ed", + "roarr;": "\u21fe", + "robrk;": "\u27e7", + "ropar;": "\u2986", + "ropf;": "\U0001d563", + "roplus;": "\u2a2e", + "rotimes;": "\u2a35", + "rpar;": ")", + "rpargt;": "\u2994", + "rppolint;": "\u2a12", + "rrarr;": "\u21c9", + "rsaquo;": "\u203a", + "rscr;": "\U0001d4c7", + "rsh;": "\u21b1", + "rsqb;": "]", + "rsquo;": "\u2019", + "rsquor;": "\u2019", + "rthree;": "\u22cc", + "rtimes;": "\u22ca", + "rtri;": "\u25b9", + "rtrie;": "\u22b5", + "rtrif;": "\u25b8", + "rtriltri;": "\u29ce", + "ruluhar;": "\u2968", + "rx;": "\u211e", + "sacute;": "\u015b", + "sbquo;": "\u201a", + "sc;": "\u227b", + "scE;": "\u2ab4", + "scap;": "\u2ab8", + "scaron;": "\u0161", + "sccue;": "\u227d", + "sce;": "\u2ab0", + "scedil;": "\u015f", + "scirc;": "\u015d", + "scnE;": "\u2ab6", + "scnap;": "\u2aba", + "scnsim;": "\u22e9", + "scpolint;": "\u2a13", + "scsim;": "\u227f", + "scy;": "\u0441", + "sdot;": "\u22c5", + "sdotb;": "\u22a1", + "sdote;": "\u2a66", + "seArr;": "\u21d8", + "searhk;": "\u2925", + "searr;": "\u2198", + "searrow;": "\u2198", + "sect": "\xa7", + "sect;": "\xa7", + "semi;": ";", + "seswar;": "\u2929", + "setminus;": "\u2216", + "setmn;": "\u2216", + "sext;": "\u2736", + "sfr;": "\U0001d530", + "sfrown;": "\u2322", + "sharp;": "\u266f", + "shchcy;": "\u0449", + "shcy;": "\u0448", + "shortmid;": "\u2223", + "shortparallel;": "\u2225", + "shy": "\xad", + "shy;": "\xad", + "sigma;": "\u03c3", + "sigmaf;": "\u03c2", + "sigmav;": "\u03c2", + "sim;": "\u223c", + "simdot;": "\u2a6a", + "sime;": "\u2243", + "simeq;": "\u2243", + "simg;": "\u2a9e", + "simgE;": "\u2aa0", + "siml;": "\u2a9d", + "simlE;": "\u2a9f", + "simne;": "\u2246", + "simplus;": "\u2a24", + "simrarr;": "\u2972", + "slarr;": "\u2190", + "smallsetminus;": "\u2216", + "smashp;": "\u2a33", + "smeparsl;": "\u29e4", + "smid;": "\u2223", + "smile;": "\u2323", + "smt;": "\u2aaa", + "smte;": "\u2aac", + "smtes;": "\u2aac\ufe00", + "softcy;": "\u044c", + "sol;": "/", + "solb;": "\u29c4", + "solbar;": "\u233f", + "sopf;": "\U0001d564", + "spades;": "\u2660", + "spadesuit;": "\u2660", + "spar;": "\u2225", + "sqcap;": "\u2293", + "sqcaps;": "\u2293\ufe00", + "sqcup;": "\u2294", + "sqcups;": "\u2294\ufe00", + "sqsub;": "\u228f", + "sqsube;": "\u2291", + "sqsubset;": "\u228f", + "sqsubseteq;": "\u2291", + "sqsup;": "\u2290", + "sqsupe;": "\u2292", + "sqsupset;": "\u2290", + "sqsupseteq;": "\u2292", + "squ;": "\u25a1", + "square;": "\u25a1", + "squarf;": "\u25aa", + "squf;": "\u25aa", + "srarr;": "\u2192", + "sscr;": "\U0001d4c8", + "ssetmn;": "\u2216", + "ssmile;": "\u2323", + "sstarf;": "\u22c6", + "star;": "\u2606", + "starf;": "\u2605", + "straightepsilon;": "\u03f5", + "straightphi;": "\u03d5", + "strns;": "\xaf", + "sub;": "\u2282", + "subE;": "\u2ac5", + "subdot;": "\u2abd", + "sube;": "\u2286", + "subedot;": "\u2ac3", + "submult;": "\u2ac1", + "subnE;": "\u2acb", + "subne;": "\u228a", + "subplus;": "\u2abf", + "subrarr;": "\u2979", + "subset;": "\u2282", + "subseteq;": "\u2286", + "subseteqq;": "\u2ac5", + "subsetneq;": "\u228a", + "subsetneqq;": "\u2acb", + "subsim;": "\u2ac7", + "subsub;": "\u2ad5", + "subsup;": "\u2ad3", + "succ;": "\u227b", + "succapprox;": "\u2ab8", + "succcurlyeq;": "\u227d", + "succeq;": "\u2ab0", + "succnapprox;": "\u2aba", + "succneqq;": "\u2ab6", + "succnsim;": "\u22e9", + "succsim;": "\u227f", + "sum;": "\u2211", + "sung;": "\u266a", + "sup1": "\xb9", + "sup1;": "\xb9", + "sup2": "\xb2", + "sup2;": "\xb2", + "sup3": "\xb3", + "sup3;": "\xb3", + "sup;": "\u2283", + "supE;": "\u2ac6", + "supdot;": "\u2abe", + "supdsub;": "\u2ad8", + "supe;": "\u2287", + "supedot;": "\u2ac4", + "suphsol;": "\u27c9", + "suphsub;": "\u2ad7", + "suplarr;": "\u297b", + "supmult;": "\u2ac2", + "supnE;": "\u2acc", + "supne;": "\u228b", + "supplus;": "\u2ac0", + "supset;": "\u2283", + "supseteq;": "\u2287", + "supseteqq;": "\u2ac6", + "supsetneq;": "\u228b", + "supsetneqq;": "\u2acc", + "supsim;": "\u2ac8", + "supsub;": "\u2ad4", + "supsup;": "\u2ad6", + "swArr;": "\u21d9", + "swarhk;": "\u2926", + "swarr;": "\u2199", + "swarrow;": "\u2199", + "swnwar;": "\u292a", + "szlig": "\xdf", + "szlig;": "\xdf", + "target;": "\u2316", + "tau;": "\u03c4", + "tbrk;": "\u23b4", + "tcaron;": "\u0165", + "tcedil;": "\u0163", + "tcy;": "\u0442", + "tdot;": "\u20db", + "telrec;": "\u2315", + "tfr;": "\U0001d531", + "there4;": "\u2234", + "therefore;": "\u2234", + "theta;": "\u03b8", + "thetasym;": "\u03d1", + "thetav;": "\u03d1", + "thickapprox;": "\u2248", + "thicksim;": "\u223c", + "thinsp;": "\u2009", + "thkap;": "\u2248", + "thksim;": "\u223c", + "thorn": "\xfe", + "thorn;": "\xfe", + "tilde;": "\u02dc", + "times": "\xd7", + "times;": "\xd7", + "timesb;": "\u22a0", + "timesbar;": "\u2a31", + "timesd;": "\u2a30", + "tint;": "\u222d", + "toea;": "\u2928", + "top;": "\u22a4", + "topbot;": "\u2336", + "topcir;": "\u2af1", + "topf;": "\U0001d565", + "topfork;": "\u2ada", + "tosa;": "\u2929", + "tprime;": "\u2034", + "trade;": "\u2122", + "triangle;": "\u25b5", + "triangledown;": "\u25bf", + "triangleleft;": "\u25c3", + "trianglelefteq;": "\u22b4", + "triangleq;": "\u225c", + "triangleright;": "\u25b9", + "trianglerighteq;": "\u22b5", + "tridot;": "\u25ec", + "trie;": "\u225c", + "triminus;": "\u2a3a", + "triplus;": "\u2a39", + "trisb;": "\u29cd", + "tritime;": "\u2a3b", + "trpezium;": "\u23e2", + "tscr;": "\U0001d4c9", + "tscy;": "\u0446", + "tshcy;": "\u045b", + "tstrok;": "\u0167", + "twixt;": "\u226c", + "twoheadleftarrow;": "\u219e", + "twoheadrightarrow;": "\u21a0", + "uArr;": "\u21d1", + "uHar;": "\u2963", + "uacute": "\xfa", + "uacute;": "\xfa", + "uarr;": "\u2191", + "ubrcy;": "\u045e", + "ubreve;": "\u016d", + "ucirc": "\xfb", + "ucirc;": "\xfb", + "ucy;": "\u0443", + "udarr;": "\u21c5", + "udblac;": "\u0171", + "udhar;": "\u296e", + "ufisht;": "\u297e", + "ufr;": "\U0001d532", + "ugrave": "\xf9", + "ugrave;": "\xf9", + "uharl;": "\u21bf", + "uharr;": "\u21be", + "uhblk;": "\u2580", + "ulcorn;": "\u231c", + "ulcorner;": "\u231c", + "ulcrop;": "\u230f", + "ultri;": "\u25f8", + "umacr;": "\u016b", + "uml": "\xa8", + "uml;": "\xa8", + "uogon;": "\u0173", + "uopf;": "\U0001d566", + "uparrow;": "\u2191", + "updownarrow;": "\u2195", + "upharpoonleft;": "\u21bf", + "upharpoonright;": "\u21be", + "uplus;": "\u228e", + "upsi;": "\u03c5", + "upsih;": "\u03d2", + "upsilon;": "\u03c5", + "upuparrows;": "\u21c8", + "urcorn;": "\u231d", + "urcorner;": "\u231d", + "urcrop;": "\u230e", + "uring;": "\u016f", + "urtri;": "\u25f9", + "uscr;": "\U0001d4ca", + "utdot;": "\u22f0", + "utilde;": "\u0169", + "utri;": "\u25b5", + "utrif;": "\u25b4", + "uuarr;": "\u21c8", + "uuml": "\xfc", + "uuml;": "\xfc", + "uwangle;": "\u29a7", + "vArr;": "\u21d5", + "vBar;": "\u2ae8", + "vBarv;": "\u2ae9", + "vDash;": "\u22a8", + "vangrt;": "\u299c", + "varepsilon;": "\u03f5", + "varkappa;": "\u03f0", + "varnothing;": "\u2205", + "varphi;": "\u03d5", + "varpi;": "\u03d6", + "varpropto;": "\u221d", + "varr;": "\u2195", + "varrho;": "\u03f1", + "varsigma;": "\u03c2", + "varsubsetneq;": "\u228a\ufe00", + "varsubsetneqq;": "\u2acb\ufe00", + "varsupsetneq;": "\u228b\ufe00", + "varsupsetneqq;": "\u2acc\ufe00", + "vartheta;": "\u03d1", + "vartriangleleft;": "\u22b2", + "vartriangleright;": "\u22b3", + "vcy;": "\u0432", + "vdash;": "\u22a2", + "vee;": "\u2228", + "veebar;": "\u22bb", + "veeeq;": "\u225a", + "vellip;": "\u22ee", + "verbar;": "|", + "vert;": "|", + "vfr;": "\U0001d533", + "vltri;": "\u22b2", + "vnsub;": "\u2282\u20d2", + "vnsup;": "\u2283\u20d2", + "vopf;": "\U0001d567", + "vprop;": "\u221d", + "vrtri;": "\u22b3", + "vscr;": "\U0001d4cb", + "vsubnE;": "\u2acb\ufe00", + "vsubne;": "\u228a\ufe00", + "vsupnE;": "\u2acc\ufe00", + "vsupne;": "\u228b\ufe00", + "vzigzag;": "\u299a", + "wcirc;": "\u0175", + "wedbar;": "\u2a5f", + "wedge;": "\u2227", + "wedgeq;": "\u2259", + "weierp;": "\u2118", + "wfr;": "\U0001d534", + "wopf;": "\U0001d568", + "wp;": "\u2118", + "wr;": "\u2240", + "wreath;": "\u2240", + "wscr;": "\U0001d4cc", + "xcap;": "\u22c2", + "xcirc;": "\u25ef", + "xcup;": "\u22c3", + "xdtri;": "\u25bd", + "xfr;": "\U0001d535", + "xhArr;": "\u27fa", + "xharr;": "\u27f7", + "xi;": "\u03be", + "xlArr;": "\u27f8", + "xlarr;": "\u27f5", + "xmap;": "\u27fc", + "xnis;": "\u22fb", + "xodot;": "\u2a00", + "xopf;": "\U0001d569", + "xoplus;": "\u2a01", + "xotime;": "\u2a02", + "xrArr;": "\u27f9", + "xrarr;": "\u27f6", + "xscr;": "\U0001d4cd", + "xsqcup;": "\u2a06", + "xuplus;": "\u2a04", + "xutri;": "\u25b3", + "xvee;": "\u22c1", + "xwedge;": "\u22c0", + "yacute": "\xfd", + "yacute;": "\xfd", + "yacy;": "\u044f", + "ycirc;": "\u0177", + "ycy;": "\u044b", + "yen": "\xa5", + "yen;": "\xa5", + "yfr;": "\U0001d536", + "yicy;": "\u0457", + "yopf;": "\U0001d56a", + "yscr;": "\U0001d4ce", + "yucy;": "\u044e", + "yuml": "\xff", + "yuml;": "\xff", + "zacute;": "\u017a", + "zcaron;": "\u017e", + "zcy;": "\u0437", + "zdot;": "\u017c", + "zeetrf;": "\u2128", + "zeta;": "\u03b6", + "zfr;": "\U0001d537", + "zhcy;": "\u0436", + "zigrarr;": "\u21dd", + "zopf;": "\U0001d56b", + "zscr;": "\U0001d4cf", + "zwj;": "\u200d", + "zwnj;": "\u200c", } replacementCharacters = { - 0x0:u"\uFFFD", - 0x0d:u"\u000D", - 0x80:u"\u20AC", - 0x81:u"\u0081", - 0x81:u"\u0081", - 0x82:u"\u201A", - 0x83:u"\u0192", - 0x84:u"\u201E", - 0x85:u"\u2026", - 0x86:u"\u2020", - 0x87:u"\u2021", - 0x88:u"\u02C6", - 0x89:u"\u2030", - 0x8A:u"\u0160", - 0x8B:u"\u2039", - 0x8C:u"\u0152", - 0x8D:u"\u008D", - 0x8E:u"\u017D", - 0x8F:u"\u008F", - 0x90:u"\u0090", - 0x91:u"\u2018", - 0x92:u"\u2019", - 0x93:u"\u201C", - 0x94:u"\u201D", - 0x95:u"\u2022", - 0x96:u"\u2013", - 0x97:u"\u2014", - 0x98:u"\u02DC", - 0x99:u"\u2122", - 0x9A:u"\u0161", - 0x9B:u"\u203A", - 0x9C:u"\u0153", - 0x9D:u"\u009D", - 0x9E:u"\u017E", - 0x9F:u"\u0178", + 0x0: "\uFFFD", + 0x0d: "\u000D", + 0x80: "\u20AC", + 0x81: "\u0081", + 0x81: "\u0081", + 0x82: "\u201A", + 0x83: "\u0192", + 0x84: "\u201E", + 0x85: "\u2026", + 0x86: "\u2020", + 0x87: "\u2021", + 0x88: "\u02C6", + 0x89: "\u2030", + 0x8A: "\u0160", + 0x8B: "\u2039", + 0x8C: "\u0152", + 0x8D: "\u008D", + 0x8E: "\u017D", + 0x8F: "\u008F", + 0x90: "\u0090", + 0x91: "\u2018", + 0x92: "\u2019", + 0x93: "\u201C", + 0x94: "\u201D", + 0x95: "\u2022", + 0x96: "\u2013", + 0x97: "\u2014", + 0x98: "\u02DC", + 0x99: "\u2122", + 0x9A: "\u0161", + 0x9B: "\u203A", + 0x9C: "\u0153", + 0x9D: "\u009D", + 0x9E: "\u017E", + 0x9F: "\u0178", } encodings = { @@ -3061,25 +3078,27 @@ encodings = { 'x-x-big5': 'big5'} tokenTypes = { - "Doctype":0, - "Characters":1, - "SpaceCharacters":2, - "StartTag":3, - "EndTag":4, - "EmptyTag":5, - "Comment":6, - "ParseError":7 + "Doctype": 0, + "Characters": 1, + "SpaceCharacters": 2, + "StartTag": 3, + "EndTag": 4, + "EmptyTag": 5, + "Comment": 6, + "ParseError": 7 } -tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"], +tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"], tokenTypes["EmptyTag"])) -prefixes = dict([(v,k) for k,v in namespaces.iteritems()]) +prefixes = dict([(v, k) for k, v in namespaces.items()]) prefixes["http://www.w3.org/1998/Math/MathML"] = "math" + class DataLossWarning(UserWarning): pass + class ReparseException(Exception): pass diff --git a/libs/html5lib/filters/_base.py b/libs/html5lib/filters/_base.py index bca94ada..c7dbaed0 100644 --- a/libs/html5lib/filters/_base.py +++ b/libs/html5lib/filters/_base.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, unicode_literals + class Filter(object): def __init__(self, source): diff --git a/libs/html5lib/filters/alphabeticalattributes.py b/libs/html5lib/filters/alphabeticalattributes.py new file mode 100644 index 00000000..fed6996c --- /dev/null +++ b/libs/html5lib/filters/alphabeticalattributes.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, division, unicode_literals + +from . import _base + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + + +class Filter(_base.Filter): + def __iter__(self): + for token in _base.Filter.__iter__(self): + if token["type"] in ("StartTag", "EmptyTag"): + attrs = OrderedDict() + for name, value in sorted(token["data"].items(), + key=lambda x: x[0]): + attrs[name] = value + token["data"] = attrs + yield token diff --git a/libs/html5lib/filters/formfiller.py b/libs/html5lib/filters/formfiller.py deleted file mode 100644 index 94001714..00000000 --- a/libs/html5lib/filters/formfiller.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# The goal is to finally have a form filler where you pass data for -# each form, using the algorithm for "Seeding a form with initial values" -# See http://www.whatwg.org/specs/web-forms/current-work/#seeding -# - -import _base - -from html5lib.constants import spaceCharacters -spaceCharacters = u"".join(spaceCharacters) - -class SimpleFilter(_base.Filter): - def __init__(self, source, fieldStorage): - _base.Filter.__init__(self, source) - self.fieldStorage = fieldStorage - - def __iter__(self): - field_indices = {} - state = None - field_name = None - for token in _base.Filter.__iter__(self): - type = token["type"] - if type in ("StartTag", "EmptyTag"): - name = token["name"].lower() - if name == "input": - field_name = None - field_type = None - input_value_index = -1 - input_checked_index = -1 - for i,(n,v) in enumerate(token["data"]): - n = n.lower() - if n == u"name": - field_name = v.strip(spaceCharacters) - elif n == u"type": - field_type = v.strip(spaceCharacters) - elif n == u"checked": - input_checked_index = i - elif n == u"value": - input_value_index = i - - value_list = self.fieldStorage.getlist(field_name) - field_index = field_indices.setdefault(field_name, 0) - if field_index < len(value_list): - value = value_list[field_index] - else: - value = "" - - if field_type in (u"checkbox", u"radio"): - if value_list: - if token["data"][input_value_index][1] == value: - if input_checked_index < 0: - token["data"].append((u"checked", u"")) - field_indices[field_name] = field_index + 1 - elif input_checked_index >= 0: - del token["data"][input_checked_index] - - elif field_type not in (u"button", u"submit", u"reset"): - if input_value_index >= 0: - token["data"][input_value_index] = (u"value", value) - else: - token["data"].append((u"value", value)) - field_indices[field_name] = field_index + 1 - - field_type = None - field_name = None - - elif name == "textarea": - field_type = "textarea" - field_name = dict((token["data"])[::-1])["name"] - - elif name == "select": - field_type = "select" - attributes = dict(token["data"][::-1]) - field_name = attributes.get("name") - is_select_multiple = "multiple" in attributes - is_selected_option_found = False - - elif field_type == "select" and field_name and name == "option": - option_selected_index = -1 - option_value = None - for i,(n,v) in enumerate(token["data"]): - n = n.lower() - if n == "selected": - option_selected_index = i - elif n == "value": - option_value = v.strip(spaceCharacters) - if option_value is None: - raise NotImplementedError("