From 1b5238f8ecc07a25fba2373ea8336f28c2f4c231 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 19 Dec 2011 13:23:07 +0100 Subject: [PATCH] Subliminal base --- couchpotato/core/providers/subtitle/base.py | 9 + .../providers/subtitle/subliminal/__init__.py | 6 + .../providers/subtitle/subliminal/main.py | 8 + libs/subliminal/__init__.py | 31 + libs/subliminal/core.py | 366 +++++++ libs/subliminal/exceptions.py | 99 ++ libs/subliminal/infos.py | 26 + libs/subliminal/languages.py | 535 +++++++++++ libs/subliminal/plugins.py | 890 ++++++++++++++++++ libs/subliminal/subtitles.py | 110 +++ libs/subliminal/tasks.py | 47 + libs/subliminal/utils.py | 51 + libs/subliminal/videos.py | 214 +++++ 13 files changed, 2392 insertions(+) create mode 100644 couchpotato/core/providers/subtitle/subliminal/__init__.py create mode 100644 couchpotato/core/providers/subtitle/subliminal/main.py create mode 100755 libs/subliminal/__init__.py create mode 100755 libs/subliminal/core.py create mode 100755 libs/subliminal/exceptions.py create mode 100755 libs/subliminal/infos.py create mode 100755 libs/subliminal/languages.py create mode 100755 libs/subliminal/plugins.py create mode 100755 libs/subliminal/subtitles.py create mode 100755 libs/subliminal/tasks.py create mode 100755 libs/subliminal/utils.py create mode 100755 libs/subliminal/videos.py diff --git a/couchpotato/core/providers/subtitle/base.py b/couchpotato/core/providers/subtitle/base.py index 57984192..46bfa81c 100644 --- a/couchpotato/core/providers/subtitle/base.py +++ b/couchpotato/core/providers/subtitle/base.py @@ -1,5 +1,14 @@ +from couchpotato.core.event import addEvent from couchpotato.core.providers.base import Provider class SubtitleProvider(Provider): + type = 'subtitle' + + def __init__(self): + + addEvent('renamer.before', self.search) + + def search(self, group): + pass diff --git a/couchpotato/core/providers/subtitle/subliminal/__init__.py b/couchpotato/core/providers/subtitle/subliminal/__init__.py new file mode 100644 index 00000000..6592c2fe --- /dev/null +++ b/couchpotato/core/providers/subtitle/subliminal/__init__.py @@ -0,0 +1,6 @@ +from .main import Subliminal + +def start(): + return Subliminal() + +config = [] diff --git a/couchpotato/core/providers/subtitle/subliminal/main.py b/couchpotato/core/providers/subtitle/subliminal/main.py new file mode 100644 index 00000000..0c0df7dc --- /dev/null +++ b/couchpotato/core/providers/subtitle/subliminal/main.py @@ -0,0 +1,8 @@ +from couchpotato.core.logger import CPLog +from couchpotato.core.providers.subtitle.base import SubtitleProvider + +log = CPLog(__name__) + + +class Subliminal(SubtitleProvider): + pass diff --git a/libs/subliminal/__init__.py b/libs/subliminal/__init__.py new file mode 100755 index 00000000..a5442a20 --- /dev/null +++ b/libs/subliminal/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + + +from infos import * +from languages import * +from utils import * +from exceptions import * +from videos import * +from tasks import * +from subtitles import * +from core import * +from plugins import * diff --git a/libs/subliminal/core.py b/libs/subliminal/core.py new file mode 100755 index 00000000..e45778fa --- /dev/null +++ b/libs/subliminal/core.py @@ -0,0 +1,366 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['PLUGINS', 'API_PLUGINS', 'IDLE', 'RUNNING', 'PAUSED', 'Subliminal', 'PluginWorker', 'matching_confidence', + 'LANGUAGE_INDEX', 'PLUGIN_INDEX', 'PLUGIN_CONFIDENCE', 'MATCHING_CONFIDENCE'] + + +from collections import defaultdict +from exceptions import InvalidLanguageError, PluginError, BadStateError, \ + WrongTaskError, DownloadFailedError +from itertools import groupby +from languages import list_languages +from subliminal.utils import NullHandler +from tasks import Task, DownloadTask, ListTask, StopTask +import Queue +import guessit +import logging +import os +import plugins +import subtitles +import threading +import utils +import videos + +# init logger +logger = logging.getLogger('subliminal') +logger.addHandler(NullHandler()) + +# const +PLUGINS = ['OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos'] +API_PLUGINS = filter(lambda p: getattr(plugins, p).api_based, PLUGINS) +IDLE, RUNNING, PAUSED = range(3) +LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE = range(4) + + +class Subliminal(object): + """Main Subliminal class""" + + def __init__(self, cache_dir=None, workers=None, multi=False, force=False, + max_depth=None, filemode=None, sort_order=None, plugins=None, languages=None): + self.multi = multi + self.sort_order = sort_order or [LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE] + self.force = force + self.max_depth = max_depth or 3 + self.taskQueue = Queue.PriorityQueue() + self.listResultQueue = Queue.Queue() + self.downloadResultQueue = Queue.Queue() + self.languages = languages or [] + self.plugins = plugins or API_PLUGINS + self._workers = workers or 4 + self.filemode = filemode + self.state = IDLE + self.cache_dir = cache_dir + try: + if cache_dir and not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + logger.debug(u'Creating cache directory: %r' % cache_dir) + except: + self.cache_dir = None + logger.error(u'Failed to use the cache directory, continue without it') + + def __enter__(self): + self.startWorkers() + return self + + def __exit__(self, *args): + self.stopWorkers(0) + + @property + def workers(self): + return self._workers + + @workers.setter + def workers(self, value): + if self.state == RUNNING: + raise BadStateError(self.state, IDLE) + self._workers = value + + @property + def languages(self): + """Getter for languages""" + return self._languages + + @languages.setter + def languages(self, languages): + """Setter for languages""" + logger.debug(u'Setting languages to %r' % languages) + self._languages = [] + for l in languages: + if l not in list_languages(1): + raise InvalidLanguageError(l) + if not l in self._languages: + self._languages.append(l) + + @property + def plugins(self): + """Getter for plugins""" + return self._plugins + + @plugins.setter + def plugins(self, plugins): + """Setter for plugins""" + logger.debug(u'Setting plugins to %r' % plugins) + self._plugins = [] + for p in plugins: + if p not in PLUGINS: + raise PluginError(p) + if not p in self._plugins: + self._plugins.append(p) + + def listSubtitles(self, entries, auto=False): + """ + Search subtitles within the plugins and return all found subtitles in a list of Subtitle object. + + Attributes: + entries -- filepath or folderpath of video file or a list of that + auto -- automaticaly manage workers (default to False)""" + if auto: + if self.state != IDLE: + raise BadStateError(self.state, IDLE) + self.startWorkers() + if isinstance(entries, basestring): + entries = [entries] + config = utils.PluginConfig(self.multi, self.cache_dir, self.filemode) + scan_result = [] + for e in entries: + if not isinstance(e, unicode): + logger.warning(u'Entry %r is not unicode' % e) + scan_result.extend(videos.scan(e)) + task_count = 0 + for video, subtitles in scan_result: + languages = set([s.language for s in subtitles if s.language]) + wanted_languages = set(self._languages) + if not wanted_languages: + wanted_languages = list_languages(1) + if not self.force and self.multi: + wanted_languages = set(wanted_languages) - languages + if not wanted_languages: + logger.debug(u'No need to list multi subtitles %r for %r because %r subtitles detected' % (self._languages, video.path, languages)) + continue + if not self.force and not self.multi and None in [s.language for s in subtitles]: + logger.debug(u'No need to list single subtitles %r for %r because one detected' % (self._languages, video.path)) + continue + logger.debug(u'Listing subtitles %r for %r with %r' % (wanted_languages, video.path, self._plugins)) + for plugin_name in self._plugins: + plugin = getattr(plugins, plugin_name) + to_list_languages = wanted_languages & plugin.availableLanguages() + if not to_list_languages: + logger.debug(u'Skipping %r: none of wanted languages %r available in %r for plugin %s' % (video.path, wanted_languages, plugin.availableLanguages(), plugin_name)) + continue + if not plugin.isValidVideo(video): + logger.debug(u'Skipping %r: video %r is not part of supported videos %r for plugin %s' % (video.path, video, plugin.videos, plugin_name)) + continue + self.taskQueue.put((5, ListTask(video, to_list_languages, plugin_name, config))) + task_count += 1 + subtitles = [] + for _ in range(task_count): + subtitles.extend(self.listResultQueue.get()) + if auto: + self.stopWorkers() + return subtitles + + def downloadSubtitles(self, entries, auto=False): + """ + Download subtitles using the plugins preferences and languages. Also use internal algorithm to find + the best match inside a plugin. + + Attributes: + entries -- filepath or folderpath of video file or a list of that + auto -- automaticaly manage workers (default to False)""" + if auto: + if self.state != IDLE: + raise BadStateError(self.state, IDLE) + self.startWorkers() + by_video = self.groupByVideo(self.listSubtitles(entries, False)) + # Define an order with LANGUAGE_INDEX first for multi sorting + order = self.sort_order + if self.multi: + order.insert(0, LANGUAGE_INDEX) + task_count = 0 + for video, subtitles in by_video.iteritems(): + ordered_subtitles = sorted(subtitles, key=lambda s: self.keySubtitles(s, video, order), reverse=True) + if not self.multi: + self.taskQueue.put((5, DownloadTask(video, list(ordered_subtitles)))) + task_count += 1 + continue + for _, by_language in groupby(ordered_subtitles, lambda s: s.language): + self.taskQueue.put((5, DownloadTask(video, list(by_language)))) + task_count += 1 + downloaded = [] + for _ in range(task_count): + downloaded.extend(self.downloadResultQueue.get()) + if auto: + self.stopWorkers() + return downloaded + + def keySubtitles(self, subtitle, video, order): + """Create a key to sort subtitle using preferences""" + key = '' + for sort_item in order: + if sort_item == LANGUAGE_INDEX: + key += '{:03d}'.format(len(self._languages) - self._languages.index(subtitle.language) - 1) + elif sort_item == PLUGIN_INDEX: + key += '{:02d}'.format(len(self._plugins) - self._plugins.index(subtitle.plugin) - 1) + elif sort_item == PLUGIN_CONFIDENCE: + key += '{:04d}'.format(int(subtitle.confidence * 1000)) + elif sort_item == MATCHING_CONFIDENCE: + confidence = 0 + if subtitle.release: + confidence = matching_confidence(video, subtitle) + key += '{:04d}'.format(int(confidence * 1000)) + return int(key) + + def groupByVideo(self, list_result): + '''Because list outputs a list of tuples from different plugins, we need to put them back + together under a single video key''' + result = defaultdict(list) + for video, subtitles in list_result: + result[video] += subtitles + return result + + def startWorkers(self): + """Create a pool of workers and start them""" + if self.state == RUNNING: + raise BadStateError(self.state, IDLE) + self.pool = [] + for _ in range(self._workers): + worker = PluginWorker(self.taskQueue, self.listResultQueue, self.downloadResultQueue) + worker.start() + self.pool.append(worker) + logger.debug(u'Worker %s added to the pool' % worker.name) + self.state = RUNNING + + def stopWorkers(self, priority=10): + """Stop workers using a lowest priority stop signal and wait for them to terminate properly""" + for _ in range(self._workers): + self.taskQueue.put((priority, StopTask())) + for worker in self.pool: + worker.join() + self.state = IDLE + if not self.taskQueue.empty(): + self.state = PAUSED + + def pauseWorkers(self): + """Pause workers using a highest priority stop signal and wait for them to terminate properly""" + self.stopWorkers(0) + + def addTask(self, task): + """Add a task with default priority""" + if not isinstance(task, Task) or isinstance(task, StopTask): + raise WrongTaskError() + self.taskQueue.put((5, task)) + + +class PluginWorker(threading.Thread): + """Threaded plugin worker""" + def __init__(self, taskQueue, listResultQueue, downloadResultQueue): + threading.Thread.__init__(self) + self.taskQueue = taskQueue + self.listResultQueue = listResultQueue + self.downloadResultQueue = downloadResultQueue + self.logger = logging.getLogger('subliminal.worker') + self.plugins = {} + + def run(self): + while True: + task = self.taskQueue.get()[1] + if isinstance(task, StopTask): + self.logger.debug(u'Poison pill received in thread %s' % self.name) + self.taskQueue.task_done() + break + result = [] + try: + if isinstance(task, ListTask): + if task.plugin not in self.plugins: # init the plugin + self.plugins[task.plugin] = getattr(plugins, task.plugin)() + self.plugins[task.plugin].init() + # Retrieve the plugin list subtitles and return [(video, [subtitle])] + plugin = self.plugins[task.plugin] + plugin.config = task.config + subtitles = plugin.list(task.video, task.languages) + result = [(task.video, subtitles)] + elif isinstance(task, DownloadTask): + # Attempt to download one subtitle from the given list + for subtitle in task.subtitles: + if subtitle.plugin not in self.plugins: # init the plugin + self.plugins[subtitle.plugin] = getattr(plugins, subtitle.plugin)() + self.plugins[subtitle.plugin].init() + plugin = self.plugins[subtitle.plugin] + try: + result = [plugin.download(subtitle)] + break + except DownloadFailedError: # try the next one + self.logger.warning(u'Could not download subtitle %r, trying next' % subtitle) + continue + if not result: + self.logger.error(u'No subtitles could be downloaded for video %r' % task.video.path or task.video.release) + except: + self.logger.error(u'Exception raised in worker %s' % self.name, exc_info=True) + finally: + # Put the result in the correct queue + if isinstance(task, ListTask): + self.listResultQueue.put(result) + elif isinstance(task, DownloadTask): + self.downloadResultQueue.put(result) + self.taskQueue.task_done() + self.terminate() + self.logger.debug(u'Thread %s terminated' % self.name) + + def terminate(self): + """Terminate instanciated plugins""" + for plugin_name, plugin in self.plugins.iteritems(): + try: + plugin.terminate() + except: + self.logger.error(u'Exception raised when terminating plugin %s' % plugin_name, exc_info=True) + + +def matching_confidence(video, subtitle): + '''Compute the confidence that the subtitle matches the video. + Returns a float between 0 and 1. 1 being the perfect match.''' + guess = guessit.guess_file_info(subtitle.release, 'autodetect') + video_keywords = utils.get_keywords(video.guess) + subtitle_keywords = utils.get_keywords(guess) | subtitle.keywords + replacement = {'keywords': len(video_keywords & subtitle_keywords)} + if isinstance(video, videos.Episode): + replacement.update({'series': 0, 'season': 0, 'episode': 0}) + matching_format = '{series:b}{season:b}{episode:b}{keywords:03b}' + best = matching_format.format(series=1, season=1, episode=1, keywords=len(video_keywords)) + if guess['type'] in ['episode', 'episodesubtitle']: + if 'series' in guess and guess['series'].lower() == video.series.lower(): + replacement['series'] = 1 + if 'season' in guess and guess['season'] == video.season: + replacement['season'] = 1 + if 'episodeNumber' in guess and guess['episodeNumber'] == video.episode: + replacement['episode'] = 1 + elif isinstance(video, videos.Movie): + replacement.update({'title': 0, 'year': 0}) + matching_format = '{title:b}{year:b}{keywords:03b}' + best = matching_format.format(title=1, year=1, keywords=len(video_keywords)) + if guess['type'] in ['movie', 'moviesubtitle']: + if 'title' in guess and guess['title'].lower() == video.title.lower(): + replacement['title'] = 1 + if 'year' in guess and guess['year'] == video.year: + replacement['year'] = 1 + else: + return 0 + confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2)) + return confidence diff --git a/libs/subliminal/exceptions.py b/libs/subliminal/exceptions.py new file mode 100755 index 00000000..29b417f7 --- /dev/null +++ b/libs/subliminal/exceptions.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + + +class Error(Exception): + """Base class for exceptions in subliminal""" + pass + + +class BadStateError(Error): + """Exception raised when an invalid action is asked + + Attributes: + current -- current state of Subliminal instance + expected -- expected state of Subliminal instance + """ + def __init__(self, current, expected): + self.current = current + self.expected = expected + + def __str__(self): + return 'Expected state %d but current state is %d' % (self.expected, self.current) + + +class InvalidLanguageError(Error): + """Exception raised when invalid language is submitted + + Attributes: + language -- language that cause the error + """ + def __init__(self, language): + self.language = language + + def __str__(self): + return self.language + + +class MissingLanguageError(Error): + """Exception raised when a missing language is found + + Attributes: + language -- the missing language + """ + def __init__(self, language): + self.language = language + + def __str__(self): + return self.language + + +class InvalidPluginError(Error): + """"Exception raised when invalid plugin is submitted + + Attributes: + plugin -- plugin that cause the error + """ + def __init__(self, plugin): + self.plugin = plugin + + def __str__(self): + return self.plugin + + +class PluginError(Error): + """"Exception raised by plugins""" + pass + + +class WrongTaskError(Error): + """"Exception raised when invalid task is submitted""" + pass + + +class DownloadFailedError(Error): + """"Exception raised when a download task has failed in plugin""" + pass + + +class UnknownVideoError(Error): + """"Exception raised when a video could not be identified""" + pass diff --git a/libs/subliminal/infos.py b/libs/subliminal/infos.py new file mode 100755 index 00000000..32ea82f1 --- /dev/null +++ b/libs/subliminal/infos.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__title__ = 'subliminal' +__description__ = 'Subtitles, faster than your thoughts' +__version__ = '0.5' +__author__ = 'Antoine Bertin' +__license__ = 'LGPLv3' +__copyright__ = 'Copyright 2010-2011 Antoine Bertin' diff --git a/libs/subliminal/languages.py b/libs/subliminal/languages.py new file mode 100755 index 00000000..d38d48ba --- /dev/null +++ b/libs/subliminal/languages.py @@ -0,0 +1,535 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['convert_language', 'list_languages', 'LANGUAGES'] + + +def convert_language(language, to_iso, from_iso=None): + # if no from_iso is given, try to guess it + if from_iso == None: + if language.startswith(language[:1].upper()): + from_iso = 0 + elif len(language) == 2: + from_iso = 1 + elif len(language) == 3: + from_iso = 2 + else: + raise ValueError('Invalid input language format') + if isinstance(language, unicode): + language = language.encode('utf-8') + converted_language = None + for language_tuple in LANGUAGES: + if language_tuple[from_iso] == language and language_tuple[to_iso]: + converted_language = language_tuple[to_iso] + break + return converted_language + + +def list_languages(iso): + return [l[iso] for l in LANGUAGES if l[iso]] + +# ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt +# + ('Brazilian', 'po', 'pob') +LANGUAGES = [('Afar', 'aa', 'aar'), + ('Abkhazian', 'ab', 'abk'), + ('Achinese', '', 'ace'), + ('Acoli', '', 'ach'), + ('Adangme', '', 'ada'), + ('Adyghe; Adygei', '', 'ady'), + ('Afro-Asiatic languages', '', 'afa'), + ('Afrihili', '', 'afh'), + ('Afrikaans', 'af', 'afr'), + ('Ainu', '', 'ain'), + ('Akan', 'ak', 'aka'), + ('Akkadian', '', 'akk'), + ('Albanian', 'sq', 'alb'), + ('Aleut', '', 'ale'), + ('Algonquian languages', '', 'alg'), + ('Southern Altai', '', 'alt'), + ('Amharic', 'am', 'amh'), + ('English, Old (ca.450-1100)', '', 'ang'), + ('Angika', '', 'anp'), + ('Apache languages', '', 'apa'), + ('Arabic', 'ar', 'ara'), + ('Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)', '', 'arc'), + ('Aragonese', 'an', 'arg'), + ('Armenian', 'hy', 'arm'), + ('Mapudungun; Mapuche', '', 'arn'), + ('Arapaho', '', 'arp'), + ('Artificial languages', '', 'art'), + ('Arawak', '', 'arw'), + ('Assamese', 'as', 'asm'), + ('Asturian; Bable; Leonese; Asturleonese', '', 'ast'), + ('Athapascan languages', '', 'ath'), + ('Australian languages', '', 'aus'), + ('Avaric', 'av', 'ava'), + ('Avestan', 'ae', 'ave'), + ('Awadhi', '', 'awa'), + ('Aymara', 'ay', 'aym'), + ('Azerbaijani', 'az', 'aze'), + ('Banda languages', '', 'bad'), + ('Bamileke languages', '', 'bai'), + ('Bashkir', 'ba', 'bak'), + ('Baluchi', '', 'bal'), + ('Bambara', 'bm', 'bam'), + ('Balinese', '', 'ban'), + ('Basque', 'eu', 'baq'), + ('Basa', '', 'bas'), + ('Baltic languages', '', 'bat'), + ('Beja; Bedawiyet', '', 'bej'), + ('Belarusian', 'be', 'bel'), + ('Bemba', '', 'bem'), + ('Bengali', 'bn', 'ben'), + ('Berber languages', '', 'ber'), + ('Bhojpuri', '', 'bho'), + ('Bihari languages', 'bh', 'bih'), + ('Bikol', '', 'bik'), + ('Bini; Edo', '', 'bin'), + ('Bislama', 'bi', 'bis'), + ('Siksika', '', 'bla'), + ('Bantu (Other)', '', 'bnt'), + ('Bosnian', 'bs', 'bos'), + ('Braj', '', 'bra'), + ('Breton', 'br', 'bre'), + ('Batak languages', '', 'btk'), + ('Buriat', '', 'bua'), + ('Buginese', '', 'bug'), + ('Bulgarian', 'bg', 'bul'), + ('Burmese', 'my', 'bur'), + ('Blin; Bilin', '', 'byn'), + ('Caddo', '', 'cad'), + ('Central American Indian languages', '', 'cai'), + ('Galibi Carib', '', 'car'), + ('Catalan; Valencian', 'ca', 'cat'), + ('Caucasian languages', '', 'cau'), + ('Cebuano', '', 'ceb'), + ('Celtic languages', '', 'cel'), + ('Chamorro', 'ch', 'cha'), + ('Chibcha', '', 'chb'), + ('Chechen', 'ce', 'che'), + ('Chagatai', '', 'chg'), + ('Chinese', 'zh', 'chi'), + ('Chuukese', '', 'chk'), + ('Mari', '', 'chm'), + ('Chinook jargon', '', 'chn'), + ('Choctaw', '', 'cho'), + ('Chipewyan; Dene Suline', '', 'chp'), + ('Cherokee', '', 'chr'), + ('Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic', 'cu', 'chu'), + ('Chuvash', 'cv', 'chv'), + ('Cheyenne', '', 'chy'), + ('Chamic languages', '', 'cmc'), + ('Coptic', '', 'cop'), + ('Cornish', 'kw', 'cor'), + ('Corsican', 'co', 'cos'), + ('Creoles and pidgins, English based', '', 'cpe'), + ('Creoles and pidgins, French-based ', '', 'cpf'), + ('Creoles and pidgins, Portuguese-based ', '', 'cpp'), + ('Cree', 'cr', 'cre'), + ('Crimean Tatar; Crimean Turkish', '', 'crh'), + ('Creoles and pidgins ', '', 'crp'), + ('Kashubian', '', 'csb'), + ('Cushitic languages', '', 'cus'), + ('Czech', 'cs', 'cze'), + ('Dakota', '', 'dak'), + ('Danish', 'da', 'dan'), + ('Dargwa', '', 'dar'), + ('Land Dayak languages', '', 'day'), + ('Delaware', '', 'del'), + ('Slave (Athapascan)', '', 'den'), + ('Dogrib', '', 'dgr'), + ('Dinka', '', 'din'), + ('Divehi; Dhivehi; Maldivian', 'dv', 'div'), + ('Dogri', '', 'doi'), + ('Dravidian languages', '', 'dra'), + ('Lower Sorbian', '', 'dsb'), + ('Duala', '', 'dua'), + ('Dutch, Middle (ca.1050-1350)', '', 'dum'), + ('Dutch; Flemish', 'nl', 'dut'), + ('Dyula', '', 'dyu'), + ('Dzongkha', 'dz', 'dzo'), + ('Efik', '', 'efi'), + ('Egyptian (Ancient)', '', 'egy'), + ('Ekajuk', '', 'eka'), + ('Elamite', '', 'elx'), + ('English', 'en', 'eng'), + ('English, Middle (1100-1500)', '', 'enm'), + ('Esperanto', 'eo', 'epo'), + ('Estonian', 'et', 'est'), + ('Ewe', 'ee', 'ewe'), + ('Ewondo', '', 'ewo'), + ('Fang', '', 'fan'), + ('Faroese', 'fo', 'fao'), + ('Fanti', '', 'fat'), + ('Fijian', 'fj', 'fij'), + ('Filipino; Pilipino', '', 'fil'), + ('Finnish', 'fi', 'fin'), + ('Finno-Ugrian languages', '', 'fiu'), + ('Fon', '', 'fon'), + ('French', 'fr', 'fre'), + ('French, Middle (ca.1400-1600)', '', 'frm'), + ('French, Old (842-ca.1400)', '', 'fro'), + ('Northern Frisian', '', 'frr'), + ('Eastern Frisian', '', 'frs'), + ('Western Frisian', 'fy', 'fry'), + ('Fulah', 'ff', 'ful'), + ('Friulian', '', 'fur'), + ('Ga', '', 'gaa'), + ('Gayo', '', 'gay'), + ('Gbaya', '', 'gba'), + ('Germanic languages', '', 'gem'), + ('Georgian', 'ka', 'geo'), + ('German', 'de', 'ger'), + ('Geez', '', 'gez'), + ('Gilbertese', '', 'gil'), + ('Gaelic; Scottish Gaelic', 'gd', 'gla'), + ('Irish', 'ga', 'gle'), + ('Galician', 'gl', 'glg'), + ('Manx', 'gv', 'glv'), + ('German, Middle High (ca.1050-1500)', '', 'gmh'), + ('German, Old High (ca.750-1050)', '', 'goh'), + ('Gondi', '', 'gon'), + ('Gorontalo', '', 'gor'), + ('Gothic', '', 'got'), + ('Grebo', '', 'grb'), + ('Greek, Ancient (to 1453)', '', 'grc'), + ('Greek, Modern (1453-)', 'el', 'gre'), + ('Guarani', 'gn', 'grn'), + ('Swiss German; Alemannic; Alsatian', '', 'gsw'), + ('Gujarati', 'gu', 'guj'), + ('Gwich\'in', '', 'gwi'), + ('Haida', '', 'hai'), + ('Haitian; Haitian Creole', 'ht', 'hat'), + ('Hausa', 'ha', 'hau'), + ('Hawaiian', '', 'haw'), + ('Hebrew', 'he', 'heb'), + ('Herero', 'hz', 'her'), + ('Hiligaynon', '', 'hil'), + ('Himachali languages; Western Pahari languages', '', 'him'), + ('Hindi', 'hi', 'hin'), + ('Hittite', '', 'hit'), + ('Hmong; Mong', '', 'hmn'), + ('Hiri Motu', 'ho', 'hmo'), + ('Croatian', 'hr', 'hrv'), + ('Upper Sorbian', '', 'hsb'), + ('Hungarian', 'hu', 'hun'), + ('Hupa', '', 'hup'), + ('Iban', '', 'iba'), + ('Igbo', 'ig', 'ibo'), + ('Icelandic', 'is', 'ice'), + ('Ido', 'io', 'ido'), + ('Sichuan Yi; Nuosu', 'ii', 'iii'), + ('Ijo languages', '', 'ijo'), + ('Inuktitut', 'iu', 'iku'), + ('Interlingue; Occidental', 'ie', 'ile'), + ('Iloko', '', 'ilo'), + ('Interlingua (International Auxiliary Language Association)', 'ia', 'ina'), + ('Indic languages', '', 'inc'), + ('Indonesian', 'id', 'ind'), + ('Indo-European languages', '', 'ine'), + ('Ingush', '', 'inh'), + ('Inupiaq', 'ik', 'ipk'), + ('Iranian languages', '', 'ira'), + ('Iroquoian languages', '', 'iro'), + ('Italian', 'it', 'ita'), + ('Javanese', 'jv', 'jav'), + ('Lojban', '', 'jbo'), + ('Japanese', 'ja', 'jpn'), + ('Judeo-Persian', '', 'jpr'), + ('Judeo-Arabic', '', 'jrb'), + ('Kara-Kalpak', '', 'kaa'), + ('Kabyle', '', 'kab'), + ('Kachin; Jingpho', '', 'kac'), + ('Kalaallisut; Greenlandic', 'kl', 'kal'), + ('Kamba', '', 'kam'), + ('Kannada', 'kn', 'kan'), + ('Karen languages', '', 'kar'), + ('Kashmiri', 'ks', 'kas'), + ('Kanuri', 'kr', 'kau'), + ('Kawi', '', 'kaw'), + ('Kazakh', 'kk', 'kaz'), + ('Kabardian', '', 'kbd'), + ('Khasi', '', 'kha'), + ('Khoisan languages', '', 'khi'), + ('Central Khmer', 'km', 'khm'), + ('Khotanese; Sakan', '', 'kho'), + ('Kikuyu; Gikuyu', 'ki', 'kik'), + ('Kinyarwanda', 'rw', 'kin'), + ('Kirghiz; Kyrgyz', 'ky', 'kir'), + ('Kimbundu', '', 'kmb'), + ('Konkani', '', 'kok'), + ('Komi', 'kv', 'kom'), + ('Kongo', 'kg', 'kon'), + ('Korean', 'ko', 'kor'), + ('Kosraean', '', 'kos'), + ('Kpelle', '', 'kpe'), + ('Karachay-Balkar', '', 'krc'), + ('Karelian', '', 'krl'), + ('Kru languages', '', 'kro'), + ('Kurukh', '', 'kru'), + ('Kuanyama; Kwanyama', 'kj', 'kua'), + ('Kumyk', '', 'kum'), + ('Kurdish', 'ku', 'kur'), + ('Kutenai', '', 'kut'), + ('Ladino', '', 'lad'), + ('Lahnda', '', 'lah'), + ('Lamba', '', 'lam'), + ('Lao', 'lo', 'lao'), + ('Latin', 'la', 'lat'), + ('Latvian', 'lv', 'lav'), + ('Lezghian', '', 'lez'), + ('Limburgan; Limburger; Limburgish', 'li', 'lim'), + ('Lingala', 'ln', 'lin'), + ('Lithuanian', 'lt', 'lit'), + ('Mongo', '', 'lol'), + ('Lozi', '', 'loz'), + ('Luxembourgish; Letzeburgesch', 'lb', 'ltz'), + ('Luba-Lulua', '', 'lua'), + ('Luba-Katanga', 'lu', 'lub'), + ('Ganda', 'lg', 'lug'), + ('Luiseno', '', 'lui'), + ('Lunda', '', 'lun'), + ('Luo (Kenya and Tanzania)', '', 'luo'), + ('Lushai', '', 'lus'), + ('Macedonian', 'mk', 'mac'), + ('Madurese', '', 'mad'), + ('Magahi', '', 'mag'), + ('Marshallese', 'mh', 'mah'), + ('Maithili', '', 'mai'), + ('Makasar', '', 'mak'), + ('Malayalam', 'ml', 'mal'), + ('Mandingo', '', 'man'), + ('Maori', 'mi', 'mao'), + ('Austronesian languages', '', 'map'), + ('Marathi', 'mr', 'mar'), + ('Masai', '', 'mas'), + ('Malay', 'ms', 'may'), + ('Moksha', '', 'mdf'), + ('Mandar', '', 'mdr'), + ('Mende', '', 'men'), + ('Irish, Middle (900-1200)', '', 'mga'), + ('Mi\'kmaq; Micmac', '', 'mic'), + ('Minangkabau', '', 'min'), + ('Uncoded languages', '', 'mis'), + ('Mon-Khmer languages', '', 'mkh'), + ('Malagasy', 'mg', 'mlg'), + ('Maltese', 'mt', 'mlt'), + ('Manchu', '', 'mnc'), + ('Manipuri', '', 'mni'), + ('Manobo languages', '', 'mno'), + ('Mohawk', '', 'moh'), + ('Mongolian', 'mn', 'mon'), + ('Mossi', '', 'mos'), + ('Multiple languages', '', 'mul'), + ('Munda languages', '', 'mun'), + ('Creek', '', 'mus'), + ('Mirandese', '', 'mwl'), + ('Marwari', '', 'mwr'), + ('Mayan languages', '', 'myn'), + ('Erzya', '', 'myv'), + ('Nahuatl languages', '', 'nah'), + ('North American Indian languages', '', 'nai'), + ('Neapolitan', '', 'nap'), + ('Nauru', 'na', 'nau'), + ('Navajo; Navaho', 'nv', 'nav'), + ('Ndebele, South; South Ndebele', 'nr', 'nbl'), + ('Ndebele, North; North Ndebele', 'nd', 'nde'), + ('Ndonga', 'ng', 'ndo'), + ('Low German; Low Saxon; German, Low; Saxon, Low', '', 'nds'), + ('Nepali', 'ne', 'nep'), + ('Nepal Bhasa; Newari', '', 'new'), + ('Nias', '', 'nia'), + ('Niger-Kordofanian languages', '', 'nic'), + ('Niuean', '', 'niu'), + ('Norwegian Nynorsk; Nynorsk, Norwegian', 'nn', 'nno'), + ('Bokmål, Norwegian; Norwegian Bokmål', 'nb', 'nob'), + ('Nogai', '', 'nog'), + ('Norse, Old', '', 'non'), + ('Norwegian', 'no', 'nor'), + ('N\'Ko', '', 'nqo'), + ('Pedi; Sepedi; Northern Sotho', '', 'nso'), + ('Nubian languages', '', 'nub'), + ('Classical Newari; Old Newari; Classical Nepal Bhasa', '', 'nwc'), + ('Chichewa; Chewa; Nyanja', 'ny', 'nya'), + ('Nyamwezi', '', 'nym'), + ('Nyankole', '', 'nyn'), + ('Nyoro', '', 'nyo'), + ('Nzima', '', 'nzi'), + ('Occitan (post 1500); Provençal', 'oc', 'oci'), + ('Ojibwa', 'oj', 'oji'), + ('Oriya', 'or', 'ori'), + ('Oromo', 'om', 'orm'), + ('Osage', '', 'osa'), + ('Ossetian; Ossetic', 'os', 'oss'), + ('Turkish, Ottoman (1500-1928)', '', 'ota'), + ('Otomian languages', '', 'oto'), + ('Papuan languages', '', 'paa'), + ('Pangasinan', '', 'pag'), + ('Pahlavi', '', 'pal'), + ('Pampanga; Kapampangan', '', 'pam'), + ('Panjabi; Punjabi', 'pa', 'pan'), + ('Papiamento', '', 'pap'), + ('Palauan', '', 'pau'), + ('Persian, Old (ca.600-400 B.C.)', '', 'peo'), + ('Persian', 'fa', 'per'), + ('Philippine languages', '', 'phi'), + ('Phoenician', '', 'phn'), + ('Pali', 'pi', 'pli'), + ('Polish', 'pl', 'pol'), + ('Pohnpeian', '', 'pon'), + ('Portuguese', 'pt', 'por'), + ('Prakrit languages', '', 'pra'), + ('Provençal, Old (to 1500)', '', 'pro'), + ('Pushto; Pashto', 'ps', 'pus'), + ('Reserved for local use', '', 'qaa-qtz'), + ('Quechua', 'qu', 'que'), + ('Rajasthani', '', 'raj'), + ('Rapanui', '', 'rap'), + ('Rarotongan; Cook Islands Maori', '', 'rar'), + ('Romance languages', '', 'roa'), + ('Romansh', 'rm', 'roh'), + ('Romany', '', 'rom'), + ('Romanian; Moldavian; Moldovan', 'ro', 'rum'), + ('Rundi', 'rn', 'run'), + ('Aromanian; Arumanian; Macedo-Romanian', '', 'rup'), + ('Russian', 'ru', 'rus'), + ('Sandawe', '', 'sad'), + ('Sango', 'sg', 'sag'), + ('Yakut', '', 'sah'), + ('South American Indian (Other)', '', 'sai'), + ('Salishan languages', '', 'sal'), + ('Samaritan Aramaic', '', 'sam'), + ('Sanskrit', 'sa', 'san'), + ('Sasak', '', 'sas'), + ('Santali', '', 'sat'), + ('Sicilian', '', 'scn'), + ('Scots', '', 'sco'), + ('Selkup', '', 'sel'), + ('Semitic languages', '', 'sem'), + ('Irish, Old (to 900)', '', 'sga'), + ('Sign Languages', '', 'sgn'), + ('Shan', '', 'shn'), + ('Sidamo', '', 'sid'), + ('Sinhala; Sinhalese', 'si', 'sin'), + ('Siouan languages', '', 'sio'), + ('Sino-Tibetan languages', '', 'sit'), + ('Slavic languages', '', 'sla'), + ('Slovak', 'sk', 'slo'), + ('Slovenian', 'sl', 'slv'), + ('Southern Sami', '', 'sma'), + ('Northern Sami', 'se', 'sme'), + ('Sami languages', '', 'smi'), + ('Lule Sami', '', 'smj'), + ('Inari Sami', '', 'smn'), + ('Samoan', 'sm', 'smo'), + ('Skolt Sami', '', 'sms'), + ('Shona', 'sn', 'sna'), + ('Sindhi', 'sd', 'snd'), + ('Soninke', '', 'snk'), + ('Sogdian', '', 'sog'), + ('Somali', 'so', 'som'), + ('Songhai languages', '', 'son'), + ('Sotho, Southern', 'st', 'sot'), + ('Spanish; Castilian', 'es', 'spa'), + ('Sardinian', 'sc', 'srd'), + ('Sranan Tongo', '', 'srn'), + ('Serbian', 'sr', 'srp'), + ('Serer', '', 'srr'), + ('Nilo-Saharan languages', '', 'ssa'), + ('Swati', 'ss', 'ssw'), + ('Sukuma', '', 'suk'), + ('Sundanese', 'su', 'sun'), + ('Susu', '', 'sus'), + ('Sumerian', '', 'sux'), + ('Swahili', 'sw', 'swa'), + ('Swedish', 'sv', 'swe'), + ('Classical Syriac', '', 'syc'), + ('Syriac', '', 'syr'), + ('Tahitian', 'ty', 'tah'), + ('Tai languages', '', 'tai'), + ('Tamil', 'ta', 'tam'), + ('Tatar', 'tt', 'tat'), + ('Telugu', 'te', 'tel'), + ('Timne', '', 'tem'), + ('Tereno', '', 'ter'), + ('Tetum', '', 'tet'), + ('Tajik', 'tg', 'tgk'), + ('Tagalog', 'tl', 'tgl'), + ('Thai', 'th', 'tha'), + ('Tibetan', 'bo', 'tib'), + ('Tigre', '', 'tig'), + ('Tigrinya', 'ti', 'tir'), + ('Tiv', '', 'tiv'), + ('Tokelau', '', 'tkl'), + ('Klingon; tlhIngan-Hol', '', 'tlh'), + ('Tlingit', '', 'tli'), + ('Tamashek', '', 'tmh'), + ('Tonga (Nyasa)', '', 'tog'), + ('Tonga (Tonga Islands)', 'to', 'ton'), + ('Tok Pisin', '', 'tpi'), + ('Tsimshian', '', 'tsi'), + ('Tswana', 'tn', 'tsn'), + ('Tsonga', 'ts', 'tso'), + ('Turkmen', 'tk', 'tuk'), + ('Tumbuka', '', 'tum'), + ('Tupi languages', '', 'tup'), + ('Turkish', 'tr', 'tur'), + ('Altaic languages', '', 'tut'), + ('Tuvalu', '', 'tvl'), + ('Twi', 'tw', 'twi'), + ('Tuvinian', '', 'tyv'), + ('Udmurt', '', 'udm'), + ('Ugaritic', '', 'uga'), + ('Uighur; Uyghur', 'ug', 'uig'), + ('Ukrainian', 'uk', 'ukr'), + ('Umbundu', '', 'umb'), + ('Undetermined', '', 'und'), + ('Urdu', 'ur', 'urd'), + ('Uzbek', 'uz', 'uzb'), + ('Vai', '', 'vai'), + ('Venda', 've', 'ven'), + ('Vietnamese', 'vi', 'vie'), + ('Volapük', 'vo', 'vol'), + ('Votic', '', 'vot'), + ('Wakashan languages', '', 'wak'), + ('Walamo', '', 'wal'), + ('Waray', '', 'war'), + ('Washo', '', 'was'), + ('Welsh', 'cy', 'wel'), + ('Sorbian languages', '', 'wen'), + ('Walloon', 'wa', 'wln'), + ('Wolof', 'wo', 'wol'), + ('Kalmyk; Oirat', '', 'xal'), + ('Xhosa', 'xh', 'xho'), + ('Yao', '', 'yao'), + ('Yapese', '', 'yap'), + ('Yiddish', 'yi', 'yid'), + ('Yoruba', 'yo', 'yor'), + ('Yupik languages', '', 'ypk'), + ('Zapotec', '', 'zap'), + ('Blissymbols; Blissymbolics; Bliss', '', 'zbl'), + ('Zenaga', '', 'zen'), + ('Zhuang; Chuang', 'za', 'zha'), + ('Zande languages', '', 'znd'), + ('Zulu', 'zu', 'zul'), + ('Zuni', '', 'zun'), + ('No linguistic content; Not applicable', '', 'zxx'), + ('Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki', '', 'zza'), + ('Brazilian', 'po', 'pob')] diff --git a/libs/subliminal/plugins.py b/libs/subliminal/plugins.py new file mode 100755 index 00000000..c86c4013 --- /dev/null +++ b/libs/subliminal/plugins.py @@ -0,0 +1,890 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['PluginBase', 'OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos'] + + +from exceptions import DownloadFailedError, MissingLanguageError, PluginError +from subliminal.utils import get_keywords, PluginConfig, split_keyword +from subliminal.videos import Episode, Movie, UnknownVideo +from subtitles import ResultSubtitle, get_subtitle_path +import BeautifulSoup +import abc +import gzip +import logging +import os +import re +import requests +import suds.client +import threading +import unicodedata +import urllib +import xmlrpclib +try: + import cPickle as pickle +except ImportError: + import pickle + + +#TODO: use ISO-639-2 in plugins instead of ISO-639-1 +class PluginBase(object): + __metaclass__ = abc.ABCMeta + site_url = '' + site_name = '' + server_url = '' + user_agent = 'Subliminal v0.5' + api_based = False + timeout = 5 + lock = threading.Lock() + languages = {} + reverted_languages = False + videos = [] + require_video = False + shared_support = False + + @abc.abstractmethod + def __init__(self, config=None): + self.config = config or PluginConfig() + self.logger = logging.getLogger('subliminal.%s' % self.__class__.__name__) + + @abc.abstractmethod + def init(self): + """Initiate connection""" + self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent}) + + @abc.abstractmethod + def terminate(self): + """Terminate connection""" + + @abc.abstractmethod + def query(self, *args): + """Make the actual query""" + + @abc.abstractmethod + def list(self, video, languages): + """List subtitles""" + + @abc.abstractmethod + def download(self, subtitle): + """Download a subtitle""" + + @classmethod + def availableLanguages(cls): + if not cls.reverted_languages: + return set(cls.languages.keys()) + if cls.reverted_languages: + return set(cls.languages.values()) + + @classmethod + def isValidVideo(cls, video): + if cls.require_video and not video.exists: + return False + if not isinstance(video, tuple(cls.videos)): + return False + return True + + @classmethod + def isValidLanguage(cls, language): + if language in cls.availableLanguages(): + return True + return False + + @classmethod + def getRevertLanguage(cls, language): + """ISO-639-1 language code from plugin language code""" + if not cls.reverted_languages and language in cls.languages.values(): + return [k for k, v in cls.languages.iteritems() if v == language][0] + if cls.reverted_languages and language in cls.languages.keys(): + return cls.languages[language] + raise MissingLanguageError(language) + + @classmethod + def getLanguage(cls, language): + """Plugin language code from ISO-639-1 language code""" + if not cls.reverted_languages and language in cls.languages.keys(): + return cls.languages[language] + if cls.reverted_languages and language in cls.languages.values(): + return [k for k, v in cls.languages.iteritems() if v == language][0] + raise MissingLanguageError(language) + + def adjustPermissions(self, filepath): + if self.config.filemode != None: + os.chmod(filepath, self.config.filemode) + + def downloadFile(self, url, filepath): + """Download a subtitle file""" + self.logger.info(u'Downloading %s' % url) + try: + r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent}) + with open(filepath, 'wb') as f: + f.write(r.content) + except Exception as e: + self.logger.error(u'Download %s failed: %s' % (url, e)) + if os.path.exists(filepath): + os.remove(filepath) + raise DownloadFailedError(str(e)) + self.logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath))) + + +class OpenSubtitles(PluginBase): + site_url = 'http://www.opensubtitles.org' + site_name = 'OpenSubtitles' + server_url = 'http://api.opensubtitles.org/xml-rpc' + user_agent = 'Subliminal v0.5' + api_based = True + languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara', + 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze', + 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis', + 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che', + 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze', + 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est', + 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful', + 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell', + 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin', + 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii', + 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav', + 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm', + 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua', + 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit', + 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao', + 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau', + 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob', + 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss', + 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que', + 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin', + 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som', + 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe', + 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha', + 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur', + 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie', + 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor', + 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'} + reverted_languages = False + videos = [Episode, Movie] + require_video = False + confidence_order = ['moviehash', 'imdbid', 'fulltext'] + + def __init__(self, config=None): + super(OpenSubtitles, self).__init__(config) + self.server = xmlrpclib.ServerProxy(self.server_url) + self.token = None + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + self.terminate() + + def init(self): + self.logger.debug(u'Initializing') + super(OpenSubtitles, self).init() + result = self.server.LogIn('', '', 'eng', self.user_agent) + if result['status'] != '200 OK': + raise PluginError('Login failed') + self.token = result['token'] + + def terminate(self): + self.logger.debug(u'Terminating') + if self.token: + self.server.LogOut(self.token) + + def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None): + searches = [] + if moviehash and size: + searches.append({'moviehash': moviehash, 'moviebytesize': size}) + if imdbid: + searches.append({'imdbid': imdbid}) + if query: + searches.append({'query': query}) + if not searches: + raise PluginError('One or more parameter missing') + for search in searches: + search['sublanguageid'] = ','.join([self.getLanguage(l) for l in languages]) + self.logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token)) + results = self.server.SearchSubtitles(self.token, searches) + if not results['data']: + self.logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token)) + return [] + subtitles = [] + for result in results['data']: + language = self.getRevertLanguage(result['SubLanguageID']) + path = get_subtitle_path(filepath, language, self.config.multi) + confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order)) + subtitle = ResultSubtitle(path, language, self.__class__.__name__, result['SubDownloadLink'], result['SubFileName'], confidence) + subtitles.append(subtitle) + return subtitles + + def list(self, video, languages): + languages = languages & self.availableLanguages() + if not languages: + self.logger.debug(u'No language available') + return [] + if not self.isValidVideo(video): + self.logger.debug(u'Not a valid video') + return [] + results = [] + if video.exists: + results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size)) + elif video.imdbid: + results = self.query(video.path or video.release, languages, imdbid=video.imdbid) + elif isinstance(video, Episode): + results = self.query(video.path or video.release, languages, query=video.series) + elif isinstance(video, Movie): + results = self.query(video.path or video.release, languages, query=video.title) + return results + + def download(self, subtitle): + #TODO: Use OpenSubtitles DownloadSubtitles method + try: + self.downloadFile(subtitle.link, subtitle.path + '.gz') + with open(subtitle.path, 'wb') as dump: + gz = gzip.open(subtitle.path + '.gz') + dump.write(gz.read()) + gz.close() + self.adjustPermissions(subtitle.path) + except Exception as e: + if os.path.exists(subtitle.path): + os.remove(subtitle.path) + raise DownloadFailedError(str(e)) + finally: + if os.path.exists(subtitle.path + '.gz'): + os.remove(subtitle.path + '.gz') + return subtitle + + +class BierDopje(PluginBase): + site_url = 'http://bierdopje.com' + site_name = 'BierDopje' + server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/' + api_based = True + languages = {'en': 'en', 'nl': 'nl'} + reverted_languages = False + videos = [Episode] + require_video = False + + def __init__(self, config=None): + super(BierDopje, self).__init__(config) + self.showids = {} + if self.config and self.config.cache_dir: + self.initCache() + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + self.terminate() + + def init(self): + self.logger.debug(u'Initializing') + super(BierDopje, self).init() + + def terminate(self): + self.logger.debug(u'Terminating') + + def initCache(self): + self.logger.debug(u'Initializing cache...') + if not self.config or not self.config.cache_dir: + raise PluginError('Cache directory is required') + self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache') + if not os.path.exists(self.showids_cache): + self.saveToCache() + + def saveToCache(self): + self.logger.debug(u'Saving showids to cache...') + with self.lock: + with open(self.showids_cache, 'w') as f: + pickle.dump(self.showids, f) + + def loadFromCache(self): + self.logger.debug(u'Loading showids from cache...') + with self.lock: + with open(self.showids_cache, 'r') as f: + self.showids = pickle.load(f) + + def query(self, season, episode, languages, filepath, tvdbid=None, series=None): + self.initCache() + self.loadFromCache() + if series: + if series.lower() in self.showids: # from cache + request_id = self.showids[series.lower()] + self.logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series)) + else: # query to get showid + self.logger.debug(u'Getting showid from show name %s...' % series) + r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower()))) + if r.status_code != 200: + self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulStoneSoup(r.content) + if soup.status.contents[0] == 'false': + self.logger.debug(u'Could not find show %s' % series) + return [] + request_id = int(soup.showid.contents[0]) + self.showids[series.lower()] = request_id + self.saveToCache() + request_source = 'showid' + request_is_tvdbid = 'false' + elif tvdbid: + request_id = tvdbid + request_source = 'tvdbid' + request_is_tvdbid = 'true' + else: + raise PluginError('One or more parameter missing') + subtitles = [] + for language in languages: + self.logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language)) + r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid)) + if r.status_code != 200: + self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulStoneSoup(r.content) + if soup.status.contents[0] == 'false': + self.logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language)) + continue + path = get_subtitle_path(filepath, language, self.config.multi) + for result in soup.results('result'): + subtitle = ResultSubtitle(path, language, self.__class__.__name__, result.downloadlink.contents[0], result.filename.contents[0]) + subtitles.append(subtitle) + return subtitles + + def list(self, video, languages): + languages = languages & self.availableLanguages() + if not languages: + self.logger.debug(u'No language available') + return [] + if not self.isValidVideo(video): + self.logger.debug(u'Not a valid video') + return [] + results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series) + return results + + def download(self, subtitle): + self.downloadFile(subtitle.link, subtitle.path) + return subtitle + + +class TheSubDB(PluginBase): + site_url = 'http://thesubdb.com' + site_name = 'SubDB' + server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead + api_based = True + user_agent = 'SubDB/1.0 (Subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API + languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi', + 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no', + 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr', + 'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages + reverted_languages = False + videos = [Movie, Episode, UnknownVideo] + require_video = True + + def __init__(self, config=None): + super(TheSubDB, self).__init__(config) + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + self.terminate() + + def init(self): + self.logger.debug(u'Initializing') + super(TheSubDB, self).init() + + def terminate(self): + self.logger.debug(u'Terminating') + + def list(self, video, languages): + languages = languages & self.availableLanguages() + if not languages: + self.logger.debug(u'No language available') + return [] + if not self.isValidVideo(video): + self.logger.debug(u'Not a valid video') + return [] + results = self.query(video.path, video.hashes['TheSubDB'], languages) + return results + + def query(self, filepath, moviehash, languages): + r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash}) + if r.status_code == 404: + self.logger.debug(u'Could not find subtitles for hash %s' % moviehash) + return [] + if r.status_code != 200: + self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + available_languages = set([self.getRevertLanguage(l) for l in r.content.split(',')]) + filtered_languages = languages & available_languages + if not filtered_languages: + self.logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages)) + return [] + subtitles = [] + for language in filtered_languages: + path = get_subtitle_path(filepath, language, self.config.multi) + subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.getLanguage(language))) + subtitles.append(subtitle) + return subtitles + + def download(self, subtitle): + self.downloadFile(subtitle.link, subtitle.path) + return subtitle + + +class SubsWiki(PluginBase): + site_url = 'http://www.subswiki.com' + site_name = 'SubsWiki' + server_url = 'http://www.subswiki.com' + api_based = False + languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po', + u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es', + u'Italian': 'it', u'Català': 'ca'} + reverted_languages = True + videos = [Episode, Movie] + require_video = False + release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs') + + def __init__(self, config=None): + super(SubsWiki, self).__init__(config) + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + self.terminate() + + def init(self): + self.logger.debug(u'Initializing') + super(SubsWiki, self).init() + + def terminate(self): + self.logger.debug(u'Terminating') + + def list(self, video, languages): + languages = languages & self.availableLanguages() + if not languages: + self.logger.debug(u'No language available') + return [] + if not self.isValidVideo(video): + self.logger.debug(u'Not a valid video') + return [] + results = [] + if isinstance(video, Episode): + results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode) + elif isinstance(video, Movie) and video.year: + results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year) + return results + + def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None): + if series and season and episode: + request_series = series.lower().replace(' ', '_') + if isinstance(request_series, unicode): + request_series = request_series.encode('utf-8') + self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode)) + if r.status_code == 404: + self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + return [] + elif movie and year: + request_movie = movie.title().replace(' ', '_') + if isinstance(request_movie, unicode): + request_movie = request_movie.encode('utf-8') + self.logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages)) + r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year)) + if r.status_code == 404: + self.logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages)) + return [] + else: + raise PluginError('One or more parameter missing') + if r.status_code != 200: + self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulSoup(r.content) + subtitles = [] + for sub in soup('td', {'class': 'NewsTitle'}): + sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower()) + if not keywords & sub_keywords: + self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) + continue + for html_language in sub.parent.parent.findAll('td', {'class': 'language'}): + language = self.getRevertLanguage(html_language.string.strip()) + if not language in languages: + self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) + continue + html_status = html_language.findNextSibling('td') + status = html_status.find('strong').string.strip() + if status != 'Completed': + self.logger.debug(u'Wrong subtitle status %s' % status) + continue + path = get_subtitle_path(filepath, language, self.config.multi) + subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href'])) + subtitles.append(subtitle) + return subtitles + + def download(self, subtitle): + self.downloadFile(subtitle.link, subtitle.path) + return subtitle + + +class Subtitulos(PluginBase): + site_url = 'http://www.subtitulos.es/' + site_name = 'Subtitulos' + server_url = 'http://www.subtitulos.es' + api_based = False + languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po', + u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es', + u'Italian': 'it', u'Català': 'ca'} + reverted_languages = True + videos = [Episode] + require_video = False + release_pattern = re.compile('Versión (.+) ([0-9]+).([0-9])+ megabytes') + + def __init__(self, config=None): + super(Subtitulos, self).__init__(config) + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + self.terminate() + + def init(self): + self.logger.debug(u'Initializing') + super(Subtitulos, self).init() + + def terminate(self): + self.logger.debug(u'Terminating') + + def list(self, video, languages): + languages = languages & self.availableLanguages() + if not languages: + self.logger.debug(u'No language available') + return [] + if not self.isValidVideo(video): + self.logger.debug(u'Not a valid video') + return [] + results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode) + return results + + def query(self, filepath, languages, keywords, series, season, episode): + request_series = series.lower().replace(' ', '_') + if isinstance(request_series, unicode): + request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore') + self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode)) + if r.status_code == 404: + self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + return [] + if r.status_code != 200: + self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulSoup(r.content) + subtitles = [] + for sub in soup('div', {'id': 'version'}): + sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower()) + if not keywords & sub_keywords: + self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) + continue + for html_language in sub.findAllNext('ul', {'class': 'sslist'}): + language = self.getRevertLanguage(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip()) + if not language in languages: + self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) + continue + html_status = html_language.findNext('li', {'class': 'li-estado green'}) + status = html_status.contents[0].string.strip() + if status != 'Completado': + self.logger.debug(u'Wrong subtitle status %s' % status) + continue + path = get_subtitle_path(filepath, language, self.config.multi) + subtitle = ResultSubtitle(path, language, self.__class__.__name__, html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords) + subtitles.append(subtitle) + return subtitles + + def download(self, subtitle): + self.downloadFile(subtitle.link, subtitle.path) + return subtitle + + +class GetSubtitle(PluginBase): + site_url = 'http://www.subtitles.com.br/' + site_name = 'GetSubtitle' + server_url = 'http://api.getsubtitle.com/server.php?wsdl' + api_based = True + languages = {'sq': 'ALB', 'ar': 'ARA', 'hy': 'ARM', 'bs': 'BOS', 'bg': 'BUL', 'ca': 'CAT', 'zh': 'CHI', 'hr': 'HRV', + 'cs': 'CZE', 'da': 'DAN', 'nl': 'NLD', 'en': 'ENG', 'eo': 'ESP', 'et': 'EST', 'fi': 'FIN', 'fr': 'FRA', + 'gl': 'GLG', 'ka': 'GEO', 'de': 'DEU', 'el': 'GRC', 'he': 'ISR', 'hi': 'HIN', 'hu': 'HUN', 'is': 'ISL', + 'id': 'IND', 'it': 'ITA', 'ja': 'JPN', 'kk': 'KAZ', 'ko': 'KOR', 'lv': 'LVA', 'lt': 'LIT', 'lb': 'LTZ', + 'mk': 'MKD', 'ms': 'MAY', 'no': 'NOR', 'oc': 'OCC', 'pl': 'POL', 'pt': 'POR', 'ro': 'RUM', 'ru': 'RUS', + 'sr': 'ZAF', 'sk': 'SLK', 'sl': 'SLV', 'es': 'SPA', 'sv': 'SWE', 'th': 'THA', 'tr': 'TUR', 'uk': 'UKR', + 'ur': 'URD', 'vi': 'VTN'} + reverted_languages = False + videos = [Movie] + require_video = False + max_results = 100 + + def __init__(self, config=None): + super(GetSubtitle, self).__init__(config) + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + pass + + def init(self): + self.logger.debug(u'Initializing') + self.server = suds.client.Client(self.server_url) + + def terminate(self): + self.logger.debug(u'Terminating') + + def query(self, *args): + #TODO + pass + + def list(self, video, languages): + languages = languages & self.availableLanguages() + if not languages: + self.logger.debug(u'No language available') + return [] + if not self.isValidVideo(video): + self.logger.debug(u'Not a valid video') + return [] + #TODO + + def download(self, subtitle): + #TODO + pass + + +''' +class Addic7ed(PluginBase.PluginBase): + site_url = 'http://www.addic7ed.com' + site_name = 'Addic7ed' + server_url = 'http://www.addic7ed.com' + api_based = False + _plugin_languages = {u'English': 'en', + u'English (US)': 'en', + u'English (UK)': 'en', + u'Italian': 'it', + u'Portuguese': 'pt', + u'Portuguese (Brazilian)': 'po', + u'Romanian': 'ro', + u'Español (Latinoamérica)': 'es', + u'Español (España)': 'es', + u'Spanish (Latin America)': 'es', + u'Español': 'es', + u'Spanish': 'es', + u'Spanish (Spain)': 'es', + u'French': 'fr', + u'Greek': 'el', + u'Arabic': 'ar', + u'German': 'de', + u'Croatian': 'hr', + u'Indonesian': 'id', + u'Hebrew': 'he', + u'Russian': 'ru', + u'Turkish': 'tr', + u'Swedish': 'se', + u'Czech': 'cs', + u'Dutch': 'nl', + u'Hungarian': 'hu', + u'Norwegian': 'no', + u'Polish': 'pl', + u'Persian': 'fa'} + + def __init__(self, config_dict=None): + super(Addic7ed, self).__init__(self._plugin_languages, config_dict, isRevert=True) + #http://www.addic7ed.com/serie/Smallville/9/11/Absolute_Justice + self.release_pattern = re.compile(' \nVersion (.+), ([0-9]+).([0-9])+ MBs') + + def list(self, filepath, languages): + if not self.checkLanguages(languages): + return [] + guess = guessit.guess_file_info(filepath, 'autodetect') + if guess['type'] != 'episode': + self.logger.debug(u'Not an episode') + return [] + # add multiple things to the release group set + release_group = set() + if 'releaseGroup' in guess: + release_group.add(guess['releaseGroup'].lower()) + else: + if 'title' in guess: + release_group.add(guess['title'].lower()) + if 'screenSize' in guess: + release_group.add(guess['screenSize'].lower()) + if 'series' not in guess or len(release_group) == 0: + self.logger.debug(u'Not enough information to proceed') + return [] + self.release_group = release_group # used to sort results + return self.query(guess['series'], guess['season'], guess['episodeNumber'], release_group, filepath, languages) + + def query(self, name, season, episode, release_group, filepath, languages=None): + searchname = name.lower().replace(' ', '_') + if isinstance(searchname, unicode): + searchname = searchname.encode('utf-8') + searchurl = '%s/serie/%s/%s/%s/%s' % (self.server_url, urllib2.quote(searchname), season, episode, urllib2.quote(searchname)) + self.logger.debug(u'Searching in %s' % searchurl) + try: + req = urllib2.Request(searchurl, headers={'User-Agent': self.user_agent}) + page = urllib2.urlopen(req, timeout=self.timeout) + except urllib2.HTTPError as inst: + self.logger.info(u'Error: %s - %s' % (searchurl, inst)) + return [] + except urllib2.URLError as inst: + self.logger.info(u'TimeOut: %s' % inst) + return [] + soup = BeautifulSoup(page.read()) + sublinks = [] + for html_sub in soup('td', {'class': 'NewsTitle', 'colspan': '3'}): + if not self.release_pattern.match(str(html_sub.contents[1])): # On not needed soup td result + continue + sub_teams = self.listTeams([self.release_pattern.match(str(html_sub.contents[1])).groups()[0].lower()], ['.', '_', ' ', '/', '-']) + if not release_group.intersection(sub_teams): # On wrong team + continue + html_language = html_sub.findNext('td', {'class': 'language'}) + sub_language = self.getRevertLanguage(html_language.contents[0].strip().replace(' ', '')) + if languages and not sub_language in languages: # On wrong language + continue + html_status = html_language.findNextSibling('td') + sub_status = html_status.find('b').string.strip() + if not sub_status == 'Completed': # On not completed subtitles + continue + sub_link = self.server_url + html_status.findNextSibling('td', {'colspan': '3'}).find('a')['href'] + self.logger.debug(u'Found a match with teams: %s' % sub_teams) + result = Subtitle(filepath, self.getSubtitlePath(filepath, sub_language), self.__class__.__name__, sub_language, sub_link, keywords=sub_teams) + sublinks.append(result) + sublinks.sort(self._cmpReleaseGroup) + return sublinks + + def download(self, subtitle): + self.downloadFile(subtitle.link, subtitle.path) + return subtitle + + +class Podnapisi(PluginBase.PluginBase): + site_url = "http://www.podnapisi.net" + site_name = "Podnapisi" + server_url = 'http://ssp.podnapisi.net:8000' + api_based = True + _plugin_languages = {"sl": "1", + "en": "2", + "no": "3", + "ko": "4", + "de": "5", + "is": "6", + "cs": "7", + "fr": "8", + "it": "9", + "bs": "10", + "ja": "11", + "ar": "12", + "ro": "13", + "es-ar": "14", + "hu": "15", + "el": "16", + "zh": "17", + "lt": "19", + "et": "20", + "lv": "21", + "he": "22", + "nl": "23", + "da": "24", + "se": "25", + "pl": "26", + "ru": "27", + "es": "28", + "sq": "29", + "tr": "30", + "fi": "31", + "pt": "32", + "bg": "33", + "mk": "35", + "sk": "37", + "hr": "38", + "zh": "40", + "hi": "42", + "th": "44", + "uk": "46", + "sr": "47", + "po": "48", + "ga": "49", + "be": "50", + "vi": "51", + "fa": "52", + "ca": "53", + "id": "54"} + + def __init__(self, config_dict=None): + super(Podnapisi, self).__init__(self._plugin_languages, config_dict) + # Podnapisi uses two reference for latin serbian and cyrillic serbian (36 and 47) + # add the 36 manually as cyrillic seems to be more used + self.revertPluginLanguages["36"] = "sr" + + def list(self, filenames, languages): + """Main method to call when you want to list subtitles""" + filepath = filenames[0] + if not os.path.isfile(filepath): + return [] + return self.query(self.hashFile(filepath), languages) + + def download(self, subtitle): + return [] + + def query(self, moviehash, languages=None): + """Makes a query on podnapisi and returns info (link, lang) about found subtitles""" + # login + self.server = xmlrpclib.ServerProxy(self.server_url) + try: + log_result = self.server.initiate(self.user_agent) + self.logger.debug(u"Result: %s" % log_result) + token = log_result["session"] + nonce = log_result["nonce"] + except Exception: + self.logger.error(u"Cannot login" % log_result) + return [] + username = 'getmesubs' + password = '99D31$$' + hash = md5() + hash.update(password) + password = hash.hexdigest() + hash = sha256() + hash.update(password) + hash.update(nonce) + password = hash.hexdigest() + self.server.authenticate(token, username, password) + self.logger.debug(u'Authenticated') + #if languages: + # self.logger.debug([self.getLanguage(l) for l in languages]) + # self.server.setFilters(token, [self.getLanguage(l) for l in languages]) + # self.logger.debug('Filers set for languages %s' % languages) + self.logger.debug(u"Starting search with token %s and hashs %s" % (token, [moviehash])) + results = self.server.search(token, [moviehash]) + return results + subs = [] + for sub in results['results']: + subs.append(sub) + self.server.terminate(token) + return subs +''' diff --git a/libs/subliminal/subtitles.py b/libs/subliminal/subtitles.py new file mode 100755 index 00000000..86a8bb73 --- /dev/null +++ b/libs/subliminal/subtitles.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path'] + + +from subliminal.languages import list_languages, convert_language +import abc +import os.path + + +EXTENSIONS = ['.srt', '.sub', '.txt'] + + +class Subtitle(object): + __metaclass__ = abc.ABCMeta + """Base class for subtitles""" + + def __init__(self, path, language): + self.path = path + self.language = language + + @property + def exists(self): + if self.path: + return os.path.exists(self.path) + return False + + @classmethod + def fromPath(cls, path): + extension = '' + for e in EXTENSIONS: + if path.endswith(e): + extension = e + break + if not extension: + raise ValueError('Not a supported subtitle extension') + language = os.path.splitext(path[:len(path) - len(extension)])[1][1:] + if not language in list_languages(1): + language = None + return cls(path, language) + + +class EmbeddedSubtitle(Subtitle): + def __init__(self, path, language, track_id): + super(EmbeddedSubtitle, self).__init__(path, language) + self.track_id = track_id + + @classmethod + def fromEnzyme(cls, path, subtitle): + language = convert_language(subtitle.language, 1, 2) + return cls(path, language, subtitle.trackno) + + +class ExternalSubtitle(Subtitle): + pass + + +class ResultSubtitle(ExternalSubtitle): + def __init__(self, path, language, plugin, link, release=None, confidence=1, keywords=set()): + super(ResultSubtitle, self).__init__(path, language) + self.plugin = plugin + self.link = link + self.release = release + self.confidence = confidence + self.keywords = keywords + + @property + def single(self): + extension = os.path.splitext(self.path)[0] + language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:] + if not language in list_languages(1): + return True + return False + + def convert(self): + converted = {'path': self.path, 'plugin': self.plugin, 'language': self.language, 'link': self.link, 'release': self.release, + 'confidence': self.confidence, 'keywords': self.keywords} + return converted + + def __str__(self): + return repr(self.convert()) + + +def get_subtitle_path(video_path, language, multi): + """Create the subtitle path from the given video path using language if multi""" + if not os.path.exists(video_path): + path = os.path.splitext(os.path.basename(video_path))[0] + else: + path = os.path.splitext(video_path)[0] + if multi and language: + return path + '.%s%s' % (language, EXTENSIONS[0]) + return path + '%s' % EXTENSIONS[0] diff --git a/libs/subliminal/tasks.py b/libs/subliminal/tasks.py new file mode 100755 index 00000000..111a08d3 --- /dev/null +++ b/libs/subliminal/tasks.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['Task', 'ListTask', 'DownloadTask', 'StopTask'] + + +class Task(object): + """Base class for tasks to use in subliminal""" + pass + + +class ListTask(Task): + """List task to list subtitles""" + def __init__(self, video, languages, plugin, config): + self.video = video + self.plugin = plugin + self.languages = languages + self.config = config + + +class DownloadTask(Task): + """Download task to download subtitles""" + def __init__(self, video, subtitles): + self.video = video + self.subtitles = subtitles + + +class StopTask(Task): + """Stop task to stop workers""" + pass diff --git a/libs/subliminal/utils.py b/libs/subliminal/utils.py new file mode 100755 index 00000000..27dddb16 --- /dev/null +++ b/libs/subliminal/utils.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['PluginConfig', 'get_keywords', 'split_keyword', 'NullHandler'] + + +import logging +import re +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + + +class PluginConfig(object): + def __init__(self, multi=None, cache_dir=None, filemode=None): + self.multi = multi + self.cache_dir = cache_dir + self.filemode = filemode + + +def get_keywords(guess): + keywords = set() + for k in ['releaseGroup', 'screenSize', 'videoCodec', 'format']: + if k in guess: + keywords = keywords | split_keyword(guess[k].lower()) + return keywords + + +def split_keyword(keyword): + split = set(re.findall(r'\w+', keyword)) + return split diff --git a/libs/subliminal/videos.py b/libs/subliminal/videos.py new file mode 100755 index 00000000..cb5a1520 --- /dev/null +++ b/libs/subliminal/videos.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +# +# Subliminal - Subtitles, faster than your thoughts +# Copyright (c) 2011 Antoine Bertin +# +# This file is part of Subliminal. +# +# Subliminal is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# +__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo', 'scan'] + + +from languages import list_languages +import abc +import enzyme +import guessit +import hashlib +import mimetypes +import os +import struct +import subprocess +import subtitles + + +EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv', '.divx', '.asf'] +MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo', 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d'] + + +class Video(object): + __metaclass__ = abc.ABCMeta + """Base class for videos""" + def __init__(self, release, guess, imdbid=None): + self.release = release + self.guess = guess + self.imdbid = imdbid + self._path = None + self.hashes = {} + if os.path.exists(release): + self.path = release + + @classmethod + def fromPath(cls, path): + """Create a Video object guessing all informations from the given release/path""" + guess = guessit.guess_file_info(path, 'autodetect') + result = None + if guess['type'] == 'episode' and 'series' in guess and 'season' in guess and 'episodeNumber' in guess: + title = None + if 'title' in guess: + title = guess['title'] + result = Episode(path, guess['series'], guess['season'], guess['episodeNumber'], title, guess) + if guess['type'] == 'movie' and 'title' in guess: + year = None + if 'year' in guess: + year = guess['year'] + result = Movie(path, guess['title'], year, guess) + if not result: + result = UnknownVideo(path, guess) + if not isinstance(result, cls): + raise ValueError('Video is not of requested type') + return result + + @property + def exists(self): + if self._path: + return os.path.exists(self._path) + return False + + @property + def path(self): + return self._path + + @path.setter + def path(self, value): + if not os.path.exists(value): + raise ValueError('Path does not exists') + self._path = value + self.size = os.path.getsize(self._path) + self._computeHashes() + + def _computeHashes(self): + self.hashes['OpenSubtitles'] = self._computeHashOpenSubtitles() + self.hashes['TheSubDB'] = self._computeHashTheSubDB() + + def _computeHashOpenSubtitles(self): + """Hash a file like OpenSubtitles""" + longlongformat = 'q' # long long + bytesize = struct.calcsize(longlongformat) + f = open(self.path, 'rb') + filesize = os.path.getsize(self.path) + filehash = filesize + if filesize < 65536 * 2: + return [] + for _ in range(65536 / bytesize): + filebuffer = f.read(bytesize) + (l_value,) = struct.unpack(longlongformat, filebuffer) + filehash += l_value + filehash = filehash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number + f.seek(max(0, filesize - 65536), 0) + for _ in range(65536 / bytesize): + filebuffer = f.read(bytesize) + (l_value,) = struct.unpack(longlongformat, filebuffer) + filehash += l_value + filehash = filehash & 0xFFFFFFFFFFFFFFFF + f.close() + returnedhash = '%016x' % filehash + return returnedhash + + def _computeHashTheSubDB(self): + """Hash a file like TheSubDB""" + readsize = 64 * 1024 + with open(self.path, 'rb') as f: + data = f.read(readsize) + f.seek(-readsize, os.SEEK_END) + data += f.read(readsize) + return hashlib.md5(data).hexdigest() + + def mkvmerge(self, subs, out=None, mkvmerge_bin='mkvmerge', title=None): + """Merge the video with subs""" + if not out: + out = self.path + '.merged.mkv' + args = [mkvmerge_bin, '-o', out, self.path] + if title: + args += ['--title', title] + for sub in subs: + if sub.language: + track_id = 0 + if isinstance(sub, subtitles.EmbeddedSubtitle): + track_id = sub.track_id + args += ['--language', str(track_id) + ':' + sub.language, sub.path] + continue + args += [sub.path] + with open(os.devnull, 'w') as devnull: + p = subprocess.Popen(args, stdout=devnull, stderr=devnull) + p.wait() + + def scan(self): + """Scan and return associated Subtitles""" + if not self.exists: + return [] + basepath = os.path.splitext(self.path)[0] + results = [] + video_infos = None + try: + video_infos = enzyme.parse(self.path) + except enzyme.ParseError: + pass + if isinstance(video_infos, enzyme.core.AVContainer): + results.extend([subtitles.EmbeddedSubtitle.fromEnzyme(self.path, s) for s in video_infos.subtitles]) + for l in list_languages(1): + for e in subtitles.EXTENSIONS: + single_path = basepath + '%s' % e + if os.path.exists(single_path): + results.append(subtitles.ExternalSubtitle(single_path, None)) + multi_path = basepath + '.%s%s' % (l, e) + if os.path.exists(multi_path): + results.append(subtitles.ExternalSubtitle(multi_path, l)) + return results + + +class Episode(Video): + """Episode class""" + def __init__(self, release, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None): + super(Episode, self).__init__(release, guess, imdbid) + self.series = series + self.title = title + self.season = season + self.episode = episode + self.tvdbid = tvdbid + + +class Movie(Video): + """Movie class""" + def __init__(self, release, title, year=None, guess=None, imdbid=None): + super(Movie, self).__init__(release, guess, imdbid) + self.title = title + self.year = year + + +class UnknownVideo(Video): + """Unknown video""" + def __init__(self, release, guess, imdbid=None): + super(UnknownVideo, self).__init__(release, guess, imdbid) + self.guess = guess + + +def scan(entry, max_depth=3, depth=0): + """Scan a path and return a list of tuples (video, [subtitle])""" + if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0 + return [] + if depth == 0: + entry = os.path.abspath(entry) + if os.path.isfile(entry): # a file? scan it + if depth != 0: # trust the user: only check for valid format if recursing + if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS: + return [] + video = Video.fromPath(entry) + return [(video, video.scan())] + if os.path.isdir(entry): # a dir? recurse + result = [] + for e in os.listdir(entry): + result.extend(scan(os.path.join(entry, e), max_depth, depth + 1)) + return result + return [] # anything else