dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update subliminal

Browse Source
This commit is contained in:
Ruud
2012-06-16 18:12:41 +02:00
parent 79e842951e
commit d3f5f1408e
18 changed files with 1235 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/subliminal/api.py
+12 -11
View File
@@ -18,8 +18,7 @@
from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
group_by_video, key_subtitles)
import guessit
from guessit.language import ALL_LANGUAGES
from .language import language_set, language_list, LANGUAGES
import logging
@@ -32,7 +31,8 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
:param paths: path(s) to video file or folder
:type paths: string or list
:param list languages: languages to search for, in preferred order
:param languages: languages to search for, in preferred order
:type languages: list of :class:`~subliminal.language.Language` or string
:param list services: services to use for the search, in preferred order
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
@@ -44,7 +44,7 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
"""
services = services or SERVICES
languages = set(map(guessit.Language, languages or []) or ALL_LANGUAGES)
languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
@@ -68,7 +68,8 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
:param paths: path(s) to video file or folder
:type paths: string or list
:param list languages: languages to search for, in preferred order
:param languages: languages to search for, in preferred order
:type languages: list of :class:`~subliminal.language.Language` or string
:param list services: services to use for the search, in preferred order
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
@@ -77,16 +78,16 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
:param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
:param order: preferred order for subtitles sorting
:type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
:return: found subtitles
:rtype: list of (:class:`~subliminal.videos.Video`, [:class:`~subliminal.subtitles.ResultSubtitle`])
:return: downloaded subtitles
:rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
"""
services = services or SERVICES
languages = map(guessit.Language, languages or []) or list(ALL_LANGUAGES)
languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth, scan_filter)
subtitles_by_video = list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
results = []
@@ -95,9 +96,9 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
for task in tasks:
try:
result = consume_task(task, service_instances)
results.append(result)
results.append((task.video, result))
except:
logger.error(u'Error consuming task %r' % task, exc_info=True)
for service_instance in service_instances.itervalues():
service_instance.terminate()
return results
return group_by_video(results)
libs/subliminal/async.py
+6 -5
View File
@@ -18,13 +18,14 @@
from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
create_download_tasks, group_by_video, key_subtitles)
from guessit.language import ALL_LANGUAGES
from .language import language_list, language_set, LANGUAGES
from .tasks import StopTask
import Queue
import logging
import threading
__all__ = ['Worker', 'Pool']
logger = logging.getLogger(__name__)
@@ -111,7 +112,7 @@ class Pool(object):
def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
"""See :meth:`subliminal.list_subtitles`"""
services = services or SERVICES
languages = set(languages or ALL_LANGUAGES)
languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
@@ -126,11 +127,11 @@ class Pool(object):
def download_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
"""See :meth:`subliminal.download_subtitles`"""
services = services or SERVICES
languages = languages or list(ALL_LANGUAGES)
languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth, scan_filter)
subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
tasks = create_download_tasks(subtitles_by_video, multi)
@@ -138,4 +139,4 @@ class Pool(object):
self.tasks.put(task)
self.join()
results = self.collect()
return results
return group_by_video(results)
libs/subliminal/cache.py
+10 -8
View File
@@ -15,25 +15,26 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
import os.path
from collections import defaultdict
import threading
from functools import wraps
import logging
import os.path
import threading
try:
import cPickle as pickle
except ImportError:
import pickle
__all__ = ['Cache', 'cachedmethod']
logger = logging.getLogger(__name__)
class Cache(object):
"""A Cache object contains cached values for methods. It can have
separate internal caches, one for each service.
"""
separate internal caches, one for each service
"""
def __init__(self, cache_dir):
self.cache_dir = cache_dir
self.cache = defaultdict(dict)
@@ -100,9 +101,12 @@ class Cache(object):
def cachedmethod(function):
"""Decorator to make a method use the cache.
WARNING: this can NOT be used with static functions, it has to be used on
methods of some class."""
.. note::
This can NOT be used with static functions, it has to be used on
methods of some class
"""
@wraps(function)
def cached(*args):
c = args[0].config.cache
@@ -126,7 +130,5 @@ def cachedmethod(function):
# meantime, but that's ok as we prefer to keep the latest value in
# the cache
func_cache[key] = result
return result
return cached
libs/subliminal/core.py
+4 -5
View File
@@ -20,10 +20,9 @@ from .services import ServiceConfig
from .tasks import DownloadTask, ListTask
from .utils import get_keywords
from .videos import Episode, Movie, scan
from guessit.language import lang_set
import bs4
from collections import defaultdict
from itertools import groupby
import bs4
import guessit
import logging
@@ -119,7 +118,7 @@ def consume_task(task, services=None):
:type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask`
:param dict services: mapping between the service name and an instance of this service
:return: the result of the task
:rtype: list of :class:`~subliminal.subtitles.ResultSubtitle` or :class:`~subliminal.subtitles.Subtitle`
:rtype: list of :class:`~subliminal.subtitles.ResultSubtitle`
"""
if services is None:
@@ -134,12 +133,11 @@ def consume_task(task, services=None):
service = get_service(services, subtitle.service)
try:
service.download(subtitle)
result = subtitle
result = [subtitle]
break
except DownloadFailedError:
logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
continue
if result is None:
logger.error(u'No subtitles could be downloaded for video %r' % task.video)
return result
@@ -225,6 +223,7 @@ def key_subtitles(subtitle, video, languages, services, order):
for sort_item in order:
if sort_item == LANGUAGE_INDEX:
key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1)
key += '{0:01d}'.format(subtitle.language == languages[languages.index(subtitle.language)])
elif sort_item == SERVICE_INDEX:
key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1)
elif sort_item == SERVICE_CONFIDENCE:
libs/subliminal/exceptions.py
-49
View File
@@ -22,60 +22,11 @@ class Error(Exception):
pass
class InvalidLanguageError(Error):
"""Exception raised when invalid language is submitted
Attributes:
language -- language that cause the error
"""
def __init__(self, language):
self.language = language
def __str__(self):
return self.language
class MissingLanguageError(Error):
"""Exception raised when a missing language is found
Attributes:
language -- the missing language
"""
def __init__(self, language):
self.language = language
def __str__(self):
return self.language
class InvalidServiceError(Error):
"""Exception raised when invalid service is submitted
:param string service: service that causes the error
"""
def __init__(self, service):
self.service = service
def __str__(self):
return self.service
class ServiceError(Error):
""""Exception raised by services"""
pass
class WrongTaskError(Error):
""""Exception raised when invalid task is submitted"""
pass
class DownloadFailedError(Error):
""""Exception raised when a download task has failed in service"""
pass
class UnknownVideoError(Error):
""""Exception raised when a video could not be identified"""
pass
libs/subliminal/language.py
Executable
+1032
View File
File diff suppressed because it is too large Load Diff
libs/subliminal/services/__init__.py
+59 -20
View File
@@ -15,10 +15,10 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .. import cache
from ..exceptions import MissingLanguageError, DownloadFailedError, ServiceError
from ..cache import Cache
from ..exceptions import DownloadFailedError, ServiceError
from ..language import language_set, Language
from ..subtitles import EXTENSIONS
from guessit.language import lang_set, UNDETERMINED
import logging
import os
import requests
@@ -49,8 +49,14 @@ class ServiceBase(object):
#: Timeout for web requests
timeout = 5
#: Mapping to Service's language codes and subliminal's
languages = {}
#: :class:`~subliminal.language.language_set` of available languages
languages = language_set()
#: Map between language objects and language codes used in the service
language_map = {}
#: Default attribute of a :class:`~subliminal.language.Language` to get with :meth:`get_code`
language_code = 'alpha2'
#: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
videos = []
@@ -63,6 +69,7 @@ class ServiceBase(object):
def __init__(self, config=None):
self.config = config or ServiceConfig()
self.session = None
def __enter__(self):
self.init()
@@ -80,33 +87,60 @@ class ServiceBase(object):
"""Initialize cache, make sure it is loaded from disk"""
if not self.config or not self.config.cache:
raise ServiceError('Cache directory is required')
service_name = self.__class__.__name__
self.config.cache.load(service_name)
self.config.cache.load(self.__class__.__name__)
def save_cache(self):
service_name = self.__class__.__name__
self.config.cache.save(service_name)
self.config.cache.save(self.__class__.__name__)
def clear_cache(self):
service_name = self.__class__.__name__
self.config.cache.clear(service_name)
self.config.cache.clear(self.__class__.__name__)
def cache_for(self, func, args, result):
service_name = self.__class__.__name__
return self.config.cache.cache_for(service_name, func, args, result)
return self.config.cache.cache_for(self.__class__.__name__, func, args, result)
def cached_value(self, func, args):
service_name = self.__class__.__name__
return self.config.cache.cached_value(service_name, func, args)
return self.config.cache.cached_value(self.__class__.__name__, func, args)
def terminate(self):
"""Terminate connection"""
logger.debug(u'Terminating %s' % self.__class__.__name__)
def get_code(self, language):
"""Get the service code for a :class:`~subliminal.language.Language`
It uses the :data:`language_map` and if there's no match, falls back
on the :data:`language_code` attribute of the given :class:`~subliminal.language.Language`
"""
if language in self.language_map:
return self.language_map[language]
if self.language_code is None:
raise ValueError('%r has no matching code' % language)
return getattr(language, self.language_code)
def get_language(self, code):
"""Get a :class:`~subliminal.language.Language` from a service code
It uses the :data:`language_map` and if there's no match, uses the
given code as ``language`` parameter for the :class:`~subliminal.language.Language`
constructor
.. note::
A warning is emitted if the generated :class:`~subliminal.language.Language`
is "Undetermined"
"""
if code in self.language_map:
return self.language_map[code]
language = Language(code, strict=False)
if language == Language('Undetermined'):
logger.warning(u'Code %s could not be identified as a language for %s' % (code, self.__class__.__name__))
return language
def query(self, *args):
"""Make the actual query"""
pass
raise NotImplementedError()
def list(self, video, languages):
"""List subtitles
@@ -119,6 +153,10 @@ class ServiceBase(object):
return []
return self.list_checked(video, languages)
def list_checked(self, video, languages):
"""List subtitles without having to check parameters for validity"""
raise NotImplementedError()
def download(self, subtitle):
"""Download a subtitle"""
self.download_file(subtitle.link, subtitle.path)
@@ -129,11 +167,12 @@ class ServiceBase(object):
:param video: the video to check
:type video: :class:`~subliminal.videos.video`
:param set languages: languages to check
:param languages: languages to check
:type languages: :class:`~subliminal.language.Language`
:rtype: bool
"""
languages = (lang_set(languages) & cls.languages) - set([UNDETERMINED])
languages = (languages & cls.languages) - language_set(['Undetermined'])
if not languages:
logger.debug(u'No language available for service %s' % cls.__name__.lower())
return False
@@ -211,7 +250,7 @@ class ServiceConfig(object):
self.cache_dir = cache_dir
self.cache = None
if cache_dir is not None:
self.cache = cache.Cache(cache_dir)
self.cache = Cache(cache_dir)
def __repr__(self):
return 'ServiceConfig(%r, %s)' % (self.multi, self.cache.cache_dir)
libs/subliminal/services/addic7ed.py
+7 -14
View File
@@ -17,12 +17,11 @@
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..cache import cachedmethod
from ..language import Language, language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import get_keywords
from ..videos import Episode
from bs4 import BeautifulSoup
from guessit.language import lang_set
from subliminal.utils import get_keywords
import guessit
import logging
import re
@@ -49,13 +48,10 @@ def matches(pattern, string):
class Addic7ed(ServiceBase):
server_url = 'http://www.addic7ed.com'
api_based = False
languages = lang_set([u'English', u'Italian', u'Portuguese',
u'Portuguese (Brazilian)', u'Romanian',
u'Spanish', u'French', u'Greek', u'Arabic',
u'German', u'Croatian', u'Indonesian', u'Hebrew',
u'Russian', u'Turkish', u'Swedish', u'Czech',
u'Dutch', u'Hungarian', u'Norwegian', u'Polish',
u'Persian'], strict=True)
#TODO: Complete this
languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'he', 'hr', 'hu', 'it',
'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre')}
videos = [Episode]
require_video = False
required_features = ['permissive']
@@ -119,7 +115,7 @@ class Addic7ed(ServiceBase):
if 'href' not in link.attrs or not (link['href'].startswith('/original') or link['href'].startswith('/updated')):
continue
suburl = link['href']
lang = guessit.Language(row.find('td', 'language').text.strip())
lang = self.get_language(row.find('td', 'language').text.strip())
result = {'suburl': suburl, 'language': lang, 'release': release}
suburls.append(result)
return suburls
@@ -135,21 +131,18 @@ class Addic7ed(ServiceBase):
except KeyError:
logger.debug(u'Could not find series id for %s' % series)
return []
try:
ep_url = self.get_episode_url(sid, season, episode)
except KeyError:
logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
return []
suburls = self.get_sub_urls(ep_url)
# filter the subtitles with our queried languages
subtitles = []
for suburl in suburls:
language = suburl['language']
if language not in languages:
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
'%s/%s' % (self.server_url, suburl['suburl']),
libs/subliminal/services/bierdopje.py
+2 -3
View File
@@ -18,11 +18,11 @@
from . import ServiceBase
from ..cache import cachedmethod
from ..exceptions import ServiceError
from ..language import language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import to_unicode
from ..videos import Episode
from bs4 import BeautifulSoup
from guessit.language import lang_set
import logging
import urllib
try:
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
class BierDopje(ServiceBase):
server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
api_based = True
languages = lang_set(['en', 'nl'])
languages = language_set(['eng', 'dut'])
videos = [Episode]
require_video = False
required_features = ['xml']
@@ -52,7 +52,6 @@ class BierDopje(ServiceBase):
if soup.status.contents[0] == 'false':
logger.debug(u'Could not find show %s' % series)
return None
return int(soup.showid.contents[0])
def load_cache(self):
libs/subliminal/services/opensubtitles.py
+47 -69
View File
@@ -17,11 +17,10 @@
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..exceptions import ServiceError, DownloadFailedError
from ..language import Language, language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import to_unicode
from ..videos import Episode, Movie
from guessit.language import lang_set
import guessit
import gzip
import logging
import os.path
@@ -34,71 +33,50 @@ logger = logging.getLogger(__name__)
class OpenSubtitles(ServiceBase):
server_url = 'http://api.opensubtitles.org/xml-rpc'
api_based = True
# language list fetched from:
# http://www.opensubtitles.org/addons/export_languages.php
languages = lang_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh',
'afr', 'ain', 'aka', 'akk', 'alb', 'ale', 'alg', 'alt',
'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava',
'ave', 'awa', 'aym', 'aze', 'bad', 'bai', 'bak', 'bal',
'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla',
'bnt', 'bod', 'bos', 'bra', 'bre', 'btk', 'bua', 'bug',
'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau',
'ceb', 'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk',
'chm', 'chn', 'cho', 'chp', 'chr', 'chu', 'chv', 'chy',
'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre',
'crh', 'crp', 'csb', 'cus', 'cym', 'cze', 'dak', 'dan',
'dar', 'day', 'del', 'den', 'deu', 'dgr', 'din', 'div',
'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi',
'egy', 'eka', 'elx', 'eng', 'enm', 'epo', 'est', 'eus',
'ewe', 'ewo', 'fan', 'fao', 'fas', 'fat', 'fij', 'fil',
'fin', 'fiu', 'fon', 'fra', 'fre', 'frm', 'fro', 'fry',
'ful', 'fur', 'gaa', 'gay', 'gba', 'gem', 'geo', 'ger',
'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh',
'gon', 'gor', 'got', 'grb', 'grc', 'ell', 'grn', 'guj',
'gwi', 'hai', 'hat', 'hau', 'haw', 'heb', 'her', 'hil',
'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup',
'hye', 'iba', 'ibo', 'ice', 'ido', 'iii', 'ijo', 'iku',
'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh', 'ipk',
'ira', 'iro', 'isl', 'ita', 'jav', 'jpn', 'jpr', 'jrb',
'kaa', 'kab', 'kac', 'kal', 'kam', 'kan', 'kar', 'kas',
'kat', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm',
'kho', 'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon',
'kor', 'kos', 'kpe', 'krc', 'kro', 'kru', 'kua', 'kum',
'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav',
'lez', 'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua',
'lub', 'lug', 'lui', 'lun', 'luo', 'lus', 'mac', 'mad',
'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map',
'mar', 'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic',
'min', 'mis', 'mkd', 'mkh', 'mlg', 'mlt', 'mnc', 'mni',
'mno', 'moh', 'mol', 'mon', 'mos', 'mri', 'msa', 'mwl',
'mul', 'mun', 'mus', 'mwr', 'mya', 'myn', 'myv', 'nah',
'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds',
'nep', 'new', 'nia', 'nic', 'niu', 'nld', 'nno', 'nob',
'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym',
'nyn', 'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa',
'oss', 'ota', 'oto', 'paa', 'pag', 'pal', 'pam', 'pan',
'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol',
'pon', 'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap',
'rar', 'roa', 'roh', 'rom', 'ron', 'run', 'rup', 'rus',
'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas',
'sat', 'scc', 'scn', 'sco', 'scr', 'sel', 'sem', 'sga',
'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla', 'slk',
'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo',
'sms', 'sna', 'snd', 'snk', 'sog', 'som', 'son', 'sot',
'spa', 'sqi', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk',
'sun', 'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai',
'tam', 'tat', 'tel', 'tem', 'ter', 'tet', 'tgk', 'tgl',
'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli',
'tmh', 'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk',
'tum', 'tup', 'tur', 'tut', 'tvl', 'twi', 'tyv', 'udm',
'uga', 'uig', 'ukr', 'umb', 'und', 'urd', 'uzb', 'vai',
'ven', 'vie', 'vol', 'vot', 'wak', 'wal', 'war', 'was',
'wel', 'wen', 'wln', 'wol', 'xal', 'xho', 'yao', 'yap',
'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'zho', 'znd',
'zul', 'zun', 'rum', 'pob', 'unk', 'ass'])
# Source: http://www.opensubtitles.org/addons/export_languages.php
languages = language_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh', 'afr', 'ain', 'aka', 'akk',
'alb', 'ale', 'alg', 'alt', 'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava', 'ave', 'awa', 'aym', 'aze',
'bad', 'bai', 'bak', 'bal', 'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla', 'bnt', 'bos', 'bra', 'bre',
'btk', 'bua', 'bug', 'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau', 'ceb',
'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk', 'chm', 'chn', 'cho', 'chp', 'chr',
'chu', 'chv', 'chy', 'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre', 'crh',
'crp', 'csb', 'cus', 'cze', 'dak', 'dan', 'dar', 'day', 'del', 'den', 'dgr', 'din',
'div', 'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi', 'egy', 'eka', 'ell',
'elx', 'eng', 'enm', 'epo', 'est', 'ewe', 'ewo', 'fan', 'fao', 'fat', 'fij', 'fil',
'fin', 'fiu', 'fon', 'fre', 'frm', 'fro', 'fry', 'ful', 'fur', 'gaa', 'gay', 'gba',
'gem', 'geo', 'ger', 'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh', 'gon',
'gor', 'got', 'grb', 'grc', 'grn', 'guj', 'gwi', 'hai', 'hat', 'hau', 'haw', 'heb',
'her', 'hil', 'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup', 'iba', 'ibo',
'ice', 'ido', 'iii', 'ijo', 'iku', 'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh',
'ipk', 'ira', 'iro', 'ita', 'jav', 'jpn', 'jpr', 'jrb', 'kaa', 'kab', 'kac', 'kal',
'kam', 'kan', 'kar', 'kas', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm', 'kho',
'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon', 'kor', 'kos', 'kpe', 'krc', 'kro',
'kru', 'kua', 'kum', 'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav', 'lez',
'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua', 'lub', 'lug', 'lui', 'lun', 'luo',
'lus', 'mac', 'mad', 'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map', 'mar',
'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic', 'min', 'mkh', 'mlg', 'mlt', 'mnc',
'mni', 'mno', 'moh', 'mon', 'mos', 'mun', 'mus', 'mwl', 'mwr', 'myn', 'myv', 'nah',
'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds', 'nep', 'new', 'nia', 'nic',
'niu', 'nno', 'nob', 'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym', 'nyn',
'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa', 'oss', 'ota', 'oto', 'paa', 'pag',
'pal', 'pam', 'pan', 'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol', 'pon',
'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap', 'rar', 'roa', 'roh', 'rom', 'rum',
'run', 'rup', 'rus', 'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas', 'sat',
'scn', 'sco', 'sel', 'sem', 'sga', 'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla',
'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo', 'sms', 'sna', 'snd', 'snk',
'sog', 'som', 'son', 'sot', 'spa', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk', 'sun',
'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai', 'tam', 'tat', 'tel', 'tem', 'ter',
'tet', 'tgk', 'tgl', 'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli', 'tmh',
'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk', 'tum', 'tup', 'tur', 'tut', 'tvl',
'twi', 'tyv', 'udm', 'uga', 'uig', 'ukr', 'umb', 'urd', 'uzb', 'vai', 'ven', 'vie',
'vol', 'vot', 'wak', 'wal', 'war', 'was', 'wel', 'wen', 'wln', 'wol', 'xal', 'xho',
'yao', 'yap', 'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'znd', 'zul', 'zun',
'por-BR', 'rum-MD'])
language_map = {'mol': Language('rum-MD'), 'scc': Language('srp'), 'pob': Language('por-BR'),
Language('rum-MD'): 'mol', Language('srp'): 'scc', Language('por-BR'): 'pob'}
language_code = 'alpha3'
videos = [Episode, Movie]
require_video = False
confidence_order = ['moviehash', 'imdbid', 'fulltext']
@@ -131,7 +109,7 @@ class OpenSubtitles(ServiceBase):
if not searches:
raise ServiceError('One or more parameter missing')
for search in searches:
search['sublanguageid'] = ','.join(l.opensubtitles for l in languages)
search['sublanguageid'] = ','.join(self.get_code(l) for l in languages)
logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
results = self.server.SearchSubtitles(self.token, searches)
if not results['data']:
@@ -139,7 +117,7 @@ class OpenSubtitles(ServiceBase):
return []
subtitles = []
for result in results['data']:
language = guessit.Language(result['SubLanguageID'])
language = self.get_language(result['SubLanguageID'])
path = get_subtitle_path(filepath, language, self.config.multi)
confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
libs/subliminal/services/podnapisi.py
+13 -9
View File
@@ -17,12 +17,11 @@
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..exceptions import ServiceError, DownloadFailedError
from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import to_unicode
from ..videos import Episode, Movie
from guessit.language import lang_set
from hashlib import md5, sha256
import guessit
import logging
import xmlrpclib
@@ -33,11 +32,16 @@ logger = logging.getLogger(__name__)
class Podnapisi(ServiceBase):
server_url = 'http://ssp.podnapisi.net:8000'
api_based = True
languages = lang_set(['sl', 'en', 'nn', 'ko', 'de', 'is', 'cs', 'fr', 'it', 'bs', 'jp', 'ar', 'ro',
'hu', 'gr', 'zh', 'lt', 'et', 'lv', 'he', 'nl', 'da', 'sv', 'pl', 'ru', 'es',
'sq', 'tr', 'fi', 'pt', 'bg', 'mk', 'sr', 'sk', 'hr', 'hi', 'th', 'ca', 'uk',
'pb', 'ga', 'be', 'vi', 'fa', 'ca', 'id', 'ms'])
#FIXME: ag and cyr not recognized by guessit
languages = language_set(['ar', 'be', 'bg', 'bs', 'ca', 'ca', 'cs', 'da', 'de', 'el', 'en',
'es', 'et', 'fa', 'fi', 'fr', 'ga', 'he', 'hi', 'hr', 'hu', 'id',
'is', 'it', 'ja', 'ko', 'lt', 'lv', 'mk', 'ms', 'nl', 'nn', 'pl',
'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'th', 'tr', 'uk',
'vi', 'zh', 'es-ar', 'pt-br'])
language_map = {'jp': Language('jpn'), Language('jpn'): 'jp',
'gr': Language('gre'), Language('gre'): 'gr',
'pb': Language('por-BR'), Language('por-BR'): 'pb',
'ag': Language('spa-AR'), Language('spa-AR'): 'ag',
'cyr': Language('srp')}
videos = [Episode, Movie]
require_video = True
@@ -71,8 +75,8 @@ class Podnapisi(ServiceBase):
return []
subtitles = []
for result in results['results'][moviehash]['subtitles']:
language = guessit.Language(result['lang'])
if language == guessit.language.UNDETERMINED or language not in languages:
language = self.get_language(result['lang'])
if language not in languages:
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['id'],
libs/subliminal/services/subswiki.py
+7 -6
View File
@@ -17,12 +17,11 @@
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..exceptions import ServiceError
from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie
from bs4 import BeautifulSoup
from guessit.language import lang_set
from subliminal.utils import get_keywords, split_keyword
import guessit
import logging
import re
import urllib
@@ -34,9 +33,11 @@ logger = logging.getLogger(__name__)
class SubsWiki(ServiceBase):
server_url = 'http://www.subswiki.com'
api_based = False
languages = lang_set([u'English (US)', u'English (UK)', u'English', u'French', u'Brazilian',
u'Portuguese', u'Español (Latinoamérica)', u'Español (España)',
u'Español', u'Italian', u'Català'], strict=True)
languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
u'English (UK)': Language('eng-GB')}
language_code = 'name'
videos = [Episode, Movie]
require_video = False
release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
@@ -82,7 +83,7 @@ class SubsWiki(ServiceBase):
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
language = guessit.Language(html_language.string.strip())
language = self.get_language(html_language.string.strip())
if language not in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
libs/subliminal/services/subtitulos.py
+7 -6
View File
@@ -16,12 +16,11 @@
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode
from bs4 import BeautifulSoup
from guessit.language import lang_set
from subliminal.utils import get_keywords, split_keyword
import guessit
import logging
import re
import unicodedata
@@ -34,9 +33,11 @@ logger = logging.getLogger(__name__)
class Subtitulos(ServiceBase):
server_url = 'http://www.subtitulos.es'
api_based = False
languages = lang_set([u'English (US)', u'English (UK)', u'English', u'French', u'Brazilian',
u'Portuguese', u'Español (Latinoamérica)', u'Español (España)', u'Español',
u'Italian', u'Català'], strict=True)
languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
u'English (UK)': Language('eng-GB')}
language_code = 'name'
videos = [Episode]
require_video = False
required_features = ['permissive']
@@ -68,7 +69,7 @@ class Subtitulos(ServiceBase):
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
language = guessit.Language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
language = self.get_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
if language not in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
libs/subliminal/services/thesubdb.py
+8 -9
View File
@@ -16,10 +16,9 @@
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..language import language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie, UnknownVideo
from guessit.language import lang_set
import guessit
import logging
@@ -27,13 +26,13 @@ logger = logging.getLogger(__name__)
class TheSubDB(ServiceBase):
server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)' # defined by the API
server_url = 'http://api.thesubdb.com'
user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)'
api_based = True
languages = lang_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi',
'fr', 'hu', 'id', 'it', 'la', 'nl', 'no',
'oc', 'pl', 'pt', 'ro', 'ru', 'sl', 'sr',
'sv', 'tr'], strict=True) # list available with the API at http://sandbox.thesubdb.com/?action=languages
# Source: http://api.thesubdb.com/?action=languages
languages = language_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'id', 'it',
'la', 'nl', 'no', 'oc', 'pl', 'pt', 'ro', 'ru', 'sl', 'sr', 'sv',
'tr'])
videos = [Movie, Episode, UnknownVideo]
require_video = True
@@ -48,7 +47,7 @@ class TheSubDB(ServiceBase):
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
available_languages = set(guessit.Language(l) for l in r.content.split(','))
available_languages = language_set(r.content.split(','))
languages &= available_languages
if not languages:
logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
libs/subliminal/services/tvsubtitles.py
+8 -13
View File
@@ -17,12 +17,11 @@
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..cache import cachedmethod
from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import get_keywords
from ..videos import Episode
from bs4 import BeautifulSoup
from guessit.language import lang_set
from subliminal.utils import get_keywords
import guessit
import logging
import re
@@ -41,12 +40,11 @@ def match(pattern, string):
class TvSubtitles(ServiceBase):
server_url = 'http://www.tvsubtitles.net'
api_based = False
languages = lang_set([u'English', u'Español', u'French', u'German',
u'Brazilian', u'Russian', u'Ukrainian', u'Italian',
u'Greek', u'Arabic', u'Hungarian', u'Polish',
u'Turkish', u'Dutch', u'Portuguese', u'Swedish',
u'Danish', u'Finnish', u'Korean', u'Chinese',
u'Japanese', u'Bulgarian', u'Czech', u'Romanian'], strict=True)
languages = language_set(['ar', 'bg', 'cs', 'da', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hu',
'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
'zh', 'pt-br'])
#TODO: Find more exceptions
language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr')}
videos = [Episode]
require_video = False
required_features = ['permissive']
@@ -78,11 +76,9 @@ class TvSubtitles(ServiceBase):
cells = row.find_all('td')
if not cells:
continue
episode_number = match('x([0-9]+)', cells[0].text)
if not episode_number:
continue
episode_number = int(episode_number)
episode_id = int(match('episode-([0-9]+)', cells[1].a['href']))
# we could just return the id of the queried episode, but as we
@@ -102,14 +98,13 @@ class TvSubtitles(ServiceBase):
if 'href' not in subdiv.attrs or not subdiv['href'].startswith('/subtitle'):
continue
subid = int(match('([0-9]+)', subdiv['href']))
lang = guessit.Language(match('flags/(.*).gif', subdiv.img['src']))
lang = self.get_language(match('flags/(.*).gif', subdiv.img['src']))
result = {'subid': subid, 'language': lang}
for p in subdiv.find_all('p'):
if 'alt' in p.attrs and p['alt'] == 'rip':
result['rip'] = p.text.strip()
if 'alt' in p.attrs and p['alt'] == 'release':
result['release'] = p.text.strip()
subids.append(result)
return subids
libs/subliminal/subtitles.py
+8 -10
View File
@@ -15,9 +15,8 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .language import Language
import os.path
import guessit
from guessit.language import is_language
__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
@@ -31,7 +30,7 @@ class Subtitle(object):
:param string path: path to the subtitle
:param language: language of the subtitle
:type language: :class:`guessit.Language`
:type language: :class:`~subliminal.language.Language`
"""
def __init__(self, path, language):
@@ -51,7 +50,7 @@ class EmbeddedSubtitle(Subtitle):
:param string path: path to the subtitle
:param language: language of the subtitle
:type language: :class:`guessit.Language`
:type language: :class:`~subliminal.language.Language`
:param int track_id: id of the subtitle track in the container
"""
@@ -61,7 +60,7 @@ class EmbeddedSubtitle(Subtitle):
@classmethod
def from_enzyme(cls, path, subtitle):
language = guessit.Language(subtitle.language) or None
language = Language(subtitle.language) or None
return cls(path, language, subtitle.trackno)
@@ -78,8 +77,7 @@ class ExternalSubtitle(Subtitle):
if not extension:
raise ValueError('Not a supported subtitle extension')
language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
language = guessit.Language(language) or None
language = Language(language) or None
return cls(path, language)
@@ -88,7 +86,7 @@ class ResultSubtitle(ExternalSubtitle):
:param string path: path to the subtitle
:param language: language of the subtitle
:type language: :class:`guessit.Language`
:type language: :class:`~subliminal.language.Language`
:param string service: name of the service
:param string link: download link for the subtitle
:param string release: release name of the video
@@ -114,7 +112,7 @@ class ResultSubtitle(ExternalSubtitle):
"""
extension = os.path.splitext(self.path)[0]
language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
return not is_language(language)
return Language(language) == Language('und')
def __repr__(self):
return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
@@ -125,7 +123,7 @@ def get_subtitle_path(video_path, language, multi):
:param string video_path: path to the video
:param language: language of the subtitle
:type language: :class:`guessit.Language`
:type language: :class:`~subliminal.language.Language`
:param bool multi: whether to use multi language naming or not
:return: path of the subtitle
:rtype: string
libs/subliminal/tasks.py
+2
View File
@@ -35,6 +35,7 @@ class ListTask(Task):
"""
def __init__(self, video, languages, service, config):
super(ListTask, self).__init__()
self.video = video
self.service = service
self.languages = languages
@@ -54,6 +55,7 @@ class DownloadTask(Task):
"""
def __init__(self, video, subtitles):
super(DownloadTask, self).__init__()
self.video = video
self.subtitles = subtitles
libs/subliminal/videos.py
+3 -1
View File
@@ -145,12 +145,14 @@ class Video(object):
lang = guessit.Language(possible_lang)
if lang:
results.append(subtitles.ExternalSubtitle(path, lang))
return results
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.release)
def __hash__(self):
return hash(self.path or self.release)
class Episode(Video):
"""Episode :class:`Video`