diff --git a/libs/subliminal/api.py b/libs/subliminal/api.py
index a7baeab6..02785ffa 100755
--- a/libs/subliminal/api.py
+++ b/libs/subliminal/api.py
@@ -18,8 +18,7 @@
from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
group_by_video, key_subtitles)
-import guessit
-from guessit.language import ALL_LANGUAGES
+from .language import language_set, language_list, LANGUAGES
import logging
@@ -32,7 +31,8 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
:param paths: path(s) to video file or folder
:type paths: string or list
- :param list languages: languages to search for, in preferred order
+ :param languages: languages to search for, in preferred order
+ :type languages: list of :class:`~subliminal.language.Language` or string
:param list services: services to use for the search, in preferred order
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
@@ -44,7 +44,7 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
"""
services = services or SERVICES
- languages = set(map(guessit.Language, languages or []) or ALL_LANGUAGES)
+ languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
@@ -68,7 +68,8 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
:param paths: path(s) to video file or folder
:type paths: string or list
- :param list languages: languages to search for, in preferred order
+ :param languages: languages to search for, in preferred order
+ :type languages: list of :class:`~subliminal.language.Language` or string
:param list services: services to use for the search, in preferred order
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
@@ -77,16 +78,16 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
:param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
:param order: preferred order for subtitles sorting
:type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
- :return: found subtitles
- :rtype: list of (:class:`~subliminal.videos.Video`, [:class:`~subliminal.subtitles.ResultSubtitle`])
+ :return: downloaded subtitles
+ :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
"""
services = services or SERVICES
- languages = map(guessit.Language, languages or []) or list(ALL_LANGUAGES)
+ languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
- subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth, scan_filter)
+ subtitles_by_video = list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
results = []
@@ -95,9 +96,9 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
for task in tasks:
try:
result = consume_task(task, service_instances)
- results.append(result)
+ results.append((task.video, result))
except:
logger.error(u'Error consuming task %r' % task, exc_info=True)
for service_instance in service_instances.itervalues():
service_instance.terminate()
- return results
+ return group_by_video(results)
diff --git a/libs/subliminal/async.py b/libs/subliminal/async.py
index e125bbf9..5d70b2b2 100755
--- a/libs/subliminal/async.py
+++ b/libs/subliminal/async.py
@@ -18,13 +18,14 @@
from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
create_download_tasks, group_by_video, key_subtitles)
-from guessit.language import ALL_LANGUAGES
+from .language import language_list, language_set, LANGUAGES
from .tasks import StopTask
import Queue
import logging
import threading
+__all__ = ['Worker', 'Pool']
logger = logging.getLogger(__name__)
@@ -111,7 +112,7 @@ class Pool(object):
def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
"""See :meth:`subliminal.list_subtitles`"""
services = services or SERVICES
- languages = set(languages or ALL_LANGUAGES)
+ languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
@@ -126,11 +127,11 @@ class Pool(object):
def download_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
"""See :meth:`subliminal.download_subtitles`"""
services = services or SERVICES
- languages = languages or list(ALL_LANGUAGES)
+ languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
- subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth, scan_filter)
+ subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
tasks = create_download_tasks(subtitles_by_video, multi)
@@ -138,4 +139,4 @@ class Pool(object):
self.tasks.put(task)
self.join()
results = self.collect()
- return results
+ return group_by_video(results)
diff --git a/libs/subliminal/cache.py b/libs/subliminal/cache.py
index f4d7d947..9add007a 100755
--- a/libs/subliminal/cache.py
+++ b/libs/subliminal/cache.py
@@ -15,25 +15,26 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
-import os.path
from collections import defaultdict
-import threading
from functools import wraps
import logging
+import os.path
+import threading
try:
import cPickle as pickle
except ImportError:
import pickle
+__all__ = ['Cache', 'cachedmethod']
logger = logging.getLogger(__name__)
class Cache(object):
"""A Cache object contains cached values for methods. It can have
- separate internal caches, one for each service.
- """
+ separate internal caches, one for each service
+ """
def __init__(self, cache_dir):
self.cache_dir = cache_dir
self.cache = defaultdict(dict)
@@ -100,9 +101,12 @@ class Cache(object):
def cachedmethod(function):
"""Decorator to make a method use the cache.
- WARNING: this can NOT be used with static functions, it has to be used on
- methods of some class."""
+ .. note::
+ This can NOT be used with static functions, it has to be used on
+ methods of some class
+
+ """
@wraps(function)
def cached(*args):
c = args[0].config.cache
@@ -126,7 +130,5 @@ def cachedmethod(function):
# meantime, but that's ok as we prefer to keep the latest value in
# the cache
func_cache[key] = result
-
return result
-
return cached
diff --git a/libs/subliminal/core.py b/libs/subliminal/core.py
index 9bbc1cd8..dc4af7f6 100755
--- a/libs/subliminal/core.py
+++ b/libs/subliminal/core.py
@@ -20,10 +20,9 @@ from .services import ServiceConfig
from .tasks import DownloadTask, ListTask
from .utils import get_keywords
from .videos import Episode, Movie, scan
-from guessit.language import lang_set
-import bs4
from collections import defaultdict
from itertools import groupby
+import bs4
import guessit
import logging
@@ -119,7 +118,7 @@ def consume_task(task, services=None):
:type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask`
:param dict services: mapping between the service name and an instance of this service
:return: the result of the task
- :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle` or :class:`~subliminal.subtitles.Subtitle`
+ :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle`
"""
if services is None:
@@ -134,12 +133,11 @@ def consume_task(task, services=None):
service = get_service(services, subtitle.service)
try:
service.download(subtitle)
- result = subtitle
+ result = [subtitle]
break
except DownloadFailedError:
logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
continue
-
if result is None:
logger.error(u'No subtitles could be downloaded for video %r' % task.video)
return result
@@ -225,6 +223,7 @@ def key_subtitles(subtitle, video, languages, services, order):
for sort_item in order:
if sort_item == LANGUAGE_INDEX:
key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1)
+ key += '{0:01d}'.format(subtitle.language == languages[languages.index(subtitle.language)])
elif sort_item == SERVICE_INDEX:
key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1)
elif sort_item == SERVICE_CONFIDENCE:
diff --git a/libs/subliminal/exceptions.py b/libs/subliminal/exceptions.py
index 93001110..66e3dd51 100755
--- a/libs/subliminal/exceptions.py
+++ b/libs/subliminal/exceptions.py
@@ -22,60 +22,11 @@ class Error(Exception):
pass
-class InvalidLanguageError(Error):
- """Exception raised when invalid language is submitted
-
- Attributes:
- language -- language that cause the error
- """
- def __init__(self, language):
- self.language = language
-
- def __str__(self):
- return self.language
-
-
-class MissingLanguageError(Error):
- """Exception raised when a missing language is found
-
- Attributes:
- language -- the missing language
- """
- def __init__(self, language):
- self.language = language
-
- def __str__(self):
- return self.language
-
-
-class InvalidServiceError(Error):
- """Exception raised when invalid service is submitted
-
- :param string service: service that causes the error
-
- """
- def __init__(self, service):
- self.service = service
-
- def __str__(self):
- return self.service
-
-
class ServiceError(Error):
""""Exception raised by services"""
pass
-class WrongTaskError(Error):
- """"Exception raised when invalid task is submitted"""
- pass
-
-
class DownloadFailedError(Error):
""""Exception raised when a download task has failed in service"""
pass
-
-
-class UnknownVideoError(Error):
- """"Exception raised when a video could not be identified"""
- pass
diff --git a/libs/subliminal/language.py b/libs/subliminal/language.py
new file mode 100755
index 00000000..e8cfa0a5
--- /dev/null
+++ b/libs/subliminal/language.py
@@ -0,0 +1,1032 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from .utils import to_unicode
+import re
+
+
+COUNTRIES = [('AF', 'AFG', '004', u'Afghanistan'),
+ ('AX', 'ALA', '248', u'Åland Islands'),
+ ('AL', 'ALB', '008', u'Albania'),
+ ('DZ', 'DZA', '012', u'Algeria'),
+ ('AS', 'ASM', '016', u'American Samoa'),
+ ('AD', 'AND', '020', u'Andorra'),
+ ('AO', 'AGO', '024', u'Angola'),
+ ('AI', 'AIA', '660', u'Anguilla'),
+ ('AQ', 'ATA', '010', u'Antarctica'),
+ ('AG', 'ATG', '028', u'Antigua and Barbuda'),
+ ('AR', 'ARG', '032', u'Argentina'),
+ ('AM', 'ARM', '051', u'Armenia'),
+ ('AW', 'ABW', '533', u'Aruba'),
+ ('AU', 'AUS', '036', u'Australia'),
+ ('AT', 'AUT', '040', u'Austria'),
+ ('AZ', 'AZE', '031', u'Azerbaijan'),
+ ('BS', 'BHS', '044', u'Bahamas'),
+ ('BH', 'BHR', '048', u'Bahrain'),
+ ('BD', 'BGD', '050', u'Bangladesh'),
+ ('BB', 'BRB', '052', u'Barbados'),
+ ('BY', 'BLR', '112', u'Belarus'),
+ ('BE', 'BEL', '056', u'Belgium'),
+ ('BZ', 'BLZ', '084', u'Belize'),
+ ('BJ', 'BEN', '204', u'Benin'),
+ ('BM', 'BMU', '060', u'Bermuda'),
+ ('BT', 'BTN', '064', u'Bhutan'),
+ ('BO', 'BOL', '068', u'Bolivia, Plurinational State of'),
+ ('BQ', 'BES', '535', u'Bonaire, Sint Eustatius and Saba'),
+ ('BA', 'BIH', '070', u'Bosnia and Herzegovina'),
+ ('BW', 'BWA', '072', u'Botswana'),
+ ('BV', 'BVT', '074', u'Bouvet Island'),
+ ('BR', 'BRA', '076', u'Brazil'),
+ ('IO', 'IOT', '086', u'British Indian Ocean Territory'),
+ ('BN', 'BRN', '096', u'Brunei Darussalam'),
+ ('BG', 'BGR', '100', u'Bulgaria'),
+ ('BF', 'BFA', '854', u'Burkina Faso'),
+ ('BI', 'BDI', '108', u'Burundi'),
+ ('KH', 'KHM', '116', u'Cambodia'),
+ ('CM', 'CMR', '120', u'Cameroon'),
+ ('CA', 'CAN', '124', u'Canada'),
+ ('CV', 'CPV', '132', u'Cape Verde'),
+ ('KY', 'CYM', '136', u'Cayman Islands'),
+ ('CF', 'CAF', '140', u'Central African Republic'),
+ ('TD', 'TCD', '148', u'Chad'),
+ ('CL', 'CHL', '152', u'Chile'),
+ ('CN', 'CHN', '156', u'China'),
+ ('CX', 'CXR', '162', u'Christmas Island'),
+ ('CC', 'CCK', '166', u'Cocos (Keeling) Islands'),
+ ('CO', 'COL', '170', u'Colombia'),
+ ('KM', 'COM', '174', u'Comoros'),
+ ('CG', 'COG', '178', u'Congo'),
+ ('CD', 'COD', '180', u'Congo, The Democratic Republic of the'),
+ ('CK', 'COK', '184', u'Cook Islands'),
+ ('CR', 'CRI', '188', u'Costa Rica'),
+ ('CI', 'CIV', '384', u'Côte d\'Ivoire'),
+ ('HR', 'HRV', '191', u'Croatia'),
+ ('CU', 'CUB', '192', u'Cuba'),
+ ('CW', 'CUW', '531', u'Curaçao'),
+ ('CY', 'CYP', '196', u'Cyprus'),
+ ('CZ', 'CZE', '203', u'Czech Republic'),
+ ('DK', 'DNK', '208', u'Denmark'),
+ ('DJ', 'DJI', '262', u'Djibouti'),
+ ('DM', 'DMA', '212', u'Dominica'),
+ ('DO', 'DOM', '214', u'Dominican Republic'),
+ ('EC', 'ECU', '218', u'Ecuador'),
+ ('EG', 'EGY', '818', u'Egypt'),
+ ('SV', 'SLV', '222', u'El Salvador'),
+ ('GQ', 'GNQ', '226', u'Equatorial Guinea'),
+ ('ER', 'ERI', '232', u'Eritrea'),
+ ('EE', 'EST', '233', u'Estonia'),
+ ('ET', 'ETH', '231', u'Ethiopia'),
+ ('FK', 'FLK', '238', u'Falkland Islands (Malvinas)'),
+ ('FO', 'FRO', '234', u'Faroe Islands'),
+ ('FJ', 'FJI', '242', u'Fiji'),
+ ('FI', 'FIN', '246', u'Finland'),
+ ('FR', 'FRA', '250', u'France'),
+ ('GF', 'GUF', '254', u'French Guiana'),
+ ('PF', 'PYF', '258', u'French Polynesia'),
+ ('TF', 'ATF', '260', u'French Southern Territories'),
+ ('GA', 'GAB', '266', u'Gabon'),
+ ('GM', 'GMB', '270', u'Gambia'),
+ ('GE', 'GEO', '268', u'Georgia'),
+ ('DE', 'DEU', '276', u'Germany'),
+ ('GH', 'GHA', '288', u'Ghana'),
+ ('GI', 'GIB', '292', u'Gibraltar'),
+ ('GR', 'GRC', '300', u'Greece'),
+ ('GL', 'GRL', '304', u'Greenland'),
+ ('GD', 'GRD', '308', u'Grenada'),
+ ('GP', 'GLP', '312', u'Guadeloupe'),
+ ('GU', 'GUM', '316', u'Guam'),
+ ('GT', 'GTM', '320', u'Guatemala'),
+ ('GG', 'GGY', '831', u'Guernsey'),
+ ('GN', 'GIN', '324', u'Guinea'),
+ ('GW', 'GNB', '624', u'Guinea-Bissau'),
+ ('GY', 'GUY', '328', u'Guyana'),
+ ('HT', 'HTI', '332', u'Haiti'),
+ ('HM', 'HMD', '334', u'Heard Island and McDonald Islands'),
+ ('VA', 'VAT', '336', u'Holy See (Vatican City State)'),
+ ('HN', 'HND', '340', u'Honduras'),
+ ('HK', 'HKG', '344', u'Hong Kong'),
+ ('HU', 'HUN', '348', u'Hungary'),
+ ('IS', 'ISL', '352', u'Iceland'),
+ ('IN', 'IND', '356', u'India'),
+ ('ID', 'IDN', '360', u'Indonesia'),
+ ('IR', 'IRN', '364', u'Iran, Islamic Republic of'),
+ ('IQ', 'IRQ', '368', u'Iraq'),
+ ('IE', 'IRL', '372', u'Ireland'),
+ ('IM', 'IMN', '833', u'Isle of Man'),
+ ('IL', 'ISR', '376', u'Israel'),
+ ('IT', 'ITA', '380', u'Italy'),
+ ('JM', 'JAM', '388', u'Jamaica'),
+ ('JP', 'JPN', '392', u'Japan'),
+ ('JE', 'JEY', '832', u'Jersey'),
+ ('JO', 'JOR', '400', u'Jordan'),
+ ('KZ', 'KAZ', '398', u'Kazakhstan'),
+ ('KE', 'KEN', '404', u'Kenya'),
+ ('KI', 'KIR', '296', u'Kiribati'),
+ ('KP', 'PRK', '408', u'Korea, Democratic People\'s Republic of'),
+ ('KR', 'KOR', '410', u'Korea, Republic of'),
+ ('KW', 'KWT', '414', u'Kuwait'),
+ ('KG', 'KGZ', '417', u'Kyrgyzstan'),
+ ('LA', 'LAO', '418', u'Lao People\'s Democratic Republic'),
+ ('LV', 'LVA', '428', u'Latvia'),
+ ('LB', 'LBN', '422', u'Lebanon'),
+ ('LS', 'LSO', '426', u'Lesotho'),
+ ('LR', 'LBR', '430', u'Liberia'),
+ ('LY', 'LBY', '434', u'Libya'),
+ ('LI', 'LIE', '438', u'Liechtenstein'),
+ ('LT', 'LTU', '440', u'Lithuania'),
+ ('LU', 'LUX', '442', u'Luxembourg'),
+ ('MO', 'MAC', '446', u'Macao'),
+ ('MK', 'MKD', '807', u'Macedonia, Republic of'),
+ ('MG', 'MDG', '450', u'Madagascar'),
+ ('MW', 'MWI', '454', u'Malawi'),
+ ('MY', 'MYS', '458', u'Malaysia'),
+ ('MV', 'MDV', '462', u'Maldives'),
+ ('ML', 'MLI', '466', u'Mali'),
+ ('MT', 'MLT', '470', u'Malta'),
+ ('MH', 'MHL', '584', u'Marshall Islands'),
+ ('MQ', 'MTQ', '474', u'Martinique'),
+ ('MR', 'MRT', '478', u'Mauritania'),
+ ('MU', 'MUS', '480', u'Mauritius'),
+ ('YT', 'MYT', '175', u'Mayotte'),
+ ('MX', 'MEX', '484', u'Mexico'),
+ ('FM', 'FSM', '583', u'Micronesia, Federated States of'),
+ ('MD', 'MDA', '498', u'Moldova, Republic of'),
+ ('MC', 'MCO', '492', u'Monaco'),
+ ('MN', 'MNG', '496', u'Mongolia'),
+ ('ME', 'MNE', '499', u'Montenegro'),
+ ('MS', 'MSR', '500', u'Montserrat'),
+ ('MA', 'MAR', '504', u'Morocco'),
+ ('MZ', 'MOZ', '508', u'Mozambique'),
+ ('MM', 'MMR', '104', u'Myanmar'),
+ ('NA', 'NAM', '516', u'Namibia'),
+ ('NR', 'NRU', '520', u'Nauru'),
+ ('NP', 'NPL', '524', u'Nepal'),
+ ('NL', 'NLD', '528', u'Netherlands'),
+ ('NC', 'NCL', '540', u'New Caledonia'),
+ ('NZ', 'NZL', '554', u'New Zealand'),
+ ('NI', 'NIC', '558', u'Nicaragua'),
+ ('NE', 'NER', '562', u'Niger'),
+ ('NG', 'NGA', '566', u'Nigeria'),
+ ('NU', 'NIU', '570', u'Niue'),
+ ('NF', 'NFK', '574', u'Norfolk Island'),
+ ('MP', 'MNP', '580', u'Northern Mariana Islands'),
+ ('NO', 'NOR', '578', u'Norway'),
+ ('OM', 'OMN', '512', u'Oman'),
+ ('PK', 'PAK', '586', u'Pakistan'),
+ ('PW', 'PLW', '585', u'Palau'),
+ ('PS', 'PSE', '275', u'Palestinian Territory, Occupied'),
+ ('PA', 'PAN', '591', u'Panama'),
+ ('PG', 'PNG', '598', u'Papua New Guinea'),
+ ('PY', 'PRY', '600', u'Paraguay'),
+ ('PE', 'PER', '604', u'Peru'),
+ ('PH', 'PHL', '608', u'Philippines'),
+ ('PN', 'PCN', '612', u'Pitcairn'),
+ ('PL', 'POL', '616', u'Poland'),
+ ('PT', 'PRT', '620', u'Portugal'),
+ ('PR', 'PRI', '630', u'Puerto Rico'),
+ ('QA', 'QAT', '634', u'Qatar'),
+ ('RE', 'REU', '638', u'Réunion'),
+ ('RO', 'ROU', '642', u'Romania'),
+ ('RU', 'RUS', '643', u'Russian Federation'),
+ ('RW', 'RWA', '646', u'Rwanda'),
+ ('BL', 'BLM', '652', u'Saint Barthélemy'),
+ ('SH', 'SHN', '654', u'Saint Helena, Ascension and Tristan da Cunha'),
+ ('KN', 'KNA', '659', u'Saint Kitts and Nevis'),
+ ('LC', 'LCA', '662', u'Saint Lucia'),
+ ('MF', 'MAF', '663', u'Saint Martin (French part)'),
+ ('PM', 'SPM', '666', u'Saint Pierre and Miquelon'),
+ ('VC', 'VCT', '670', u'Saint Vincent and the Grenadines'),
+ ('WS', 'WSM', '882', u'Samoa'),
+ ('SM', 'SMR', '674', u'San Marino'),
+ ('ST', 'STP', '678', u'Sao Tome and Principe'),
+ ('SA', 'SAU', '682', u'Saudi Arabia'),
+ ('SN', 'SEN', '686', u'Senegal'),
+ ('RS', 'SRB', '688', u'Serbia'),
+ ('SC', 'SYC', '690', u'Seychelles'),
+ ('SL', 'SLE', '694', u'Sierra Leone'),
+ ('SG', 'SGP', '702', u'Singapore'),
+ ('SX', 'SXM', '534', u'Sint Maarten (Dutch part)'),
+ ('SK', 'SVK', '703', u'Slovakia'),
+ ('SI', 'SVN', '705', u'Slovenia'),
+ ('SB', 'SLB', '090', u'Solomon Islands'),
+ ('SO', 'SOM', '706', u'Somalia'),
+ ('ZA', 'ZAF', '710', u'South Africa'),
+ ('GS', 'SGS', '239', u'South Georgia and the South Sandwich Islands'),
+ ('ES', 'ESP', '724', u'Spain'),
+ ('LK', 'LKA', '144', u'Sri Lanka'),
+ ('SD', 'SDN', '729', u'Sudan'),
+ ('SR', 'SUR', '740', u'Suriname'),
+ ('SS', 'SSD', '728', u'South Sudan'),
+ ('SJ', 'SJM', '744', u'Svalbard and Jan Mayen'),
+ ('SZ', 'SWZ', '748', u'Swaziland'),
+ ('SE', 'SWE', '752', u'Sweden'),
+ ('CH', 'CHE', '756', u'Switzerland'),
+ ('SY', 'SYR', '760', u'Syrian Arab Republic'),
+ ('TW', 'TWN', '158', u'Taiwan, Province of China'),
+ ('TJ', 'TJK', '762', u'Tajikistan'),
+ ('TZ', 'TZA', '834', u'Tanzania, United Republic of'),
+ ('TH', 'THA', '764', u'Thailand'),
+ ('TL', 'TLS', '626', u'Timor-Leste'),
+ ('TG', 'TGO', '768', u'Togo'),
+ ('TK', 'TKL', '772', u'Tokelau'),
+ ('TO', 'TON', '776', u'Tonga'),
+ ('TT', 'TTO', '780', u'Trinidad and Tobago'),
+ ('TN', 'TUN', '788', u'Tunisia'),
+ ('TR', 'TUR', '792', u'Turkey'),
+ ('TM', 'TKM', '795', u'Turkmenistan'),
+ ('TC', 'TCA', '796', u'Turks and Caicos Islands'),
+ ('TV', 'TUV', '798', u'Tuvalu'),
+ ('UG', 'UGA', '800', u'Uganda'),
+ ('UA', 'UKR', '804', u'Ukraine'),
+ ('AE', 'ARE', '784', u'United Arab Emirates'),
+ ('GB', 'GBR', '826', u'United Kingdom'),
+ ('US', 'USA', '840', u'United States'),
+ ('UM', 'UMI', '581', u'United States Minor Outlying Islands'),
+ ('UY', 'URY', '858', u'Uruguay'),
+ ('UZ', 'UZB', '860', u'Uzbekistan'),
+ ('VU', 'VUT', '548', u'Vanuatu'),
+ ('VE', 'VEN', '862', u'Venezuela, Bolivarian Republic of'),
+ ('VN', 'VNM', '704', u'Viet Nam'),
+ ('VG', 'VGB', '092', u'Virgin Islands, British'),
+ ('VI', 'VIR', '850', u'Virgin Islands, U.S.'),
+ ('WF', 'WLF', '876', u'Wallis and Futuna'),
+ ('EH', 'ESH', '732', u'Western Sahara'),
+ ('YE', 'YEM', '887', u'Yemen'),
+ ('ZM', 'ZMB', '894', u'Zambia'),
+ ('ZW', 'ZWE', '716', u'Zimbabwe')]
+
+
+LANGUAGES = [('aar', '', 'aa', u'Afar', u'afar'),
+ ('abk', '', 'ab', u'Abkhazian', u'abkhaze'),
+ ('ace', '', '', u'Achinese', u'aceh'),
+ ('ach', '', '', u'Acoli', u'acoli'),
+ ('ada', '', '', u'Adangme', u'adangme'),
+ ('ady', '', '', u'Adyghe; Adygei', u'adyghé'),
+ ('afa', '', '', u'Afro-Asiatic languages', u'afro-asiatiques, langues'),
+ ('afh', '', '', u'Afrihili', u'afrihili'),
+ ('afr', '', 'af', u'Afrikaans', u'afrikaans'),
+ ('ain', '', '', u'Ainu', u'aïnou'),
+ ('aka', '', 'ak', u'Akan', u'akan'),
+ ('akk', '', '', u'Akkadian', u'akkadien'),
+ ('alb', 'sqi', 'sq', u'Albanian', u'albanais'),
+ ('ale', '', '', u'Aleut', u'aléoute'),
+ ('alg', '', '', u'Algonquian languages', u'algonquines, langues'),
+ ('alt', '', '', u'Southern Altai', u'altai du Sud'),
+ ('amh', '', 'am', u'Amharic', u'amharique'),
+ ('ang', '', '', u'English, Old (ca.450-1100)', u'anglo-saxon (ca.450-1100)'),
+ ('anp', '', '', u'Angika', u'angika'),
+ ('apa', '', '', u'Apache languages', u'apaches, langues'),
+ ('ara', '', 'ar', u'Arabic', u'arabe'),
+ ('arc', '', '', u'Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)', u'araméen d\'empire (700-300 BCE)'),
+ ('arg', '', 'an', u'Aragonese', u'aragonais'),
+ ('arm', 'hye', 'hy', u'Armenian', u'arménien'),
+ ('arn', '', '', u'Mapudungun; Mapuche', u'mapudungun; mapuche; mapuce'),
+ ('arp', '', '', u'Arapaho', u'arapaho'),
+ ('art', '', '', u'Artificial languages', u'artificielles, langues'),
+ ('arw', '', '', u'Arawak', u'arawak'),
+ ('asm', '', 'as', u'Assamese', u'assamais'),
+ ('ast', '', '', u'Asturian; Bable; Leonese; Asturleonese', u'asturien; bable; léonais; asturoléonais'),
+ ('ath', '', '', u'Athapascan languages', u'athapascanes, langues'),
+ ('aus', '', '', u'Australian languages', u'australiennes, langues'),
+ ('ava', '', 'av', u'Avaric', u'avar'),
+ ('ave', '', 'ae', u'Avestan', u'avestique'),
+ ('awa', '', '', u'Awadhi', u'awadhi'),
+ ('aym', '', 'ay', u'Aymara', u'aymara'),
+ ('aze', '', 'az', u'Azerbaijani', u'azéri'),
+ ('bad', '', '', u'Banda languages', u'banda, langues'),
+ ('bai', '', '', u'Bamileke languages', u'bamiléké, langues'),
+ ('bak', '', 'ba', u'Bashkir', u'bachkir'),
+ ('bal', '', '', u'Baluchi', u'baloutchi'),
+ ('bam', '', 'bm', u'Bambara', u'bambara'),
+ ('ban', '', '', u'Balinese', u'balinais'),
+ ('baq', 'eus', 'eu', u'Basque', u'basque'),
+ ('bas', '', '', u'Basa', u'basa'),
+ ('bat', '', '', u'Baltic languages', u'baltes, langues'),
+ ('bej', '', '', u'Beja; Bedawiyet', u'bedja'),
+ ('bel', '', 'be', u'Belarusian', u'biélorusse'),
+ ('bem', '', '', u'Bemba', u'bemba'),
+ ('ben', '', 'bn', u'Bengali', u'bengali'),
+ ('ber', '', '', u'Berber languages', u'berbères, langues'),
+ ('bho', '', '', u'Bhojpuri', u'bhojpuri'),
+ ('bih', '', 'bh', u'Bihari languages', u'langues biharis'),
+ ('bik', '', '', u'Bikol', u'bikol'),
+ ('bin', '', '', u'Bini; Edo', u'bini; edo'),
+ ('bis', '', 'bi', u'Bislama', u'bichlamar'),
+ ('bla', '', '', u'Siksika', u'blackfoot'),
+ ('bnt', '', '', u'Bantu (Other)', u'bantoues, autres langues'),
+ ('bos', '', 'bs', u'Bosnian', u'bosniaque'),
+ ('bra', '', '', u'Braj', u'braj'),
+ ('bre', '', 'br', u'Breton', u'breton'),
+ ('btk', '', '', u'Batak languages', u'batak, langues'),
+ ('bua', '', '', u'Buriat', u'bouriate'),
+ ('bug', '', '', u'Buginese', u'bugi'),
+ ('bul', '', 'bg', u'Bulgarian', u'bulgare'),
+ ('bur', 'mya', 'my', u'Burmese', u'birman'),
+ ('byn', '', '', u'Blin; Bilin', u'blin; bilen'),
+ ('cad', '', '', u'Caddo', u'caddo'),
+ ('cai', '', '', u'Central American Indian languages', u'amérindiennes de L\'Amérique centrale, langues'),
+ ('car', '', '', u'Galibi Carib', u'karib; galibi; carib'),
+ ('cat', '', 'ca', u'Catalan; Valencian', u'catalan; valencien'),
+ ('cau', '', '', u'Caucasian languages', u'caucasiennes, langues'),
+ ('ceb', '', '', u'Cebuano', u'cebuano'),
+ ('cel', '', '', u'Celtic languages', u'celtiques, langues; celtes, langues'),
+ ('cha', '', 'ch', u'Chamorro', u'chamorro'),
+ ('chb', '', '', u'Chibcha', u'chibcha'),
+ ('che', '', 'ce', u'Chechen', u'tchétchène'),
+ ('chg', '', '', u'Chagatai', u'djaghataï'),
+ ('chi', 'zho', 'zh', u'Chinese', u'chinois'),
+ ('chk', '', '', u'Chuukese', u'chuuk'),
+ ('chm', '', '', u'Mari', u'mari'),
+ ('chn', '', '', u'Chinook jargon', u'chinook, jargon'),
+ ('cho', '', '', u'Choctaw', u'choctaw'),
+ ('chp', '', '', u'Chipewyan; Dene Suline', u'chipewyan'),
+ ('chr', '', '', u'Cherokee', u'cherokee'),
+ ('chu', '', 'cu', u'Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic', u'slavon d\'église; vieux slave; slavon liturgique; vieux bulgare'),
+ ('chv', '', 'cv', u'Chuvash', u'tchouvache'),
+ ('chy', '', '', u'Cheyenne', u'cheyenne'),
+ ('cmc', '', '', u'Chamic languages', u'chames, langues'),
+ ('cop', '', '', u'Coptic', u'copte'),
+ ('cor', '', 'kw', u'Cornish', u'cornique'),
+ ('cos', '', 'co', u'Corsican', u'corse'),
+ ('cpe', '', '', u'Creoles and pidgins, English based', u'créoles et pidgins basés sur l\'anglais'),
+ ('cpf', '', '', u'Creoles and pidgins, French-based ', u'créoles et pidgins basés sur le français'),
+ ('cpp', '', '', u'Creoles and pidgins, Portuguese-based ', u'créoles et pidgins basés sur le portugais'),
+ ('cre', '', 'cr', u'Cree', u'cree'),
+ ('crh', '', '', u'Crimean Tatar; Crimean Turkish', u'tatar de Crimé'),
+ ('crp', '', '', u'Creoles and pidgins ', u'créoles et pidgins'),
+ ('csb', '', '', u'Kashubian', u'kachoube'),
+ ('cus', '', '', u'Cushitic languages', u'couchitiques, langues'),
+ ('cze', 'ces', 'cs', u'Czech', u'tchèque'),
+ ('dak', '', '', u'Dakota', u'dakota'),
+ ('dan', '', 'da', u'Danish', u'danois'),
+ ('dar', '', '', u'Dargwa', u'dargwa'),
+ ('day', '', '', u'Land Dayak languages', u'dayak, langues'),
+ ('del', '', '', u'Delaware', u'delaware'),
+ ('den', '', '', u'Slave (Athapascan)', u'esclave (athapascan)'),
+ ('dgr', '', '', u'Dogrib', u'dogrib'),
+ ('din', '', '', u'Dinka', u'dinka'),
+ ('div', '', 'dv', u'Divehi; Dhivehi; Maldivian', u'maldivien'),
+ ('doi', '', '', u'Dogri', u'dogri'),
+ ('dra', '', '', u'Dravidian languages', u'dravidiennes, langues'),
+ ('dsb', '', '', u'Lower Sorbian', u'bas-sorabe'),
+ ('dua', '', '', u'Duala', u'douala'),
+ ('dum', '', '', u'Dutch, Middle (ca.1050-1350)', u'néerlandais moyen (ca. 1050-1350)'),
+ ('dut', 'nld', 'nl', u'Dutch; Flemish', u'néerlandais; flamand'),
+ ('dyu', '', '', u'Dyula', u'dioula'),
+ ('dzo', '', 'dz', u'Dzongkha', u'dzongkha'),
+ ('efi', '', '', u'Efik', u'efik'),
+ ('egy', '', '', u'Egyptian (Ancient)', u'égyptien'),
+ ('eka', '', '', u'Ekajuk', u'ekajuk'),
+ ('elx', '', '', u'Elamite', u'élamite'),
+ ('eng', '', 'en', u'English', u'anglais'),
+ ('enm', '', '', u'English, Middle (1100-1500)', u'anglais moyen (1100-1500)'),
+ ('epo', '', 'eo', u'Esperanto', u'espéranto'),
+ ('est', '', 'et', u'Estonian', u'estonien'),
+ ('ewe', '', 'ee', u'Ewe', u'éwé'),
+ ('ewo', '', '', u'Ewondo', u'éwondo'),
+ ('fan', '', '', u'Fang', u'fang'),
+ ('fao', '', 'fo', u'Faroese', u'féroïen'),
+ ('fat', '', '', u'Fanti', u'fanti'),
+ ('fij', '', 'fj', u'Fijian', u'fidjien'),
+ ('fil', '', '', u'Filipino; Pilipino', u'filipino; pilipino'),
+ ('fin', '', 'fi', u'Finnish', u'finnois'),
+ ('fiu', '', '', u'Finno-Ugrian languages', u'finno-ougriennes, langues'),
+ ('fon', '', '', u'Fon', u'fon'),
+ ('fre', 'fra', 'fr', u'French', u'français'),
+ ('frm', '', '', u'French, Middle (ca.1400-1600)', u'français moyen (1400-1600)'),
+ ('fro', '', '', u'French, Old (842-ca.1400)', u'français ancien (842-ca.1400)'),
+ ('frr', '', '', u'Northern Frisian', u'frison septentrional'),
+ ('frs', '', '', u'Eastern Frisian', u'frison oriental'),
+ ('fry', '', 'fy', u'Western Frisian', u'frison occidental'),
+ ('ful', '', 'ff', u'Fulah', u'peul'),
+ ('fur', '', '', u'Friulian', u'frioulan'),
+ ('gaa', '', '', u'Ga', u'ga'),
+ ('gay', '', '', u'Gayo', u'gayo'),
+ ('gba', '', '', u'Gbaya', u'gbaya'),
+ ('gem', '', '', u'Germanic languages', u'germaniques, langues'),
+ ('geo', 'kat', 'ka', u'Georgian', u'géorgien'),
+ ('ger', 'deu', 'de', u'German', u'allemand'),
+ ('gez', '', '', u'Geez', u'guèze'),
+ ('gil', '', '', u'Gilbertese', u'kiribati'),
+ ('gla', '', 'gd', u'Gaelic; Scottish Gaelic', u'gaélique; gaélique écossais'),
+ ('gle', '', 'ga', u'Irish', u'irlandais'),
+ ('glg', '', 'gl', u'Galician', u'galicien'),
+ ('glv', '', 'gv', u'Manx', u'manx; mannois'),
+ ('gmh', '', '', u'German, Middle High (ca.1050-1500)', u'allemand, moyen haut (ca. 1050-1500)'),
+ ('goh', '', '', u'German, Old High (ca.750-1050)', u'allemand, vieux haut (ca. 750-1050)'),
+ ('gon', '', '', u'Gondi', u'gond'),
+ ('gor', '', '', u'Gorontalo', u'gorontalo'),
+ ('got', '', '', u'Gothic', u'gothique'),
+ ('grb', '', '', u'Grebo', u'grebo'),
+ ('grc', '', '', u'Greek, Ancient (to 1453)', u'grec ancien (jusqu\'à 1453)'),
+ ('gre', 'ell', 'el', u'Greek, Modern (1453-)', u'grec moderne (après 1453)'),
+ ('grn', '', 'gn', u'Guarani', u'guarani'),
+ ('gsw', '', '', u'Swiss German; Alemannic; Alsatian', u'suisse alémanique; alémanique; alsacien'),
+ ('guj', '', 'gu', u'Gujarati', u'goudjrati'),
+ ('gwi', '', '', u'Gwich\'in', u'gwich\'in'),
+ ('hai', '', '', u'Haida', u'haida'),
+ ('hat', '', 'ht', u'Haitian; Haitian Creole', u'haïtien; créole haïtien'),
+ ('hau', '', 'ha', u'Hausa', u'haoussa'),
+ ('haw', '', '', u'Hawaiian', u'hawaïen'),
+ ('heb', '', 'he', u'Hebrew', u'hébreu'),
+ ('her', '', 'hz', u'Herero', u'herero'),
+ ('hil', '', '', u'Hiligaynon', u'hiligaynon'),
+ ('him', '', '', u'Himachali languages; Western Pahari languages', u'langues himachalis; langues paharis occidentales'),
+ ('hin', '', 'hi', u'Hindi', u'hindi'),
+ ('hit', '', '', u'Hittite', u'hittite'),
+ ('hmn', '', '', u'Hmong; Mong', u'hmong'),
+ ('hmo', '', 'ho', u'Hiri Motu', u'hiri motu'),
+ ('hrv', '', 'hr', u'Croatian', u'croate'),
+ ('hsb', '', '', u'Upper Sorbian', u'haut-sorabe'),
+ ('hun', '', 'hu', u'Hungarian', u'hongrois'),
+ ('hup', '', '', u'Hupa', u'hupa'),
+ ('iba', '', '', u'Iban', u'iban'),
+ ('ibo', '', 'ig', u'Igbo', u'igbo'),
+ ('ice', 'isl', 'is', u'Icelandic', u'islandais'),
+ ('ido', '', 'io', u'Ido', u'ido'),
+ ('iii', '', 'ii', u'Sichuan Yi; Nuosu', u'yi de Sichuan'),
+ ('ijo', '', '', u'Ijo languages', u'ijo, langues'),
+ ('iku', '', 'iu', u'Inuktitut', u'inuktitut'),
+ ('ile', '', 'ie', u'Interlingue; Occidental', u'interlingue'),
+ ('ilo', '', '', u'Iloko', u'ilocano'),
+ ('ina', '', 'ia', u'Interlingua (International Auxiliary Language Association)', u'interlingua (langue auxiliaire internationale)'),
+ ('inc', '', '', u'Indic languages', u'indo-aryennes, langues'),
+ ('ind', '', 'id', u'Indonesian', u'indonésien'),
+ ('ine', '', '', u'Indo-European languages', u'indo-européennes, langues'),
+ ('inh', '', '', u'Ingush', u'ingouche'),
+ ('ipk', '', 'ik', u'Inupiaq', u'inupiaq'),
+ ('ira', '', '', u'Iranian languages', u'iraniennes, langues'),
+ ('iro', '', '', u'Iroquoian languages', u'iroquoises, langues'),
+ ('ita', '', 'it', u'Italian', u'italien'),
+ ('jav', '', 'jv', u'Javanese', u'javanais'),
+ ('jbo', '', '', u'Lojban', u'lojban'),
+ ('jpn', '', 'ja', u'Japanese', u'japonais'),
+ ('jpr', '', '', u'Judeo-Persian', u'judéo-persan'),
+ ('jrb', '', '', u'Judeo-Arabic', u'judéo-arabe'),
+ ('kaa', '', '', u'Kara-Kalpak', u'karakalpak'),
+ ('kab', '', '', u'Kabyle', u'kabyle'),
+ ('kac', '', '', u'Kachin; Jingpho', u'kachin; jingpho'),
+ ('kal', '', 'kl', u'Kalaallisut; Greenlandic', u'groenlandais'),
+ ('kam', '', '', u'Kamba', u'kamba'),
+ ('kan', '', 'kn', u'Kannada', u'kannada'),
+ ('kar', '', '', u'Karen languages', u'karen, langues'),
+ ('kas', '', 'ks', u'Kashmiri', u'kashmiri'),
+ ('kau', '', 'kr', u'Kanuri', u'kanouri'),
+ ('kaw', '', '', u'Kawi', u'kawi'),
+ ('kaz', '', 'kk', u'Kazakh', u'kazakh'),
+ ('kbd', '', '', u'Kabardian', u'kabardien'),
+ ('kha', '', '', u'Khasi', u'khasi'),
+ ('khi', '', '', u'Khoisan languages', u'khoïsan, langues'),
+ ('khm', '', 'km', u'Central Khmer', u'khmer central'),
+ ('kho', '', '', u'Khotanese; Sakan', u'khotanais; sakan'),
+ ('kik', '', 'ki', u'Kikuyu; Gikuyu', u'kikuyu'),
+ ('kin', '', 'rw', u'Kinyarwanda', u'rwanda'),
+ ('kir', '', 'ky', u'Kirghiz; Kyrgyz', u'kirghiz'),
+ ('kmb', '', '', u'Kimbundu', u'kimbundu'),
+ ('kok', '', '', u'Konkani', u'konkani'),
+ ('kom', '', 'kv', u'Komi', u'kom'),
+ ('kon', '', 'kg', u'Kongo', u'kongo'),
+ ('kor', '', 'ko', u'Korean', u'coréen'),
+ ('kos', '', '', u'Kosraean', u'kosrae'),
+ ('kpe', '', '', u'Kpelle', u'kpellé'),
+ ('krc', '', '', u'Karachay-Balkar', u'karatchai balkar'),
+ ('krl', '', '', u'Karelian', u'carélien'),
+ ('kro', '', '', u'Kru languages', u'krou, langues'),
+ ('kru', '', '', u'Kurukh', u'kurukh'),
+ ('kua', '', 'kj', u'Kuanyama; Kwanyama', u'kuanyama; kwanyama'),
+ ('kum', '', '', u'Kumyk', u'koumyk'),
+ ('kur', '', 'ku', u'Kurdish', u'kurde'),
+ ('kut', '', '', u'Kutenai', u'kutenai'),
+ ('lad', '', '', u'Ladino', u'judéo-espagnol'),
+ ('lah', '', '', u'Lahnda', u'lahnda'),
+ ('lam', '', '', u'Lamba', u'lamba'),
+ ('lao', '', 'lo', u'Lao', u'lao'),
+ ('lat', '', 'la', u'Latin', u'latin'),
+ ('lav', '', 'lv', u'Latvian', u'letton'),
+ ('lez', '', '', u'Lezghian', u'lezghien'),
+ ('lim', '', 'li', u'Limburgan; Limburger; Limburgish', u'limbourgeois'),
+ ('lin', '', 'ln', u'Lingala', u'lingala'),
+ ('lit', '', 'lt', u'Lithuanian', u'lituanien'),
+ ('lol', '', '', u'Mongo', u'mongo'),
+ ('loz', '', '', u'Lozi', u'lozi'),
+ ('ltz', '', 'lb', u'Luxembourgish; Letzeburgesch', u'luxembourgeois'),
+ ('lua', '', '', u'Luba-Lulua', u'luba-lulua'),
+ ('lub', '', 'lu', u'Luba-Katanga', u'luba-katanga'),
+ ('lug', '', 'lg', u'Ganda', u'ganda'),
+ ('lui', '', '', u'Luiseno', u'luiseno'),
+ ('lun', '', '', u'Lunda', u'lunda'),
+ ('luo', '', '', u'Luo (Kenya and Tanzania)', u'luo (Kenya et Tanzanie)'),
+ ('lus', '', '', u'Lushai', u'lushai'),
+ ('mac', 'mkd', 'mk', u'Macedonian', u'macédonien'),
+ ('mad', '', '', u'Madurese', u'madourais'),
+ ('mag', '', '', u'Magahi', u'magahi'),
+ ('mah', '', 'mh', u'Marshallese', u'marshall'),
+ ('mai', '', '', u'Maithili', u'maithili'),
+ ('mak', '', '', u'Makasar', u'makassar'),
+ ('mal', '', 'ml', u'Malayalam', u'malayalam'),
+ ('man', '', '', u'Mandingo', u'mandingue'),
+ ('mao', 'mri', 'mi', u'Maori', u'maori'),
+ ('map', '', '', u'Austronesian languages', u'austronésiennes, langues'),
+ ('mar', '', 'mr', u'Marathi', u'marathe'),
+ ('mas', '', '', u'Masai', u'massaï'),
+ ('may', 'msa', 'ms', u'Malay', u'malais'),
+ ('mdf', '', '', u'Moksha', u'moksa'),
+ ('mdr', '', '', u'Mandar', u'mandar'),
+ ('men', '', '', u'Mende', u'mendé'),
+ ('mga', '', '', u'Irish, Middle (900-1200)', u'irlandais moyen (900-1200)'),
+ ('mic', '', '', u'Mi\'kmaq; Micmac', u'mi\'kmaq; micmac'),
+ ('min', '', '', u'Minangkabau', u'minangkabau'),
+ ('mkh', '', '', u'Mon-Khmer languages', u'môn-khmer, langues'),
+ ('mlg', '', 'mg', u'Malagasy', u'malgache'),
+ ('mlt', '', 'mt', u'Maltese', u'maltais'),
+ ('mnc', '', '', u'Manchu', u'mandchou'),
+ ('mni', '', '', u'Manipuri', u'manipuri'),
+ ('mno', '', '', u'Manobo languages', u'manobo, langues'),
+ ('moh', '', '', u'Mohawk', u'mohawk'),
+ ('mon', '', 'mn', u'Mongolian', u'mongol'),
+ ('mos', '', '', u'Mossi', u'moré'),
+ ('mun', '', '', u'Munda languages', u'mounda, langues'),
+ ('mus', '', '', u'Creek', u'muskogee'),
+ ('mwl', '', '', u'Mirandese', u'mirandais'),
+ ('mwr', '', '', u'Marwari', u'marvari'),
+ ('myn', '', '', u'Mayan languages', u'maya, langues'),
+ ('myv', '', '', u'Erzya', u'erza'),
+ ('nah', '', '', u'Nahuatl languages', u'nahuatl, langues'),
+ ('nai', '', '', u'North American Indian languages', u'nord-amérindiennes, langues'),
+ ('nap', '', '', u'Neapolitan', u'napolitain'),
+ ('nau', '', 'na', u'Nauru', u'nauruan'),
+ ('nav', '', 'nv', u'Navajo; Navaho', u'navaho'),
+ ('nbl', '', 'nr', u'Ndebele, South; South Ndebele', u'ndébélé du Sud'),
+ ('nde', '', 'nd', u'Ndebele, North; North Ndebele', u'ndébélé du Nord'),
+ ('ndo', '', 'ng', u'Ndonga', u'ndonga'),
+ ('nds', '', '', u'Low German; Low Saxon; German, Low; Saxon, Low', u'bas allemand; bas saxon; allemand, bas; saxon, bas'),
+ ('nep', '', 'ne', u'Nepali', u'népalais'),
+ ('new', '', '', u'Nepal Bhasa; Newari', u'nepal bhasa; newari'),
+ ('nia', '', '', u'Nias', u'nias'),
+ ('nic', '', '', u'Niger-Kordofanian languages', u'nigéro-kordofaniennes, langues'),
+ ('niu', '', '', u'Niuean', u'niué'),
+ ('nno', '', 'nn', u'Norwegian Nynorsk; Nynorsk, Norwegian', u'norvégien nynorsk; nynorsk, norvégien'),
+ ('nob', '', 'nb', u'Bokmål, Norwegian; Norwegian Bokmål', u'norvégien bokmål'),
+ ('nog', '', '', u'Nogai', u'nogaï; nogay'),
+ ('non', '', '', u'Norse, Old', u'norrois, vieux'),
+ ('nor', '', 'no', u'Norwegian', u'norvégien'),
+ ('nqo', '', '', u'N\'Ko', u'n\'ko'),
+ ('nso', '', '', u'Pedi; Sepedi; Northern Sotho', u'pedi; sepedi; sotho du Nord'),
+ ('nub', '', '', u'Nubian languages', u'nubiennes, langues'),
+ ('nwc', '', '', u'Classical Newari; Old Newari; Classical Nepal Bhasa', u'newari classique'),
+ ('nya', '', 'ny', u'Chichewa; Chewa; Nyanja', u'chichewa; chewa; nyanja'),
+ ('nym', '', '', u'Nyamwezi', u'nyamwezi'),
+ ('nyn', '', '', u'Nyankole', u'nyankolé'),
+ ('nyo', '', '', u'Nyoro', u'nyoro'),
+ ('nzi', '', '', u'Nzima', u'nzema'),
+ ('oci', '', 'oc', u'Occitan (post 1500); Provençal', u'occitan (après 1500); provençal'),
+ ('oji', '', 'oj', u'Ojibwa', u'ojibwa'),
+ ('ori', '', 'or', u'Oriya', u'oriya'),
+ ('orm', '', 'om', u'Oromo', u'galla'),
+ ('osa', '', '', u'Osage', u'osage'),
+ ('oss', '', 'os', u'Ossetian; Ossetic', u'ossète'),
+ ('ota', '', '', u'Turkish, Ottoman (1500-1928)', u'turc ottoman (1500-1928)'),
+ ('oto', '', '', u'Otomian languages', u'otomi, langues'),
+ ('paa', '', '', u'Papuan languages', u'papoues, langues'),
+ ('pag', '', '', u'Pangasinan', u'pangasinan'),
+ ('pal', '', '', u'Pahlavi', u'pahlavi'),
+ ('pam', '', '', u'Pampanga; Kapampangan', u'pampangan'),
+ ('pan', '', 'pa', u'Panjabi; Punjabi', u'pendjabi'),
+ ('pap', '', '', u'Papiamento', u'papiamento'),
+ ('pau', '', '', u'Palauan', u'palau'),
+ ('peo', '', '', u'Persian, Old (ca.600-400 B.C.)', u'perse, vieux (ca. 600-400 av. J.-C.)'),
+ ('per', 'fas', 'fa', u'Persian', u'persan'),
+ ('phi', '', '', u'Philippine languages', u'philippines, langues'),
+ ('phn', '', '', u'Phoenician', u'phénicien'),
+ ('pli', '', 'pi', u'Pali', u'pali'),
+ ('pol', '', 'pl', u'Polish', u'polonais'),
+ ('pon', '', '', u'Pohnpeian', u'pohnpei'),
+ ('por', '', 'pt', u'Portuguese', u'portugais'),
+ ('pra', '', '', u'Prakrit languages', u'prâkrit, langues'),
+ ('pro', '', '', u'Provençal, Old (to 1500)', u'provençal ancien (jusqu\'à 1500)'),
+ ('pus', '', 'ps', u'Pushto; Pashto', u'pachto'),
+ ('que', '', 'qu', u'Quechua', u'quechua'),
+ ('raj', '', '', u'Rajasthani', u'rajasthani'),
+ ('rap', '', '', u'Rapanui', u'rapanui'),
+ ('rar', '', '', u'Rarotongan; Cook Islands Maori', u'rarotonga; maori des îles Cook'),
+ ('roa', '', '', u'Romance languages', u'romanes, langues'),
+ ('roh', '', 'rm', u'Romansh', u'romanche'),
+ ('rom', '', '', u'Romany', u'tsigane'),
+ ('rum', 'ron', 'ro', u'Romanian; Moldavian; Moldovan', u'roumain; moldave'),
+ ('run', '', 'rn', u'Rundi', u'rundi'),
+ ('rup', '', '', u'Aromanian; Arumanian; Macedo-Romanian', u'aroumain; macédo-roumain'),
+ ('rus', '', 'ru', u'Russian', u'russe'),
+ ('sad', '', '', u'Sandawe', u'sandawe'),
+ ('sag', '', 'sg', u'Sango', u'sango'),
+ ('sah', '', '', u'Yakut', u'iakoute'),
+ ('sai', '', '', u'South American Indian (Other)', u'indiennes d\'Amérique du Sud, autres langues'),
+ ('sal', '', '', u'Salishan languages', u'salishennes, langues'),
+ ('sam', '', '', u'Samaritan Aramaic', u'samaritain'),
+ ('san', '', 'sa', u'Sanskrit', u'sanskrit'),
+ ('sas', '', '', u'Sasak', u'sasak'),
+ ('sat', '', '', u'Santali', u'santal'),
+ ('scn', '', '', u'Sicilian', u'sicilien'),
+ ('sco', '', '', u'Scots', u'écossais'),
+ ('sel', '', '', u'Selkup', u'selkoupe'),
+ ('sem', '', '', u'Semitic languages', u'sémitiques, langues'),
+ ('sga', '', '', u'Irish, Old (to 900)', u'irlandais ancien (jusqu\'à 900)'),
+ ('sgn', '', '', u'Sign Languages', u'langues des signes'),
+ ('shn', '', '', u'Shan', u'chan'),
+ ('sid', '', '', u'Sidamo', u'sidamo'),
+ ('sin', '', 'si', u'Sinhala; Sinhalese', u'singhalais'),
+ ('sio', '', '', u'Siouan languages', u'sioux, langues'),
+ ('sit', '', '', u'Sino-Tibetan languages', u'sino-tibétaines, langues'),
+ ('sla', '', '', u'Slavic languages', u'slaves, langues'),
+ ('slo', 'slk', 'sk', u'Slovak', u'slovaque'),
+ ('slv', '', 'sl', u'Slovenian', u'slovène'),
+ ('sma', '', '', u'Southern Sami', u'sami du Sud'),
+ ('sme', '', 'se', u'Northern Sami', u'sami du Nord'),
+ ('smi', '', '', u'Sami languages', u'sames, langues'),
+ ('smj', '', '', u'Lule Sami', u'sami de Lule'),
+ ('smn', '', '', u'Inari Sami', u'sami d\'Inari'),
+ ('smo', '', 'sm', u'Samoan', u'samoan'),
+ ('sms', '', '', u'Skolt Sami', u'sami skolt'),
+ ('sna', '', 'sn', u'Shona', u'shona'),
+ ('snd', '', 'sd', u'Sindhi', u'sindhi'),
+ ('snk', '', '', u'Soninke', u'soninké'),
+ ('sog', '', '', u'Sogdian', u'sogdien'),
+ ('som', '', 'so', u'Somali', u'somali'),
+ ('son', '', '', u'Songhai languages', u'songhai, langues'),
+ ('sot', '', 'st', u'Sotho, Southern', u'sotho du Sud'),
+ ('spa', '', 'es', u'Spanish; Castilian', u'espagnol; castillan'),
+ ('srd', '', 'sc', u'Sardinian', u'sarde'),
+ ('srn', '', '', u'Sranan Tongo', u'sranan tongo'),
+ ('srp', '', 'sr', u'Serbian', u'serbe'),
+ ('srr', '', '', u'Serer', u'sérère'),
+ ('ssa', '', '', u'Nilo-Saharan languages', u'nilo-sahariennes, langues'),
+ ('ssw', '', 'ss', u'Swati', u'swati'),
+ ('suk', '', '', u'Sukuma', u'sukuma'),
+ ('sun', '', 'su', u'Sundanese', u'soundanais'),
+ ('sus', '', '', u'Susu', u'soussou'),
+ ('sux', '', '', u'Sumerian', u'sumérien'),
+ ('swa', '', 'sw', u'Swahili', u'swahili'),
+ ('swe', '', 'sv', u'Swedish', u'suédois'),
+ ('syc', '', '', u'Classical Syriac', u'syriaque classique'),
+ ('syr', '', '', u'Syriac', u'syriaque'),
+ ('tah', '', 'ty', u'Tahitian', u'tahitien'),
+ ('tai', '', '', u'Tai languages', u'tai, langues'),
+ ('tam', '', 'ta', u'Tamil', u'tamoul'),
+ ('tat', '', 'tt', u'Tatar', u'tatar'),
+ ('tel', '', 'te', u'Telugu', u'télougou'),
+ ('tem', '', '', u'Timne', u'temne'),
+ ('ter', '', '', u'Tereno', u'tereno'),
+ ('tet', '', '', u'Tetum', u'tetum'),
+ ('tgk', '', 'tg', u'Tajik', u'tadjik'),
+ ('tgl', '', 'tl', u'Tagalog', u'tagalog'),
+ ('tha', '', 'th', u'Thai', u'thaï'),
+ ('tib', 'bod', 'bo', u'Tibetan', u'tibétain'),
+ ('tig', '', '', u'Tigre', u'tigré'),
+ ('tir', '', 'ti', u'Tigrinya', u'tigrigna'),
+ ('tiv', '', '', u'Tiv', u'tiv'),
+ ('tkl', '', '', u'Tokelau', u'tokelau'),
+ ('tlh', '', '', u'Klingon; tlhIngan-Hol', u'klingon'),
+ ('tli', '', '', u'Tlingit', u'tlingit'),
+ ('tmh', '', '', u'Tamashek', u'tamacheq'),
+ ('tog', '', '', u'Tonga (Nyasa)', u'tonga (Nyasa)'),
+ ('ton', '', 'to', u'Tonga (Tonga Islands)', u'tongan (Îles Tonga)'),
+ ('tpi', '', '', u'Tok Pisin', u'tok pisin'),
+ ('tsi', '', '', u'Tsimshian', u'tsimshian'),
+ ('tsn', '', 'tn', u'Tswana', u'tswana'),
+ ('tso', '', 'ts', u'Tsonga', u'tsonga'),
+ ('tuk', '', 'tk', u'Turkmen', u'turkmène'),
+ ('tum', '', '', u'Tumbuka', u'tumbuka'),
+ ('tup', '', '', u'Tupi languages', u'tupi, langues'),
+ ('tur', '', 'tr', u'Turkish', u'turc'),
+ ('tut', '', '', u'Altaic languages', u'altaïques, langues'),
+ ('tvl', '', '', u'Tuvalu', u'tuvalu'),
+ ('twi', '', 'tw', u'Twi', u'twi'),
+ ('tyv', '', '', u'Tuvinian', u'touva'),
+ ('udm', '', '', u'Udmurt', u'oudmourte'),
+ ('uga', '', '', u'Ugaritic', u'ougaritique'),
+ ('uig', '', 'ug', u'Uighur; Uyghur', u'ouïgour'),
+ ('ukr', '', 'uk', u'Ukrainian', u'ukrainien'),
+ ('umb', '', '', u'Umbundu', u'umbundu'),
+ ('und', '', '', u'Undetermined', u'indéterminée'),
+ ('urd', '', 'ur', u'Urdu', u'ourdou'),
+ ('uzb', '', 'uz', u'Uzbek', u'ouszbek'),
+ ('vai', '', '', u'Vai', u'vaï'),
+ ('ven', '', 've', u'Venda', u'venda'),
+ ('vie', '', 'vi', u'Vietnamese', u'vietnamien'),
+ ('vol', '', 'vo', u'Volapük', u'volapük'),
+ ('vot', '', '', u'Votic', u'vote'),
+ ('wak', '', '', u'Wakashan languages', u'wakashanes, langues'),
+ ('wal', '', '', u'Walamo', u'walamo'),
+ ('war', '', '', u'Waray', u'waray'),
+ ('was', '', '', u'Washo', u'washo'),
+ ('wel', 'cym', 'cy', u'Welsh', u'gallois'),
+ ('wen', '', '', u'Sorbian languages', u'sorabes, langues'),
+ ('wln', '', 'wa', u'Walloon', u'wallon'),
+ ('wol', '', 'wo', u'Wolof', u'wolof'),
+ ('xal', '', '', u'Kalmyk; Oirat', u'kalmouk; oïrat'),
+ ('xho', '', 'xh', u'Xhosa', u'xhosa'),
+ ('yao', '', '', u'Yao', u'yao'),
+ ('yap', '', '', u'Yapese', u'yapois'),
+ ('yid', '', 'yi', u'Yiddish', u'yiddish'),
+ ('yor', '', 'yo', u'Yoruba', u'yoruba'),
+ ('ypk', '', '', u'Yupik languages', u'yupik, langues'),
+ ('zap', '', '', u'Zapotec', u'zapotèque'),
+ ('zbl', '', '', u'Blissymbols; Blissymbolics; Bliss', u'symboles Bliss; Bliss'),
+ ('zen', '', '', u'Zenaga', u'zenaga'),
+ ('zha', '', 'za', u'Zhuang; Chuang', u'zhuang; chuang'),
+ ('znd', '', '', u'Zande languages', u'zandé, langues'),
+ ('zul', '', 'zu', u'Zulu', u'zoulou'),
+ ('zun', '', '', u'Zuni', u'zuni'),
+ ('zza', '', '', u'Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki', u'zaza; dimili; dimli; kirdki; kirmanjki; zazaki')]
+
+
+class Country(object):
+ """Country according to ISO-3166
+
+ :param string country: country name, alpha2 code, alpha3 code or numeric code
+ :param list countries: all countries
+ :type countries: see :data:`~subliminal.language.COUNTRIES`
+
+ """
+ def __init__(self, country, countries=None):
+ countries = countries or COUNTRIES
+ country = to_unicode(country.strip().lower())
+ country_tuple = None
+
+ # Try to find the country
+ if len(country) == 2:
+ country_tuple = dict((c[0].lower(), c) for c in countries).get(country)
+ elif len(country) == 3 and not country.isdigit():
+ country_tuple = dict((c[1].lower(), c) for c in countries).get(country)
+ elif len(country) == 3 and country.isdigit():
+ country_tuple = dict((c[2].lower(), c) for c in countries).get(country)
+ if country_tuple is None:
+ country_tuple = dict((c[3].lower(), c) for c in countries).get(country)
+
+ # Raise ValueError if nothing is found
+ if country_tuple is None:
+ raise ValueError('Country %s does not exist' % country)
+
+ # Set default attrs
+ self.alpha2 = country_tuple[0]
+ self.alpha3 = country_tuple[1]
+ self.numeric = country_tuple[2]
+ self.name = country_tuple[3]
+
+ def __hash__(self):
+ return hash(self.alpha3)
+
+ def __eq__(self, other):
+ if isinstance(other, Country):
+ return self.alpha3 == other.alpha3
+ return False
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __unicode__(self):
+ return self.name
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ def __repr__(self):
+ return 'Country(%s)' % self
+
+
+class Language(object):
+ """Language according to ISO-639
+
+ :param string language: language name (english or french), alpha2 code, alpha3 code, terminologic code or numeric code, eventually with a country
+ :param country: country of the language
+ :type country: :class:`Country` or string
+ :param languages: all languages
+ :type languages: see :data:`~subliminal.language.LANGUAGES`
+ :param countries: all countries
+ :type countries: see :data:`~subliminal.language.COUNTRIES`
+ :param bool strict: whether to raise a ValueError on unknown language or not
+
+ :class:`Language` implements the inclusion test, with the ``in`` keyword::
+
+ >>> Language('pt-BR') in Language('pt') # Portuguese (Brazil) is included in Portuguese
+ True
+ >>> Language('pt') in Language('pt-BR') # Portuguese is not included in Portuguese (Brazil)
+ False
+
+ """
+ with_country_regexps = [re.compile('(.*)\((.*)\)'), re.compile('(.*)[-_](.*)')]
+
+ def __init__(self, language, country=None, languages=None, countries=None, strict=True):
+ languages = languages or LANGUAGES
+ countries = countries or COUNTRIES
+
+ # Get the country
+ self.country = None
+ if isinstance(country, Country):
+ self.country = country
+ elif isinstance(country, basestring):
+ self.country = Country(country, countries)
+
+ # Language + Country format
+ #TODO: Improve this part
+ for regexp in [r.match(language) for r in self.with_country_regexps]:
+ if regexp:
+ language = regexp.group(1)
+ self.country = Country(regexp.group(2), countries)
+ break
+
+ # Try to find the language
+ language = to_unicode(language.strip().lower())
+ language_tuple = None
+ if len(language) == 2:
+ language_tuple = dict((l[2].lower(), l) for l in languages).get(language)
+ elif len(language) == 3:
+ language_tuple = dict((l[0].lower(), l) for l in languages).get(language)
+ if language_tuple is None:
+ language_tuple = dict((l[1].lower(), l) for l in languages).get(language)
+ if language_tuple is None:
+ language_tuple = dict((l[3].split('; ')[0].lower(), l) for l in languages).get(language)
+ if language_tuple is None:
+ language_tuple = dict((l[4].split('; ')[0].lower(), l) for l in languages).get(language)
+
+ # Raise ValueError if strict or continue with Undetermined
+ if language_tuple is None:
+ if strict:
+ raise ValueError('Language %s does not exist' % language)
+ language_tuple = dict((l[0].lower(), l) for l in languages).get('und')
+
+ # Set attributes
+ self.alpha2 = language_tuple[2]
+ self.alpha3 = language_tuple[0]
+ self.terminologic = language_tuple[1]
+ self.name = language_tuple[3]
+ self.french_name = language_tuple[4]
+
+ def __hash__(self):
+ if self.country is None:
+ return hash(self.alpha3)
+ return hash(self.alpha3 + self.country.alpha3)
+
+ def __eq__(self, other):
+ if isinstance(other, Language):
+ return self.alpha3 == other.alpha3 and self.country == other.country
+ return False
+
+ def __contains__(self, item):
+ if isinstance(item, Language):
+ if self == item:
+ return True
+ if self.country is None:
+ return self.alpha3 == item.alpha3
+ return False
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __nonzero__(self):
+ return self.alpha3 != 'und'
+
+ def __unicode__(self):
+ if self.country is None:
+ return self.name
+ return '%s (%s)' % (self.name, self.country)
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ def __repr__(self):
+ if self.country is None:
+ return 'Language(%s)' % self.name.encode('utf-8')
+ return 'Language(%s, country=%s)' % (self.name.encode('utf-8'), self.country)
+
+
+class language_set(set):
+ """Set of :class:`Language` with some specificities.
+
+ :param iterable: where to take elements from
+ :type iterable: iterable of :class:`Languages ` or string
+ :param languages: all languages
+ :type languages: see :data:`~subliminal.language.LANGUAGES`
+ :param bool strict: whether to raise a ValueError on invalid language or not
+
+ The following redefinitions are meant to reflect the inclusion logic in :class:`Language`
+
+ * Inclusion test, with the ``in`` keyword
+ * Intersection
+ * Substraction
+
+ Here is an illustration of the previous points::
+
+ >>> Language('en') in language_set(['en-US', 'en-CA'])
+ False
+ >>> Language('en-US') in language_set(['en', 'fr'])
+ True
+ >>> language_set(['en']) & language_set(['en-US', 'en-CA'])
+ language_set([Language(English, country=Canada), Language(English, country=United States)])
+ >>> language_set(['en-US', 'en-CA', 'fr']) - language_set(['en'])
+ language_set([Language(French)])
+
+ """
+ def __init__(self, iterable=None, languages=None, strict=True):
+ iterable = iterable or []
+ languages = languages or LANGUAGES
+ items = []
+ for i in iterable:
+ if isinstance(i, Language):
+ items.append(i)
+ continue
+ if isinstance(i, tuple):
+ items.append(Language(i[0], languages=languages, strict=strict))
+ continue
+ items.append(Language(i, languages=languages, strict=strict))
+ super(language_set, self).__init__(items)
+
+ def __contains__(self, item):
+ for i in self:
+ if item in i:
+ return True
+ return super(language_set, self).__contains__(item)
+
+ def __and__(self, other):
+ results = language_set()
+ for i in self:
+ for j in other:
+ if i in j:
+ results.add(i)
+ for i in other:
+ for j in self:
+ if i in j:
+ results.add(i)
+ return results
+
+ def __sub__(self, other):
+ results = language_set()
+ for i in self:
+ if i not in other:
+ results.add(i)
+ return results
+
+
+class language_list(list):
+ """List of :class:`Language` with some specificities.
+
+ :param iterable: where to take elements from
+ :type iterable: iterable of :class:`Languages ` or string
+ :param languages: all languages
+ :type languages: see :data:`~subliminal.language.LANGUAGES`
+ :param bool strict: whether to raise a ValueError on invalid language or not
+
+ The following redefinitions are meant to reflect the inclusion logic in :class:`Language`
+
+ * Inclusion test, with the ``in`` keyword
+ * Index
+
+ Here is an illustration of the previous points::
+
+ >>> Language('en') in language_list(['en-US', 'en-CA'])
+ False
+ >>> Language('en-US') in language_list(['en', 'fr-BE'])
+ True
+ >>> language_list(['en', 'fr-BE']).index(Language('en-US'))
+ 0
+
+ """
+ def __init__(self, iterable=None, languages=None, strict=True):
+ iterable = iterable or []
+ languages = languages or LANGUAGES
+ items = []
+ for i in iterable:
+ if isinstance(i, Language):
+ items.append(i)
+ continue
+ if isinstance(i, tuple):
+ items.append(Language(i[0], languages=languages, strict=strict))
+ continue
+ items.append(Language(i, languages=languages, strict=strict))
+ super(language_list, self).__init__(items)
+
+ def __contains__(self, item):
+ for i in self:
+ if item in i:
+ return True
+ return super(language_list, self).__contains__(item)
+
+ def index(self, x, strict=False):
+ if not strict:
+ for i in range(len(self)):
+ if x in self[i]:
+ return i
+ return super(language_list, self).index(x)
diff --git a/libs/subliminal/services/__init__.py b/libs/subliminal/services/__init__.py
index e1aea2d4..fe4ac08e 100755
--- a/libs/subliminal/services/__init__.py
+++ b/libs/subliminal/services/__init__.py
@@ -15,10 +15,10 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
-from .. import cache
-from ..exceptions import MissingLanguageError, DownloadFailedError, ServiceError
+from ..cache import Cache
+from ..exceptions import DownloadFailedError, ServiceError
+from ..language import language_set, Language
from ..subtitles import EXTENSIONS
-from guessit.language import lang_set, UNDETERMINED
import logging
import os
import requests
@@ -49,8 +49,14 @@ class ServiceBase(object):
#: Timeout for web requests
timeout = 5
- #: Mapping to Service's language codes and subliminal's
- languages = {}
+ #: :class:`~subliminal.language.language_set` of available languages
+ languages = language_set()
+
+ #: Map between language objects and language codes used in the service
+ language_map = {}
+
+ #: Default attribute of a :class:`~subliminal.language.Language` to get with :meth:`get_code`
+ language_code = 'alpha2'
#: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
videos = []
@@ -63,6 +69,7 @@ class ServiceBase(object):
def __init__(self, config=None):
self.config = config or ServiceConfig()
+ self.session = None
def __enter__(self):
self.init()
@@ -80,33 +87,60 @@ class ServiceBase(object):
"""Initialize cache, make sure it is loaded from disk"""
if not self.config or not self.config.cache:
raise ServiceError('Cache directory is required')
-
- service_name = self.__class__.__name__
- self.config.cache.load(service_name)
+ self.config.cache.load(self.__class__.__name__)
def save_cache(self):
- service_name = self.__class__.__name__
- self.config.cache.save(service_name)
+ self.config.cache.save(self.__class__.__name__)
def clear_cache(self):
- service_name = self.__class__.__name__
- self.config.cache.clear(service_name)
+ self.config.cache.clear(self.__class__.__name__)
def cache_for(self, func, args, result):
- service_name = self.__class__.__name__
- return self.config.cache.cache_for(service_name, func, args, result)
+ return self.config.cache.cache_for(self.__class__.__name__, func, args, result)
def cached_value(self, func, args):
- service_name = self.__class__.__name__
- return self.config.cache.cached_value(service_name, func, args)
+ return self.config.cache.cached_value(self.__class__.__name__, func, args)
def terminate(self):
"""Terminate connection"""
logger.debug(u'Terminating %s' % self.__class__.__name__)
+ def get_code(self, language):
+ """Get the service code for a :class:`~subliminal.language.Language`
+
+ It uses the :data:`language_map` and if there's no match, falls back
+ on the :data:`language_code` attribute of the given :class:`~subliminal.language.Language`
+
+ """
+ if language in self.language_map:
+ return self.language_map[language]
+ if self.language_code is None:
+ raise ValueError('%r has no matching code' % language)
+ return getattr(language, self.language_code)
+
+ def get_language(self, code):
+ """Get a :class:`~subliminal.language.Language` from a service code
+
+ It uses the :data:`language_map` and if there's no match, uses the
+ given code as ``language`` parameter for the :class:`~subliminal.language.Language`
+ constructor
+
+ .. note::
+
+ A warning is emitted if the generated :class:`~subliminal.language.Language`
+ is "Undetermined"
+
+ """
+ if code in self.language_map:
+ return self.language_map[code]
+ language = Language(code, strict=False)
+ if language == Language('Undetermined'):
+ logger.warning(u'Code %s could not be identified as a language for %s' % (code, self.__class__.__name__))
+ return language
+
def query(self, *args):
"""Make the actual query"""
- pass
+ raise NotImplementedError()
def list(self, video, languages):
"""List subtitles
@@ -119,6 +153,10 @@ class ServiceBase(object):
return []
return self.list_checked(video, languages)
+ def list_checked(self, video, languages):
+ """List subtitles without having to check parameters for validity"""
+ raise NotImplementedError()
+
def download(self, subtitle):
"""Download a subtitle"""
self.download_file(subtitle.link, subtitle.path)
@@ -129,11 +167,12 @@ class ServiceBase(object):
:param video: the video to check
:type video: :class:`~subliminal.videos.video`
- :param set languages: languages to check
+ :param languages: languages to check
+ :type languages: :class:`~subliminal.language.Language`
:rtype: bool
"""
- languages = (lang_set(languages) & cls.languages) - set([UNDETERMINED])
+ languages = (languages & cls.languages) - language_set(['Undetermined'])
if not languages:
logger.debug(u'No language available for service %s' % cls.__name__.lower())
return False
@@ -211,7 +250,7 @@ class ServiceConfig(object):
self.cache_dir = cache_dir
self.cache = None
if cache_dir is not None:
- self.cache = cache.Cache(cache_dir)
+ self.cache = Cache(cache_dir)
def __repr__(self):
return 'ServiceConfig(%r, %s)' % (self.multi, self.cache.cache_dir)
diff --git a/libs/subliminal/services/addic7ed.py b/libs/subliminal/services/addic7ed.py
index c754de55..7ca5c0cb 100755
--- a/libs/subliminal/services/addic7ed.py
+++ b/libs/subliminal/services/addic7ed.py
@@ -17,12 +17,11 @@
# along with subliminal. If not, see .
from . import ServiceBase
from ..cache import cachedmethod
+from ..language import Language, language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import get_keywords
from ..videos import Episode
from bs4 import BeautifulSoup
-from guessit.language import lang_set
-from subliminal.utils import get_keywords
-import guessit
import logging
import re
@@ -49,13 +48,10 @@ def matches(pattern, string):
class Addic7ed(ServiceBase):
server_url = 'http://www.addic7ed.com'
api_based = False
- languages = lang_set([u'English', u'Italian', u'Portuguese',
- u'Portuguese (Brazilian)', u'Romanian',
- u'Spanish', u'French', u'Greek', u'Arabic',
- u'German', u'Croatian', u'Indonesian', u'Hebrew',
- u'Russian', u'Turkish', u'Swedish', u'Czech',
- u'Dutch', u'Hungarian', u'Norwegian', u'Polish',
- u'Persian'], strict=True)
+ #TODO: Complete this
+ languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'he', 'hr', 'hu', 'it',
+ 'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
+ language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre')}
videos = [Episode]
require_video = False
required_features = ['permissive']
@@ -119,7 +115,7 @@ class Addic7ed(ServiceBase):
if 'href' not in link.attrs or not (link['href'].startswith('/original') or link['href'].startswith('/updated')):
continue
suburl = link['href']
- lang = guessit.Language(row.find('td', 'language').text.strip())
+ lang = self.get_language(row.find('td', 'language').text.strip())
result = {'suburl': suburl, 'language': lang, 'release': release}
suburls.append(result)
return suburls
@@ -135,21 +131,18 @@ class Addic7ed(ServiceBase):
except KeyError:
logger.debug(u'Could not find series id for %s' % series)
return []
-
try:
ep_url = self.get_episode_url(sid, season, episode)
except KeyError:
logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
return []
suburls = self.get_sub_urls(ep_url)
-
# filter the subtitles with our queried languages
subtitles = []
for suburl in suburls:
language = suburl['language']
if language not in languages:
continue
-
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
'%s/%s' % (self.server_url, suburl['suburl']),
diff --git a/libs/subliminal/services/bierdopje.py b/libs/subliminal/services/bierdopje.py
index ab6c577c..a996cb00 100755
--- a/libs/subliminal/services/bierdopje.py
+++ b/libs/subliminal/services/bierdopje.py
@@ -18,11 +18,11 @@
from . import ServiceBase
from ..cache import cachedmethod
from ..exceptions import ServiceError
+from ..language import language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import to_unicode
from ..videos import Episode
from bs4 import BeautifulSoup
-from guessit.language import lang_set
import logging
import urllib
try:
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
class BierDopje(ServiceBase):
server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
api_based = True
- languages = lang_set(['en', 'nl'])
+ languages = language_set(['eng', 'dut'])
videos = [Episode]
require_video = False
required_features = ['xml']
@@ -52,7 +52,6 @@ class BierDopje(ServiceBase):
if soup.status.contents[0] == 'false':
logger.debug(u'Could not find show %s' % series)
return None
-
return int(soup.showid.contents[0])
def load_cache(self):
diff --git a/libs/subliminal/services/opensubtitles.py b/libs/subliminal/services/opensubtitles.py
index cf493805..04d44c4c 100755
--- a/libs/subliminal/services/opensubtitles.py
+++ b/libs/subliminal/services/opensubtitles.py
@@ -17,11 +17,10 @@
# along with subliminal. If not, see .
from . import ServiceBase
from ..exceptions import ServiceError, DownloadFailedError
+from ..language import Language, language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import to_unicode
from ..videos import Episode, Movie
-from guessit.language import lang_set
-import guessit
import gzip
import logging
import os.path
@@ -34,71 +33,50 @@ logger = logging.getLogger(__name__)
class OpenSubtitles(ServiceBase):
server_url = 'http://api.opensubtitles.org/xml-rpc'
api_based = True
- # language list fetched from:
- # http://www.opensubtitles.org/addons/export_languages.php
- languages = lang_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh',
- 'afr', 'ain', 'aka', 'akk', 'alb', 'ale', 'alg', 'alt',
- 'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
- 'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava',
- 'ave', 'awa', 'aym', 'aze', 'bad', 'bai', 'bak', 'bal',
- 'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
- 'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla',
- 'bnt', 'bod', 'bos', 'bra', 'bre', 'btk', 'bua', 'bug',
- 'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau',
- 'ceb', 'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk',
- 'chm', 'chn', 'cho', 'chp', 'chr', 'chu', 'chv', 'chy',
- 'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre',
- 'crh', 'crp', 'csb', 'cus', 'cym', 'cze', 'dak', 'dan',
- 'dar', 'day', 'del', 'den', 'deu', 'dgr', 'din', 'div',
- 'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi',
- 'egy', 'eka', 'elx', 'eng', 'enm', 'epo', 'est', 'eus',
- 'ewe', 'ewo', 'fan', 'fao', 'fas', 'fat', 'fij', 'fil',
- 'fin', 'fiu', 'fon', 'fra', 'fre', 'frm', 'fro', 'fry',
- 'ful', 'fur', 'gaa', 'gay', 'gba', 'gem', 'geo', 'ger',
- 'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh',
- 'gon', 'gor', 'got', 'grb', 'grc', 'ell', 'grn', 'guj',
- 'gwi', 'hai', 'hat', 'hau', 'haw', 'heb', 'her', 'hil',
- 'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup',
- 'hye', 'iba', 'ibo', 'ice', 'ido', 'iii', 'ijo', 'iku',
- 'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh', 'ipk',
- 'ira', 'iro', 'isl', 'ita', 'jav', 'jpn', 'jpr', 'jrb',
- 'kaa', 'kab', 'kac', 'kal', 'kam', 'kan', 'kar', 'kas',
- 'kat', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm',
- 'kho', 'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon',
- 'kor', 'kos', 'kpe', 'krc', 'kro', 'kru', 'kua', 'kum',
- 'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav',
- 'lez', 'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua',
- 'lub', 'lug', 'lui', 'lun', 'luo', 'lus', 'mac', 'mad',
- 'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map',
- 'mar', 'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic',
- 'min', 'mis', 'mkd', 'mkh', 'mlg', 'mlt', 'mnc', 'mni',
- 'mno', 'moh', 'mol', 'mon', 'mos', 'mri', 'msa', 'mwl',
- 'mul', 'mun', 'mus', 'mwr', 'mya', 'myn', 'myv', 'nah',
- 'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds',
- 'nep', 'new', 'nia', 'nic', 'niu', 'nld', 'nno', 'nob',
- 'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym',
- 'nyn', 'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa',
- 'oss', 'ota', 'oto', 'paa', 'pag', 'pal', 'pam', 'pan',
- 'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol',
- 'pon', 'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap',
- 'rar', 'roa', 'roh', 'rom', 'ron', 'run', 'rup', 'rus',
- 'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas',
- 'sat', 'scc', 'scn', 'sco', 'scr', 'sel', 'sem', 'sga',
- 'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla', 'slk',
- 'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo',
- 'sms', 'sna', 'snd', 'snk', 'sog', 'som', 'son', 'sot',
- 'spa', 'sqi', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk',
- 'sun', 'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai',
- 'tam', 'tat', 'tel', 'tem', 'ter', 'tet', 'tgk', 'tgl',
- 'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli',
- 'tmh', 'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk',
- 'tum', 'tup', 'tur', 'tut', 'tvl', 'twi', 'tyv', 'udm',
- 'uga', 'uig', 'ukr', 'umb', 'und', 'urd', 'uzb', 'vai',
- 'ven', 'vie', 'vol', 'vot', 'wak', 'wal', 'war', 'was',
- 'wel', 'wen', 'wln', 'wol', 'xal', 'xho', 'yao', 'yap',
- 'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'zho', 'znd',
- 'zul', 'zun', 'rum', 'pob', 'unk', 'ass'])
-
+ # Source: http://www.opensubtitles.org/addons/export_languages.php
+ languages = language_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh', 'afr', 'ain', 'aka', 'akk',
+ 'alb', 'ale', 'alg', 'alt', 'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
+ 'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava', 'ave', 'awa', 'aym', 'aze',
+ 'bad', 'bai', 'bak', 'bal', 'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
+ 'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla', 'bnt', 'bos', 'bra', 'bre',
+ 'btk', 'bua', 'bug', 'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau', 'ceb',
+ 'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk', 'chm', 'chn', 'cho', 'chp', 'chr',
+ 'chu', 'chv', 'chy', 'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre', 'crh',
+ 'crp', 'csb', 'cus', 'cze', 'dak', 'dan', 'dar', 'day', 'del', 'den', 'dgr', 'din',
+ 'div', 'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi', 'egy', 'eka', 'ell',
+ 'elx', 'eng', 'enm', 'epo', 'est', 'ewe', 'ewo', 'fan', 'fao', 'fat', 'fij', 'fil',
+ 'fin', 'fiu', 'fon', 'fre', 'frm', 'fro', 'fry', 'ful', 'fur', 'gaa', 'gay', 'gba',
+ 'gem', 'geo', 'ger', 'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh', 'gon',
+ 'gor', 'got', 'grb', 'grc', 'grn', 'guj', 'gwi', 'hai', 'hat', 'hau', 'haw', 'heb',
+ 'her', 'hil', 'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup', 'iba', 'ibo',
+ 'ice', 'ido', 'iii', 'ijo', 'iku', 'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh',
+ 'ipk', 'ira', 'iro', 'ita', 'jav', 'jpn', 'jpr', 'jrb', 'kaa', 'kab', 'kac', 'kal',
+ 'kam', 'kan', 'kar', 'kas', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm', 'kho',
+ 'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon', 'kor', 'kos', 'kpe', 'krc', 'kro',
+ 'kru', 'kua', 'kum', 'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav', 'lez',
+ 'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua', 'lub', 'lug', 'lui', 'lun', 'luo',
+ 'lus', 'mac', 'mad', 'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map', 'mar',
+ 'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic', 'min', 'mkh', 'mlg', 'mlt', 'mnc',
+ 'mni', 'mno', 'moh', 'mon', 'mos', 'mun', 'mus', 'mwl', 'mwr', 'myn', 'myv', 'nah',
+ 'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds', 'nep', 'new', 'nia', 'nic',
+ 'niu', 'nno', 'nob', 'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym', 'nyn',
+ 'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa', 'oss', 'ota', 'oto', 'paa', 'pag',
+ 'pal', 'pam', 'pan', 'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol', 'pon',
+ 'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap', 'rar', 'roa', 'roh', 'rom', 'rum',
+ 'run', 'rup', 'rus', 'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas', 'sat',
+ 'scn', 'sco', 'sel', 'sem', 'sga', 'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla',
+ 'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo', 'sms', 'sna', 'snd', 'snk',
+ 'sog', 'som', 'son', 'sot', 'spa', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk', 'sun',
+ 'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai', 'tam', 'tat', 'tel', 'tem', 'ter',
+ 'tet', 'tgk', 'tgl', 'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli', 'tmh',
+ 'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk', 'tum', 'tup', 'tur', 'tut', 'tvl',
+ 'twi', 'tyv', 'udm', 'uga', 'uig', 'ukr', 'umb', 'urd', 'uzb', 'vai', 'ven', 'vie',
+ 'vol', 'vot', 'wak', 'wal', 'war', 'was', 'wel', 'wen', 'wln', 'wol', 'xal', 'xho',
+ 'yao', 'yap', 'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'znd', 'zul', 'zun',
+ 'por-BR', 'rum-MD'])
+ language_map = {'mol': Language('rum-MD'), 'scc': Language('srp'), 'pob': Language('por-BR'),
+ Language('rum-MD'): 'mol', Language('srp'): 'scc', Language('por-BR'): 'pob'}
+ language_code = 'alpha3'
videos = [Episode, Movie]
require_video = False
confidence_order = ['moviehash', 'imdbid', 'fulltext']
@@ -131,7 +109,7 @@ class OpenSubtitles(ServiceBase):
if not searches:
raise ServiceError('One or more parameter missing')
for search in searches:
- search['sublanguageid'] = ','.join(l.opensubtitles for l in languages)
+ search['sublanguageid'] = ','.join(self.get_code(l) for l in languages)
logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
results = self.server.SearchSubtitles(self.token, searches)
if not results['data']:
@@ -139,7 +117,7 @@ class OpenSubtitles(ServiceBase):
return []
subtitles = []
for result in results['data']:
- language = guessit.Language(result['SubLanguageID'])
+ language = self.get_language(result['SubLanguageID'])
path = get_subtitle_path(filepath, language, self.config.multi)
confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
diff --git a/libs/subliminal/services/podnapisi.py b/libs/subliminal/services/podnapisi.py
index 0e12fa00..39025bc5 100755
--- a/libs/subliminal/services/podnapisi.py
+++ b/libs/subliminal/services/podnapisi.py
@@ -17,12 +17,11 @@
# along with subliminal. If not, see .
from . import ServiceBase
from ..exceptions import ServiceError, DownloadFailedError
+from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..utils import to_unicode
from ..videos import Episode, Movie
-from guessit.language import lang_set
from hashlib import md5, sha256
-import guessit
import logging
import xmlrpclib
@@ -33,11 +32,16 @@ logger = logging.getLogger(__name__)
class Podnapisi(ServiceBase):
server_url = 'http://ssp.podnapisi.net:8000'
api_based = True
- languages = lang_set(['sl', 'en', 'nn', 'ko', 'de', 'is', 'cs', 'fr', 'it', 'bs', 'jp', 'ar', 'ro',
- 'hu', 'gr', 'zh', 'lt', 'et', 'lv', 'he', 'nl', 'da', 'sv', 'pl', 'ru', 'es',
- 'sq', 'tr', 'fi', 'pt', 'bg', 'mk', 'sr', 'sk', 'hr', 'hi', 'th', 'ca', 'uk',
- 'pb', 'ga', 'be', 'vi', 'fa', 'ca', 'id', 'ms'])
- #FIXME: ag and cyr not recognized by guessit
+ languages = language_set(['ar', 'be', 'bg', 'bs', 'ca', 'ca', 'cs', 'da', 'de', 'el', 'en',
+ 'es', 'et', 'fa', 'fi', 'fr', 'ga', 'he', 'hi', 'hr', 'hu', 'id',
+ 'is', 'it', 'ja', 'ko', 'lt', 'lv', 'mk', 'ms', 'nl', 'nn', 'pl',
+ 'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'th', 'tr', 'uk',
+ 'vi', 'zh', 'es-ar', 'pt-br'])
+ language_map = {'jp': Language('jpn'), Language('jpn'): 'jp',
+ 'gr': Language('gre'), Language('gre'): 'gr',
+ 'pb': Language('por-BR'), Language('por-BR'): 'pb',
+ 'ag': Language('spa-AR'), Language('spa-AR'): 'ag',
+ 'cyr': Language('srp')}
videos = [Episode, Movie]
require_video = True
@@ -71,8 +75,8 @@ class Podnapisi(ServiceBase):
return []
subtitles = []
for result in results['results'][moviehash]['subtitles']:
- language = guessit.Language(result['lang'])
- if language == guessit.language.UNDETERMINED or language not in languages:
+ language = self.get_language(result['lang'])
+ if language not in languages:
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['id'],
diff --git a/libs/subliminal/services/subswiki.py b/libs/subliminal/services/subswiki.py
index 6671a8de..008948f3 100755
--- a/libs/subliminal/services/subswiki.py
+++ b/libs/subliminal/services/subswiki.py
@@ -17,12 +17,11 @@
# along with subliminal. If not, see .
from . import ServiceBase
from ..exceptions import ServiceError
+from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie
from bs4 import BeautifulSoup
-from guessit.language import lang_set
from subliminal.utils import get_keywords, split_keyword
-import guessit
import logging
import re
import urllib
@@ -34,9 +33,11 @@ logger = logging.getLogger(__name__)
class SubsWiki(ServiceBase):
server_url = 'http://www.subswiki.com'
api_based = False
- languages = lang_set([u'English (US)', u'English (UK)', u'English', u'French', u'Brazilian',
- u'Portuguese', u'Español (Latinoamérica)', u'Español (España)',
- u'Español', u'Italian', u'Català'], strict=True)
+ languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
+ language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
+ u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
+ u'English (UK)': Language('eng-GB')}
+ language_code = 'name'
videos = [Episode, Movie]
require_video = False
release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
@@ -82,7 +83,7 @@ class SubsWiki(ServiceBase):
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
- language = guessit.Language(html_language.string.strip())
+ language = self.get_language(html_language.string.strip())
if language not in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
diff --git a/libs/subliminal/services/subtitulos.py b/libs/subliminal/services/subtitulos.py
index 8154aaaf..d346a75b 100755
--- a/libs/subliminal/services/subtitulos.py
+++ b/libs/subliminal/services/subtitulos.py
@@ -16,12 +16,11 @@
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
from . import ServiceBase
+from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode
from bs4 import BeautifulSoup
-from guessit.language import lang_set
from subliminal.utils import get_keywords, split_keyword
-import guessit
import logging
import re
import unicodedata
@@ -34,9 +33,11 @@ logger = logging.getLogger(__name__)
class Subtitulos(ServiceBase):
server_url = 'http://www.subtitulos.es'
api_based = False
- languages = lang_set([u'English (US)', u'English (UK)', u'English', u'French', u'Brazilian',
- u'Portuguese', u'Español (Latinoamérica)', u'Español (España)', u'Español',
- u'Italian', u'Català'], strict=True)
+ languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
+ language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
+ u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
+ u'English (UK)': Language('eng-GB')}
+ language_code = 'name'
videos = [Episode]
require_video = False
required_features = ['permissive']
@@ -68,7 +69,7 @@ class Subtitulos(ServiceBase):
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
- language = guessit.Language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
+ language = self.get_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
if language not in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
diff --git a/libs/subliminal/services/thesubdb.py b/libs/subliminal/services/thesubdb.py
index 1800dcf7..274c775f 100755
--- a/libs/subliminal/services/thesubdb.py
+++ b/libs/subliminal/services/thesubdb.py
@@ -16,10 +16,9 @@
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
from . import ServiceBase
+from ..language import language_set
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie, UnknownVideo
-from guessit.language import lang_set
-import guessit
import logging
@@ -27,13 +26,13 @@ logger = logging.getLogger(__name__)
class TheSubDB(ServiceBase):
- server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
- user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)' # defined by the API
+ server_url = 'http://api.thesubdb.com'
+ user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)'
api_based = True
- languages = lang_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi',
- 'fr', 'hu', 'id', 'it', 'la', 'nl', 'no',
- 'oc', 'pl', 'pt', 'ro', 'ru', 'sl', 'sr',
- 'sv', 'tr'], strict=True) # list available with the API at http://sandbox.thesubdb.com/?action=languages
+ # Source: http://api.thesubdb.com/?action=languages
+ languages = language_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'id', 'it',
+ 'la', 'nl', 'no', 'oc', 'pl', 'pt', 'ro', 'ru', 'sl', 'sr', 'sv',
+ 'tr'])
videos = [Movie, Episode, UnknownVideo]
require_video = True
@@ -48,7 +47,7 @@ class TheSubDB(ServiceBase):
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
- available_languages = set(guessit.Language(l) for l in r.content.split(','))
+ available_languages = language_set(r.content.split(','))
languages &= available_languages
if not languages:
logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
diff --git a/libs/subliminal/services/tvsubtitles.py b/libs/subliminal/services/tvsubtitles.py
index 5c74cc1e..44bb743b 100755
--- a/libs/subliminal/services/tvsubtitles.py
+++ b/libs/subliminal/services/tvsubtitles.py
@@ -17,12 +17,11 @@
# along with subliminal. If not, see .
from . import ServiceBase
from ..cache import cachedmethod
+from ..language import language_set, Language
from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import get_keywords
from ..videos import Episode
from bs4 import BeautifulSoup
-from guessit.language import lang_set
-from subliminal.utils import get_keywords
-import guessit
import logging
import re
@@ -41,12 +40,11 @@ def match(pattern, string):
class TvSubtitles(ServiceBase):
server_url = 'http://www.tvsubtitles.net'
api_based = False
- languages = lang_set([u'English', u'Español', u'French', u'German',
- u'Brazilian', u'Russian', u'Ukrainian', u'Italian',
- u'Greek', u'Arabic', u'Hungarian', u'Polish',
- u'Turkish', u'Dutch', u'Portuguese', u'Swedish',
- u'Danish', u'Finnish', u'Korean', u'Chinese',
- u'Japanese', u'Bulgarian', u'Czech', u'Romanian'], strict=True)
+ languages = language_set(['ar', 'bg', 'cs', 'da', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hu',
+ 'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
+ 'zh', 'pt-br'])
+ #TODO: Find more exceptions
+ language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr')}
videos = [Episode]
require_video = False
required_features = ['permissive']
@@ -78,11 +76,9 @@ class TvSubtitles(ServiceBase):
cells = row.find_all('td')
if not cells:
continue
-
episode_number = match('x([0-9]+)', cells[0].text)
if not episode_number:
continue
-
episode_number = int(episode_number)
episode_id = int(match('episode-([0-9]+)', cells[1].a['href']))
# we could just return the id of the queried episode, but as we
@@ -102,14 +98,13 @@ class TvSubtitles(ServiceBase):
if 'href' not in subdiv.attrs or not subdiv['href'].startswith('/subtitle'):
continue
subid = int(match('([0-9]+)', subdiv['href']))
- lang = guessit.Language(match('flags/(.*).gif', subdiv.img['src']))
+ lang = self.get_language(match('flags/(.*).gif', subdiv.img['src']))
result = {'subid': subid, 'language': lang}
for p in subdiv.find_all('p'):
if 'alt' in p.attrs and p['alt'] == 'rip':
result['rip'] = p.text.strip()
if 'alt' in p.attrs and p['alt'] == 'release':
result['release'] = p.text.strip()
-
subids.append(result)
return subids
diff --git a/libs/subliminal/subtitles.py b/libs/subliminal/subtitles.py
index a33c35c3..2d27b0f1 100755
--- a/libs/subliminal/subtitles.py
+++ b/libs/subliminal/subtitles.py
@@ -15,9 +15,8 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
+from .language import Language
import os.path
-import guessit
-from guessit.language import is_language
__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
@@ -31,7 +30,7 @@ class Subtitle(object):
:param string path: path to the subtitle
:param language: language of the subtitle
- :type language: :class:`guessit.Language`
+ :type language: :class:`~subliminal.language.Language`
"""
def __init__(self, path, language):
@@ -51,7 +50,7 @@ class EmbeddedSubtitle(Subtitle):
:param string path: path to the subtitle
:param language: language of the subtitle
- :type language: :class:`guessit.Language`
+ :type language: :class:`~subliminal.language.Language`
:param int track_id: id of the subtitle track in the container
"""
@@ -61,7 +60,7 @@ class EmbeddedSubtitle(Subtitle):
@classmethod
def from_enzyme(cls, path, subtitle):
- language = guessit.Language(subtitle.language) or None
+ language = Language(subtitle.language) or None
return cls(path, language, subtitle.trackno)
@@ -78,8 +77,7 @@ class ExternalSubtitle(Subtitle):
if not extension:
raise ValueError('Not a supported subtitle extension')
language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
- language = guessit.Language(language) or None
-
+ language = Language(language) or None
return cls(path, language)
@@ -88,7 +86,7 @@ class ResultSubtitle(ExternalSubtitle):
:param string path: path to the subtitle
:param language: language of the subtitle
- :type language: :class:`guessit.Language`
+ :type language: :class:`~subliminal.language.Language`
:param string service: name of the service
:param string link: download link for the subtitle
:param string release: release name of the video
@@ -114,7 +112,7 @@ class ResultSubtitle(ExternalSubtitle):
"""
extension = os.path.splitext(self.path)[0]
language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
- return not is_language(language)
+ return Language(language) == Language('und')
def __repr__(self):
return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
@@ -125,7 +123,7 @@ def get_subtitle_path(video_path, language, multi):
:param string video_path: path to the video
:param language: language of the subtitle
- :type language: :class:`guessit.Language`
+ :type language: :class:`~subliminal.language.Language`
:param bool multi: whether to use multi language naming or not
:return: path of the subtitle
:rtype: string
diff --git a/libs/subliminal/tasks.py b/libs/subliminal/tasks.py
index 448a64f1..bccf9ab5 100755
--- a/libs/subliminal/tasks.py
+++ b/libs/subliminal/tasks.py
@@ -35,6 +35,7 @@ class ListTask(Task):
"""
def __init__(self, video, languages, service, config):
+ super(ListTask, self).__init__()
self.video = video
self.service = service
self.languages = languages
@@ -54,6 +55,7 @@ class DownloadTask(Task):
"""
def __init__(self, video, subtitles):
+ super(DownloadTask, self).__init__()
self.video = video
self.subtitles = subtitles
diff --git a/libs/subliminal/videos.py b/libs/subliminal/videos.py
index adfcafe7..34050722 100755
--- a/libs/subliminal/videos.py
+++ b/libs/subliminal/videos.py
@@ -145,12 +145,14 @@ class Video(object):
lang = guessit.Language(possible_lang)
if lang:
results.append(subtitles.ExternalSubtitle(path, lang))
-
return results
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.release)
+ def __hash__(self):
+ return hash(self.path or self.release)
+
class Episode(Video):
"""Episode :class:`Video`