Update libs

libs/subliminal/api.py

@@ -81,6 +81,11 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
     :return: downloaded subtitles
     :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
 
+    .. note::
+
+        If you use ``multi=True``, :data:`~subliminal.core.LANGUAGE_INDEX` has to be the first item of the ``order`` list
+        or you might get unexpected results.
+
     """
     services = services or SERVICES
     languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
@@ -92,7 +97,7 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
         subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
     results = []
     service_instances = {}
-    tasks = create_download_tasks(subtitles_by_video, multi)
+    tasks = create_download_tasks(subtitles_by_video, languages, multi)
     for task in tasks:
         try:
             result = consume_task(task, service_instances)

libs/subliminal/async.py

@@ -134,7 +134,7 @@ class Pool(object):
         subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
         for video, subtitles in subtitles_by_video.iteritems():
             subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
-        tasks = create_download_tasks(subtitles_by_video, multi)
+        tasks = create_download_tasks(subtitles_by_video, languages, multi)
         for task in tasks:
             self.tasks.put(task)
         self.join()

libs/subliminal/core.py

@@ -20,6 +20,7 @@ from .services import ServiceConfig
 from .tasks import DownloadTask, ListTask
 from .utils import get_keywords
 from .videos import Episode, Movie, scan
+from .language import Language
 from collections import defaultdict
 from itertools import groupby
 import bs4
@@ -66,7 +67,7 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
             if not wanted_languages:
                 logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages))
                 continue
-        if not force and not multi and None in detected_languages:
+        if not force and not multi and Language('Undetermined') in detected_languages:
             logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video))
             continue
         logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services))
@@ -81,13 +82,13 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
     return tasks
 
 
-def create_download_tasks(subtitles_by_video, multi):
+def create_download_tasks(subtitles_by_video, languages, multi):
     """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
 
-    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted
+    :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results with ordered subtitles
     :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
-    :param order: preferred order for subtitles sorting
-    :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
+    :param languages: languages in preferred order
+    :type languages: :class:`~subliminal.language.language_list`
     :param bool multi: download multiple languages for the same video
     :return: the created tasks
     :rtype: list of :class:`~subliminal.tasks.DownloadTask`
@@ -102,7 +103,7 @@ def create_download_tasks(subtitles_by_video, multi):
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
             continue
-        for _, by_language in groupby(subtitles, lambda s: s.language):
+        for _, by_language in groupby(subtitles, lambda s: languages.index(s.language)):
             task = DownloadTask(video, list(by_language))
             logger.debug(u'Created task %r' % task)
             tasks.append(task)
@@ -157,6 +158,7 @@ def matching_confidence(video, subtitle):
     guess = guessit.guess_file_info(subtitle.release, 'autodetect')
     video_keywords = get_keywords(video.guess)
     subtitle_keywords = get_keywords(guess) | subtitle.keywords
+    logger.debug(u'Video keywords %r - Subtitle keywords %r' % (video_keywords, subtitle_keywords))
     replacement = {'keywords': len(video_keywords & subtitle_keywords)}
     if isinstance(video, Episode):
         replacement.update({'series': 0, 'season': 0, 'episode': 0})
@@ -179,8 +181,11 @@ def matching_confidence(video, subtitle):
             if 'year' in guess and guess['year'] == video.year:
                 replacement['year'] = 1
     else:
-        return 0
+        logger.debug(u'Not able to compute confidence for %r' % video)
+        return 0.0
+    logger.debug(u'Found %r' % replacement)
     confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
+    logger.info(u'Computed confidence %.4f for %r and %r' % (confidence, video, subtitle))
     return confidence
 
 
@@ -248,7 +253,7 @@ def group_by_video(list_results):
     """
     result = defaultdict(list)
     for video, subtitles in list_results:
-        result[video] += subtitles
+        result[video] += subtitles or []
     return result
 
 

libs/subliminal/infos.py

@@ -15,4 +15,4 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.6.0'
+__version__ = '0.6.1'

libs/subliminal/language.py

@@ -17,6 +17,10 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from .utils import to_unicode
 import re
+import logging
+
+
+logger = logging.getLogger(__name__)
 
 
 COUNTRIES = [('AF', 'AFG', '004', u'Afghanistan'),
@@ -838,15 +842,26 @@ class Language(object):
         if isinstance(country, Country):
             self.country = country
         elif isinstance(country, basestring):
-            self.country = Country(country, countries)
+            try:
+                self.country = Country(country, countries)
+            except ValueError:
+                logger.warning(u'Country %s could not be identified' % country)
+                if strict:
+                    raise
 
         # Language + Country format
         #TODO: Improve this part
-        for regexp in [r.match(language) for r in self.with_country_regexps]:
-            if regexp:
-                language = regexp.group(1)
-                self.country = Country(regexp.group(2), countries)
-                break
+        if country is None:
+            for regexp in [r.match(language) for r in self.with_country_regexps]:
+                if regexp:
+                    language = regexp.group(1)
+                    try:
+                        self.country = Country(regexp.group(2), countries)
+                    except ValueError:
+                        logger.warning(u'Country %s could not be identified' % country)
+                        if strict:
+                            raise
+                    break
 
         # Try to find the language
         language = to_unicode(language.strip().lower())

libs/subliminal/services/__init__.py

@@ -160,6 +160,7 @@ class ServiceBase(object):
     def download(self, subtitle):
         """Download a subtitle"""
         self.download_file(subtitle.link, subtitle.path)
+        return subtitle
 
     @classmethod
     def check_validity(cls, video, languages):
@@ -188,17 +189,17 @@ class ServiceBase(object):
         :param string filepath: destination path
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
             with open(filepath, 'wb') as f:
                 f.write(r.content)
         except Exception as e:
-            logger.error(u'Download %s failed: %s' % (url, e))
+            logger.error(u'Download failed: %s' % e)
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
-        logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+        logger.debug(u'Download finished')
 
     def download_zip_file(self, url, filepath):
         """Attempt to download a zip file and extract any subtitle file from it, if any.
@@ -208,7 +209,7 @@ class ServiceBase(object):
         :param string filepath: destination path for the subtitle
 
         """
-        logger.info(u'Downloading %s' % url)
+        logger.info(u'Downloading %s in %s' % (url, filepath))
         try:
             zippath = filepath + '.zip'
             r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
@@ -218,17 +219,15 @@ class ServiceBase(object):
                 # TODO: could check if maybe we already have a text file and
                 # download it directly
                 raise DownloadFailedError('Downloaded file is not a zip file')
-            zipsub = zipfile.ZipFile(zippath)
-            for subfile in zipsub.namelist():
-                if os.path.splitext(subfile)[1] in EXTENSIONS:
-                    open(filepath, 'w').write(zipsub.open(subfile).read())
-                    break
-            else:
-                logger.debug(u'No subtitles found in zip file')
-                raise DownloadFailedError('No subtitles found in zip file')
+            with zipfile.ZipFile(zippath) as zipsub:
+                for subfile in zipsub.namelist():
+                    if os.path.splitext(subfile)[1] in EXTENSIONS:
+                        with open(filepath, 'w') as f:
+                            f.write(zipsub.open(subfile).read())
+                        break
+                else:
+                    raise DownloadFailedError('No subtitles found in zip file')
             os.remove(zippath)
-            logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
-            return
         except Exception as e:
             logger.error(u'Download %s failed: %s' % (url, e))
             if os.path.exists(zippath):
@@ -236,6 +235,7 @@ class ServiceBase(object):
             if os.path.exists(filepath):
                 os.remove(filepath)
             raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
 
 
 class ServiceConfig(object):

libs/subliminal/services/addic7ed.py

@@ -17,12 +17,14 @@
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import ServiceBase
 from ..cache import cachedmethod
+from ..exceptions import DownloadFailedError
 from ..language import Language, language_set
 from ..subtitles import get_subtitle_path, ResultSubtitle
 from ..utils import get_keywords
 from ..videos import Episode
 from bs4 import BeautifulSoup
 import logging
+import os
 import re
 
 
@@ -51,7 +53,8 @@ class Addic7ed(ServiceBase):
     #TODO: Complete this
     languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'he', 'hr', 'hu', 'it',
                               'pl', 'pt', 'ro', 'ru', 'se', 'pt-br'])
-    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre')}
+    language_map = {'Portuguese (Brazilian)': Language('por-BR'), 'Greek': Language('gre'),
+                    'Spanish (Latin America)': Language('spa'), }
     videos = [Episode]
     require_video = False
     required_features = ['permissive']
@@ -144,11 +147,27 @@ class Addic7ed(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
-                                      '%s/%s' % (self.server_url, suburl['suburl']),
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/%s' % (self.server_url, suburl['suburl']),
                                       keywords=[suburl['release']])
             subtitles.append(subtitle)
         return subtitles
 
+    def download(self, subtitle):
+        logger.info(u'Downloading %s in %s' % (subtitle.link, subtitle.path))
+        try:
+            r = self.session.get(subtitle.link, headers={'Referer': subtitle.link, 'User-Agent': self.user_agent})
+            soup = BeautifulSoup(r.content, self.required_features)
+            if soup.title is not None and u'Addic7ed.com' in soup.title.text.strip():
+                raise DownloadFailedError('Download limit exceeded')
+            with open(subtitle.path, 'wb') as f:
+                f.write(r.content)
+        except Exception as e:
+            logger.error(u'Download failed: %s' % e)
+            if os.path.exists(subtitle.path):
+                os.remove(subtitle.path)
+            raise DownloadFailedError(str(e))
+        logger.debug(u'Download finished')
+        return subtitle
+
 
 Service = Addic7ed

libs/subliminal/services/bierdopje.py

@@ -19,7 +19,7 @@ from . import ServiceBase
 from ..cache import cachedmethod
 from ..exceptions import ServiceError
 from ..language import language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..subtitles import get_subtitle_path, ResultSubtitle, EXTENSIONS
 from ..utils import to_unicode
 from ..videos import Episode
 from bs4 import BeautifulSoup
@@ -87,8 +87,11 @@ class BierDopje(ServiceBase):
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
             for result in soup.results('result'):
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0],
-                                          release=to_unicode(result.filename.contents[0]))
+                release = to_unicode(result.filename.contents[0])
+                if not release.endswith(tuple(EXTENSIONS)):
+                    release += '.srt'
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result.downloadlink.contents[0],
+                                          release=release)
                 subtitles.append(subtitle)
         return subtitles
 

libs/subliminal/services/opensubtitles.py

@@ -120,7 +120,7 @@ class OpenSubtitles(ServiceBase):
             language = self.get_language(result['SubLanguageID'])
             path = get_subtitle_path(filepath, language, self.config.multi)
             confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['SubDownloadLink'],
                                       release=to_unicode(result['SubFileName']), confidence=confidence)
             subtitles.append(subtitle)
         return subtitles

libs/subliminal/services/podnapisi.py

@@ -79,7 +79,7 @@ class Podnapisi(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['id'],
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'],
                                       release=to_unicode(result['release']), confidence=result['weight'])
             subtitles.append(subtitle)
         if not subtitles:

libs/subliminal/services/subswiki.py

@@ -93,7 +93,7 @@ class SubsWiki(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
                 subtitles.append(subtitle)
         return subtitles
 

libs/subliminal/services/subtitulos.py

@@ -36,7 +36,7 @@ class Subtitulos(ServiceBase):
     languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'por-BR', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
     language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
                     u'Català': Language('cat'), u'Brazilian': Language('por-BR'), u'English (US)': Language('eng-US'),
-                    u'English (UK)': Language('eng-GB')}
+                    u'English (UK)': Language('eng-GB'), 'Galego': Language('glg')}
     language_code = 'name'
     videos = [Episode]
     require_video = False
@@ -79,7 +79,8 @@ class Subtitulos(ServiceBase):
                     logger.debug(u'Wrong subtitle status %s' % status)
                     continue
                 path = get_subtitle_path(filepath, language, self.config.multi)
-                subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
+                subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'],
+                                          keywords=sub_keywords)
                 subtitles.append(subtitle)
         return subtitles
 

libs/subliminal/services/tvsubtitles.py

@@ -44,7 +44,8 @@ class TvSubtitles(ServiceBase):
                               'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
                               'zh', 'pt-br'])
     #TODO: Find more exceptions
-    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr')}
+    language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr'),
+                    'cn': Language('chi')}
     videos = [Episode]
     require_video = False
     required_features = ['permissive']
@@ -128,14 +129,14 @@ class TvSubtitles(ServiceBase):
             if language not in languages:
                 continue
             path = get_subtitle_path(filepath, language, self.config.multi)
-            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
-                                      '%s/download-%d.html' % (self.server_url, subid['subid']),
+            subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/download-%d.html' % (self.server_url, subid['subid']),
                                       keywords=[subid['rip'], subid['release']])
             subtitles.append(subtitle)
         return subtitles
 
     def download(self, subtitle):
         self.download_zip_file(subtitle.link, subtitle.path)
+        return subtitle
 
 
 Service = TvSubtitles

libs/subliminal/subtitles.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from .language import Language
+from .utils import to_unicode
 import os.path
 
 
@@ -34,6 +35,8 @@ class Subtitle(object):
 
     """
     def __init__(self, path, language):
+        if not isinstance(language, Language):
+            raise TypeError('%r is not an instance of Language')
         self.path = path
         self.language = language
 
@@ -44,6 +47,15 @@ class Subtitle(object):
             return os.path.exists(self.path)
         return False
 
+    def __unicode__(self):
+        return to_unicode(self.path)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __repr__(self):
+        return '%s(%s, %s)' % (self.__class__.__name__, self, self.language)
+
 
 class EmbeddedSubtitle(Subtitle):
     """Subtitle embedded in a container
@@ -60,7 +72,7 @@ class EmbeddedSubtitle(Subtitle):
 
     @classmethod
     def from_enzyme(cls, path, subtitle):
-        language = Language(subtitle.language) or None
+        language = Language(subtitle.language, strict=False)
         return cls(path, language, subtitle.trackno)
 
 
@@ -69,15 +81,14 @@ class ExternalSubtitle(Subtitle):
     @classmethod
     def from_path(cls, path):
         """Create an :class:`ExternalSubtitle` from path"""
-        extension = ''
+        extension = None
         for e in EXTENSIONS:
             if path.endswith(e):
                 extension = e
                 break
-        if not extension:
+        if extension is None:
             raise ValueError('Not a supported subtitle extension')
-        language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
-        language = Language(language) or None
+        language = Language(os.path.splitext(path[:len(path) - len(extension)])[1][1:], strict=False)
         return cls(path, language)
 
 
@@ -94,13 +105,13 @@ class ResultSubtitle(ExternalSubtitle):
     :param set keywords: keywords that describe the subtitle
 
     """
-    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()):
+    def __init__(self, path, language, service, link, release=None, confidence=1, keywords=None):
         super(ResultSubtitle, self).__init__(path, language)
         self.service = service
         self.link = link
         self.release = release
         self.confidence = confidence
-        self.keywords = keywords
+        self.keywords = keywords or set()
 
     @property
     def single(self):
@@ -110,12 +121,12 @@ class ResultSubtitle(ExternalSubtitle):
         :rtype: bool
 
         """
-        extension = os.path.splitext(self.path)[0]
-        language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
-        return Language(language) == Language('und')
+        return self.language == Language('Undetermined')
 
     def __repr__(self):
-        return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
+        if not self.release:
+            return 'ResultSubtitle(%s, %s, %s, %.2f)' % (self.path, self.language, self.service, self.confidence)
+        return 'ResultSubtitle(%s, %s, %s, %.2f, release=%s)' % (self.path, self.language, self.service, self.confidence, self.release.encode('ascii', 'ignore'))
 
 
 def get_subtitle_path(video_path, language, multi):

libs/subliminal/utils.py

@@ -61,4 +61,9 @@ def to_unicode(data):
         raise ValueError('Basestring expected')
     if isinstance(data, unicode):
         return data
+    for encoding in ('utf-8', 'latin-1'):
+        try:
+            return unicode(data, encoding)
+        except UnicodeDecodeError:
+            pass
     return unicode(data, 'utf-8', 'replace')

libs/subliminal/videos.py

@@ -16,6 +16,8 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
 from . import subtitles
+from .language import Language
+from .utils import to_unicode
 import enzyme
 import guessit
 import hashlib
@@ -129,7 +131,6 @@ class Video(object):
             logger.debug(u'Failed parsing %s with enzyme' % self.path)
         if isinstance(video_infos, enzyme.core.AVContainer):
             results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
-
         # cannot use glob here because it chokes if there are any square
         # brackets inside the filename, so we have to use basic string
         # startswith/endswith comparisons
@@ -138,17 +139,18 @@ class Video(object):
         for path in existing:
             for ext in subtitles.EXTENSIONS:
                 if path.endswith(ext):
-                    possible_lang = path[len(basename) + 1:-len(ext)]
-                    if possible_lang == '':
-                        results.append(subtitles.ExternalSubtitle(path, None))
-                    else:
-                        lang = guessit.Language(possible_lang)
-                        if lang:
-                            results.append(subtitles.ExternalSubtitle(path, lang))
+                    language = Language(path[len(basename) + 1:-len(ext)], strict=False)
+                    results.append(subtitles.ExternalSubtitle(path, language))
         return results
 
+    def __unicode__(self):
+        return to_unicode(self.path or self.release)
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
     def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, self.release)
+        return '%s(%s)' % (self.__class__.__name__, self)
 
     def __hash__(self):
         return hash(self.path or self.release)
@@ -212,8 +214,6 @@ def scan(entry, max_depth=3, scan_filter=None, depth=0):
     """
     if depth > max_depth and max_depth != 0:  # we do not want to search the whole file system except if max_depth = 0
         return []
-    if depth == 0:
-        entry = os.path.abspath(entry)
     if os.path.isdir(entry):  # a dir? recurse
         logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
         result = []
@@ -273,6 +273,8 @@ def hash_thesubdb(path):
 
     """
     readsize = 64 * 1024
+    if os.path.getsize(path) < readsize:
+        return None
     with open(path, 'rb') as f:
         data = f.read(readsize)
         f.seek(-readsize, os.SEEK_END)