dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subliminal update

Browse Source
This commit is contained in:
Ruud
2012-04-08 17:35:39 +02:00
parent 5321037f08
commit 148e1940ac
85 changed files with 4375 additions and 3420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/guessit/language.py
+53 -44
View File
@@ -19,28 +19,28 @@
#
from guessit import fileutils
import os.path
import re
import logging
log = logging.getLogger('guessit.language')
# downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
#
# Description of the fields:
# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
# an alpha-2 code (when given), an English name, and a French name of a language
# are all separated by pipe (|) characters."
_iso639_contents = fileutils.load_file_in_same_dir(__file__,
'ISO-639-2_utf-8.txt')
language_matrix = [ l.strip().decode('utf-8').split('|')
for l in fileutils.load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt').split('\n') ]
for l in _iso639_contents.split('\n') ]
lng3 = frozenset(filter(bool, (l[0] for l in language_matrix)))
lng3term = frozenset(filter(bool, (l[1] for l in language_matrix)))
lng2 = frozenset(filter(bool, (l[2] for l in language_matrix)))
lng_en_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[3].lower().split('; '))))
lng_fr_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[4].lower().split('; '))))
lng3 = frozenset(l[0] for l in language_matrix if l[0])
lng3term = frozenset(l[1] for l in language_matrix if l[1])
lng2 = frozenset(l[2] for l in language_matrix if l[2])
lng_en_name = frozenset(lng for l in language_matrix
for lng in l[3].lower().split('; ') if lng)
lng_fr_name = frozenset(lng for l in language_matrix
for lng in l[4].lower().split('; ') if lng)
lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name
lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1])
@@ -50,22 +50,29 @@ lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2])
lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2])
# we only return the first given english name, hoping it is the most used one
lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) for l in language_matrix if l[3])
lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) for l in language_matrix if l[3] for en_name in l[3].split('; '))
lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0])
for l in language_matrix if l[3])
lng_en_name_to_lng3 = dict((en_name.lower(), l[0])
for l in language_matrix if l[3]
for en_name in l[3].split('; '))
# we only return the first given french name, hoping it is the most used one
lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) for l in language_matrix if l[4])
lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) for l in language_matrix if l[4] for fr_name in l[4].split('; '))
lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0])
for l in language_matrix if l[4])
lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0])
for l in language_matrix if l[4]
for fr_name in l[4].split('; '))
def is_language(language):
return language.lower() in lng_all_names
class Language(object):
"""This class represents a human language.
You can initialize it with pretty much everything, as it knows conversion from
ISO-639 2-letter and 3-letter codes, English and French names.
You can initialize it with pretty much everything, as it knows conversion
from ISO-639 2-letter and 3-letter codes, English and French names.
>>> Language('fr')
Language(French)
@@ -79,12 +86,16 @@ class Language(object):
if len(language) == 2:
lang = lng2_to_lng3.get(language)
elif len(language) == 3:
lang = language if language in lng3 else lng3term_to_lng3.get(language)
lang = (language
if language in lng3
else lng3term_to_lng3.get(language))
else:
lang = lng_en_name_to_lng3.get(language) or lng_fr_name_to_lng3.get(language)
lang = (lng_en_name_to_lng3.get(language) or
lng_fr_name_to_lng3.get(language))
if lang is None:
raise ValueError, 'The given string "%s" could not be identified as a language' % language
msg = 'The given string "%s" could not be identified as a language'
raise ValueError(msg % language)
self.lang = lang
@@ -103,7 +114,6 @@ class Language(object):
def french_name(self):
return lng3_to_lng_fr_name[self.lang]
def __hash__(self):
return hash(self.lang)
@@ -132,19 +142,15 @@ class Language(object):
return 'Language(%s)' % self
def search_language(string, lang_filter = None):
def search_language(string, lang_filter=None):
"""Looks for language patterns, and if found return the language object,
its group span and an associated confidence.
you can specify a list of allowed languages using the lang_filter argument,
as in lang_filter = [ 'fr', 'eng', 'spanish' ]
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-letter delimiting the language.
>>> search_language('movie [en].avi')
(Language(English), (7, 9), 0.80000000000000004)
(Language(English), (7, 9), 0.8)
>>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es'])
(None, None, None)
@@ -153,25 +159,27 @@ def search_language(string, lang_filter = None):
# list of common words which could be interpreted as languages, but which
# are far too common to be able to say they represent a language in the
# middle of a string (where they most likely carry their commmon meaning)
lng_common_words = frozenset([ # english words
'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
# french words
'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
'mal', 'est', 'vol', 'or', 'mon', 'se',
# spanish words
'la', 'el', 'del', 'por', 'mar',
# other
'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san'
])
lng_common_words = frozenset([
# english words
'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
# french words
'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
'mal', 'est', 'vol', 'or', 'mon', 'se',
# spanish words
'la', 'el', 'del', 'por', 'mar',
# other
'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii',
'vi'
])
sep = r'[](){} \._-+'
if lang_filter:
lang_filter = set(Language(l) for l in lang_filter)
slow = string.lower()
slow = ' %s ' % string.lower()
confidence = 1.0 # for all of them
for lang in lng_all_names:
@@ -183,7 +191,7 @@ def search_language(string, lang_filter = None):
if pos != -1:
end = pos + len(lang)
# make sure our word is always surrounded by separators
if slow[pos-1] not in sep or slow[end] not in sep:
if slow[pos - 1] not in sep or slow[end] not in sep:
continue
language = Language(slow[pos:end])
@@ -201,10 +209,11 @@ def search_language(string, lang_filter = None):
elif len(lang) == 3:
confidence = 0.9
else:
# Note: we could either be really confident that we found a language
# or assume that full language names are too common words
# Note: we could either be really confident that we found a
# language or assume that full language names are too
# common words
confidence = 0.3 # going with the low-confidence route here
return language, (pos, end), confidence
return language, (pos - 1, end - 1), confidence
return None, None, None