dinislage/CouchPotatoServer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subliminal update

Browse Source
This commit is contained in:
Ruud committed 2012-10-14 17:58:09 +02:00
1 parent 4dfd8b4cd5
commit 67c87444de
163 files changed
+22377 -804
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

libs/guessit/__init__.py
+66 -8
View File
@@ -18,12 +18,58 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.4.2'
__version__ = '0.5.2'
__all__ = ['Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info']
# Do python3 detection before importing any other module, to be sure that
# it will then always be available
# with code from http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/
import sys
if sys.version_info[0] >= 3:
PY3 = True
unicode_text_type = str
native_text_type = str
base_text_type = str
def u(x):
return str(x)
def s(x):
return x
class UnicodeMixin(object):
__str__ = lambda x: x.__unicode__()
import binascii
def to_hex(x):
return binascii.hexlify(x).decode('utf-8')
else:
PY3 = False
__all__ = [ str(s) for s in __all__ ] # fix imports for python2
unicode_text_type = unicode
native_text_type = str
base_text_type = basestring
def u(x):
if isinstance(x, str):
return x.decode('utf-8')
return unicode(x)
def s(x):
if isinstance(x, unicode):
return x.encode('utf-8')
if isinstance(x, list):
return [ s(y) for y in x ]
if isinstance(x, tuple):
return tuple(s(y) for y in x)
if isinstance(x, dict):
return dict((s(key), s(value)) for key, value in x.items())
return x
class UnicodeMixin(object):
__str__ = lambda x: unicode(x).encode('utf-8')
def to_hex(x):
return x.encode('hex')
from guessit.guess import Guess, merge_all
from guessit.language import Language
from guessit.matcher import IterativeMatcher
@@ -32,6 +78,7 @@ import logging
log = logging.getLogger(__name__)
class NullHandler(logging.Handler):
def emit(self, record):
pass
@@ -45,7 +92,7 @@ def guess_file_info(filename, filetype, info=None):
"""info can contain the names of the various plugins, such as 'filename' to
detect filename info, or 'hash_md5' to get the md5 hash of the file.
>>> guess_file_info('test/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1'])
>>> guess_file_info('tests/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1'])
{'hash_md5': 'e781de9b94ba2753a8e2945b2c0a123d', 'hash_sha1': 'bfd18e2f4e5d59775c2bc14d80f56971891ed620'}
"""
result = []
@@ -54,7 +101,7 @@ def guess_file_info(filename, filetype, info=None):
if info is None:
info = ['filename']
if isinstance(info, basestring):
if isinstance(info, base_text_type):
info = [info]
for infotype in info:
@@ -67,7 +114,7 @@ def guess_file_info(filename, filetype, info=None):
try:
result.append(Guess({'hash_mpc': hash_file(filename)},
confidence=1.0))
except Exception, e:
except Exception as e:
log.warning('Could not compute MPC-style hash because: %s' % e)
elif infotype == 'hash_ed2k':
@@ -75,7 +122,7 @@ def guess_file_info(filename, filetype, info=None):
try:
result.append(Guess({'hash_ed2k': hash_file(filename)},
confidence=1.0))
except Exception, e:
except Exception as e:
log.warning('Could not compute ed2k hash because: %s' % e)
elif infotype.startswith('hash_'):
@@ -97,17 +144,28 @@ def guess_file_info(filename, filetype, info=None):
hasherobjs = dict(hashers).values()
with open(filename, 'rb') as f:
for chunk in iter(lambda: f.read(blocksize), ''):
chunk = f.read(blocksize)
while chunk:
for hasher in hasherobjs:
hasher.update(chunk)
chunk = f.read(blocksize)
for infotype, hasher in hashers:
result.append(Guess({infotype: hasher.hexdigest()},
confidence=1.0))
except Exception, e:
except Exception as e:
log.warning('Could not compute hash because: %s' % e)
return merge_all(result)
result = merge_all(result)
# last minute adjustments
# if country is in the guessed properties, make it part of the filename
if 'country' in result:
result['series'] += ' (%s)' % result['country'].alpha2.upper()
return result
def guess_video_info(filename, info=None):
libs/guessit/country.py
+8 -11
View File
@@ -19,10 +19,12 @@
#
from __future__ import unicode_literals
from guessit import fileutils
from guessit.textutils import to_unicode
from guessit import UnicodeMixin, base_text_type, u
from guessit.fileutils import load_file_in_same_dir
import logging
__all__ = [ 'Country' ]
log = logging.getLogger(__name__)
@@ -32,8 +34,7 @@ log = logging.getLogger(__name__)
# "An English name, an alpha-2 code (when given),
# an alpha-3 code (when given), a numeric code, and an ISO 31666-2 code
# are all separated by pipe (|) characters."
_iso3166_contents = fileutils.load_file_in_same_dir(__file__,
'ISO-3166-1_utf8.txt').decode('utf-8')
_iso3166_contents = load_file_in_same_dir(__file__, 'ISO-3166-1_utf8.txt')
country_matrix = [ l.strip().split('|')
for l in _iso3166_contents.strip().split('\n') ]
@@ -59,7 +60,7 @@ country_alpha3_to_alpha2 = dict((c[2].lower(), c[1].lower()) for c in country_ma
class Country(object):
class Country(UnicodeMixin):
"""This class represents a country.
You can initialize it with pretty much anything, as it knows conversion
@@ -67,7 +68,7 @@ class Country(object):
"""
def __init__(self, country, strict=False):
country = to_unicode(country.strip().lower())
country = u(country.strip().lower())
self.alpha3 = country_to_alpha3.get(country)
if self.alpha3 is None and strict:
@@ -93,7 +94,7 @@ class Country(object):
if isinstance(other, Country):
return self.alpha3 == other.alpha3
if isinstance(other, basestring):
if isinstance(other, base_text_type):
try:
return self == Country(other)
except ValueError:
@@ -107,9 +108,5 @@ class Country(object):
def __unicode__(self):
return self.english_name
def __str__(self):
return unicode(self).encode('utf-8')
def __repr__(self):
return 'Country(%s)' % self.english_name
libs/guessit/date.py
+4 -1
View File
@@ -18,9 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
import datetime
import re
def valid_year(year):
return 1920 < year < datetime.date.today().year + 5
def search_year(string):
"""Looks for year patterns, and if found return the year and group span.
@@ -40,7 +43,7 @@ def search_year(string):
match = re.search(r'[^0-9]([0-9]{4})[^0-9]', string)
if match:
year = int(match.group(1))
if 1920 < year < datetime.date.today().year + 5:
if valid_year(year):
return (year, match.span(1))
return (None, None)
libs/guessit/fileutils.py
+6 -4
View File
@@ -18,6 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import s, u
import os.path
import zipfile
@@ -31,10 +33,10 @@ def split_path(path):
- the drive letter on Windows systems (eg: r'C:\')
- the mount point '\\' on Windows systems (eg: r'\\host\share')
>>> split_path('/usr/bin/smewt')
>>> s(split_path('/usr/bin/smewt'))
['/', 'usr', 'bin', 'smewt']
>>> split_path('relative_path/to/my_folder/')
>>> s(split_path('relative_path/to/my_folder/'))
['relative_path', 'to', 'my_folder']
"""
@@ -65,7 +67,7 @@ def split_path(path):
def file_in_same_dir(ref_file, desired_file):
"""Return the path for a file in the same dir as a given reference file.
>>> file_in_same_dir('~/smewt/smewt.db', 'smewt.settings')
>>> s(file_in_same_dir('~/smewt/smewt.db', 'smewt.settings'))
'~/smewt/smewt.settings'
"""
@@ -82,4 +84,4 @@ def load_file_in_same_dir(ref_file, filename):
zfile = zipfile.ZipFile(zfilename)
return zfile.read('/'.join(path[i + 1:]))
return open(os.path.join(*path)).read()
return u(open(os.path.join(*path)).read())
libs/guessit/guess.py
+22 -20
View File
@@ -18,6 +18,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import UnicodeMixin, s, u, base_text_type
from guessit.language import Language
from guessit.country import Country
import json
import datetime
import logging
@@ -25,7 +29,7 @@ import logging
log = logging.getLogger(__name__)
class Guess(dict):
class Guess(UnicodeMixin, dict):
"""A Guess is a dictionary which has an associated confidence for each of
its values.
@@ -44,23 +48,21 @@ class Guess(dict):
for prop in self:
self._confidence[prop] = confidence
def to_utf8_dict(self):
from guessit.language import Language
def to_dict(self):
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, Language):
data[prop] = str(value)
elif isinstance(value, unicode):
data[prop] = value.encode('utf-8')
elif isinstance(value, (Language, Country, base_text_type)):
data[prop] = u(value)
elif isinstance(value, list):
data[prop] = [str(x) for x in value]
data[prop] = [u(x) for x in value]
return data
def nice_string(self):
data = self.to_utf8_dict()
data = self.to_dict()
parts = json.dumps(data, indent=4).split('\n')
for i, p in enumerate(parts):
@@ -72,8 +74,8 @@ class Guess(dict):
return '\n'.join(parts)
def __str__(self):
return str(self.to_utf8_dict())
def __unicode__(self):
return u(self.to_dict())
def confidence(self, prop):
return self._confidence.get(prop, -1)
@@ -138,16 +140,16 @@ def choose_string(g1, g2):
differ very little, such as one string being the other one with the 'the' word
prepended to it.
>>> choose_string(('Hello', 0.75), ('World', 0.5))
>>> s(choose_string(('Hello', 0.75), ('World', 0.5)))
('Hello', 0.25)
>>> choose_string(('Hello', 0.5), ('hello', 0.5))
>>> s(choose_string(('Hello', 0.5), ('hello', 0.5)))
('Hello', 0.75)
>>> choose_string(('Hello', 0.4), ('Hello World', 0.4))
>>> s(choose_string(('Hello', 0.4), ('Hello World', 0.4)))
('Hello', 0.64)
>>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5))
>>> s(choose_string(('simpsons', 0.5), ('The Simpsons', 0.5)))
('The Simpsons', 0.75)
"""
@@ -285,12 +287,12 @@ def merge_all(guesses, append=None):
You can specify a list of properties that should be appended into a list
instead of being merged.
>>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ])
>>> s(merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ]))
{'season': 2, 'episodeNumber': 13}
>>> merge_all([ Guess({ 'episodeNumber': 27 }, confidence = 0.02),
... Guess({ 'season': 1 }, confidence = 0.2) ])
>>> s(merge_all([ Guess({ 'episodeNumber': 27 }, confidence = 0.02),
... Guess({ 'season': 1 }, confidence = 0.2) ]))
{'season': 1}
"""
@@ -320,7 +322,7 @@ def merge_all(guesses, append=None):
result.update_highest_confidence(g)
# delete very unlikely values
for p in result.keys():
for p in list(result.keys()):
if result.confidence(p) < 0.05:
del result[p]
libs/guessit/hash_ed2k.py
+4 -2
View File
@@ -18,6 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import s, to_hex
import hashlib
import os.path
@@ -25,7 +27,7 @@ import os.path
def hash_file(filename):
"""Returns the ed2k hash of a given file.
>>> hash_file('test/dummy.srt')
>>> s(hash_file('tests/dummy.srt'))
'ed2k://|file|dummy.srt|44|1CA0B9DED3473B926AA93A0A546138BB|/'
"""
return 'ed2k://|file|%s|%d|%s|/' % (os.path.basename(filename),
@@ -58,6 +60,6 @@ def hash_filehash(filename):
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return hashes[0].encode("hex")
return to_hex(hashes[0])
else:
return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexd
libs/guessit/hash_mpc.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
import struct
import os
libs/guessit/language.py
+17 -22
View File
@@ -19,9 +19,9 @@
#
from __future__ import unicode_literals
from guessit import fileutils
from guessit import UnicodeMixin, base_text_type, u, s
from guessit.fileutils import load_file_in_same_dir
from guessit.country import Country
from guessit.textutils import to_unicode
import re
import logging
@@ -39,8 +39,7 @@ log = logging.getLogger(__name__)
# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
# an alpha-2 code (when given), an English name, and a French name of a language
# are all separated by pipe (|) characters."
_iso639_contents = fileutils.load_file_in_same_dir(__file__,
'ISO-639-2_utf-8.txt').decode('utf-8')
_iso639_contents = load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt')
# drop the BOM from the beginning of the file
_iso639_contents = _iso639_contents[1:]
@@ -116,7 +115,6 @@ lng_exceptions = { 'unknown': ('und', None),
'cn': ('chi', None),
'chs': ('chi', None),
'jp': ('jpn', None),
'scc': ('srp', None),
'scr': ('hrv', None)
}
@@ -137,7 +135,7 @@ def lang_set(languages, strict=False):
return set(Language(l, strict=strict) for l in languages)
class Language(object):
class Language(UnicodeMixin):
"""This class represents a human language.
You can initialize it with pretty much anything, as it knows conversion
@@ -154,30 +152,30 @@ class Language(object):
>>> Language('fr')
Language(French)
>>> Language('eng').french_name
u'anglais'
>>> s(Language('eng').french_name)
'anglais'
>>> Language('pt(br)').country.english_name
u'Brazil'
>>> s(Language('pt(br)').country.english_name)
'Brazil'
>>> Language('Español (Latinoamérica)').country.english_name
u'Latin America'
>>> s(Language('Español (Latinoamérica)').country.english_name)
'Latin America'
>>> Language('Spanish (Latin America)') == Language('Español (Latinoamérica)')
True
>>> Language('zz', strict=False).english_name
u'Undetermined'
>>> s(Language('zz', strict=False).english_name)
'Undetermined'
>>> Language('pt(br)').opensubtitles
u'pob'
>>> s(Language('pt(br)').opensubtitles)
'pob'
"""
_with_country_regexp = re.compile('(.*)\((.*)\)')
_with_country_regexp2 = re.compile('(.*)-(.*)')
def __init__(self, language, country=None, strict=False, scheme=None):
language = to_unicode(language.strip().lower())
language = u(language.strip().lower())
with_country = (Language._with_country_regexp.match(language) or
Language._with_country_regexp2.match(language))
if with_country:
@@ -249,7 +247,7 @@ class Language(object):
def opensubtitles(self):
if self.lang == 'por' and self.country and self.country.alpha2 == 'br':
return 'pob'
elif self.lang in ['gre', 'eus', 'ice', 'srp']:
elif self.lang in ['gre', 'srp']:
return self.alpha3term
return self.alpha3
@@ -266,7 +264,7 @@ class Language(object):
if isinstance(other, Language):
return self.lang == other.lang
if isinstance(other, basestring):
if isinstance(other, base_text_type):
try:
return self == Language(other)
except ValueError:
@@ -286,9 +284,6 @@ class Language(object):
else:
return self.english_name
def __str__(self):
return unicode(self).encode('utf-8')
def __repr__(self):
if self.country:
return 'Language(%s, country=%s)' % (self.english_name, self.country)
libs/guessit/matcher.py
+24 -16
View File
@@ -18,8 +18,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import PY3, u
from guessit.matchtree import MatchTree
from guessit.textutils import to_utf8
from guessit.guess import (merge_similar_guesses, merge_all,
choose_int, choose_string)
import copy
@@ -69,12 +70,12 @@ class IterativeMatcher(object):
"""
valid_filetypes = ('autodetect', 'subtitle', 'video',
'movie', 'moviesubtitle',
'episode', 'episodesubtitle')
'movie', 'moviesubtitle',
'episode', 'episodesubtitle')
if filetype not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" % valid_filetypes)
if not isinstance(filename, unicode):
log.debug('WARNING: given filename to matcher is not unicode...')
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
self.match_tree = MatchTree(filename)
mtree = self.match_tree
@@ -99,21 +100,26 @@ class IterativeMatcher(object):
apply_transfo('split_explicit_groups')
# 4- try to match information for specific patterns
# NOTE: order needs to comply to the following:
# - website before language (eg: tvu.org.ru vs russian)
# - language before episodes_rexps
# - properties before language (eg: he-aac vs hebrew)
# - release_group before properties (eg: XviD-?? vs xvid)
if mtree.guess['type'] in ('episode', 'episodesubtitle'):
strategy = ['guess_date', 'guess_video_rexps',
'guess_episodes_rexps', 'guess_website',
'guess_release_group', 'guess_properties',
'guess_weak_episodes_rexps', 'guess_language']
strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
'guess_properties', 'guess_language',
'guess_video_rexps',
'guess_episodes_rexps', 'guess_weak_episodes_rexps' ]
else:
strategy = ['guess_date', 'guess_video_rexps',
'guess_website', 'guess_release_group',
'guess_properties', 'guess_language']
strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
'guess_properties', 'guess_language',
'guess_video_rexps' ]
for name in strategy:
apply_transfo(name)
# more guessers for both movies and episodes
for name in ['guess_bonus_features']:
for name in ['guess_bonus_features', 'guess_year', 'guess_country']:
apply_transfo(name)
# split into '-' separated subgroups (with required separator chars
@@ -125,13 +131,12 @@ class IterativeMatcher(object):
if mtree.guess['type'] in ('episode', 'episodesubtitle'):
apply_transfo('guess_episode_info_from_position')
else:
apply_transfo('guess_year')
apply_transfo('guess_movie_title_from_position')
# 6- perform some post-processing steps
apply_transfo('post_process')
log.debug('Found match tree:\n%s' % (to_utf8(unicode(mtree))))
log.debug('Found match tree:\n%s' % u(mtree))
def matched(self):
# we need to make a copy here, as the merge functions work in place and
@@ -147,9 +152,12 @@ class IterativeMatcher(object):
for string_part in ('title', 'series', 'container', 'format',
'releaseGroup', 'website', 'audioCodec',
'videoCodec', 'screenSize', 'episodeFormat'):
'videoCodec', 'screenSize', 'episodeFormat',
'audioChannels'):
merge_similar_guesses(parts, string_part, choose_string)
# 2- merge the rest, potentially discarding information not properly
# merged before
result = merge_all(parts,
append=['language', 'subtitleLanguage', 'other'])
libs/guessit/matchtree.py
+6 -7
View File
@@ -18,15 +18,16 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.textutils import clean_string, str_fill, to_utf8
from __future__ import unicode_literals
from guessit import UnicodeMixin, base_text_type, Guess
from guessit.textutils import clean_string, str_fill
from guessit.patterns import group_delimiters
import logging
log = logging.getLogger(__name__)
class BaseMatchTree(object):
class BaseMatchTree(UnicodeMixin):
"""A MatchTree represents the hierarchical split of a string into its
constituent semantic groups."""
@@ -154,6 +155,7 @@ class BaseMatchTree(object):
'extension': 'e',
'format': 'f',
'language': 'l',
'country': 'C',
'videoCodec': 'v',
'audioCodec': 'a',
'website': 'w',
@@ -198,9 +200,6 @@ class BaseMatchTree(object):
def __unicode__(self):
return self.to_string()
def __str__(self):
return to_utf8(unicode(self))
class MatchTree(BaseMatchTree):
"""The MatchTree contains a few "utility" methods which are not necessary
@@ -218,7 +217,7 @@ class MatchTree(BaseMatchTree):
return list(self._unidentified_leaves(valid))
def _leaves_containing(self, property_name):
if isinstance(property_name, basestring):
if isinstance(property_name, base_text_type):
property_name = [ property_name ]
for leaf in self._leaves():
libs/guessit/patterns.py
+18 -11
View File
@@ -19,6 +19,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa', 'txt' ]
@@ -40,7 +42,10 @@ episode_rexps = [ # ... Season 2 ...
(r'saison (?P<season>[0-9]+)', 1.0, (0, 0)),
# ... s02e13 ...
(r'[Ss](?P<season>[0-9]{1,2}).{,3}(?P<episodeNumber>(?:[EeXx][0-9]{1,2})+)[^0-9]', 1.0, (0, -1)),
(r'[Ss](?P<season>[0-9]{1,2}).{,3}(?P<episodeNumber>(?:[Ee][0-9]{1,2})+)[^0-9]', 1.0, (0, -1)),
# ... s03-x02 ...
(r'[Ss](?P<season>[0-9]{1,2}).{,3}(?P<bonusNumber>(?:[Xx][0-9]{1,2})+)[^0-9]', 1.0, (0, -1)),
# ... 2x13 ...
(r'[^0-9](?P<season>[0-9]{1,2})(?P<episodeNumber>(?:[xX][0-9]{1,2})+)[^0-9]', 0.8, (1, -1)),
@@ -53,18 +58,16 @@ episode_rexps = [ # ... Season 2 ...
(r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
# ... ep 23 ...
('ep' + sep + r'(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.7, (0, -1))
('ep' + sep + r'(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.7, (0, -1)),
# ... e13 ... for a mini-series without a season number
(r'e(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.6, (0, -1))
]
weak_episode_rexps = [ # ... 213 or 0106 ...
(sep + r'(?P<episodeNumber>[0-9]{1,4})' + sep, (1, -1)),
# ... 2x13 ...
(sep + r'[^0-9](?P<season>[0-9]{1,2})\.(?P<episodeNumber>[0-9]{1,2})[^0-9]' + sep, (1, -1)),
# ... e13 ... for a mini-series without a season number
(r'e(?P<episodeNumber>[0-9]{1,4})[^0-9]', (0, -1)),
(sep + r'(?P<episodeNumber>[0-9]{2,4})' + sep, (1, -1))
]
non_episode_title = [ 'extras', 'rip' ]
@@ -102,7 +105,7 @@ unlikely_series = ['series']
properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'Blu-ray', 'BDRip', 'BRRip',
'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'DVBRip', 'PDTV', 'WEBRip',
'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS' ],
'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS', 'WEB-DL', 'WEBDL' ],
'screenSize': [ '720p', '720', '1080p', '1080' ],
@@ -116,7 +119,10 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE',
'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe',
'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL',
'HDBRiSe' ],
'SiNNERS', 'DiRTY', 'REWARD', 'ECI', 'KiNGS', 'CLUE',
'CtrlHD', 'POD', 'WiKi', 'DIMENSION', 'IMMERSE', 'FQM',
'2HD', 'REPTiLE', 'CTU', 'HALCYON', 'EbP', 'SiTV', 'SAiNTS',
'HDBRiSe', 'AlFleNi-TeaM', 'EVOLVE', '0TV' ],
'episodeFormat': [ 'Minisode', 'Minisodes' ],
@@ -149,6 +155,7 @@ def find_properties(filename):
property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
'HD-DVD': [ 'HDDVD', 'HDDVDRip' ],
'BluRay': [ 'BDRip', 'BRRip', 'Blu-ray' ],
'WEB-DL': [ 'WEBDL' ],
'DVB': [ 'DVBRip', 'PDTV' ],
'Screener': [ 'DVDSCR' ],
'DivX': [ 'DVDivX' ],
libs/guessit/slogging.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
import logging
import sys
libs/guessit/textutils.py
+13 -50
View File
@@ -18,7 +18,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import s
from guessit.patterns import sep
import functools
import unicodedata
import copy
@@ -61,45 +64,6 @@ def str_fill(string, region, c):
return string[:start] + c * (end - start) + string[end:]
def to_utf8(o):
"""Convert all unicode strings found in the given object to utf-8
strings."""
if isinstance(o, unicode):
return o.encode('utf-8')
elif isinstance(o, list):
return [ to_utf8(i) for i in o ]
elif isinstance(o, dict):
# need to do it like that to handle Guess instances correctly
# FIXME: why is that necessary?
result = copy.deepcopy(o)
for key, value in o.items():
result[to_utf8(key)] = to_utf8(value)
return result
else:
return o
def to_unicode(o):
"""Convert all strings found in the given object to normalized
unicode strings, using the UTF-8 codec if needed."""
if isinstance(o, unicode):
return unicodedata.normalize('NFC', o)
if isinstance(o, str):
return unicodedata.normalize('NFC', o.decode('utf-8'))
elif isinstance(o, list):
return [ to_unicode(i) for i in o ]
elif isinstance(o, dict):
# need to do it like that to handle Guess instances correctly
#result = copy.deepcopy(o)
for key, value in o.items():
result[to_unicode(key)] = to_unicode(value)
return result
else:
return o
def levenshtein(a, b):
if not a:
@@ -176,21 +140,20 @@ def find_first_level_groups_span(string, enclosing):
def split_on_groups(string, groups):
"""Split the given string using the different known groups for boundaries.
>>> split_on_groups('0123456789', [ (2, 4) ])
>>> s(split_on_groups('0123456789', [ (2, 4) ]))
['01', '23', '456789']
>>> split_on_groups('0123456789', [ (2, 4), (4, 6) ])
>>> s(split_on_groups('0123456789', [ (2, 4), (4, 6) ]))
['01', '23', '45', '6789']
>>> split_on_groups('0123456789', [ (5, 7), (2, 4) ])
>>> s(split_on_groups('0123456789', [ (5, 7), (2, 4) ]))
['01', '23', '4', '56', '789']
"""
if not groups:
return [ string ]
boundaries = sorted(set(reduce(lambda l, x: l + list(x), groups, [])))
boundaries = sorted(set(functools.reduce(lambda l, x: l + list(x), groups, [])))
if boundaries[0] != 0:
boundaries.insert(0, 0)
if boundaries[-1] != len(string):
@@ -212,22 +175,22 @@ def find_first_level_groups(string, enclosing, blank_sep=None):
This does not return nested groups, ie: '(ab(c)(d))' will return a single group
containing the whole string.
>>> find_first_level_groups('', '()')
>>> s(find_first_level_groups('', '()'))
['']
>>> find_first_level_groups('abcd', '()')
>>> s(find_first_level_groups('abcd', '()'))
['abcd']
>>> find_first_level_groups('abc(de)fgh', '()')
>>> s(find_first_level_groups('abc(de)fgh', '()'))
['abc', '(de)', 'fgh']
>>> find_first_level_groups('(ab(c)(d))', '()', blank_sep = '_')
>>> s(find_first_level_groups('(ab(c)(d))', '()', blank_sep = '_'))
['_ab(c)(d)_']
>>> find_first_level_groups('ab[c]de[f]gh(i)', '[]')
>>> s(find_first_level_groups('ab[c]de[f]gh(i)', '[]'))
['ab', '[c]', 'de', '[f]', 'gh(i)']
>>> find_first_level_groups('()[]()', '()', blank_sep = '-')
>>> s(find_first_level_groups('()[]()', '()', blank_sep = '-'))
['--', '[]', '--']
"""
libs/guessit/transfo/__init__.py
+4 -3
View File
@@ -18,7 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from __future__ import unicode_literals
from guessit import base_text_type, Guess
from guessit.patterns import canonical_form
from guessit.textutils import clean_string
import logging
@@ -41,7 +42,7 @@ def format_guess(guess):
if prop in ('season', 'episodeNumber', 'year', 'cdNumber',
'cdNumberTotal', 'bonusNumber', 'filmNumber'):
guess[prop] = int(guess[prop])
elif isinstance(value, basestring):
elif isinstance(value, base_text_type):
if prop in ('edition',):
value = clean_string(value)
guess[prop] = canonical_form(value)
@@ -63,7 +64,7 @@ def find_and_split_node(node, strategy, logger):
if isinstance(result, Guess):
if confidence is None:
confidence = result.confidence(result.keys()[0])
confidence = result.confidence(list(result.keys())[0])
else:
if confidence is None:
confidence = 1.0
libs/guessit/transfo/guess_bonus_features.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import found_property
import logging
libs/guessit/transfo/guess_country.py
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
#from guessit.transfo import SingleNodeGuesser
#from guessit.date import search_year
from guessit.country import Country
from guessit import Guess
import logging
log = logging.getLogger(__name__)
def process(mtree):
for node in mtree.unidentified_leaves():
# only keep explicit groups (enclosed in parentheses/brackets)
if len(node.node_idx) == 2:
try:
country = Country(node.value[1:-1], strict=True)
if node.value[0] + node.value[-1] not in ['()', '[]', '{}']:
continue
node.guess = Guess(country=country, confidence=1.0)
except ValueError:
pass
libs/guessit/transfo/guess_date.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import SingleNodeGuesser
from guessit.date import search_date
import logging
libs/guessit/transfo/guess_episode_info_from_position.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import found_property
from guessit.patterns import non_episode_title, unlikely_series
import logging
libs/guessit/transfo/guess_episodes_rexps.py
+17 -11
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import episode_rexps
@@ -27,30 +28,35 @@ import logging
log = logging.getLogger(__name__)
def number_list(s):
return re.sub('[^0-9]+', ' ', s).split()
return list(re.sub('[^0-9]+', ' ', s).split())
def guess_episodes_rexps(string):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, string, re.IGNORECASE)
if match:
result = (Guess(match.groupdict(), confidence=confidence),
(match.start() + span_adjust[0],
match.end() + span_adjust[1]))
guess = Guess(match.groupdict(), confidence=confidence)
span = (match.start() + span_adjust[0],
match.end() + span_adjust[1])
# episodes which have a season > 25 are most likely errors
# (Simpsons is at 23!)
if int(result[0].get('season', 0)) > 25:
# (Simpsons is at 24!)
if int(guess.get('season', 0)) > 25:
continue
# decide whether we have only a single episode number or an
# episode list
if result[0].get('episodeNumber'):
eplist = number_list(result[0]['episodeNumber'])
result[0].set('episodeNumber', int(eplist[0]), confidence=confidence)
if guess.get('episodeNumber'):
eplist = number_list(guess['episodeNumber'])
guess.set('episodeNumber', int(eplist[0]), confidence=confidence)
if len(eplist) > 1:
result[0].set('episodeList', map(int, eplist), confidence=confidence)
guess.set('episodeList', list(map(int, eplist)), confidence=confidence)
return result
if guess.get('bonusNumber'):
eplist = number_list(guess['bonusNumber'])
guess.set('bonusNumber', int(eplist[0]), confidence=confidence)
return guess, span
return None, None
libs/guessit/transfo/guess_filetype.py
+83 -23
View File
@@ -18,9 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import Guess
from guessit.patterns import (subtitle_exts, video_exts, episode_rexps,
find_properties, canonical_form)
from guessit.date import valid_year
from guessit.textutils import clean_string
import os.path
import re
import mimetypes
@@ -28,34 +31,22 @@ import logging
log = logging.getLogger(__name__)
# List of well known movies and series, hardcoded because they cannot be
# guessed appropriately otherwise
MOVIES = [ 'OSS 117' ]
SERIES = [ 'Band of Brothers' ]
def guess_filetype(filename, filetype):
other = {}
# look at the extension first
fileext = os.path.splitext(filename)[1][1:].lower()
if fileext in subtitle_exts:
if 'movie' in filetype:
filetype = 'moviesubtitle'
elif 'episode' in filetype:
filetype = 'episodesubtitle'
else:
filetype = 'subtitle'
other = { 'container': fileext }
elif fileext in video_exts:
if filetype == 'autodetect':
filetype = 'video'
other = { 'container': fileext }
else:
if filetype == 'autodetect':
filetype = 'unknown'
other = { 'extension': fileext }
MOVIES = [ m.lower() for m in MOVIES ]
SERIES = [ s.lower() for s in SERIES ]
def guess_filetype(mtree, filetype):
# put the filetype inside a dummy container to be able to have the
# following functions work correctly as closures
# this is a workaround for python 2 which doesn't have the
# 'nonlocal' keyword (python 3 does have it)
filetype_container = [filetype]
other = {}
filename = mtree.string
def upgrade_episode():
if filetype_container[0] == 'video':
@@ -69,14 +60,82 @@ def guess_filetype(filename, filetype):
elif filetype_container[0] == 'subtitle':
filetype_container[0] = 'moviesubtitle'
def upgrade_subtitle():
if 'movie' in filetype_container[0]:
filetype_container[0] = 'moviesubtitle'
elif 'episode' in filetype_container[0]:
filetype_container[0] = 'episodesubtitle'
else:
filetype_container[0] = 'subtitle'
def upgrade(type='unknown'):
if filetype_container[0] == 'autodetect':
filetype_container[0] = type
# look at the extension first
fileext = os.path.splitext(filename)[1][1:].lower()
if fileext in subtitle_exts:
upgrade_subtitle()
other = { 'container': fileext }
elif fileext in video_exts:
upgrade(type='video')
other = { 'container': fileext }
else:
upgrade(type='unknown')
other = { 'extension': fileext }
# check whether we are in a 'Movies', 'Tv Shows', ... folder
folder_rexps = [ (r'Movies?', upgrade_movie),
(r'Tv ?Shows?', upgrade_episode),
(r'Series', upgrade_episode)
]
for frexp, upgrade_func in folder_rexps:
frexp = re.compile(frexp, re.IGNORECASE)
for pathgroup in mtree.children:
if frexp.match(pathgroup.value):
upgrade_func()
# check for a few specific cases which will unintentionally make the
# following heuristics confused (eg: OSS 117 will look like an episode,
# season 1, epnum 17, when it is in fact a movie)
fname = clean_string(filename).lower()
for m in MOVIES:
if m in fname:
upgrade_movie()
for s in SERIES:
if s in fname:
upgrade_episode()
# now look whether there are some specific hints for episode vs movie
if filetype in ('video', 'subtitle'):
if filetype_container[0] in ('video', 'subtitle'):
# if we have an episode_rexp (eg: s02e13), it is an episode
for rexp, _, _ in episode_rexps:
match = re.search(rexp, filename, re.IGNORECASE)
if match:
upgrade_episode()
break
# if we have a 3-4 digit number that's not a year, maybe an episode
match = re.search(r'[^0-9]([0-9]{3,4})[^0-9]', filename)
if match:
fullnumber = int(match.group()[1:-1])
#season = fullnumber // 100
epnumber = fullnumber % 100
possible = True
# check for validity
if epnumber > 40:
possible = False
if valid_year(fullnumber):
possible = False
if possible:
upgrade_episode()
# if we have certain properties characteristic of episodes, it is an ep
for prop, value, _, _ in find_properties(filename):
log.debug('prop: %s = %s' % (prop, value))
if prop == 'episodeFormat':
@@ -87,6 +146,7 @@ def guess_filetype(filename, filetype):
upgrade_episode()
break
# origin-specific type
if 'tvu.org.ru' in filename:
upgrade_episode()
@@ -98,7 +158,7 @@ def guess_filetype(filename, filetype):
def process(mtree, filetype='autodetect'):
filetype, other = guess_filetype(mtree.string, filetype)
filetype, other = guess_filetype(mtree, filetype)
mtree.guess.set('type', filetype, confidence=1.0)
log.debug('Found with confidence %.2f: %s' % (1.0, mtree.guess))
libs/guessit/transfo/guess_language.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.language import search_language
libs/guessit/transfo/guess_movie_title_from_position.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import Guess
import logging
libs/guessit/transfo/guess_properties.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import find_properties
import logging
libs/guessit/transfo/guess_release_group.py
+31 -2
View File
@@ -18,24 +18,53 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import properties, canonical_form
import re
import logging
log = logging.getLogger(__name__)
CODECS = properties['videoCodec']
FORMATS = properties['format']
def adjust_metadata(md):
codec = canonical_form(md['videoCodec'])
if codec in FORMATS:
md['format'] = codec
del md['videoCodec']
return md
def guess_release_group(string):
group_names = [ r'\.(Xvid)-(?P<releaseGroup>.*?)[ \.]',
r'\.(DivX)-(?P<releaseGroup>.*?)[\. ]',
r'\.(DVDivX)-(?P<releaseGroup>.*?)[\. ]',
]
# first try to see whether we have both a known codec and a known release group
group_names = [ r'\.(?P<videoCodec>' + codec + r')-(?P<releaseGroup>.*?)[ \.]'
for codec in (CODECS + FORMATS) ]
for rexp in group_names:
match = re.search(rexp, string, re.IGNORECASE)
if match:
metadata = match.groupdict()
metadata.update({ 'videoCodec': match.group(1) })
return metadata, (match.start() + 1, match.end() - 1)
if canonical_form(metadata['releaseGroup']) in properties['releaseGroup']:
return adjust_metadata(metadata), (match.start(1), match.end(2))
# pick anything as releaseGroup as long as we have a codec in front
# this doesn't include a potential dash ('-') ending the release group
# eg: [...].X264-HiS@SiLUHD-English.[...]
group_names = [ r'\.(?P<videoCodec>' + codec + r')-(?P<releaseGroup>.*?)(-(.*?))?[ \.]'
for codec in (CODECS + FORMATS) ]
for rexp in group_names:
match = re.search(rexp, string, re.IGNORECASE)
if match:
return adjust_metadata(match.groupdict()), (match.start(1), match.end(2))
return None, None
libs/guessit/transfo/guess_video_rexps.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import video_rexps, sep
libs/guessit/transfo/guess_weak_episodes_rexps.py
+8 -2
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import weak_episode_rexps
@@ -40,8 +41,13 @@ def guess_weak_episodes_rexps(string, node):
epnum = int(metadata['episodeNumber'])
if epnum > 100:
return Guess({ 'season': epnum // 100,
'episodeNumber': epnum % 100 },
season, epnum = epnum // 100, epnum % 100
# episodes which have a season > 25 are most likely errors
# (Simpsons is at 23!)
if season > 25:
continue
return Guess({ 'season': season,
'episodeNumber': epnum },
confidence=0.6), span
else:
return Guess(metadata, confidence=0.3), span
libs/guessit/transfo/guess_website.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import websites
import logging
libs/guessit/transfo/guess_year.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.transfo import SingleNodeGuesser
from guessit.date import search_year
import logging
libs/guessit/transfo/post_process.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.patterns import subtitle_exts
import logging
libs/guessit/transfo/split_explicit_groups.py
+4 -2
View File
@@ -18,8 +18,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.textutils import find_first_level_groups
from guessit.patterns import group_delimiters
import functools
import logging
log = logging.getLogger(__name__)
@@ -33,10 +35,10 @@ def process(mtree):
groups = find_first_level_groups(c.value, group_delimiters[0])
for delimiters in group_delimiters:
flatten = lambda l, x: l + find_first_level_groups(x, delimiters)
groups = reduce(flatten, groups, [])
groups = functools.reduce(flatten, groups, [])
# do not do this at this moment, it is not strong enough and can break other
# patterns, such as dates, etc...
#groups = reduce(lambda l, x: l + x.split('-'), groups, [])
#groups = functools.reduce(lambda l, x: l + x.split('-'), groups, [])
c.split_on_components(groups)
libs/guessit/transfo/split_on_dash.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit.patterns import sep
import re
import logging
libs/guessit/transfo/split_path_components.py
+1
View File
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import unicode_literals
from guessit import fileutils
import os.path
import logging