dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subliminal update

Browse Source
This commit is contained in:
Ruud
2012-04-08 17:35:39 +02:00
parent 5321037f08
commit 148e1940ac
85 changed files with 4375 additions and 3420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/enzyme/__init__.py
+15 -8
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,14 +17,13 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
import mimetypes
import os
import sys
from exceptions import *
PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']),
('flv', ['video/flv'], ['flv']),
('mkv', ['video/x-matroska', 'application/mkv'], ['mkv', 'mka', 'webm']),
@@ -32,10 +31,18 @@ PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']),
('mpeg', ['video/mpeg'], ['mpeg', 'mpg', 'mp4', 'ts']),
('ogm', ['application/ogg'], ['ogm', 'ogg', 'ogv']),
('real', ['video/real'], ['rm', 'ra', 'ram']),
('riff', ['video/avi'], ['wav', 'avi'])]
('riff', ['video/avi'], ['wav', 'avi'])
]
def parse(path):
"""Parse metadata of the given video
:param string path: path to the video file to parse
:return: a parser corresponding to the video's mimetype or extension
:rtype: :class:`~enzyme.core.AVContainer`
"""
if not os.path.isfile(path):
raise ValueError('Invalid path')
extension = os.path.splitext(path)[1][1:]
@@ -47,7 +54,7 @@ def parse(path):
parser_mime = parser_name
if extension in parser_extensions:
parser_ext = parser_name
parser = parser_mime or parser_ext
parser = parser_mime or parser_ext
if not parser:
raise NoParserError()
mod = __import__(parser, globals=globals(), locals=locals(), fromlist=[], level=-1)
libs/enzyme/asf.py
+62 -64
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,30 +17,29 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
from exceptions import ParseError
import core
import logging
import string
import struct
__all__ = ['Parser']
import struct
import string
import logging
from exceptions import *
import core
# get logging object
log = logging.getLogger(__name__)
def _guid(input):
# Remove any '-'
s = string.join(string.split(input,'-'), '')
s = string.join(string.split(input, '-'), '')
r = ''
if len(s) != 32:
return ''
x = ''
for i in range(0,16):
r+=chr(int(s[2*i:2*i+2],16))
guid = struct.unpack('>IHHBB6s',r)
for i in range(0, 16):
r += chr(int(s[2 * i:2 * i + 2], 16))
guid = struct.unpack('>IHHBB6s', r)
return guid
GUIDS = {
@@ -93,8 +92,7 @@ GUIDS = {
'ASF_Web_Stream_Format' : _guid('DA1E6B13-8359-4050-B398-388E965BF00C'),
'ASF_No_Error_Correction' : _guid('20FB5700-5B55-11CF-A8FD-00805F5C442B'),
'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220'),
}
'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220')}
class Asf(core.AVContainer):
@@ -114,7 +112,7 @@ class Asf(core.AVContainer):
raise ParseError()
(guidstr, objsize, objnum, reserved1, \
reserved2) = struct.unpack('<16sQIBB',h)
reserved2) = struct.unpack('<16sQIBB', h)
guid = self._parseguid(guidstr)
if (guid != GUIDS['ASF_Header_Object']):
@@ -122,9 +120,9 @@ class Asf(core.AVContainer):
if reserved1 != 0x01 or reserved2 != 0x02:
raise ParseError()
log.debug("asf header size: %d / %d objects" % (objsize,objnum))
header = file.read(objsize-30)
for i in range(0,objnum):
log.debug(u'Header size: %d / %d objects' % (objsize, objnum))
header = file.read(objsize - 30)
for _ in range(0, objnum):
h = self._getnextheader(header)
header = header[h[1]:]
@@ -148,19 +146,19 @@ class Asf(core.AVContainer):
stream._appendtable('ASFMETADATA', metadata)
def _parseguid(self,string):
def _parseguid(self, string):
return struct.unpack('<IHHBB6s', string[:16])
def _parsekv(self,s):
def _parsekv(self, s):
pos = 0
(descriptorlen,) = struct.unpack('<H', s[pos:pos+2])
(descriptorlen,) = struct.unpack('<H', s[pos:pos + 2])
pos += 2
descriptorname = s[pos:pos+descriptorlen]
descriptorname = s[pos:pos + descriptorlen]
pos += descriptorlen
descriptortype, valuelen = struct.unpack('<HH', s[pos:pos+4])
descriptortype, valuelen = struct.unpack('<HH', s[pos:pos + 4])
pos += 4
descriptorvalue = s[pos:pos+valuelen]
descriptorvalue = s[pos:pos + valuelen]
pos += valuelen
value = None
if descriptortype == 0x0000:
@@ -182,17 +180,17 @@ class Asf(core.AVContainer):
# WORD
value = struct.unpack('<H', descriptorvalue)[0]
else:
log.debug("Unknown Descriptor Type %d" % descriptortype)
return (pos,descriptorname,value)
log.debug(u'Unknown Descriptor Type %d' % descriptortype)
return (pos, descriptorname, value)
def _parsekv2(self,s):
def _parsekv2(self, s):
pos = 0
strno, descriptorlen, descriptortype, valuelen = struct.unpack('<2xHHHI', s[pos:pos+12])
strno, descriptorlen, descriptortype, valuelen = struct.unpack('<2xHHHI', s[pos:pos + 12])
pos += 12
descriptorname = s[pos:pos+descriptorlen]
descriptorname = s[pos:pos + descriptorlen]
pos += descriptorlen
descriptorvalue = s[pos:pos+valuelen]
descriptorvalue = s[pos:pos + valuelen]
pos += valuelen
value = None
@@ -216,25 +214,25 @@ class Asf(core.AVContainer):
# WORD
value = struct.unpack('<H', descriptorvalue)[0]
else:
log.debug("Unknown Descriptor Type %d" % descriptortype)
return (pos,descriptorname,value,strno)
log.debug(u'Unknown Descriptor Type %d' % descriptortype)
return (pos, descriptorname, value, strno)
def _getnextheader(self,s):
r = struct.unpack('<16sQ',s[:24])
(guidstr,objsize) = r
def _getnextheader(self, s):
r = struct.unpack('<16sQ', s[:24])
(guidstr, objsize) = r
guid = self._parseguid(guidstr)
if guid == GUIDS['ASF_File_Properties_Object']:
log.debug("File Properties Object")
val = struct.unpack('<16s6Q4I',s[24:24+80])
log.debug(u'File Properties Object')
val = struct.unpack('<16s6Q4I', s[24:24 + 80])
(fileid, size, date, packetcount, duration, \
senddur, preroll, flags, minpack, maxpack, maxbr) = \
val
# FIXME: parse date to timestamp
self.length = duration/10000000.0
self.length = duration / 10000000.0
elif guid == GUIDS['ASF_Stream_Properties_Object']:
log.debug("Stream Properties Object [%d]" % objsize)
log.debug(u'Stream Properties Object [%d]' % objsize)
streamtype = self._parseguid(s[24:40])
errortype = self._parseguid(s[40:56])
offset, typelen, errorlen, flags = struct.unpack('<QIIH', s[56:74])
@@ -244,15 +242,15 @@ class Asf(core.AVContainer):
self._set('encrypted', True)
if streamtype == GUIDS['ASF_Video_Media']:
vi = core.VideoStream()
vi.width, vi.height, depth, codec, = struct.unpack('<4xII2xH4s', s[89:89+20])
vi.width, vi.height, depth, codec, = struct.unpack('<4xII2xH4s', s[89:89 + 20])
vi.codec = codec
vi.id = strno
self.video.append(vi)
elif streamtype == GUIDS['ASF_Audio_Media']:
ai = core.AudioStream()
twocc, ai.channels, ai.samplerate, bitrate, block, \
ai.samplebits, = struct.unpack('<HHIIHH', s[78:78+16])
ai.bitrate = 8*bitrate
ai.samplebits, = struct.unpack('<HHIIHH', s[78:78 + 16])
ai.bitrate = 8 * bitrate
ai.codec = twocc
ai.id = strno
self.audio.append(ai)
@@ -260,8 +258,8 @@ class Asf(core.AVContainer):
self._apply_extinfo(strno)
elif guid == GUIDS['ASF_Extended_Stream_Properties_Object']:
streamid, langid, frametime = struct.unpack('<HHQ',s[72:84])
(bitrate,) = struct.unpack('<I', s[40:40+4])
streamid, langid, frametime = struct.unpack('<HHQ', s[72:84])
(bitrate,) = struct.unpack('<I', s[40:40 + 4])
if streamid not in self._extinfo:
self._extinfo[streamid] = [None, None, None, {}]
if frametime == 0:
@@ -271,31 +269,31 @@ class Asf(core.AVContainer):
self._apply_extinfo(streamid)
elif guid == GUIDS['ASF_Header_Extension_Object']:
log.debug("ASF_Header_Extension_Object %d" % objsize)
size = struct.unpack('<I',s[42:46])[0]
data = s[46:46+size]
log.debug(u'ASF_Header_Extension_Object %d' % objsize)
size = struct.unpack('<I', s[42:46])[0]
data = s[46:46 + size]
while len(data):
log.debug("Sub:")
log.debug(u'Sub:')
h = self._getnextheader(data)
data = data[h[1]:]
elif guid == GUIDS['ASF_Codec_List_Object']:
log.debug("List Object")
log.debug(u'List Object')
pass
elif guid == GUIDS['ASF_Error_Correction_Object']:
log.debug("Error Correction")
log.debug(u'Error Correction')
pass
elif guid == GUIDS['ASF_Content_Description_Object']:
log.debug("Content Description Object")
val = struct.unpack('<5H', s[24:24+10])
log.debug(u'Content Description Object')
val = struct.unpack('<5H', s[24:24 + 10])
pos = 34
strings = []
for i in val:
ss = s[pos:pos+i].replace('\0', '').lstrip().rstrip()
ss = s[pos:pos + i].replace('\0', '').lstrip().rstrip()
strings.append(ss)
pos+=i
pos += i
# Set empty strings to None
strings = [x or None for x in strings]
@@ -318,7 +316,7 @@ class Asf(core.AVContainer):
streams = {}
for i in range(0, count):
# Read additional content descriptors
size,key,value,strno = self._parsekv2(s[pos:])
size, key, value, strno = self._parsekv2(s[pos:])
if strno not in streams:
streams[strno] = {}
streams[strno][key] = value
@@ -334,12 +332,12 @@ class Asf(core.AVContainer):
count = struct.unpack('<H', s[24:26])[0]
pos = 26
for i in range(0, count):
idlen = struct.unpack('<B', s[pos:pos+1])[0]
idstring = s[pos+1:pos+1+idlen]
idlen = struct.unpack('<B', s[pos:pos + 1])[0]
idstring = s[pos + 1:pos + 1 + idlen]
idstring = unicode(idstring, 'utf-16').replace('\0', '')
log.debug("Language: %d/%d: %s" % (i+1, count, idstring))
log.debug(u'Language: %d/%d: %r' % (i + 1, count, idstring))
self._languages.append(idstring)
pos += 1+idlen
pos += 1 + idlen
elif guid == GUIDS['ASF_Stream_Bitrate_Properties_Object']:
# This record contains stream bitrate with payload overhead. For
@@ -356,11 +354,11 @@ class Asf(core.AVContainer):
# Just print the type:
for h in GUIDS.keys():
if GUIDS[h] == guid:
log.debug("Unparsed %s [%d]" % (h,objsize))
log.debug(u'Unparsed %r [%d]' % (h, objsize))
break
else:
u = "%.8X-%.4X-%.4X-%.2X%.2X-%s" % guid
log.debug("unknown: len=%d [%d]" % (len(u), objsize))
log.debug(u'unknown: len=%d [%d]' % (len(u), objsize))
return r
libs/enzyme/core.py
+26 -33
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,21 +17,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
import re
import logging
import fourcc
import language
from exceptions import *
from strutils import str_to_unicode, unicode_to_str
UNPRINTABLE_KEYS = ['thumbnail', 'url', 'codec_private']
EXTENSION_DEVICE = 'device'
EXTENSION_DIRECTORY = 'directory'
EXTENSION_STREAM = 'stream'
MEDIACORE = ['title', 'caption', 'comment', 'size', 'type', 'subtype', 'timestamp',
'keywords', 'country', 'language', 'langcode', 'url', 'artist',
'mime', 'datetime', 'tags', 'hash']
@@ -52,8 +45,6 @@ log = logging.getLogger(__name__)
class Media(object):
media = None
"""
Media is the base class to all Media Metadata Containers. It defines
the basic structures that handle metadata. Media and its derivates
@@ -61,10 +52,11 @@ class Media(object):
Specific derivates contain additional keys to the dublin core set that is
defined in Media.
"""
media = None
_keys = MEDIACORE
table_mapping = {}
def __init__(self, hash = None):
def __init__(self, hash=None):
if hash is not None:
# create Media based on dict
for key, value in hash.items():
@@ -139,20 +131,21 @@ class Media(object):
result += '| ' + re.sub(r'\n(.)', r'\n| \1', unicode(item))
# print tables
if log.level >= 10:
for name, table in self.tables.items():
result += '+-- Table %s\n' % str(name)
for key, value in table.items():
try:
value = unicode(value)
if len(value) > 50:
value = u'<unprintable data, size=%d>' % len(value)
except (UnicodeDecodeError, TypeError), e:
try:
value = u'<unprintable data, size=%d>' % len(value)
except AttributeError:
value = u'<unprintable data>'
result += u'| | %s: %s\n' % (unicode(key), value)
#FIXME: WTH?
# if log.level >= 10:
# for name, table in self.tables.items():
# result += '+-- Table %s\n' % str(name)
# for key, value in table.items():
# try:
# value = unicode(value)
# if len(value) > 50:
# value = u'<unprintable data, size=%d>' % len(value)
# except (UnicodeDecodeError, TypeError):
# try:
# value = u'<unprintable data, size=%d>' % len(value)
# except AttributeError:
# value = u'<unprintable data>'
# result += u'| | %s: %s\n' % (unicode(key), value)
return result
def __str__(self):
@@ -253,7 +246,7 @@ class Media(object):
"""
return hasattr(self, key)
def get(self, attr, default = None):
def get(self, attr, default=None):
"""
Returns the given attribute. If the attribute is not set by
the parser return 'default'.
@@ -315,7 +308,7 @@ class Tag(object):
Tag values are strings (for binary data), unicode objects, or datetime
objects for tags that represent dates or times.
"""
def __init__(self, value = None, langcode = 'und', binary = False):
def __init__(self, value=None, langcode='und', binary=False):
super(Tag, self).__init__()
self.value = value
self.langcode = langcode
@@ -363,7 +356,7 @@ class Tags(dict, Tag):
The attribute RATING has a value (PG), but it also has a child tag
COUNTRY that specifies the country code the rating belongs to.
"""
def __init__(self, value = None, langcode = 'und', binary = False):
def __init__(self, value=None, langcode='und', binary=False):
super(Tags, self).__init__()
self.value = value
self.langcode = langcode
@@ -410,7 +403,7 @@ class Chapter(Media):
"""
_keys = ['enabled', 'name', 'pos', 'id']
def __init__(self, name = None, pos = 0):
def __init__(self, name=None, pos=0):
Media.__init__(self)
self.name = name
self.pos = pos
@@ -424,7 +417,7 @@ class Subtitle(Media):
_keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title',
'id', 'codec']
def __init__(self, language = None):
def __init__(self, language=None):
Media.__init__(self)
self.language = language
libs/enzyme/exceptions.py
+2 -5
View File
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of enzyme.
#
@@ -15,10 +15,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
class Error(Exception):
pass
libs/enzyme/flv.py
+17 -18
View File
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -16,16 +16,15 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
from exceptions import *
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
from exceptions import ParseError
import core
import logging
import struct
__all__ = ['Parser']
# get logging object
log = logging.getLogger(__name__)
@@ -44,7 +43,7 @@ FLV_AUDIO_CODECID = (0x0001, 0x0002, 0x0055, 0x0001)
# video flags
FLV_VIDEO_CODECID_MASK = 0x0f
FLV_VIDEO_CODECID = ( 'FLV1', 'MSS1', 'VP60') # wild guess
FLV_VIDEO_CODECID = ('FLV1', 'MSS1', 'VP60') # wild guess
FLV_DATA_TYPE_NUMBER = 0x00
FLV_DATA_TYPE_BOOL = 0x01
@@ -70,7 +69,7 @@ class FlashVideo(core.AVContainer):
"""
table_mapping = { 'FLVINFO' : FLVINFO }
def __init__(self,file):
def __init__(self, file):
core.AVContainer.__init__(self)
self.mime = 'video/flv'
self.type = 'Flash Video'
@@ -78,7 +77,7 @@ class FlashVideo(core.AVContainer):
if len(data) < 13 or struct.unpack('>3sBBII', data)[0] != 'FLV':
raise ParseError()
for i in range(10):
for _ in range(10):
if self.audio and self.video:
break
data = file.read(11)
@@ -116,13 +115,13 @@ class FlashVideo(core.AVContainer):
file.seek(size - 1, 1)
elif chunk[0] == FLV_TAG_TYPE_META:
log.info('metadata %s', str(chunk))
log.info(u'metadata %r', str(chunk))
metadata = file.read(size)
try:
while metadata:
length, value = self._parse_value(metadata)
if isinstance(value, dict):
log.info('metadata: %s', value)
log.info(u'metadata: %r', value)
if value.get('creator'):
self.copyright = value.get('creator')
if value.get('width'):
@@ -139,7 +138,7 @@ class FlashVideo(core.AVContainer):
except (IndexError, struct.error, TypeError):
pass
else:
log.info('unkown %s', str(chunk))
log.info(u'unkown %r', str(chunk))
file.seek(size, 1)
file.seek(4, 1)
@@ -157,16 +156,16 @@ class FlashVideo(core.AVContainer):
if ord(data[0]) == FLV_DATA_TYPE_STRING:
length = (ord(data[1]) << 8) + ord(data[2])
return length + 3, data[3:length+3]
return length + 3, data[3:length + 3]
if ord(data[0]) == FLV_DATA_TYPE_ECMARRAY:
init_length = len(data)
num = struct.unpack('>I', data[1:5])[0]
data = data[5:]
result = {}
for i in range(num):
for _ in range(num):
length = (ord(data[0]) << 8) + ord(data[1])
key = data[2:length+2]
key = data[2:length + 2]
data = data[length + 2:]
length, value = self._parse_value(data)
if not length:
@@ -175,7 +174,7 @@ class FlashVideo(core.AVContainer):
data = data[length:]
return init_length - len(data), result
log.info('unknown code: %x. Stop metadata parser', ord(data[0]))
log.info(u'unknown code: %x. Stop metadata parser', ord(data[0]))
return 0, None
libs/enzyme/fourcc.py
+3 -6
View File
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -16,10 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
import string
import re
import struct
libs/enzyme/infos.py
+3 -10
View File
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of enzyme.
#
@@ -15,12 +15,5 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__title__ = 'enzyme'
__description__ = 'Video metadata parser'
__version__ = '0.1'
__author__ = 'Antoine Bertin'
__email__ = 'diaoulael@gmail.com'
__license__ = 'GPLv3'
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__version__ = '0.2'
libs/enzyme/language.py
+4 -7
View File
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -16,10 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
import re
__all__ = ['resolve']
@@ -44,7 +41,7 @@ def resolve(code):
if code in spec[:-1]:
return code, spec[-1]
return code, u'Unknown (%s)' % code
return code, u'Unknown (%r)' % code
# Parsed from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
libs/enzyme/mkv.py
+33 -34
View File
@@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright (C) 2003-2006 Jason Tackaberry <tack@urandom.ca>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2003-2006 Jason Tackaberry <tack@urandom.ca>
#
# This file is part of enzyme.
#
@@ -18,18 +18,17 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
from exceptions import *
from exceptions import ParseError
from struct import unpack
import core
import logging
import re
__all__ = ['Parser']
# get logging object
log = logging.getLogger(__name__)
@@ -236,7 +235,7 @@ class EbmlEntity:
self.compute_id(inbuf)
if self.id_len == 0:
log.error("EBML entity not found, bad file format")
log.error(u'EBML entity not found, bad file format')
raise ParseError()
self.entity_len, self.len_size = self.compute_len(inbuf[self.id_len:])
@@ -247,7 +246,7 @@ class EbmlEntity:
# if the data size is 8 or less, it could be a numeric value
self.value = 0
if self.entity_len <= 8:
for pos, shift in zip(range(self.entity_len), range((self.entity_len-1)*8, -1, -8)):
for pos, shift in zip(range(self.entity_len), range((self.entity_len - 1) * 8, -1, -8)):
self.value |= ord(self.entity_data[pos]) << shift
@@ -271,19 +270,19 @@ class EbmlEntity:
if len(inbuf) < 2:
return 0
self.id_len = 2
self.entity_id = ord(inbuf[0])<<8 | ord(inbuf[1])
self.entity_id = ord(inbuf[0]) << 8 | ord(inbuf[1])
elif first & 0x20:
if len(inbuf) < 3:
return 0
self.id_len = 3
self.entity_id = (ord(inbuf[0])<<16) | (ord(inbuf[1])<<8) | \
self.entity_id = (ord(inbuf[0]) << 16) | (ord(inbuf[1]) << 8) | \
(ord(inbuf[2]))
elif first & 0x10:
if len(inbuf) < 4:
return 0
self.id_len = 4
self.entity_id = (ord(inbuf[0])<<24) | (ord(inbuf[1])<<16) | \
(ord(inbuf[2])<<8) | (ord(inbuf[3]))
self.entity_id = (ord(inbuf[0]) << 24) | (ord(inbuf[1]) << 16) | \
(ord(inbuf[2]) << 8) | (ord(inbuf[3]))
self.entity_str = inbuf[0:self.id_len]
@@ -381,7 +380,7 @@ class Matroska(core.AVContainer):
if header.get_id() != MATROSKA_HEADER_ID:
raise ParseError()
log.debug("HEADER ID found %08X" % header.get_id() )
log.debug(u'HEADER ID found %08X' % header.get_id())
self.mime = 'video/x-matroska'
self.type = 'Matroska'
self.has_idx = False
@@ -392,10 +391,10 @@ class Matroska(core.AVContainer):
# Record file offset of segment data for seekheads
self.segment.offset = header.get_total_len() + segment.get_header_len()
if segment.get_id() != MATROSKA_SEGMENT_ID:
log.debug("SEGMENT ID not found %08X" % segment.get_id())
log.debug(u'SEGMENT ID not found %08X' % segment.get_id())
return
log.debug("SEGMENT ID found %08X" % segment.get_id())
log.debug(u'SEGMENT ID found %08X' % segment.get_id())
try:
for elem in self.process_one_level(segment):
if elem.get_id() == MATROSKA_SEEKHEAD_ID:
@@ -404,12 +403,12 @@ class Matroska(core.AVContainer):
pass
if not self.has_idx:
log.warning('File has no index')
log.warning(u'File has no index')
self._set('corrupt', True)
def process_elem(self, elem):
elem_id = elem.get_id()
log.debug('BEGIN: process element %s' % hex(elem_id))
log.debug(u'BEGIN: process element %r' % hex(elem_id))
if elem_id == MATROSKA_SEGMENT_INFO_ID:
duration = 0
scalecode = 1000000.0
@@ -423,7 +422,7 @@ class Matroska(core.AVContainer):
elif ielem_id == MATROSKA_TITLE_ID:
self.title = ielem.get_utf8()
elif ielem_id == MATROSKA_DATE_UTC_ID:
timestamp = unpack('!q', ielem.get_data())[0] / 10.0**9
timestamp = unpack('!q', ielem.get_data())[0] / 10.0 ** 9
# Date is offset 2001-01-01 00:00:00 (timestamp 978307200.0)
self.timestamp = int(timestamp + 978307200)
@@ -447,7 +446,7 @@ class Matroska(core.AVContainer):
elif elem_id == MATROSKA_CUES_ID:
self.has_idx = True
log.debug('END: process element %s' % hex(elem_id))
log.debug(u'END: process element %r' % hex(elem_id))
return True
@@ -479,7 +478,7 @@ class Matroska(core.AVContainer):
index = 0
while index < tracks.get_len():
trackelem = EbmlEntity(tracksbuf[index:])
log.debug ("ELEMENT %X found" % trackelem.get_id())
log.debug (u'ELEMENT %X found' % trackelem.get_id())
self.process_track(trackelem)
index += trackelem.get_total_len() + trackelem.get_crc_len()
@@ -503,20 +502,20 @@ class Matroska(core.AVContainer):
elements = [x for x in self.process_one_level(track)]
track_type = [x.get_value() for x in elements if x.get_id() == MATROSKA_TRACK_TYPE_ID]
if not track_type:
log.debug('Bad track: no type id found')
log.debug(u'Bad track: no type id found')
return
track_type = track_type[0]
track = None
if track_type == MATROSKA_VIDEO_TRACK:
log.debug("Video track found")
log.debug(u'Video track found')
track = self.process_video_track(elements)
elif track_type == MATROSKA_AUDIO_TRACK:
log.debug("Audio track found")
log.debug(u'Audio track found')
track = self.process_audio_track(elements)
elif track_type == MATROSKA_SUBTITLES_TRACK:
log.debug("Subtitle track found")
log.debug(u'Subtitle track found')
track = core.Subtitle()
self.set_track_defaults(track)
track.id = len(self.subtitles)
@@ -529,7 +528,7 @@ class Matroska(core.AVContainer):
elem_id = elem.get_id()
if elem_id == MATROSKA_TRACK_LANGUAGE_ID:
track.language = elem.get_str()
log.debug("Track language found: %s" % track.language)
log.debug(u'Track language found: %r' % track.language)
elif elem_id == MATROSKA_NAME_ID:
track.title = elem.get_utf8()
elif elem_id == MATROSKA_TRACK_NUMBER_ID:
@@ -671,7 +670,7 @@ class Matroska(core.AVContainer):
elif elem_id == MATROSKA_CHAPTER_UID_ID:
self.objects_by_uid[elem.get_value()] = chap
log.debug('Chapter "%s" found', chap.name)
log.debug(u'Chapter %r found', chap.name)
chap.id = len(self.chapters)
self.chapters.append(chap)
@@ -704,10 +703,10 @@ class Matroska(core.AVContainer):
# Right now we only support attachments that could be cover images.
# Make a guess to see if this attachment is a cover image.
if mimetype.startswith("image/") and u"cover" in (name+desc).lower() and data:
if mimetype.startswith("image/") and u"cover" in (name + desc).lower() and data:
self.thumbnail = data
log.debug('Attachment "%s" found' % name)
log.debug(u'Attachment %r found' % name)
def process_tags(self, tags):
@@ -744,7 +743,7 @@ class Matroska(core.AVContainer):
self.objects_by_uid[target].tags.update(tags_dict)
self.tags_to_attributes(self.objects_by_uid[target], tags_dict)
except KeyError:
log.warning('Tags assigned to unknown/unsupported target uid %d', target)
log.warning(u'Tags assigned to unknown/unsupported target uid %d', target)
else:
self.tags.update(tags_dict)
self.tags_to_attributes(self, tags_dict)
@@ -816,7 +815,7 @@ class Matroska(core.AVContainer):
try:
value = [filter(item) for item in value] if isinstance(value, list) else filter(value)
except Exception, e:
log.warning('Failed to convert tag to core attribute: %s', e)
log.warning(u'Failed to convert tag to core attribute: %r', e)
# Special handling for tv series recordings. The 'title' tag
# can be used for both the series and the episode name. The
# same is true for trackno which may refer to the season
libs/enzyme/mp4.py
+64 -66
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2007 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2007 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2007 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2007 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,16 +17,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['Parser']
import zlib
import logging
import StringIO
import struct
from exceptions import *
from exceptions import ParseError
import core
# get logging object
@@ -169,7 +167,7 @@ class MPEG4(core.AVContainer):
self.type = 'Quicktime Video'
h = file.read(8)
try:
(size, type) = struct.unpack('>I4s',h)
(size, type) = struct.unpack('>I4s', h)
except struct.error:
# EOF.
raise ParseError()
@@ -183,18 +181,18 @@ class MPEG4(core.AVContainer):
self.mime = 'video/mp4'
self.type = 'MPEG-4 Video'
size -= 4
file.seek(size-8, 1)
file.seek(size - 8, 1)
h = file.read(8)
(size, type) = struct.unpack('>I4s',h)
(size, type) = struct.unpack('>I4s', h)
while type in ['mdat', 'skip']:
# movie data at the beginning, skip
file.seek(size-8, 1)
file.seek(size - 8, 1)
h = file.read(8)
(size, type) = struct.unpack('>I4s',h)
(size, type) = struct.unpack('>I4s', h)
if not type in ['moov', 'wide', 'free']:
log.debug('invalid header: %r' % type)
log.debug(u'invalid header: %r' % type)
raise ParseError()
# Extended size
@@ -216,37 +214,37 @@ class MPEG4(core.AVContainer):
if len(s) < 8:
return 0
atomsize,atomtype = struct.unpack('>I4s', s)
atomsize, atomtype = struct.unpack('>I4s', s)
if not str(atomtype).decode('latin1').isalnum():
# stop at nonsense data
return 0
log.debug('%s [%X]' % (atomtype,atomsize))
log.debug(u'%r [%X]' % (atomtype, atomsize))
if atomtype == 'udta':
# Userdata (Metadata)
pos = 0
tabl = {}
i18ntabl = {}
atomdata = file.read(atomsize-8)
while pos < atomsize-12:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
atomdata = file.read(atomsize - 8)
while pos < atomsize - 12:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
if ord(datatype[0]) == 169:
# i18n Metadata...
mypos = 8+pos
while mypos + 4 < datasize+pos:
mypos = 8 + pos
while mypos + 4 < datasize + pos:
# first 4 Bytes are i18n header
(tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos+4])
(tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4])
i18ntabl[lang] = i18ntabl.get(lang, {})
l = atomdata[mypos+4:mypos+tlen+4]
l = atomdata[mypos + 4:mypos + tlen + 4]
i18ntabl[lang][datatype[1:]] = l
mypos += tlen+4
mypos += tlen + 4
elif datatype == 'WLOC':
# Drop Window Location
pass
else:
if ord(atomdata[pos+8:pos+datasize][0]) > 1:
tabl[datatype] = atomdata[pos+8:pos+datasize]
if ord(atomdata[pos + 8:pos + datasize][0]) > 1:
tabl[datatype] = atomdata[pos + 8:pos + datasize]
pos += datasize
if len(i18ntabl.keys()) > 0:
for k in i18ntabl.keys():
@@ -254,19 +252,19 @@ class MPEG4(core.AVContainer):
self._appendtable('QTUDTA', i18ntabl[k])
self._appendtable('QTUDTA', tabl)
else:
log.debug('NO i18')
log.debug(u'NO i18')
self._appendtable('QTUDTA', tabl)
elif atomtype == 'trak':
atomdata = file.read(atomsize-8)
atomdata = file.read(atomsize - 8)
pos = 0
trackinfo = {}
tracktype = None
while pos < atomsize-8:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
while pos < atomsize - 8:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
if datatype == 'tkhd':
tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos+8:pos+datasize])
tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize])
trackinfo['width'] = tkhd[10] >> 16
trackinfo['height'] = tkhd[11] >> 16
trackinfo['id'] = tkhd[3]
@@ -277,23 +275,23 @@ class MPEG4(core.AVContainer):
# XXX Apple time. FIXME to work on Apple, too
self.timestamp = int(tkhd[1]) - 2082844800
except Exception, e:
log.exception('There was trouble extracting timestamp')
log.exception(u'There was trouble extracting timestamp')
elif datatype == 'mdia':
pos += 8
datasize -= 8
log.debug('--> mdia information')
log.debug(u'--> mdia information')
while datasize:
mdia = struct.unpack('>I4s', atomdata[pos:pos+8])
mdia = struct.unpack('>I4s', atomdata[pos:pos + 8])
if mdia[1] == 'mdhd':
# Parse based on version of mdhd header. See
# http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
ver = ord(atomdata[pos + 8])
if ver == 0:
mdhd = struct.unpack('>IIIIIhh', atomdata[pos+8:pos+8+24])
mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24])
elif ver == 1:
mdhd = struct.unpack('>IQQIQhh', atomdata[pos+8:pos+8+36])
mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36])
else:
mdhd = None
@@ -313,34 +311,34 @@ class MPEG4(core.AVContainer):
pos -= (mdia[0] - 8)
datasize += (mdia[0] - 8)
elif mdia[1] == 'hdlr':
hdlr = struct.unpack('>I4s4s', atomdata[pos+8:pos+8+12])
hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12])
if hdlr[1] == 'mhlr':
if hdlr[2] == 'vide':
tracktype = 'video'
if hdlr[2] == 'soun':
tracktype = 'audio'
elif mdia[1] == 'stsd':
stsd = struct.unpack('>2I', atomdata[pos+8:pos+8+8])
stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8])
if stsd[1] > 0:
codec = atomdata[pos+16:pos+16+8]
codec = atomdata[pos + 16:pos + 16 + 8]
codec = struct.unpack('>I4s', codec)
trackinfo['codec'] = codec[1]
if codec[1] == 'jpeg':
tracktype = 'image'
elif mdia[1] == 'dinf':
dref = struct.unpack('>I4s', atomdata[pos+8:pos+8+8])
log.debug(' --> %s, %s (useless)' % mdia)
dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8])
log.debug(u' --> %r, %r (useless)' % mdia)
if dref[1] == 'dref':
num = struct.unpack('>I', atomdata[pos+20:pos+20+4])[0]
rpos = pos+20+4
num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0]
rpos = pos + 20 + 4
for ref in range(num):
# FIXME: do somthing if this references
ref = struct.unpack('>I3s', atomdata[rpos:rpos+7])
data = atomdata[rpos+7:rpos+ref[0]]
ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7])
data = atomdata[rpos + 7:rpos + ref[0]]
rpos += ref[0]
else:
if mdia[1].startswith('st'):
log.debug(' --> %s, %s (sample)' % mdia)
log.debug(u' --> %r, %r (sample)' % mdia)
elif mdia[1] == 'vmhd' and not tracktype:
# indicates that this track is video
tracktype = 'video'
@@ -348,19 +346,19 @@ class MPEG4(core.AVContainer):
# indicates that this track is audio
tracktype = 'audio'
else:
log.debug(' --> %s, %s (unknown)' % mdia)
log.debug(u' --> %r, %r (unknown)' % mdia)
pos += mdia[0]
datasize -= mdia[0]
elif datatype == 'udta':
log.debug(struct.unpack('>I4s', atomdata[:8]))
log.debug(u'udta: %r' % struct.unpack('>I4s', atomdata[:8]))
else:
if datatype == 'edts':
log.debug('--> %s [%d] (edit list)' % \
log.debug(u'--> %r [%d] (edit list)' % \
(datatype, datasize))
else:
log.debug('--> %s [%d] (unknown)' % \
log.debug(u'--> %r [%d] (unknown)' % \
(datatype, datasize))
pos += datasize
@@ -380,7 +378,7 @@ class MPEG4(core.AVContainer):
mvhd = struct.unpack('>6I2h', file.read(28))
self.length = max(self.length, mvhd[4] / mvhd[3])
self.volume = mvhd[6]
file.seek(atomsize-8-28,1)
file.seek(atomsize - 8 - 28, 1)
elif atomtype == 'cmov':
@@ -389,21 +387,21 @@ class MPEG4(core.AVContainer):
if not atomtype == 'dcom':
return atomsize
method = struct.unpack('>4s', file.read(datasize-8))[0]
method = struct.unpack('>4s', file.read(datasize - 8))[0]
datasize, atomtype = struct.unpack('>I4s', file.read(8))
if not atomtype == 'cmvd':
return atomsize
if method == 'zlib':
data = file.read(datasize-8)
data = file.read(datasize - 8)
try:
decompressed = zlib.decompress(data)
except Exception, e:
try:
decompressed = zlib.decompress(data[4:])
except Exception, e:
log.exception('There was a proble decompressiong atom')
log.exception(u'There was a proble decompressiong atom')
return atomsize
decompressedIO = StringIO.StringIO(decompressed)
@@ -411,9 +409,9 @@ class MPEG4(core.AVContainer):
pass
else:
log.info('unknown compression %s' % method)
log.info(u'unknown compression %r' % method)
# unknown compression method
file.seek(datasize-8,1)
file.seek(datasize - 8, 1)
elif atomtype == 'moov':
# decompressed movie info
@@ -423,10 +421,10 @@ class MPEG4(core.AVContainer):
elif atomtype == 'mdat':
pos = file.tell() + atomsize - 8
# maybe there is data inside the mdat
log.info('parsing mdat')
log.info(u'parsing mdat')
while self._readatom(file):
pass
log.info('end of mdat')
log.info(u'end of mdat')
file.seek(pos, 0)
@@ -437,24 +435,24 @@ class MPEG4(core.AVContainer):
elif atomtype == 'rmda':
# reference
atomdata = file.read(atomsize-8)
atomdata = file.read(atomsize - 8)
pos = 0
url = ''
quality = 0
datarate = 0
while pos < atomsize-8:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
while pos < atomsize - 8:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
if datatype == 'rdrf':
rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos+8:pos+20])
rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20])
if rtype == 'url ':
url = atomdata[pos+20:pos+20+rlen]
url = atomdata[pos + 20:pos + 20 + rlen]
if url.find('\0') > 0:
url = url[:url.find('\0')]
elif datatype == 'rmqu':
quality = struct.unpack('>I', atomdata[pos+8:pos+12])[0]
quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0]
elif datatype == 'rmdr':
datarate = struct.unpack('>I', atomdata[pos+12:pos+16])[0]
datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0]
pos += datasize
if url:
@@ -462,11 +460,11 @@ class MPEG4(core.AVContainer):
else:
if not atomtype in ['wide', 'free']:
log.info('unhandled base atom %s' % atomtype)
log.info(u'unhandled base atom %r' % atomtype)
# Skip unknown atoms
try:
file.seek(atomsize-8,1)
file.seek(atomsize - 8, 1)
except IOError:
return 0
libs/enzyme/mpeg.py
+89 -91
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,16 +17,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['Parser']
import os
import struct
import logging
import stat
from exceptions import *
from exceptions import ParseError
import core
# get logging object
@@ -49,7 +47,7 @@ START_CODE = {
0xB7 : 'sequence end',
0xB8 : 'group of pictures',
}
for i in range(0x01,0xAF):
for i in range(0x01, 0xAF):
START_CODE[i] = 'slice_start_code'
##------------------------------------------------------------------------
@@ -96,14 +94,14 @@ TS_SYNC = 0x47
##------------------------------------------------------------------------
FRAME_RATE = [
0,
24000.0/1001, ## 3-2 pulldown NTSC (CPB/Main Level)
24, ## Film (CPB/Main Level)
25, ## PAL/SECAM or 625/60 video
30000.0/1001, ## NTSC (CPB/Main Level)
30, ## drop-frame NTSC or component 525/60 (CPB/Main Level)
50, ## double-rate PAL
60000.0/1001, ## double-rate NTSC
60, ## double-rate, drop-frame NTSC/component 525/60 video
24000.0 / 1001, ## 3-2 pulldown NTSC (CPB/Main Level)
24, ## Film (CPB/Main Level)
25, ## PAL/SECAM or 625/60 video
30000.0 / 1001, ## NTSC (CPB/Main Level)
30, ## drop-frame NTSC or component 525/60 (CPB/Main Level)
50, ## double-rate PAL
60000.0 / 1001, ## double-rate NTSC
60, ## double-rate, drop-frame NTSC/component 525/60 video
]
##------------------------------------------------------------------------
@@ -114,9 +112,9 @@ FRAME_RATE = [
## it, a stream that doesn't adhere to one of these aspect ratios is
## technically considered non-compliant.
##------------------------------------------------------------------------
ASPECT_RATIO = ( None, # Forbidden
1.0, # 1/1 (VGA)
4.0 / 3, # 4/3 (TV)
ASPECT_RATIO = (None, # Forbidden
1.0, # 1/1 (VGA)
4.0 / 3, # 4/3 (TV)
16.0 / 9, # 16/9 (Widescreen)
2.21 # (Cinema)
)
@@ -131,7 +129,7 @@ class MPEG(core.AVContainer):
no additional metadata like title, etc; only codecs, length and
resolution is reported back.
"""
def __init__(self,file):
def __init__(self, file):
core.AVContainer.__init__(self)
self.sequence_header_offset = 0
self.mpeg_version = 2
@@ -190,29 +188,29 @@ class MPEG(core.AVContainer):
a.codec = ac
def dxy(self,file):
def dxy(self, file):
"""
get width and height of the video
"""
file.seek(self.sequence_header_offset+4,0)
file.seek(self.sequence_header_offset + 4, 0)
v = file.read(4)
x = struct.unpack('>H',v[:2])[0] >> 4
y = struct.unpack('>H',v[1:3])[0] & 0x0FFF
return (x,y)
x = struct.unpack('>H', v[:2])[0] >> 4
y = struct.unpack('>H', v[1:3])[0] & 0x0FFF
return (x, y)
def framerate_aspect(self,file):
def framerate_aspect(self, file):
"""
read framerate and aspect ratio
"""
file.seek(self.sequence_header_offset+7,0)
v = struct.unpack( '>B', file.read(1) )[0]
file.seek(self.sequence_header_offset + 7, 0)
v = struct.unpack('>B', file.read(1))[0]
try:
fps = FRAME_RATE[v&0xf]
fps = FRAME_RATE[v & 0xf]
except IndexError:
fps = None
if v>>4 < len(ASPECT_RATIO):
aspect = ASPECT_RATIO[v>>4]
if v >> 4 < len(ASPECT_RATIO):
aspect = ASPECT_RATIO[v >> 4]
else:
aspect = None
return (fps, aspect)
@@ -238,18 +236,18 @@ class MPEG(core.AVContainer):
if pos == -1 or len(buffer) - pos < 5:
buffer = buffer[-10:]
continue
ext = (ord(buffer[pos+4]) >> 4)
ext = (ord(buffer[pos + 4]) >> 4)
if ext == 8:
pass
elif ext == 1:
if (ord(buffer[pos+5]) >> 3) & 1:
if (ord(buffer[pos + 5]) >> 3) & 1:
self._set('progressive', True)
else:
self._set('interlaced', True)
return True
else:
log.debug('ext', ext)
buffer = buffer[pos+4:]
log.debug(u'ext: %r' % ext)
buffer = buffer[pos + 4:]
return False
@@ -292,12 +290,12 @@ class MPEG(core.AVContainer):
## Some parts in the code are based on mpgtx (mpgtx.sf.net)
def bitrate(self,file):
def bitrate(self, file):
"""
read the bitrate (most of the time broken)
"""
file.seek(self.sequence_header_offset+8,0)
t,b = struct.unpack( '>HB', file.read(3) )
file.seek(self.sequence_header_offset + 8, 0)
t, b = struct.unpack('>HB', file.read(3))
vrate = t << 2 | b >> 6
return vrate * 400
@@ -309,9 +307,9 @@ class MPEG(core.AVContainer):
if len(buffer) < 6:
return None
highbit = (ord(buffer[0])&0x20)>>5
highbit = (ord(buffer[0]) & 0x20) >> 5
low4Bytes= ((long(ord(buffer[0])) & 0x18) >> 3) << 30
low4Bytes = ((long(ord(buffer[0])) & 0x18) >> 3) << 30
low4Bytes |= (ord(buffer[0]) & 0x03) << 28
low4Bytes |= ord(buffer[1]) << 20
low4Bytes |= (ord(buffer[2]) & 0xF8) << 12
@@ -319,10 +317,10 @@ class MPEG(core.AVContainer):
low4Bytes |= ord(buffer[3]) << 5
low4Bytes |= (ord(buffer[4])) >> 3
sys_clock_ref=(ord(buffer[4]) & 0x3) << 7
sys_clock_ref|=(ord(buffer[5]) >> 1)
sys_clock_ref = (ord(buffer[4]) & 0x3) << 7
sys_clock_ref |= (ord(buffer[5]) >> 1)
return (long(highbit * (1<<16) * (1<<16)) + low4Bytes) / 90000
return (long(highbit * (1 << 16) * (1 << 16)) + low4Bytes) / 90000
def ReadSCRMpeg1(self, buffer):
@@ -340,7 +338,7 @@ class MPEG(core.AVContainer):
low4Bytes |= ord(buffer[3]) << 7;
low4Bytes |= ord(buffer[4]) >> 1;
return (long(highbit) * (1<<16) * (1<<16) + low4Bytes) / 90000;
return (long(highbit) * (1 << 16) * (1 << 16) + low4Bytes) / 90000;
def ReadPTS(self, buffer):
@@ -350,7 +348,7 @@ class MPEG(core.AVContainer):
high = ((ord(buffer[0]) & 0xF) >> 1)
med = (ord(buffer[1]) << 7) + (ord(buffer[2]) >> 1)
low = (ord(buffer[3]) << 7) + (ord(buffer[4]) >> 1)
return ((long(high) << 30 ) + (med << 15) + low) / 90000
return ((long(high) << 30) + (med << 15) + low) / 90000
def ReadHeader(self, buffer, offset):
@@ -358,25 +356,25 @@ class MPEG(core.AVContainer):
Handle MPEG header in buffer on position offset
Return None on error, new offset or 0 if the new offset can't be scanned
"""
if buffer[offset:offset+3] != '\x00\x00\x01':
if buffer[offset:offset + 3] != '\x00\x00\x01':
return None
id = ord(buffer[offset+3])
id = ord(buffer[offset + 3])
if id == PADDING_PKT:
return offset + (ord(buffer[offset+4]) << 8) + \
ord(buffer[offset+5]) + 6
return offset + (ord(buffer[offset + 4]) << 8) + \
ord(buffer[offset + 5]) + 6
if id == PACK_PKT:
if ord(buffer[offset+4]) & 0xF0 == 0x20:
if ord(buffer[offset + 4]) & 0xF0 == 0x20:
self.type = 'MPEG-1 Video'
self.get_time = self.ReadSCRMpeg1
self.mpeg_version = 1
return offset + 12
elif (ord(buffer[offset+4]) & 0xC0) == 0x40:
elif (ord(buffer[offset + 4]) & 0xC0) == 0x40:
self.type = 'MPEG-2 Video'
self.get_time = self.ReadSCRMpeg2
return offset + (ord(buffer[offset+13]) & 0x07) + 14
return offset + (ord(buffer[offset + 13]) & 0x07) + 14
else:
# I have no idea what just happened, but for some DVB
# recordings done with mencoder this points to a
@@ -412,10 +410,10 @@ class MPEG(core.AVContainer):
if id in [PRIVATE_STREAM1, PRIVATE_STREAM2]:
# private stream. we don't know, but maybe we can guess later
add = ord(buffer[offset+8])
add = ord(buffer[offset + 8])
# if (ord(buffer[offset+6]) & 4) or 1:
# id = ord(buffer[offset+10+add])
if buffer[offset+11+add:offset+15+add].find('\x0b\x77') != -1:
if buffer[offset + 11 + add:offset + 15 + add].find('\x0b\x77') != -1:
# AC3 stream
for a in self.audio:
if a.id == id:
@@ -442,18 +440,18 @@ class MPEG(core.AVContainer):
This MPEG starts with a sequence of 0x00 followed by a PACK Header
http://dvd.sourceforge.net/dvdinfo/packhdr.html
"""
file.seek(0,0)
file.seek(0, 0)
buffer = file.read(10000)
offset = 0
# seek until the 0 byte stop
while offset < len(buffer)-100 and buffer[offset] == '\0':
while offset < len(buffer) - 100 and buffer[offset] == '\0':
offset += 1
offset -= 2
# test for mpeg header 0x00 0x00 0x01
header = '\x00\x00\x01%s' % chr(PACK_PKT)
if offset < 0 or not buffer[offset:offset+4] == header:
if offset < 0 or not buffer[offset:offset + 4] == header:
if not force:
return 0
# brute force and try to find the pack header in the first
@@ -470,9 +468,9 @@ class MPEG(core.AVContainer):
self.ReadHeader(buffer, offset)
# store first timestamp
self.start = self.get_time(buffer[offset+4:])
self.start = self.get_time(buffer[offset + 4:])
while len(buffer) > offset + 1000 and \
buffer[offset:offset+3] == '\x00\x00\x01':
buffer[offset:offset + 3] == '\x00\x00\x01':
# read the mpeg header
new_offset = self.ReadHeader(buffer, offset)
@@ -486,12 +484,12 @@ class MPEG(core.AVContainer):
# skip padding 0 before a new header
while len(buffer) > offset + 10 and \
not ord(buffer[offset+2]):
not ord(buffer[offset + 2]):
offset += 1
else:
# seek to new header by brute force
offset += buffer[offset+4:].find('\x00\x00\x01') + 4
offset += buffer[offset + 4:].find('\x00\x00\x01') + 4
# fill in values for support functions:
self.__seek_size__ = 1000000
@@ -555,15 +553,15 @@ class MPEG(core.AVContainer):
self.video[-1]._set('id', id)
# new mpeg starting
if buffer[header_length+9:header_length+13] == \
if buffer[header_length + 9:header_length + 13] == \
'\x00\x00\x01\xB3' and not self.sequence_header_offset:
# yes, remember offset for later use
self.sequence_header_offset = offset + header_length+9
self.sequence_header_offset = offset + header_length + 9
elif ord(buffer[3]) == 189 or ord(buffer[3]) == 191:
# private stream. we don't know, but maybe we can guess later
id = id or ord(buffer[3]) & 0xF
if align and \
buffer[header_length+9:header_length+11] == '\x0b\x77':
buffer[header_length + 9:header_length + 11] == '\x0b\x77':
# AC3 stream
for a in self.audio:
if a.id == id:
@@ -581,7 +579,7 @@ class MPEG(core.AVContainer):
if ptsdts and ptsdts == ord(buffer[9]) >> 4:
if ord(buffer[9]) >> 4 != ptsdts:
log.warning('WARNING: bad PTS/DTS, please contact us')
log.warning(u'WARNING: bad PTS/DTS, please contact us')
return packet_length, None
# timestamp = self.ReadPTS(buffer[9:14])
@@ -595,8 +593,8 @@ class MPEG(core.AVContainer):
def isPES(self, file):
log.info('trying mpeg-pes scan')
file.seek(0,0)
log.info(u'trying mpeg-pes scan')
file.seek(0, 0)
buffer = file.read(3)
# header (also valid for all mpegs)
@@ -613,7 +611,7 @@ class MPEG(core.AVContainer):
return 0
if timestamp != None and not hasattr(self, 'start'):
self.get_time = self.ReadPTS
bpos = buffer[offset+timestamp:offset+timestamp+5]
bpos = buffer[offset + timestamp:offset + timestamp + 5]
self.start = self.get_time(bpos)
if self.sequence_header_offset and hasattr(self, 'start'):
# we have all informations we need
@@ -702,13 +700,13 @@ class MPEG(core.AVContainer):
# Transport Stream ===============================================
def isTS(self, file):
file.seek(0,0)
file.seek(0, 0)
buffer = file.read(TS_PACKET_LENGTH * 2)
c = 0
while c + TS_PACKET_LENGTH < len(buffer):
if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC:
if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC:
break
c += 1
else:
@@ -718,7 +716,7 @@ class MPEG(core.AVContainer):
self.type = 'MPEG-TS'
while c + TS_PACKET_LENGTH < len(buffer):
start = ord(buffer[c+1]) & 0x40
start = ord(buffer[c + 1]) & 0x40
# maybe load more into the buffer
if c + 2 * TS_PACKET_LENGTH > len(buffer) and c < 500000:
buffer += file.read(10000)
@@ -728,30 +726,30 @@ class MPEG(core.AVContainer):
c += TS_PACKET_LENGTH
continue
tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2])
adapt = (ord(buffer[c+3]) & 0x30) >> 4
tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2])
adapt = (ord(buffer[c + 3]) & 0x30) >> 4
offset = 4
if adapt & 0x02:
# meta info present, skip it for now
adapt_len = ord(buffer[c+offset])
adapt_len = ord(buffer[c + offset])
offset += adapt_len + 1
if not ord(buffer[c+1]) & 0x40:
if not ord(buffer[c + 1]) & 0x40:
# no new pes or psi in stream payload starting
pass
elif adapt & 0x01:
# PES
timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:],
timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:],
tsid)[1]
if timestamp != None:
if not hasattr(self, 'start'):
self.get_time = self.ReadPTS
timestamp = c + offset + timestamp
self.start = self.get_time(buffer[timestamp:timestamp+5])
self.start = self.get_time(buffer[timestamp:timestamp + 5])
elif not hasattr(self, 'audio_ok'):
timestamp = c + offset + timestamp
start = self.get_time(buffer[timestamp:timestamp+5])
start = self.get_time(buffer[timestamp:timestamp + 5])
if start is not None and self.start is not None and \
abs(start - self.start) < 10:
# looks ok
@@ -759,7 +757,7 @@ class MPEG(core.AVContainer):
else:
# timestamp broken
del self.start
log.warning('Timestamp error, correcting')
log.warning(u'Timestamp error, correcting')
if hasattr(self, 'start') and self.start and \
self.sequence_header_offset and self.video and self.audio:
@@ -786,31 +784,31 @@ class MPEG(core.AVContainer):
c = 0
while c + TS_PACKET_LENGTH < len(buffer):
if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC:
if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC:
break
c += 1
else:
return None
while c + TS_PACKET_LENGTH < len(buffer):
start = ord(buffer[c+1]) & 0x40
start = ord(buffer[c + 1]) & 0x40
if not start:
c += TS_PACKET_LENGTH
continue
tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2])
adapt = (ord(buffer[c+3]) & 0x30) >> 4
tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2])
adapt = (ord(buffer[c + 3]) & 0x30) >> 4
offset = 4
if adapt & 0x02:
# meta info present, skip it for now
offset += ord(buffer[c+offset]) + 1
offset += ord(buffer[c + offset]) + 1
if adapt & 0x01:
timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], tsid)[1]
timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:], tsid)[1]
if timestamp is None:
# this should not happen
log.error('bad TS')
log.error(u'bad TS')
return None
return c + offset + timestamp
c += TS_PACKET_LENGTH
@@ -841,7 +839,7 @@ class MPEG(core.AVContainer):
if pos == None:
break
end = self.get_time(buffer[pos:]) or end
buffer = buffer[pos+100:]
buffer = buffer[pos + 100:]
file.close()
return end
@@ -855,7 +853,7 @@ class MPEG(core.AVContainer):
if end == None or self.start == None:
return None
if self.start > end:
return int(((long(1) << 33) - 1 ) / 90000) - self.start + end
return int(((long(1) << 33) - 1) / 90000) - self.start + end
return end - self.start
@@ -897,7 +895,7 @@ class MPEG(core.AVContainer):
return 0
file = open(self.filename)
log.debug('scanning file...')
log.debug(u'scanning file...')
while 1:
file.seek(self.__seek_size__ * 10, 1)
buffer = file.read(self.__sample_size__)
@@ -906,10 +904,10 @@ class MPEG(core.AVContainer):
pos = self.__search__(buffer)
if pos == None:
continue
log.debug('buffer position: %s' % self.get_time(buffer[pos:]))
log.debug(u'buffer position: %r' % self.get_time(buffer[pos:]))
file.close()
log.debug('done scanning file')
log.debug(u'done scanning file')
Parser = MPEG
libs/enzyme/ogm.py
+29 -31
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,9 +17,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['Parser']
import struct
@@ -27,7 +25,7 @@ import re
import stat
import os
import logging
from exceptions import *
from exceptions import ParseError
import core
# get logging object
@@ -82,7 +80,7 @@ class Ogm(core.AVContainer):
# seek to the end of the stream, to avoid scanning the whole file
if (os.stat(file.name)[stat.ST_SIZE] > 50000):
file.seek(os.stat(file.name)[stat.ST_SIZE]-49000)
file.seek(os.stat(file.name)[stat.ST_SIZE] - 49000)
# read the rest of the file into a buffer
h = file.read()
@@ -147,21 +145,21 @@ class Ogm(core.AVContainer):
self._appendtable('VORBISCOMMENT', header)
def _parseOGGS(self,file):
def _parseOGGS(self, file):
h = file.read(27)
if len(h) == 0:
# Regular File end
return None, None
elif len(h) < 27:
log.debug("%d Bytes of Garbage found after End." % len(h))
log.debug(u'%d Bytes of Garbage found after End.' % len(h))
return None, None
if h[:4] != "OggS":
log.debug("Invalid Ogg")
log.debug(u'Invalid Ogg')
raise ParseError()
version = ord(h[4])
if version != 0:
log.debug("Unsupported OGG/OGM Version %d." % version)
log.debug(u'Unsupported OGG/OGM Version %d' % version)
return None, None
head = struct.unpack('<BQIIIB', h[5:])
@@ -178,13 +176,13 @@ class Ogm(core.AVContainer):
h = file.read(1)
packettype = ord(h[0]) & PACKET_TYPE_BITS
if packettype == PACKET_TYPE_HEADER:
h += file.read(nextlen-1)
h += file.read(nextlen - 1)
self._parseHeader(h, granulepos)
elif packettype == PACKED_TYPE_METADATA:
h += file.read(nextlen-1)
h += file.read(nextlen - 1)
self._parseMeta(h)
else:
file.seek(nextlen-1,1)
file.seek(nextlen - 1, 1)
if len(self.all_streams) > serial:
stream = self.all_streams[serial]
if hasattr(stream, 'samplerate') and \
@@ -197,27 +195,27 @@ class Ogm(core.AVContainer):
return granulepos, nextlen + 27 + pageSegCount
def _parseMeta(self,h):
def _parseMeta(self, h):
flags = ord(h[0])
headerlen = len(h)
if headerlen >= 7 and h[1:7] == 'vorbis':
header = {}
nextlen, self.encoder = self._extractHeaderString(h[7:])
numItems = struct.unpack('<I',h[7+nextlen:7+nextlen+4])[0]
start = 7+4+nextlen
for i in range(numItems):
numItems = struct.unpack('<I', h[7 + nextlen:7 + nextlen + 4])[0]
start = 7 + 4 + nextlen
for _ in range(numItems):
(nextlen, s) = self._extractHeaderString(h[start:])
start += nextlen
if s:
a = re.split('=',s)
header[(a[0]).upper()]=a[1]
a = re.split('=', s)
header[(a[0]).upper()] = a[1]
# Put Header fields into info fields
self.type = 'OGG Vorbis'
self.subtype = ''
self.all_header.append(header)
def _parseHeader(self,header,granule):
def _parseHeader(self, header, granule):
headerlen = len(header)
flags = ord(header[0])
@@ -225,7 +223,7 @@ class Ogm(core.AVContainer):
ai = core.AudioStream()
ai.version, ai.channels, ai.samplerate, bitrate_max, ai.bitrate, \
bitrate_min, blocksize, framing = \
struct.unpack('<IBIiiiBB',header[7:7+23])
struct.unpack('<IBIiiiBB', header[7:7 + 23])
ai.codec = 'Vorbis'
#ai.granule = granule
#ai.length = granule / ai.samplerate
@@ -250,12 +248,12 @@ class Ogm(core.AVContainer):
self.all_streams.append(vi)
elif flags & PACKET_TYPE_BITS == PACKET_TYPE_HEADER and \
headerlen >= struct.calcsize(STREAM_HEADER_VIDEO)+1:
headerlen >= struct.calcsize(STREAM_HEADER_VIDEO) + 1:
# New Directshow Format
htype = header[1:9]
if htype[:5] == 'video':
sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO)+9]
sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO) + 9]
streamheader = struct.unpack(STREAM_HEADER_VIDEO, sh)
vi = core.VideoStream()
(type, ssize, timeunit, samplerate, vi.length, buffersize, \
@@ -270,13 +268,13 @@ class Ogm(core.AVContainer):
self.all_streams.append(vi)
elif htype[:5] == 'audio':
sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO)+9]
sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO) + 9]
streamheader = struct.unpack(STREAM_HEADER_AUDIO, sha)
ai = core.AudioStream()
(type, ssize, timeunit, ai.samplerate, ai.length, buffersize, \
ai.bitrate, ai.channels, bloc, ai.bitrate) = streamheader
self.samplerate = ai.samplerate
log.debug("Samplerate %d" % self.samplerate)
log.debug(u'Samplerate %d' % self.samplerate)
self.audio.append(ai)
self.all_streams.append(ai)
@@ -287,15 +285,15 @@ class Ogm(core.AVContainer):
self.all_streams.append(subtitle)
else:
log.debug("Unknown Header")
log.debug(u'Unknown Header')
def _extractHeaderString(self,header):
def _extractHeaderString(self, header):
len = struct.unpack('<I', header[:4])[0]
try:
return (len+4,unicode(header[4:4+len], 'utf-8'))
return (len + 4, unicode(header[4:4 + len], 'utf-8'))
except (KeyError, IndexError, UnicodeDecodeError):
return (len+4,None)
return (len + 4, None)
Parser = Ogm
libs/enzyme/real.py
+39 -41
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,14 +17,12 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['Parser']
import struct
import logging
from exceptions import *
from exceptions import ParseError
import core
# http://www.pcisys.net/~melanson/codecs/rmff.htm
@@ -34,13 +32,13 @@ import core
log = logging.getLogger(__name__)
class RealVideo(core.AVContainer):
def __init__(self,file):
def __init__(self, file):
core.AVContainer.__init__(self)
self.mime = 'video/real'
self.type = 'Real Video'
h = file.read(10)
try:
(object_id,object_size,object_version) = struct.unpack('>4sIH',h)
(object_id, object_size, object_version) = struct.unpack('>4sIH', h)
except struct.error:
# EOF.
raise ParseError()
@@ -49,47 +47,47 @@ class RealVideo(core.AVContainer):
raise ParseError()
file_version, num_headers = struct.unpack('>II', file.read(8))
log.debug("size: %d, ver: %d, headers: %d" % \
(object_size, file_version,num_headers))
for i in range(0,num_headers):
log.debug(u'size: %d, ver: %d, headers: %d' % \
(object_size, file_version, num_headers))
for _ in range(0, num_headers):
try:
oi = struct.unpack('>4sIH',file.read(10))
oi = struct.unpack('>4sIH', file.read(10))
except (struct.error, IOError):
# Header data we expected wasn't there. File may be
# only partially complete.
break
if object_id == 'DATA' and oi[0] != 'INDX':
log.debug('INDX chunk expected after DATA but not found -- file corrupt')
log.debug(u'INDX chunk expected after DATA but not found -- file corrupt')
break
(object_id,object_size,object_version) = oi
(object_id, object_size, object_version) = oi
if object_id == 'DATA':
# Seek over the data chunk rather than reading it in.
file.seek(object_size - 10, 1)
else:
self._read_header(object_id, file.read(object_size-10))
log.debug("%s [%d]" % (object_id,object_size-10))
self._read_header(object_id, file.read(object_size - 10))
log.debug(u'%r [%d]' % (object_id, object_size - 10))
# Read all the following headers
def _read_header(self,object_id,s):
def _read_header(self, object_id, s):
if object_id == 'PROP':
prop = struct.unpack('>9IHH', s)
log.debug(prop)
log.debug(u'PROP: %r' % prop)
if object_id == 'MDPR':
mdpr = struct.unpack('>H7I', s[:30])
log.debug(mdpr)
self.length = mdpr[7]/1000.0
log.debug(u'MDPR: %r' % mdpr)
self.length = mdpr[7] / 1000.0
(stream_name_size,) = struct.unpack('>B', s[30:31])
stream_name = s[31:31+stream_name_size]
pos = 31+stream_name_size
(mime_type_size,) = struct.unpack('>B', s[pos:pos+1])
mime = s[pos+1:pos+1+mime_type_size]
pos += mime_type_size+1
(type_specific_len,) = struct.unpack('>I', s[pos:pos+4])
type_specific = s[pos+4:pos+4+type_specific_len]
pos += 4+type_specific_len
stream_name = s[31:31 + stream_name_size]
pos = 31 + stream_name_size
(mime_type_size,) = struct.unpack('>B', s[pos:pos + 1])
mime = s[pos + 1:pos + 1 + mime_type_size]
pos += mime_type_size + 1
(type_specific_len,) = struct.unpack('>I', s[pos:pos + 4])
type_specific = s[pos + 4:pos + 4 + type_specific_len]
pos += 4 + type_specific_len
if mime[:5] == 'audio':
ai = core.AudioStream()
ai.id = mdpr[0]
@@ -101,20 +99,20 @@ class RealVideo(core.AVContainer):
vi.bitrate = mdpr[2]
self.video.append(vi)
else:
log.debug("Unknown: %s" % mime)
log.debug(u'Unknown: %r' % mime)
if object_id == 'CONT':
pos = 0
(title_len,) = struct.unpack('>H', s[pos:pos+2])
self.title = s[2:title_len+2]
pos += title_len+2
(author_len,) = struct.unpack('>H', s[pos:pos+2])
self.artist = s[pos+2:pos+author_len+2]
pos += author_len+2
(copyright_len,) = struct.unpack('>H', s[pos:pos+2])
self.copyright = s[pos+2:pos+copyright_len+2]
pos += copyright_len+2
(comment_len,) = struct.unpack('>H', s[pos:pos+2])
self.comment = s[pos+2:pos+comment_len+2]
(title_len,) = struct.unpack('>H', s[pos:pos + 2])
self.title = s[2:title_len + 2]
pos += title_len + 2
(author_len,) = struct.unpack('>H', s[pos:pos + 2])
self.artist = s[pos + 2:pos + author_len + 2]
pos += author_len + 2
(copyright_len,) = struct.unpack('>H', s[pos:pos + 2])
self.copyright = s[pos + 2:pos + copyright_len + 2]
pos += copyright_len + 2
(comment_len,) = struct.unpack('>H', s[pos:pos + 2])
self.comment = s[pos + 2:pos + comment_len + 2]
Parser = RealVideo
libs/enzyme/riff.py
+85 -87
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
@@ -17,9 +17,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['Parser']
import os
@@ -27,7 +25,7 @@ import struct
import string
import logging
import time
from exceptions import *
from exceptions import ParseError
import core
# get logging object
@@ -70,7 +68,7 @@ class Riff(core.AVContainer):
"""
table_mapping = { 'AVIINFO' : AVIINFO }
def __init__(self,file):
def __init__(self, file):
core.AVContainer.__init__(self)
# read the header
h = file.read(12)
@@ -90,12 +88,12 @@ class Riff(core.AVContainer):
while self._parseRIFFChunk(file):
pass
except IOError:
log.exception('error in file, stop parsing')
log.exception(u'error in file, stop parsing')
self._find_subtitles(file.name)
if not self.has_idx and isinstance(self, core.AVContainer):
log.debug('WARNING: avi has no index')
log.debug(u'WARNING: avi has no index')
self._set('corrupt', True)
@@ -104,9 +102,9 @@ class Riff(core.AVContainer):
Search for subtitle files. Right now only VobSub is supported
"""
base = os.path.splitext(filename)[0]
if os.path.isfile(base+'.idx') and \
(os.path.isfile(base+'.sub') or os.path.isfile(base+'.rar')):
file = open(base+'.idx')
if os.path.isfile(base + '.idx') and \
(os.path.isfile(base + '.sub') or os.path.isfile(base + '.rar')):
file = open(base + '.idx')
if file.readline().find('VobSub index file') > 0:
for line in file.readlines():
if line.find('id') == 0:
@@ -117,10 +115,10 @@ class Riff(core.AVContainer):
file.close()
def _parseAVIH(self,t):
def _parseAVIH(self, t):
retval = {}
v = struct.unpack('<IIIIIIIIIIIIII',t[0:56])
( retval['dwMicroSecPerFrame'],
v = struct.unpack('<IIIIIIIIIIIIII', t[0:56])
(retval['dwMicroSecPerFrame'],
retval['dwMaxBytesPerSec'],
retval['dwPaddingGranularity'],
retval['dwFlags'],
@@ -133,22 +131,22 @@ class Riff(core.AVContainer):
retval['dwScale'],
retval['dwRate'],
retval['dwStart'],
retval['dwLength'] ) = v
retval['dwLength']) = v
if retval['dwMicroSecPerFrame'] == 0:
log.warning("ERROR: Corrupt AVI")
log.warning(u'ERROR: Corrupt AVI')
raise ParseError()
return retval
def _parseSTRH(self,t):
def _parseSTRH(self, t):
retval = {}
retval['fccType'] = t[0:4]
log.debug("_parseSTRH(%s) : %d bytes" % (retval['fccType'], len(t)))
log.debug(u'_parseSTRH(%r) : %d bytes' % (retval['fccType'], len(t)))
if retval['fccType'] != 'auds':
retval['fccHandler'] = t[4:8]
v = struct.unpack('<IHHIIIIIIIII',t[8:52])
( retval['dwFlags'],
v = struct.unpack('<IHHIIIIIIIII', t[8:52])
(retval['dwFlags'],
retval['wPriority'],
retval['wLanguage'],
retval['dwInitialFrames'],
@@ -159,11 +157,11 @@ class Riff(core.AVContainer):
retval['dwSuggestedBufferSize'],
retval['dwQuality'],
retval['dwSampleSize'],
retval['rcFrame'] ) = v
retval['rcFrame']) = v
else:
try:
v = struct.unpack('<IHHIIIIIIIII',t[8:52])
( retval['dwFlags'],
v = struct.unpack('<IHHIIIIIIIII', t[8:52])
(retval['dwFlags'],
retval['wPriority'],
retval['wLanguage'],
retval['dwInitialFrames'],
@@ -174,7 +172,7 @@ class Riff(core.AVContainer):
retval['dwSuggestedBufferSize'],
retval['dwQuality'],
retval['dwSampleSize'],
retval['rcFrame'] ) = v
retval['rcFrame']) = v
self.delay = float(retval['dwStart']) / \
(float(retval['dwRate']) / retval['dwScale'])
except (KeyError, IndexError, ValueError, ZeroDivisionError):
@@ -183,12 +181,12 @@ class Riff(core.AVContainer):
return retval
def _parseSTRF(self,t,strh):
def _parseSTRF(self, t, strh):
fccType = strh['fccType']
retval = {}
if fccType == 'auds':
v = struct.unpack('<HHHHHH',t[0:12])
( retval['wFormatTag'],
v = struct.unpack('<HHHHHH', t[0:12])
(retval['wFormatTag'],
retval['nChannels'],
retval['nSamplesPerSec'],
retval['nAvgBytesPerSec'],
@@ -209,18 +207,18 @@ class Riff(core.AVContainer):
ai.codec = retval['wFormatTag']
self.audio.append(ai)
elif fccType == 'vids':
v = struct.unpack('<IIIHH',t[0:16])
( retval['biSize'],
v = struct.unpack('<IIIHH', t[0:16])
(retval['biSize'],
retval['biWidth'],
retval['biHeight'],
retval['biPlanes'],
retval['biBitCount'] ) = v
v = struct.unpack('IIIII',t[20:40])
( retval['biSizeImage'],
retval['biBitCount']) = v
v = struct.unpack('IIIII', t[20:40])
(retval['biSizeImage'],
retval['biXPelsPerMeter'],
retval['biYPelsPerMeter'],
retval['biClrUsed'],
retval['biClrImportant'] ) = v
retval['biClrImportant']) = v
vi = core.VideoStream()
vi.codec = t[16:20]
vi.width = retval['biWidth']
@@ -233,15 +231,15 @@ class Riff(core.AVContainer):
return retval
def _parseSTRL(self,t):
def _parseSTRL(self, t):
retval = {}
size = len(t)
i = 0
while i < len(t) - 8:
key = t[i:i+4]
sz = struct.unpack('<I',t[i+4:i+8])[0]
i+=8
key = t[i:i + 4]
sz = struct.unpack('<I', t[i + 4:i + 8])[0]
i += 8
value = t[i:]
if key == 'strh':
@@ -249,40 +247,40 @@ class Riff(core.AVContainer):
elif key == 'strf':
retval[key] = self._parseSTRF(value, retval['strh'])
else:
log.debug("_parseSTRL: unsupported stream tag '%s'", key)
log.debug(u'_parseSTRL: unsupported stream tag %r', key)
i += sz
return retval, i
def _parseODML(self,t):
def _parseODML(self, t):
retval = {}
size = len(t)
i = 0
key = t[i:i+4]
sz = struct.unpack('<I',t[i+4:i+8])[0]
key = t[i:i + 4]
sz = struct.unpack('<I', t[i + 4:i + 8])[0]
i += 8
value = t[i:]
if key != 'dmlh':
log.debug("_parseODML: Error")
log.debug(u'_parseODML: Error')
i += sz - 8
return (retval, i)
def _parseVPRP(self,t):
def _parseVPRP(self, t):
retval = {}
v = struct.unpack('<IIIIIIIIII',t[:4*10])
v = struct.unpack('<IIIIIIIIII', t[:4 * 10])
( retval['VideoFormat'],
(retval['VideoFormat'],
retval['VideoStandard'],
retval['RefreshRate'],
retval['HTotalIn'],
retval['VTotalIn'],
retval['FrameAspectRatio'],
retval['wPixel'],
retval['hPixel'] ) = v[1:-1]
retval['hPixel']) = v[1:-1]
# I need an avi with more informations
# enum {FORMAT_UNKNOWN, FORMAT_PAL_SQUARE, FORMAT_PAL_CCIR_601,
@@ -309,7 +307,7 @@ class Riff(core.AVContainer):
# If the VOL header doesn't appear within 5MB or 5 video chunks,
# give up. The 5MB limit is not likely to apply except in
# pathological cases.
while i < min(1024*1024*5, size - 8) and n_dc < 5:
while i < min(1024 * 1024 * 5, size - 8) and n_dc < 5:
data = file.read(8)
if ord(data[0]) == 0:
# Eat leading nulls.
@@ -317,7 +315,7 @@ class Riff(core.AVContainer):
i += 1
key, sz = struct.unpack('<4sI', data)
if key[2:] != 'dc' or sz > 1024*500:
if key[2:] != 'dc' or sz > 1024 * 500:
# This chunk is not video or is unusually big (> 500KB);
# skip it.
file.seek(sz, 1)
@@ -338,7 +336,7 @@ class Riff(core.AVContainer):
startcode = 0xff
def bits(v, o, n):
# Returns n bits in v, offset o bits.
return (v & 2**n-1 << (64-n-o)) >> 64-n-o
return (v & 2 ** n - 1 << (64 - n - o)) >> 64 - n - o
while pos < sz:
startcode = ((startcode << 8) | ord(data[pos])) & 0xffffffff
@@ -350,7 +348,7 @@ class Riff(core.AVContainer):
if startcode >= 0x120 and startcode <= 0x12F:
# We have the VOL startcode. Pull 64 bits of it and treat
# as a bitstream
v = struct.unpack(">Q", data[pos : pos+8])[0]
v = struct.unpack(">Q", data[pos : pos + 8])[0]
offset = 10
if bits(v, 9, 1):
# is_ol_id, skip over vo_ver_id and vo_priority
@@ -385,36 +383,36 @@ class Riff(core.AVContainer):
if i < size:
# Seek past whatever might be remaining of the movi list.
file.seek(size-i,1)
file.seek(size - i, 1)
def _parseLIST(self,t):
def _parseLIST(self, t):
retval = {}
i = 0
size = len(t)
while i < size-8:
while i < size - 8:
# skip zero
if ord(t[i]) == 0: i += 1
key = t[i:i+4]
key = t[i:i + 4]
sz = 0
if key == 'LIST':
sz = struct.unpack('<I',t[i+4:i+8])[0]
i+=8
key = "LIST:"+t[i:i+4]
value = self._parseLIST(t[i:i+sz])
sz = struct.unpack('<I', t[i + 4:i + 8])[0]
i += 8
key = "LIST:" + t[i:i + 4]
value = self._parseLIST(t[i:i + sz])
if key == 'strl':
for k in value.keys():
retval[k] = value[k]
else:
retval[key] = value
i+=sz
i += sz
elif key == 'avih':
sz = struct.unpack('<I',t[i+4:i+8])[0]
sz = struct.unpack('<I', t[i + 4:i + 8])[0]
i += 8
value = self._parseAVIH(t[i:i+sz])
value = self._parseAVIH(t[i:i + sz])
i += sz
retval[key] = value
elif key == 'strl':
@@ -433,15 +431,15 @@ class Riff(core.AVContainer):
retval[key] = value
i += sz
elif key == 'JUNK':
sz = struct.unpack('<I',t[i+4:i+8])[0]
sz = struct.unpack('<I', t[i + 4:i + 8])[0]
i += sz + 8
else:
sz = struct.unpack('<I',t[i+4:i+8])[0]
i+=8
sz = struct.unpack('<I', t[i + 4:i + 8])[0]
i += 8
# in most cases this is some info stuff
if not key in AVIINFO.keys() and key != 'IDIT':
log.debug("Unknown Key: %s, len: %d" % (key,sz))
value = t[i:i+sz]
log.debug(u'Unknown Key: %r, len: %d' % (key, sz))
value = t[i:i + sz]
if key == 'ISFT':
# product information
if value.find('\0') > 0:
@@ -466,17 +464,17 @@ class Riff(core.AVContainer):
except ValueError:
pass
else:
log.debug('no support for time format %s', value)
i+=sz
log.debug(u'no support for time format %r', value)
i += sz
return retval
def _parseRIFFChunk(self,file):
def _parseRIFFChunk(self, file):
h = file.read(8)
if len(h) < 8:
return False
name = h[:4]
size = struct.unpack('<I',h[4:8])[0]
size = struct.unpack('<I', h[4:8])[0]
if name == 'LIST':
pos = file.tell() - 8
@@ -488,18 +486,18 @@ class Riff(core.AVContainer):
# header), but we do know the video's dimensions, and
# we're dealing with an mpeg4 stream, try to get the aspect
# from the VOL header in the mpeg4 stream.
self._parseLISTmovi(size-4, file)
self._parseLISTmovi(size - 4, file)
return True
elif size > 80000:
log.debug('RIFF LIST "%s" too long to parse: %s bytes' % (key, size))
t = file.seek(size-4,1)
log.debug(u'RIFF LIST %r too long to parse: %r bytes' % (key, size))
t = file.seek(size - 4, 1)
return True
elif size < 5:
log.debug('RIFF LIST "%s" too short: %s bytes' % (key, size))
log.debug(u'RIFF LIST %r too short: %r bytes' % (key, size))
return True
t = file.read(size-4)
log.debug('parse RIFF LIST "%s": %d bytes' % (key, size))
t = file.read(size - 4)
log.debug(u'parse RIFF LIST %r: %d bytes' % (key, size))
value = self._parseLIST(t)
self.header[key] = value
if key == 'INFO':
@@ -511,7 +509,7 @@ class Riff(core.AVContainer):
# no need to add this info to a table
pass
else:
log.debug('Skipping table info %s' % key)
log.debug(u'Skipping table info %r' % key)
elif name == 'JUNK':
self.junkStart = file.tell() - 8
@@ -519,15 +517,15 @@ class Riff(core.AVContainer):
file.seek(size, 1)
elif name == 'idx1':
self.has_idx = True
log.debug('idx1: %s bytes' % size)
log.debug(u'idx1: %r bytes' % size)
# no need to parse this
t = file.seek(size,1)
t = file.seek(size, 1)
elif name == 'RIFF':
log.debug("New RIFF chunk, extended avi [%i]" % size)
log.debug(u'New RIFF chunk, extended avi [%i]' % size)
type = file.read(4)
if type != 'AVIX':
log.debug("Second RIFF chunk is %s, not AVIX, skipping", type)
file.seek(size-4, 1)
log.debug(u'Second RIFF chunk is %r, not AVIX, skipping', type)
file.seek(size - 4, 1)
# that's it, no new informations should be in AVIX
return False
elif name == 'fmt ' and size <= 50:
@@ -556,11 +554,11 @@ class Riff(core.AVContainer):
elif not name.strip(string.printable + string.whitespace):
# check if name is something usefull at all, maybe it is no
# avi or broken
t = file.seek(size,1)
log.debug("Skipping %s [%i]" % (name,size))
t = file.seek(size, 1)
log.debug(u'Skipping %r [%i]' % (name, size))
else:
# bad avi
log.debug("Bad or broken avi")
log.debug(u'Bad or broken avi')
return False
return True
libs/enzyme/strutils.py
+4 -6
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2006-2009 Dirk Meyer <dischi@freevo.org>
# Copyright (C) 2006-2009 Jason Tackaberry
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
# Copyright 2006-2009 Dirk Meyer <dischi@freevo.org>
# Copyright 2006-2009 Jason Tackaberry
#
# This file is part of enzyme.
#
@@ -17,9 +17,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['ENCODING', 'str_to_unicode', 'unicode_to_str']
import locale
libs/guessit/__init__.py
+24 -35
View File
@@ -18,10 +18,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.3-dev'
__all__ = [ 'Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info' ]
__version__ = '0.3.1'
__all__ = ['Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info']
from guessit.guess import Guess, merge_all
@@ -31,6 +31,7 @@ import logging
log = logging.getLogger("guessit")
class NullHandler(logging.Handler):
def emit(self, record):
pass
@@ -40,9 +41,7 @@ h = NullHandler()
log.addHandler(h)
def guess_file_info(filename, filetype, info = [ 'filename' ]):
def guess_file_info(filename, filetype, info=None):
"""info can contain the names of the various plugins, such as 'filename' to
detect filename info, or 'hash_md5' to get the md5 hash of the file.
@@ -52,27 +51,30 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
result = []
hashers = []
if info is None:
info = ['filename']
if isinstance(info, basestring):
info = [ info ]
info = [info]
for infotype in info:
if infotype == 'filename':
m = IterativeMatcher(filename, filetype = filetype)
m = IterativeMatcher(filename, filetype=filetype)
result.append(m.matched())
elif infotype == 'hash_mpc':
import hash_mpc
from guessit.hash_mpc import hash_file
try:
result.append(Guess({ 'hash_mpc': hash_mpc.hash_file(filename) },
confidence = 1.0))
result.append(Guess({'hash_mpc': hash_file(filename)},
confidence=1.0))
except Exception, e:
log.warning('Could not compute MPC-style hash because: %s' % e)
elif infotype == 'hash_ed2k':
import hash_ed2k
from guessit.hash_ed2k import hash_file
try:
result.append(Guess({ 'hash_ed2k': hash_ed2k.hash_file(filename) },
confidence = 1.0))
result.append(Guess({'hash_ed2k': hash_file(filename)},
confidence=1.0))
except Exception, e:
log.warning('Could not compute ed2k hash because: %s' % e)
@@ -88,18 +90,6 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
else:
log.warning('Invalid infotype: %s' % infotype)
"""For plugins which depend on some optional library, import them like that:
if infotype == 'plugin_name':
try:
import optional_lib
except ImportError:
raise Exception, 'The plugin module cannot be loaded because the optional_lib lib is missing'
# do some stuff
"""
# do all the hashes now, but on a single pass
if hashers:
try:
@@ -112,22 +102,21 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
hasher.update(chunk)
for infotype, hasher in hashers:
result.append(Guess({ infotype: hasher.hexdigest() },
confidence = 1.0))
result.append(Guess({infotype: hasher.hexdigest()},
confidence=1.0))
except Exception, e:
log.warning('Could not compute hash because: %s' % e)
return merge_all(result)
def guess_video_info(filename, info = [ 'filename' ]):
def guess_video_info(filename, info=None):
return guess_file_info(filename, 'autodetect', info)
def guess_movie_info(filename, info = [ 'filename' ]):
def guess_movie_info(filename, info=None):
return guess_file_info(filename, 'movie', info)
def guess_episode_info(filename, info = [ 'filename' ]):
def guess_episode_info(filename, info=None):
return guess_file_info(filename, 'episode', info)
libs/guessit/autodetect.py
-75
View File
@@ -1,75 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#from guessit import movie, episode
import os, os.path
import logging
log = logging.getLogger('guessit.autodetect')
def within(x, nrange):
"""Return whether a number is inside a given range, specified as a list or tuple
of the lower and upper bounds."""
low, high = nrange
return low <= x <= high
def guess_filename_info(filename):
log.debug('Trying to guess info for file: ' + filename)
# try to guess info as if it were an episode
episode_info = episode.guess_episode_filename(filename)
# 1- if we found either season/episodeNumber, then we're pretty sure it must
# be an episode
if 'season' in episode_info or 'episodeNumber' in episode_info:
log.debug('Likely an episode as it contains season and/or episodeNumber: ' + filename)
episode_info.update({ 'type': 'episode' }, confidence = 0.9)
return episode_info
# try to guess info as if it were a movie
movie_info = movie.guess_movie_filename(filename)
# 2- if the file exists, try to guess its type using its size
if os.path.exists(filename):
size = os.stat(filename).st_size / (1024 * 1024)
# if size <= 1/2 of 1CD -> episode (very unlikely a movie so small)
if size < 400:
log.debug('Likely an episode due to its small size (%dMB): %s' % (size, filename))
episode_info.update({ 'type': 'episode' }, confidence = 0.8)
return episode_info
# if size > 2G -> movie (even fullHD eps aren't that big yet)
if size > 2048:
log.debug('Likely a movie due to its big size (%dMB): %s' % (size, filename))
movie_info.update({ 'type': 'movie' }, confidence = 0.8)
return movie_info
# if size == 1CD or 2CDs -> movie
if within(size, [690, 710]) or within(size, [1380, 1420]):
log.debug('Likely a movie due to its size close to a CD size (%dMB): %s' % (size, filename))
movie_info.update({ 'type': 'movie' }, confidence = 0.8)
return movie_info
# 3- if all else fails, assume it's a movie
log.debug('Couldn\'t make an informed guess... Assuming file is a movie: %s' % filename)
movie_info.update({ 'type': 'movie' }, confidence = 0.5)
return movie_info
libs/guessit/date.py
+31 -28
View File
@@ -21,6 +21,7 @@
import datetime
import re
def search_year(string):
"""Looks for year patterns, and if found return the year and group span.
Assumes there are sentinels at the beginning and end of the string that
@@ -62,34 +63,35 @@ def search_date(string):
dsep = r'[-/ \.]'
date_rexps = [ # 20010823
r'[^0-9]' +
r'(?P<year>[0-9]{4})' +
r'(?P<month>[0-9]{2})' +
r'(?P<day>[0-9]{2})' +
r'[^0-9]',
date_rexps = [
# 20010823
r'[^0-9]' +
r'(?P<year>[0-9]{4})' +
r'(?P<month>[0-9]{2})' +
r'(?P<day>[0-9]{2})' +
r'[^0-9]',
# 2001-08-23
r'[^0-9]' +
r'(?P<year>[0-9]{4})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<day>[0-9]{2})' +
r'[^0-9]',
# 2001-08-23
r'[^0-9]' +
r'(?P<year>[0-9]{4})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<day>[0-9]{2})' +
r'[^0-9]',
# 23-08-2001
r'[^0-9]' +
r'(?P<day>[0-9]{2})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<year>[0-9]{4})' +
r'[^0-9]',
# 23-08-2001
r'[^0-9]' +
r'(?P<day>[0-9]{2})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<year>[0-9]{4})' +
r'[^0-9]',
# 23-08-01
r'[^0-9]' +
r'(?P<day>[0-9]{2})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<year>[0-9]{2})' +
r'[^0-9]',
]
# 23-08-01
r'[^0-9]' +
r'(?P<day>[0-9]{2})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<year>[0-9]{2})' +
r'[^0-9]',
]
for drexp in date_rexps:
match = re.search(drexp, string)
@@ -98,7 +100,7 @@ def search_date(string):
year, month, day = int(d['year']), int(d['month']), int(d['day'])
# years specified as 2 digits should be adjusted here
if year < 100:
if year > (datetime.date.today().year % 100)+ 5:
if year > (datetime.date.today().year % 100) + 5:
year = 1900 + year
else:
year = 2000 + year
@@ -120,8 +122,9 @@ def search_date(string):
continue
# looks like we have a valid date
# note: span is [+1,-1] because we don't want to include the non-digit char
# note: span is [+1,-1] because we don't want to include the
# non-digit char
start, end = match.span()
return (date, (start+1, end-1))
return (date, (start + 1, end - 1))
return None, None
libs/guessit/filetype.py
-76
View File
@@ -1,76 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import subtitle_exts, video_exts, episode_rexps, find_properties, canonical_form
import os.path
import re
import logging
log = logging.getLogger("guessit.filetype")
def guess_filetype(filename, filetype = 'autodetect'):
other = {}
# look at the extension first
fileext = os.path.splitext(filename)[1][1:].lower()
if fileext in subtitle_exts:
if 'movie' in filetype:
filetype = 'moviesubtitle'
elif 'episode' in filetype:
filetype = 'episodesubtitle'
else:
filetype = 'subtitle'
other = { 'container': fileext }
elif fileext in video_exts:
if filetype == 'autodetect':
filetype = 'video'
other = { 'container': fileext }
else:
if filetype == 'autodetect':
filetype = 'unknown'
other = { 'extension': fileext }
# now look whether there are some specific hints for episode vs movie
if filetype in ('video', 'subtitle'):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, filename, re.IGNORECASE)
if match:
if filetype == 'video':
filetype = 'episode'
elif filetype == 'subtitle':
filetype = 'episodesubtitle'
break
for prop, value, start, end in find_properties(filename):
if canonical_form(value) == 'DVB':
if filetype == 'video':
filetype = 'episode'
elif filetype == 'subtitle':
filetype = 'episodesubtitle'
break
# if no episode info found, assume it's a movie
if filetype == 'video':
filetype = 'movie'
elif filetype == 'subtitle':
filetype = 'moviesubtitle'
return filetype, other
libs/guessit/fileutils.py
+7 -14
View File
@@ -50,11 +50,11 @@ def split_path(path):
# on Unix systems, the root folder is '/'
if head == '/' and tail == '':
return [ '/' ] + result
return ['/'] + result
# on Windows, the root folder is a drive letter (eg: 'C:\')
if len(head) == 3 and head[1:] == ':\\' and tail == '':
return [ head ] + result
return [head] + result
if head == '' and tail == '':
return result
@@ -64,17 +64,10 @@ def split_path(path):
path = head
continue
result = [ tail ] + result
result = [tail] + result
path = head
def split_path_components(filename):
"""Returns the filename split into [ dir*, basename, ext ]."""
result = split_path(filename)
basename = result.pop(-1)
return result + list(os.path.splitext(basename))
def file_in_same_dir(ref_file, desired_file):
"""Return the path for a file in the same dir as a given reference file.
@@ -82,17 +75,17 @@ def file_in_same_dir(ref_file, desired_file):
'~/smewt/smewt.settings'
"""
return os.path.join(*(split_path(ref_file)[:-1] + [ desired_file ]))
return os.path.join(*(split_path(ref_file)[:-1] + [desired_file]))
def load_file_in_same_dir(ref_file, filename):
"""Load a given file. Works even when the file is contained inside a zip."""
path = split_path(ref_file)[:-1] + [ filename ]
path = split_path(ref_file)[:-1] + [filename]
for i, p in enumerate(path):
if p.endswith('.zip'):
zfilename = os.path.join(*path[:i+1])
zfilename = os.path.join(*path[:i + 1])
zfile = zipfile.ZipFile(zfilename)
return zfile.read('/'.join(path[i+1:]))
return zfile.read('/'.join(path[i + 1:]))
return open(os.path.join(*path)).read()
libs/guessit/guess.py
+60 -45
View File
@@ -26,9 +26,12 @@ log = logging.getLogger("guessit.guess")
class Guess(dict):
"""A Guess is a dictionary which has an associated confidence for each of its values.
"""A Guess is a dictionary which has an associated confidence for each of
its values.
As it is a subclass of dict, you can use it everywhere you expect a
simple dict."""
As it is a subclass of dict, you can use it everywhere you expect a simple dict"""
def __init__(self, *args, **kwargs):
try:
confidence = kwargs.pop('confidence')
@@ -52,20 +55,20 @@ class Guess(dict):
elif isinstance(value, unicode):
data[prop] = value.encode('utf-8')
elif isinstance(value, list):
data[prop] = [ str(x) for x in value ]
data[prop] = [str(x) for x in value]
return data
def nice_string(self):
data = self.to_utf8_dict()
parts = json.dumps(data, indent = 4).split('\n')
parts = json.dumps(data, indent=4).split('\n')
for i, p in enumerate(parts):
if p[:5] != ' "':
continue
prop = p.split('"')[1]
parts[i] = (' [%.2f] "' % (self._confidence.get(prop) or -1)) + p[5:]
parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:]
return '\n'.join(parts)
@@ -73,9 +76,9 @@ class Guess(dict):
return str(self.to_utf8_dict())
def confidence(self, prop):
return self._confidence[prop]
return self._confidence.get(prop, -1)
def set(self, prop, value, confidence = None):
def set(self, prop, value, confidence=None):
self[prop] = value
if confidence is not None:
self._confidence[prop] = confidence
@@ -83,7 +86,7 @@ class Guess(dict):
def set_confidence(self, prop, value):
self._confidence[prop] = value
def update(self, other, confidence = None):
def update(self, other, confidence=None):
dict.update(self, other)
if isinstance(other, Guess):
for prop in other:
@@ -94,36 +97,36 @@ class Guess(dict):
self._confidence[prop] = confidence
def update_highest_confidence(self, other):
"""Update this guess with the values from the given one. In case there is
property present in both, only the one with the highest one is kept."""
"""Update this guess with the values from the given one. In case
there is property present in both, only the one with the highest one
is kept."""
if not isinstance(other, Guess):
raise ValueError, 'Can only call this function on Guess instances'
raise ValueError('Can only call this function on Guess instances')
for prop in other:
if prop in self and self._confidence[prop] >= other._confidence[prop]:
if prop in self and self.confidence(prop) >= other.confidence(prop):
continue
self[prop] = other[prop]
self._confidence[prop] = other._confidence[prop]
self._confidence[prop] = other.confidence(prop)
def choose_int(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible properties
when they are integers."""
"""Function used by merge_similar_guesses to choose between 2 possible
properties when they are integers."""
v1, c1 = g1 # value, confidence
v2, c2 = g2
if (v1 == v2):
return (v1, 1 - (1-c1)*(1-c2))
return (v1, 1 - (1 - c1) * (1 - c2))
else:
if c1 > c2:
return (v1, c1 - c2)
else:
return (v2, c2 - c1)
def choose_string(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible properties
when they are strings.
"""Function used by merge_similar_guesses to choose between 2 possible
properties when they are strings.
If the 2 strings are similar, or one is contained in the other, the latter is returned
with an increased confidence.
@@ -142,7 +145,7 @@ def choose_string(g1, g2):
('Hello', 0.75)
>>> choose_string(('Hello', 0.4), ('Hello World', 0.4))
('Hello', 0.64000000000000001)
('Hello', 0.64)
>>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5))
('The Simpsons', 0.75)
@@ -159,7 +162,7 @@ def choose_string(g1, g2):
v1, v2 = v1.strip(), v2.strip()
v1l, v2l = v1.lower(), v2.lower()
combined_prob = 1 - (1-c1)*(1-c2)
combined_prob = 1 - (1 - c1) * (1 - c2)
if v1l == v2l:
return (v1, combined_prob)
@@ -191,26 +194,31 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose):
This function assumes there are at least 2 valid guesses."""
similar = [ guess for guess in guesses if prop in guess ]
similar = [guess for guess in guesses if prop in guess]
g1, g2 = similar[0], similar[1]
other_props = set(g1) & set(g2) - set([prop])
if other_props:
log.debug('guess 1: %s' % g1)
log.debug('guess 2: %s' % g2)
for prop in other_props:
if g1[prop] != g2[prop]:
log.warning('both guesses to be merged have more than one different property in common, bailing out...')
log.warning('both guesses to be merged have more than one '
'different property in common, bailing out...')
return
# merge all props of s2 into s1, updating the confidence for the considered property
# merge all props of s2 into s1, updating the confidence for the
# considered property
v1, v2 = g1[prop], g2[prop]
c1, c2 = g1.confidence(prop), g2.confidence(prop)
new_value, new_confidence = choose((v1, c1), (v2, c2))
if new_confidence >= c1:
log.debug("Updating matching property '%s' with confidence %.2f" % (prop, new_confidence))
msg = "Updating matching property '%s' with confidence %.2f"
else:
log.debug("Updating non-matching property '%s' with confidence %.2f" % (prop, new_confidence))
msg = "Updating non-matching property '%s' with confidence %.2f"
log.debug(msg % (prop, new_confidence))
g2[prop] = new_value
g2.set_confidence(prop, new_confidence)
@@ -218,12 +226,13 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose):
g1.update(g2)
guesses.remove(g2)
def merge_similar_guesses(guesses, prop, choose):
"""Take a list of guesses and merge those which have the same properties,
increasing or decreasing the confidence depending on whether their values
are similar."""
similar = [ guess for guess in guesses if prop in guess ]
similar = [guess for guess in guesses if prop in guess]
if len(similar) < 2:
# nothing to merge
return
@@ -233,9 +242,13 @@ def merge_similar_guesses(guesses, prop, choose):
if len(similar) > 2:
log.debug('complex merge, trying our best...')
before = len(guesses)
_merge_similar_guesses_nocheck(guesses, prop, choose)
merge_similar_guesses(guesses, prop, choose)
return
after = len(guesses)
if after < before:
# recurse only when the previous call actually did something,
# otherwise we end up in an infinite loop
merge_similar_guesses(guesses, prop, choose)
def merge_append_guesses(guesses, prop):
@@ -245,14 +258,12 @@ def merge_append_guesses(guesses, prop):
DEPRECATED, remove with old guessers
"""
similar = [ guess for guess in guesses if prop in guess ]
similar = [guess for guess in guesses if prop in guess]
if not similar:
return
merged = similar[0]
merged[prop] = [ merged[prop] ]
merged[prop] = [merged[prop]]
# TODO: what to do with global confidence? mean of them all?
for m in similar[1:]:
@@ -261,17 +272,18 @@ def merge_append_guesses(guesses, prop):
merged[prop].append(m[prop])
else:
if prop2 in m:
log.warning('overwriting property "%s" with value ' % (prop2, m[prop2]))
log.warning('overwriting property "%s" with value %s' % (prop2, m[prop2]))
merged[prop2] = m[prop2]
# TODO: confidence also
guesses.remove(m)
def merge_all(guesses, append = []):
"""Merges all the guesses in a single result, removes very unlikely values, and returns it.
You can specify a list of properties that should be appended into a list instead of being
merged.
def merge_all(guesses, append=None):
"""Merge all the guesses in a single result, remove very unlikely values,
and return it.
You can specify a list of properties that should be appended into a list
instead of being merged.
>>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ])
@@ -286,20 +298,24 @@ def merge_all(guesses, append = []):
return Guess()
result = guesses[0]
if append is None:
append = []
for g in guesses[1:]:
# first append our appendable properties
for prop in append:
if prop in g:
result.set(prop, result.get(prop, []) + [ g[prop] ],
# TODO: what to do with confidence here? maybe an arithmetic mean...
confidence = g.confidence(prop))
result.set(prop, result.get(prop, []) + [g[prop]],
# TODO: what to do with confidence here? maybe an
# arithmetic mean...
confidence=g.confidence(prop))
del g[prop]
# then merge the remaining ones
if set(result) & set(g):
log.warning('duplicate properties %s in merged result...' % (set(result) & set(g)))
dups = set(result) & set(g)
if dups:
log.warning('duplicate properties %s in merged result...' % dups)
result.update_highest_confidence(g)
@@ -314,4 +330,3 @@ def merge_all(guesses, append = []):
result[prop] = list(set(result[prop]))
return result
libs/guessit/hash_ed2k.py
+10 -5
View File
@@ -18,8 +18,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
import hashlib, os.path
import hashlib
import os.path
def hash_file(filename):
"""Returns the ed2k hash of a given file.
@@ -31,6 +32,7 @@ def hash_file(filename):
os.path.getsize(filename),
hash_filehash(filename).upper())
def hash_filehash(filename):
"""Returns the ed2k hash of a given file.
@@ -42,8 +44,10 @@ def hash_filehash(filename):
def gen(f):
while True:
x = f.read(9728000)
if x: yield x
else: return
if x:
yield x
else:
return
def md4_hash(data):
m = md4()
@@ -55,4 +59,5 @@ def hash_filehash(filename):
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return hashes[0].encode("hex")
else: return md4_hash(reduce(lambda a,d: a + d, hashes, "")).hexd
else:
return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexd
libs/guessit/hash_mpc.py
+18 -18
View File
@@ -18,8 +18,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
import struct, os
import struct
import os
def hash_file(filename):
"""This function is taken from:
@@ -32,25 +33,24 @@ def hash_file(filename):
f = open(filename, "rb")
filesize = os.path.getsize(filename)
hash = filesize
hash_value = filesize
if filesize < 65536 * 2:
raise Exception, "SizeError: size is %d, should be > 132K..." % filesize
raise Exception("SizeError: size is %d, should be > 132K..." % filesize)
for x in range(65536/bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
for x in range(65536 / bytesize):
buf = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, buf)
hash_value += l_value
hash_value = hash_value & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
f.seek(max(0,filesize-65536),0)
for x in range(65536/bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF
f.seek(max(0, filesize - 65536), 0)
for x in range(65536 / bytesize):
buf = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, buf)
hash_value += l_value
hash_value = hash_value & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = "%016x" % hash
return returnedhash
return "%016x" % hash_value
libs/guessit/language.py
+53 -44
View File
@@ -19,28 +19,28 @@
#
from guessit import fileutils
import os.path
import re
import logging
log = logging.getLogger('guessit.language')
# downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
#
# Description of the fields:
# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
# an alpha-2 code (when given), an English name, and a French name of a language
# are all separated by pipe (|) characters."
_iso639_contents = fileutils.load_file_in_same_dir(__file__,
'ISO-639-2_utf-8.txt')
language_matrix = [ l.strip().decode('utf-8').split('|')
for l in fileutils.load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt').split('\n') ]
for l in _iso639_contents.split('\n') ]
lng3 = frozenset(filter(bool, (l[0] for l in language_matrix)))
lng3term = frozenset(filter(bool, (l[1] for l in language_matrix)))
lng2 = frozenset(filter(bool, (l[2] for l in language_matrix)))
lng_en_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[3].lower().split('; '))))
lng_fr_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[4].lower().split('; '))))
lng3 = frozenset(l[0] for l in language_matrix if l[0])
lng3term = frozenset(l[1] for l in language_matrix if l[1])
lng2 = frozenset(l[2] for l in language_matrix if l[2])
lng_en_name = frozenset(lng for l in language_matrix
for lng in l[3].lower().split('; ') if lng)
lng_fr_name = frozenset(lng for l in language_matrix
for lng in l[4].lower().split('; ') if lng)
lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name
lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1])
@@ -50,22 +50,29 @@ lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2])
lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2])
# we only return the first given english name, hoping it is the most used one
lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) for l in language_matrix if l[3])
lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) for l in language_matrix if l[3] for en_name in l[3].split('; '))
lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0])
for l in language_matrix if l[3])
lng_en_name_to_lng3 = dict((en_name.lower(), l[0])
for l in language_matrix if l[3]
for en_name in l[3].split('; '))
# we only return the first given french name, hoping it is the most used one
lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) for l in language_matrix if l[4])
lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) for l in language_matrix if l[4] for fr_name in l[4].split('; '))
lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0])
for l in language_matrix if l[4])
lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0])
for l in language_matrix if l[4]
for fr_name in l[4].split('; '))
def is_language(language):
return language.lower() in lng_all_names
class Language(object):
"""This class represents a human language.
You can initialize it with pretty much everything, as it knows conversion from
ISO-639 2-letter and 3-letter codes, English and French names.
You can initialize it with pretty much everything, as it knows conversion
from ISO-639 2-letter and 3-letter codes, English and French names.
>>> Language('fr')
Language(French)
@@ -79,12 +86,16 @@ class Language(object):
if len(language) == 2:
lang = lng2_to_lng3.get(language)
elif len(language) == 3:
lang = language if language in lng3 else lng3term_to_lng3.get(language)
lang = (language
if language in lng3
else lng3term_to_lng3.get(language))
else:
lang = lng_en_name_to_lng3.get(language) or lng_fr_name_to_lng3.get(language)
lang = (lng_en_name_to_lng3.get(language) or
lng_fr_name_to_lng3.get(language))
if lang is None:
raise ValueError, 'The given string "%s" could not be identified as a language' % language
msg = 'The given string "%s" could not be identified as a language'
raise ValueError(msg % language)
self.lang = lang
@@ -103,7 +114,6 @@ class Language(object):
def french_name(self):
return lng3_to_lng_fr_name[self.lang]
def __hash__(self):
return hash(self.lang)
@@ -132,19 +142,15 @@ class Language(object):
return 'Language(%s)' % self
def search_language(string, lang_filter = None):
def search_language(string, lang_filter=None):
"""Looks for language patterns, and if found return the language object,
its group span and an associated confidence.
you can specify a list of allowed languages using the lang_filter argument,
as in lang_filter = [ 'fr', 'eng', 'spanish' ]
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-letter delimiting the language.
>>> search_language('movie [en].avi')
(Language(English), (7, 9), 0.80000000000000004)
(Language(English), (7, 9), 0.8)
>>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es'])
(None, None, None)
@@ -153,25 +159,27 @@ def search_language(string, lang_filter = None):
# list of common words which could be interpreted as languages, but which
# are far too common to be able to say they represent a language in the
# middle of a string (where they most likely carry their commmon meaning)
lng_common_words = frozenset([ # english words
'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
# french words
'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
'mal', 'est', 'vol', 'or', 'mon', 'se',
# spanish words
'la', 'el', 'del', 'por', 'mar',
# other
'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san'
])
lng_common_words = frozenset([
# english words
'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
# french words
'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
'mal', 'est', 'vol', 'or', 'mon', 'se',
# spanish words
'la', 'el', 'del', 'por', 'mar',
# other
'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii',
'vi'
])
sep = r'[](){} \._-+'
if lang_filter:
lang_filter = set(Language(l) for l in lang_filter)
slow = string.lower()
slow = ' %s ' % string.lower()
confidence = 1.0 # for all of them
for lang in lng_all_names:
@@ -183,7 +191,7 @@ def search_language(string, lang_filter = None):
if pos != -1:
end = pos + len(lang)
# make sure our word is always surrounded by separators
if slow[pos-1] not in sep or slow[end] not in sep:
if slow[pos - 1] not in sep or slow[end] not in sep:
continue
language = Language(slow[pos:end])
@@ -201,10 +209,11 @@ def search_language(string, lang_filter = None):
elif len(lang) == 3:
confidence = 0.9
else:
# Note: we could either be really confident that we found a language
# or assume that full language names are too common words
# Note: we could either be really confident that we found a
# language or assume that full language names are too
# common words
confidence = 0.3 # going with the low-confidence route here
return language, (pos, end), confidence
return language, (pos - 1, end - 1), confidence
return None, None, None
libs/guessit/matcher.py
+63 -505
View File
@@ -2,8 +2,7 @@
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2011 Ricard Marxer <ricardmp@gmail.com>
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -19,282 +18,18 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import fileutils, textutils
from guessit.guess import Guess, merge_similar_guesses, merge_all, choose_int, choose_string
from guessit.date import search_date, search_year
from guessit.language import search_language
from guessit.filetype import guess_filetype
from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, find_properties, canonical_form, unlikely_series
from guessit.matchtree import get_group, find_group, leftover_valid_groups, tree_to_string
from guessit.textutils import find_first_level_groups, split_on_groups, blank_region, clean_string, to_utf8
from guessit.fileutils import split_path_components
import datetime
import os.path
import re
from guessit.matchtree import MatchTree
from guessit.textutils import to_utf8
from guessit.guess import (merge_similar_guesses, merge_all,
choose_int, choose_string)
import copy
import logging
import mimetypes
log = logging.getLogger("guessit.matcher")
def split_explicit_groups(string):
"""return the string split into explicit groups, that is, those either
between parenthese, square brackets or curly braces, and those separated
by a dash."""
result = find_first_level_groups(string, '()')
result = reduce(lambda l, x: l + find_first_level_groups(x, '[]'), result, [])
result = reduce(lambda l, x: l + find_first_level_groups(x, '{}'), result, [])
# do not do this at this moment, it is not strong enough and can break other
# patterns, such as dates, etc...
#result = reduce(lambda l, x: l + x.split('-'), result, [])
return result
def format_guess(guess):
"""Format all the found values to their natural type.
For instance, a year would be stored as an int value, etc...
Note that this modifies the dictionary given as input.
"""
for prop, value in guess.items():
if prop in ('season', 'episodeNumber', 'year', 'cdNumber', 'cdNumberTotal'):
guess[prop] = int(guess[prop])
elif isinstance(value, basestring):
if prop in ('edition',):
value = clean_string(value)
guess[prop] = canonical_form(value)
return guess
def guess_groups(string, result, filetype):
# add sentinels so we can match a separator char at either end of
# our groups, even when they are at the beginning or end of the string
# we will adjust the span accordingly later
#
# filetype can either be movie, moviesubtitle, episode, episodesubtitle
current = ' ' + string + ' '
regions = [] # list of (start, end) of matched regions
def guessed(match_dict, confidence):
guess = format_guess(Guess(match_dict, confidence = confidence))
result.append(guess)
log.debug('Found with confidence %.2f: %s' % (confidence, guess))
return guess
def update_found(string, guess, span, span_adjust = (0,0)):
span = (span[0] + span_adjust[0],
span[1] + span_adjust[1])
regions.append((span, guess))
return blank_region(string, span)
# try to find dates first, as they are very specific
date, span = search_date(current)
if date:
guess = guessed({ 'date': date }, confidence = 1.0)
current = update_found(current, guess, span)
# for non episodes only, look for year information
if filetype not in ('episode', 'episodesubtitle'):
year, span = search_year(current)
if year:
guess = guessed({ 'year': year }, confidence = 1.0)
current = update_found(current, guess, span)
# specific regexps (ie: cd number, season X episode, ...)
for rexp, confidence, span_adjust in video_rexps:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
# is this the better place to put it? (maybe, as it is at least the soonest that we can catch it)
if 'cdNumberTotal' in metadata and metadata['cdNumberTotal'] is None:
del metadata['cdNumberTotal']
guess = guessed(metadata, confidence = confidence)
current = update_found(current, guess, match.span(), span_adjust)
if filetype in ('episode', 'episodesubtitle'):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
guess = guessed(metadata, confidence = confidence)
current = update_found(current, guess, match.span(), span_adjust)
# Now websites, but as exact string instead of regexps
clow = current.lower()
for site in websites:
pos = clow.find(site.lower())
if pos != -1:
guess = guessed({ 'website': site }, confidence = confidence)
current = update_found(current, guess, (pos, pos+len(site)))
clow = current.lower()
# release groups have certain constraints, cannot be included in the previous general regexps
group_names = [ r'\.(Xvid)-(?P<releaseGroup>.*?)[ \.]',
r'\.(DivX)-(?P<releaseGroup>.*?)[\. ]',
r'\.(DVDivX)-(?P<releaseGroup>.*?)[\. ]',
]
for rexp in group_names:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
metadata.update({ 'videoCodec': match.group(1) })
guess = guessed(metadata, confidence = 0.8)
current = update_found(current, guess, match.span(), span_adjust = (1, -1))
# common well-defined words and regexps
confidence = 1.0 # for all of them
for prop, value, pos, end in find_properties(current):
guess = guessed({ prop: value }, confidence = confidence)
current = update_found(current, guess, (pos, end))
# weak guesses for episode number, only run it if we don't have an estimate already
if filetype in ('episode', 'episodesubtitle'):
if not any('episodeNumber' in match for match in result):
for rexp, _, span_adjust in weak_episode_rexps:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
epnum = int(metadata['episodeNumber'])
if epnum > 100:
guess = guessed({ 'season': epnum // 100,
'episodeNumber': epnum % 100 }, confidence = 0.6)
else:
guess = guessed(metadata, confidence = 0.3)
current = update_found(current, guess, match.span(), span_adjust)
# try to find languages now
language, span, confidence = search_language(current)
while language:
# is it a subtitle language?
if 'sub' in clean_string(current[:span[0]]).lower().split(' '):
guess = guessed({ 'subtitleLanguage': language }, confidence = confidence)
else:
guess = guessed({ 'language': language }, confidence = confidence)
current = update_found(current, guess, span)
language, span, confidence = search_language(current)
# remove our sentinels now and ajust spans accordingly
assert(current[0] == ' ' and current[-1] == ' ')
current = current[1:-1]
regions = [ ((start-1, end-1), guess) for (start, end), guess in regions ]
# split into '-' separated subgroups (with required separator chars
# around the dash)
didx = current.find('-')
while didx > 0:
regions.append(((didx, didx), None))
didx = current.find('-', didx+1)
# cut our final groups, and rematch the guesses to the group that created
# id, None if it is a leftover group
region_spans = [ span for span, guess in regions ]
string_groups = split_on_groups(string, region_spans)
remaining_groups = split_on_groups(current, region_spans)
guesses = []
pos = 0
for group in string_groups:
found = False
for span, guess in regions:
if span[0] == pos:
guesses.append(guess)
found = True
if not found:
guesses.append(None)
pos += len(group)
return zip(string_groups,
remaining_groups,
guesses)
def match_from_epnum_position(match_tree, epnum_pos, guessed, update_found):
"""guessed is a callback function to call with the guessed group
update_found is a callback to update the match group and returns leftover groups."""
pidx, eidx, gidx = epnum_pos
# a few helper functions to be able to filter using high-level semantics
def same_pgroup_before(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and (eeidx, ggidx) < (eidx, gidx)
def same_pgroup_after(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and (eeidx, ggidx) > (eidx, gidx)
def same_egroup_before(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and eeidx == eidx and ggidx < gidx
def same_egroup_after(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and eeidx == eidx and ggidx > gidx
leftover = leftover_valid_groups(match_tree)
# if we have at least 1 valid group before the episodeNumber, then it's probably
# the series name
series_candidates = filter(same_pgroup_before, leftover)
if len(series_candidates) >= 1:
guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.7)
leftover = update_found(leftover, series_candidates[0][1], guess)
# only 1 group after (in the same path group) and it's probably the episode title
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(same_pgroup_after, leftover))
if len(title_candidates) == 1:
guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.5)
leftover = update_found(leftover, title_candidates[0][1], guess)
else:
# try in the same explicit group, with lower confidence
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(same_egroup_after, leftover))
if len(title_candidates) == 1:
guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[0][1], guess)
# epnumber is the first group and there are only 2 after it in same path group
# -> season title - episode title
already_has_title = (find_group(match_tree, 'title') != [])
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(same_pgroup_after, leftover))
if (not already_has_title and # no title
not filter(same_pgroup_before, leftover) and # no groups before
len(title_candidates) == 2): # only 2 groups after
guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[0][1], guess)
guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[1][1], guess)
# if we only have 1 remaining valid group in the pathpart before the filename,
# then it's likely that it is the series name
series_candidates = [ group for group in leftover if group[1][0] == pidx-1 ]
if len(series_candidates) == 1:
guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.5)
leftover = update_found(leftover, series_candidates[0][1], guess)
return match_tree
class IterativeMatcher(object):
def __init__(self, filename, filetype = 'autodetect'):
def __init__(self, filename, filetype='autodetect'):
"""An iterative matcher tries to match different patterns that appear
in the filename.
@@ -325,7 +60,7 @@ class IterativeMatcher(object):
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, x264 is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter'v' in the 4th line.
(for more info, see guess.matchtree.tree_to_string)
(for more info, see guess.matchtree.to_string)
Second, it tries to merge all this information into a single object
@@ -333,266 +68,89 @@ class IterativeMatcher(object):
resolution when they arise.
"""
if filetype not in ('autodetect', 'subtitle', 'video',
valid_filetypes = ('autodetect', 'subtitle', 'video',
'movie', 'moviesubtitle',
'episode', 'episodesubtitle'):
raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'video', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')"
'episode', 'episodesubtitle')
if filetype not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" % valid_filetypes)
if not isinstance(filename, unicode):
log.debug('WARNING: given filename to matcher is not unicode...')
match_tree = []
result = [] # list of found metadata
def guessed(match_dict, confidence):
guess = format_guess(Guess(match_dict, confidence = confidence))
result.append(guess)
log.debug('Found with confidence %.2f: %s' % (confidence, guess))
return guess
def update_found(leftover, group_pos, guess):
pidx, eidx, gidx = group_pos
group = match_tree[pidx][eidx][gidx]
match_tree[pidx][eidx][gidx] = (group[0],
deleted * len(group[0]),
guess)
return [ g for g in leftover if g[1] != group_pos ]
self.match_tree = MatchTree(filename)
mtree = self.match_tree
mtree.guess.set('type', filetype, confidence=1.0)
def apply_transfo(transfo_name, *args, **kwargs):
transfo = __import__('guessit.transfo.' + transfo_name,
globals=globals(), locals=locals(),
fromlist=['process'], level=-1)
transfo.process(mtree, *args, **kwargs)
# 1- first split our path into dirs + basename + ext
match_tree = split_path_components(filename)
apply_transfo('split_path_components')
# try to detect the file type
filetype, other = guess_filetype(filename, filetype)
guessed({ 'type': filetype }, confidence = 1.0)
extguess = guessed(other, confidence = 1.0)
# 2- guess the file type now (will be useful later)
apply_transfo('guess_filetype', filetype)
if mtree.guess['type'] == 'unknown':
return
# guess the mimetype of the filename
# TODO: handle other mimetypes not found on the default type_maps
# mimetypes.types_map['.srt']='text/subtitle'
mime, _ = mimetypes.guess_type(filename, strict=False)
if mime is not None:
guessed({ 'mimetype': mime }, confidence = 1.0)
# 3- split each of those into explicit groups (separated by parentheses
# or square brackets)
apply_transfo('split_explicit_groups')
# remove the extension from the match tree, as all indices relative
# the the filename groups assume the basename is the last one
fileext = match_tree.pop(-1)[1:].lower()
# 4- try to match information for specific patterns
if mtree.guess['type'] in ('episode', 'episodesubtitle'):
strategy = ['guess_date', 'guess_video_rexps',
'guess_episodes_rexps', 'guess_website',
'guess_release_group', 'guess_properties',
'guess_weak_episodes_rexps', 'guess_language']
else:
strategy = ['guess_date', 'guess_year', 'guess_video_rexps',
'guess_website', 'guess_release_group',
'guess_properties', 'guess_language']
for name in strategy:
apply_transfo(name)
# 2- split each of those into explicit groups, if any
# note: be careful, as this might split some regexps with more confidence such as
# Alfleni-Team, or [XCT] or split a date such as (14-01-2008)
match_tree = [ split_explicit_groups(part) for part in match_tree ]
# more guessers for both movies and episodes
for name in ['guess_bonus_features']:
apply_transfo(name)
# split into '-' separated subgroups (with required separator chars
# around the dash)
apply_transfo('split_on_dash')
# 3- try to match information in decreasing order of confidence and
# blank the matching group in the string if we found something
for pathpart in match_tree:
for gidx, explicit_group in enumerate(pathpart):
pathpart[gidx] = guess_groups(explicit_group, result, filetype = filetype)
# 5- try to identify the remaining unknown groups by looking at their
# position relative to other known elements
if mtree.guess['type'] in ('episode', 'episodesubtitle'):
apply_transfo('guess_episode_info_from_position')
else:
apply_transfo('guess_movie_title_from_position')
# 4- try to identify the remaining unknown groups by looking at their position
# relative to other known elements
if filetype in ('episode', 'episodesubtitle'):
eps = find_group(match_tree, 'episodeNumber')
if eps:
match_tree = match_from_epnum_position(match_tree, eps[0], guessed, update_found)
leftover = leftover_valid_groups(match_tree)
if not eps:
# if we don't have the episode number, but at least 2 groups in the
# last path group, then it's probably series - eptitle
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(lambda g: g[1][0] == len(match_tree)-1,
leftover_valid_groups(match_tree)))
if len(title_candidates) >= 2:
guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[0][1], guess)
guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[1][1], guess)
# if there's a path group that only contains the season info, then the previous one
# is most likely the series title (ie: .../series/season X/...)
eps = [ gpos for gpos in find_group(match_tree, 'season')
if 'episodeNumber' not in get_group(match_tree, gpos)[2] ]
if eps:
pidx, eidx, gidx = eps[0]
previous = [ group for group in leftover if group[1][0] == pidx - 1 ]
if len(previous) == 1:
guess = guessed({ 'series': previous[0][0] }, confidence = 0.5)
leftover = update_found(leftover, previous[0][1], guess)
# reduce the confidence of unlikely series
for guess in result:
if 'series' in guess:
if guess['series'].lower() in unlikely_series:
guess.set_confidence('series', guess.confidence('series') * 0.5)
elif filetype in ('movie', 'moviesubtitle'):
leftover_all = leftover_valid_groups(match_tree)
# specific cases:
# - movies/tttttt (yyyy)/tttttt.ccc
try:
if match_tree[-3][0][0][0].lower() == 'movies':
# Note:too generic, might solve all the unittests as they all contain 'movies'
# in their path
#
#if len(match_tree[-2][0]) == 1:
# title = match_tree[-2][0][0]
# guess = guessed({ 'title': clean_string(title[0]) }, confidence = 0.7)
# update_found(leftover_all, title, guess)
year_group = filter(lambda gpos: gpos[0] == len(match_tree)-2,
find_group(match_tree, 'year'))[0]
leftover = leftover_valid_groups(match_tree,
valid = lambda g: ((g[0] and g[0][0] not in sep) and
g[1][0] == len(match_tree) - 2))
if len(match_tree[-2]) == 2 and year_group[1] == 1:
title = leftover[0]
guess = guessed({ 'title': clean_string(title[0]) },
confidence = 0.8)
update_found(leftover_all, title[1], guess)
raise Exception # to exit the try catch now
leftover = [ g for g in leftover_all if (g[1][0] == year_group[0] and
g[1][1] < year_group[1] and
g[1][2] < year_group[2]) ]
leftover = sorted(leftover, key = lambda x:x[1])
title = leftover[0]
guess = guessed({ 'title': title[0] }, confidence = 0.8)
leftover = update_found(leftover, title[1], guess)
except:
pass
# if we have either format or videoCodec in the folder containing the file
# or one of its parents, then we should probably look for the title in
# there rather than in the basename
props = filter(lambda g: g[0] <= len(match_tree) - 2,
find_group(match_tree, 'videoCodec') +
find_group(match_tree, 'format') +
find_group(match_tree, 'language'))
leftover = None
if props and all(g[0] == props[0][0] for g in props):
leftover = [ g for g in leftover_all if g[1][0] == props[0][0] ]
if props and leftover:
guess = guessed({ 'title': leftover[0][0] }, confidence = 0.7)
leftover = update_found(leftover, leftover[0][1], guess)
else:
# first leftover group in the last path part sounds like a good candidate for title,
# except if it's only one word and that the first group before has at least 3 words in it
# (case where the filename contains an 8 chars short name and the movie title is
# actually in the parent directory name)
leftover = [ g for g in leftover_all if g[1][0] == len(match_tree)-1 ]
if leftover:
title, (pidx, eidx, gidx) = leftover[0]
previous_pgroup_leftover = filter(lambda g: g[1][0] == pidx-1, leftover_all)
if (title.count(' ') == 0 and
previous_pgroup_leftover and
previous_pgroup_leftover[0][0].count(' ') >= 2):
guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6)
leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess)
else:
guess = guessed({ 'title': title }, confidence = 0.6)
leftover = update_found(leftover, leftover[0][1], guess)
else:
# if there were no leftover groups in the last path part, look in the one before that
previous_pgroup_leftover = filter(lambda g: g[1][0] == len(match_tree)-2, leftover_all)
if previous_pgroup_leftover:
guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6)
leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess)
# 5- perform some post-processing steps
# 5.1- try to promote language to subtitle language where it makes sense
for pidx, eidx, gidx in find_group(match_tree, 'language'):
string, remaining, guess = get_group(match_tree, (pidx, eidx, gidx))
def promote_subtitle():
guess.set('subtitleLanguage', guess['language'], confidence = guess.confidence('language'))
del guess['language']
# - if we matched a language in a file with a sub extension and that the group
# is the last group of the filename, it is probably the language of the subtitle
# (eg: 'xxx.english.srt')
if (fileext in subtitle_exts and
pidx == len(match_tree) - 1 and
eidx == len(match_tree[pidx]) - 1):
promote_subtitle()
# - if a language is in an explicit group just preceded by "st", it is a subtitle
# language (eg: '...st[fr-eng]...')
if eidx > 0:
previous = get_group(match_tree, (pidx, eidx-1, -1))
if previous[0][-2:].lower() == 'st':
promote_subtitle()
# re-append the extension now
match_tree.append([[(fileext, deleted*len(fileext), extguess)]])
self.parts = result
self.match_tree = match_tree
if filename.startswith('/'):
filename = ' ' + filename
log.debug('Found match tree:\n%s\n%s' % (to_utf8(tree_to_string(match_tree)),
to_utf8(filename)))
# 6- perform some post-processing steps
apply_transfo('post_process')
log.debug('Found match tree:\n%s' % (to_utf8(unicode(mtree))))
def matched(self):
# we need to make a copy here, as the merge functions work in place and
# calling them on the match tree would modify it
parts = copy.deepcopy(self.parts)
# 1- start by doing some common preprocessing tasks
parts = [node.guess for node in self.match_tree.nodes() if node.guess]
parts = copy.deepcopy(parts)
# 1.1- ", the" at the end of a series title should be prepended to it
for part in parts:
if 'series' not in part:
continue
series = part['series']
lseries = series.lower()
if lseries[-4:] == ',the':
part['series'] = 'The ' + series[:-4]
if lseries[-5:] == ', the':
part['series'] = 'The ' + series[:-5]
# 2- try to merge similar information together and give it a higher confidence
# 1- try to merge similar information together and give it a higher
# confidence
for int_part in ('year', 'season', 'episodeNumber'):
merge_similar_guesses(parts, int_part, choose_int)
for string_part in ('title', 'series', 'container', 'format', 'releaseGroup', 'website',
'audioCodec', 'videoCodec', 'screenSize', 'episodeFormat'):
for string_part in ('title', 'series', 'container', 'format',
'releaseGroup', 'website', 'audioCodec',
'videoCodec', 'screenSize', 'episodeFormat'):
merge_similar_guesses(parts, string_part, choose_string)
result = merge_all(parts, append = ['language', 'subtitleLanguage', 'other'])
# 3- some last minute post-processing
if (result['type'] == 'episode' and
'season' not in result and
result.get('episodeFormat', '') == 'Minisode'):
result['season'] = 0
result = merge_all(parts,
append=['language', 'subtitleLanguage', 'other'])
log.debug('Final result: ' + result.nice_string())
return result
libs/guessit/matchtree.py
+206 -99
View File
@@ -18,136 +18,243 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import deleted
from guessit.textutils import clean_string
from guessit import Guess
from guessit.textutils import clean_string, str_fill, to_utf8
from guessit.patterns import group_delimiters
import logging
log = logging.getLogger("guessit.matchtree")
class BaseMatchTree(object):
"""A MatchTree represents the hierarchical split of a string into its
constituent semantic groups."""
def tree_to_string(tree):
"""Return a string representation for the given tree.
def __init__(self, string='', span=None, parent=None):
self.string = string
self.span = span or (0, len(string))
self.parent = parent
self.children = []
self.guess = Guess()
The lines convey the following information:
- line 1: path idx
- line 2: explicit group idx
- line 3: group index
- line 4: remaining info
- line 5: meaning conveyed
@property
def value(self):
return self.string[self.span[0]:self.span[1]]
Meaning is a letter indicating what type of info was matched by this group,
for instance 't' = title, 'f' = format, 'l' = language, etc...
@property
def clean_value(self):
return clean_string(self.value)
An example is the following:
@property
def offset(self):
return self.span[0]
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
@property
def info(self):
result = dict(self.guess)
(note: the last line representing the filename is not pat of the tree representation)
"""
m_tree = [ '', # path level index
'', # explicit group index
'', # matched regexp and dash-separated
'', # groups leftover that couldn't be matched
'', # meaning conveyed: E = episodenumber, S = season, ...
]
for c in self.children:
result.update(c.info)
return result
@property
def root(self):
if not self.parent:
return self
return self.parent.root
@property
def depth(self):
if self.is_leaf():
return 0
return 1 + max(c.depth for c in self.children)
def is_leaf(self):
return self.children == []
def add_child(self, span):
child = MatchTree(self.string, span=span, parent=self)
self.children.append(child)
def partition(self, indices):
indices = sorted(indices)
if indices[0] != 0:
indices.insert(0, 0)
if indices[-1] != len(self.value):
indices.append(len(self.value))
for start, end in zip(indices[:-1], indices[1:]):
self.add_child(span=(self.offset + start,
self.offset + end))
def split_on_components(self, components):
offset = 0
for c in components:
start = self.value.find(c, offset)
end = start + len(c)
self.add_child(span=(self.offset + start,
self.offset + end))
offset = end
def nodes_at_depth(self, depth):
if depth == 0:
yield self
for child in self.children:
for node in child.nodes_at_depth(depth - 1):
yield node
@property
def node_idx(self):
if self.parent is None:
return ()
return self.parent.node_idx + (self.parent.children.index(self),)
def node_at(self, idx):
if not idx:
return self
try:
return self.children[idx[0]].node_at(idx[1:])
except:
raise ValueError('Non-existent node index: %s' % (idx,))
def nodes(self):
yield self
for child in self.children:
for node in child.nodes():
yield node
def _leaves(self):
if self.is_leaf():
yield self
else:
for child in self.children:
# pylint: disable=W0212
for leaf in child._leaves():
yield leaf
def leaves(self):
return list(self._leaves())
def to_string(self):
empty_line = ' ' * len(self.string)
def add_char(pidx, eidx, gidx, remaining, meaning = None):
nr = len(remaining)
def to_hex(x):
if isinstance(x, int):
return str(x) if x < 10 else chr(55+x)
return str(x) if x < 10 else chr(55 + x)
return x
m_tree[0] = m_tree[0] + to_hex(pidx) * nr
m_tree[1] = m_tree[1] + to_hex(eidx) * nr
m_tree[2] = m_tree[2] + to_hex(gidx) * nr
m_tree[3] = m_tree[3] + remaining
m_tree[4] = m_tree[4] + str(meaning or ' ') * nr
def meaning(result):
mmap = { 'episodeNumber': 'E',
'season': 'S',
'extension': 'e',
'format': 'f',
'language': 'l',
'videoCodec': 'v',
'audioCodec': 'a',
'website': 'w',
'container': 'c',
'series': 'T',
'title': 't',
'date': 'd',
'year': 'y',
'releaseGroup': 'r',
'screenSize': 's'
}
def meaning(result):
mmap = { 'episodeNumber': 'E',
'season': 'S',
'extension': 'e',
'format': 'f',
'language': 'l',
'videoCodec': 'v',
'audioCodec': 'a',
'website': 'w',
'container': 'c',
'series': 'T',
'title': 't',
'date': 'd',
'year': 'y',
'releaseGroup': 'r',
'screenSize': 's'
}
if result is None:
return ' '
if result is None:
return ' '
for prop, l in mmap.items():
if prop in result:
return l
for prop, l in mmap.items():
if prop in result:
return l
return 'x'
return 'x'
for pidx, pathpart in enumerate(tree):
for eidx, explicit_group in enumerate(pathpart):
for gidx, (group, remaining, result) in enumerate(explicit_group):
add_char(pidx, eidx, gidx, remaining, meaning(result))
lines = [ empty_line ] * (self.depth + 2) # +2: remaining, meaning
lines[-2] = self.string
# special conditions for the path separator
if pidx < len(tree) - 2:
add_char(' ', ' ', ' ', '/')
elif pidx == len(tree) - 2:
add_char(' ', ' ', ' ', '.')
for node in self.nodes():
if node == self:
continue
return '\n'.join(m_tree)
idx = node.node_idx
depth = len(idx) - 1
if idx:
lines[depth] = str_fill(lines[depth], node.span,
to_hex(idx[-1]))
if node.guess:
lines[-2] = str_fill(lines[-2], node.span, '_')
lines[-1] = str_fill(lines[-1], node.span, meaning(node.guess))
lines.append(self.string)
return '\n'.join(lines)
def __unicode__(self):
return self.to_string()
def __str__(self):
return to_utf8(unicode(self))
class MatchTree(BaseMatchTree):
"""The MatchTree contains a few "utility" methods which are not necessary
for the BaseMatchTree, but add a lot of convenience for writing
higher-level rules."""
def iterate_groups(match_tree):
"""Iterate over all the groups in a match_tree and return them as pairs
of (group_pos, group) where:
- group_pos = (pidx, eidx, gidx)
- group = (string, remaining, guess)
"""
for pidx, pathpart in enumerate(match_tree):
for eidx, explicit_group in enumerate(pathpart):
for gidx, group in enumerate(explicit_group):
yield (pidx, eidx, gidx), group
def _unidentified_leaves(self,
valid=lambda leaf: len(leaf.clean_value) >= 2):
for leaf in self._leaves():
if not leaf.guess and valid(leaf):
yield leaf
def unidentified_leaves(self,
valid=lambda leaf: len(leaf.clean_value) >= 2):
return list(self._unidentified_leaves(valid))
def find_group(match_tree, prop):
"""Find the list of groups that resulted in a guess that contains the
asked property."""
result = []
for gpos, (string, remaining, guess) in iterate_groups(match_tree):
if guess and prop in guess:
result.append(gpos)
return result
def _leaves_containing(self, property_name):
if isinstance(property_name, basestring):
property_name = [ property_name ]
def get_group(match_tree, gpos):
pidx, eidx, gidx = gpos
return match_tree[pidx][eidx][gidx]
for leaf in self._leaves():
for prop in property_name:
if prop in leaf.guess:
yield leaf
break
def leaves_containing(self, property_name):
return list(self._leaves_containing(property_name))
def leftover_valid_groups(match_tree, valid = lambda s: len(s[0]) > 3):
"""Return the list of valid string groups (eg: len(s) > 3) that could not be
matched to anything as a list of pairs (cleaned_str, group_pos)."""
leftover = []
for gpos, (group, remaining, guess) in iterate_groups(match_tree):
if not guess:
clean_str = clean_string(remaining)
if valid((clean_str, gpos)):
leftover.append((clean_str, gpos))
def first_leaf_containing(self, property_name):
try:
return next(self._leaves_containing(property_name))
except StopIteration:
return None
return leftover
def _previous_unidentified_leaves(self, node):
node_idx = node.node_idx
for leaf in self._unidentified_leaves():
if leaf.node_idx < node_idx:
yield leaf
def previous_unidentified_leaves(self, node):
return list(self._previous_unidentified_leaves(node))
def _previous_leaves_containing(self, node, property_name):
node_idx = node.node_idx
for leaf in self._leaves_containing(property_name):
if leaf.node_idx < node_idx:
yield leaf
def previous_leaves_containing(self, node, property_name):
return list(self._previous_leaves_containing(node, property_name))
def is_explicit(self):
"""Return whether the group was explicitly enclosed by
parentheses/square brackets/etc."""
return (self.value[0] + self.value[-1]) in group_delimiters
libs/guessit/patterns.py
Regular → Executable
+45 -23
View File
@@ -22,10 +22,13 @@
subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa', 'txt' ]
video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv', 'wmv', 'divx' ]
video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv',
'wmv', 'divx' ]
group_delimiters = [ '()', '[]', '{}' ]
# separator character regexp
sep = r'[][)(}{+ \._-]' # regexp art, hehe :D
sep = r'[][)(}{+ /\._-]' # regexp art, hehe :D
# character used to represent a deleted char (when matching groups)
deleted = '_'
@@ -35,28 +38,33 @@ episode_rexps = [ # ... Season 2 ...
(r'season (?P<season>[0-9]+)', 1.0, (0, 0)),
(r'saison (?P<season>[0-9]+)', 1.0, (0, 0)),
# ... s02-x01 ...
(r's(?P<season>[0-9]{1,2})-x(?P<bonusNumber>[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
# ... s02e13 ...
(r'[Ss](?P<season>[0-9]{1,2}).{,3}[EeXx](?P<episodeNumber>[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
# ... 2x13 ...
(r'[^0-9](?P<season>[0-9]{1,2})[x\.](?P<episodeNumber>[0-9]{2})[^0-9]', 0.8, (1, -1)),
(r'[^0-9](?P<season>[0-9]{1,2})x(?P<episodeNumber>[0-9]{2})[^0-9]', 0.8, (1, -1)),
# ... s02 ...
(sep + r's(?P<season>[0-9]{1,2})' + sep + '?', 0.6, (1, -1)),
#(sep + r's(?P<season>[0-9]{1,2})' + sep, 0.6, (1, -1)),
(r's(?P<season>[0-9]{1,2})[^0-9]', 0.6, (0, -1)),
# v2 or v3 for some mangas which have multiples rips
(sep + r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
(r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
# ... ep 23 ...
('ep' + sep + r'(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.7, (0, -1))
]
weak_episode_rexps = [ # ... 213 or 0106 ...
(sep + r'(?P<episodeNumber>[0-9]{1,4})' + sep, 0.3, (1, -1)),
(sep + r'(?P<episodeNumber>[0-9]{1,4})' + sep, (1, -1)),
# ... 2x13 ...
(sep + r'[^0-9](?P<season>[0-9]{1,2})\.(?P<episodeNumber>[0-9]{2})[^0-9]' + sep, (1, -1)),
]
non_episode_title = [ 'extras' ]
non_episode_title = [ 'extras', 'rip' ]
video_rexps = [ # cd number
@@ -76,7 +84,13 @@ video_rexps = [ # cd number
(r'(?P<width>[0-9]{3,4})x(?P<height>[0-9]{3,4})', 0.9, (0, 0)),
# website
(r'(?P<website>www(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0))
(r'(?P<website>www(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0)),
# bonusNumber: ... x01 ...
(r'x(?P<bonusNumber>[0-9]{1,2})', 1.0, (0, 0)),
# filmNumber: ... f01 ...
(r'f(?P<filmNumber>[0-9]{1,2})', 1.0, (0, 0))
]
websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com', 'sharethefiles.com' ]
@@ -87,7 +101,7 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'DVBRip', 'PDTV', 'WEBRip',
'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS' ],
'screenSize': [ '720p', '720' ],
'screenSize': [ '720p', '720', '1080p', '1080' ],
'videoCodec': [ 'XviD', 'DivX', 'x264', 'h264', 'Rv10' ],
@@ -98,18 +112,18 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', '[XCT]', 'iNT', 'PUKKA',
'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE',
'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe',
'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL' ],
'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL',
'HDBRiSe' ],
'episodeFormat': [ 'Minisode', 'Minisodes' ],
'other': [ '5ch', 'PROPER', 'REPACK', 'LIMITED', 'DualAudio', 'iNTERNAL', 'Audiofixed', 'R5',
'complete', 'classic', # not so sure about these ones, could appear in a title
'ws', # widescreen
#'SE', # special edition
# TODO: director's cut
],
}
def find_properties(filename):
result = []
clow = filename.lower()
@@ -119,7 +133,7 @@ def find_properties(filename):
if pos != -1:
end = pos + len(value)
# make sure our word is always surrounded by separators
if ((pos > 0 and clow[pos-1] not in sep) or
if ((pos > 0 and clow[pos - 1] not in sep) or
(end < len(clow) and clow[end] not in sep)):
# note: sep is a regexp, but in this case using it as
# a sequence achieves the same goal
@@ -137,6 +151,7 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
'DivX': [ 'DVDivX' ],
'h264': [ 'x264' ],
'720p': [ '720' ],
'1080p': [ '1080' ],
'AAC': [ 'He-AAC', 'AAC-He' ],
'Special Edition': [ 'Special' ],
'Collector Edition': [ 'Collector' ],
@@ -145,14 +160,21 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
}
reverse_synonyms = {}
for prop, values in properties.items():
for value in values:
reverse_synonyms[value.lower()] = value
def revert_synonyms():
reverse = {}
for _, values in properties.items():
for value in values:
reverse[value.lower()] = value
for canonical, synonyms in property_synonyms.items():
for synonym in synonyms:
reverse[synonym.lower()] = canonical
return reverse
reverse_synonyms = revert_synonyms()
for canonical, synonyms in property_synonyms.items():
for synonym in synonyms:
reverse_synonyms[synonym.lower()] = canonical
def canonical_form(string):
return reverse_synonyms.get(string.lower(), string)
libs/guessit/slogging.py
+5 -5
View File
@@ -28,8 +28,8 @@ RED_FONT = "\x1B[0;31m"
RESET_FONT = "\x1B[0m"
def setupLogging(colored = True):
"""Sets up a nice colored logger as the main application logger (not only smewt itself)."""
def setupLogging(colored=True):
"""Set up a nice colored logger as the main application logger."""
class SimpleFormatter(logging.Formatter):
def __init__(self):
@@ -38,7 +38,9 @@ def setupLogging(colored = True):
class ColoredFormatter(logging.Formatter):
def __init__(self):
self.fmt = '%(levelname)-8s ' + BLUE_FONT + '%(module)s:%(funcName)s' + RESET_FONT + ' -- %(message)s'
self.fmt = ('%(levelname)-8s ' +
BLUE_FONT + '%(name)s:%(funcName)s' +
RESET_FONT + ' -- %(message)s')
logging.Formatter.__init__(self, self.fmt)
def format(self, record):
@@ -50,11 +52,9 @@ def setupLogging(colored = True):
else:
return RED_FONT + result
ch = logging.StreamHandler()
if colored and sys.platform != 'win32':
ch.setFormatter(ColoredFormatter())
else:
ch.setFormatter(SimpleFormatter())
logging.getLogger().addHandler(ch)
libs/guessit/textutils.py
+33 -25
View File
@@ -18,47 +18,59 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import sep, deleted
from guessit.patterns import sep
import copy
# string-related functions
def strip_brackets(s):
if not s:
return s
if s[0] == '[' and s[-1] == ']': return s[1:-1]
if s[0] == '(' and s[-1] == ')': return s[1:-1]
if s[0] == '{' and s[-1] == '}': return s[1:-1]
if ((s[0] == '[' and s[-1] == ']') or
(s[0] == '(' and s[-1] == ')') or
(s[0] == '{' and s[-1] == '}')):
return s[1:-1]
return s
def clean_string(s):
for c in sep:
for c in sep[:-2]: # do not remove dashes ('-')
s = s.replace(c, ' ')
parts = s.split()
return ' '.join(p for p in parts if p != '')
result = ' '.join(p for p in parts if p != '')
# now also remove dashes on the outer part of the string
while result and result[0] in sep:
result = result[1:]
while result and result[-1] in sep:
result = result[:-1]
return result
def str_replace(string, pos, c):
return string[:pos] + c + string[pos+1:]
def blank_region(string, region, blank_sep = deleted):
def str_fill(string, region, c):
start, end = region
return string[:start] + blank_sep * (end - start) + string[end:]
return string[:start] + c * (end - start) + string[end:]
def between(s, left, right):
return s.split(left)[1].split(right)[0]
def to_utf8(o):
'''converts all unicode strings found in the given object to utf-8 strings'''
"""Convert all unicode strings found in the given object to utf-8
strings."""
if isinstance(o, unicode):
return o.encode('utf-8')
elif isinstance(o, list):
return [ to_utf8(i) for i in o ]
elif isinstance(o, dict):
result = copy.deepcopy(o) # need to do it like that to handle Guess instances correctly
# need to do it like that to handle Guess instances correctly
result = copy.deepcopy(o)
for key, value in o.items():
result[to_utf8(key)] = to_utf8(value)
return result
@@ -68,8 +80,10 @@ def to_utf8(o):
def levenshtein(a, b):
if not a: return len(b)
if not b: return len(a)
if not a:
return len(b)
if not b:
return len(a)
m = len(a)
n = len(b)
@@ -160,14 +174,13 @@ def split_on_groups(string, groups):
if boundaries[-1] != len(string):
boundaries.append(len(string))
groups = [ string[start:end] for start, end in zip(boundaries[:-1], boundaries[1:]) ]
groups = [ string[start:end] for start, end in zip(boundaries[:-1],
boundaries[1:]) ]
return filter(bool, groups) # return only non-empty groups
return [ g for g in groups if g ] # return only non-empty groups
def find_first_level_groups(string, enclosing, blank_sep = None):
def find_first_level_groups(string, enclosing, blank_sep=None):
"""Return a list of groups that could be split because of explicit grouping.
The groups are delimited by the given enclosing characters.
@@ -203,8 +216,3 @@ def find_first_level_groups(string, enclosing, blank_sep = None):
string = str_replace(string, end-1, blank_sep)
return split_on_groups(string, groups)
libs/guessit/transfo/__init__.py
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.patterns import canonical_form
from guessit.textutils import clean_string
import logging
log = logging.getLogger('guessit.transfo')
def found_property(node, name, confidence):
node.guess = Guess({name: node.clean_value}, confidence=confidence)
log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
def format_guess(guess):
"""Format all the found values to their natural type.
For instance, a year would be stored as an int value, etc...
Note that this modifies the dictionary given as input.
"""
for prop, value in guess.items():
if prop in ('season', 'episodeNumber', 'year', 'cdNumber',
'cdNumberTotal', 'bonusNumber', 'filmNumber'):
guess[prop] = int(guess[prop])
elif isinstance(value, basestring):
if prop in ('edition',):
value = clean_string(value)
guess[prop] = canonical_form(value)
return guess
def find_and_split_node(node, strategy, logger):
string = ' %s ' % node.value # add sentinels
for matcher, confidence in strategy:
if getattr(matcher, 'use_node', False):
result, span = matcher(string, node)
else:
result, span = matcher(string)
if result:
# readjust span to compensate for sentinels
span = (span[0] - 1, span[1] - 1)
if isinstance(result, Guess):
if confidence is None:
confidence = result.confidence(result.keys()[0])
else:
if confidence is None:
confidence = 1.0
guess = format_guess(Guess(result, confidence=confidence))
msg = 'Found with confidence %.2f: %s' % (confidence, guess)
(logger or log).debug(msg)
node.partition(span)
absolute_span = (span[0] + node.offset, span[1] + node.offset)
for child in node.children:
if child.span == absolute_span:
child.guess = guess
else:
find_and_split_node(child, strategy, logger)
return
class SingleNodeGuesser(object):
def __init__(self, guess_func, confidence, logger=None):
self.guess_func = guess_func
self.confidence = confidence
self.logger = logger
def process(self, mtree):
# strategy is a list of pairs (guesser, confidence)
# - if the guesser returns a guessit.Guess and confidence is specified,
# it will override it, otherwise it will leave the guess confidence
# - if the guesser returns a simple dict as a guess and confidence is
# specified, it will use it, or 1.0 otherwise
strategy = [ (self.guess_func, self.confidence) ]
for node in mtree.unidentified_leaves():
find_and_split_node(node, strategy, self.logger)
libs/guessit/transfo/guess_bonus_features.py
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import found_property
import logging
log = logging.getLogger("guessit.transfo.guess_bonus_features")
def process(mtree):
def previous_group(g):
for leaf in mtree.unidentified_leaves()[::-1]:
if leaf.node_idx < g.node_idx:
return leaf
def next_group(g):
for leaf in mtree.unidentified_leaves():
if leaf.node_idx > g.node_idx:
return leaf
def same_group(g1, g2):
return g1.node_idx[:2] == g2.node_idx[:2]
bonus = [ node for node in mtree.leaves() if 'bonusNumber' in node.guess ]
if bonus:
bonusTitle = next_group(bonus[0])
if same_group(bonusTitle, bonus[0]):
found_property(bonusTitle, 'bonusTitle', 0.8)
filmNumber = [ node for node in mtree.leaves()
if 'filmNumber' in node.guess ]
if filmNumber:
filmSeries = previous_group(filmNumber[0])
found_property(filmSeries, 'filmSeries', 0.9)
title = next_group(filmNumber[0])
found_property(title, 'title', 0.9)
season = [ node for node in mtree.leaves() if 'season' in node.guess ]
if season and 'bonusNumber' in mtree.info:
series = previous_group(season[0])
if same_group(series, season[0]):
found_property(series, 'series', 0.9)
libs/guessit/transfo/guess_date.py
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import SingleNodeGuesser
from guessit.date import search_date
import logging
log = logging.getLogger("guessit.transfo.guess_date")
def guess_date(string):
date, span = search_date(string)
if date:
return { 'date': date }, span
else:
return None, None
def process(mtree):
SingleNodeGuesser(guess_date, 1.0, log).process(mtree)
libs/guessit/transfo/guess_episode_info_from_position.py
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import found_property
from guessit.patterns import non_episode_title, unlikely_series
import logging
log = logging.getLogger("guessit.transfo.guess_episode_info_from_position")
def match_from_epnum_position(mtree, node):
epnum_idx = node.node_idx
# a few helper functions to be able to filter using high-level semantics
def before_epnum_in_same_pathgroup():
return [ leaf for leaf in mtree.unidentified_leaves()
if (leaf.node_idx[0] == epnum_idx[0] and
leaf.node_idx[1:] < epnum_idx[1:]) ]
def after_epnum_in_same_pathgroup():
return [ leaf for leaf in mtree.unidentified_leaves()
if (leaf.node_idx[0] == epnum_idx[0] and
leaf.node_idx[1:] > epnum_idx[1:]) ]
def after_epnum_in_same_explicitgroup():
return [ leaf for leaf in mtree.unidentified_leaves()
if (leaf.node_idx[:2] == epnum_idx[:2] and
leaf.node_idx[2:] > epnum_idx[2:]) ]
# epnumber is the first group and there are only 2 after it in same
# path group
# -> series title - episode title
title_candidates = [ n for n in after_epnum_in_same_pathgroup()
if n.clean_value.lower() not in non_episode_title ]
if ('title' not in mtree.info and # no title
before_epnum_in_same_pathgroup() == [] and # no groups before
len(title_candidates) == 2): # only 2 groups after
found_property(title_candidates[0], 'series', confidence=0.4)
found_property(title_candidates[1], 'title', confidence=0.4)
return
# if we have at least 1 valid group before the episodeNumber, then it's
# probably the series name
series_candidates = before_epnum_in_same_pathgroup()
if len(series_candidates) >= 1:
found_property(series_candidates[0], 'series', confidence=0.7)
# only 1 group after (in the same path group) and it's probably the
# episode title
title_candidates = [ n for n in after_epnum_in_same_pathgroup()
if n.clean_value.lower() not in non_episode_title ]
if len(title_candidates) == 1:
found_property(title_candidates[0], 'title', confidence=0.5)
return
else:
# try in the same explicit group, with lower confidence
title_candidates = [ n for n in after_epnum_in_same_explicitgroup()
if n.clean_value.lower() not in non_episode_title
]
if len(title_candidates) == 1:
found_property(title_candidates[0], 'title', confidence=0.4)
return
elif len(title_candidates) > 1:
found_property(title_candidates[0], 'title', confidence=0.3)
return
# get the one with the longest value
title_candidates = [ n for n in after_epnum_in_same_pathgroup()
if n.clean_value.lower() not in non_episode_title ]
if title_candidates:
maxidx = -1
maxv = -1
for i, c in enumerate(title_candidates):
if len(c.clean_value) > maxv:
maxidx = i
maxv = len(c.clean_value)
found_property(title_candidates[maxidx], 'title', confidence=0.3)
def process(mtree):
eps = [node for node in mtree.leaves() if 'episodeNumber' in node.guess]
if eps:
match_from_epnum_position(mtree, eps[0])
else:
# if we don't have the episode number, but at least 2 groups in the
# basename, then it's probably series - eptitle
basename = mtree.node_at((-2,))
title_candidates = [ n for n in basename.unidentified_leaves()
if n.clean_value.lower() not in non_episode_title
]
if len(title_candidates) >= 2:
found_property(title_candidates[0], 'series', 0.4)
found_property(title_candidates[1], 'title', 0.4)
# if we only have 1 remaining valid group in the folder containing the
# file, then it's likely that it is the series name
try:
series_candidates = mtree.node_at((-3,)).unidentified_leaves()
except ValueError:
series_candidates = []
if len(series_candidates) == 1:
found_property(series_candidates[0], 'series', 0.3)
# if there's a path group that only contains the season info, then the
# previous one is most likely the series title (ie: ../series/season X/..)
eps = [ node for node in mtree.nodes()
if 'season' in node.guess and 'episodeNumber' not in node.guess ]
if eps:
previous = [ node for node in mtree.unidentified_leaves()
if node.node_idx[0] == eps[0].node_idx[0] - 1 ]
if len(previous) == 1:
found_property(previous[0], 'series', 0.5)
# reduce the confidence of unlikely series
for node in mtree.nodes():
if 'series' in node.guess:
if node.guess['series'].lower() in unlikely_series:
new_confidence = node.guess.confidence('series') * 0.5
node.guess.set_confidence('series', new_confidence)
libs/guessit/transfo/guess_episodes_rexps.py
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import episode_rexps
import re
import logging
log = logging.getLogger("guessit.transfo.guess_episodes_rexps")
def guess_episodes_rexps(string):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, string, re.IGNORECASE)
if match:
return (Guess(match.groupdict(), confidence=confidence),
(match.start() + span_adjust[0],
match.end() + span_adjust[1]))
return None, None
def process(mtree):
SingleNodeGuesser(guess_episodes_rexps, None, log).process(mtree)
libs/guessit/transfo/guess_filetype.py
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.patterns import (subtitle_exts, video_exts, episode_rexps,
find_properties, canonical_form)
import os.path
import re
import mimetypes
import logging
log = logging.getLogger("guessit.transfo.guess_filetype")
def guess_filetype(filename, filetype):
other = {}
# look at the extension first
fileext = os.path.splitext(filename)[1][1:].lower()
if fileext in subtitle_exts:
if 'movie' in filetype:
filetype = 'moviesubtitle'
elif 'episode' in filetype:
filetype = 'episodesubtitle'
else:
filetype = 'subtitle'
other = { 'container': fileext }
elif fileext in video_exts:
if filetype == 'autodetect':
filetype = 'video'
other = { 'container': fileext }
else:
if filetype == 'autodetect':
filetype = 'unknown'
other = { 'extension': fileext }
# put the filetype inside a dummy container to be able to have the
# following functions work correctly as closures
# this is a workaround for python 2 which doesn't have the
# 'nonlocal' keyword (python 3 does have it)
filetype_container = [filetype]
def upgrade_episode():
if filetype_container[0] == 'video':
filetype_container[0] = 'episode'
elif filetype_container[0] == 'subtitle':
filetype_container[0] = 'episodesubtitle'
def upgrade_movie():
if filetype_container[0] == 'video':
filetype_container[0] = 'movie'
elif filetype_container[0] == 'subtitle':
filetype_container[0] = 'moviesubtitle'
# now look whether there are some specific hints for episode vs movie
if filetype in ('video', 'subtitle'):
for rexp, _, _ in episode_rexps:
match = re.search(rexp, filename, re.IGNORECASE)
if match:
upgrade_episode()
break
for prop, value, _, _ in find_properties(filename):
log.debug('prop: %s = %s' % (prop, value))
if prop == 'episodeFormat':
upgrade_episode()
break
elif canonical_form(value) == 'DVB':
upgrade_episode()
break
# if no episode info found, assume it's a movie
upgrade_movie()
filetype = filetype_container[0]
return filetype, other
def process(mtree, filetype='autodetect'):
filetype, other = guess_filetype(mtree.string, filetype)
mtree.guess.set('type', filetype, confidence=1.0)
log.debug('Found with confidence %.2f: %s' % (1.0, mtree.guess))
filetype_info = Guess(other, confidence=1.0)
# guess the mimetype of the filename
# TODO: handle other mimetypes not found on the default type_maps
# mimetypes.types_map['.srt']='text/subtitle'
mime, _ = mimetypes.guess_type(mtree.string, strict=False)
if mime is not None:
filetype_info.update({'mimetype': mime}, confidence=1.0)
node_ext = mtree.node_at((-1,))
node_ext.guess = filetype_info
log.debug('Found with confidence %.2f: %s' % (1.0, node_ext.guess))
libs/guessit/transfo/guess_language.py
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.language import search_language
from guessit.textutils import clean_string
import logging
log = logging.getLogger("guessit.transfo.guess_language")
def guess_language(string):
language, span, confidence = search_language(string)
if language:
# is it a subtitle language?
if 'sub' in clean_string(string[:span[0]]).lower().split(' '):
return (Guess({'subtitleLanguage': language},
confidence=confidence),
span)
else:
return (Guess({'language': language},
confidence=confidence),
span)
return None, None
def process(mtree):
SingleNodeGuesser(guess_language, None, log).process(mtree)
libs/guessit/transfo/guess_movie_title_from_position.py
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
import logging
log = logging.getLogger("guessit.transfo.guess_movie_title_from_position")
def process(mtree):
def found_property(node, name, value, confidence):
node.guess = Guess({ name: value },
confidence=confidence)
log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
def found_title(node, confidence):
found_property(node, 'title', node.clean_value, confidence)
basename = mtree.node_at((-2,))
all_valid = lambda leaf: len(leaf.clean_value) > 0
basename_leftover = basename.unidentified_leaves(valid=all_valid)
try:
folder = mtree.node_at((-3,))
folder_leftover = folder.unidentified_leaves()
except ValueError:
folder = None
folder_leftover = []
log.debug('folder: %s' % folder_leftover)
log.debug('basename: %s' % basename_leftover)
# specific cases:
# if we find the same group both in the folder name and the filename,
# it's a good candidate for title
if (folder_leftover and basename_leftover and
folder_leftover[0].clean_value == basename_leftover[0].clean_value):
found_title(folder_leftover[0], confidence=0.8)
return
# specific cases:
# if the basename contains a number first followed by an unidentified
# group, and the folder only contains 1 unidentified one, then we have
# a series
# ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
try:
series = folder_leftover[0]
filmNumber = basename_leftover[0]
title = basename_leftover[1]
basename_leaves = basename.leaves()
num = int(filmNumber.clean_value)
log.debug('series: %s' % series.clean_value)
log.debug('title: %s' % title.clean_value)
if (series.clean_value != title.clean_value and
series.clean_value != filmNumber.clean_value and
basename_leaves.index(filmNumber) == 0 and
basename_leaves.index(title) == 1):
found_title(title, confidence=0.6)
found_property(series, 'filmSeries',
series.clean_value, confidence=0.6)
found_property(filmNumber, 'filmNumber',
num, confidence=0.6)
return
except Exception:
pass
# specific cases:
# - movies/tttttt (yyyy)/tttttt.ccc
try:
if mtree.node_at((-4, 0)).value.lower() == 'movies':
folder = mtree.node_at((-3,))
# Note:too generic, might solve all the unittests as they all
# contain 'movies' in their path
#
#if containing_folder.is_leaf() and not containing_folder.guess:
# containing_folder.guess =
# Guess({ 'title': clean_string(containing_folder.value) },
# confidence=0.7)
year_group = folder.first_leaf_containing('year')
groups_before = folder.previous_unidentified_leaves(year_group)
found_title(groups_before[0], confidence=0.8)
return
except Exception:
pass
# if we have either format or videoCodec in the folder containing the file
# or one of its parents, then we should probably look for the title in
# there rather than in the basename
try:
props = mtree.previous_leaves_containing(mtree.children[-2],
[ 'videoCodec', 'format',
'language' ])
except IndexError:
props = []
if props:
group_idx = props[0].node_idx[0]
if all(g.node_idx[0] == group_idx for g in props):
# if they're all in the same group, take leftover info from there
leftover = mtree.node_at((group_idx,)).unidentified_leaves()
if leftover:
found_title(leftover[0], confidence=0.7)
return
# look for title in basename if there are some remaining undidentified
# groups there
if basename_leftover:
title_candidate = basename_leftover[0]
# if basename is only one word and the containing folder has at least
# 3 words in it, we should take the title from the folder name
# ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
# ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
if (title_candidate.clean_value.count(' ') == 0 and
folder_leftover and
folder_leftover[0].clean_value.count(' ') >= 2):
found_title(folder_leftover[0], confidence=0.7)
return
# if there are only 2 unidentified groups, the first of which is inside
# brackets or parentheses, we take the second one for the title:
# ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
if len(basename_leftover) == 2 and basename_leftover[0].is_explicit():
found_title(basename_leftover[1], confidence=0.8)
return
# if all else fails, take the first remaining unidentified group in the
# basename as title
found_title(title_candidate, confidence=0.6)
return
# if there are no leftover groups in the basename, look in the folder name
if folder_leftover:
found_title(folder_leftover[0], confidence=0.5)
return
# if nothing worked, look if we have a very small group at the beginning
# of the basename
basename = mtree.node_at((-2,))
basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True)
if basename_leftover:
found_title(basename_leftover[0], confidence=0.4)
return
libs/guessit/transfo/guess_properties.py
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import find_properties
import logging
log = logging.getLogger("guessit.transfo.guess_properties")
def guess_properties(string):
try:
prop, value, pos, end = find_properties(string)[0]
return { prop: value }, (pos, end)
except IndexError:
return None, None
def process(mtree):
SingleNodeGuesser(guess_properties, 1.0, log).process(mtree)
libs/guessit/transfo/guess_release_group.py
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import SingleNodeGuesser
import re
import logging
log = logging.getLogger("guessit.transfo.guess_release_group")
def guess_release_group(string):
group_names = [ r'\.(Xvid)-(?P<releaseGroup>.*?)[ \.]',
r'\.(DivX)-(?P<releaseGroup>.*?)[\. ]',
r'\.(DVDivX)-(?P<releaseGroup>.*?)[\. ]',
]
for rexp in group_names:
match = re.search(rexp, string, re.IGNORECASE)
if match:
metadata = match.groupdict()
metadata.update({ 'videoCodec': match.group(1) })
return metadata, (match.start() + 1, match.end() - 1)
return None, None
def process(mtree):
SingleNodeGuesser(guess_release_group, 0.8, log).process(mtree)
libs/guessit/transfo/guess_video_rexps.py
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import video_rexps, sep
import re
import logging
log = logging.getLogger("guessit.transfo.guess_video_rexps")
def guess_video_rexps(string):
string = '-' + string + '-'
for rexp, confidence, span_adjust in video_rexps:
match = re.search(sep + rexp + sep, string, re.IGNORECASE)
if match:
metadata = match.groupdict()
# is this the better place to put it? (maybe, as it is at least
# the soonest that we can catch it)
if metadata.get('cdNumberTotal', -1) is None:
del metadata['cdNumberTotal']
return (Guess(metadata, confidence=confidence),
(match.start() + span_adjust[0],
match.end() + span_adjust[1] - 2))
return None, None
def process(mtree):
SingleNodeGuesser(guess_video_rexps, None, log).process(mtree)
libs/guessit/transfo/guess_weak_episodes_rexps.py
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import weak_episode_rexps
import re
import logging
log = logging.getLogger("guessit.transfo.guess_weak_episodes_rexps")
def guess_weak_episodes_rexps(string, node):
if 'episodeNumber' in node.root.info:
return None, None
for rexp, span_adjust in weak_episode_rexps:
match = re.search(rexp, string, re.IGNORECASE)
if match:
metadata = match.groupdict()
span = (match.start() + span_adjust[0],
match.end() + span_adjust[1])
epnum = int(metadata['episodeNumber'])
if epnum > 100:
return Guess({ 'season': epnum // 100,
'episodeNumber': epnum % 100 },
confidence=0.6), span
else:
return Guess(metadata, confidence=0.3), span
return None, None
guess_weak_episodes_rexps.use_node = True
def process(mtree):
SingleNodeGuesser(guess_weak_episodes_rexps, 0.6, log).process(mtree)
libs/guessit/transfo/guess_website.py
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import SingleNodeGuesser
from guessit.patterns import websites
import logging
log = logging.getLogger("guessit.transfo.guess_website")
def guess_website(string):
low = string.lower()
for site in websites:
pos = low.find(site.lower())
if pos != -1:
return {'website': site}, (pos, pos + len(site))
return None, None
def process(mtree):
SingleNodeGuesser(guess_website, 1.0, log).process(mtree)
libs/guessit/transfo/guess_year.py
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.transfo import SingleNodeGuesser
from guessit.date import search_year
import logging
log = logging.getLogger("guessit.transfo.guess_year")
def guess_year(string):
year, span = search_year(string)
if year:
return { 'year': year }, span
else:
return None, None
def process(mtree):
SingleNodeGuesser(guess_year, 1.0, log).process(mtree)
libs/guessit/transfo/post_process.py
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import subtitle_exts
import logging
log = logging.getLogger("guessit.transfo.post_process")
def process(mtree):
# 1- try to promote language to subtitle language where it makes sense
for node in mtree.nodes():
if 'language' not in node.guess:
continue
def promote_subtitle():
# pylint: disable=W0631
node.guess.set('subtitleLanguage', node.guess['language'],
confidence=node.guess.confidence('language'))
del node.guess['language']
# - if we matched a language in a file with a sub extension and that
# the group is the last group of the filename, it is probably the
# language of the subtitle
# (eg: 'xxx.english.srt')
if (mtree.node_at((-1,)).value.lower() in subtitle_exts and
node == mtree.leaves()[-2]):
promote_subtitle()
# - if a language is in an explicit group just preceded by "st",
# it is a subtitle language (eg: '...st[fr-eng]...')
try:
idx = node.node_idx
previous = mtree.node_at((idx[0], idx[1] - 1)).leaves()[-1]
if previous.value.lower()[-2:] == 'st':
promote_subtitle()
except IndexError:
pass
# 2- ", the" at the end of a series title should be prepended to it
for node in mtree.nodes():
if 'series' not in node.guess:
continue
series = node.guess['series']
lseries = series.lower()
if lseries[-4:] == ',the':
node.guess['series'] = 'The ' + series[:-4]
if lseries[-5:] == ', the':
node.guess['series'] = 'The ' + series[:-5]
libs/guessit/transfo/split_explicit_groups.py
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.textutils import find_first_level_groups
from guessit.patterns import group_delimiters
import logging
log = logging.getLogger("guessit.transfo.split_explicit_groups")
def process(mtree):
"""return the string split into explicit groups, that is, those either
between parenthese, square brackets or curly braces, and those separated
by a dash."""
for c in mtree.children:
groups = find_first_level_groups(c.value, group_delimiters[0])
for delimiters in group_delimiters:
flatten = lambda l, x: l + find_first_level_groups(x, delimiters)
groups = reduce(flatten, groups, [])
# do not do this at this moment, it is not strong enough and can break other
# patterns, such as dates, etc...
#groups = reduce(lambda l, x: l + x.split('-'), groups, [])
c.split_on_components(groups)
libs/guessit/transfo/split_on_dash.py
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import sep
import re
import logging
log = logging.getLogger("guessit.transfo.split_on_dash")
def process(mtree):
for node in mtree.unidentified_leaves():
indices = []
didx = 0
pattern = re.compile(sep + '-' + sep)
match = pattern.search(node.value)
while match:
span = match.span()
indices.extend([ span[0], span[1] ])
match = pattern.search(node.value, span[1])
didx = node.value.find('-')
while didx > 0:
if (didx > 10 and
(didx - 1 not in indices and
didx + 2 not in indices)):
indices.extend([ didx, didx + 1 ])
didx = node.value.find('-', didx + 1)
if indices:
node.partition(indices)
libs/guessit/transfo/split_path_components.py
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import fileutils
import os.path
import logging
log = logging.getLogger("guessit.transfo.split_path_components")
def process(mtree):
"""Returns the filename split into [ dir*, basename, ext ]."""
components = fileutils.split_path(mtree.value)
basename = components.pop(-1)
components += list(os.path.splitext(basename))
components[-1] = components[-1][1:] # remove the '.' from the extension
mtree.split_on_components(components)
libs/requests/__init__.py
+2 -3
View File
@@ -15,14 +15,13 @@ requests
"""
__title__ = 'requests'
__version__ = '0.10.1'
__build__ = 0x001001
__version__ = '0.11.1'
__build__ = 0x001101
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2012 Kenneth Reitz'
from . import utils
from .models import Request, Response
from .api import request, get, head, post, patch, put, delete, options
libs/requests/api.py
+9 -8
View File
@@ -33,6 +33,7 @@ def request(method, url, **kwargs):
:param config: (optional) A configuration dictionary.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
"""
s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
@@ -44,7 +45,7 @@ def get(url, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -55,7 +56,7 @@ def options(url, **kwargs):
"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -66,10 +67,10 @@ def head(url, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
kwargs.setdefault('allow_redirects', False)
return request('head', url, **kwargs)
@@ -78,7 +79,7 @@ def post(url, data=None, **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('post', url, data=data, **kwargs)
@@ -89,7 +90,7 @@ def put(url, data=None, **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('put', url, data=data, **kwargs)
@@ -100,7 +101,7 @@ def patch(url, data=None, **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('patch', url, data=data, **kwargs)
@@ -110,7 +111,7 @@ def delete(url, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('delete', url, **kwargs)
libs/requests/async.py
+28 -10
View File
@@ -17,13 +17,13 @@ except ImportError:
raise RuntimeError('Gevent is required for requests.async.')
# Monkey-patch.
curious_george.patch_all(thread=False)
curious_george.patch_all(thread=False, select=False)
from . import api
__all__ = (
'map',
'map', 'imap',
'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
)
@@ -46,15 +46,15 @@ def patched(f):
return wrapped
def send(r, pool=None):
"""Sends the request object using the specified pool. If a pool isn't
def send(r, pool=None, prefetch=False):
"""Sends the request object using the specified pool. If a pool isn't
specified this method blocks. Pools are useful because you can specify size
and can hence limit concurrency."""
if pool != None:
return pool.spawn(r.send)
return pool.spawn(r.send, prefetch=prefetch)
return gevent.spawn(r.send)
return gevent.spawn(r.send, prefetch=prefetch)
# Patched requests.api functions.
@@ -79,10 +79,28 @@ def map(requests, prefetch=True, size=None):
requests = list(requests)
pool = Pool(size) if size else None
jobs = [send(r, pool) for r in requests]
jobs = [send(r, pool, prefetch=prefetch) for r in requests]
gevent.joinall(jobs)
if prefetch:
[r.response.content for r in requests]
return [r.response for r in requests]
return [r.response for r in requests]
def imap(requests, prefetch=True, size=2):
"""Concurrently converts a generator object of Requests to
a generator of Responses.
:param requests: a generator of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. default is 2
"""
pool = Pool(size)
def send(r):
r.send(prefetch)
return r.response
for r in pool.imap_unordered(send, requests):
yield r
pool.join()
libs/requests/auth.py
+2 -4
View File
@@ -7,13 +7,11 @@ requests.auth
This module contains the authentication handlers for Requests.
"""
from __future__ import unicode_literals
import time
import hashlib
from base64 import b64encode
from .compat import urlparse, str, bytes
from .compat import urlparse, str
from .utils import randombytes, parse_dict_header
@@ -21,7 +19,7 @@ from .utils import randombytes, parse_dict_header
def _basic_auth_str(username, password):
"""Returns a Basic Auth string."""
return 'Basic ' + b64encode(("%s:%s" % (username, password)).encode('utf-8')).strip().decode('utf-8')
return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1')
class AuthBase(object):
libs/requests/compat.py
+4 -1
View File
@@ -86,8 +86,10 @@ if is_py2:
from .packages.oreos.monkeys import SimpleCookie
from StringIO import StringIO
str = unicode
bytes = str
str = unicode
basestring = basestring
elif is_py3:
@@ -99,4 +101,5 @@ elif is_py3:
str = str
bytes = bytes
basestring = (str,bytes)
libs/requests/defaults.py
+11
View File
@@ -15,10 +15,16 @@ Configurations:
:max_retries: The number of times a request should be retried in the event of a connection failure.
:danger_mode: If true, Requests will raise errors immediately.
:safe_mode: If true, Requests will catch all errors.
:strict_mode: If true, Requests will do its best to follow RFCs (e.g. POST redirects).
:pool_maxsize: The maximium size of an HTTP connection pool.
:pool_connections: The number of active HTTP connection pools to use.
:encode_uri: If true, URIs will automatically be percent-encoded.
:trust_env: If true, the surrouding environment will be trusted (environ, netrc).
"""
SCHEMAS = ['http', 'https']
from . import __version__
defaults = dict()
@@ -37,4 +43,9 @@ defaults['pool_maxsize'] = 10
defaults['max_retries'] = 0
defaults['danger_mode'] = False
defaults['safe_mode'] = False
defaults['strict_mode'] = False
defaults['keep_alive'] = True
defaults['encode_uri'] = True
defaults['trust_env'] = True
libs/requests/exceptions.py
+8 -1
View File
@@ -8,12 +8,13 @@ This module contains the set of Requests' exceptions.
"""
class RequestException(Exception):
class RequestException(RuntimeError):
"""There was an ambiguous exception that occurred while handling your
request."""
class HTTPError(RequestException):
"""An HTTP error occurred."""
response = None
class ConnectionError(RequestException):
"""A Connection error occurred."""
@@ -29,3 +30,9 @@ class URLRequired(RequestException):
class TooManyRedirects(RequestException):
"""Too many redirects."""
class MissingSchema(RequestException, ValueError):
"""The URL schema (e.g. http or https) is missing."""
class InvalidSchema(RequestException, ValueError):
"""See defaults.py for valid schemas."""
libs/requests/models.py
+152 -139
View File
@@ -21,14 +21,16 @@ from .packages.urllib3.exceptions import SSLError as _SSLError
from .packages.urllib3.exceptions import HTTPError as _HTTPError
from .packages.urllib3 import connectionpool, poolmanager
from .packages.urllib3.filepost import encode_multipart_formdata
from .defaults import SCHEMAS
from .exceptions import (
ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects,
URLRequired, SSLError)
URLRequired, SSLError, MissingSchema, InvalidSchema)
from .utils import (
get_encoding_from_headers, stream_decode_response_unicode,
stream_decompress, guess_filename, requote_path, dict_from_string)
from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2
get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
dict_from_string, stream_decode_response_unicode, get_netrc_auth)
from .compat import (
urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
SimpleCookie, is_py2)
# Import chardet if it is available.
try:
@@ -39,7 +41,6 @@ except ImportError:
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
class Request(object):
"""The :class:`Request <Request>` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
@@ -62,18 +63,17 @@ class Request(object):
config=None,
_poolmanager=None,
verify=None,
session=None):
session=None,
cert=None):
#: Dictionary of configurations for this request.
self.config = dict(config or [])
#: Float describes the timeout of the request.
# (Use socket.setdefaulttimeout() as fallback)
self.timeout = timeout
#: Request URL.
# if isinstance(url, str):
# url = url.encode('utf-8')
# print(dir(url))
self.url = url
#: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
@@ -103,6 +103,14 @@ class Request(object):
# Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
self.proxies = dict(proxies or [])
# If no proxies are given, allow configuration by environment variables
# HTTP_PROXY and HTTPS_PROXY.
if not self.proxies and self.config.get('trust_env'):
if 'HTTP_PROXY' in os.environ:
self.proxies['http'] = os.environ['HTTP_PROXY']
if 'HTTPS_PROXY' in os.environ:
self.proxies['https'] = os.environ['HTTPS_PROXY']
self.data, self._enc_data = self._encode_params(data)
self.params, self._enc_params = self._encode_params(params)
@@ -116,9 +124,6 @@ class Request(object):
#: CookieJar to attach to :class:`Request <Request>`.
self.cookies = dict(cookies or [])
#: Dictionary of configurations for this request.
self.config = dict(config or [])
#: True if Request has been sent.
self.sent = False
@@ -139,6 +144,9 @@ class Request(object):
#: SSL Verification.
self.verify = verify
#: SSL Certificate
self.cert = cert
if headers:
headers = CaseInsensitiveDict(self.headers)
else:
@@ -152,15 +160,9 @@ class Request(object):
self.headers = headers
self._poolmanager = _poolmanager
# Pre-request hook.
r = dispatch_hook('pre_request', hooks, self)
self.__dict__.update(r.__dict__)
def __repr__(self):
return '<Request [%s]>' % (self.method)
def _build_response(self, resp):
"""Build internal :class:`Response <Response>` object
from given response.
@@ -200,29 +202,36 @@ class Request(object):
# Save original response for later.
response.raw = resp
response.url = self.full_url
if isinstance(self.full_url, bytes):
response.url = self.full_url.decode('utf-8')
else:
response.url = self.full_url
return response
history = []
r = build(resp)
cookies = self.cookies
self.cookies.update(r.cookies)
if r.status_code in REDIRECT_STATI and not self.redirect:
while (
('location' in r.headers) and
((r.status_code is codes.see_other) or (self.allow_redirects))
):
while (('location' in r.headers) and
((r.status_code is codes.see_other) or (self.allow_redirects))):
r.content # Consume socket so it can be released
if not len(history) < self.config.get('max_redirects'):
raise TooManyRedirects()
# Release the connection back into the pool.
r.raw.release_conn()
history.append(r)
url = r.headers['location']
data = self.data
# Handle redirection without scheme (see: RFC 1808 Section 4)
if url.startswith('//'):
@@ -232,14 +241,29 @@ class Request(object):
# Facilitate non-RFC2616-compliant 'location' headers
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
if not urlparse(url).netloc:
url = urljoin(r.url, url)
url = urljoin(r.url,
# Compliant with RFC3986, we percent
# encode the url.
requote_uri(url))
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
if r.status_code is codes.see_other:
method = 'GET'
data = None
else:
method = self.method
# Do what the browsers do if strict_mode is off...
if (not self.config.get('strict_mode')):
if r.status_code in (codes.moved, codes.found) and self.method == 'POST':
method = 'GET'
data = None
if (r.status_code == 303) and self.method != 'HEAD':
method = 'GET'
data = None
# Remove the cookie headers that were sent.
headers = self.headers
try:
@@ -254,18 +278,19 @@ class Request(object):
method=method,
params=self.session.params,
auth=self.auth,
cookies=cookies,
cookies=self.cookies,
redirect=True,
data=data,
config=self.config,
timeout=self.timeout,
_poolmanager=self._poolmanager,
proxies = self.proxies,
verify = self.verify,
session = self.session
proxies=self.proxies,
verify=self.verify,
session=self.session,
cert=self.cert
)
request.send()
cookies.update(request.response.cookies)
r = request.response
self.cookies.update(r.cookies)
@@ -275,7 +300,6 @@ class Request(object):
self.response.request = self
self.response.cookies.update(self.cookies)
@staticmethod
def _encode_params(data):
"""Encode parameters in a piece of data.
@@ -288,10 +312,12 @@ class Request(object):
returns it twice.
"""
if isinstance(data, bytes):
return data, data
if hasattr(data, '__iter__') and not isinstance(data, str):
data = dict(data)
if hasattr(data, 'items'):
result = []
for k, vs in list(data.items()):
@@ -314,30 +340,44 @@ class Request(object):
# Support for unicode domain names and paths.
scheme, netloc, path, params, query, fragment = urlparse(url)
if not scheme:
raise ValueError("Invalid URL %r: No schema supplied" % url)
raise MissingSchema("Invalid URL %r: No schema supplied" % url)
if not scheme in SCHEMAS:
raise InvalidSchema("Invalid scheme %r" % scheme)
netloc = netloc.encode('idna').decode('utf-8')
if not path:
path = '/'
if is_py2:
if isinstance(scheme, str):
scheme = scheme.encode('utf-8')
if isinstance(netloc, str):
netloc = netloc.encode('utf-8')
if isinstance(path, str):
path = path.encode('utf-8')
if isinstance(params, str):
params = params.encode('utf-8')
if isinstance(query, str):
query = query.encode('utf-8')
if isinstance(fragment, str):
fragment = fragment.encode('utf-8')
path = requote_path(path)
# print([ scheme, netloc, path, params, query, fragment ])
# print('---------------------')
url = (urlunparse([ scheme, netloc, path, params, query, fragment ]))
url = (urlunparse([scheme, netloc, path, params, query, fragment]))
if self._enc_params:
if urlparse(url).query:
return '%s&%s' % (url, self._enc_params)
url = '%s&%s' % (url, self._enc_params)
else:
return '%s?%s' % (url, self._enc_params)
else:
return url
url = '%s?%s' % (url, self._enc_params)
if self.config.get('encode_uri', True):
url = requote_uri(url)
return url
@property
def path_url(self):
@@ -355,9 +395,6 @@ class Request(object):
if not path:
path = '/'
# if is_py3:
path = quote(path.encode('utf-8'))
url.append(path)
query = p.query
@@ -365,19 +402,15 @@ class Request(object):
url.append('?')
url.append(query)
# print(url)
return ''.join(url)
def register_hook(self, event, hook):
"""Properly register a hook."""
return self.hooks[event].append(hook)
def send(self, anyway=False, prefetch=False):
"""Sends the request. Returns True of successful, false if not.
"""Sends the request. Returns True of successful, False if not.
If there was an HTTPError during transmission,
self.response.status_code will contain the HTTPError code.
@@ -435,6 +468,10 @@ class Request(object):
if (content_type) and (not 'content-type' in self.headers):
self.headers['Content-Type'] = content_type
# Use .netrc auth if none was provided.
if not self.auth and self.config.get('trust_env'):
self.auth = get_netrc_auth(url)
if self.auth:
if isinstance(self.auth, tuple) and len(self.auth) == 2:
# special-case basic HTTP auth
@@ -472,13 +509,12 @@ class Request(object):
if self.verify is not True:
cert_loc = self.verify
# Look for configuration.
if not cert_loc:
if not cert_loc and self.config.get('trust_env'):
cert_loc = os.environ.get('REQUESTS_CA_BUNDLE')
# Curl compatiblity.
if not cert_loc:
if not cert_loc and self.config.get('trust_env'):
cert_loc = os.environ.get('CURL_CA_BUNDLE')
# Use the awesome certifi list.
@@ -491,6 +527,13 @@ class Request(object):
conn.cert_reqs = 'CERT_NONE'
conn.ca_certs = None
if self.cert and self.verify:
if len(self.cert) == 2:
conn.cert_file = self.cert[0]
conn.key_file = self.cert[1]
else:
conn.cert_file = self.cert
if not self.sent or anyway:
if self.cookies:
@@ -509,6 +552,10 @@ class Request(object):
# Attach Cookie header to request.
self.headers['Cookie'] = cookie_header
# Pre-request hook.
r = dispatch_hook('pre_request', self.hooks, self)
self.__dict__.update(r.__dict__)
try:
# The inner try .. except re-raises certain exceptions as
# internal exception types; the outer suppresses exceptions
@@ -523,7 +570,7 @@ class Request(object):
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=True,
decode_content=False,
retries=self.config.get('max_retries', 0),
timeout=self.timeout,
)
@@ -613,7 +660,6 @@ class Response(object):
#: Dictionary of configurations for this request.
self.config = {}
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
@@ -629,11 +675,10 @@ class Response(object):
def ok(self):
try:
self.raise_for_status()
except HTTPError:
except RequestException:
return False
return True
def iter_content(self, chunk_size=10 * 1024, decode_unicode=False):
"""Iterates over the response data. This avoids reading the content
at once into memory for large responses. The chunk size is the number
@@ -653,81 +698,39 @@ class Response(object):
yield chunk
self._content_consumed = True
def generate_chunked():
resp = self.raw._original_response
fp = resp.fp
if resp.chunk_left is not None:
pending_bytes = resp.chunk_left
while pending_bytes:
chunk = fp.read(min(chunk_size, pending_bytes))
pending_bytes-=len(chunk)
yield chunk
fp.read(2) # throw away crlf
while 1:
#XXX correct line size? (httplib has 64kb, seems insane)
pending_bytes = fp.readline(40).strip()
pending_bytes = int(pending_bytes, 16)
if pending_bytes == 0:
break
while pending_bytes:
chunk = fp.read(min(chunk_size, pending_bytes))
pending_bytes-=len(chunk)
yield chunk
fp.read(2) # throw away crlf
self._content_consumed = True
fp.close()
if getattr(getattr(self.raw, '_original_response', None), 'chunked', False):
gen = generate_chunked()
else:
gen = generate()
if 'gzip' in self.headers.get('content-encoding', ''):
gen = stream_decompress(gen, mode='gzip')
elif 'deflate' in self.headers.get('content-encoding', ''):
gen = stream_decompress(gen, mode='deflate')
gen = stream_untransfer(generate(), self)
if decode_unicode:
gen = stream_decode_response_unicode(gen, self)
return gen
def iter_lines(self, chunk_size=10 * 1024, decode_unicode=None):
"""Iterates over the response data, one line at a time. This
avoids reading the content at once into memory for large
responses.
"""
#TODO: why rstrip by default
pending = None
for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode):
for chunk in self.iter_content(
chunk_size=chunk_size,
decode_unicode=decode_unicode):
if pending is not None:
chunk = pending + chunk
lines = chunk.splitlines(True)
lines = chunk.splitlines()
for line in lines[:-1]:
yield line.rstrip()
# Save the last part of the chunk for next iteration, to keep full line together
# lines may be empty for the last chunk of a chunked response
if lines:
pending = lines[-1]
#if pending is a complete line, give it baack
if pending[-1] == '\n':
yield pending.rstrip()
pending = None
if lines[-1][-1] == chunk[-1]:
pending = lines.pop()
else:
pending = None
# Yield the last line
if pending is not None:
yield pending.rstrip()
for line in lines:
yield line
if pending is not None:
yield pending
@property
def content(self):
@@ -740,13 +743,26 @@ class Response(object):
raise RuntimeError(
'The content for this response was already consumed')
self._content = self.raw.read()
if self.status_code is 0:
self._content = None
else:
self._content = bytes().join(self.iter_content()) or bytes()
except AttributeError:
self._content = None
self._content_consumed = True
return self._content
def _detected_encoding(self):
try:
detected = chardet.detect(self.content) or {}
return detected.get('encoding')
# Trust that chardet isn't available or something went terribly wrong.
except Exception:
pass
@property
def text(self):
@@ -762,43 +778,40 @@ class Response(object):
# Fallback to auto-detected encoding if chardet is available.
if self.encoding is None:
try:
detected = chardet.detect(self.content) or {}
encoding = detected.get('encoding')
# Trust that chardet isn't available or something went terribly wrong.
except Exception:
pass
encoding = self._detected_encoding()
# Decode unicode from given encoding.
try:
content = str(self.content, encoding)
content = str(self.content, encoding, errors='replace')
except LookupError:
# A LookupError is raised if the encoding was not found which could
# indicate a misspelling or similar mistake.
#
# So we try blindly encoding.
content = str(self.content, errors='replace')
except (UnicodeError, TypeError):
pass
# Try to fall back:
if not content:
try:
content = str(content, encoding, errors='replace')
except (UnicodeError, TypeError):
pass
return content
def raise_for_status(self):
def raise_for_status(self, allow_redirects=True):
"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
if self.error:
raise self.error
if (self.status_code >= 300) and (self.status_code < 400):
raise HTTPError('%s Redirection' % self.status_code)
if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects:
http_error = HTTPError('%s Redirection' % self.status_code)
http_error.response = self
raise http_error
elif (self.status_code >= 400) and (self.status_code < 500):
raise HTTPError('%s Client Error' % self.status_code)
http_error = HTTPError('%s Client Error' % self.status_code)
http_error.response = self
raise http_error
elif (self.status_code >= 500) and (self.status_code < 600):
raise HTTPError('%s Server Error' % self.status_code)
http_error = HTTPError('%s Server Error' % self.status_code)
http_error.response = self
raise http_error
libs/requests/packages/oreos/monkeys.py
+4 -1
View File
@@ -249,10 +249,13 @@ class CookieError(Exception):
# quoted with a preceeding '\' slash.
#
# These are taken from RFC2068 and RFC2109.
# _RFC2965Forbidden is the list of forbidden chars we accept anyway
# _LegalChars is the list of chars which don't require "'s
# _Translator hash-table for fast quoting
#
_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~[]_"
_RFC2965Forbidden = "[]:{}="
_LegalChars = ( string.ascii_letters + string.digits +
"!#$%&'*+-.^_`|~_@" + _RFC2965Forbidden )
_Translator = {
'\000' : '\\000', '\001' : '\\001', '\002' : '\\002',
'\003' : '\\003', '\004' : '\\004', '\005' : '\\005',
libs/requests/packages/oreos/structures.py
+5 -5
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
"""
oreos.sructures
~~~~~~~~~~~~~~~
oreos.structures
~~~~~~~~~~~~~~~~
The plastic blue packaging.
@@ -362,7 +362,7 @@ class MultiDict(TypeConversionDict):
"""
try:
return dict.pop(self, key)[0]
except KeyError, e:
except KeyError as e:
if default is not _missing:
return default
raise KeyError(str(e))
@@ -372,7 +372,7 @@ class MultiDict(TypeConversionDict):
try:
item = dict.popitem(self)
return (item[0], item[1][0])
except KeyError, e:
except KeyError as e:
raise KeyError(str(e))
def poplist(self, key):
@@ -389,7 +389,7 @@ class MultiDict(TypeConversionDict):
"""Pop a ``(key, list)`` tuple from the dict."""
try:
return dict.popitem(self)
except KeyError, e:
except KeyError as e:
raise KeyError(str(e))
def __copy__(self):
libs/requests/packages/urllib3/__init__.py
+6 -12
View File
@@ -10,26 +10,20 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.1'
__version__ = '1.3'
from .connectionpool import (
HTTPConnectionPool,
HTTPSConnectionPool,
connection_from_url,
get_host,
make_headers)
from .exceptions import (
HTTPError,
MaxRetryError,
SSLError,
TimeoutError)
connection_from_url
)
from . import exceptions
from .filepost import encode_multipart_formdata
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import HTTPResponse
from .filepost import encode_multipart_formdata
from .util import make_headers, get_host
# Set default logging handler to avoid "No handler found" warnings.
libs/requests/packages/urllib3/connectionpool.py
+43 -126
View File
@@ -9,45 +9,51 @@ import socket
from socket import error as SocketError, timeout as SocketTimeout
try:
from select import poll, POLLIN
except ImportError: # Doesn't exist on OSX and other platforms
from select import select
poll = False
try: # Python 3
from http.client import HTTPConnection, HTTPSConnection, HTTPException
from http.client import HTTPConnection, HTTPException
from http.client import HTTP_PORT, HTTPS_PORT
except ImportError:
from httplib import HTTPConnection, HTTPSConnection, HTTPException
from httplib import HTTPConnection, HTTPException
from httplib import HTTP_PORT, HTTPS_PORT
try: # Python 3
from queue import Queue, Empty, Full
from queue import LifoQueue, Empty, Full
except ImportError:
from Queue import Queue, Empty, Full
from Queue import LifoQueue, Empty, Full
try: # Compiled with SSL?
HTTPSConnection = object
BaseSSLError = None
ssl = None
try: # Python 3
from http.client import HTTPSConnection
except ImportError:
from httplib import HTTPSConnection
import ssl
BaseSSLError = ssl.SSLError
except ImportError:
ssl = None
BaseSSLError = None
except (ImportError, AttributeError):
pass
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .request import RequestMethods
from .response import HTTPResponse
from .exceptions import (SSLError,
MaxRetryError,
TimeoutError,
HostChangedError,
from .util import get_host, is_connection_dropped
from .exceptions import (
EmptyPoolError,
HostChangedError,
MaxRetryError,
SSLError,
TimeoutError,
)
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .packages import six
xrange = six.moves.xrange
log = logging.getLogger(__name__)
@@ -59,6 +65,7 @@ port_by_scheme = {
'https': HTTPS_PORT,
}
## Connection objects (extension of httplib)
class VerifiedHTTPSConnection(HTTPSConnection):
@@ -94,6 +101,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
if self.ca_certs:
match_hostname(self.sock.getpeercert(), self.host)
## Pool objects
class ConnectionPool(object):
@@ -103,6 +111,7 @@ class ConnectionPool(object):
"""
scheme = None
QueueCls = LifoQueue
def __init__(self, host, port=None):
self.host = host
@@ -156,11 +165,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1,
block=False, headers=None):
self.host = host
self.port = port
super(HTTPConnectionPool, self).__init__(host, port)
self.strict = strict
self.timeout = timeout
self.pool = Queue(maxsize)
self.pool = self.QueueCls(maxsize)
self.block = block
self.headers = headers or {}
@@ -198,7 +207,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn = self.pool.get(block=self.block, timeout=timeout)
# If this is a persistent connection, check if it got disconnected
if conn and conn.sock and is_connection_dropped(conn):
if conn and is_connection_dropped(conn):
log.info("Resetting dropped connection: %s" % self.host)
conn.close()
@@ -242,9 +251,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
timeout = self.timeout
conn.timeout = timeout # This only does anything in Py26+
conn.request(method, url, **httplib_request_kw)
conn.sock.settimeout(timeout)
# Set timeout
sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr.
if sock:
sock.settimeout(timeout)
httplib_response = conn.getresponse()
log.debug("\"%s %s %s\" %s %s" %
@@ -281,7 +294,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
.. note::
More commonly, it's appropriate to use a convenience method provided
by :class:`.RequestMethods`, such as :meth:`.request`.
by :class:`.RequestMethods`, such as :meth:`request`.
.. note::
@@ -468,7 +481,11 @@ class HTTPSConnectionPool(HTTPConnectionPool):
log.info("Starting new HTTPS connection (%d): %s"
% (self.num_connections, self.host))
if not ssl:
if not ssl: # Platform-specific: Python compiled without +ssl
if not HTTPSConnection or HTTPSConnection is object:
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
return HTTPSConnection(host=self.host, port=self.port)
connection = VerifiedHTTPSConnection(host=self.host, port=self.port)
@@ -477,87 +494,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
return connection
## Helpers
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
basic_auth=None):
"""
Shortcuts for generating request headers.
:param keep_alive:
If ``True``, adds 'connection: keep-alive' header.
:param accept_encoding:
Can be a boolean, list, or string.
``True`` translates to 'gzip,deflate'.
List will get joined by comma.
String will be used as provided.
:param user_agent:
String representing the user-agent you want, such as
"python-urllib3/0.6"
:param basic_auth:
Colon-separated username:password string for 'authorization: basic ...'
auth header.
Example: ::
>>> make_headers(keep_alive=True, user_agent="Batman/1.0")
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
>>> make_headers(accept_encoding=True)
{'accept-encoding': 'gzip,deflate'}
"""
headers = {}
if accept_encoding:
if isinstance(accept_encoding, str):
pass
elif isinstance(accept_encoding, list):
accept_encoding = ','.join(accept_encoding)
else:
accept_encoding = 'gzip,deflate'
headers['accept-encoding'] = accept_encoding
if user_agent:
headers['user-agent'] = user_agent
if keep_alive:
headers['connection'] = 'keep-alive'
if basic_auth:
headers['authorization'] = 'Basic ' + \
basic_auth.encode('base64').strip()
return headers
def get_host(url):
"""
Given a url, return its scheme, host and port (None if it's not there).
For example: ::
>>> get_host('http://google.com/mail/')
('http', 'google.com', None)
>>> get_host('google.com:80')
('http', 'google.com', 80)
"""
# This code is actually similar to urlparse.urlsplit, but much
# simplified for our needs.
port = None
scheme = 'http'
if '://' in url:
scheme, url = url.split('://', 1)
if '/' in url:
url, _path = url.split('/', 1)
if '@' in url:
_auth, url = url.split('@', 1)
if ':' in url:
url, port = url.split(':', 1)
port = int(port)
return scheme, url, port
def connection_from_url(url, **kw):
"""
Given a url, return an :class:`.ConnectionPool` instance of its host.
@@ -583,22 +519,3 @@ def connection_from_url(url, **kw):
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
def is_connection_dropped(conn):
"""
Returns True if the connection is dropped and should be closed.
:param conn:
``HTTPConnection`` object.
"""
if not poll:
return select([conn.sock], [], [], 0.0)[0]
# This version is better on platforms that support it.
p = poll()
p.register(conn.sock, POLLIN)
for (fno, ev) in p.poll(0.0):
if fno == conn.sock.fileno():
# Either data is buffered (bad), or the connection is dropped.
return True
libs/requests/packages/urllib3/exceptions.py
+17 -4
View File
@@ -4,6 +4,7 @@
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
## Base Exceptions
class HTTPError(Exception):
@@ -27,18 +28,20 @@ class SSLError(HTTPError):
class MaxRetryError(PoolError):
"Raised when the maximum number of retries is exceeded."
def __init__(self, pool, url):
PoolError.__init__(self, pool,
"Max retries exceeded with url: %s" % url)
message = "Max retries exceeded with url: %s" % url
PoolError.__init__(self, pool, message)
self.url = url
class HostChangedError(PoolError):
"Raised when an existing pool gets a request for a foreign host."
def __init__(self, pool, url, retries=3):
PoolError.__init__(self, pool,
"Tried to open a foreign host with url: %s" % url)
message = "Tried to open a foreign host with url: %s" % url
PoolError.__init__(self, pool, message)
self.url = url
self.retries = retries
@@ -52,3 +55,13 @@ class TimeoutError(PoolError):
class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
class LocationParseError(ValueError, HTTPError):
"Raised when get_host or similar fails to parse the URL input."
def __init__(self, location):
message = "Failed to parse: %s" % location
super(LocationParseError, self).__init__(self, message)
self.location = location
libs/requests/packages/urllib3/filepost.py
+19 -5
View File
@@ -24,15 +24,29 @@ def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def iter_fields(fields):
"""
Iterate over fields.
Supports list of (k, v) tuples and dicts.
"""
if isinstance(fields, dict):
return ((k, v) for k, v in six.iteritems(fields))
return ((k, v) for k, v in fields)
def encode_multipart_formdata(fields, boundary=None):
"""
Encode a dictionary of ``fields`` using the multipart/form-data mime format.
:param fields:
Dictionary of fields. The key is treated as the field name, and the
value as the body of the form-data. If the value is a tuple of two
elements, then the first element is treated as the filename of the
form-data section.
Dictionary of fields or list of (key, value) field tuples. The key is
treated as the field name, and the value as the body of the form-data
bytes. If the value is a tuple of two elements, then the first element
is treated as the filename of the form-data section.
Field names and filenames must be unicode.
:param boundary:
If not specified, then a random boundary will be generated using
@@ -42,7 +56,7 @@ def encode_multipart_formdata(fields, boundary=None):
if boundary is None:
boundary = choose_boundary()
for fieldname, value in six.iteritems(fields):
for fieldname, value in iter_fields(fields):
body.write(b('--%s\r\n' % (boundary)))
if isinstance(value, tuple):
libs/requests/packages/urllib3/poolmanager.py
+12 -2
View File
@@ -39,11 +39,11 @@ class PoolManager(RequestMethods):
Example: ::
>>> manager = PoolManager()
>>> manager = PoolManager(num_pools=2)
>>> r = manager.urlopen("http://google.com/")
>>> r = manager.urlopen("http://google.com/mail")
>>> r = manager.urlopen("http://yahoo.com/")
>>> len(r.pools)
>>> len(manager.pools)
2
"""
@@ -117,9 +117,19 @@ class ProxyManager(RequestMethods):
def __init__(self, proxy_pool):
self.proxy_pool = proxy_pool
def _set_proxy_headers(self, headers=None):
headers = headers or {}
# Same headers are curl passes for --proxy1.0
headers['Accept'] = '*/*'
headers['Proxy-Connection'] = 'Keep-Alive'
return headers
def urlopen(self, method, url, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
kw['assert_same_host'] = False
kw['headers'] = self._set_proxy_headers(kw.get('headers'))
return self.proxy_pool.urlopen(method, url, **kw)
libs/requests/packages/urllib3/request.py
+1 -20
View File
@@ -44,7 +44,7 @@ class RequestMethods(object):
def urlopen(self, method, url, body=None, headers=None,
encode_multipart=True, multipart_boundary=None,
**kw):
**kw): # Abstract
raise NotImplemented("Classes extending RequestMethods must implement "
"their own ``urlopen`` method.")
@@ -126,22 +126,3 @@ class RequestMethods(object):
return self.urlopen(method, url, body=body, headers=headers,
**urlopen_kw)
# Deprecated:
def get_url(self, url, fields=None, **urlopen_kw):
"""
.. deprecated:: 1.0
Use :meth:`request` instead.
"""
return self.request_encode_url('GET', url, fields=fields,
**urlopen_kw)
def post_url(self, url, fields=None, headers=None, **urlopen_kw):
"""
.. deprecated:: 1.0
Use :meth:`request` instead.
"""
return self.request_encode_body('POST', url, fields=fields,
headers=headers,
**urlopen_kw)
libs/requests/packages/urllib3/response.py
+14 -8
View File
@@ -11,12 +11,7 @@ import zlib
from io import BytesIO
from .exceptions import HTTPError
try:
basestring = basestring
except NameError: # Python 3
basestring = (str, bytes)
from .packages.six import string_types as basestring
log = logging.getLogger(__name__)
@@ -176,11 +171,22 @@ class HTTPResponse(object):
with ``original_response=r``.
"""
# Normalize headers between different versions of Python
headers = {}
for k, v in r.getheaders():
# Python 3: Header keys are returned capitalised
k = k.lower()
has_value = headers.get(k)
if has_value: # Python 3: Repeating header keys are unmerged.
v = ', '.join([has_value, v])
headers[k] = v
# HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, 'strict', 0)
return ResponseCls(body=r,
# In Python 3, the header keys are returned capitalised
headers=dict((k.lower(), v) for k,v in r.getheaders()),
headers=headers,
status=r.status,
version=r.version,
reason=r.reason,
libs/requests/packages/urllib3/util.py
+136
View File
@@ -0,0 +1,136 @@
# urllib3/util.py
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
from base64 import b64encode
try:
from select import poll, POLLIN
except ImportError: # `poll` doesn't exist on OSX and other platforms
poll = False
try:
from select import select
except ImportError: # `select` doesn't exist on AppEngine.
select = False
from .packages import six
from .exceptions import LocationParseError
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
basic_auth=None):
"""
Shortcuts for generating request headers.
:param keep_alive:
If ``True``, adds 'connection: keep-alive' header.
:param accept_encoding:
Can be a boolean, list, or string.
``True`` translates to 'gzip,deflate'.
List will get joined by comma.
String will be used as provided.
:param user_agent:
String representing the user-agent you want, such as
"python-urllib3/0.6"
:param basic_auth:
Colon-separated username:password string for 'authorization: basic ...'
auth header.
Example: ::
>>> make_headers(keep_alive=True, user_agent="Batman/1.0")
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
>>> make_headers(accept_encoding=True)
{'accept-encoding': 'gzip,deflate'}
"""
headers = {}
if accept_encoding:
if isinstance(accept_encoding, str):
pass
elif isinstance(accept_encoding, list):
accept_encoding = ','.join(accept_encoding)
else:
accept_encoding = 'gzip,deflate'
headers['accept-encoding'] = accept_encoding
if user_agent:
headers['user-agent'] = user_agent
if keep_alive:
headers['connection'] = 'keep-alive'
if basic_auth:
headers['authorization'] = 'Basic ' + \
b64encode(six.b(basic_auth)).decode('utf-8')
return headers
def get_host(url):
"""
Given a url, return its scheme, host and port (None if it's not there).
For example: ::
>>> get_host('http://google.com/mail/')
('http', 'google.com', None)
>>> get_host('google.com:80')
('http', 'google.com', 80)
"""
# This code is actually similar to urlparse.urlsplit, but much
# simplified for our needs.
port = None
scheme = 'http'
if '://' in url:
scheme, url = url.split('://', 1)
if '/' in url:
url, _path = url.split('/', 1)
if '@' in url:
_auth, url = url.split('@', 1)
if ':' in url:
url, port = url.split(':', 1)
if not port.isdigit():
raise LocationParseError("Failed to parse: %s" % url)
port = int(port)
return scheme, url, port
def is_connection_dropped(conn):
"""
Returns True if the connection is dropped and should be closed.
:param conn:
``HTTPConnection`` object.
Note: For platforms like AppEngine, this will always return ``False`` to
let the platform handle connection recycling transparently for us.
"""
sock = getattr(conn, 'sock', False)
if not sock: #Platform-specific: AppEngine
return False
if not poll: # Platform-specific
if not select: #Platform-specific: AppEngine
return False
return select([sock], [], [], 0.0)[0]
# This version is better on platforms that support it.
p = poll()
p.register(sock, POLLIN)
for (fno, ev) in p.poll(0.0):
if fno == sock.fileno():
# Either data is buffered (bad), or the connection is dropped.
return True
libs/requests/sessions.py
+38 -21
View File
@@ -40,7 +40,7 @@ def merge_kwargs(local_kwarg, default_kwarg):
kwargs.update(local_kwarg)
# Remove keys that are set to None.
for (k,v) in list(local_kwarg.items()):
for (k, v) in list(local_kwarg.items()):
if v is None:
del kwargs[k]
@@ -52,7 +52,7 @@ class Session(object):
__attrs__ = [
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
'params', 'config']
'params', 'config', 'verify', 'cert']
def __init__(self,
@@ -64,7 +64,9 @@ class Session(object):
hooks=None,
params=None,
config=None,
verify=True):
prefetch=False,
verify=True,
cert=None):
self.headers = headers or {}
self.cookies = cookies or {}
@@ -74,15 +76,14 @@ class Session(object):
self.hooks = hooks or {}
self.params = params or {}
self.config = config or {}
self.prefetch = prefetch
self.verify = verify
self.cert = cert
for (k, v) in list(defaults.items()):
self.config.setdefault(k, v)
self.poolmanager = PoolManager(
num_pools=self.config.get('pool_connections'),
maxsize=self.config.get('pool_maxsize')
)
self.init_poolmanager()
# Set up a CookieJar to be used by default
self.cookies = {}
@@ -91,6 +92,12 @@ class Session(object):
if cookies is not None:
self.cookies.update(cookies)
def init_poolmanager(self):
self.poolmanager = PoolManager(
num_pools=self.config.get('pool_connections'),
maxsize=self.config.get('pool_maxsize')
)
def __repr__(self):
return '<requests-client at 0x%x>' % (id(self))
@@ -108,13 +115,14 @@ class Session(object):
files=None,
auth=None,
timeout=None,
allow_redirects=False,
allow_redirects=True,
proxies=None,
hooks=None,
return_response=True,
config=None,
prefetch=False,
verify=None):
verify=None,
cert=None):
"""Constructs and sends a :class:`Request <Request>`.
Returns :class:`Response <Response>` object.
@@ -128,12 +136,13 @@ class Session(object):
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
:param allow_redirects: (optional) Boolean. Set to True by default.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param return_response: (optional) If False, an un-sent Request object will returned.
:param config: (optional) A configuration dictionary.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
"""
method = str(method).upper()
@@ -145,9 +154,7 @@ class Session(object):
headers = {} if headers is None else headers
params = {} if params is None else params
hooks = {} if hooks is None else hooks
if verify is None:
verify = self.verify
prefetch = self.prefetch or prefetch
# use session's hooks as defaults
for key, cb in list(self.hooks.items()):
@@ -173,6 +180,7 @@ class Session(object):
proxies=proxies,
config=config,
verify=verify,
cert=cert,
_poolmanager=self.poolmanager
)
@@ -210,7 +218,7 @@ class Session(object):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -221,7 +229,7 @@ class Session(object):
"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -232,10 +240,10 @@ class Session(object):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
kwargs.setdefault('allow_redirects', False)
return self.request('head', url, **kwargs)
@@ -244,7 +252,7 @@ class Session(object):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('post', url, data=data, **kwargs)
@@ -255,7 +263,7 @@ class Session(object):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('put', url, data=data, **kwargs)
@@ -266,7 +274,7 @@ class Session(object):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('patch', url, data=data, **kwargs)
@@ -276,11 +284,20 @@ class Session(object):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
:param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('delete', url, **kwargs)
def __getstate__(self):
return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
def __setstate__(self, state):
for attr, value in state.items():
setattr(self, attr, value)
self.init_poolmanager()
def session(**kwargs):
"""Returns a :class:`Session` for context-management."""
libs/requests/utils.py
+106 -42
View File
@@ -15,9 +15,54 @@ import os
import random
import re
import zlib
from netrc import netrc, NetrcParseError
from .compat import parse_http_list as _parse_list_header
from .compat import quote, unquote, cookielib, SimpleCookie, is_py2
from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse
from .compat import basestring, bytes, str
NETRC_FILES = ('.netrc', '_netrc')
def dict_to_sequence(d):
"""Returns an internal sequence dictionary update."""
if hasattr(d, 'items'):
d = d.items()
return d
def get_netrc_auth(url):
"""Returns the Requests tuple auth for a given url from netrc."""
locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES)
netrc_path = None
for loc in locations:
if os.path.exists(loc) and not netrc_path:
netrc_path = loc
# Abort early if there isn't one.
if netrc_path is None:
return netrc_path
ri = urlparse(url)
# Strip port numbers from netloc
host = ri.netloc.split(':')[0]
try:
_netrc = netrc(netrc_path).authenticators(host)
if _netrc:
# Return with login / password
login_i = (0 if _netrc[0] else 1)
return (_netrc[login_i], _netrc[2])
except (NetrcParseError, IOError, AttributeError):
# If there was a parsing error or a permissions issue reading the file,
# we'll just skip netrc auth
pass
def dict_from_string(s):
@@ -25,20 +70,26 @@ def dict_from_string(s):
cookies = dict()
c = SimpleCookie()
c.load(s)
try:
c = SimpleCookie()
c.load(s)
for k,v in list(c.items()):
cookies.update({k: v.value})
for k, v in list(c.items()):
cookies.update({k: v.value})
# This stuff is not to be trusted.
except Exception:
pass
return cookies
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
if name and name[0] != '<' and name[-1] != '>':
return name
# From mitsuhiko/werkzeug (used with permission).
def parse_list_header(value):
"""Parse lists as described by RFC 2068 Section 2.
@@ -145,8 +196,14 @@ def header_expand(headers):
if isinstance(headers, dict):
headers = list(headers.items())
elif isinstance(headers, basestring):
return headers
elif isinstance(headers, str):
# As discussed in https://github.com/kennethreitz/requests/issues/400
# latin-1 is the most conservative encoding used on the web. Anyone
# who needs more can encode to a byte-string before calling
return headers.encode("latin-1")
elif headers is None:
return headers
for i, (value, params) in enumerate(headers):
@@ -164,10 +221,9 @@ def header_expand(headers):
collector.append('; '.join(_params))
if not len(headers) == i+1:
if not len(headers) == i + 1:
collector.append(', ')
# Remove trailing separators.
if collector[-1] in (', ', '; '):
del collector[-1]
@@ -175,7 +231,6 @@ def header_expand(headers):
return ''.join(collector)
def randombytes(n):
"""Return n random bytes."""
if is_py2:
@@ -286,23 +341,6 @@ def get_encoding_from_headers(headers):
return 'ISO-8859-1'
def unicode_from_html(content):
"""Attempts to decode an HTML string into unicode.
If unsuccessful, the original content is returned.
"""
encodings = get_encodings_from_content(content)
for encoding in encodings:
try:
return str(content, encoding)
except (UnicodeError, TypeError):
pass
return content
def stream_decode_response_unicode(iterator, r):
"""Stream decodes a iterator."""
@@ -354,15 +392,6 @@ def get_unicode_from_response(r):
return r.content
def decode_gzip(content):
"""Return gzip-decoded string.
:param content: bytestring to gzip-decode.
"""
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
def stream_decompress(iterator, mode='gzip'):
"""
Stream decodes an iterator over compressed data
@@ -390,18 +419,53 @@ def stream_decompress(iterator, mode='gzip'):
yield chunk
else:
# Make sure everything has been returned from the decompression object
buf = dec.decompress('')
buf = dec.decompress(bytes())
rv = buf + dec.flush()
if rv:
yield rv
def requote_path(path):
"""Re-quote the given URL path component.
def stream_untransfer(gen, resp):
if 'gzip' in resp.headers.get('content-encoding', ''):
gen = stream_decompress(gen, mode='gzip')
elif 'deflate' in resp.headers.get('content-encoding', ''):
gen = stream_decompress(gen, mode='deflate')
This function passes the given path through an unquote/quote cycle to
return gen
# The unreserved URI characters (RFC 3986)
UNRESERVED_SET = frozenset(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+ "0123456789-._~")
def unquote_unreserved(uri):
"""Un-escape any percent-escape sequences in a URI that are unreserved
characters.
This leaves all reserved, illegal and non-ASCII bytes encoded.
"""
parts = uri.split('%')
for i in range(1, len(parts)):
h = parts[i][0:2]
if len(h) == 2:
c = chr(int(h, 16))
if c in UNRESERVED_SET:
parts[i] = c + parts[i][2:]
else:
parts[i] = '%' + parts[i]
else:
parts[i] = '%' + parts[i]
return ''.join(parts)
def requote_uri(uri):
"""Re-quote the given URI.
This function passes the given URI through an unquote/quote cycle to
ensure that it is fully and consistently quoted.
"""
parts = path.split(b"/")
parts = (quote(unquote(part), safe=b"") for part in parts)
return b"/".join(parts)
# Unquote only the unreserved characters
# Then quote only illegal characters (do not quote reserved, unreserved,
# or '%')
return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
libs/subliminal/__init__.py
+24 -13
View File
@@ -1,23 +1,34 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Subliminal']
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .api import list_subtitles, download_subtitles
from .async import Pool
from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
MATCHING_CONFIDENCE)
from .infos import __version__
import logging
try:
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
def emit(self, record):
pass
from core import Subliminal
__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE',
'MATCHING_CONFIDENCE', 'list_subtitles', 'download_subtitles', 'Pool']
logging.getLogger(__name__).addHandler(NullHandler())
libs/subliminal/api.py
Executable
+100
View File
@@ -0,0 +1,100 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
group_by_video, key_subtitles)
from .languages import list_languages
import logging
__all__ = ['list_subtitles', 'download_subtitles']
logger = logging.getLogger(__name__)
def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
"""List subtitles in given paths according to the criteria
:param paths: path(s) to video file or folder
:type paths: string or list
:param list languages: languages to search for, in preferred order
:param list services: services to use for the search, in preferred order
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
:param string cache_dir: path to the cache directory to use
:param int max_depth: maximum depth for scanning entries
:return: found subtitles
:rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
"""
services = services or SERVICES
languages = set(languages or list_languages(1))
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
logger.warning(u'Not all entries are unicode')
results = []
service_instances = {}
tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
for task in tasks:
try:
result = consume_task(task, service_instances)
results.append((task.video, result))
except:
logger.error(u'Error consuming task %r' % task, exc_info=True)
for service_instance in service_instances.itervalues():
service_instance.terminate()
return group_by_video(results)
def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, order=None):
"""Download subtitles in given paths according to the criteria
:param paths: path(s) to video file or folder
:type paths: string or list
:param list languages: languages to search for, in preferred order
:param list services: services to use for the search, in preferred order
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
:param string cache_dir: path to the cache directory to use
:param int max_depth: maximum depth for scanning entries
:param order: preferred order for subtitles sorting
:type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
:return: found subtitles
:rtype: list of (:class:`~subliminal.videos.Video`, [:class:`~subliminal.subtitles.ResultSubtitle`])
"""
services = services or SERVICES
languages = languages or list_languages(1)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
results = []
service_instances = {}
tasks = create_download_tasks(subtitles_by_video, multi)
for task in tasks:
try:
result = consume_task(task, service_instances)
results.append(result)
except:
logger.error(u'Error consuming task %r' % task, exc_info=True)
for service_instance in service_instances.itervalues():
service_instance.terminate()
return results
libs/subliminal/async.py
Executable
+141
View File
@@ -0,0 +1,141 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
create_download_tasks, group_by_video, key_subtitles)
from .languages import list_languages
from .tasks import StopTask
import Queue
import logging
import threading
logger = logging.getLogger(__name__)
class Worker(threading.Thread):
"""Consume tasks and put the result in the queue"""
def __init__(self, tasks, results):
super(Worker, self).__init__()
self.tasks = tasks
self.results = results
self.services = {}
def run(self):
while 1:
result = []
try:
task = self.tasks.get(block=True)
if isinstance(task, StopTask):
break
result = consume_task(task, self.services)
self.results.put((task.video, result))
except:
logger.error(u'Exception raised in worker %s' % self.name, exc_info=True)
finally:
self.tasks.task_done()
self.terminate()
logger.debug(u'Thread %s terminated' % self.name)
def terminate(self):
"""Terminate instantiated services"""
for service_name, service in self.services.iteritems():
try:
service.terminate()
except:
logger.error(u'Exception raised when terminating service %s' % service_name, exc_info=True)
class Pool(object):
"""Pool of workers"""
def __init__(self, size):
self.tasks = Queue.Queue()
self.results = Queue.Queue()
self.workers = []
for _ in range(size):
self.workers.append(Worker(self.tasks, self.results))
def __enter__(self):
self.start()
return self
def __exit__(self, *args):
self.stop()
self.join()
def start(self):
"""Start workers"""
for worker in self.workers:
worker.start()
def stop(self):
"""Stop workers"""
for _ in self.workers:
self.tasks.put(StopTask())
def join(self):
"""Join the task queue"""
self.tasks.join()
def collect(self):
"""Collect available results
:return: results of tasks
:rtype: list of :class:`~subliminal.tasks.Task`
"""
results = []
while 1:
try:
result = self.results.get(block=False)
results.append(result)
except Queue.Empty:
break
return results
def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
"""See :meth:`subliminal.list_subtitles`"""
services = services or SERVICES
languages = set(languages or list_languages(1))
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
logger.warning(u'Not all entries are unicode')
tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
for task in tasks:
self.tasks.put(task)
self.join()
results = self.collect()
return group_by_video(results)
def download_subtitles(self, paths, languages=None, services=None, cache_dir=None, max_depth=3, force=True, multi=False, order=None):
"""See :meth:`subliminal.download_subtitles`"""
services = services or SERVICES
languages = languages or list_languages(1)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
tasks = create_download_tasks(subtitles_by_video, multi)
for task in tasks:
self.tasks.put(task)
self.join()
results = self.collect()
return results
libs/subliminal/core.py
+187 -313
View File
@@ -1,346 +1,173 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['PLUGINS', 'API_PLUGINS', 'IDLE', 'RUNNING', 'PAUSED', 'Subliminal', 'PluginWorker', 'matching_confidence',
'LANGUAGE_INDEX', 'PLUGIN_INDEX', 'PLUGIN_CONFIDENCE', 'MATCHING_CONFIDENCE']
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .exceptions import DownloadFailedError
from .services import ServiceConfig
from .tasks import DownloadTask, ListTask
from .utils import get_keywords
from .videos import Episode, Movie, scan
from collections import defaultdict
from exceptions import InvalidLanguageError, PluginError, BadStateError, \
WrongTaskError, DownloadFailedError
from itertools import groupby
from languages import list_languages
from utils import NullHandler
from tasks import Task, DownloadTask, ListTask, StopTask
import Queue
import guessit
import logging
import os
import plugins
import subtitles
import threading
import utils
import videos
# init logger
logger = logging.getLogger('subliminal')
logger.addHandler(NullHandler())
# const
PLUGINS = ['OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos']
API_PLUGINS = filter(lambda p: getattr(plugins, p).api_based, PLUGINS)
IDLE, RUNNING, PAUSED = range(3)
LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
class Subliminal(object):
"""Main Subliminal class"""
__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE', 'MATCHING_CONFIDENCE',
'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence',
'key_subtitles', 'group_by_video']
logger = logging.getLogger(__name__)
SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb']
LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
def __init__(self, cache_dir=None, workers=None, multi=False, force=False,
max_depth=None, filemode=None, sort_order=None, plugins=None, languages=None):
self.multi = multi
self.sort_order = sort_order or [LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE]
self.force = force
self.max_depth = max_depth or 3
self.taskQueue = Queue.PriorityQueue()
self.listResultQueue = Queue.Queue()
self.downloadResultQueue = Queue.Queue()
self.languages = languages or []
self.plugins = plugins or API_PLUGINS
self._workers = workers or 4
self.filemode = filemode
self.state = IDLE
self.cache_dir = cache_dir
try:
if cache_dir and not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
logger.debug(u'Creating cache directory: %r' % cache_dir)
except:
self.cache_dir = None
logger.error(u'Failed to use the cache directory, continue without it')
def __enter__(self):
self.startWorkers()
return self
def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth):
"""Create a list of :class:`~subliminal.tasks.ListTask` from one or more paths using the given criteria
def __exit__(self, *args):
self.stopWorkers(0)
:param paths: path(s) to video file or folder
:type paths: string or list
:param set languages: languages to search for
:param list services: services to use for the search
:param bool force: force searching for subtitles even if some are detected
:param bool multi: search multiple languages for the same video
:param string cache_dir: path to the cache directory to use
:param int max_depth: maximum depth for scanning entries
:return: the created tasks
:rtype: list of :class:`~subliminal.tasks.ListTask`
@property
def workers(self):
return self._workers
@workers.setter
def workers(self, value):
if self.state == RUNNING:
raise BadStateError(self.state, IDLE)
self._workers = value
@property
def languages(self):
"""Getter for languages"""
return self._languages
@languages.setter
def languages(self, languages):
"""Setter for languages"""
logger.debug(u'Setting languages to %r' % languages)
self._languages = []
for l in languages:
if l not in list_languages(1):
raise InvalidLanguageError(l)
if not l in self._languages:
self._languages.append(l)
@property
def plugins(self):
"""Getter for plugins"""
return self._plugins
@plugins.setter
def plugins(self, plugins):
"""Setter for plugins"""
logger.debug(u'Setting plugins to %r' % plugins)
self._plugins = []
for p in plugins:
if p not in PLUGINS:
raise PluginError(p)
if not p in self._plugins:
self._plugins.append(p)
def listSubtitles(self, entries, auto=False):
"""
Search subtitles within the plugins and return all found subtitles in a list of Subtitle object.
Attributes:
entries -- filepath or folderpath of video file or a list of that
auto -- automaticaly manage workers (default to False)"""
if auto:
if self.state != IDLE:
raise BadStateError(self.state, IDLE)
self.startWorkers()
if isinstance(entries, basestring):
entries = [entries]
config = utils.PluginConfig(self.multi, self.cache_dir, self.filemode)
scan_result = []
for e in entries:
if not isinstance(e, unicode):
logger.warning(u'Entry %r is not unicode' % e)
scan_result.extend(videos.scan(e))
task_count = 0
for video, subtitles in scan_result:
languages = set([s.language for s in subtitles if s.language])
wanted_languages = set(self._languages)
"""
scan_result = []
for p in paths:
scan_result.extend(scan(p, max_depth))
logger.debug(u'Found %d videos in %r with maximum depth %d' % (len(scan_result), paths, max_depth))
tasks = []
config = ServiceConfig(multi, cache_dir)
for video, detected_subtitles in scan_result:
detected_languages = set([s.language for s in detected_subtitles])
wanted_languages = languages.copy()
if not force and multi:
wanted_languages -= detected_languages
if not wanted_languages:
wanted_languages = list_languages(1)
if not self.force and self.multi:
wanted_languages = set(wanted_languages) - languages
if not wanted_languages:
logger.debug(u'No need to list multi subtitles %r for %r because %r subtitles detected' % (self._languages, video.path, languages))
continue
if not self.force and not self.multi and None in [s.language for s in subtitles]:
logger.debug(u'No need to list single subtitles %r for %r because one detected' % (self._languages, video.path))
logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages))
continue
logger.debug(u'Listing subtitles %r for %r with %r' % (wanted_languages, video.path, self._plugins))
for plugin_name in self._plugins:
plugin = getattr(plugins, plugin_name)
to_list_languages = wanted_languages & plugin.availableLanguages()
if not to_list_languages:
logger.debug(u'Skipping %r: none of wanted languages %r available in %r for plugin %s' % (video.path, wanted_languages, plugin.availableLanguages(), plugin_name))
continue
if not plugin.isValidVideo(video):
logger.debug(u'Skipping %r: video %r is not part of supported videos %r for plugin %s' % (video.path, video, plugin.videos, plugin_name))
continue
self.taskQueue.put((5, ListTask(video, to_list_languages, plugin_name, config)))
task_count += 1
subtitles = []
for _ in range(task_count):
subtitles.extend(self.listResultQueue.get())
if auto:
self.stopWorkers()
return subtitles
def downloadSubtitles(self, entries, auto=False):
"""
Download subtitles using the plugins preferences and languages. Also use internal algorithm to find
the best match inside a plugin.
Attributes:
entries -- filepath or folderpath of video file or a list of that
auto -- automaticaly manage workers (default to False)"""
if auto:
if self.state != IDLE:
raise BadStateError(self.state, IDLE)
self.startWorkers()
by_video = self.groupByVideo(self.listSubtitles(entries, False))
# Define an order with LANGUAGE_INDEX first for multi sorting
order = self.sort_order
if self.multi:
order.insert(0, LANGUAGE_INDEX)
task_count = 0
for video, subtitles in by_video.iteritems():
ordered_subtitles = sorted(subtitles, key=lambda s: self.keySubtitles(s, video, order), reverse=True)
if not self.multi:
self.taskQueue.put((5, DownloadTask(video, list(ordered_subtitles))))
task_count += 1
if not force and not multi and None in detected_languages:
logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video))
continue
logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services))
for service_name in services:
mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
service = mod.Service
service_languages = wanted_languages & service.available_languages()
if not service_languages:
logger.debug(u'Skipping %r: none of wanted languages %r available for service %s' % (video, wanted_languages, service_name))
continue
for _, by_language in groupby(ordered_subtitles, lambda s: s.language):
self.taskQueue.put((5, DownloadTask(video, list(by_language))))
task_count += 1
downloaded = []
for _ in range(task_count):
downloaded.extend(self.downloadResultQueue.get())
if auto:
self.stopWorkers()
return downloaded
def keySubtitles(self, subtitle, video, order):
"""Create a key to sort subtitle using preferences"""
key = ''
for sort_item in order:
if sort_item == LANGUAGE_INDEX:
key += '{0:03d}'.format(len(self._languages) - self._languages.index(subtitle.language) - 1)
elif sort_item == PLUGIN_INDEX:
key += '{0:02d}'.format(len(self._plugins) - self._plugins.index(subtitle.plugin) - 1)
elif sort_item == PLUGIN_CONFIDENCE:
key += '{0:04d}'.format(int(subtitle.confidence * 1000))
elif sort_item == MATCHING_CONFIDENCE:
confidence = 0
if subtitle.release:
confidence = matching_confidence(video, subtitle)
key += '{0:04d}'.format(int(confidence * 1000))
return int(key)
def groupByVideo(self, list_result):
'''Because list outputs a list of tuples from different plugins, we need to put them back
together under a single video key'''
result = defaultdict(list)
for video, subtitles in list_result:
result[video] += subtitles
return result
def startWorkers(self):
"""Create a pool of workers and start them"""
if self.state == RUNNING:
raise BadStateError(self.state, IDLE)
self.pool = []
for _ in range(self._workers):
worker = PluginWorker(self.taskQueue, self.listResultQueue, self.downloadResultQueue)
worker.start()
self.pool.append(worker)
logger.debug(u'Worker %s added to the pool' % worker.name)
self.state = RUNNING
def stopWorkers(self, priority=10):
"""Stop workers using a lowest priority stop signal and wait for them to terminate properly"""
for _ in range(self._workers):
self.taskQueue.put((priority, StopTask()))
for worker in self.pool:
worker.join()
self.state = IDLE
if not self.taskQueue.empty():
self.state = PAUSED
def pauseWorkers(self):
"""Pause workers using a highest priority stop signal and wait for them to terminate properly"""
self.stopWorkers(0)
def addTask(self, task):
"""Add a task with default priority"""
if not isinstance(task, Task) or isinstance(task, StopTask):
raise WrongTaskError()
self.taskQueue.put((5, task))
if not service.is_valid_video(video):
logger.debug(u'Skipping %r: not part of supported videos %r for service %s' % (video, service.videos, service_name))
continue
task = ListTask(video, service_languages, service_name, config)
logger.debug(u'Created task %r' % task)
tasks.append(task)
return tasks
class PluginWorker(threading.Thread):
"""Threaded plugin worker"""
def __init__(self, taskQueue, listResultQueue, downloadResultQueue):
threading.Thread.__init__(self)
self.taskQueue = taskQueue
self.listResultQueue = listResultQueue
self.downloadResultQueue = downloadResultQueue
self.logger = logging.getLogger('subliminal.worker')
self.plugins = {}
def create_download_tasks(subtitles_by_video, multi):
"""Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
def run(self):
while True:
task = self.taskQueue.get()[1]
if isinstance(task, StopTask):
self.logger.debug(u'Poison pill received in thread %s' % self.name)
self.taskQueue.task_done()
:param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted
:type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
:param order: preferred order for subtitles sorting
:type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
:param bool multi: download multiple languages for the same video
:return: the created tasks
:rtype: list of :class:`~subliminal.tasks.DownloadTask`
"""
tasks = []
for video, subtitles in subtitles_by_video.iteritems():
if not subtitles:
continue
if not multi:
task = DownloadTask(video, list(subtitles))
logger.debug(u'Created task %r' % task)
tasks.append(task)
continue
for _, by_language in groupby(subtitles, lambda s: s.language):
task = DownloadTask(video, list(by_language))
logger.debug(u'Created task %r' % task)
tasks.append(task)
return tasks
def consume_task(task, services=None):
"""Consume a task. If the ``services`` parameter is given, the function will attempt
to get the service from it. In case the service is not in ``services``, it will be initialized
and put in ``services``
:param task: task to consume
:type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask`
:param dict services: mapping between the service name and an instance of this service
:return: the result of the task
:rtype: list of :class:`~subliminal.subtitles.ResultSubtitle` or :class:`~subliminal.subtitles.Subtitle`
"""
if services is None:
services = {}
logger.info(u'Consuming %r' % task)
result = None
if isinstance(task, ListTask):
if task.service not in services:
mod = __import__('services.' + task.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
services[task.service] = mod.Service(task.config)
services[task.service].init()
subtitles = services[task.service].list(task.video, task.languages)
result = subtitles
elif isinstance(task, DownloadTask):
for subtitle in task.subtitles:
if subtitle.service not in services:
mod = __import__('services.' + subtitle.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
services[subtitle.service] = mod.Service()
services[subtitle.service].init()
try:
services[subtitle.service].download(subtitle)
result = subtitle
break
result = []
try:
if isinstance(task, ListTask):
if task.plugin not in self.plugins: # init the plugin
self.plugins[task.plugin] = getattr(plugins, task.plugin)()
self.plugins[task.plugin].init()
# Retrieve the plugin list subtitles and return [(video, [subtitle])]
plugin = self.plugins[task.plugin]
plugin.config = task.config
subtitles = plugin.list(task.video, task.languages)
result = [(task.video, subtitles)]
elif isinstance(task, DownloadTask):
# Attempt to download one subtitle from the given list
for subtitle in task.subtitles:
if subtitle.plugin not in self.plugins: # init the plugin
self.plugins[subtitle.plugin] = getattr(plugins, subtitle.plugin)()
self.plugins[subtitle.plugin].init()
plugin = self.plugins[subtitle.plugin]
try:
result = [plugin.download(subtitle)]
break
except DownloadFailedError: # try the next one
self.logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
continue
if not result:
self.logger.error(u'No subtitles could be downloaded for video %r' % task.video.path or task.video.release)
except:
self.logger.error(u'Exception raised in worker %s' % self.name, exc_info=True)
finally:
# Put the result in the correct queue
if isinstance(task, ListTask):
self.listResultQueue.put(result)
elif isinstance(task, DownloadTask):
self.downloadResultQueue.put(result)
self.taskQueue.task_done()
self.terminate()
self.logger.debug(u'Thread %s terminated' % self.name)
def terminate(self):
"""Terminate instanciated plugins"""
for plugin_name, plugin in self.plugins.iteritems():
try:
plugin.terminate()
except:
self.logger.error(u'Exception raised when terminating plugin %s' % plugin_name, exc_info=True)
except DownloadFailedError:
logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
continue
if result is None:
logger.error(u'No subtitles could be downloaded for video %r' % task.video)
return result
def matching_confidence(video, subtitle):
'''Compute the confidence that the subtitle matches the video.
Returns a float between 0 and 1. 1 being the perfect match.'''
"""Compute the probability (confidence) that the subtitle matches the video
:param video: video to match
:type video: :class:`~subliminal.videos.Video`
:param subtitle: subtitle to match
:type subtitle: :class:`~subliminal.subtitles.Subtitle`
:return: the matching probability
:rtype: float
"""
guess = guessit.guess_file_info(subtitle.release, 'autodetect')
video_keywords = utils.get_keywords(video.guess)
subtitle_keywords = utils.get_keywords(guess) | subtitle.keywords
video_keywords = get_keywords(video.guess)
subtitle_keywords = get_keywords(guess) | subtitle.keywords
replacement = {'keywords': len(video_keywords & subtitle_keywords)}
if isinstance(video, videos.Episode):
if isinstance(video, Episode):
replacement.update({'series': 0, 'season': 0, 'episode': 0})
matching_format = '{series:b}{season:b}{episode:b}{keywords:03b}'
best = matching_format.format(series=1, season=1, episode=1, keywords=len(video_keywords))
@@ -351,7 +178,7 @@ def matching_confidence(video, subtitle):
replacement['season'] = 1
if 'episodeNumber' in guess and guess['episodeNumber'] == video.episode:
replacement['episode'] = 1
elif isinstance(video, videos.Movie):
elif isinstance(video, Movie):
replacement.update({'title': 0, 'year': 0})
matching_format = '{title:b}{year:b}{keywords:03b}'
best = matching_format.format(title=1, year=1, keywords=len(video_keywords))
@@ -364,3 +191,50 @@ def matching_confidence(video, subtitle):
return 0
confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
return confidence
def key_subtitles(subtitle, video, languages, services, order):
"""Create a key to sort subtitle using the given order
:param subtitle: subtitle to sort
:type subtitle: :class:`~subliminal.subtitles.ResultSubtitle`
:param video: video to match
:type video: :class:`~subliminal.videos.Video`
:param list languages: languages in preferred order
:param list services: services in preferred order
:param order: preferred order for subtitles sorting
:type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
:return: a key ready to use for subtitles sorting
:rtype: int
"""
key = ''
for sort_item in order:
if sort_item == LANGUAGE_INDEX:
key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1)
elif sort_item == SERVICE_INDEX:
key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1)
elif sort_item == SERVICE_CONFIDENCE:
key += '{0:04d}'.format(int(subtitle.confidence * 1000))
elif sort_item == MATCHING_CONFIDENCE:
confidence = 0
if subtitle.release:
confidence = matching_confidence(video, subtitle)
key += '{0:04d}'.format(int(confidence * 1000))
return int(key)
def group_by_video(list_results):
"""Group the results of :class:`ListTasks <subliminal.tasks.ListTask>` into a
dictionary of :class:`~subliminal.videos.Video` => :class:`~subliminal.subtitles.Subtitle`
:param list_results:
:type list_results: list of result of :class:`~subliminal.tasks.ListTask`
:return: subtitles grouped by videos
:rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
"""
result = defaultdict(list)
for video, subtitles in list_results:
result[video] += subtitles
return result
libs/subliminal/exceptions.py
+18 -36
View File
@@ -1,23 +1,20 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
class Error(Exception):
@@ -25,21 +22,6 @@ class Error(Exception):
pass
class BadStateError(Error):
"""Exception raised when an invalid action is asked
Attributes:
current -- current state of Subliminal instance
expected -- expected state of Subliminal instance
"""
def __init__(self, current, expected):
self.current = current
self.expected = expected
def __str__(self):
return 'Expected state %d but current state is %d' % (self.expected, self.current)
class InvalidLanguageError(Error):
"""Exception raised when invalid language is submitted
@@ -66,21 +48,21 @@ class MissingLanguageError(Error):
return self.language
class InvalidPluginError(Error):
""""Exception raised when invalid plugin is submitted
class InvalidServiceError(Error):
"""Exception raised when invalid service is submitted
:param string service: service that causes the error
Attributes:
plugin -- plugin that cause the error
"""
def __init__(self, plugin):
self.plugin = plugin
def __init__(self, service):
self.service = service
def __str__(self):
return self.plugin
return self.service
class PluginError(Error):
""""Exception raised by plugins"""
class ServiceError(Error):
""""Exception raised by services"""
pass
@@ -90,7 +72,7 @@ class WrongTaskError(Error):
class DownloadFailedError(Error):
""""Exception raised when a download task has failed in plugin"""
""""Exception raised when a download task has failed in service"""
pass
libs/subliminal/infos.py
+9 -17
View File
@@ -1,26 +1,18 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__title__ = 'subliminal'
__description__ = 'Subtitles, faster than your thoughts'
__version__ = '0.5'
__author__ = 'Antoine Bertin'
__license__ = 'LGPLv3'
__copyright__ = 'Copyright 2010-2011 Antoine Bertin'
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
__version__ = '0.5.1'
libs/subliminal/languages.py
+27 -15
View File
@@ -1,29 +1,34 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['convert_language', 'list_languages', 'LANGUAGES']
def convert_language(language, to_iso, from_iso=None):
# if no from_iso is given, try to guess it
if from_iso == None:
"""Convert a language into another format
:param string language: language
:param int to_iso: convert language to ISO-639-x
:param int from_iso: convert language from ISO-639-x
:return: converted language
:rtype: string
"""
if from_iso == None: # if no from_iso is given, try to guess it
if language.startswith(language[:1].upper()):
from_iso = 0
elif len(language) == 2:
@@ -43,10 +48,17 @@ def convert_language(language, to_iso, from_iso=None):
def list_languages(iso):
"""List languages in the given ISO-639-x format
:param int iso: ISO-639-x format to list
:return: languages in the requested format
:rtype: list
"""
return [l[iso] for l in LANGUAGES if l[iso]]
# ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
# + ('Brazilian', 'po', 'pob')
#: ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
#: + ('Brazilian', 'po', 'pob')
LANGUAGES = [('Afar', 'aa', 'aar'),
('Abkhazian', 'ab', 'abk'),
('Achinese', '', 'ace'),
libs/subliminal/plugins.py
-890
View File
@@ -1,890 +0,0 @@
# -*- coding: utf-8 -*-
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['PluginBase', 'OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos']
from exceptions import DownloadFailedError, MissingLanguageError, PluginError
from utils import get_keywords, PluginConfig, split_keyword
from videos import Episode, Movie, UnknownVideo
from subtitles import ResultSubtitle, get_subtitle_path
import BeautifulSoup
import abc
import gzip
import logging
import os
import re
import requests
import suds.client
import threading
import unicodedata
import urllib
import xmlrpclib
try:
import cPickle as pickle
except ImportError:
import pickle
#TODO: use ISO-639-2 in plugins instead of ISO-639-1
class PluginBase(object):
__metaclass__ = abc.ABCMeta
site_url = ''
site_name = ''
server_url = ''
user_agent = 'Subliminal v0.5'
api_based = False
timeout = 5
lock = threading.Lock()
languages = {}
reverted_languages = False
videos = []
require_video = False
shared_support = False
@abc.abstractmethod
def __init__(self, config=None):
self.config = config or PluginConfig()
self.logger = logging.getLogger('subliminal.%s' % self.__class__.__name__)
@abc.abstractmethod
def init(self):
"""Initiate connection"""
self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent})
@abc.abstractmethod
def terminate(self):
"""Terminate connection"""
@abc.abstractmethod
def query(self, *args):
"""Make the actual query"""
@abc.abstractmethod
def list(self, video, languages):
"""List subtitles"""
@abc.abstractmethod
def download(self, subtitle):
"""Download a subtitle"""
@classmethod
def availableLanguages(cls):
if not cls.reverted_languages:
return set(cls.languages.keys())
if cls.reverted_languages:
return set(cls.languages.values())
@classmethod
def isValidVideo(cls, video):
if cls.require_video and not video.exists:
return False
if not isinstance(video, tuple(cls.videos)):
return False
return True
@classmethod
def isValidLanguage(cls, language):
if language in cls.availableLanguages():
return True
return False
@classmethod
def getRevertLanguage(cls, language):
"""ISO-639-1 language code from plugin language code"""
if not cls.reverted_languages and language in cls.languages.values():
return [k for k, v in cls.languages.iteritems() if v == language][0]
if cls.reverted_languages and language in cls.languages.keys():
return cls.languages[language]
raise MissingLanguageError(language)
@classmethod
def getLanguage(cls, language):
"""Plugin language code from ISO-639-1 language code"""
if not cls.reverted_languages and language in cls.languages.keys():
return cls.languages[language]
if cls.reverted_languages and language in cls.languages.values():
return [k for k, v in cls.languages.iteritems() if v == language][0]
raise MissingLanguageError(language)
def adjustPermissions(self, filepath):
if self.config.filemode != None:
os.chmod(filepath, self.config.filemode)
def downloadFile(self, url, filepath):
"""Download a subtitle file"""
self.logger.info(u'Downloading %s' % url)
try:
r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
with open(filepath, 'wb') as f:
f.write(r.content)
except Exception as e:
self.logger.error(u'Download %s failed: %s' % (url, e))
if os.path.exists(filepath):
os.remove(filepath)
raise DownloadFailedError(str(e))
self.logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
class OpenSubtitles(PluginBase):
site_url = 'http://www.opensubtitles.org'
site_name = 'OpenSubtitles'
server_url = 'http://api.opensubtitles.org/xml-rpc'
user_agent = 'Subliminal v0.5'
api_based = True
languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara',
'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze',
'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis',
'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che',
'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze',
'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful',
'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin',
'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii',
'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav',
'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm',
'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit',
'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao',
'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau',
'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob',
'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss',
'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que',
'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin',
'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe',
'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur',
'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie',
'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor',
'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'}
reverted_languages = False
videos = [Episode, Movie]
require_video = False
confidence_order = ['moviehash', 'imdbid', 'fulltext']
def __init__(self, config=None):
super(OpenSubtitles, self).__init__(config)
self.server = xmlrpclib.ServerProxy(self.server_url)
self.token = None
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
self.terminate()
def init(self):
self.logger.debug(u'Initializing')
super(OpenSubtitles, self).init()
result = self.server.LogIn('', '', 'eng', self.user_agent)
if result['status'] != '200 OK':
raise PluginError('Login failed')
self.token = result['token']
def terminate(self):
self.logger.debug(u'Terminating')
if self.token:
self.server.LogOut(self.token)
def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None):
searches = []
if moviehash and size:
searches.append({'moviehash': moviehash, 'moviebytesize': size})
if imdbid:
searches.append({'imdbid': imdbid})
if query:
searches.append({'query': query})
if not searches:
raise PluginError('One or more parameter missing')
for search in searches:
search['sublanguageid'] = ','.join([self.getLanguage(l) for l in languages])
self.logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
results = self.server.SearchSubtitles(self.token, searches)
if not results['data']:
self.logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token))
return []
subtitles = []
for result in results['data']:
language = self.getRevertLanguage(result['SubLanguageID'])
path = get_subtitle_path(filepath, language, self.config.multi)
confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
subtitle = ResultSubtitle(path, language, self.__class__.__name__, result['SubDownloadLink'], result['SubFileName'], confidence)
subtitles.append(subtitle)
return subtitles
def list(self, video, languages):
languages = languages & self.availableLanguages()
if not languages:
self.logger.debug(u'No language available')
return []
if not self.isValidVideo(video):
self.logger.debug(u'Not a valid video')
return []
results = []
if video.exists:
results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))
elif video.imdbid:
results = self.query(video.path or video.release, languages, imdbid=video.imdbid)
elif isinstance(video, Episode):
results = self.query(video.path or video.release, languages, query=video.series)
elif isinstance(video, Movie):
results = self.query(video.path or video.release, languages, query=video.title)
return results
def download(self, subtitle):
#TODO: Use OpenSubtitles DownloadSubtitles method
try:
self.downloadFile(subtitle.link, subtitle.path + '.gz')
with open(subtitle.path, 'wb') as dump:
gz = gzip.open(subtitle.path + '.gz')
dump.write(gz.read())
gz.close()
self.adjustPermissions(subtitle.path)
except Exception as e:
if os.path.exists(subtitle.path):
os.remove(subtitle.path)
raise DownloadFailedError(str(e))
finally:
if os.path.exists(subtitle.path + '.gz'):
os.remove(subtitle.path + '.gz')
return subtitle
class BierDopje(PluginBase):
site_url = 'http://bierdopje.com'
site_name = 'BierDopje'
server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
api_based = True
languages = {'en': 'en', 'nl': 'nl'}
reverted_languages = False
videos = [Episode]
require_video = False
def __init__(self, config=None):
super(BierDopje, self).__init__(config)
self.showids = {}
if self.config and self.config.cache_dir:
self.initCache()
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
self.terminate()
def init(self):
self.logger.debug(u'Initializing')
super(BierDopje, self).init()
def terminate(self):
self.logger.debug(u'Terminating')
def initCache(self):
self.logger.debug(u'Initializing cache...')
if not self.config or not self.config.cache_dir:
raise PluginError('Cache directory is required')
self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache')
if not os.path.exists(self.showids_cache):
self.saveToCache()
def saveToCache(self):
self.logger.debug(u'Saving showids to cache...')
with self.lock:
with open(self.showids_cache, 'w') as f:
pickle.dump(self.showids, f)
def loadFromCache(self):
self.logger.debug(u'Loading showids from cache...')
with self.lock:
with open(self.showids_cache, 'r') as f:
self.showids = pickle.load(f)
def query(self, season, episode, languages, filepath, tvdbid=None, series=None):
self.initCache()
self.loadFromCache()
if series:
if series.lower() in self.showids: # from cache
request_id = self.showids[series.lower()]
self.logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series))
else: # query to get showid
self.logger.debug(u'Getting showid from show name %s...' % series)
r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
if r.status_code != 200:
self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulStoneSoup(r.content)
if soup.status.contents[0] == 'false':
self.logger.debug(u'Could not find show %s' % series)
return []
request_id = int(soup.showid.contents[0])
self.showids[series.lower()] = request_id
self.saveToCache()
request_source = 'showid'
request_is_tvdbid = 'false'
elif tvdbid:
request_id = tvdbid
request_source = 'tvdbid'
request_is_tvdbid = 'true'
else:
raise PluginError('One or more parameter missing')
subtitles = []
for language in languages:
self.logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid))
if r.status_code != 200:
self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulStoneSoup(r.content)
if soup.status.contents[0] == 'false':
self.logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
continue
path = get_subtitle_path(filepath, language, self.config.multi)
for result in soup.results('result'):
subtitle = ResultSubtitle(path, language, self.__class__.__name__, result.downloadlink.contents[0], result.filename.contents[0])
subtitles.append(subtitle)
return subtitles
def list(self, video, languages):
languages = languages & self.availableLanguages()
if not languages:
self.logger.debug(u'No language available')
return []
if not self.isValidVideo(video):
self.logger.debug(u'Not a valid video')
return []
results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series)
return results
def download(self, subtitle):
self.downloadFile(subtitle.link, subtitle.path)
return subtitle
class TheSubDB(PluginBase):
site_url = 'http://thesubdb.com'
site_name = 'SubDB'
server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
api_based = True
user_agent = 'SubDB/1.0 (Subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API
languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages
reverted_languages = False
videos = [Movie, Episode, UnknownVideo]
require_video = True
def __init__(self, config=None):
super(TheSubDB, self).__init__(config)
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
self.terminate()
def init(self):
self.logger.debug(u'Initializing')
super(TheSubDB, self).init()
def terminate(self):
self.logger.debug(u'Terminating')
def list(self, video, languages):
languages = languages & self.availableLanguages()
if not languages:
self.logger.debug(u'No language available')
return []
if not self.isValidVideo(video):
self.logger.debug(u'Not a valid video')
return []
results = self.query(video.path, video.hashes['TheSubDB'], languages)
return results
def query(self, filepath, moviehash, languages):
r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
if r.status_code == 404:
self.logger.debug(u'Could not find subtitles for hash %s' % moviehash)
return []
if r.status_code != 200:
self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
available_languages = set([self.getRevertLanguage(l) for l in r.content.split(',')])
filtered_languages = languages & available_languages
if not filtered_languages:
self.logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
return []
subtitles = []
for language in filtered_languages:
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.getLanguage(language)))
subtitles.append(subtitle)
return subtitles
def download(self, subtitle):
self.downloadFile(subtitle.link, subtitle.path)
return subtitle
class SubsWiki(PluginBase):
site_url = 'http://www.subswiki.com'
site_name = 'SubsWiki'
server_url = 'http://www.subswiki.com'
api_based = False
languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
u'Italian': 'it', u'Català': 'ca'}
reverted_languages = True
videos = [Episode, Movie]
require_video = False
release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
def __init__(self, config=None):
super(SubsWiki, self).__init__(config)
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
self.terminate()
def init(self):
self.logger.debug(u'Initializing')
super(SubsWiki, self).init()
def terminate(self):
self.logger.debug(u'Terminating')
def list(self, video, languages):
languages = languages & self.availableLanguages()
if not languages:
self.logger.debug(u'No language available')
return []
if not self.isValidVideo(video):
self.logger.debug(u'Not a valid video')
return []
results = []
if isinstance(video, Episode):
results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
elif isinstance(video, Movie) and video.year:
results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year)
return results
def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None):
if series and season and episode:
request_series = series.lower().replace(' ', '_')
if isinstance(request_series, unicode):
request_series = request_series.encode('utf-8')
self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode))
if r.status_code == 404:
self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
return []
elif movie and year:
request_movie = movie.title().replace(' ', '_')
if isinstance(request_movie, unicode):
request_movie = request_movie.encode('utf-8')
self.logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages))
r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year))
if r.status_code == 404:
self.logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages))
return []
else:
raise PluginError('One or more parameter missing')
if r.status_code != 200:
self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulSoup(r.content)
subtitles = []
for sub in soup('td', {'class': 'NewsTitle'}):
sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
if not keywords & sub_keywords:
self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
language = self.getRevertLanguage(html_language.string.strip())
if not language in languages:
self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
html_status = html_language.findNextSibling('td')
status = html_status.find('strong').string.strip()
if status != 'Completed':
self.logger.debug(u'Wrong subtitle status %s' % status)
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
subtitles.append(subtitle)
return subtitles
def download(self, subtitle):
self.downloadFile(subtitle.link, subtitle.path)
return subtitle
class Subtitulos(PluginBase):
site_url = 'http://www.subtitulos.es/'
site_name = 'Subtitulos'
server_url = 'http://www.subtitulos.es'
api_based = False
languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
u'Italian': 'it', u'Català': 'ca'}
reverted_languages = True
videos = [Episode]
require_video = False
release_pattern = re.compile('Versi&oacute;n (.+) ([0-9]+).([0-9])+ megabytes')
def __init__(self, config=None):
super(Subtitulos, self).__init__(config)
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
self.terminate()
def init(self):
self.logger.debug(u'Initializing')
super(Subtitulos, self).init()
def terminate(self):
self.logger.debug(u'Terminating')
def list(self, video, languages):
languages = languages & self.availableLanguages()
if not languages:
self.logger.debug(u'No language available')
return []
if not self.isValidVideo(video):
self.logger.debug(u'Not a valid video')
return []
results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
return results
def query(self, filepath, languages, keywords, series, season, episode):
request_series = series.lower().replace(' ', '_')
if isinstance(request_series, unicode):
request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore')
self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode))
if r.status_code == 404:
self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
return []
if r.status_code != 200:
self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulSoup(r.content)
subtitles = []
for sub in soup('div', {'id': 'version'}):
sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
if not keywords & sub_keywords:
self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
language = self.getRevertLanguage(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
if not language in languages:
self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
html_status = html_language.findNext('li', {'class': 'li-estado green'})
status = html_status.contents[0].string.strip()
if status != 'Completado':
self.logger.debug(u'Wrong subtitle status %s' % status)
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, self.__class__.__name__, html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
subtitles.append(subtitle)
return subtitles
def download(self, subtitle):
self.downloadFile(subtitle.link, subtitle.path)
return subtitle
class GetSubtitle(PluginBase):
site_url = 'http://www.subtitles.com.br/'
site_name = 'GetSubtitle'
server_url = 'http://api.getsubtitle.com/server.php?wsdl'
api_based = True
languages = {'sq': 'ALB', 'ar': 'ARA', 'hy': 'ARM', 'bs': 'BOS', 'bg': 'BUL', 'ca': 'CAT', 'zh': 'CHI', 'hr': 'HRV',
'cs': 'CZE', 'da': 'DAN', 'nl': 'NLD', 'en': 'ENG', 'eo': 'ESP', 'et': 'EST', 'fi': 'FIN', 'fr': 'FRA',
'gl': 'GLG', 'ka': 'GEO', 'de': 'DEU', 'el': 'GRC', 'he': 'ISR', 'hi': 'HIN', 'hu': 'HUN', 'is': 'ISL',
'id': 'IND', 'it': 'ITA', 'ja': 'JPN', 'kk': 'KAZ', 'ko': 'KOR', 'lv': 'LVA', 'lt': 'LIT', 'lb': 'LTZ',
'mk': 'MKD', 'ms': 'MAY', 'no': 'NOR', 'oc': 'OCC', 'pl': 'POL', 'pt': 'POR', 'ro': 'RUM', 'ru': 'RUS',
'sr': 'ZAF', 'sk': 'SLK', 'sl': 'SLV', 'es': 'SPA', 'sv': 'SWE', 'th': 'THA', 'tr': 'TUR', 'uk': 'UKR',
'ur': 'URD', 'vi': 'VTN'}
reverted_languages = False
videos = [Movie]
require_video = False
max_results = 100
def __init__(self, config=None):
super(GetSubtitle, self).__init__(config)
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
pass
def init(self):
self.logger.debug(u'Initializing')
self.server = suds.client.Client(self.server_url)
def terminate(self):
self.logger.debug(u'Terminating')
def query(self, *args):
#TODO
pass
def list(self, video, languages):
languages = languages & self.availableLanguages()
if not languages:
self.logger.debug(u'No language available')
return []
if not self.isValidVideo(video):
self.logger.debug(u'Not a valid video')
return []
#TODO
def download(self, subtitle):
#TODO
pass
'''
class Addic7ed(PluginBase.PluginBase):
site_url = 'http://www.addic7ed.com'
site_name = 'Addic7ed'
server_url = 'http://www.addic7ed.com'
api_based = False
_plugin_languages = {u'English': 'en',
u'English (US)': 'en',
u'English (UK)': 'en',
u'Italian': 'it',
u'Portuguese': 'pt',
u'Portuguese (Brazilian)': 'po',
u'Romanian': 'ro',
u'Español (Latinoamérica)': 'es',
u'Español (España)': 'es',
u'Spanish (Latin America)': 'es',
u'Español': 'es',
u'Spanish': 'es',
u'Spanish (Spain)': 'es',
u'French': 'fr',
u'Greek': 'el',
u'Arabic': 'ar',
u'German': 'de',
u'Croatian': 'hr',
u'Indonesian': 'id',
u'Hebrew': 'he',
u'Russian': 'ru',
u'Turkish': 'tr',
u'Swedish': 'se',
u'Czech': 'cs',
u'Dutch': 'nl',
u'Hungarian': 'hu',
u'Norwegian': 'no',
u'Polish': 'pl',
u'Persian': 'fa'}
def __init__(self, config_dict=None):
super(Addic7ed, self).__init__(self._plugin_languages, config_dict, isRevert=True)
#http://www.addic7ed.com/serie/Smallville/9/11/Absolute_Justice
self.release_pattern = re.compile(' \nVersion (.+), ([0-9]+).([0-9])+ MBs')
def list(self, filepath, languages):
if not self.checkLanguages(languages):
return []
guess = guessit.guess_file_info(filepath, 'autodetect')
if guess['type'] != 'episode':
self.logger.debug(u'Not an episode')
return []
# add multiple things to the release group set
release_group = set()
if 'releaseGroup' in guess:
release_group.add(guess['releaseGroup'].lower())
else:
if 'title' in guess:
release_group.add(guess['title'].lower())
if 'screenSize' in guess:
release_group.add(guess['screenSize'].lower())
if 'series' not in guess or len(release_group) == 0:
self.logger.debug(u'Not enough information to proceed')
return []
self.release_group = release_group # used to sort results
return self.query(guess['series'], guess['season'], guess['episodeNumber'], release_group, filepath, languages)
def query(self, name, season, episode, release_group, filepath, languages=None):
searchname = name.lower().replace(' ', '_')
if isinstance(searchname, unicode):
searchname = searchname.encode('utf-8')
searchurl = '%s/serie/%s/%s/%s/%s' % (self.server_url, urllib2.quote(searchname), season, episode, urllib2.quote(searchname))
self.logger.debug(u'Searching in %s' % searchurl)
try:
req = urllib2.Request(searchurl, headers={'User-Agent': self.user_agent})
page = urllib2.urlopen(req, timeout=self.timeout)
except urllib2.HTTPError as inst:
self.logger.info(u'Error: %s - %s' % (searchurl, inst))
return []
except urllib2.URLError as inst:
self.logger.info(u'TimeOut: %s' % inst)
return []
soup = BeautifulSoup(page.read())
sublinks = []
for html_sub in soup('td', {'class': 'NewsTitle', 'colspan': '3'}):
if not self.release_pattern.match(str(html_sub.contents[1])): # On not needed soup td result
continue
sub_teams = self.listTeams([self.release_pattern.match(str(html_sub.contents[1])).groups()[0].lower()], ['.', '_', ' ', '/', '-'])
if not release_group.intersection(sub_teams): # On wrong team
continue
html_language = html_sub.findNext('td', {'class': 'language'})
sub_language = self.getRevertLanguage(html_language.contents[0].strip().replace('&nbsp;', ''))
if languages and not sub_language in languages: # On wrong language
continue
html_status = html_language.findNextSibling('td')
sub_status = html_status.find('b').string.strip()
if not sub_status == 'Completed': # On not completed subtitles
continue
sub_link = self.server_url + html_status.findNextSibling('td', {'colspan': '3'}).find('a')['href']
self.logger.debug(u'Found a match with teams: %s' % sub_teams)
result = Subtitle(filepath, self.getSubtitlePath(filepath, sub_language), self.__class__.__name__, sub_language, sub_link, keywords=sub_teams)
sublinks.append(result)
sublinks.sort(self._cmpReleaseGroup)
return sublinks
def download(self, subtitle):
self.downloadFile(subtitle.link, subtitle.path)
return subtitle
class Podnapisi(PluginBase.PluginBase):
site_url = "http://www.podnapisi.net"
site_name = "Podnapisi"
server_url = 'http://ssp.podnapisi.net:8000'
api_based = True
_plugin_languages = {"sl": "1",
"en": "2",
"no": "3",
"ko": "4",
"de": "5",
"is": "6",
"cs": "7",
"fr": "8",
"it": "9",
"bs": "10",
"ja": "11",
"ar": "12",
"ro": "13",
"es-ar": "14",
"hu": "15",
"el": "16",
"zh": "17",
"lt": "19",
"et": "20",
"lv": "21",
"he": "22",
"nl": "23",
"da": "24",
"se": "25",
"pl": "26",
"ru": "27",
"es": "28",
"sq": "29",
"tr": "30",
"fi": "31",
"pt": "32",
"bg": "33",
"mk": "35",
"sk": "37",
"hr": "38",
"zh": "40",
"hi": "42",
"th": "44",
"uk": "46",
"sr": "47",
"po": "48",
"ga": "49",
"be": "50",
"vi": "51",
"fa": "52",
"ca": "53",
"id": "54"}
def __init__(self, config_dict=None):
super(Podnapisi, self).__init__(self._plugin_languages, config_dict)
# Podnapisi uses two reference for latin serbian and cyrillic serbian (36 and 47)
# add the 36 manually as cyrillic seems to be more used
self.revertPluginLanguages["36"] = "sr"
def list(self, filenames, languages):
"""Main method to call when you want to list subtitles"""
filepath = filenames[0]
if not os.path.isfile(filepath):
return []
return self.query(self.hashFile(filepath), languages)
def download(self, subtitle):
return []
def query(self, moviehash, languages=None):
"""Makes a query on podnapisi and returns info (link, lang) about found subtitles"""
# login
self.server = xmlrpclib.ServerProxy(self.server_url)
try:
log_result = self.server.initiate(self.user_agent)
self.logger.debug(u"Result: %s" % log_result)
token = log_result["session"]
nonce = log_result["nonce"]
except Exception:
self.logger.error(u"Cannot login" % log_result)
return []
username = 'getmesubs'
password = '99D31$$'
hash = md5()
hash.update(password)
password = hash.hexdigest()
hash = sha256()
hash.update(password)
hash.update(nonce)
password = hash.hexdigest()
self.server.authenticate(token, username, password)
self.logger.debug(u'Authenticated')
#if languages:
# self.logger.debug([self.getLanguage(l) for l in languages])
# self.server.setFilters(token, [self.getLanguage(l) for l in languages])
# self.logger.debug('Filers set for languages %s' % languages)
self.logger.debug(u"Starting search with token %s and hashs %s" % (token, [moviehash]))
results = self.server.search(token, [moviehash])
return results
subs = []
for sub in results['results']:
subs.append(sub)
self.server.terminate(token)
return subs
'''
libs/subliminal/services/__init__.py
Executable
+214
View File
@@ -0,0 +1,214 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from ..exceptions import MissingLanguageError, DownloadFailedError
import logging
import os
import requests
import threading
__all__ = ['ServiceBase', 'ServiceConfig']
logger = logging.getLogger(__name__)
class ServiceBase(object):
"""Service base class
:param config: service configuration
:type config: :class:`ServiceConfig`
"""
#: URL to the service server
server_url = ''
#: User Agent for any HTTP-based requests
user_agent = 'subliminal v0.5'
#: Whether based on an API or not
api_based = False
#: Timeout for web requests
timeout = 5
#: Lock for cache interactions
lock = threading.Lock()
#: Mapping to Service's language codes and subliminal's
languages = {}
#: Whether the mapping is reverted or not
reverted_languages = False
#: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
videos = []
#: Whether the video has to exist or not
require_video = False
def __init__(self, config=None):
self.config = config or ServiceConfig()
def __enter__(self):
self.init()
return self
def __exit__(self, *args):
self.terminate()
def init(self):
"""Initialize connection"""
logger.debug(u'Initializing %s' % self.__class__.__name__)
self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent})
def terminate(self):
"""Terminate connection"""
logger.debug(u'Terminating %s' % self.__class__.__name__)
def query(self, *args):
"""Make the actual query"""
pass
def list(self, video, languages):
"""List subtitles"""
pass
def download(self, subtitle):
"""Download a subtitle"""
self.download_file(subtitle.link, subtitle.path)
@classmethod
def available_languages(cls):
"""Available languages in the Service
:return: available languages
:rtype: set
"""
if not cls.reverted_languages:
return set(cls.languages.keys())
return set(cls.languages.values())
@classmethod
def check_validity(cls, video, languages):
"""Check for video and languages validity in the Service
:param video: the video to check
:type video: :class:`~subliminal.videos.video`
:param set languages: languages to check
:rtype: bool
"""
languages &= cls.available_languages()
if not languages:
logger.debug(u'No language available for service %s' % cls.__class__.__name__.lower())
return False
if not cls.is_valid_video(video):
logger.debug(u'%r is not valid for service %s' % (video, cls.__class__.__name__.lower()))
return False
return True
@classmethod
def is_valid_video(cls, video):
"""Check if video is valid in the Service
:param video: the video to check
:type video: :class:`~subliminal.videos.Video`
:rtype: bool
"""
if cls.require_video and not video.exists:
return False
if not isinstance(video, tuple(cls.videos)):
return False
return True
@classmethod
def is_valid_language(cls, language):
"""Check if language is valid in the Service
:param string language: the language to check
:rtype: bool
"""
if language in cls.available_languages():
return True
return False
@classmethod
def get_revert_language(cls, language):
"""ISO-639-1 language code from service language code
:param string language: service language code
:return: ISO-639-1 language code
:rtype: string
"""
if not cls.reverted_languages and language in cls.languages.values():
return [k for k, v in cls.languages.iteritems() if v == language][0]
if cls.reverted_languages and language in cls.languages.keys():
return cls.languages[language]
raise MissingLanguageError(language)
@classmethod
def get_language(cls, language):
"""Service language code from ISO-639-1 language code
:param string language: ISO-639-1 language code
:return: service language code
:rtype: string
"""
if not cls.reverted_languages and language in cls.languages.keys():
return cls.languages[language]
if cls.reverted_languages and language in cls.languages.values():
return [k for k, v in cls.languages.iteritems() if v == language][0]
raise MissingLanguageError(language)
def download_file(self, url, filepath):
"""Attempt to download a file and remove it in case of failure
:param string url: URL to download
:param string filepath: destination path
"""
logger.info(u'Downloading %s' % url)
try:
r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
with open(filepath, 'wb') as f:
f.write(r.content)
except Exception as e:
logger.error(u'Download %s failed: %s' % (url, e))
if os.path.exists(filepath):
os.remove(filepath)
raise DownloadFailedError(str(e))
logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
class ServiceConfig(object):
"""Configuration for any :class:`Service`
:param bool multi: whether to download one subtitle per language or not
:param string cache_dir: cache directory
"""
def __init__(self, multi=False, cache_dir=None):
self.multi = multi
self.cache_dir = cache_dir
def __repr__(self):
return 'ServiceConfig(%r, %s)' % (self.multi, self.cache_dir)
libs/subliminal/services/bierdopje.py
Executable
+122
View File
@@ -0,0 +1,122 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..exceptions import ServiceError
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode
from ..utils import to_unicode
import BeautifulSoup
import logging
import os.path
import urllib
try:
import cPickle as pickle
except ImportError:
import pickle
logger = logging.getLogger(__name__)
class BierDopje(ServiceBase):
server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
api_based = True
languages = {'en': 'en', 'nl': 'nl'}
reverted_languages = False
videos = [Episode]
require_video = False
def __init__(self, config=None):
super(BierDopje, self).__init__(config)
self.showids = {}
if self.config and self.config.cache_dir:
self.init_cache()
def init_cache(self):
logger.debug(u'Initializing cache...')
if not self.config or not self.config.cache_dir:
raise ServiceError('Cache directory is required')
self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache')
if not os.path.exists(self.showids_cache):
self.save_cache()
def save_cache(self):
logger.debug(u'Saving showids to cache...')
with self.lock:
with open(self.showids_cache, 'w') as f:
pickle.dump(self.showids, f)
def load_cache(self):
logger.debug(u'Loading showids from cache...')
with self.lock:
with open(self.showids_cache, 'r') as f:
self.showids = pickle.load(f)
def query(self, season, episode, languages, filepath, tvdbid=None, series=None):
self.load_cache()
if series:
if series.lower() in self.showids: # from cache
request_id = self.showids[series.lower()]
logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series))
else: # query to get showid
logger.debug(u'Getting showid from show name %s...' % series)
r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulStoneSoup(r.content)
if soup.status.contents[0] == 'false':
logger.debug(u'Could not find show %s' % series)
return []
request_id = int(soup.showid.contents[0])
self.showids[series.lower()] = request_id
self.save_cache()
request_source = 'showid'
request_is_tvdbid = 'false'
elif tvdbid:
request_id = tvdbid
request_source = 'tvdbid'
request_is_tvdbid = 'true'
else:
raise ServiceError('One or more parameter missing')
subtitles = []
for language in languages:
logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid))
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulStoneSoup(r.content)
if soup.status.contents[0] == 'false':
logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
continue
path = get_subtitle_path(filepath, language, self.config.multi)
for result in soup.results('result'):
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0],
release=to_unicode(result.filename.contents[0]))
subtitles.append(subtitle)
return subtitles
def list(self, video, languages):
if not self.check_validity(video, languages):
return []
results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series)
return results
Service = BierDopje
libs/subliminal/services/opensubtitles.py
Executable
+143
View File
@@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..exceptions import ServiceError, DownloadFailedError
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie
from ..utils import to_unicode
import gzip
import logging
import os.path
import xmlrpclib
logger = logging.getLogger(__name__)
class OpenSubtitles(ServiceBase):
server_url = 'http://api.opensubtitles.org/xml-rpc'
api_based = True
languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara',
'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze',
'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis',
'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che',
'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze',
'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful',
'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin',
'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii',
'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav',
'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm',
'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit',
'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao',
'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau',
'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob',
'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss',
'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que',
'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin',
'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe',
'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur',
'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie',
'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor',
'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'}
reverted_languages = False
videos = [Episode, Movie]
require_video = False
confidence_order = ['moviehash', 'imdbid', 'fulltext']
def __init__(self, config=None):
super(OpenSubtitles, self).__init__(config)
self.server = xmlrpclib.ServerProxy(self.server_url)
self.token = None
def init(self):
super(OpenSubtitles, self).init()
result = self.server.LogIn('', '', 'eng', self.user_agent)
if result['status'] != '200 OK':
raise ServiceError('Login failed')
self.token = result['token']
def terminate(self):
super(OpenSubtitles, self).terminate()
if self.token:
self.server.LogOut(self.token)
def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None):
searches = []
if moviehash and size:
searches.append({'moviehash': moviehash, 'moviebytesize': size})
if imdbid:
searches.append({'imdbid': imdbid})
if query:
searches.append({'query': query})
if not searches:
raise ServiceError('One or more parameter missing')
for search in searches:
search['sublanguageid'] = ','.join([self.get_language(l) for l in languages])
logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
results = self.server.SearchSubtitles(self.token, searches)
if not results['data']:
logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token))
return []
subtitles = []
for result in results['data']:
language = self.get_revert_language(result['SubLanguageID'])
path = get_subtitle_path(filepath, language, self.config.multi)
confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
release=to_unicode(result['SubFileName']), confidence=confidence)
subtitles.append(subtitle)
return subtitles
def list(self, video, languages):
if not self.check_validity(video, languages):
return []
results = []
if video.exists:
results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))
elif video.imdbid:
results = self.query(video.path or video.release, languages, imdbid=video.imdbid)
elif isinstance(video, Episode):
results = self.query(video.path or video.release, languages, query=video.series)
elif isinstance(video, Movie):
results = self.query(video.path or video.release, languages, query=video.title)
return results
def download(self, subtitle):
#TODO: Use OpenSubtitles DownloadSubtitles method
try:
self.download_file(subtitle.link, subtitle.path + '.gz')
with open(subtitle.path, 'wb') as dump:
gz = gzip.open(subtitle.path + '.gz')
dump.write(gz.read())
gz.close()
except Exception as e:
if os.path.exists(subtitle.path):
os.remove(subtitle.path)
raise DownloadFailedError(str(e))
finally:
if os.path.exists(subtitle.path + '.gz'):
os.remove(subtitle.path + '.gz')
return subtitle
Service = OpenSubtitles
libs/subliminal/services/subswiki.py
Executable
+99
View File
@@ -0,0 +1,99 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..exceptions import ServiceError
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie
from subliminal.utils import get_keywords, split_keyword
import BeautifulSoup
import logging
import re
import urllib
logger = logging.getLogger(__name__)
class SubsWiki(ServiceBase):
server_url = 'http://www.subswiki.com'
api_based = False
languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
u'Italian': 'it', u'Català': 'ca'}
reverted_languages = True
videos = [Episode, Movie]
require_video = False
release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
def list(self, video, languages):
if not self.check_validity(video, languages):
return []
results = []
if isinstance(video, Episode):
results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
elif isinstance(video, Movie) and video.year:
results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year)
return results
def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None):
if series and season and episode:
request_series = series.lower().replace(' ', '_')
if isinstance(request_series, unicode):
request_series = request_series.encode('utf-8')
logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode))
if r.status_code == 404:
logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
return []
elif movie and year:
request_movie = movie.title().replace(' ', '_')
if isinstance(request_movie, unicode):
request_movie = request_movie.encode('utf-8')
logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages))
r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year))
if r.status_code == 404:
logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages))
return []
else:
raise ServiceError('One or more parameter missing')
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulSoup(r.content)
subtitles = []
for sub in soup('td', {'class': 'NewsTitle'}):
sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
if not keywords & sub_keywords:
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
language = self.get_revert_language(html_language.string.strip())
if not language in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
html_status = html_language.findNextSibling('td')
status = html_status.find('strong').string.strip()
if status != 'Completed':
logger.debug(u'Wrong subtitle status %s' % status)
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
subtitles.append(subtitle)
return subtitles
Service = SubsWiki
libs/subliminal/services/subtitulos.py
Executable
+83
View File
@@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode
from subliminal.utils import get_keywords, split_keyword
import BeautifulSoup
import logging
import re
import unicodedata
import urllib
logger = logging.getLogger(__name__)
class Subtitulos(ServiceBase):
server_url = 'http://www.subtitulos.es'
api_based = False
languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
u'Italian': 'it', u'Català': 'ca'}
reverted_languages = True
videos = [Episode]
require_video = False
release_pattern = re.compile('Versi&oacute;n (.+) ([0-9]+).([0-9])+ megabytes')
def list(self, video, languages):
if not self.check_validity(video, languages):
return []
results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
return results
def query(self, filepath, languages, keywords, series, season, episode):
request_series = series.lower().replace(' ', '_')
if isinstance(request_series, unicode):
request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore')
logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode))
if r.status_code == 404:
logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
return []
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
soup = BeautifulSoup.BeautifulSoup(r.content)
subtitles = []
for sub in soup('div', {'id': 'version'}):
sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
if not keywords & sub_keywords:
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
language = self.get_revert_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
if not language in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
html_status = html_language.findNext('li', {'class': 'li-estado green'})
status = html_status.contents[0].string.strip()
if status != 'Completado':
logger.debug(u'Wrong subtitle status %s' % status)
continue
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
subtitles.append(subtitle)
return subtitles
Service = Subtitulos
libs/subliminal/services/thesubdb.py
Executable
+65
View File
@@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of subliminal.
#
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import ServiceBase
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie, UnknownVideo
import logging
logger = logging.getLogger(__name__)
class TheSubDB(ServiceBase):
server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
user_agent = 'SubDB/1.0 (subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API
api_based = True
languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages
reverted_languages = False
videos = [Movie, Episode, UnknownVideo]
require_video = True
def list(self, video, languages):
if not self.check_validity(video, languages):
return []
results = self.query(video.path, video.hashes['TheSubDB'], languages)
return results
def query(self, filepath, moviehash, languages):
r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
if r.status_code == 404:
logger.debug(u'Could not find subtitles for hash %s' % moviehash)
return []
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
available_languages = set([self.get_revert_language(l) for l in r.content.split(',')])
languages &= available_languages
if not languages:
logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
return []
subtitles = []
for language in languages:
path = get_subtitle_path(filepath, language, self.config.multi)
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.get_language(language)))
subtitles.append(subtitle)
return subtitles
Service = TheSubDB
libs/subliminal/subtitles.py
+63 -43
View File
@@ -1,50 +1,73 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
from languages import list_languages, convert_language
import abc
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from .languages import list_languages, convert_language
import os.path
__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
#: Subtitles extensions
EXTENSIONS = ['.srt', '.sub', '.txt']
class Subtitle(object):
__metaclass__ = abc.ABCMeta
"""Base class for subtitles"""
"""Base class for subtitles
:param string path: path to the subtitle
:param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
"""
def __init__(self, path, language):
self.path = path
self.language = language
@property
def exists(self):
"""Whether the subtitle exists or not"""
if self.path:
return os.path.exists(self.path)
return False
class EmbeddedSubtitle(Subtitle):
"""Subtitle embedded in a container
:param string path: path to the subtitle
:param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
:param int track_id: id of the subtitle track in the container
"""
def __init__(self, path, language, track_id):
super(EmbeddedSubtitle, self).__init__(path, language)
self.track_id = track_id
@classmethod
def fromPath(cls, path):
def from_enzyme(cls, path, subtitle):
language = convert_language(subtitle.language, 1, 2)
return cls(path, language, subtitle.trackno)
class ExternalSubtitle(Subtitle):
"""Subtitle in a file next to the video file"""
@classmethod
def from_path(cls, path):
"""Create an :class:`ExternalSubtitle` from path"""
extension = ''
for e in EXTENSIONS:
if path.endswith(e):
@@ -58,25 +81,21 @@ class Subtitle(object):
return cls(path, language)
class EmbeddedSubtitle(Subtitle):
def __init__(self, path, language, track_id):
super(EmbeddedSubtitle, self).__init__(path, language)
self.track_id = track_id
@classmethod
def fromEnzyme(cls, path, subtitle):
language = convert_language(subtitle.language, 1, 2)
return cls(path, language, subtitle.trackno)
class ExternalSubtitle(Subtitle):
pass
class ResultSubtitle(ExternalSubtitle):
def __init__(self, path, language, plugin, link, release=None, confidence=1, keywords=set()):
"""Subtitle found using :mod:`~subliminal.services`
:param string path: path to the subtitle
:param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
:param string service: name of the service
:param string link: download link for the subtitle
:param string release: release name of the video
:param float confidence: confidence that the subtitle matches the video according to the service
:param set keywords: keywords that describe the subtitle
"""
def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()):
super(ResultSubtitle, self).__init__(path, language)
self.plugin = plugin
self.service = service
self.link = link
self.release = release
self.confidence = confidence
@@ -84,19 +103,20 @@ class ResultSubtitle(ExternalSubtitle):
@property
def single(self):
"""Whether this is a single subtitle or not. A single subtitle does not have
a language indicator in its file name
:rtype: bool
"""
extension = os.path.splitext(self.path)[0]
language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
if not language in list_languages(1):
return True
return False
def convert(self):
converted = {'path': self.path, 'plugin': self.plugin, 'language': self.language, 'link': self.link, 'release': self.release,
'confidence': self.confidence, 'keywords': self.keywords}
return converted
def __str__(self):
return repr(self.convert())
def __repr__(self):
return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
def get_subtitle_path(video_path, language, multi):
libs/subliminal/tasks.py
+35 -16
View File
@@ -1,23 +1,20 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['Task', 'ListTask', 'DownloadTask', 'StopTask']
@@ -27,21 +24,43 @@ class Task(object):
class ListTask(Task):
"""List task to list subtitles"""
def __init__(self, video, languages, plugin, config):
"""List task used by the worker to search for subtitles
:param video: video to search subtitles for
:type video: :class:`~subliminal.videos.Video`
:param list languages: languages to search for
:param string service: name of the service to use
:param config: configuration for the service
:type config: :class:`~subliminal.services.ServiceConfig`
"""
def __init__(self, video, languages, service, config):
self.video = video
self.plugin = plugin
self.service = service
self.languages = languages
self.config = config
def __repr__(self):
return 'ListTask(%r, %r, %s, %r)' % (self.video, self.languages, self.service, self.config)
class DownloadTask(Task):
"""Download task to download subtitles"""
"""Download task used by the worker to download subtitles
:param video: video to download subtitles for
:type video: :class:`~subliminal.videos.Video`
:param subtitles: subtitles to download in order of preference
:type subtitles: list of :class:`~subliminal.subtitles.Subtitle`
"""
def __init__(self, video, subtitles):
self.video = video
self.subtitles = subtitles
def __repr__(self):
return 'DownloadTask(%r, %r)' % (self.video, self.subtitles)
class StopTask(Task):
"""Stop task to stop workers"""
"""Stop task that will stop the worker"""
pass
libs/subliminal/utils.py
+39 -26
View File
@@ -1,44 +1,35 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['PluginConfig', 'get_keywords', 'split_keyword', 'NullHandler']
import logging
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
import re
try:
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
def emit(self, record):
pass
class PluginConfig(object):
def __init__(self, multi=None, cache_dir=None, filemode=None):
self.multi = multi
self.cache_dir = cache_dir
self.filemode = filemode
__all__ = ['get_keywords', 'split_keyword', 'to_unicode']
def get_keywords(guess):
"""Retrieve keywords from guessed informations
:param guess: guessed informations
:type guess: :class:`guessit.guess.Guess`
:return: lower case alphanumeric keywords
:rtype: set
"""
keywords = set()
for k in ['releaseGroup', 'screenSize', 'videoCodec', 'format']:
if k in guess:
@@ -47,5 +38,27 @@ def get_keywords(guess):
def split_keyword(keyword):
"""Split a keyword in multiple ones on any non-alphanumeric character
:param string keyword: keyword
:return: keywords
:rtype: set
"""
split = set(re.findall(r'\w+', keyword))
return split
def to_unicode(data):
"""Convert a basestring to unicode
:param basestring data: data to decode
:return: data as unicode
:rtype: unicode
"""
if not isinstance(data, basestring):
raise ValueError('Basestring expected')
if isinstance(data, unicode):
return data
return unicode(data, 'utf-8', 'replace')
libs/subliminal/videos.py
+190 -135
View File
@@ -1,57 +1,73 @@
# -*- coding: utf-8 -*-
# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
#
# Subliminal - Subtitles, faster than your thoughts
# Copyright (c) 2011 Antoine Bertin <diaoulael@gmail.com>
# This file is part of subliminal.
#
# This file is part of Subliminal.
#
# Subliminal is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# subliminal is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Subliminal is distributed in the hope that it will be useful,
# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo', 'scan']
from languages import list_languages
import abc
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
from . import subtitles
from .languages import list_languages
import enzyme
import guessit
import hashlib
import logging
import mimetypes
import os
import struct
import subprocess
import subtitles
EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv', '.divx', '.asf']
MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo', 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d']
__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo',
'scan', 'hash_opensubtitles', 'hash_thesubdb']
logger = logging.getLogger(__name__)
#: Video extensions
EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv',
'.divx', '.asf']
#: Video mimetypes
MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo',
'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d']
class Video(object):
__metaclass__ = abc.ABCMeta
"""Base class for videos"""
def __init__(self, release, guess, imdbid=None):
self.release = release
"""Base class for videos
:param string path: path
:param guess: guessed informations
:type guess: :class:`~guessit.guess.Guess`
:param string imdbid: imdbid
"""
def __init__(self, path, guess, imdbid=None):
self.release = path
self.guess = guess
self.imdbid = imdbid
self._path = None
self.hashes = {}
if os.path.exists(release):
self.path = release
if os.path.exists(path):
self._path = path
self.size = os.path.getsize(self._path)
self._compute_hashes()
@classmethod
def fromPath(cls, path):
"""Create a Video object guessing all informations from the given release/path"""
def from_path(cls, path):
"""Create a :class:`Video` subclass guessing all informations from the given path
:param string path: path
:return: video object
:rtype: :class:`Episode` or :class:`Movie` or :class:`UnknownVideo`
"""
guess = guessit.guess_file_info(path, 'autodetect')
result = None
if guess['type'] == 'episode' and 'series' in guess and 'season' in guess and 'episodeNumber' in guess:
@@ -72,12 +88,14 @@ class Video(object):
@property
def exists(self):
"""Whether the video exists or not"""
if self._path:
return os.path.exists(self._path)
return False
@property
def path(self):
"""Path to the video"""
return self._path
@path.setter
@@ -86,21 +104,137 @@ class Video(object):
raise ValueError('Path does not exists')
self._path = value
self.size = os.path.getsize(self._path)
self._computeHashes()
self._compute_hashes()
def _computeHashes(self):
self.hashes['OpenSubtitles'] = self._computeHashOpenSubtitles()
self.hashes['TheSubDB'] = self._computeHashTheSubDB()
def _compute_hashes(self):
"""Compute different hashes"""
self.hashes['OpenSubtitles'] = hash_opensubtitles(self.path)
self.hashes['TheSubDB'] = hash_thesubdb(self.path)
def _computeHashOpenSubtitles(self):
"""Hash a file like OpenSubtitles"""
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(self.path, 'rb')
filesize = os.path.getsize(self.path)
def scan(self):
"""Scan and return associated subtitles
:return: associated subtitles
:rtype: list of :class:`~subliminal.subtitles.Subtitle`
"""
if not self.exists:
return []
basepath = os.path.splitext(self.path)[0]
results = []
video_infos = None
try:
video_infos = enzyme.parse(self.path)
logger.debug(u'Succeeded parsing %s with enzyme: %r' % (self.path, video_infos))
except:
logger.debug(u'Failed parsing %s with enzyme' % self.path)
if isinstance(video_infos, enzyme.core.AVContainer):
results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
for l in list_languages(1):
for e in subtitles.EXTENSIONS:
single_path = basepath + '%s' % e
if os.path.exists(single_path):
results.append(subtitles.ExternalSubtitle(single_path, None))
multi_path = basepath + '.%s%s' % (l, e)
if os.path.exists(multi_path):
results.append(subtitles.ExternalSubtitle(multi_path, l))
return results
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.release)
class Episode(Video):
"""Episode :class:`Video`
:param string path: path
:param string series: series
:param int season: season number
:param int episode: episode number
:param string title: title
:param guess: guessed informations
:type guess: :class:`~guessit.guess.Guess`
:param string tvdbid: tvdbid
:param string imdbid: imdbid
"""
def __init__(self, path, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None):
super(Episode, self).__init__(path, guess, imdbid)
self.series = series
self.title = title
self.season = season
self.episode = episode
self.tvdbid = tvdbid
class Movie(Video):
"""Movie :class:`Video`
:param string path: path
:param string title: title
:param int year: year
:param guess: guessed informations
:type guess: :class:`~guessit.guess.Guess`
:param string imdbid: imdbid
"""
def __init__(self, path, title, year=None, guess=None, imdbid=None):
super(Movie, self).__init__(path, guess, imdbid)
self.title = title
self.year = year
class UnknownVideo(Video):
"""Unknown video"""
pass
def scan(entry, max_depth=3, depth=0):
"""Scan a path for videos and subtitles
:param string entry: path
:param int max_depth: maximum folder depth
:param int depth: starting depth
:return: found videos and subtitles
:rtype: list of (:class:`Video`, [:class:`~subliminal.subtitles.Subtitle`])
"""
if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0
return []
if depth == 0:
entry = os.path.abspath(entry)
if os.path.isdir(entry): # a dir? recurse
logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
result = []
for e in os.listdir(entry):
result.extend(scan(os.path.join(entry, e), max_depth, depth + 1))
return result
if os.path.isfile(entry) or depth == 0:
logger.debug(u'Scanning file %s with depth %d/%d' % (entry, depth, max_depth))
if depth != 0: # trust the user: only check for valid format if recursing
if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
return []
video = Video.from_path(entry)
return [(video, video.scan())]
logger.warning(u'Scanning entry %s failed with depth %d/%d' % (entry, depth, max_depth))
return [] # anything else
def hash_opensubtitles(path):
"""Compute a hash using OpenSubtitles' algorithm
:param string path: path
:return: hash
:rtype: string
"""
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
with open(path, 'rb') as f:
filesize = os.path.getsize(path)
filehash = filesize
if filesize < 65536 * 2:
return []
return None
for _ in range(65536 / bytesize):
filebuffer = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, filebuffer)
@@ -112,103 +246,24 @@ class Video(object):
(l_value,) = struct.unpack(longlongformat, filebuffer)
filehash += l_value
filehash = filehash & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = '%016x' % filehash
return returnedhash
def _computeHashTheSubDB(self):
"""Hash a file like TheSubDB"""
readsize = 64 * 1024
with open(self.path, 'rb') as f:
data = f.read(readsize)
f.seek(-readsize, os.SEEK_END)
data += f.read(readsize)
return hashlib.md5(data).hexdigest()
def mkvmerge(self, subs, out=None, mkvmerge_bin='mkvmerge', title=None):
"""Merge the video with subs"""
if not out:
out = self.path + '.merged.mkv'
args = [mkvmerge_bin, '-o', out, self.path]
if title:
args += ['--title', title]
for sub in subs:
if sub.language:
track_id = 0
if isinstance(sub, subtitles.EmbeddedSubtitle):
track_id = sub.track_id
args += ['--language', str(track_id) + ':' + sub.language, sub.path]
continue
args += [sub.path]
with open(os.devnull, 'w') as devnull:
p = subprocess.Popen(args, stdout=devnull, stderr=devnull)
p.wait()
def scan(self):
"""Scan and return associated Subtitles"""
if not self.exists:
return []
basepath = os.path.splitext(self.path)[0]
results = []
video_infos = None
try:
video_infos = enzyme.parse(self.path)
except enzyme.ParseError:
pass
if isinstance(video_infos, enzyme.core.AVContainer):
results.extend([subtitles.EmbeddedSubtitle.fromEnzyme(self.path, s) for s in video_infos.subtitles])
for l in list_languages(1):
for e in subtitles.EXTENSIONS:
single_path = basepath + '%s' % e
if os.path.exists(single_path):
results.append(subtitles.ExternalSubtitle(single_path, None))
multi_path = basepath + '.%s%s' % (l, e)
if os.path.exists(multi_path):
results.append(subtitles.ExternalSubtitle(multi_path, l))
return results
returnedhash = '%016x' % filehash
logger.debug(u'Computed OpenSubtitle hash %s for %s' % (returnedhash, path))
return returnedhash
class Episode(Video):
"""Episode class"""
def __init__(self, release, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None):
super(Episode, self).__init__(release, guess, imdbid)
self.series = series
self.title = title
self.season = season
self.episode = episode
self.tvdbid = tvdbid
def hash_thesubdb(path):
"""Compute a hash using TheSubDB's algorithm
:param string path: path
:return: hash
:rtype: string
class Movie(Video):
"""Movie class"""
def __init__(self, release, title, year=None, guess=None, imdbid=None):
super(Movie, self).__init__(release, guess, imdbid)
self.title = title
self.year = year
class UnknownVideo(Video):
"""Unknown video"""
def __init__(self, release, guess, imdbid=None):
super(UnknownVideo, self).__init__(release, guess, imdbid)
self.guess = guess
def scan(entry, max_depth=3, depth=0):
"""Scan a path and return a list of tuples (video, [subtitle])"""
if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0
return []
if depth == 0:
entry = os.path.abspath(entry)
if os.path.isfile(entry): # a file? scan it
if depth != 0: # trust the user: only check for valid format if recursing
if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
return []
video = Video.fromPath(entry)
return [(video, video.scan())]
if os.path.isdir(entry): # a dir? recurse
result = []
for e in os.listdir(entry):
result.extend(scan(os.path.join(entry, e), max_depth, depth + 1))
return result
return [] # anything else
"""
readsize = 64 * 1024
with open(path, 'rb') as f:
data = f.read(readsize)
f.seek(-readsize, os.SEEK_END)
data += f.read(readsize)
returnedhash = hashlib.md5(data).hexdigest()
logger.debug(u'Computed TheSubDB hash %s for %s' % (returnedhash, path))
return returnedhash