dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Libraries for Subliminal

Browse Source
This commit is contained in:
Ruud
2011-12-23 16:02:48 +01:00
parent a03c33c1c3
commit 8efc88e691
109 changed files with 26894 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/enzyme/__init__.py
+56
View File
@@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import mimetypes
import os
import sys
from exceptions import *
PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']),
('flv', ['video/flv'], ['flv']),
('mkv', ['video/x-matroska', 'application/mkv'], ['mkv', 'mka', 'webm']),
('mp4', ['video/quicktime', 'video/mp4'], ['mov', 'qt', 'mp4', 'mp4a', '3gp', '3gp2', '3g2', 'mk2']),
('mpeg', ['video/mpeg'], ['mpeg', 'mpg', 'mp4', 'ts']),
('ogm', ['application/ogg'], ['ogm', 'ogg', 'ogv']),
('real', ['video/real'], ['rm', 'ra', 'ram']),
('riff', ['video/avi'], ['wav', 'avi'])]
def parse(path):
if not os.path.isfile(path):
raise ValueError('Invalid path')
extension = os.path.splitext(path)[1][1:]
mimetype = mimetypes.guess_type(path)[0]
parser_ext = None
parser_mime = None
for (parser_name, parser_mimetypes, parser_extensions) in PARSERS:
if mimetype in parser_mimetypes:
parser_mime = parser_name
if extension in parser_extensions:
parser_ext = parser_name
parser = parser_mime or parser_ext
if not parser:
raise NoParserError()
mod = __import__(parser, globals=globals(), locals=locals(), fromlist=[], level=-1)
with open(path, 'rb') as f:
p = mod.Parser(f)
return p
libs/enzyme/asf.py
+391
View File
@@ -0,0 +1,391 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
import struct
import string
import logging
from exceptions import *
import core
# get logging object
log = logging.getLogger(__name__)
def _guid(input):
# Remove any '-'
s = string.join(string.split(input,'-'), '')
r = ''
if len(s) != 32:
return ''
x = ''
for i in range(0,16):
r+=chr(int(s[2*i:2*i+2],16))
guid = struct.unpack('>IHHBB6s',r)
return guid
GUIDS = {
'ASF_Header_Object' : _guid('75B22630-668E-11CF-A6D9-00AA0062CE6C'),
'ASF_Data_Object' : _guid('75B22636-668E-11CF-A6D9-00AA0062CE6C'),
'ASF_Simple_Index_Object' : _guid('33000890-E5B1-11CF-89F4-00A0C90349CB'),
'ASF_Index_Object' : _guid('D6E229D3-35DA-11D1-9034-00A0C90349BE'),
'ASF_Media_Object_Index_Object' : _guid('FEB103F8-12AD-4C64-840F-2A1D2F7AD48C'),
'ASF_Timecode_Index_Object' : _guid('3CB73FD0-0C4A-4803-953D-EDF7B6228F0C'),
'ASF_File_Properties_Object' : _guid('8CABDCA1-A947-11CF-8EE4-00C00C205365'),
'ASF_Stream_Properties_Object' : _guid('B7DC0791-A9B7-11CF-8EE6-00C00C205365'),
'ASF_Header_Extension_Object' : _guid('5FBF03B5-A92E-11CF-8EE3-00C00C205365'),
'ASF_Codec_List_Object' : _guid('86D15240-311D-11D0-A3A4-00A0C90348F6'),
'ASF_Script_Command_Object' : _guid('1EFB1A30-0B62-11D0-A39B-00A0C90348F6'),
'ASF_Marker_Object' : _guid('F487CD01-A951-11CF-8EE6-00C00C205365'),
'ASF_Bitrate_Mutual_Exclusion_Object' : _guid('D6E229DC-35DA-11D1-9034-00A0C90349BE'),
'ASF_Error_Correction_Object' : _guid('75B22635-668E-11CF-A6D9-00AA0062CE6C'),
'ASF_Content_Description_Object' : _guid('75B22633-668E-11CF-A6D9-00AA0062CE6C'),
'ASF_Extended_Content_Description_Object' : _guid('D2D0A440-E307-11D2-97F0-00A0C95EA850'),
'ASF_Content_Branding_Object' : _guid('2211B3FA-BD23-11D2-B4B7-00A0C955FC6E'),
'ASF_Stream_Bitrate_Properties_Object' : _guid('7BF875CE-468D-11D1-8D82-006097C9A2B2'),
'ASF_Content_Encryption_Object' : _guid('2211B3FB-BD23-11D2-B4B7-00A0C955FC6E'),
'ASF_Extended_Content_Encryption_Object' : _guid('298AE614-2622-4C17-B935-DAE07EE9289C'),
'ASF_Alt_Extended_Content_Encryption_Obj' : _guid('FF889EF1-ADEE-40DA-9E71-98704BB928CE'),
'ASF_Digital_Signature_Object' : _guid('2211B3FC-BD23-11D2-B4B7-00A0C955FC6E'),
'ASF_Padding_Object' : _guid('1806D474-CADF-4509-A4BA-9AABCB96AAE8'),
'ASF_Extended_Stream_Properties_Object' : _guid('14E6A5CB-C672-4332-8399-A96952065B5A'),
'ASF_Advanced_Mutual_Exclusion_Object' : _guid('A08649CF-4775-4670-8A16-6E35357566CD'),
'ASF_Group_Mutual_Exclusion_Object' : _guid('D1465A40-5A79-4338-B71B-E36B8FD6C249'),
'ASF_Stream_Prioritization_Object' : _guid('D4FED15B-88D3-454F-81F0-ED5C45999E24'),
'ASF_Bandwidth_Sharing_Object' : _guid('A69609E6-517B-11D2-B6AF-00C04FD908E9'),
'ASF_Language_List_Object' : _guid('7C4346A9-EFE0-4BFC-B229-393EDE415C85'),
'ASF_Metadata_Object' : _guid('C5F8CBEA-5BAF-4877-8467-AA8C44FA4CCA'),
'ASF_Metadata_Library_Object' : _guid('44231C94-9498-49D1-A141-1D134E457054'),
'ASF_Index_Parameters_Object' : _guid('D6E229DF-35DA-11D1-9034-00A0C90349BE'),
'ASF_Media_Object_Index_Parameters_Obj' : _guid('6B203BAD-3F11-4E84-ACA8-D7613DE2CFA7'),
'ASF_Timecode_Index_Parameters_Object' : _guid('F55E496D-9797-4B5D-8C8B-604DFE9BFB24'),
'ASF_Audio_Media' : _guid('F8699E40-5B4D-11CF-A8FD-00805F5C442B'),
'ASF_Video_Media' : _guid('BC19EFC0-5B4D-11CF-A8FD-00805F5C442B'),
'ASF_Command_Media' : _guid('59DACFC0-59E6-11D0-A3AC-00A0C90348F6'),
'ASF_JFIF_Media' : _guid('B61BE100-5B4E-11CF-A8FD-00805F5C442B'),
'ASF_Degradable_JPEG_Media' : _guid('35907DE0-E415-11CF-A917-00805F5C442B'),
'ASF_File_Transfer_Media' : _guid('91BD222C-F21C-497A-8B6D-5AA86BFC0185'),
'ASF_Binary_Media' : _guid('3AFB65E2-47EF-40F2-AC2C-70A90D71D343'),
'ASF_Web_Stream_Media_Subtype' : _guid('776257D4-C627-41CB-8F81-7AC7FF1C40CC'),
'ASF_Web_Stream_Format' : _guid('DA1E6B13-8359-4050-B398-388E965BF00C'),
'ASF_No_Error_Correction' : _guid('20FB5700-5B55-11CF-A8FD-00805F5C442B'),
'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220'),
}
class Asf(core.AVContainer):
"""
ASF video parser. The ASF format is also used for Microsft Windows
Media files like wmv.
"""
def __init__(self, file):
core.AVContainer.__init__(self)
self.mime = 'video/x-ms-asf'
self.type = 'asf format'
self._languages = []
self._extinfo = {}
h = file.read(30)
if len(h) < 30:
raise ParseError()
(guidstr, objsize, objnum, reserved1, \
reserved2) = struct.unpack('<16sQIBB',h)
guid = self._parseguid(guidstr)
if (guid != GUIDS['ASF_Header_Object']):
raise ParseError()
if reserved1 != 0x01 or reserved2 != 0x02:
raise ParseError()
log.debug("asf header size: %d / %d objects" % (objsize,objnum))
header = file.read(objsize-30)
for i in range(0,objnum):
h = self._getnextheader(header)
header = header[h[1]:]
del self._languages
del self._extinfo
def _findstream(self, id):
for stream in self.video + self.audio:
if stream.id == id:
return stream
def _apply_extinfo(self, streamid):
stream = self._findstream(streamid)
if not stream or streamid not in self._extinfo:
return
stream.bitrate, stream.fps, langid, metadata = self._extinfo[streamid]
if langid is not None and langid >= 0 and langid < len(self._languages):
stream.language = self._languages[langid]
if metadata:
stream._appendtable('ASFMETADATA', metadata)
def _parseguid(self,string):
return struct.unpack('<IHHBB6s', string[:16])
def _parsekv(self,s):
pos = 0
(descriptorlen,) = struct.unpack('<H', s[pos:pos+2])
pos += 2
descriptorname = s[pos:pos+descriptorlen]
pos += descriptorlen
descriptortype, valuelen = struct.unpack('<HH', s[pos:pos+4])
pos += 4
descriptorvalue = s[pos:pos+valuelen]
pos += valuelen
value = None
if descriptortype == 0x0000:
# Unicode string
value = descriptorvalue
elif descriptortype == 0x0001:
# Byte Array
value = descriptorvalue
elif descriptortype == 0x0002:
# Bool (?)
value = struct.unpack('<I', descriptorvalue)[0] != 0
elif descriptortype == 0x0003:
# DWORD
value = struct.unpack('<I', descriptorvalue)[0]
elif descriptortype == 0x0004:
# QWORD
value = struct.unpack('<Q', descriptorvalue)[0]
elif descriptortype == 0x0005:
# WORD
value = struct.unpack('<H', descriptorvalue)[0]
else:
log.debug("Unknown Descriptor Type %d" % descriptortype)
return (pos,descriptorname,value)
def _parsekv2(self,s):
pos = 0
strno, descriptorlen, descriptortype, valuelen = struct.unpack('<2xHHHI', s[pos:pos+12])
pos += 12
descriptorname = s[pos:pos+descriptorlen]
pos += descriptorlen
descriptorvalue = s[pos:pos+valuelen]
pos += valuelen
value = None
if descriptortype == 0x0000:
# Unicode string
value = descriptorvalue
elif descriptortype == 0x0001:
# Byte Array
value = descriptorvalue
elif descriptortype == 0x0002:
# Bool
value = struct.unpack('<H', descriptorvalue)[0] != 0
pass
elif descriptortype == 0x0003:
# DWORD
value = struct.unpack('<I', descriptorvalue)[0]
elif descriptortype == 0x0004:
# QWORD
value = struct.unpack('<Q', descriptorvalue)[0]
elif descriptortype == 0x0005:
# WORD
value = struct.unpack('<H', descriptorvalue)[0]
else:
log.debug("Unknown Descriptor Type %d" % descriptortype)
return (pos,descriptorname,value,strno)
def _getnextheader(self,s):
r = struct.unpack('<16sQ',s[:24])
(guidstr,objsize) = r
guid = self._parseguid(guidstr)
if guid == GUIDS['ASF_File_Properties_Object']:
log.debug("File Properties Object")
val = struct.unpack('<16s6Q4I',s[24:24+80])
(fileid, size, date, packetcount, duration, \
senddur, preroll, flags, minpack, maxpack, maxbr) = \
val
# FIXME: parse date to timestamp
self.length = duration/10000000.0
elif guid == GUIDS['ASF_Stream_Properties_Object']:
log.debug("Stream Properties Object [%d]" % objsize)
streamtype = self._parseguid(s[24:40])
errortype = self._parseguid(s[40:56])
offset, typelen, errorlen, flags = struct.unpack('<QIIH', s[56:74])
strno = flags & 0x7f
encrypted = flags >> 15
if encrypted:
self._set('encrypted', True)
if streamtype == GUIDS['ASF_Video_Media']:
vi = core.VideoStream()
vi.width, vi.height, depth, codec, = struct.unpack('<4xII2xH4s', s[89:89+20])
vi.codec = codec
vi.id = strno
self.video.append(vi)
elif streamtype == GUIDS['ASF_Audio_Media']:
ai = core.AudioStream()
twocc, ai.channels, ai.samplerate, bitrate, block, \
ai.samplebits, = struct.unpack('<HHIIHH', s[78:78+16])
ai.bitrate = 8*bitrate
ai.codec = twocc
ai.id = strno
self.audio.append(ai)
self._apply_extinfo(strno)
elif guid == GUIDS['ASF_Extended_Stream_Properties_Object']:
streamid, langid, frametime = struct.unpack('<HHQ',s[72:84])
(bitrate,) = struct.unpack('<I', s[40:40+4])
if streamid not in self._extinfo:
self._extinfo[streamid] = [None, None, None, {}]
if frametime == 0:
# Problaby VFR, report as 1000fps (which is what MPlayer does)
frametime = 10000.0
self._extinfo[streamid][:3] = [bitrate, 10000000.0 / frametime, langid]
self._apply_extinfo(streamid)
elif guid == GUIDS['ASF_Header_Extension_Object']:
log.debug("ASF_Header_Extension_Object %d" % objsize)
size = struct.unpack('<I',s[42:46])[0]
data = s[46:46+size]
while len(data):
log.debug("Sub:")
h = self._getnextheader(data)
data = data[h[1]:]
elif guid == GUIDS['ASF_Codec_List_Object']:
log.debug("List Object")
pass
elif guid == GUIDS['ASF_Error_Correction_Object']:
log.debug("Error Correction")
pass
elif guid == GUIDS['ASF_Content_Description_Object']:
log.debug("Content Description Object")
val = struct.unpack('<5H', s[24:24+10])
pos = 34
strings = []
for i in val:
ss = s[pos:pos+i].replace('\0', '').lstrip().rstrip()
strings.append(ss)
pos+=i
# Set empty strings to None
strings = [x or None for x in strings]
self.title, self.artist, self.copyright, self.caption, rating = strings
elif guid == GUIDS['ASF_Extended_Content_Description_Object']:
(count,) = struct.unpack('<H', s[24:26])
pos = 26
descriptor = {}
for i in range(0, count):
# Read additional content descriptors
d = self._parsekv(s[pos:])
pos += d[0]
descriptor[d[1]] = d[2]
self._appendtable('ASFDESCRIPTOR', descriptor)
elif guid == GUIDS['ASF_Metadata_Object']:
(count,) = struct.unpack('<H', s[24:26])
pos = 26
streams = {}
for i in range(0, count):
# Read additional content descriptors
size,key,value,strno = self._parsekv2(s[pos:])
if strno not in streams:
streams[strno] = {}
streams[strno][key] = value
pos += size
for strno, metadata in streams.items():
if strno not in self._extinfo:
self._extinfo[strno] = [None, None, None, {}]
self._extinfo[strno][3].update(metadata)
self._apply_extinfo(strno)
elif guid == GUIDS['ASF_Language_List_Object']:
count = struct.unpack('<H', s[24:26])[0]
pos = 26
for i in range(0, count):
idlen = struct.unpack('<B', s[pos:pos+1])[0]
idstring = s[pos+1:pos+1+idlen]
idstring = unicode(idstring, 'utf-16').replace('\0', '')
log.debug("Language: %d/%d: %s" % (i+1, count, idstring))
self._languages.append(idstring)
pos += 1+idlen
elif guid == GUIDS['ASF_Stream_Bitrate_Properties_Object']:
# This record contains stream bitrate with payload overhead. For
# audio streams, we should have the average bitrate from
# ASF_Stream_Properties_Object. For video streams, we get it from
# ASF_Extended_Stream_Properties_Object. So this record is not
# used.
pass
elif guid == GUIDS['ASF_Content_Encryption_Object'] or \
guid == GUIDS['ASF_Extended_Content_Encryption_Object']:
self._set('encrypted', True)
else:
# Just print the type:
for h in GUIDS.keys():
if GUIDS[h] == guid:
log.debug("Unparsed %s [%d]" % (h,objsize))
break
else:
u = "%.8X-%.4X-%.4X-%.2X%.2X-%s" % guid
log.debug("unknown: len=%d [%d]" % (len(u), objsize))
return r
class AsfAudio(core.AudioStream):
"""
ASF audio parser for wma files.
"""
def __init__(self):
core.AudioStream.__init__(self)
self.mime = 'audio/x-ms-asf'
self.type = 'asf format'
def Parser(file):
"""
Wrapper around audio and av content.
"""
asf = Asf(file)
if not len(asf.audio) or len(asf.video):
# AV container
return asf
# No video but audio streams. Handle has audio core
audio = AsfAudio()
for key in audio._keys:
if key in asf._keys:
if not getattr(audio, key, None):
setattr(audio, key, getattr(asf, key))
return audio
libs/enzyme/core.py
+457
View File
@@ -0,0 +1,457 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import re
import logging
import fourcc
import language
from exceptions import *
from strutils import str_to_unicode, unicode_to_str
UNPRINTABLE_KEYS = ['thumbnail', 'url', 'codec_private']
EXTENSION_DEVICE = 'device'
EXTENSION_DIRECTORY = 'directory'
EXTENSION_STREAM = 'stream'
MEDIACORE = ['title', 'caption', 'comment', 'size', 'type', 'subtype', 'timestamp',
'keywords', 'country', 'language', 'langcode', 'url', 'artist',
'mime', 'datetime', 'tags', 'hash']
AUDIOCORE = ['channels', 'samplerate', 'length', 'encoder', 'codec', 'format',
'samplebits', 'bitrate', 'fourcc', 'trackno', 'id', 'userdate',
'enabled', 'default', 'codec_private']
MUSICCORE = ['trackof', 'album', 'genre', 'discs', 'thumbnail']
VIDEOCORE = ['length', 'encoder', 'bitrate', 'samplerate', 'codec', 'format',
'samplebits', 'width', 'height', 'fps', 'aspect', 'trackno',
'fourcc', 'id', 'enabled', 'default', 'codec_private']
AVCORE = ['length', 'encoder', 'trackno', 'trackof', 'copyright', 'product',
'genre', 'writer', 'producer', 'studio', 'rating', 'actors', 'thumbnail',
'delay', 'image', 'video', 'audio', 'subtitles', 'chapters', 'software',
'summary', 'synopsis', 'season', 'episode', 'series']
# get logging object
log = logging.getLogger(__name__)
class Media(object):
media = None
"""
Media is the base class to all Media Metadata Containers. It defines
the basic structures that handle metadata. Media and its derivates
contain a common set of metadata attributes that is listed in keys.
Specific derivates contain additional keys to the dublin core set that is
defined in Media.
"""
_keys = MEDIACORE
table_mapping = {}
def __init__(self, hash=None):
if hash is not None:
# create Media based on dict
for key, value in hash.items():
if isinstance(value, list) and value and isinstance(value[0], dict):
value = [Media(x) for x in value]
self._set(key, value)
return
self._keys = self._keys[:]
self.tables = {}
# Tags, unlike tables, are more well-defined dicts whose values are
# either Tag objects, other dicts (for nested tags), or lists of either
# (for multiple instances of the tag, e.g. actor). Where possible,
# parsers should transform tag names to conform to the Official
# Matroska tags defined at http://www.matroska.org/technical/specs/tagging/index.html
# All tag names will be lower-cased.
self.tags = Tags()
for key in set(self._keys) - set(['media', 'tags']):
setattr(self, key, None)
#
# unicode and string convertion for debugging
#
#TODO: Fix that mess
def __unicode__(self):
result = u''
# print normal attributes
lists = []
for key in self._keys:
value = getattr(self, key, None)
if value == None or key == 'url':
continue
if isinstance(value, list):
if not value:
continue
elif isinstance(value[0], basestring):
# Just a list of strings (keywords?), so don't treat it specially.
value = u', '.join(value)
else:
lists.append((key, value))
continue
elif isinstance(value, dict):
# Tables or tags treated separately.
continue
if key in UNPRINTABLE_KEYS:
value = '<unprintable data, size=%d>' % len(value)
result += u'| %10s: %s\n' % (unicode(key), unicode(value))
# print tags (recursively, to support nested tags).
def print_tags(tags, suffix, show_label):
result = ''
for n, (name, tag) in enumerate(tags.items()):
result += u'| %12s%s%s = ' % (u'tags: ' if n == 0 and show_label else '', suffix, name)
if isinstance(tag, list):
# TODO: doesn't support lists/dicts within lists.
result += u'%s\n' % ', '.join(subtag.value for subtag in tag)
else:
result += u'%s\n' % (tag.value or '')
if isinstance(tag, dict):
result += print_tags(tag, ' ', False)
return result
result += print_tags(self.tags, '', True)
# print lists
for key, l in lists:
for n, item in enumerate(l):
label = '+-- ' + key.rstrip('s').capitalize()
if key not in ['tracks', 'subtitles', 'chapters']:
label += ' Track'
result += u'%s #%d\n' % (label, n + 1)
result += '| ' + re.sub(r'\n(.)', r'\n| \1', unicode(item))
# print tables
if log.level >= 10:
for name, table in self.tables.items():
result += '+-- Table %s\n' % str(name)
for key, value in table.items():
try:
value = unicode(value)
if len(value) > 50:
value = u'<unprintable data, size=%d>' % len(value)
except (UnicodeDecodeError, TypeError), e:
try:
value = u'<unprintable data, size=%d>' % len(value)
except AttributeError:
value = u'<unprintable data>'
result += u'| | %s: %s\n' % (unicode(key), value)
return result
def __str__(self):
return unicode(self).encode()
def __repr__(self):
if hasattr(self, 'url'):
return '<%s %s>' % (str(self.__class__)[8:-2], self.url)
else:
return '<%s>' % (str(self.__class__)[8:-2])
#
# internal functions
#
def _appendtable(self, name, hashmap):
"""
Appends a tables of additional metadata to the Object.
If such a table already exists, the given tables items are
added to the existing one.
"""
if name not in self.tables:
self.tables[name] = hashmap
else:
# Append to the already existing table
for k in hashmap.keys():
self.tables[name][k] = hashmap[k]
def _set(self, key, value):
"""
Set key to value and add the key to the internal keys list if
missing.
"""
if value is None and getattr(self, key, None) is None:
return
if isinstance(value, str):
value = str_to_unicode(value)
setattr(self, key, value)
if not key in self._keys:
self._keys.append(key)
def _set_url(self, url):
"""
Set the URL of the source
"""
self.url = url
def _finalize(self):
"""
Correct same data based on specific rules
"""
# make sure all strings are unicode
for key in self._keys:
if key in UNPRINTABLE_KEYS:
continue
value = getattr(self, key)
if value is None:
continue
if key == 'image':
if isinstance(value, unicode):
setattr(self, key, unicode_to_str(value))
continue
if isinstance(value, str):
setattr(self, key, str_to_unicode(value))
if isinstance(value, unicode):
setattr(self, key, value.strip().rstrip().replace(u'\0', u''))
if isinstance(value, list) and value and isinstance(value[0], Media):
for submenu in value:
submenu._finalize()
# copy needed tags from tables
for name, table in self.tables.items():
mapping = self.table_mapping.get(name, {})
for tag, attr in mapping.items():
if self.get(attr):
continue
value = table.get(tag, None)
if value is not None:
if not isinstance(value, (str, unicode)):
value = str_to_unicode(str(value))
elif isinstance(value, str):
value = str_to_unicode(value)
value = value.strip().rstrip().replace(u'\0', u'')
setattr(self, attr, value)
if 'fourcc' in self._keys and 'codec' in self._keys and self.codec is not None:
# Codec may be a fourcc, in which case we resolve it to its actual
# name and set the fourcc attribute.
self.fourcc, self.codec = fourcc.resolve(self.codec)
if 'language' in self._keys:
self.langcode, self.language = language.resolve(self.language)
#
# data access
#
def __contains__(self, key):
"""
Test if key exists in the dict
"""
return hasattr(self, key)
def get(self, attr, default=None):
"""
Returns the given attribute. If the attribute is not set by
the parser return 'default'.
"""
return getattr(self, attr, default)
def __getitem__(self, attr):
"""
Get the value of the given attribute
"""
return getattr(self, attr, None)
def __setitem__(self, key, value):
"""
Set the value of 'key' to 'value'
"""
setattr(self, key, value)
def has_key(self, key):
"""
Check if the object has an attribute 'key'
"""
return hasattr(self, key)
def convert(self):
"""
Convert Media to dict.
"""
result = {}
for k in self._keys:
value = getattr(self, k, None)
if isinstance(value, list) and value and isinstance(value[0], Media):
value = [x.convert() for x in value]
result[k] = value
return result
def keys(self):
"""
Return all keys for the attributes set by the parser.
"""
return self._keys
class Collection(Media):
"""
Collection of Digial Media like CD, DVD, Directory, Playlist
"""
_keys = Media._keys + ['id', 'tracks']
def __init__(self):
Media.__init__(self)
self.tracks = []
class Tag(object):
"""
An individual tag, which will be a value stored in a Tags object.
Tag values are strings (for binary data), unicode objects, or datetime
objects for tags that represent dates or times.
"""
def __init__(self, value=None, langcode='und', binary=False):
super(Tag, self).__init__()
self.value = value
self.langcode = langcode
self.binary = binary
def __unicode__(self):
return unicode(self.value)
def __str__(self):
return str(self.value)
def __repr__(self):
if not self.binary:
return '<Tag object: %s>' % repr(self.value)
else:
return '<Binary Tag object: size=%d>' % len(self.value)
@property
def langcode(self):
return self._langcode
@langcode.setter
def langcode(self, code):
self._langcode, self.language = language.resolve(code)
class Tags(dict, Tag):
"""
A dictionary containing Tag objects. Values can be other Tags objects
(for nested tags), lists, or Tag objects.
A Tags object is more or less a dictionary but it also contains a value.
This is necessary in order to represent this kind of tag specification
(e.g. for Matroska)::
<Simple>
<Name>LAW_RATING</Name>
<String>PG</String>
<Simple>
<Name>COUNTRY</Name>
<String>US</String>
</Simple>
</Simple>
The attribute RATING has a value (PG), but it also has a child tag
COUNTRY that specifies the country code the rating belongs to.
"""
def __init__(self, value=None, langcode='und', binary=False):
super(Tags, self).__init__()
self.value = value
self.langcode = langcode
self.binary = False
class AudioStream(Media):
"""
Audio Tracks in a Multiplexed Container.
"""
_keys = Media._keys + AUDIOCORE
class Music(AudioStream):
"""
Digital Music.
"""
_keys = AudioStream._keys + MUSICCORE
def _finalize(self):
"""
Correct same data based on specific rules
"""
AudioStream._finalize(self)
if self.trackof:
try:
# XXX Why is this needed anyway?
if int(self.trackno) < 10:
self.trackno = u'0%s' % int(self.trackno)
except (AttributeError, ValueError):
pass
class VideoStream(Media):
"""
Video Tracks in a Multiplexed Container.
"""
_keys = Media._keys + VIDEOCORE
class Chapter(Media):
"""
Chapter in a Multiplexed Container.
"""
_keys = ['enabled', 'name', 'pos', 'id']
def __init__(self, name=None, pos=0):
Media.__init__(self)
self.name = name
self.pos = pos
self.enabled = True
class Subtitle(Media):
"""
Subtitle Tracks in a Multiplexed Container.
"""
_keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title',
'id', 'codec']
def __init__(self, language=None):
Media.__init__(self)
self.language = language
class AVContainer(Media):
"""
Container for Audio and Video streams. This is the Container Type for
all media, that contain more than one stream.
"""
_keys = Media._keys + AVCORE
def __init__(self):
Media.__init__(self)
self.audio = []
self.video = []
self.subtitles = []
self.chapters = []
def _finalize(self):
"""
Correct same data based on specific rules
"""
Media._finalize(self)
if not self.length and len(self.video) and self.video[0].length:
self.length = 0
# Length not specified for container, so use the largest length
# of its tracks as container length.
for track in self.video + self.audio:
if track.length:
self.length = max(self.length, track.length)
libs/enzyme/exceptions.py
+31
View File
@@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
class Error(Exception):
pass
class NoParserError(Error):
pass
class ParseError(Error):
pass
libs/enzyme/flv.py
+182
View File
@@ -0,0 +1,182 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
from exceptions import *
import core
import logging
import struct
# get logging object
log = logging.getLogger(__name__)
FLV_TAG_TYPE_AUDIO = 0x08
FLV_TAG_TYPE_VIDEO = 0x09
FLV_TAG_TYPE_META = 0x12
# audio flags
FLV_AUDIO_CHANNEL_MASK = 0x01
FLV_AUDIO_SAMPLERATE_MASK = 0x0c
FLV_AUDIO_CODECID_MASK = 0xf0
FLV_AUDIO_SAMPLERATE_OFFSET = 2
FLV_AUDIO_CODECID_OFFSET = 4
FLV_AUDIO_CODECID = (0x0001, 0x0002, 0x0055, 0x0001)
# video flags
FLV_VIDEO_CODECID_MASK = 0x0f
FLV_VIDEO_CODECID = ( 'FLV1', 'MSS1', 'VP60') # wild guess
FLV_DATA_TYPE_NUMBER = 0x00
FLV_DATA_TYPE_BOOL = 0x01
FLV_DATA_TYPE_STRING = 0x02
FLV_DATA_TYPE_OBJECT = 0x03
FLC_DATA_TYPE_CLIP = 0x04
FLV_DATA_TYPE_REFERENCE = 0x07
FLV_DATA_TYPE_ECMARRAY = 0x08
FLV_DATA_TYPE_ENDOBJECT = 0x09
FLV_DATA_TYPE_ARRAY = 0x0a
FLV_DATA_TYPE_DATE = 0x0b
FLV_DATA_TYPE_LONGSTRING = 0x0c
FLVINFO = {
'creator': 'copyright',
}
class FlashVideo(core.AVContainer):
"""
Experimental parser for Flash videos. It requires certain flags to
be set to report video resolutions and in most cases it does not
provide that information.
"""
table_mapping = { 'FLVINFO' : FLVINFO }
def __init__(self,file):
core.AVContainer.__init__(self)
self.mime = 'video/flv'
self.type = 'Flash Video'
data = file.read(13)
if len(data) < 13 or struct.unpack('>3sBBII', data)[0] != 'FLV':
raise ParseError()
for i in range(10):
if self.audio and self.video:
break
data = file.read(11)
if len(data) < 11:
break
chunk = struct.unpack('>BH4BI', data)
size = (chunk[1] << 8) + chunk[2]
if chunk[0] == FLV_TAG_TYPE_AUDIO:
flags = ord(file.read(1))
if not self.audio:
a = core.AudioStream()
a.channels = (flags & FLV_AUDIO_CHANNEL_MASK) + 1
srate = (flags & FLV_AUDIO_SAMPLERATE_MASK)
a.samplerate = (44100 << (srate >> FLV_AUDIO_SAMPLERATE_OFFSET) >> 3)
codec = (flags & FLV_AUDIO_CODECID_MASK) >> FLV_AUDIO_CODECID_OFFSET
if codec < len(FLV_AUDIO_CODECID):
a.codec = FLV_AUDIO_CODECID[codec]
self.audio.append(a)
file.seek(size - 1, 1)
elif chunk[0] == FLV_TAG_TYPE_VIDEO:
flags = ord(file.read(1))
if not self.video:
v = core.VideoStream()
codec = (flags & FLV_VIDEO_CODECID_MASK) - 2
if codec < len(FLV_VIDEO_CODECID):
v.codec = FLV_VIDEO_CODECID[codec]
# width and height are in the meta packet, but I have
# no file with such a packet inside. So maybe we have
# to decode some parts of the video.
self.video.append(v)
file.seek(size - 1, 1)
elif chunk[0] == FLV_TAG_TYPE_META:
log.info('metadata %s', str(chunk))
metadata = file.read(size)
try:
while metadata:
length, value = self._parse_value(metadata)
if isinstance(value, dict):
log.info('metadata: %s', value)
if value.get('creator'):
self.copyright = value.get('creator')
if value.get('width'):
self.width = value.get('width')
if value.get('height'):
self.height = value.get('height')
if value.get('duration'):
self.length = value.get('duration')
self._appendtable('FLVINFO', value)
if not length:
# parse error
break
metadata = metadata[length:]
except (IndexError, struct.error, TypeError):
pass
else:
log.info('unkown %s', str(chunk))
file.seek(size, 1)
file.seek(4, 1)
def _parse_value(self, data):
"""
Parse the next metadata value.
"""
if ord(data[0]) == FLV_DATA_TYPE_NUMBER:
value = struct.unpack('>d', data[1:9])[0]
return 9, value
if ord(data[0]) == FLV_DATA_TYPE_BOOL:
return 2, bool(data[1])
if ord(data[0]) == FLV_DATA_TYPE_STRING:
length = (ord(data[1]) << 8) + ord(data[2])
return length + 3, data[3:length+3]
if ord(data[0]) == FLV_DATA_TYPE_ECMARRAY:
init_length = len(data)
num = struct.unpack('>I', data[1:5])[0]
data = data[5:]
result = {}
for i in range(num):
length = (ord(data[0]) << 8) + ord(data[1])
key = data[2:length+2]
data = data[length + 2:]
length, value = self._parse_value(data)
if not length:
return 0, result
result[key] = value
data = data[length:]
return init_length - len(data), result
log.info('unknown code: %x. Stop metadata parser', ord(data[0]))
return 0, None
Parser = FlashVideo
libs/enzyme/fourcc.py
+853
View File
@@ -0,0 +1,853 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import string
import re
import struct
__all__ = ['resolve']
def resolve(code):
"""
Transform a twocc or fourcc code into a name. Returns a 2-tuple of (cc,
codec) where both are strings and cc is a string in the form '0xXX' if it's
a twocc, or 'ABCD' if it's a fourcc. If the given code is not a known
twocc or fourcc, the return value will be (None, 'Unknown'), unless the
code is otherwise a printable string in which case it will be returned as
the codec.
"""
if isinstance(code, basestring):
codec = u'Unknown'
# Check for twocc
if re.match(r'^0x[\da-f]{1,4}$', code, re.I):
# Twocc in hex form
return code, TWOCC.get(int(code, 16), codec)
elif code.isdigit() and 0 <= int(code) <= 0xff:
# Twocc in decimal form
return hex(int(code)), TWOCC.get(int(code), codec)
elif len(code) == 2:
code = struct.unpack('H', code)[0]
return hex(code), TWOCC.get(code, codec)
elif len(code) != 4 and len([x for x in code if x not in string.printable]) == 0:
# Code is a printable string.
codec = unicode(code)
if code[:2] == 'MS' and code[2:].upper() in FOURCC:
code = code[2:]
if code.upper() in FOURCC:
return code.upper(), unicode(FOURCC[code.upper()])
return None, codec
elif isinstance(code, (int, long)):
return hex(code), TWOCC.get(code, u'Unknown')
return None, u'Unknown'
TWOCC = {
0x0000: 'Unknown Wave Format',
0x0001: 'PCM',
0x0002: 'Microsoft ADPCM',
0x0003: 'IEEE Float',
0x0004: 'Compaq Computer VSELP',
0x0005: 'IBM CVSD',
0x0006: 'A-Law',
0x0007: 'mu-Law',
0x0008: 'Microsoft DTS',
0x0009: 'Microsoft DRM',
0x0010: 'OKI ADPCM',
0x0011: 'Intel DVI/IMA ADPCM',
0x0012: 'Videologic MediaSpace ADPCM',
0x0013: 'Sierra Semiconductor ADPCM',
0x0014: 'Antex Electronics G.723 ADPCM',
0x0015: 'DSP Solutions DigiSTD',
0x0016: 'DSP Solutions DigiFIX',
0x0017: 'Dialogic OKI ADPCM',
0x0018: 'MediaVision ADPCM',
0x0019: 'Hewlett-Packard CU',
0x0020: 'Yamaha ADPCM',
0x0021: 'Speech Compression Sonarc',
0x0022: 'DSP Group TrueSpeech',
0x0023: 'Echo Speech EchoSC1',
0x0024: 'Audiofile AF36',
0x0025: 'Audio Processing Technology APTX',
0x0026: 'AudioFile AF10',
0x0027: 'Prosody 1612',
0x0028: 'LRC',
0x0030: 'Dolby AC2',
0x0031: 'Microsoft GSM 6.10',
0x0032: 'MSNAudio',
0x0033: 'Antex Electronics ADPCME',
0x0034: 'Control Resources VQLPC',
0x0035: 'DSP Solutions DigiREAL',
0x0036: 'DSP Solutions DigiADPCM',
0x0037: 'Control Resources CR10',
0x0038: 'Natural MicroSystems VBXADPCM',
0x0039: 'Crystal Semiconductor IMA ADPCM',
0x003A: 'EchoSC3',
0x003B: 'Rockwell ADPCM',
0x003C: 'Rockwell Digit LK',
0x003D: 'Xebec',
0x0040: 'Antex Electronics G.721 ADPCM',
0x0041: 'G.728 CELP',
0x0042: 'MSG723',
0x0043: 'IBM AVC ADPCM',
0x0045: 'ITU-T G.726 ADPCM',
0x0050: 'MPEG 1, Layer 1,2',
0x0052: 'RT24',
0x0053: 'PAC',
0x0055: 'MPEG Layer 3',
0x0059: 'Lucent G.723',
0x0060: 'Cirrus',
0x0061: 'ESPCM',
0x0062: 'Voxware',
0x0063: 'Canopus Atrac',
0x0064: 'G.726 ADPCM',
0x0065: 'G.722 ADPCM',
0x0066: 'DSAT',
0x0067: 'DSAT Display',
0x0069: 'Voxware Byte Aligned',
0x0070: 'Voxware AC8',
0x0071: 'Voxware AC10',
0x0072: 'Voxware AC16',
0x0073: 'Voxware AC20',
0x0074: 'Voxware MetaVoice',
0x0075: 'Voxware MetaSound',
0x0076: 'Voxware RT29HW',
0x0077: 'Voxware VR12',
0x0078: 'Voxware VR18',
0x0079: 'Voxware TQ40',
0x0080: 'Softsound',
0x0081: 'Voxware TQ60',
0x0082: 'MSRT24',
0x0083: 'G.729A',
0x0084: 'MVI MV12',
0x0085: 'DF G.726',
0x0086: 'DF GSM610',
0x0088: 'ISIAudio',
0x0089: 'Onlive',
0x0091: 'SBC24',
0x0092: 'Dolby AC3 SPDIF',
0x0093: 'MediaSonic G.723',
0x0094: 'Aculab PLC Prosody 8KBPS',
0x0097: 'ZyXEL ADPCM',
0x0098: 'Philips LPCBB',
0x0099: 'Packed',
0x00A0: 'Malden Electronics PHONYTALK',
0x00FF: 'AAC',
0x0100: 'Rhetorex ADPCM',
0x0101: 'IBM mu-law',
0x0102: 'IBM A-law',
0x0103: 'IBM AVC Adaptive Differential Pulse Code Modulation',
0x0111: 'Vivo G.723',
0x0112: 'Vivo Siren',
0x0123: 'Digital G.723',
0x0125: 'Sanyo LD ADPCM',
0x0130: 'Sipro Lab Telecom ACELP.net',
0x0131: 'Sipro Lab Telecom ACELP.4800',
0x0132: 'Sipro Lab Telecom ACELP.8V3',
0x0133: 'Sipro Lab Telecom ACELP.G.729',
0x0134: 'Sipro Lab Telecom ACELP.G.729A',
0x0135: 'Sipro Lab Telecom ACELP.KELVIN',
0x0140: 'Windows Media Video V8',
0x0150: 'Qualcomm PureVoice',
0x0151: 'Qualcomm HalfRate',
0x0155: 'Ring Zero Systems TUB GSM',
0x0160: 'Windows Media Audio V1 / DivX audio (WMA)',
0x0161: 'Windows Media Audio V7 / V8 / V9',
0x0162: 'Windows Media Audio Professional V9',
0x0163: 'Windows Media Audio Lossless V9',
0x0170: 'UNISYS NAP ADPCM',
0x0171: 'UNISYS NAP ULAW',
0x0172: 'UNISYS NAP ALAW',
0x0173: 'UNISYS NAP 16K',
0x0200: 'Creative Labs ADPCM',
0x0202: 'Creative Labs Fastspeech8',
0x0203: 'Creative Labs Fastspeech10',
0x0210: 'UHER Informatic ADPCM',
0x0215: 'Ulead DV ACM',
0x0216: 'Ulead DV ACM',
0x0220: 'Quarterdeck',
0x0230: 'I-link Worldwide ILINK VC',
0x0240: 'Aureal Semiconductor RAW SPORT',
0x0241: 'ESST AC3',
0x0250: 'Interactive Products HSX',
0x0251: 'Interactive Products RPELP',
0x0260: 'Consistent Software CS2',
0x0270: 'Sony ATRAC3 (SCX, same as MiniDisk LP2)',
0x0300: 'Fujitsu FM Towns Snd',
0x0400: 'BTV Digital',
0x0401: 'Intel Music Coder (IMC)',
0x0402: 'Ligos Indeo Audio',
0x0450: 'QDesign Music',
0x0680: 'VME VMPCM',
0x0681: 'AT&T Labs TPC',
0x0700: 'YMPEG Alpha',
0x08AE: 'ClearJump LiteWave',
0x1000: 'Olivetti GSM',
0x1001: 'Olivetti ADPCM',
0x1002: 'Olivetti CELP',
0x1003: 'Olivetti SBC',
0x1004: 'Olivetti OPR',
0x1100: 'Lernout & Hauspie LH Codec',
0x1101: 'Lernout & Hauspie CELP codec',
0x1102: 'Lernout & Hauspie SBC codec',
0x1103: 'Lernout & Hauspie SBC codec',
0x1104: 'Lernout & Hauspie SBC codec',
0x1400: 'Norris',
0x1401: 'AT&T ISIAudio',
0x1500: 'Soundspace Music Compression',
0x181C: 'VoxWare RT24 speech codec',
0x181E: 'Lucent elemedia AX24000P Music codec',
0x1C07: 'Lucent SX8300P speech codec',
0x1C0C: 'Lucent SX5363S G.723 compliant codec',
0x1F03: 'CUseeMe DigiTalk (ex-Rocwell)',
0x1FC4: 'NCT Soft ALF2CD ACM',
0x2000: 'AC3',
0x2001: 'Dolby DTS (Digital Theater System)',
0x2002: 'RealAudio 1 / 2 14.4',
0x2003: 'RealAudio 1 / 2 28.8',
0x2004: 'RealAudio G2 / 8 Cook (low bitrate)',
0x2005: 'RealAudio 3 / 4 / 5 Music (DNET)',
0x2006: 'RealAudio 10 AAC (RAAC)',
0x2007: 'RealAudio 10 AAC+ (RACP)',
0x3313: 'makeAVIS',
0x4143: 'Divio MPEG-4 AAC audio',
0x434C: 'LEAD Speech',
0x564C: 'LEAD Vorbis',
0x674F: 'Ogg Vorbis (mode 1)',
0x6750: 'Ogg Vorbis (mode 2)',
0x6751: 'Ogg Vorbis (mode 3)',
0x676F: 'Ogg Vorbis (mode 1+)',
0x6770: 'Ogg Vorbis (mode 2+)',
0x6771: 'Ogg Vorbis (mode 3+)',
0x7A21: 'GSM-AMR (CBR, no SID)',
0x7A22: 'GSM-AMR (VBR, including SID)',
0xDFAC: 'DebugMode SonicFoundry Vegas FrameServer ACM Codec',
0xF1AC: 'Free Lossless Audio Codec FLAC',
0xFFFE: 'Extensible wave format',
0xFFFF: 'development'
}
FOURCC = {
'1978': 'A.M.Paredes predictor (LossLess)',
'2VUY': 'Optibase VideoPump 8-bit 4:2:2 Component YCbCr',
'3IV0': 'MPEG4-based codec 3ivx',
'3IV1': '3ivx v1',
'3IV2': '3ivx v2',
'3IVD': 'FFmpeg DivX ;-) (MS MPEG-4 v3)',
'3IVX': 'MPEG4-based codec 3ivx',
'8BPS': 'Apple QuickTime Planar RGB with Alpha-channel',
'AAS4': 'Autodesk Animator codec (RLE)',
'AASC': 'Autodesk Animator',
'ABYR': 'Kensington ABYR',
'ACTL': 'Streambox ACT-L2',
'ADV1': 'Loronix WaveCodec',
'ADVJ': 'Avid M-JPEG Avid Technology Also known as AVRn',
'AEIK': 'Intel Indeo Video 3.2',
'AEMI': 'Array VideoONE MPEG1-I Capture',
'AFLC': 'Autodesk Animator FLC',
'AFLI': 'Autodesk Animator FLI',
'AHDV': 'CineForm 10-bit Visually Perfect HD',
'AJPG': '22fps JPEG-based codec for digital cameras',
'AMPG': 'Array VideoONE MPEG',
'ANIM': 'Intel RDX (ANIM)',
'AP41': 'AngelPotion Definitive',
'AP42': 'AngelPotion Definitive',
'ASLC': 'AlparySoft Lossless Codec',
'ASV1': 'Asus Video v1',
'ASV2': 'Asus Video v2',
'ASVX': 'Asus Video 2.0 (audio)',
'ATM4': 'Ahead Nero Digital MPEG-4 Codec',
'AUR2': 'Aura 2 Codec - YUV 4:2:2',
'AURA': 'Aura 1 Codec - YUV 4:1:1',
'AV1X': 'Avid 1:1x (Quick Time)',
'AVC1': 'H.264 AVC',
'AVD1': 'Avid DV (Quick Time)',
'AVDJ': 'Avid Meridien JFIF with Alpha-channel',
'AVDN': 'Avid DNxHD (Quick Time)',
'AVDV': 'Avid DV',
'AVI1': 'MainConcept Motion JPEG Codec',
'AVI2': 'MainConcept Motion JPEG Codec',
'AVID': 'Avid Motion JPEG',
'AVIS': 'Wrapper for AviSynth',
'AVMP': 'Avid IMX (Quick Time)',
'AVR ': 'Avid ABVB/NuVista MJPEG with Alpha-channel',
'AVRN': 'Avid Motion JPEG',
'AVUI': 'Avid Meridien Uncompressed with Alpha-channel',
'AVUP': 'Avid 10bit Packed (Quick Time)',
'AYUV': '4:4:4 YUV (AYUV)',
'AZPR': 'Quicktime Apple Video',
'AZRP': 'Quicktime Apple Video',
'BGR ': 'Uncompressed BGR32 8:8:8:8',
'BGR(15)': 'Uncompressed BGR15 5:5:5',
'BGR(16)': 'Uncompressed BGR16 5:6:5',
'BGR(24)': 'Uncompressed BGR24 8:8:8',
'BHIV': 'BeHere iVideo',
'BINK': 'RAD Game Tools Bink Video',
'BIT ': 'BI_BITFIELDS (Raw RGB)',
'BITM': 'Microsoft H.261',
'BLOX': 'Jan Jezabek BLOX MPEG Codec',
'BLZ0': 'DivX for Blizzard Decoder Filter',
'BT20': 'Conexant Prosumer Video',
'BTCV': 'Conexant Composite Video Codec',
'BTVC': 'Conexant Composite Video',
'BW00': 'BergWave (Wavelet)',
'BW10': 'Data Translation Broadway MPEG Capture',
'BXBG': 'BOXX BGR',
'BXRG': 'BOXX RGB',
'BXY2': 'BOXX 10-bit YUV',
'BXYV': 'BOXX YUV',
'CC12': 'Intel YUV12',
'CDV5': 'Canopus SD50/DVHD',
'CDVC': 'Canopus DV',
'CDVH': 'Canopus SD50/DVHD',
'CFCC': 'Digital Processing Systems DPS Perception',
'CFHD': 'CineForm 10-bit Visually Perfect HD',
'CGDI': 'Microsoft Office 97 Camcorder Video',
'CHAM': 'Winnov Caviara Champagne',
'CJPG': 'Creative WebCam JPEG',
'CLJR': 'Cirrus Logic YUV 4 pixels',
'CLLC': 'Canopus LossLess',
'CLPL': 'YV12',
'CMYK': 'Common Data Format in Printing',
'COL0': 'FFmpeg DivX ;-) (MS MPEG-4 v3)',
'COL1': 'FFmpeg DivX ;-) (MS MPEG-4 v3)',
'CPLA': 'Weitek 4:2:0 YUV Planar',
'CRAM': 'Microsoft Video 1 (CRAM)',
'CSCD': 'RenderSoft CamStudio lossless Codec',
'CTRX': 'Citrix Scalable Video Codec',
'CUVC': 'Canopus HQ',
'CVID': 'Radius Cinepak',
'CWLT': 'Microsoft Color WLT DIB',
'CYUV': 'Creative Labs YUV',
'CYUY': 'ATI YUV',
'D261': 'H.261',
'D263': 'H.263',
'DAVC': 'Dicas MPEGable H.264/MPEG-4 AVC base profile codec',
'DC25': 'MainConcept ProDV Codec',
'DCAP': 'Pinnacle DV25 Codec',
'DCL1': 'Data Connection Conferencing Codec',
'DCT0': 'WniWni Codec',
'DFSC': 'DebugMode FrameServer VFW Codec',
'DIB ': 'Full Frames (Uncompressed)',
'DIV1': 'FFmpeg-4 V1 (hacked MS MPEG-4 V1)',
'DIV2': 'MS MPEG-4 V2',
'DIV3': 'DivX v3 MPEG-4 Low-Motion',
'DIV4': 'DivX v3 MPEG-4 Fast-Motion',
'DIV5': 'DIV5',
'DIV6': 'DivX MPEG-4',
'DIVX': 'DivX',
'DM4V': 'Dicas MPEGable MPEG-4',
'DMB1': 'Matrox Rainbow Runner hardware MJPEG',
'DMB2': 'Paradigm MJPEG',
'DMK2': 'ViewSonic V36 PDA Video',
'DP02': 'DynaPel MPEG-4',
'DPS0': 'DPS Reality Motion JPEG',
'DPSC': 'DPS PAR Motion JPEG',
'DRWX': 'Pinnacle DV25 Codec',
'DSVD': 'DSVD',
'DTMT': 'Media-100 Codec',
'DTNT': 'Media-100 Codec',
'DUCK': 'Duck True Motion 1.0',
'DV10': 'BlueFish444 (lossless RGBA, YUV 10-bit)',
'DV25': 'Matrox DVCPRO codec',
'DV50': 'Matrox DVCPRO50 codec',
'DVAN': 'DVAN',
'DVC ': 'Apple QuickTime DV (DVCPRO NTSC)',
'DVCP': 'Apple QuickTime DV (DVCPRO PAL)',
'DVCS': 'MainConcept DV Codec',
'DVE2': 'InSoft DVE-2 Videoconferencing',
'DVH1': 'Pinnacle DVHD100',
'DVHD': 'DV 1125 lines at 30.00 Hz or 1250 lines at 25.00 Hz',
'DVIS': 'VSYNC DualMoon Iris DV codec',
'DVL ': 'Radius SoftDV 16:9 NTSC',
'DVLP': 'Radius SoftDV 16:9 PAL',
'DVMA': 'Darim Vision DVMPEG',
'DVOR': 'BlueFish444 (lossless RGBA, YUV 10-bit)',
'DVPN': 'Apple QuickTime DV (DV NTSC)',
'DVPP': 'Apple QuickTime DV (DV PAL)',
'DVR1': 'TARGA2000 Codec',
'DVRS': 'VSYNC DualMoon Iris DV codec',
'DVSD': 'DV',
'DVSL': 'DV compressed in SD (SDL)',
'DVX1': 'DVX1000SP Video Decoder',
'DVX2': 'DVX2000S Video Decoder',
'DVX3': 'DVX3000S Video Decoder',
'DX50': 'DivX v5',
'DXGM': 'Electronic Arts Game Video codec',
'DXSB': 'DivX Subtitles Codec',
'DXT1': 'Microsoft DirectX Compressed Texture (DXT1)',
'DXT2': 'Microsoft DirectX Compressed Texture (DXT2)',
'DXT3': 'Microsoft DirectX Compressed Texture (DXT3)',
'DXT4': 'Microsoft DirectX Compressed Texture (DXT4)',
'DXT5': 'Microsoft DirectX Compressed Texture (DXT5)',
'DXTC': 'Microsoft DirectX Compressed Texture (DXTC)',
'DXTN': 'Microsoft DirectX Compressed Texture (DXTn)',
'EKQ0': 'Elsa EKQ0',
'ELK0': 'Elsa ELK0',
'EM2V': 'Etymonix MPEG-2 I-frame',
'EQK0': 'Elsa graphics card quick codec',
'ESCP': 'Eidos Escape',
'ETV1': 'eTreppid Video ETV1',
'ETV2': 'eTreppid Video ETV2',
'ETVC': 'eTreppid Video ETVC',
'FFDS': 'FFDShow supported',
'FFV1': 'FFDShow supported',
'FFVH': 'FFVH codec',
'FLIC': 'Autodesk FLI/FLC Animation',
'FLJP': 'D-Vision Field Encoded Motion JPEG',
'FLV1': 'FLV1 codec',
'FMJP': 'D-Vision fieldbased ISO MJPEG',
'FRLE': 'SoftLab-NSK Y16 + Alpha RLE',
'FRWA': 'SoftLab-Nsk Forward Motion JPEG w/ alpha channel',
'FRWD': 'SoftLab-Nsk Forward Motion JPEG',
'FRWT': 'SoftLab-NSK Vision Forward Motion JPEG with Alpha-channel',
'FRWU': 'SoftLab-NSK Vision Forward Uncompressed',
'FVF1': 'Iterated Systems Fractal Video Frame',
'FVFW': 'ff MPEG-4 based on XviD codec',
'GEPJ': 'White Pine (ex Paradigm Matrix) Motion JPEG Codec',
'GJPG': 'Grand Tech GT891x Codec',
'GLCC': 'GigaLink AV Capture codec',
'GLZW': 'Motion LZW',
'GPEG': 'Motion JPEG',
'GPJM': 'Pinnacle ReelTime MJPEG Codec',
'GREY': 'Apparently a duplicate of Y800',
'GWLT': 'Microsoft Greyscale WLT DIB',
'H260': 'H.260',
'H261': 'H.261',
'H262': 'H.262',
'H263': 'H.263',
'H264': 'H.264 AVC',
'H265': 'H.265',
'H266': 'H.266',
'H267': 'H.267',
'H268': 'H.268',
'H269': 'H.269',
'HD10': 'BlueFish444 (lossless RGBA, YUV 10-bit)',
'HDX4': 'Jomigo HDX4',
'HFYU': 'Huffman Lossless Codec',
'HMCR': 'Rendition Motion Compensation Format (HMCR)',
'HMRR': 'Rendition Motion Compensation Format (HMRR)',
'I263': 'Intel ITU H.263 Videoconferencing (i263)',
'I420': 'Intel Indeo 4',
'IAN ': 'Intel RDX',
'ICLB': 'InSoft CellB Videoconferencing',
'IDM0': 'IDM Motion Wavelets 2.0',
'IF09': 'Microsoft H.261',
'IGOR': 'Power DVD',
'IJPG': 'Intergraph JPEG',
'ILVC': 'Intel Layered Video',
'ILVR': 'ITU-T H.263+',
'IMC1': 'IMC1',
'IMC2': 'IMC2',
'IMC3': 'IMC3',
'IMC4': 'IMC4',
'IMJG': 'Accom SphereOUS MJPEG with Alpha-channel',
'IPDV': 'I-O Data Device Giga AVI DV Codec',
'IPJ2': 'Image Power JPEG2000',
'IR21': 'Intel Indeo 2.1',
'IRAW': 'Intel YUV Uncompressed',
'IUYV': 'Interlaced version of UYVY (line order 0,2,4 then 1,3,5 etc)',
'IV30': 'Ligos Indeo 3.0',
'IV31': 'Ligos Indeo 3.1',
'IV32': 'Ligos Indeo 3.2',
'IV33': 'Ligos Indeo 3.3',
'IV34': 'Ligos Indeo 3.4',
'IV35': 'Ligos Indeo 3.5',
'IV36': 'Ligos Indeo 3.6',
'IV37': 'Ligos Indeo 3.7',
'IV38': 'Ligos Indeo 3.8',
'IV39': 'Ligos Indeo 3.9',
'IV40': 'Ligos Indeo Interactive 4.0',
'IV41': 'Ligos Indeo Interactive 4.1',
'IV42': 'Ligos Indeo Interactive 4.2',
'IV43': 'Ligos Indeo Interactive 4.3',
'IV44': 'Ligos Indeo Interactive 4.4',
'IV45': 'Ligos Indeo Interactive 4.5',
'IV46': 'Ligos Indeo Interactive 4.6',
'IV47': 'Ligos Indeo Interactive 4.7',
'IV48': 'Ligos Indeo Interactive 4.8',
'IV49': 'Ligos Indeo Interactive 4.9',
'IV50': 'Ligos Indeo Interactive 5.0',
'IY41': 'Interlaced version of Y41P (line order 0,2,4,...,1,3,5...)',
'IYU1': '12 bit format used in mode 2 of the IEEE 1394 Digital Camera 1.04 spec',
'IYU2': '24 bit format used in mode 2 of the IEEE 1394 Digital Camera 1.04 spec',
'IYUV': 'Intel Indeo iYUV 4:2:0',
'JBYR': 'Kensington JBYR',
'JFIF': 'Motion JPEG (FFmpeg)',
'JPEG': 'Still Image JPEG DIB',
'JPG ': 'JPEG compressed',
'JPGL': 'Webcam JPEG Light',
'KMVC': 'Karl Morton\'s Video Codec',
'KPCD': 'Kodak Photo CD',
'L261': 'Lead Technologies H.261',
'L263': 'Lead Technologies H.263',
'LAGS': 'Lagarith LossLess',
'LBYR': 'Creative WebCam codec',
'LCMW': 'Lead Technologies Motion CMW Codec',
'LCW2': 'LEADTools MCMW 9Motion Wavelet)',
'LEAD': 'LEAD Video Codec',
'LGRY': 'Lead Technologies Grayscale Image',
'LJ2K': 'LEADTools JPEG2000',
'LJPG': 'LEAD MJPEG Codec',
'LMP2': 'LEADTools MPEG2',
'LOCO': 'LOCO Lossless Codec',
'LSCR': 'LEAD Screen Capture',
'LSVM': 'Vianet Lighting Strike Vmail (Streaming)',
'LZO1': 'LZO compressed (lossless codec)',
'M261': 'Microsoft H.261',
'M263': 'Microsoft H.263',
'M4CC': 'ESS MPEG4 Divio codec',
'M4S2': 'Microsoft MPEG-4 (M4S2)',
'MC12': 'ATI Motion Compensation Format (MC12)',
'MC24': 'MainConcept Motion JPEG Codec',
'MCAM': 'ATI Motion Compensation Format (MCAM)',
'MCZM': 'Theory MicroCosm Lossless 64bit RGB with Alpha-channel',
'MDVD': 'Alex MicroDVD Video (hacked MS MPEG-4)',
'MDVF': 'Pinnacle DV/DV50/DVHD100',
'MHFY': 'A.M.Paredes mhuffyYUV (LossLess)',
'MJ2C': 'Morgan Multimedia Motion JPEG2000',
'MJPA': 'Pinnacle ReelTime MJPG hardware codec',
'MJPB': 'Motion JPEG codec',
'MJPG': 'Motion JPEG DIB',
'MJPX': 'Pegasus PICVideo Motion JPEG',
'MMES': 'Matrox MPEG-2 I-frame',
'MNVD': 'MindBend MindVid LossLess',
'MP2A': 'MPEG-2 Audio',
'MP2T': 'MPEG-2 Transport Stream',
'MP2V': 'MPEG-2 Video',
'MP41': 'Microsoft MPEG-4 V1 (enhansed H263)',
'MP42': 'Microsoft MPEG-4 (low-motion)',
'MP43': 'Microsoft MPEG-4 (fast-motion)',
'MP4A': 'MPEG-4 Audio',
'MP4S': 'Microsoft MPEG-4 (MP4S)',
'MP4T': 'MPEG-4 Transport Stream',
'MP4V': 'Apple QuickTime MPEG-4 native',
'MPEG': 'MPEG-1',
'MPG1': 'FFmpeg-1',
'MPG2': 'FFmpeg-1',
'MPG3': 'Same as Low motion DivX MPEG-4',
'MPG4': 'Microsoft MPEG-4 Video High Speed Compressor',
'MPGI': 'Sigma Designs MPEG',
'MPNG': 'Motion PNG codec',
'MRCA': 'Martin Regen Codec',
'MRLE': 'Run Length Encoding',
'MSS1': 'Windows Screen Video',
'MSS2': 'Windows Media 9',
'MSUC': 'MSU LossLess',
'MSVC': 'Microsoft Video 1',
'MSZH': 'Lossless codec (ZIP compression)',
'MTGA': 'Motion TGA images (24, 32 bpp)',
'MTX1': 'Matrox MTX1',
'MTX2': 'Matrox MTX2',
'MTX3': 'Matrox MTX3',
'MTX4': 'Matrox MTX4',
'MTX5': 'Matrox MTX5',
'MTX6': 'Matrox MTX6',
'MTX7': 'Matrox MTX7',
'MTX8': 'Matrox MTX8',
'MTX9': 'Matrox MTX9',
'MV12': 'MV12',
'MVI1': 'Motion Pixels MVI',
'MVI2': 'Motion Pixels MVI',
'MWV1': 'Aware Motion Wavelets',
'MYUV': 'Media-100 844/X Uncompressed',
'NAVI': 'nAVI',
'NDIG': 'Ahead Nero Digital MPEG-4 Codec',
'NHVU': 'NVidia Texture Format (GEForce 3)',
'NO16': 'Theory None16 64bit uncompressed RAW',
'NT00': 'NewTek LigtWave HDTV YUV with Alpha-channel',
'NTN1': 'Nogatech Video Compression 1',
'NTN2': 'Nogatech Video Compression 2 (GrabBee hardware coder)',
'NUV1': 'NuppelVideo',
'NV12': '8-bit Y plane followed by an interleaved U/V plane with 2x2 subsampling',
'NV21': 'As NV12 with U and V reversed in the interleaved plane',
'NVDS': 'nVidia Texture Format',
'NVHS': 'NVidia Texture Format (GEForce 3)',
'NVS0': 'nVidia GeForce Texture',
'NVS1': 'nVidia GeForce Texture',
'NVS2': 'nVidia GeForce Texture',
'NVS3': 'nVidia GeForce Texture',
'NVS4': 'nVidia GeForce Texture',
'NVS5': 'nVidia GeForce Texture',
'NVT0': 'nVidia GeForce Texture',
'NVT1': 'nVidia GeForce Texture',
'NVT2': 'nVidia GeForce Texture',
'NVT3': 'nVidia GeForce Texture',
'NVT4': 'nVidia GeForce Texture',
'NVT5': 'nVidia GeForce Texture',
'PDVC': 'I-O Data Device Digital Video Capture DV codec',
'PGVV': 'Radius Video Vision',
'PHMO': 'IBM Photomotion',
'PIM1': 'Pegasus Imaging',
'PIM2': 'Pegasus Imaging',
'PIMJ': 'Pegasus Imaging Lossless JPEG',
'PIXL': 'MiroVideo XL (Motion JPEG)',
'PNG ': 'Apple PNG',
'PNG1': 'Corecodec.org CorePNG Codec',
'PVEZ': 'Horizons Technology PowerEZ',
'PVMM': 'PacketVideo Corporation MPEG-4',
'PVW2': 'Pegasus Imaging Wavelet Compression',
'PVWV': 'Pegasus Imaging Wavelet 2000',
'PXLT': 'Apple Pixlet (Wavelet)',
'Q1.0': 'Q-Team QPEG 1.0 (www.q-team.de)',
'Q1.1': 'Q-Team QPEG 1.1 (www.q-team.de)',
'QDGX': 'Apple QuickDraw GX',
'QPEG': 'Q-Team QPEG 1.0',
'QPEQ': 'Q-Team QPEG 1.1',
'R210': 'BlackMagic YUV (Quick Time)',
'R411': 'Radius DV NTSC YUV',
'R420': 'Radius DV PAL YUV',
'RAVI': 'GroupTRON ReferenceAVI codec (dummy for MPEG compressor)',
'RAV_': 'GroupTRON ReferenceAVI codec (dummy for MPEG compressor)',
'RAW ': 'Full Frames (Uncompressed)',
'RGB ': 'Full Frames (Uncompressed)',
'RGB(15)': 'Uncompressed RGB15 5:5:5',
'RGB(16)': 'Uncompressed RGB16 5:6:5',
'RGB(24)': 'Uncompressed RGB24 8:8:8',
'RGB1': 'Uncompressed RGB332 3:3:2',
'RGBA': 'Raw RGB with alpha',
'RGBO': 'Uncompressed RGB555 5:5:5',
'RGBP': 'Uncompressed RGB565 5:6:5',
'RGBQ': 'Uncompressed RGB555X 5:5:5 BE',
'RGBR': 'Uncompressed RGB565X 5:6:5 BE',
'RGBT': 'Computer Concepts 32-bit support',
'RL4 ': 'RLE 4bpp RGB',
'RL8 ': 'RLE 8bpp RGB',
'RLE ': 'Microsoft Run Length Encoder',
'RLE4': 'Run Length Encoded 4',
'RLE8': 'Run Length Encoded 8',
'RMP4': 'REALmagic MPEG-4 Video Codec',
'ROQV': 'Id RoQ File Video Decoder',
'RPZA': 'Apple Video 16 bit "road pizza"',
'RT21': 'Intel Real Time Video 2.1',
'RTV0': 'NewTek VideoToaster',
'RUD0': 'Rududu video codec',
'RV10': 'RealVideo codec',
'RV13': 'RealVideo codec',
'RV20': 'RealVideo G2',
'RV30': 'RealVideo 8',
'RV40': 'RealVideo 9',
'RVX ': 'Intel RDX (RVX )',
'S263': 'Sorenson Vision H.263',
'S422': 'Tekram VideoCap C210 YUV 4:2:2',
'SAMR': 'Adaptive Multi-Rate (AMR) audio codec',
'SAN3': 'MPEG-4 codec (direct copy of DivX 3.11a)',
'SDCC': 'Sun Communication Digital Camera Codec',
'SEDG': 'Samsung MPEG-4 codec',
'SFMC': 'CrystalNet Surface Fitting Method',
'SHR0': 'BitJazz SheerVideo',
'SHR1': 'BitJazz SheerVideo',
'SHR2': 'BitJazz SheerVideo',
'SHR3': 'BitJazz SheerVideo',
'SHR4': 'BitJazz SheerVideo',
'SHR5': 'BitJazz SheerVideo',
'SHR6': 'BitJazz SheerVideo',
'SHR7': 'BitJazz SheerVideo',
'SJPG': 'CUseeMe Networks Codec',
'SL25': 'SoftLab-NSK DVCPRO',
'SL50': 'SoftLab-NSK DVCPRO50',
'SLDV': 'SoftLab-NSK Forward DV Draw codec',
'SLIF': 'SoftLab-NSK MPEG2 I-frames',
'SLMJ': 'SoftLab-NSK Forward MJPEG',
'SMC ': 'Apple Graphics (SMC) codec (256 color)',
'SMSC': 'Radius SMSC',
'SMSD': 'Radius SMSD',
'SMSV': 'WorldConnect Wavelet Video',
'SNOW': 'SNOW codec',
'SP40': 'SunPlus YUV',
'SP44': 'SunPlus Aiptek MegaCam Codec',
'SP53': 'SunPlus Aiptek MegaCam Codec',
'SP54': 'SunPlus Aiptek MegaCam Codec',
'SP55': 'SunPlus Aiptek MegaCam Codec',
'SP56': 'SunPlus Aiptek MegaCam Codec',
'SP57': 'SunPlus Aiptek MegaCam Codec',
'SP58': 'SunPlus Aiptek MegaCam Codec',
'SPIG': 'Radius Spigot',
'SPLC': 'Splash Studios ACM Audio Codec',
'SPRK': 'Sorenson Spark',
'SQZ2': 'Microsoft VXTreme Video Codec V2',
'STVA': 'ST CMOS Imager Data (Bayer)',
'STVB': 'ST CMOS Imager Data (Nudged Bayer)',
'STVC': 'ST CMOS Imager Data (Bunched)',
'STVX': 'ST CMOS Imager Data (Extended CODEC Data Format)',
'STVY': 'ST CMOS Imager Data (Extended CODEC Data Format with Correction Data)',
'SV10': 'Sorenson Video R1',
'SVQ1': 'Sorenson Video R3',
'SVQ3': 'Sorenson Video 3 (Apple Quicktime 5)',
'SWC1': 'MainConcept Motion JPEG Codec',
'T420': 'Toshiba YUV 4:2:0',
'TGA ': 'Apple TGA (with Alpha-channel)',
'THEO': 'FFVFW Supported Codec',
'TIFF': 'Apple TIFF (with Alpha-channel)',
'TIM2': 'Pinnacle RAL DVI',
'TLMS': 'TeraLogic Motion Intraframe Codec (TLMS)',
'TLST': 'TeraLogic Motion Intraframe Codec (TLST)',
'TM20': 'Duck TrueMotion 2.0',
'TM2A': 'Duck TrueMotion Archiver 2.0',
'TM2X': 'Duck TrueMotion 2X',
'TMIC': 'TeraLogic Motion Intraframe Codec (TMIC)',
'TMOT': 'Horizons Technology TrueMotion S',
'TR20': 'Duck TrueMotion RealTime 2.0',
'TRLE': 'Akula Alpha Pro Custom AVI (LossLess)',
'TSCC': 'TechSmith Screen Capture Codec',
'TV10': 'Tecomac Low-Bit Rate Codec',
'TVJP': 'TrueVision Field Encoded Motion JPEG',
'TVMJ': 'Truevision TARGA MJPEG Hardware Codec',
'TY0N': 'Trident TY0N',
'TY2C': 'Trident TY2C',
'TY2N': 'Trident TY2N',
'U263': 'UB Video StreamForce H.263',
'U<Y ': 'Discreet UC YUV 4:2:2:4 10 bit',
'U<YA': 'Discreet UC YUV 4:2:2:4 10 bit (with Alpha-channel)',
'UCOD': 'eMajix.com ClearVideo',
'ULTI': 'IBM Ultimotion',
'UMP4': 'UB Video MPEG 4',
'UYNV': 'UYVY',
'UYVP': 'YCbCr 4:2:2',
'UYVU': 'SoftLab-NSK Forward YUV codec',
'UYVY': 'UYVY 4:2:2 byte ordering',
'V210': 'Optibase VideoPump 10-bit 4:2:2 Component YCbCr',
'V261': 'Lucent VX2000S',
'V422': '24 bit YUV 4:2:2 Format',
'V655': '16 bit YUV 4:2:2 Format',
'VBLE': 'MarcFD VBLE Lossless Codec',
'VCR1': 'ATI VCR 1.0',
'VCR2': 'ATI VCR 2.0',
'VCR3': 'ATI VCR 3.0',
'VCR4': 'ATI VCR 4.0',
'VCR5': 'ATI VCR 5.0',
'VCR6': 'ATI VCR 6.0',
'VCR7': 'ATI VCR 7.0',
'VCR8': 'ATI VCR 8.0',
'VCR9': 'ATI VCR 9.0',
'VDCT': 'Video Maker Pro DIB',
'VDOM': 'VDOnet VDOWave',
'VDOW': 'VDOnet VDOLive (H.263)',
'VDST': 'VirtualDub remote frameclient ICM driver',
'VDTZ': 'Darim Vison VideoTizer YUV',
'VGPX': 'VGPixel Codec',
'VIDM': 'DivX 5.0 Pro Supported Codec',
'VIDS': 'YUV 4:2:2 CCIR 601 for V422',
'VIFP': 'VIFP',
'VIV1': 'Vivo H.263',
'VIV2': 'Vivo H.263',
'VIVO': 'Vivo H.263 v2.00',
'VIXL': 'Miro Video XL',
'VLV1': 'Videologic VLCAP.DRV',
'VP30': 'On2 VP3.0',
'VP31': 'On2 VP3.1',
'VP40': 'On2 TrueCast VP4',
'VP50': 'On2 TrueCast VP5',
'VP60': 'On2 TrueCast VP6',
'VP61': 'On2 TrueCast VP6.1',
'VP62': 'On2 TrueCast VP6.2',
'VP70': 'On2 TrueMotion VP7',
'VQC1': 'Vector-quantised codec 1',
'VQC2': 'Vector-quantised codec 2',
'VR21': 'BlackMagic YUV (Quick Time)',
'VSSH': 'Vanguard VSS H.264',
'VSSV': 'Vanguard Software Solutions Video Codec',
'VSSW': 'Vanguard VSS H.264',
'VTLP': 'Alaris VideoGramPixel Codec',
'VX1K': 'VX1000S Video Codec',
'VX2K': 'VX2000S Video Codec',
'VXSP': 'VX1000SP Video Codec',
'VYU9': 'ATI Technologies YUV',
'VYUY': 'ATI Packed YUV Data',
'WBVC': 'Winbond W9960',
'WHAM': 'Microsoft Video 1 (WHAM)',
'WINX': 'Winnov Software Compression',
'WJPG': 'AverMedia Winbond JPEG',
'WMV1': 'Windows Media Video V7',
'WMV2': 'Windows Media Video V8',
'WMV3': 'Windows Media Video V9',
'WMVA': 'WMVA codec',
'WMVP': 'Windows Media Video V9',
'WNIX': 'WniWni Codec',
'WNV1': 'Winnov Hardware Compression',
'WNVA': 'Winnov hw compress',
'WRLE': 'Apple QuickTime BMP Codec',
'WRPR': 'VideoTools VideoServer Client Codec',
'WV1F': 'WV1F codec',
'WVLT': 'IllusionHope Wavelet 9/7',
'WVP2': 'WVP2 codec',
'X263': 'Xirlink H.263',
'X264': 'XiWave GNU GPL x264 MPEG-4 Codec',
'XLV0': 'NetXL Video Decoder',
'XMPG': 'Xing MPEG (I-Frame only)',
'XVID': 'XviD MPEG-4',
'XVIX': 'Based on XviD MPEG-4 codec',
'XWV0': 'XiWave Video Codec',
'XWV1': 'XiWave Video Codec',
'XWV2': 'XiWave Video Codec',
'XWV3': 'XiWave Video Codec (Xi-3 Video)',
'XWV4': 'XiWave Video Codec',
'XWV5': 'XiWave Video Codec',
'XWV6': 'XiWave Video Codec',
'XWV7': 'XiWave Video Codec',
'XWV8': 'XiWave Video Codec',
'XWV9': 'XiWave Video Codec',
'XXAN': 'XXAN',
'XYZP': 'Extended PAL format XYZ palette',
'Y211': 'YUV 2:1:1 Packed',
'Y216': 'Pinnacle TARGA CineWave YUV (Quick Time)',
'Y411': 'YUV 4:1:1 Packed',
'Y41B': 'YUV 4:1:1 Planar',
'Y41P': 'PC1 4:1:1',
'Y41T': 'PC1 4:1:1 with transparency',
'Y422': 'Y422',
'Y42B': 'YUV 4:2:2 Planar',
'Y42T': 'PCI 4:2:2 with transparency',
'Y444': 'IYU2',
'Y8 ': 'Grayscale video',
'Y800': 'Simple grayscale video',
'YC12': 'Intel YUV12 Codec',
'YMPG': 'YMPEG Alpha',
'YU12': 'ATI YV12 4:2:0 Planar',
'YU92': 'Intel - YUV',
'YUNV': 'YUNV',
'YUV2': 'Apple Component Video (YUV 4:2:2)',
'YUV8': 'Winnov Caviar YUV8',
'YUV9': 'Intel YUV9',
'YUVP': 'YCbCr 4:2:2',
'YUY2': 'Uncompressed YUV 4:2:2',
'YUYV': 'Canopus YUV',
'YV12': 'YVU12 Planar',
'YV16': 'Elecard YUV 4:2:2 Planar',
'YV92': 'Intel Smart Video Recorder YVU9',
'YVU9': 'Intel YVU9 Planar',
'YVYU': 'YVYU 4:2:2 byte ordering',
'ZLIB': 'ZLIB',
'ZPEG': 'Metheus Video Zipper',
'ZYGO': 'ZyGo Video Codec'
}
# make it fool prove
for code, value in FOURCC.items():
if not code.upper() in FOURCC:
FOURCC[code.upper()] = value
if code.endswith(' '):
FOURCC[code.strip().upper()] = value
libs/enzyme/infos.py
+26
View File
@@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__title__ = 'enzyme'
__description__ = 'Video metadata parser'
__version__ = '0.1'
__author__ = 'Antoine Bertin'
__email__ = 'diaoulael@gmail.com'
__license__ = 'GPLv3'
libs/enzyme/language.py
+538
View File
@@ -0,0 +1,538 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import re
__all__ = ['resolve']
def resolve(code):
"""
Transform the given (2- or 3-letter) language code to a human readable
language name. The return value is a 2-tuple containing the given
language code and the language name. If the language code cannot be
resolved, name will be 'Unknown (<code>)'.
"""
if not code:
return None, None
if not isinstance(code, basestring):
raise ValueError('Invalid language code specified by parser')
# Take up to 3 letters from the code.
code = re.split(r'[^a-z]', code.lower())[0][:3]
for spec in codes:
if code in spec[:-1]:
return code, spec[-1]
return code, u'Unknown (%s)' % code
# Parsed from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
codes = (
('aar', 'aa', u'Afar'),
('abk', 'ab', u'Abkhazian'),
('ace', u'Achinese'),
('ach', u'Acoli'),
('ada', u'Adangme'),
('ady', u'Adyghe'),
('afa', u'Afro-Asiatic '),
('afh', u'Afrihili'),
('afr', 'af', u'Afrikaans'),
('ain', u'Ainu'),
('aka', 'ak', u'Akan'),
('akk', u'Akkadian'),
('alb', 'sq', u'Albanian'),
('ale', u'Aleut'),
('alg', u'Algonquian languages'),
('alt', u'Southern Altai'),
('amh', 'am', u'Amharic'),
('ang', u'English, Old '),
('anp', u'Angika'),
('apa', u'Apache languages'),
('ara', 'ar', u'Arabic'),
('arc', u'Official Aramaic '),
('arg', 'an', u'Aragonese'),
('arm', 'hy', u'Armenian'),
('arn', u'Mapudungun'),
('arp', u'Arapaho'),
('art', u'Artificial '),
('arw', u'Arawak'),
('asm', 'as', u'Assamese'),
('ast', u'Asturian'),
('ath', u'Athapascan languages'),
('aus', u'Australian languages'),
('ava', 'av', u'Avaric'),
('ave', 'ae', u'Avestan'),
('awa', u'Awadhi'),
('aym', 'ay', u'Aymara'),
('aze', 'az', u'Azerbaijani'),
('bad', u'Banda languages'),
('bai', u'Bamileke languages'),
('bak', 'ba', u'Bashkir'),
('bal', u'Baluchi'),
('bam', 'bm', u'Bambara'),
('ban', u'Balinese'),
('baq', 'eu', u'Basque'),
('bas', u'Basa'),
('bat', u'Baltic '),
('bej', u'Beja'),
('bel', 'be', u'Belarusian'),
('bem', u'Bemba'),
('ben', 'bn', u'Bengali'),
('ber', u'Berber '),
('bho', u'Bhojpuri'),
('bih', 'bh', u'Bihari'),
('bik', u'Bikol'),
('bin', u'Bini'),
('bis', 'bi', u'Bislama'),
('bla', u'Siksika'),
('bnt', u'Bantu '),
('bos', 'bs', u'Bosnian'),
('bra', u'Braj'),
('bre', 'br', u'Breton'),
('btk', u'Batak languages'),
('bua', u'Buriat'),
('bug', u'Buginese'),
('bul', 'bg', u'Bulgarian'),
('bur', 'my', u'Burmese'),
('byn', u'Blin'),
('cad', u'Caddo'),
('cai', u'Central American Indian '),
('car', u'Galibi Carib'),
('cat', 'ca', u'Catalan'),
('cau', u'Caucasian '),
('ceb', u'Cebuano'),
('cel', u'Celtic '),
('cha', 'ch', u'Chamorro'),
('chb', u'Chibcha'),
('che', 'ce', u'Chechen'),
('chg', u'Chagatai'),
('chi', 'zh', u'Chinese'),
('chk', u'Chuukese'),
('chm', u'Mari'),
('chn', u'Chinook jargon'),
('cho', u'Choctaw'),
('chp', u'Chipewyan'),
('chr', u'Cherokee'),
('chu', 'cu', u'Church Slavic'),
('chv', 'cv', u'Chuvash'),
('chy', u'Cheyenne'),
('cmc', u'Chamic languages'),
('cop', u'Coptic'),
('cor', 'kw', u'Cornish'),
('cos', 'co', u'Corsican'),
('cpe', u'Creoles and pidgins, English based '),
('cpf', u'Creoles and pidgins, French-based '),
('cpp', u'Creoles and pidgins, Portuguese-based '),
('cre', 'cr', u'Cree'),
('crh', u'Crimean Tatar'),
('crp', u'Creoles and pidgins '),
('csb', u'Kashubian'),
('cus', u'Cushitic '),
('cze', 'cs', u'Czech'),
('dak', u'Dakota'),
('dan', 'da', u'Danish'),
('dar', u'Dargwa'),
('day', u'Land Dayak languages'),
('del', u'Delaware'),
('den', u'Slave '),
('dgr', u'Dogrib'),
('din', u'Dinka'),
('div', 'dv', u'Divehi'),
('doi', u'Dogri'),
('dra', u'Dravidian '),
('dsb', u'Lower Sorbian'),
('dua', u'Duala'),
('dum', u'Dutch, Middle '),
('dut', 'nl', u'Dutch'),
('dyu', u'Dyula'),
('dzo', 'dz', u'Dzongkha'),
('efi', u'Efik'),
('egy', u'Egyptian '),
('eka', u'Ekajuk'),
('elx', u'Elamite'),
('eng', 'en', u'English'),
('enm', u'English, Middle '),
('epo', 'eo', u'Esperanto'),
('est', 'et', u'Estonian'),
('ewe', 'ee', u'Ewe'),
('ewo', u'Ewondo'),
('fan', u'Fang'),
('fao', 'fo', u'Faroese'),
('fat', u'Fanti'),
('fij', 'fj', u'Fijian'),
('fil', u'Filipino'),
('fin', 'fi', u'Finnish'),
('fiu', u'Finno-Ugrian '),
('fon', u'Fon'),
('fre', 'fr', u'French'),
('frm', u'French, Middle '),
('fro', u'French, Old '),
('frr', u'Northern Frisian'),
('frs', u'Eastern Frisian'),
('fry', 'fy', u'Western Frisian'),
('ful', 'ff', u'Fulah'),
('fur', u'Friulian'),
('gaa', u'Ga'),
('gay', u'Gayo'),
('gba', u'Gbaya'),
('gem', u'Germanic '),
('geo', 'ka', u'Georgian'),
('ger', 'de', u'German'),
('gez', u'Geez'),
('gil', u'Gilbertese'),
('gla', 'gd', u'Gaelic'),
('gle', 'ga', u'Irish'),
('glg', 'gl', u'Galician'),
('glv', 'gv', u'Manx'),
('gmh', u'German, Middle High '),
('goh', u'German, Old High '),
('gon', u'Gondi'),
('gor', u'Gorontalo'),
('got', u'Gothic'),
('grb', u'Grebo'),
('grc', u'Greek, Ancient '),
('gre', 'el', u'Greek, Modern '),
('grn', 'gn', u'Guarani'),
('gsw', u'Swiss German'),
('guj', 'gu', u'Gujarati'),
('gwi', u"Gwich'in"),
('hai', u'Haida'),
('hat', 'ht', u'Haitian'),
('hau', 'ha', u'Hausa'),
('haw', u'Hawaiian'),
('heb', 'he', u'Hebrew'),
('her', 'hz', u'Herero'),
('hil', u'Hiligaynon'),
('him', u'Himachali'),
('hin', 'hi', u'Hindi'),
('hit', u'Hittite'),
('hmn', u'Hmong'),
('hmo', 'ho', u'Hiri Motu'),
('hsb', u'Upper Sorbian'),
('hun', 'hu', u'Hungarian'),
('hup', u'Hupa'),
('iba', u'Iban'),
('ibo', 'ig', u'Igbo'),
('ice', 'is', u'Icelandic'),
('ido', 'io', u'Ido'),
('iii', 'ii', u'Sichuan Yi'),
('ijo', u'Ijo languages'),
('iku', 'iu', u'Inuktitut'),
('ile', 'ie', u'Interlingue'),
('ilo', u'Iloko'),
('ina', 'ia', u'Interlingua '),
('inc', u'Indic '),
('ind', 'id', u'Indonesian'),
('ine', u'Indo-European '),
('inh', u'Ingush'),
('ipk', 'ik', u'Inupiaq'),
('ira', u'Iranian '),
('iro', u'Iroquoian languages'),
('ita', 'it', u'Italian'),
('jav', 'jv', u'Javanese'),
('jbo', u'Lojban'),
('jpn', 'ja', u'Japanese'),
('jpr', u'Judeo-Persian'),
('jrb', u'Judeo-Arabic'),
('kaa', u'Kara-Kalpak'),
('kab', u'Kabyle'),
('kac', u'Kachin'),
('kal', 'kl', u'Kalaallisut'),
('kam', u'Kamba'),
('kan', 'kn', u'Kannada'),
('kar', u'Karen languages'),
('kas', 'ks', u'Kashmiri'),
('kau', 'kr', u'Kanuri'),
('kaw', u'Kawi'),
('kaz', 'kk', u'Kazakh'),
('kbd', u'Kabardian'),
('kha', u'Khasi'),
('khi', u'Khoisan '),
('khm', 'km', u'Central Khmer'),
('kho', u'Khotanese'),
('kik', 'ki', u'Kikuyu'),
('kin', 'rw', u'Kinyarwanda'),
('kir', 'ky', u'Kirghiz'),
('kmb', u'Kimbundu'),
('kok', u'Konkani'),
('kom', 'kv', u'Komi'),
('kon', 'kg', u'Kongo'),
('kor', 'ko', u'Korean'),
('kos', u'Kosraean'),
('kpe', u'Kpelle'),
('krc', u'Karachay-Balkar'),
('krl', u'Karelian'),
('kro', u'Kru languages'),
('kru', u'Kurukh'),
('kua', 'kj', u'Kuanyama'),
('kum', u'Kumyk'),
('kur', 'ku', u'Kurdish'),
('kut', u'Kutenai'),
('lad', u'Ladino'),
('lah', u'Lahnda'),
('lam', u'Lamba'),
('lao', 'lo', u'Lao'),
('lat', 'la', u'Latin'),
('lav', 'lv', u'Latvian'),
('lez', u'Lezghian'),
('lim', 'li', u'Limburgan'),
('lin', 'ln', u'Lingala'),
('lit', 'lt', u'Lithuanian'),
('lol', u'Mongo'),
('loz', u'Lozi'),
('ltz', 'lb', u'Luxembourgish'),
('lua', u'Luba-Lulua'),
('lub', 'lu', u'Luba-Katanga'),
('lug', 'lg', u'Ganda'),
('lui', u'Luiseno'),
('lun', u'Lunda'),
('luo', u'Luo '),
('lus', u'Lushai'),
('mac', 'mk', u'Macedonian'),
('mad', u'Madurese'),
('mag', u'Magahi'),
('mah', 'mh', u'Marshallese'),
('mai', u'Maithili'),
('mak', u'Makasar'),
('mal', 'ml', u'Malayalam'),
('man', u'Mandingo'),
('mao', 'mi', u'Maori'),
('map', u'Austronesian '),
('mar', 'mr', u'Marathi'),
('mas', u'Masai'),
('may', 'ms', u'Malay'),
('mdf', u'Moksha'),
('mdr', u'Mandar'),
('men', u'Mende'),
('mga', u'Irish, Middle '),
('mic', u"Mi'kmaq"),
('min', u'Minangkabau'),
('mis', u'Uncoded languages'),
('mkh', u'Mon-Khmer '),
('mlg', 'mg', u'Malagasy'),
('mlt', 'mt', u'Maltese'),
('mnc', u'Manchu'),
('mni', u'Manipuri'),
('mno', u'Manobo languages'),
('moh', u'Mohawk'),
('mol', 'mo', u'Moldavian'),
('mon', 'mn', u'Mongolian'),
('mos', u'Mossi'),
('mul', u'Multiple languages'),
('mun', u'Munda languages'),
('mus', u'Creek'),
('mwl', u'Mirandese'),
('mwr', u'Marwari'),
('myn', u'Mayan languages'),
('myv', u'Erzya'),
('nah', u'Nahuatl languages'),
('nai', u'North American Indian'),
('nap', u'Neapolitan'),
('nau', 'na', u'Nauru'),
('nav', 'nv', u'Navajo'),
('nbl', 'nr', u'Ndebele, South'),
('nde', 'nd', u'Ndebele, North'),
('ndo', 'ng', u'Ndonga'),
('nds', u'Low German'),
('nep', 'ne', u'Nepali'),
('new', u'Nepal Bhasa'),
('nia', u'Nias'),
('nic', u'Niger-Kordofanian '),
('niu', u'Niuean'),
('nno', 'nn', u'Norwegian Nynorsk'),
('nob', 'nb', u'Bokm\xe5l, Norwegian'),
('nog', u'Nogai'),
('non', u'Norse, Old'),
('nor', 'no', u'Norwegian'),
('nqo', u"N'Ko"),
('nso', u'Pedi'),
('nub', u'Nubian languages'),
('nwc', u'Classical Newari'),
('nya', 'ny', u'Chichewa'),
('nym', u'Nyamwezi'),
('nyn', u'Nyankole'),
('nyo', u'Nyoro'),
('nzi', u'Nzima'),
('oci', 'oc', u'Occitan '),
('oji', 'oj', u'Ojibwa'),
('ori', 'or', u'Oriya'),
('orm', 'om', u'Oromo'),
('osa', u'Osage'),
('oss', 'os', u'Ossetian'),
('ota', u'Turkish, Ottoman '),
('oto', u'Otomian languages'),
('paa', u'Papuan '),
('pag', u'Pangasinan'),
('pal', u'Pahlavi'),
('pam', u'Pampanga'),
('pan', 'pa', u'Panjabi'),
('pap', u'Papiamento'),
('pau', u'Palauan'),
('peo', u'Persian, Old '),
('per', 'fa', u'Persian'),
('phi', u'Philippine '),
('phn', u'Phoenician'),
('pli', 'pi', u'Pali'),
('pol', 'pl', u'Polish'),
('pon', u'Pohnpeian'),
('por', 'pt', u'Portuguese'),
('pra', u'Prakrit languages'),
('pro', u'Proven\xe7al, Old '),
('pus', 'ps', u'Pushto'),
('qaa-qtz', u'Reserved for local use'),
('que', 'qu', u'Quechua'),
('raj', u'Rajasthani'),
('rap', u'Rapanui'),
('rar', u'Rarotongan'),
('roa', u'Romance '),
('roh', 'rm', u'Romansh'),
('rom', u'Romany'),
('rum', 'ro', u'Romanian'),
('run', 'rn', u'Rundi'),
('rup', u'Aromanian'),
('rus', 'ru', u'Russian'),
('sad', u'Sandawe'),
('sag', 'sg', u'Sango'),
('sah', u'Yakut'),
('sai', u'South American Indian '),
('sal', u'Salishan languages'),
('sam', u'Samaritan Aramaic'),
('san', 'sa', u'Sanskrit'),
('sas', u'Sasak'),
('sat', u'Santali'),
('scc', 'sr', u'Serbian'),
('scn', u'Sicilian'),
('sco', u'Scots'),
('scr', 'hr', u'Croatian'),
('sel', u'Selkup'),
('sem', u'Semitic '),
('sga', u'Irish, Old '),
('sgn', u'Sign Languages'),
('shn', u'Shan'),
('sid', u'Sidamo'),
('sin', 'si', u'Sinhala'),
('sio', u'Siouan languages'),
('sit', u'Sino-Tibetan '),
('sla', u'Slavic '),
('slo', 'sk', u'Slovak'),
('slv', 'sl', u'Slovenian'),
('sma', u'Southern Sami'),
('sme', 'se', u'Northern Sami'),
('smi', u'Sami languages '),
('smj', u'Lule Sami'),
('smn', u'Inari Sami'),
('smo', 'sm', u'Samoan'),
('sms', u'Skolt Sami'),
('sna', 'sn', u'Shona'),
('snd', 'sd', u'Sindhi'),
('snk', u'Soninke'),
('sog', u'Sogdian'),
('som', 'so', u'Somali'),
('son', u'Songhai languages'),
('sot', 'st', u'Sotho, Southern'),
('spa', 'es', u'Spanish'),
('srd', 'sc', u'Sardinian'),
('srn', u'Sranan Tongo'),
('srr', u'Serer'),
('ssa', u'Nilo-Saharan '),
('ssw', 'ss', u'Swati'),
('suk', u'Sukuma'),
('sun', 'su', u'Sundanese'),
('sus', u'Susu'),
('sux', u'Sumerian'),
('swa', 'sw', u'Swahili'),
('swe', 'sv', u'Swedish'),
('syc', u'Classical Syriac'),
('syr', u'Syriac'),
('tah', 'ty', u'Tahitian'),
('tai', u'Tai '),
('tam', 'ta', u'Tamil'),
('tat', 'tt', u'Tatar'),
('tel', 'te', u'Telugu'),
('tem', u'Timne'),
('ter', u'Tereno'),
('tet', u'Tetum'),
('tgk', 'tg', u'Tajik'),
('tgl', 'tl', u'Tagalog'),
('tha', 'th', u'Thai'),
('tib', 'bo', u'Tibetan'),
('tig', u'Tigre'),
('tir', 'ti', u'Tigrinya'),
('tiv', u'Tiv'),
('tkl', u'Tokelau'),
('tlh', u'Klingon'),
('tli', u'Tlingit'),
('tmh', u'Tamashek'),
('tog', u'Tonga '),
('ton', 'to', u'Tonga '),
('tpi', u'Tok Pisin'),
('tsi', u'Tsimshian'),
('tsn', 'tn', u'Tswana'),
('tso', 'ts', u'Tsonga'),
('tuk', 'tk', u'Turkmen'),
('tum', u'Tumbuka'),
('tup', u'Tupi languages'),
('tur', 'tr', u'Turkish'),
('tut', u'Altaic '),
('tvl', u'Tuvalu'),
('twi', 'tw', u'Twi'),
('tyv', u'Tuvinian'),
('udm', u'Udmurt'),
('uga', u'Ugaritic'),
('uig', 'ug', u'Uighur'),
('ukr', 'uk', u'Ukrainian'),
('umb', u'Umbundu'),
('und', u'Undetermined'),
('urd', 'ur', u'Urdu'),
('uzb', 'uz', u'Uzbek'),
('vai', u'Vai'),
('ven', 've', u'Venda'),
('vie', 'vi', u'Vietnamese'),
('vol', 'vo', u'Volap\xfck'),
('vot', u'Votic'),
('wak', u'Wakashan languages'),
('wal', u'Walamo'),
('war', u'Waray'),
('was', u'Washo'),
('wel', 'cy', u'Welsh'),
('wen', u'Sorbian languages'),
('wln', 'wa', u'Walloon'),
('wol', 'wo', u'Wolof'),
('xal', u'Kalmyk'),
('xho', 'xh', u'Xhosa'),
('yao', u'Yao'),
('yap', u'Yapese'),
('yid', 'yi', u'Yiddish'),
('yor', 'yo', u'Yoruba'),
('ypk', u'Yupik languages'),
('zap', u'Zapotec'),
('zbl', u'Blissymbols'),
('zen', u'Zenaga'),
('zha', 'za', u'Zhuang'),
('znd', u'Zande languages'),
('zul', 'zu', u'Zulu'),
('zun', u'Zuni'),
('zxx', u'No linguistic content'),
('zza', u'Zaza'),
)
libs/enzyme/mkv.py
+841
View File
@@ -0,0 +1,841 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
# Copyright (C) 2003-2006 Jason Tackaberry <tack@urandom.ca>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
from datetime import datetime
from exceptions import *
from struct import unpack
import core
import logging
import re
# get logging object
log = logging.getLogger(__name__)
# Main IDs for the Matroska streams
MATROSKA_VIDEO_TRACK = 0x01
MATROSKA_AUDIO_TRACK = 0x02
MATROSKA_SUBTITLES_TRACK = 0x11
MATROSKA_HEADER_ID = 0x1A45DFA3
MATROSKA_TRACKS_ID = 0x1654AE6B
MATROSKA_CUES_ID = 0x1C53BB6B
MATROSKA_SEGMENT_ID = 0x18538067
MATROSKA_SEGMENT_INFO_ID = 0x1549A966
MATROSKA_CLUSTER_ID = 0x1F43B675
MATROSKA_VOID_ID = 0xEC
MATROSKA_CRC_ID = 0xBF
MATROSKA_TIMECODESCALE_ID = 0x2AD7B1
MATROSKA_DURATION_ID = 0x4489
MATROSKA_CRC32_ID = 0xBF
MATROSKA_TIMECODESCALE_ID = 0x2AD7B1
MATROSKA_MUXING_APP_ID = 0x4D80
MATROSKA_WRITING_APP_ID = 0x5741
MATROSKA_CODEC_ID = 0x86
MATROSKA_CODEC_PRIVATE_ID = 0x63A2
MATROSKA_FRAME_DURATION_ID = 0x23E383
MATROSKA_VIDEO_SETTINGS_ID = 0xE0
MATROSKA_VIDEO_WIDTH_ID = 0xB0
MATROSKA_VIDEO_HEIGHT_ID = 0xBA
MATROSKA_VIDEO_INTERLACED_ID = 0x9A
MATROSKA_VIDEO_DISPLAY_WIDTH_ID = 0x54B0
MATROSKA_VIDEO_DISPLAY_HEIGHT_ID = 0x54BA
MATROSKA_AUDIO_SETTINGS_ID = 0xE1
MATROSKA_AUDIO_SAMPLERATE_ID = 0xB5
MATROSKA_AUDIO_CHANNELS_ID = 0x9F
MATROSKA_TRACK_UID_ID = 0x73C5
MATROSKA_TRACK_NUMBER_ID = 0xD7
MATROSKA_TRACK_TYPE_ID = 0x83
MATROSKA_TRACK_LANGUAGE_ID = 0x22B59C
MATROSKA_TRACK_OFFSET = 0x537F
MATROSKA_TRACK_FLAG_DEFAULT_ID = 0x88
MATROSKA_TRACK_FLAG_ENABLED_ID = 0xB9
MATROSKA_TITLE_ID = 0x7BA9
MATROSKA_DATE_UTC_ID = 0x4461
MATROSKA_NAME_ID = 0x536E
MATROSKA_CHAPTERS_ID = 0x1043A770
MATROSKA_CHAPTER_UID_ID = 0x73C4
MATROSKA_EDITION_ENTRY_ID = 0x45B9
MATROSKA_CHAPTER_ATOM_ID = 0xB6
MATROSKA_CHAPTER_TIME_START_ID = 0x91
MATROSKA_CHAPTER_TIME_END_ID = 0x92
MATROSKA_CHAPTER_FLAG_ENABLED_ID = 0x4598
MATROSKA_CHAPTER_DISPLAY_ID = 0x80
MATROSKA_CHAPTER_LANGUAGE_ID = 0x437C
MATROSKA_CHAPTER_STRING_ID = 0x85
MATROSKA_ATTACHMENTS_ID = 0x1941A469
MATROSKA_ATTACHED_FILE_ID = 0x61A7
MATROSKA_FILE_DESC_ID = 0x467E
MATROSKA_FILE_NAME_ID = 0x466E
MATROSKA_FILE_MIME_TYPE_ID = 0x4660
MATROSKA_FILE_DATA_ID = 0x465C
MATROSKA_SEEKHEAD_ID = 0x114D9B74
MATROSKA_SEEK_ID = 0x4DBB
MATROSKA_SEEKID_ID = 0x53AB
MATROSKA_SEEK_POSITION_ID = 0x53AC
MATROSKA_TAGS_ID = 0x1254C367
MATROSKA_TAG_ID = 0x7373
MATROSKA_TARGETS_ID = 0x63C0
MATROSKA_TARGET_TYPE_VALUE_ID = 0x68CA
MATROSKA_TARGET_TYPE_ID = 0x63CA
MATRSOKA_TAGS_TRACK_UID_ID = 0x63C5
MATRSOKA_TAGS_EDITION_UID_ID = 0x63C9
MATRSOKA_TAGS_CHAPTER_UID_ID = 0x63C4
MATRSOKA_TAGS_ATTACHMENT_UID_ID = 0x63C6
MATROSKA_SIMPLE_TAG_ID = 0x67C8
MATROSKA_TAG_NAME_ID = 0x45A3
MATROSKA_TAG_LANGUAGE_ID = 0x447A
MATROSKA_TAG_STRING_ID = 0x4487
MATROSKA_TAG_BINARY_ID = 0x4485
# See mkv spec for details:
# http://www.matroska.org/technical/specs/index.html
# Map to convert to well known codes
# http://haali.cs.msu.ru/mkv/codecs.pdf
FOURCCMap = {
'V_THEORA': 'THEO',
'V_SNOW': 'SNOW',
'V_MPEG4/ISO/ASP': 'MP4V',
'V_MPEG4/ISO/AVC': 'AVC1',
'A_AC3': 0x2000,
'A_MPEG/L3': 0x0055,
'A_MPEG/L2': 0x0050,
'A_MPEG/L1': 0x0050,
'A_DTS': 0x2001,
'A_PCM/INT/LIT': 0x0001,
'A_PCM/FLOAT/IEEE': 0x003,
'A_TTA1': 0x77a1,
'A_WAVPACK4': 0x5756,
'A_VORBIS': 0x6750,
'A_FLAC': 0xF1AC,
'A_AAC': 0x00ff,
'A_AAC/': 0x00ff
}
def matroska_date_to_datetime(date):
"""
Converts a date in Matroska's date format to a python datetime object.
Returns the given date string if it could not be converted.
"""
# From the specs:
# The fields with dates should have the following format: YYYY-MM-DD
# HH:MM:SS.MSS [...] To store less accuracy, you remove items starting
# from the right. To store only the year, you would use, "2004". To store
# a specific day such as May 1st, 2003, you would use "2003-05-01".
format = re.split(r'([-:. ])', '%Y-%m-%d %H:%M:%S.%f')
while format:
try:
return datetime.strptime(date, ''.join(format))
except ValueError:
format = format[:-2]
return date
def matroska_bps_to_bitrate(bps):
"""
Tries to convert a free-form bps string into a bitrate (bits per second).
"""
m = re.search('([\d.]+)\s*(\D.*)', bps)
if m:
bps, suffix = m.groups()
if 'kbit' in suffix:
return float(bps) * 1024
elif 'kbyte' in suffix:
return float(bps) * 1024 * 8
elif 'byte' in suffix:
return float(bps) * 8
elif 'bps' in suffix or 'bit' in suffix:
return float(bps)
if bps.replace('.', '').isdigit():
if float(bps) < 30000:
# Assume kilobits and convert to bps
return float(bps) * 1024
return float(bps)
# Used to convert the official matroska tag names (only lower-cased) to core
# attributes. tag name -> attr, filter
TAGS_MAP = {
# From Media core
u'title': ('title', None),
u'subtitle': ('caption', None),
u'comment': ('comment', None),
u'url': ('url', None),
u'artist': ('artist', None),
u'keywords': ('keywords', lambda s: [word.strip() for word in s.split(',')]),
u'composer_nationality': ('country', None),
u'date_released': ('datetime', None),
u'date_recorded': ('datetime', None),
u'date_written': ('datetime', None),
# From Video core
u'encoder': ('encoder', None),
u'bps': ('bitrate', matroska_bps_to_bitrate),
u'part_number': ('trackno', int),
u'total_parts': ('trackof', int),
u'copyright': ('copyright', None),
u'genre': ('genre', None),
u'actor': ('actors', None),
u'written_by': ('writer', None),
u'producer': ('producer', None),
u'production_studio': ('studio', None),
u'law_rating': ('rating', None),
u'summary': ('summary', None),
u'synopsis': ('synopsis', None),
}
class EbmlEntity:
"""
This is class that is responsible to handle one Ebml entity as described in
the Matroska/Ebml spec
"""
def __init__(self, inbuf):
# Compute the EBML id
# Set the CRC len to zero
self.crc_len = 0
# Now loop until we find an entity without CRC
try:
self.build_entity(inbuf)
except IndexError:
raise ParseError()
while self.get_id() == MATROSKA_CRC32_ID:
self.crc_len += self.get_total_len()
inbuf = inbuf[self.get_total_len():]
self.build_entity(inbuf)
def build_entity(self, inbuf):
self.compute_id(inbuf)
if self.id_len == 0:
log.error("EBML entity not found, bad file format")
raise ParseError()
self.entity_len, self.len_size = self.compute_len(inbuf[self.id_len:])
self.entity_data = inbuf[self.get_header_len() : self.get_total_len()]
self.ebml_length = self.entity_len
self.entity_len = min(len(self.entity_data), self.entity_len)
# if the data size is 8 or less, it could be a numeric value
self.value = 0
if self.entity_len <= 8:
for pos, shift in zip(range(self.entity_len), range((self.entity_len-1)*8, -1, -8)):
self.value |= ord(self.entity_data[pos]) << shift
def add_data(self, data):
maxlen = self.ebml_length - len(self.entity_data)
if maxlen <= 0:
return
self.entity_data += data[:maxlen]
self.entity_len = len(self.entity_data)
def compute_id(self, inbuf):
self.id_len = 0
if len(inbuf) < 1:
return 0
first = ord(inbuf[0])
if first & 0x80:
self.id_len = 1
self.entity_id = first
elif first & 0x40:
if len(inbuf) < 2:
return 0
self.id_len = 2
self.entity_id = ord(inbuf[0])<<8 | ord(inbuf[1])
elif first & 0x20:
if len(inbuf) < 3:
return 0
self.id_len = 3
self.entity_id = (ord(inbuf[0])<<16) | (ord(inbuf[1])<<8) | \
(ord(inbuf[2]))
elif first & 0x10:
if len(inbuf) < 4:
return 0
self.id_len = 4
self.entity_id = (ord(inbuf[0])<<24) | (ord(inbuf[1])<<16) | \
(ord(inbuf[2])<<8) | (ord(inbuf[3]))
self.entity_str = inbuf[0:self.id_len]
def compute_len(self, inbuf):
if not inbuf:
return 0, 0
i = num_ffs = 0
len_mask = 0x80
len = ord(inbuf[0])
while not len & len_mask:
i += 1
len_mask >>= 1
if i >= 8:
return 0, 0
len &= len_mask - 1
if len == len_mask - 1:
num_ffs += 1
for p in range(i):
len = (len << 8) | ord(inbuf[p + 1])
if len & 0xff == 0xff:
num_ffs += 1
if num_ffs == i + 1:
len = 0
return len, i + 1
def get_crc_len(self):
return self.crc_len
def get_value(self):
return self.value
def get_float_value(self):
if len(self.entity_data) == 4:
return unpack('!f', self.entity_data)[0]
elif len(self.entity_data) == 8:
return unpack('!d', self.entity_data)[0]
return 0.0
def get_data(self):
return self.entity_data
def get_utf8(self):
return unicode(self.entity_data, 'utf-8', 'replace')
def get_str(self):
return unicode(self.entity_data, 'ascii', 'replace')
def get_id(self):
return self.entity_id
def get_str_id(self):
return self.entity_str
def get_len(self):
return self.entity_len
def get_total_len(self):
return self.entity_len + self.id_len + self.len_size
def get_header_len(self):
return self.id_len + self.len_size
class Matroska(core.AVContainer):
"""
Matroska video and audio parser. If at least one video stream is
detected it will set the type to MEDIA_AV.
"""
def __init__(self, file):
core.AVContainer.__init__(self)
self.samplerate = 1
self.file = file
# Read enough that we're likely to get the full seekhead (FIXME: kludge)
buffer = file.read(2000)
if len(buffer) == 0:
# Regular File end
raise ParseError()
# Check the Matroska header
header = EbmlEntity(buffer)
if header.get_id() != MATROSKA_HEADER_ID:
raise ParseError()
log.debug("HEADER ID found %08X" % header.get_id() )
self.mime = 'video/x-matroska'
self.type = 'Matroska'
self.has_idx = False
self.objects_by_uid = {}
# Now get the segment
self.segment = segment = EbmlEntity(buffer[header.get_total_len():])
# Record file offset of segment data for seekheads
self.segment.offset = header.get_total_len() + segment.get_header_len()
if segment.get_id() != MATROSKA_SEGMENT_ID:
log.debug("SEGMENT ID not found %08X" % segment.get_id())
return
log.debug("SEGMENT ID found %08X" % segment.get_id())
try:
for elem in self.process_one_level(segment):
if elem.get_id() == MATROSKA_SEEKHEAD_ID:
self.process_elem(elem)
except ParseError:
pass
if not self.has_idx:
log.warning('File has no index')
self._set('corrupt', True)
def process_elem(self, elem):
elem_id = elem.get_id()
log.debug('BEGIN: process element %s' % hex(elem_id))
if elem_id == MATROSKA_SEGMENT_INFO_ID:
duration = 0
scalecode = 1000000.0
for ielem in self.process_one_level(elem):
ielem_id = ielem.get_id()
if ielem_id == MATROSKA_TIMECODESCALE_ID:
scalecode = ielem.get_value()
elif ielem_id == MATROSKA_DURATION_ID:
duration = ielem.get_float_value()
elif ielem_id == MATROSKA_TITLE_ID:
self.title = ielem.get_utf8()
elif ielem_id == MATROSKA_DATE_UTC_ID:
timestamp = unpack('!q', ielem.get_data())[0] / 10.0**9
# Date is offset 2001-01-01 00:00:00 (timestamp 978307200.0)
self.timestamp = int(timestamp + 978307200)
self.length = duration * scalecode / 1000000000.0
elif elem_id == MATROSKA_TRACKS_ID:
self.process_tracks(elem)
elif elem_id == MATROSKA_CHAPTERS_ID:
self.process_chapters(elem)
elif elem_id == MATROSKA_ATTACHMENTS_ID:
self.process_attachments(elem)
elif elem_id == MATROSKA_SEEKHEAD_ID:
self.process_seekhead(elem)
elif elem_id == MATROSKA_TAGS_ID:
self.process_tags(elem)
elif elem_id == MATROSKA_CUES_ID:
self.has_idx = True
log.debug('END: process element %s' % hex(elem_id))
return True
def process_seekhead(self, elem):
for seek_elem in self.process_one_level(elem):
if seek_elem.get_id() != MATROSKA_SEEK_ID:
continue
for sub_elem in self.process_one_level(seek_elem):
if sub_elem.get_id() == MATROSKA_SEEKID_ID:
if sub_elem.get_value() == MATROSKA_CLUSTER_ID:
# Not interested in these.
return
elif sub_elem.get_id() == MATROSKA_SEEK_POSITION_ID:
self.file.seek(self.segment.offset + sub_elem.get_value())
buffer = self.file.read(100)
try:
elem = EbmlEntity(buffer)
except ParseError:
continue
# Fetch all data necessary for this element.
elem.add_data(self.file.read(elem.ebml_length))
self.process_elem(elem)
def process_tracks(self, tracks):
tracksbuf = tracks.get_data()
index = 0
while index < tracks.get_len():
trackelem = EbmlEntity(tracksbuf[index:])
log.debug ("ELEMENT %X found" % trackelem.get_id())
self.process_track(trackelem)
index += trackelem.get_total_len() + trackelem.get_crc_len()
def process_one_level(self, item):
buf = item.get_data()
index = 0
while index < item.get_len():
if len(buf[index:]) == 0:
break
elem = EbmlEntity(buf[index:])
yield elem
index += elem.get_total_len() + elem.get_crc_len()
def set_track_defaults(self, track):
track.language = 'eng'
def process_track(self, track):
# Collapse generator into a list since we need to iterate over it
# twice.
elements = [x for x in self.process_one_level(track)]
track_type = [x.get_value() for x in elements if x.get_id() == MATROSKA_TRACK_TYPE_ID]
if not track_type:
log.debug('Bad track: no type id found')
return
track_type = track_type[0]
track = None
if track_type == MATROSKA_VIDEO_TRACK:
log.debug("Video track found")
track = self.process_video_track(elements)
elif track_type == MATROSKA_AUDIO_TRACK:
log.debug("Audio track found")
track = self.process_audio_track(elements)
elif track_type == MATROSKA_SUBTITLES_TRACK:
log.debug("Subtitle track found")
track = core.Subtitle()
self.set_track_defaults(track)
track.id = len(self.subtitles)
self.subtitles.append(track)
for elem in elements:
self.process_track_common(elem, track)
def process_track_common(self, elem, track):
elem_id = elem.get_id()
if elem_id == MATROSKA_TRACK_LANGUAGE_ID:
track.language = elem.get_str()
log.debug("Track language found: %s" % track.language)
elif elem_id == MATROSKA_NAME_ID:
track.title = elem.get_utf8()
elif elem_id == MATROSKA_TRACK_NUMBER_ID:
track.trackno = elem.get_value()
elif elem_id == MATROSKA_TRACK_FLAG_ENABLED_ID:
track.enabled = bool(elem.get_value())
elif elem_id == MATROSKA_TRACK_FLAG_DEFAULT_ID:
track.default = bool(elem.get_value())
elif elem_id == MATROSKA_CODEC_ID:
track.codec = elem.get_str()
elif elem_id == MATROSKA_CODEC_PRIVATE_ID:
track.codec_private = elem.get_data()
elif elem_id == MATROSKA_TRACK_UID_ID:
self.objects_by_uid[elem.get_value()] = track
def process_video_track(self, elements):
track = core.VideoStream()
# Defaults
track.codec = u'Unknown'
track.fps = 0
self.set_track_defaults(track)
for elem in elements:
elem_id = elem.get_id()
if elem_id == MATROSKA_CODEC_ID:
track.codec = elem.get_str()
elif elem_id == MATROSKA_FRAME_DURATION_ID:
try:
track.fps = 1 / (pow(10, -9) * (elem.get_value()))
except ZeroDivisionError:
pass
elif elem_id == MATROSKA_VIDEO_SETTINGS_ID:
d_width = d_height = None
for settings_elem in self.process_one_level(elem):
settings_elem_id = settings_elem.get_id()
if settings_elem_id == MATROSKA_VIDEO_WIDTH_ID:
track.width = settings_elem.get_value()
elif settings_elem_id == MATROSKA_VIDEO_HEIGHT_ID:
track.height = settings_elem.get_value()
elif settings_elem_id == MATROSKA_VIDEO_DISPLAY_WIDTH_ID:
d_width = settings_elem.get_value()
elif settings_elem_id == MATROSKA_VIDEO_DISPLAY_HEIGHT_ID:
d_height = settings_elem.get_value()
elif settings_elem_id == MATROSKA_VIDEO_INTERLACED_ID:
value = int(settings_elem.get_value())
self._set('interlaced', value)
if None not in [d_width, d_height]:
track.aspect = float(d_width) / d_height
else:
self.process_track_common(elem, track)
# convert codec information
# http://haali.cs.msu.ru/mkv/codecs.pdf
if track.codec in FOURCCMap:
track.codec = FOURCCMap[track.codec]
elif '/' in track.codec and track.codec.split('/')[0] + '/' in FOURCCMap:
track.codec = FOURCCMap[track.codec.split('/')[0] + '/']
elif track.codec.endswith('FOURCC') and len(track.codec_private or '') == 40:
track.codec = track.codec_private[16:20]
elif track.codec.startswith('V_REAL/'):
track.codec = track.codec[7:]
elif track.codec.startswith('V_'):
# FIXME: add more video codecs here
track.codec = track.codec[2:]
track.id = len(self.video)
self.video.append(track)
return track
def process_audio_track(self, elements):
track = core.AudioStream()
track.codec = u'Unknown'
self.set_track_defaults(track)
for elem in elements:
elem_id = elem.get_id()
if elem_id == MATROSKA_CODEC_ID:
track.codec = elem.get_str()
elif elem_id == MATROSKA_AUDIO_SETTINGS_ID:
for settings_elem in self.process_one_level(elem):
settings_elem_id = settings_elem.get_id()
if settings_elem_id == MATROSKA_AUDIO_SAMPLERATE_ID:
track.samplerate = settings_elem.get_float_value()
elif settings_elem_id == MATROSKA_AUDIO_CHANNELS_ID:
track.channels = settings_elem.get_value()
else:
self.process_track_common(elem, track)
if track.codec in FOURCCMap:
track.codec = FOURCCMap[track.codec]
elif '/' in track.codec and track.codec.split('/')[0] + '/' in FOURCCMap:
track.codec = FOURCCMap[track.codec.split('/')[0] + '/']
elif track.codec.startswith('A_'):
track.codec = track.codec[2:]
track.id = len(self.audio)
self.audio.append(track)
return track
def process_chapters(self, chapters):
elements = self.process_one_level(chapters)
for elem in elements:
if elem.get_id() == MATROSKA_EDITION_ENTRY_ID:
buf = elem.get_data()
index = 0
while index < elem.get_len():
sub_elem = EbmlEntity(buf[index:])
if sub_elem.get_id() == MATROSKA_CHAPTER_ATOM_ID:
self.process_chapter_atom(sub_elem)
index += sub_elem.get_total_len() + sub_elem.get_crc_len()
def process_chapter_atom(self, atom):
elements = self.process_one_level(atom)
chap = core.Chapter()
for elem in elements:
elem_id = elem.get_id()
if elem_id == MATROSKA_CHAPTER_TIME_START_ID:
# Scale timecode to seconds (float)
chap.pos = elem.get_value() / 1000000 / 1000.0
elif elem_id == MATROSKA_CHAPTER_FLAG_ENABLED_ID:
chap.enabled = elem.get_value()
elif elem_id == MATROSKA_CHAPTER_DISPLAY_ID:
# Matroska supports multiple (chapter name, language) pairs for
# each chapter, so chapter names can be internationalized. This
# logic will only take the last one in the list.
for display_elem in self.process_one_level(elem):
if display_elem.get_id() == MATROSKA_CHAPTER_STRING_ID:
chap.name = display_elem.get_utf8()
elif elem_id == MATROSKA_CHAPTER_UID_ID:
self.objects_by_uid[elem.get_value()] = chap
log.debug('Chapter "%s" found', chap.name)
chap.id = len(self.chapters)
self.chapters.append(chap)
def process_attachments(self, attachments):
buf = attachments.get_data()
index = 0
while index < attachments.get_len():
elem = EbmlEntity(buf[index:])
if elem.get_id() == MATROSKA_ATTACHED_FILE_ID:
self.process_attachment(elem)
index += elem.get_total_len() + elem.get_crc_len()
def process_attachment(self, attachment):
elements = self.process_one_level(attachment)
name = desc = mimetype = ""
data = None
for elem in elements:
elem_id = elem.get_id()
if elem_id == MATROSKA_FILE_NAME_ID:
name = elem.get_utf8()
elif elem_id == MATROSKA_FILE_DESC_ID:
desc = elem.get_utf8()
elif elem_id == MATROSKA_FILE_MIME_TYPE_ID:
mimetype = elem.get_data()
elif elem_id == MATROSKA_FILE_DATA_ID:
data = elem.get_data()
# Right now we only support attachments that could be cover images.
# Make a guess to see if this attachment is a cover image.
if mimetype.startswith("image/") and u"cover" in (name+desc).lower() and data:
self.thumbnail = data
log.debug('Attachment "%s" found' % name)
def process_tags(self, tags):
# Tags spec: http://www.matroska.org/technical/specs/tagging/index.html
# Iterate over Tags children. Tags element children is a
# Tag element (whose children are SimpleTags) and a Targets element
# whose children specific what objects the tags apply to.
for tag_elem in self.process_one_level(tags):
# Start a new dict to hold all SimpleTag elements.
tags_dict = core.Tags()
# A list of target uids this tags dict applies too. If empty,
# tags are global.
targets = []
for sub_elem in self.process_one_level(tag_elem):
if sub_elem.get_id() == MATROSKA_SIMPLE_TAG_ID:
self.process_simple_tag(sub_elem, tags_dict)
elif sub_elem.get_id() == MATROSKA_TARGETS_ID:
# Targets element: if there is no uid child (track uid,
# chapter uid, etc.) then the tags dict applies to the
# whole file (top-level Media object).
for target_elem in self.process_one_level(sub_elem):
target_elem_id = target_elem.get_id()
if target_elem_id in (MATRSOKA_TAGS_TRACK_UID_ID, MATRSOKA_TAGS_EDITION_UID_ID,
MATRSOKA_TAGS_CHAPTER_UID_ID, MATRSOKA_TAGS_ATTACHMENT_UID_ID):
targets.append(target_elem.get_value())
elif target_elem_id == MATROSKA_TARGET_TYPE_VALUE_ID:
# Target types not supported for now. (Unclear how this
# would fit with kaa.metadata.)
pass
if targets:
# Assign tags to all listed uids
for target in targets:
try:
self.objects_by_uid[target].tags.update(tags_dict)
self.tags_to_attributes(self.objects_by_uid[target], tags_dict)
except KeyError:
log.warning('Tags assigned to unknown/unsupported target uid %d', target)
else:
self.tags.update(tags_dict)
self.tags_to_attributes(self, tags_dict)
def process_simple_tag(self, simple_tag_elem, tags_dict):
"""
Returns a dict representing the Tag element.
"""
name = lang = value = children = None
binary = False
for elem in self.process_one_level(simple_tag_elem):
elem_id = elem.get_id()
if elem_id == MATROSKA_TAG_NAME_ID:
name = elem.get_utf8().lower()
elif elem_id == MATROSKA_TAG_STRING_ID:
value = elem.get_utf8()
elif elem_id == MATROSKA_TAG_BINARY_ID:
value = elem.get_data()
binary = True
elif elem_id == MATROSKA_TAG_LANGUAGE_ID:
lang = elem.get_utf8()
elif elem_id == MATROSKA_SIMPLE_TAG_ID:
if children is None:
children = core.Tags()
self.process_simple_tag(elem, children)
if children:
# Convert ourselves to a Tags object.
children.value = value
children.langcode = lang
value = children
else:
if name.startswith('date_'):
# Try to convert date to a datetime object.
value = matroska_date_to_datetime(value)
value = core.Tag(value, lang, binary)
if name in tags_dict:
# Multiple items of this tag name.
if not isinstance(tags_dict[name], list):
# Convert to a list
tags_dict[name] = [tags_dict[name]]
# Append to list
tags_dict[name].append(value)
else:
tags_dict[name] = value
def tags_to_attributes(self, obj, tags):
# Convert tags to core attributes.
for name, tag in tags.items():
if isinstance(tag, dict):
# Nested tags dict, recurse.
self.tags_to_attributes(obj, tag)
continue
elif name not in TAGS_MAP:
continue
attr, filter = TAGS_MAP[name]
if attr not in obj._keys and attr not in self._keys:
# Tag is not in any core attribute for this object or global,
# so skip.
continue
# Pull value out of Tag object or list of Tag objects.
value = [item.value for item in tag] if isinstance(tag, list) else tag.value
if filter:
try:
value = [filter(item) for item in value] if isinstance(value, list) else filter(value)
except Exception, e:
log.warning('Failed to convert tag to core attribute: %s', e)
# Special handling for tv series recordings. The 'title' tag
# can be used for both the series and the episode name. The
# same is true for trackno which may refer to the season
# and the episode number. Therefore, if we find these
# attributes already set we try some guessing.
if attr == 'trackno' and getattr(self, attr) is not None:
# delete trackno and save season and episode
self.season = self.trackno
self.episode = value
self.trackno = None
continue
if attr == 'title' and getattr(self, attr) is not None:
# store current value of title as series and use current
# value of title as title
self.series = self.title
if attr in obj._keys:
setattr(obj, attr, value)
else:
setattr(self, attr, value)
Parser = Matroska
libs/enzyme/mp4.py
+476
View File
@@ -0,0 +1,476 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2007 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2007 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
import zlib
import logging
import StringIO
import struct
from exceptions import *
import core
# get logging object
log = logging.getLogger(__name__)
# http://developer.apple.com/documentation/QuickTime/QTFF/index.html
# http://developer.apple.com/documentation/QuickTime/QTFF/QTFFChap4/\
# chapter_5_section_2.html#//apple_ref/doc/uid/TP40000939-CH206-BBCBIICE
# Note: May need to define custom log level to work like ATOM_DEBUG did here
QTUDTA = {
'nam': 'title',
'aut': 'artist',
'cpy': 'copyright'
}
QTLANGUAGES = {
0: "en",
1: "fr",
2: "de",
3: "it",
4: "nl",
5: "sv",
6: "es",
7: "da",
8: "pt",
9: "no",
10: "he",
11: "ja",
12: "ar",
13: "fi",
14: "el",
15: "is",
16: "mt",
17: "tr",
18: "hr",
19: "Traditional Chinese",
20: "ur",
21: "hi",
22: "th",
23: "ko",
24: "lt",
25: "pl",
26: "hu",
27: "et",
28: "lv",
29: "Lappish",
30: "fo",
31: "Farsi",
32: "ru",
33: "Simplified Chinese",
34: "Flemish",
35: "ga",
36: "sq",
37: "ro",
38: "cs",
39: "sk",
40: "sl",
41: "yi",
42: "sr",
43: "mk",
44: "bg",
45: "uk",
46: "be",
47: "uz",
48: "kk",
49: "az",
50: "AzerbaijanAr",
51: "hy",
52: "ka",
53: "mo",
54: "ky",
55: "tg",
56: "tk",
57: "mn",
58: "MongolianCyr",
59: "ps",
60: "ku",
61: "ks",
62: "sd",
63: "bo",
64: "ne",
65: "sa",
66: "mr",
67: "bn",
68: "as",
69: "gu",
70: "pa",
71: "or",
72: "ml",
73: "kn",
74: "ta",
75: "te",
76: "si",
77: "my",
78: "Khmer",
79: "lo",
80: "vi",
81: "id",
82: "tl",
83: "MalayRoman",
84: "MalayArabic",
85: "am",
86: "ti",
87: "om",
88: "so",
89: "sw",
90: "Ruanda",
91: "Rundi",
92: "Chewa",
93: "mg",
94: "eo",
128: "cy",
129: "eu",
130: "ca",
131: "la",
132: "qu",
133: "gn",
134: "ay",
135: "tt",
136: "ug",
137: "Dzongkha",
138: "JavaneseRom",
}
class MPEG4(core.AVContainer):
"""
Parser for the MP4 container format. This format is mostly
identical to Apple Quicktime and 3GP files. It maps to mp4, mov,
qt and some other extensions.
"""
table_mapping = {'QTUDTA': QTUDTA}
def __init__(self, file):
core.AVContainer.__init__(self)
self._references = []
self.mime = 'video/quicktime'
self.type = 'Quicktime Video'
h = file.read(8)
try:
(size, type) = struct.unpack('>I4s',h)
except struct.error:
# EOF.
raise ParseError()
if type == 'ftyp':
# file type information
if size >= 12:
# this should always happen
if file.read(4) != 'qt ':
# not a quicktime movie, it is a mpeg4 container
self.mime = 'video/mp4'
self.type = 'MPEG-4 Video'
size -= 4
file.seek(size-8, 1)
h = file.read(8)
(size, type) = struct.unpack('>I4s',h)
while type in ['mdat', 'skip']:
# movie data at the beginning, skip
file.seek(size-8, 1)
h = file.read(8)
(size, type) = struct.unpack('>I4s',h)
if not type in ['moov', 'wide', 'free']:
log.debug('invalid header: %r' % type)
raise ParseError()
# Extended size
if size == 1:
size = struct.unpack('>Q', file.read(8))
# Back over the atom header we just read, since _readatom expects the
# file position to be at the start of an atom.
file.seek(-8, 1)
while self._readatom(file):
pass
if self._references:
self._set('references', self._references)
def _readatom(self, file):
s = file.read(8)
if len(s) < 8:
return 0
atomsize,atomtype = struct.unpack('>I4s', s)
if not str(atomtype).decode('latin1').isalnum():
# stop at nonsense data
return 0
log.debug('%s [%X]' % (atomtype,atomsize))
if atomtype == 'udta':
# Userdata (Metadata)
pos = 0
tabl = {}
i18ntabl = {}
atomdata = file.read(atomsize-8)
while pos < atomsize-12:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
if ord(datatype[0]) == 169:
# i18n Metadata...
mypos = 8+pos
while mypos + 4 < datasize+pos:
# first 4 Bytes are i18n header
(tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos+4])
i18ntabl[lang] = i18ntabl.get(lang, {})
l = atomdata[mypos+4:mypos+tlen+4]
i18ntabl[lang][datatype[1:]] = l
mypos += tlen+4
elif datatype == 'WLOC':
# Drop Window Location
pass
else:
if ord(atomdata[pos+8:pos+datasize][0]) > 1:
tabl[datatype] = atomdata[pos+8:pos+datasize]
pos += datasize
if len(i18ntabl.keys()) > 0:
for k in i18ntabl.keys():
if QTLANGUAGES.has_key(k) and QTLANGUAGES[k] == 'en':
self._appendtable('QTUDTA', i18ntabl[k])
self._appendtable('QTUDTA', tabl)
else:
log.debug('NO i18')
self._appendtable('QTUDTA', tabl)
elif atomtype == 'trak':
atomdata = file.read(atomsize-8)
pos = 0
trackinfo = {}
tracktype = None
while pos < atomsize-8:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
if datatype == 'tkhd':
tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos+8:pos+datasize])
trackinfo['width'] = tkhd[10] >> 16
trackinfo['height'] = tkhd[11] >> 16
trackinfo['id'] = tkhd[3]
try:
# XXX Timestamp of Seconds is since January 1st 1904!
# XXX 2082844800 is the difference between Unix and
# XXX Apple time. FIXME to work on Apple, too
self.timestamp = int(tkhd[1]) - 2082844800
except Exception, e:
log.exception('There was trouble extracting timestamp')
elif datatype == 'mdia':
pos += 8
datasize -= 8
log.debug('--> mdia information')
while datasize:
mdia = struct.unpack('>I4s', atomdata[pos:pos+8])
if mdia[1] == 'mdhd':
# Parse based on version of mdhd header. See
# http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
ver = ord(atomdata[pos + 8])
if ver == 0:
mdhd = struct.unpack('>IIIIIhh', atomdata[pos+8:pos+8+24])
elif ver == 1:
mdhd = struct.unpack('>IQQIQhh', atomdata[pos+8:pos+8+36])
else:
mdhd = None
if mdhd:
# duration / time scale
trackinfo['length'] = mdhd[4] / mdhd[3]
if mdhd[5] in QTLANGUAGES:
trackinfo['language'] = QTLANGUAGES[mdhd[5]]
# mdhd[6] == quality
self.length = max(self.length, mdhd[4] / mdhd[3])
elif mdia[1] == 'minf':
# minf has only atoms inside
pos -= (mdia[0] - 8)
datasize += (mdia[0] - 8)
elif mdia[1] == 'stbl':
# stbl has only atoms inside
pos -= (mdia[0] - 8)
datasize += (mdia[0] - 8)
elif mdia[1] == 'hdlr':
hdlr = struct.unpack('>I4s4s', atomdata[pos+8:pos+8+12])
if hdlr[1] == 'mhlr':
if hdlr[2] == 'vide':
tracktype = 'video'
if hdlr[2] == 'soun':
tracktype = 'audio'
elif mdia[1] == 'stsd':
stsd = struct.unpack('>2I', atomdata[pos+8:pos+8+8])
if stsd[1] > 0:
codec = atomdata[pos+16:pos+16+8]
codec = struct.unpack('>I4s', codec)
trackinfo['codec'] = codec[1]
if codec[1] == 'jpeg':
tracktype = 'image'
elif mdia[1] == 'dinf':
dref = struct.unpack('>I4s', atomdata[pos+8:pos+8+8])
log.debug(' --> %s, %s (useless)' % mdia)
if dref[1] == 'dref':
num = struct.unpack('>I', atomdata[pos+20:pos+20+4])[0]
rpos = pos+20+4
for ref in range(num):
# FIXME: do somthing if this references
ref = struct.unpack('>I3s', atomdata[rpos:rpos+7])
data = atomdata[rpos+7:rpos+ref[0]]
rpos += ref[0]
else:
if mdia[1].startswith('st'):
log.debug(' --> %s, %s (sample)' % mdia)
elif mdia[1] == 'vmhd' and not tracktype:
# indicates that this track is video
tracktype = 'video'
elif mdia[1] in ['vmhd', 'smhd'] and not tracktype:
# indicates that this track is audio
tracktype = 'audio'
else:
log.debug(' --> %s, %s (unknown)' % mdia)
pos += mdia[0]
datasize -= mdia[0]
elif datatype == 'udta':
log.debug(struct.unpack('>I4s', atomdata[:8]))
else:
if datatype == 'edts':
log.debug('--> %s [%d] (edit list)' % \
(datatype, datasize))
else:
log.debug('--> %s [%d] (unknown)' % \
(datatype, datasize))
pos += datasize
info = None
if tracktype == 'video':
info = core.VideoStream()
self.video.append(info)
if tracktype == 'audio':
info = core.AudioStream()
self.audio.append(info)
if info:
for key, value in trackinfo.items():
setattr(info, key, value)
elif atomtype == 'mvhd':
# movie header
mvhd = struct.unpack('>6I2h', file.read(28))
self.length = max(self.length, mvhd[4] / mvhd[3])
self.volume = mvhd[6]
file.seek(atomsize-8-28,1)
elif atomtype == 'cmov':
# compressed movie
datasize, atomtype = struct.unpack('>I4s', file.read(8))
if not atomtype == 'dcom':
return atomsize
method = struct.unpack('>4s', file.read(datasize-8))[0]
datasize, atomtype = struct.unpack('>I4s', file.read(8))
if not atomtype == 'cmvd':
return atomsize
if method == 'zlib':
data = file.read(datasize-8)
try:
decompressed = zlib.decompress(data)
except Exception, e:
try:
decompressed = zlib.decompress(data[4:])
except Exception, e:
log.exception('There was a proble decompressiong atom')
return atomsize
decompressedIO = StringIO.StringIO(decompressed)
while self._readatom(decompressedIO):
pass
else:
log.info('unknown compression %s' % method)
# unknown compression method
file.seek(datasize-8,1)
elif atomtype == 'moov':
# decompressed movie info
while self._readatom(file):
pass
elif atomtype == 'mdat':
pos = file.tell() + atomsize - 8
# maybe there is data inside the mdat
log.info('parsing mdat')
while self._readatom(file):
pass
log.info('end of mdat')
file.seek(pos, 0)
elif atomtype == 'rmra':
# reference list
while self._readatom(file):
pass
elif atomtype == 'rmda':
# reference
atomdata = file.read(atomsize-8)
pos = 0
url = ''
quality = 0
datarate = 0
while pos < atomsize-8:
(datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
if datatype == 'rdrf':
rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos+8:pos+20])
if rtype == 'url ':
url = atomdata[pos+20:pos+20+rlen]
if url.find('\0') > 0:
url = url[:url.find('\0')]
elif datatype == 'rmqu':
quality = struct.unpack('>I', atomdata[pos+8:pos+12])[0]
elif datatype == 'rmdr':
datarate = struct.unpack('>I', atomdata[pos+12:pos+16])[0]
pos += datasize
if url:
self._references.append((url, quality, datarate))
else:
if not atomtype in ['wide', 'free']:
log.info('unhandled base atom %s' % atomtype)
# Skip unknown atoms
try:
file.seek(atomsize-8,1)
except IOError:
return 0
return atomsize
Parser = MPEG4
libs/enzyme/mpeg.py
+915
View File
@@ -0,0 +1,915 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
import os
import struct
import logging
import stat
from exceptions import *
import core
# get logging object
log = logging.getLogger(__name__)
##------------------------------------------------------------------------
## START_CODE
##
## Start Codes, with 'slice' occupying 0x01..0xAF
##------------------------------------------------------------------------
START_CODE = {
0x00 : 'picture_start_code',
0xB0 : 'reserved',
0xB1 : 'reserved',
0xB2 : 'user_data_start_code',
0xB3 : 'sequence_header_code',
0xB4 : 'sequence_error_code',
0xB5 : 'extension_start_code',
0xB6 : 'reserved',
0xB7 : 'sequence end',
0xB8 : 'group of pictures',
}
for i in range(0x01,0xAF):
START_CODE[i] = 'slice_start_code'
##------------------------------------------------------------------------
## START CODES
##------------------------------------------------------------------------
PICTURE = 0x00
USERDATA = 0xB2
SEQ_HEAD = 0xB3
SEQ_ERR = 0xB4
EXT_START = 0xB5
SEQ_END = 0xB7
GOP = 0xB8
SEQ_START_CODE = 0xB3
PACK_PKT = 0xBA
SYS_PKT = 0xBB
PADDING_PKT = 0xBE
AUDIO_PKT = 0xC0
VIDEO_PKT = 0xE0
PRIVATE_STREAM1 = 0xBD
PRIVATE_STREAM2 = 0xBf
TS_PACKET_LENGTH = 188
TS_SYNC = 0x47
##------------------------------------------------------------------------
## FRAME_RATE
##
## A lookup table of all the standard frame rates. Some rates adhere to
## a particular profile that ensures compatibility with VLSI capabilities
## of the early to mid 1990s.
##
## CPB
## Constrained Parameters Bitstreams, an MPEG-1 set of sampling and
## bitstream parameters designed to normalize decoder computational
## complexity, buffer size, and memory bandwidth while still addressing
## the widest possible range of applications.
##
## Main Level
## MPEG-2 Video Main Profile and Main Level is analogous to MPEG-1's
## CPB, with sampling limits at CCIR 601 parameters (720x480x30 Hz or
## 720x576x24 Hz).
##
##------------------------------------------------------------------------
FRAME_RATE = [
0,
24000.0/1001, ## 3-2 pulldown NTSC (CPB/Main Level)
24, ## Film (CPB/Main Level)
25, ## PAL/SECAM or 625/60 video
30000.0/1001, ## NTSC (CPB/Main Level)
30, ## drop-frame NTSC or component 525/60 (CPB/Main Level)
50, ## double-rate PAL
60000.0/1001, ## double-rate NTSC
60, ## double-rate, drop-frame NTSC/component 525/60 video
]
##------------------------------------------------------------------------
## ASPECT_RATIO -- INCOMPLETE?
##
## This lookup table maps the header aspect ratio index to a float value.
## These are just the defined ratios for CPB I believe. As I understand
## it, a stream that doesn't adhere to one of these aspect ratios is
## technically considered non-compliant.
##------------------------------------------------------------------------
ASPECT_RATIO = ( None, # Forbidden
1.0, # 1/1 (VGA)
4.0 / 3, # 4/3 (TV)
16.0 / 9, # 16/9 (Widescreen)
2.21 # (Cinema)
)
class MPEG(core.AVContainer):
"""
Parser for various MPEG files. This includes MPEG-1 and MPEG-2
program streams, elementary streams and transport streams. The
reported length differs from the length reported by most video
players but the provides length here is correct. An MPEG file has
no additional metadata like title, etc; only codecs, length and
resolution is reported back.
"""
def __init__(self,file):
core.AVContainer.__init__(self)
self.sequence_header_offset = 0
self.mpeg_version = 2
# detect TS (fast scan)
if not self.isTS(file):
# detect system mpeg (many infos)
if not self.isMPEG(file):
# detect PES
if not self.isPES(file):
# Maybe it's MPEG-ES
if self.isES(file):
# If isES() succeeds, we needn't do anything further.
return
if file.name.lower().endswith('mpeg') or \
file.name.lower().endswith('mpg'):
# This has to be an mpeg file. It could be a bad
# recording from an ivtv based hardware encoder with
# same bytes missing at the beginning.
# Do some more digging...
if not self.isMPEG(file, force=True) or \
not self.video or not self.audio:
# does not look like an mpeg at all
raise ParseError()
else:
# no mpeg at all
raise ParseError()
self.mime = 'video/mpeg'
if not self.video:
self.video.append(core.VideoStream())
if self.sequence_header_offset <= 0:
return
self.progressive(file)
for vi in self.video:
vi.width, vi.height = self.dxy(file)
vi.fps, vi.aspect = self.framerate_aspect(file)
vi.bitrate = self.bitrate(file)
if self.length:
vi.length = self.length
if not self.type:
self.type = 'MPEG Video'
# set fourcc codec for video and audio
vc, ac = 'MP2V', 'MP2A'
if self.mpeg_version == 1:
vc, ac = 'MPEG', 0x0050
for v in self.video:
v.codec = vc
for a in self.audio:
if not a.codec:
a.codec = ac
def dxy(self,file):
"""
get width and height of the video
"""
file.seek(self.sequence_header_offset+4,0)
v = file.read(4)
x = struct.unpack('>H',v[:2])[0] >> 4
y = struct.unpack('>H',v[1:3])[0] & 0x0FFF
return (x,y)
def framerate_aspect(self,file):
"""
read framerate and aspect ratio
"""
file.seek(self.sequence_header_offset+7,0)
v = struct.unpack( '>B', file.read(1) )[0]
try:
fps = FRAME_RATE[v&0xf]
except IndexError:
fps = None
if v>>4 < len(ASPECT_RATIO):
aspect = ASPECT_RATIO[v>>4]
else:
aspect = None
return (fps, aspect)
def progressive(self, file):
"""
Try to find out with brute force if the mpeg is interlaced or not.
Search for the Sequence_Extension in the extension header (01B5)
"""
file.seek(0)
buffer = ''
count = 0
while 1:
if len(buffer) < 1000:
count += 1
if count > 1000:
break
buffer += file.read(1024)
if len(buffer) < 1000:
break
pos = buffer.find('\x00\x00\x01\xb5')
if pos == -1 or len(buffer) - pos < 5:
buffer = buffer[-10:]
continue
ext = (ord(buffer[pos+4]) >> 4)
if ext == 8:
pass
elif ext == 1:
if (ord(buffer[pos+5]) >> 3) & 1:
self._set('progressive', True)
else:
self._set('interlaced', True)
return True
else:
log.debug('ext', ext)
buffer = buffer[pos+4:]
return False
##------------------------------------------------------------------------
## bitrate()
##
## From the MPEG-2.2 spec:
##
## bit_rate -- This is a 30-bit integer. The lower 18 bits of the
## integer are in bit_rate_value and the upper 12 bits are in
## bit_rate_extension. The 30-bit integer specifies the bitrate of the
## bitstream measured in units of 400 bits/second, rounded upwards.
## The value zero is forbidden.
##
## So ignoring all the variable bitrate stuff for now, this 30 bit integer
## multiplied times 400 bits/sec should give the rate in bits/sec.
##
## TODO: Variable bitrates? I need one that implements this.
##
## Continued from the MPEG-2.2 spec:
##
## If the bitstream is a constant bitrate stream, the bitrate specified
## is the actual rate of operation of the VBV specified in annex C. If
## the bitstream is a variable bitrate stream, the STD specifications in
## ISO/IEC 13818-1 supersede the VBV, and the bitrate specified here is
## used to dimension the transport stream STD (2.4.2 in ITU-T Rec. xxx |
## ISO/IEC 13818-1), or the program stream STD (2.4.5 in ITU-T Rec. xxx |
## ISO/IEC 13818-1).
##
## If the bitstream is not a constant rate bitstream the vbv_delay
## field shall have the value FFFF in hexadecimal.
##
## Given the value encoded in the bitrate field, the bitstream shall be
## generated so that the video encoding and the worst case multiplex
## jitter do not cause STD buffer overflow or underflow.
##
##
##------------------------------------------------------------------------
## Some parts in the code are based on mpgtx (mpgtx.sf.net)
def bitrate(self,file):
"""
read the bitrate (most of the time broken)
"""
file.seek(self.sequence_header_offset+8,0)
t,b = struct.unpack( '>HB', file.read(3) )
vrate = t << 2 | b >> 6
return vrate * 400
def ReadSCRMpeg2(self, buffer):
"""
read SCR (timestamp) for MPEG2 at the buffer beginning (6 Bytes)
"""
if len(buffer) < 6:
return None
highbit = (ord(buffer[0])&0x20)>>5
low4Bytes= ((long(ord(buffer[0])) & 0x18) >> 3) << 30
low4Bytes |= (ord(buffer[0]) & 0x03) << 28
low4Bytes |= ord(buffer[1]) << 20
low4Bytes |= (ord(buffer[2]) & 0xF8) << 12
low4Bytes |= (ord(buffer[2]) & 0x03) << 13
low4Bytes |= ord(buffer[3]) << 5
low4Bytes |= (ord(buffer[4])) >> 3
sys_clock_ref=(ord(buffer[4]) & 0x3) << 7
sys_clock_ref|=(ord(buffer[5]) >> 1)
return (long(highbit * (1<<16) * (1<<16)) + low4Bytes) / 90000
def ReadSCRMpeg1(self, buffer):
"""
read SCR (timestamp) for MPEG1 at the buffer beginning (5 Bytes)
"""
if len(buffer) < 5:
return None
highbit = (ord(buffer[0]) >> 3) & 0x01
low4Bytes = ((long(ord(buffer[0])) >> 1) & 0x03) << 30
low4Bytes |= ord(buffer[1]) << 22;
low4Bytes |= (ord(buffer[2]) >> 1) << 15;
low4Bytes |= ord(buffer[3]) << 7;
low4Bytes |= ord(buffer[4]) >> 1;
return (long(highbit) * (1<<16) * (1<<16) + low4Bytes) / 90000;
def ReadPTS(self, buffer):
"""
read PTS (PES timestamp) at the buffer beginning (5 Bytes)
"""
high = ((ord(buffer[0]) & 0xF) >> 1)
med = (ord(buffer[1]) << 7) + (ord(buffer[2]) >> 1)
low = (ord(buffer[3]) << 7) + (ord(buffer[4]) >> 1)
return ((long(high) << 30 ) + (med << 15) + low) / 90000
def ReadHeader(self, buffer, offset):
"""
Handle MPEG header in buffer on position offset
Return None on error, new offset or 0 if the new offset can't be scanned
"""
if buffer[offset:offset+3] != '\x00\x00\x01':
return None
id = ord(buffer[offset+3])
if id == PADDING_PKT:
return offset + (ord(buffer[offset+4]) << 8) + \
ord(buffer[offset+5]) + 6
if id == PACK_PKT:
if ord(buffer[offset+4]) & 0xF0 == 0x20:
self.type = 'MPEG-1 Video'
self.get_time = self.ReadSCRMpeg1
self.mpeg_version = 1
return offset + 12
elif (ord(buffer[offset+4]) & 0xC0) == 0x40:
self.type = 'MPEG-2 Video'
self.get_time = self.ReadSCRMpeg2
return offset + (ord(buffer[offset+13]) & 0x07) + 14
else:
# I have no idea what just happened, but for some DVB
# recordings done with mencoder this points to a
# PACK_PKT describing something odd. Returning 0 here
# (let's hope there are no extensions in the header)
# fixes it.
return 0
if 0xC0 <= id <= 0xDF:
# code for audio stream
for a in self.audio:
if a.id == id:
break
else:
self.audio.append(core.AudioStream())
self.audio[-1]._set('id', id)
return 0
if 0xE0 <= id <= 0xEF:
# code for video stream
for v in self.video:
if v.id == id:
break
else:
self.video.append(core.VideoStream())
self.video[-1]._set('id', id)
return 0
if id == SEQ_HEAD:
# sequence header, remember that position for later use
self.sequence_header_offset = offset
return 0
if id in [PRIVATE_STREAM1, PRIVATE_STREAM2]:
# private stream. we don't know, but maybe we can guess later
add = ord(buffer[offset+8])
# if (ord(buffer[offset+6]) & 4) or 1:
# id = ord(buffer[offset+10+add])
if buffer[offset+11+add:offset+15+add].find('\x0b\x77') != -1:
# AC3 stream
for a in self.audio:
if a.id == id:
break
else:
self.audio.append(core.AudioStream())
self.audio[-1]._set('id', id)
self.audio[-1].codec = 0x2000 # AC3
return 0
if id == SYS_PKT:
return 0
if id == EXT_START:
return 0
return 0
# Normal MPEG (VCD, SVCD) ========================================
def isMPEG(self, file, force=False):
"""
This MPEG starts with a sequence of 0x00 followed by a PACK Header
http://dvd.sourceforge.net/dvdinfo/packhdr.html
"""
file.seek(0,0)
buffer = file.read(10000)
offset = 0
# seek until the 0 byte stop
while offset < len(buffer)-100 and buffer[offset] == '\0':
offset += 1
offset -= 2
# test for mpeg header 0x00 0x00 0x01
header = '\x00\x00\x01%s' % chr(PACK_PKT)
if offset < 0 or not buffer[offset:offset+4] == header:
if not force:
return 0
# brute force and try to find the pack header in the first
# 10000 bytes somehow
offset = buffer.find(header)
if offset < 0:
return 0
# scan the 100000 bytes of data
buffer += file.read(100000)
# scan first header, to get basic info about
# how to read a timestamp
self.ReadHeader(buffer, offset)
# store first timestamp
self.start = self.get_time(buffer[offset+4:])
while len(buffer) > offset + 1000 and \
buffer[offset:offset+3] == '\x00\x00\x01':
# read the mpeg header
new_offset = self.ReadHeader(buffer, offset)
# header scanning detected error, this is no mpeg
if new_offset == None:
return 0
if new_offset:
# we have a new offset
offset = new_offset
# skip padding 0 before a new header
while len(buffer) > offset + 10 and \
not ord(buffer[offset+2]):
offset += 1
else:
# seek to new header by brute force
offset += buffer[offset+4:].find('\x00\x00\x01') + 4
# fill in values for support functions:
self.__seek_size__ = 1000000
self.__sample_size__ = 10000
self.__search__ = self._find_timer_
self.filename = file.name
# get length of the file
self.length = self.get_length()
return 1
def _find_timer_(self, buffer):
"""
Return position of timer in buffer or None if not found.
This function is valid for 'normal' mpeg files
"""
pos = buffer.find('\x00\x00\x01%s' % chr(PACK_PKT))
if pos == -1:
return None
return pos + 4
# PES ============================================================
def ReadPESHeader(self, offset, buffer, id=0):
"""
Parse a PES header.
Since it starts with 0x00 0x00 0x01 like 'normal' mpegs, this
function will return (0, None) when it is no PES header or
(packet length, timestamp position (maybe None))
http://dvd.sourceforge.net/dvdinfo/pes-hdr.html
"""
if not buffer[0:3] == '\x00\x00\x01':
return 0, None
packet_length = (ord(buffer[4]) << 8) + ord(buffer[5]) + 6
align = ord(buffer[6]) & 4
header_length = ord(buffer[8])
# PES ID (starting with 001)
if ord(buffer[3]) & 0xE0 == 0xC0:
id = id or ord(buffer[3]) & 0x1F
for a in self.audio:
if a.id == id:
break
else:
self.audio.append(core.AudioStream())
self.audio[-1]._set('id', id)
elif ord(buffer[3]) & 0xF0 == 0xE0:
id = id or ord(buffer[3]) & 0xF
for v in self.video:
if v.id == id:
break
else:
self.video.append(core.VideoStream())
self.video[-1]._set('id', id)
# new mpeg starting
if buffer[header_length+9:header_length+13] == \
'\x00\x00\x01\xB3' and not self.sequence_header_offset:
# yes, remember offset for later use
self.sequence_header_offset = offset + header_length+9
elif ord(buffer[3]) == 189 or ord(buffer[3]) == 191:
# private stream. we don't know, but maybe we can guess later
id = id or ord(buffer[3]) & 0xF
if align and \
buffer[header_length+9:header_length+11] == '\x0b\x77':
# AC3 stream
for a in self.audio:
if a.id == id:
break
else:
self.audio.append(core.AudioStream())
self.audio[-1]._set('id', id)
self.audio[-1].codec = 0x2000 # AC3
else:
# unknown content
pass
ptsdts = ord(buffer[7]) >> 6
if ptsdts and ptsdts == ord(buffer[9]) >> 4:
if ord(buffer[9]) >> 4 != ptsdts:
log.warning('WARNING: bad PTS/DTS, please contact us')
return packet_length, None
# timestamp = self.ReadPTS(buffer[9:14])
high = ((ord(buffer[9]) & 0xF) >> 1)
med = (ord(buffer[10]) << 7) + (ord(buffer[11]) >> 1)
low = (ord(buffer[12]) << 7) + (ord(buffer[13]) >> 1)
return packet_length, 9
return packet_length, None
def isPES(self, file):
log.info('trying mpeg-pes scan')
file.seek(0,0)
buffer = file.read(3)
# header (also valid for all mpegs)
if not buffer == '\x00\x00\x01':
return 0
self.sequence_header_offset = 0
buffer += file.read(10000)
offset = 0
while offset + 1000 < len(buffer):
pos, timestamp = self.ReadPESHeader(offset, buffer[offset:])
if not pos:
return 0
if timestamp != None and not hasattr(self, 'start'):
self.get_time = self.ReadPTS
bpos = buffer[offset+timestamp:offset+timestamp+5]
self.start = self.get_time(bpos)
if self.sequence_header_offset and hasattr(self, 'start'):
# we have all informations we need
break
offset += pos
if offset + 1000 < len(buffer) and len(buffer) < 1000000 or 1:
# looks like a pes, read more
buffer += file.read(10000)
if not self.video and not self.audio:
# no video and no audio?
return 0
self.type = 'MPEG-PES'
# fill in values for support functions:
self.__seek_size__ = 10000000 # 10 MB
self.__sample_size__ = 500000 # 500 k scanning
self.__search__ = self._find_timer_PES_
self.filename = file.name
# get length of the file
self.length = self.get_length()
return 1
def _find_timer_PES_(self, buffer):
"""
Return position of timer in buffer or -1 if not found.
This function is valid for PES files
"""
pos = buffer.find('\x00\x00\x01')
offset = 0
if pos == -1 or offset + 1000 >= len(buffer):
return None
retpos = -1
ackcount = 0
while offset + 1000 < len(buffer):
pos, timestamp = self.ReadPESHeader(offset, buffer[offset:])
if timestamp != None and retpos == -1:
retpos = offset + timestamp
if pos == 0:
# Oops, that was a mpeg header, no PES header
offset += buffer[offset:].find('\x00\x00\x01')
retpos = -1
ackcount = 0
else:
offset += pos
if retpos != -1:
ackcount += 1
if ackcount > 10:
# looks ok to me
return retpos
return None
# Elementary Stream ===============================================
def isES(self, file):
file.seek(0, 0)
try:
header = struct.unpack('>LL', file.read(8))
except (struct.error, IOError):
return False
if header[0] != 0x1B3:
return False
# Is an mpeg video elementary stream
self.mime = 'video/mpeg'
video = core.VideoStream()
video.width = header[1] >> 20
video.height = (header[1] >> 8) & 0xfff
if header[1] & 0xf < len(FRAME_RATE):
video.fps = FRAME_RATE[header[1] & 0xf]
if (header[1] >> 4) & 0xf < len(ASPECT_RATIO):
# FIXME: Empirically the aspect looks like PAR rather than DAR
video.aspect = ASPECT_RATIO[(header[1] >> 4) & 0xf]
self.video.append(video)
return True
# Transport Stream ===============================================
def isTS(self, file):
file.seek(0,0)
buffer = file.read(TS_PACKET_LENGTH * 2)
c = 0
while c + TS_PACKET_LENGTH < len(buffer):
if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC:
break
c += 1
else:
return 0
buffer += file.read(10000)
self.type = 'MPEG-TS'
while c + TS_PACKET_LENGTH < len(buffer):
start = ord(buffer[c+1]) & 0x40
# maybe load more into the buffer
if c + 2 * TS_PACKET_LENGTH > len(buffer) and c < 500000:
buffer += file.read(10000)
# wait until the ts payload contains a payload header
if not start:
c += TS_PACKET_LENGTH
continue
tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2])
adapt = (ord(buffer[c+3]) & 0x30) >> 4
offset = 4
if adapt & 0x02:
# meta info present, skip it for now
adapt_len = ord(buffer[c+offset])
offset += adapt_len + 1
if not ord(buffer[c+1]) & 0x40:
# no new pes or psi in stream payload starting
pass
elif adapt & 0x01:
# PES
timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:],
tsid)[1]
if timestamp != None:
if not hasattr(self, 'start'):
self.get_time = self.ReadPTS
timestamp = c + offset + timestamp
self.start = self.get_time(buffer[timestamp:timestamp+5])
elif not hasattr(self, 'audio_ok'):
timestamp = c + offset + timestamp
start = self.get_time(buffer[timestamp:timestamp+5])
if start is not None and self.start is not None and \
abs(start - self.start) < 10:
# looks ok
self.audio_ok = True
else:
# timestamp broken
del self.start
log.warning('Timestamp error, correcting')
if hasattr(self, 'start') and self.start and \
self.sequence_header_offset and self.video and self.audio:
break
c += TS_PACKET_LENGTH
if not self.sequence_header_offset:
return 0
# fill in values for support functions:
self.__seek_size__ = 10000000 # 10 MB
self.__sample_size__ = 100000 # 100 k scanning
self.__search__ = self._find_timer_TS_
self.filename = file.name
# get length of the file
self.length = self.get_length()
return 1
def _find_timer_TS_(self, buffer):
c = 0
while c + TS_PACKET_LENGTH < len(buffer):
if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC:
break
c += 1
else:
return None
while c + TS_PACKET_LENGTH < len(buffer):
start = ord(buffer[c+1]) & 0x40
if not start:
c += TS_PACKET_LENGTH
continue
tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2])
adapt = (ord(buffer[c+3]) & 0x30) >> 4
offset = 4
if adapt & 0x02:
# meta info present, skip it for now
offset += ord(buffer[c+offset]) + 1
if adapt & 0x01:
timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], tsid)[1]
if timestamp is None:
# this should not happen
log.error('bad TS')
return None
return c + offset + timestamp
c += TS_PACKET_LENGTH
return None
# Support functions ==============================================
def get_endpos(self):
"""
get the last timestamp of the mpeg, return -1 if this is not possible
"""
if not hasattr(self, 'filename') or not hasattr(self, 'start'):
return None
length = os.stat(self.filename)[stat.ST_SIZE]
if length < self.__sample_size__:
return
file = open(self.filename)
file.seek(length - self.__sample_size__)
buffer = file.read(self.__sample_size__)
end = None
while 1:
pos = self.__search__(buffer)
if pos == None:
break
end = self.get_time(buffer[pos:]) or end
buffer = buffer[pos+100:]
file.close()
return end
def get_length(self):
"""
get the length in seconds, return -1 if this is not possible
"""
end = self.get_endpos()
if end == None or self.start == None:
return None
if self.start > end:
return int(((long(1) << 33) - 1 ) / 90000) - self.start + end
return end - self.start
def seek(self, end_time):
"""
Return the byte position in the file where the time position
is 'pos' seconds. Return 0 if this is not possible
"""
if not hasattr(self, 'filename') or not hasattr(self, 'start'):
return 0
file = open(self.filename)
seek_to = 0
while 1:
file.seek(self.__seek_size__, 1)
buffer = file.read(self.__sample_size__)
if len(buffer) < 10000:
break
pos = self.__search__(buffer)
if pos != None:
# found something
nt = self.get_time(buffer[pos:])
if nt is not None and nt >= end_time:
# too much, break
break
# that wasn't enough
seek_to = file.tell()
file.close()
return seek_to
def __scan__(self):
"""
scan file for timestamps (may take a long time)
"""
if not hasattr(self, 'filename') or not hasattr(self, 'start'):
return 0
file = open(self.filename)
log.debug('scanning file...')
while 1:
file.seek(self.__seek_size__ * 10, 1)
buffer = file.read(self.__sample_size__)
if len(buffer) < 10000:
break
pos = self.__search__(buffer)
if pos == None:
continue
log.debug('buffer position: %s' % self.get_time(buffer[pos:]))
file.close()
log.debug('done scanning file')
Parser = MPEG
libs/enzyme/ogm.py
+301
View File
@@ -0,0 +1,301 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
import struct
import re
import stat
import os
import logging
from exceptions import *
import core
# get logging object
log = logging.getLogger(__name__)
PACKET_TYPE_HEADER = 0x01
PACKED_TYPE_METADATA = 0x03
PACKED_TYPE_SETUP = 0x05
PACKET_TYPE_BITS = 0x07
PACKET_IS_SYNCPOINT = 0x08
#VORBIS_VIDEO_PACKET_INFO = 'video'
STREAM_HEADER_VIDEO = '<4sIQQIIHII'
STREAM_HEADER_AUDIO = '<4sIQQIIHHHI'
VORBISCOMMENT = { 'TITLE': 'title',
'ALBUM': 'album',
'ARTIST': 'artist',
'COMMENT': 'comment',
'ENCODER': 'encoder',
'TRACKNUMBER': 'trackno',
'LANGUAGE': 'language',
'GENRE': 'genre',
}
# FIXME: check VORBISCOMMENT date and convert to timestamp
# Deactived tag: 'DATE': 'date',
MAXITERATIONS = 30
class Ogm(core.AVContainer):
table_mapping = { 'VORBISCOMMENT' : VORBISCOMMENT }
def __init__(self, file):
core.AVContainer.__init__(self)
self.samplerate = 1
self.all_streams = [] # used to add meta data to streams
self.all_header = []
for i in range(MAXITERATIONS):
granule, nextlen = self._parseOGGS(file)
if granule == None:
if i == 0:
# oops, bad file
raise ParseError()
break
elif granule > 0:
# ok, file started
break
# seek to the end of the stream, to avoid scanning the whole file
if (os.stat(file.name)[stat.ST_SIZE] > 50000):
file.seek(os.stat(file.name)[stat.ST_SIZE]-49000)
# read the rest of the file into a buffer
h = file.read()
# find last OggS to get length info
if len(h) > 200:
idx = h.find('OggS')
pos = -49000 + idx
if idx:
file.seek(os.stat(file.name)[stat.ST_SIZE] + pos)
while 1:
granule, nextlen = self._parseOGGS(file)
if not nextlen:
break
# Copy metadata to the streams
if len(self.all_header) == len(self.all_streams):
for i in range(len(self.all_header)):
# get meta info
for key in self.all_streams[i].keys():
if self.all_header[i].has_key(key):
self.all_streams[i][key] = self.all_header[i][key]
del self.all_header[i][key]
if self.all_header[i].has_key(key.upper()):
asi = self.all_header[i][key.upper()]
self.all_streams[i][key] = asi
del self.all_header[i][key.upper()]
# Chapter parser
if self.all_header[i].has_key('CHAPTER01') and \
not self.chapters:
while 1:
s = 'CHAPTER%02d' % (len(self.chapters) + 1)
if self.all_header[i].has_key(s) and \
self.all_header[i].has_key(s + 'NAME'):
pos = self.all_header[i][s]
try:
pos = int(pos)
except ValueError:
new_pos = 0
for v in pos.split(':'):
new_pos = new_pos * 60 + float(v)
pos = int(new_pos)
c = self.all_header[i][s + 'NAME']
c = core.Chapter(c, pos)
del self.all_header[i][s + 'NAME']
del self.all_header[i][s]
self.chapters.append(c)
else:
break
# If there are no video streams in this ogg container, it
# must be an audio file. Raise an exception to cause the
# factory to fall back to audio.ogg.
if len(self.video) == 0:
raise ParseError
# Copy Metadata from tables into the main set of attributes
for header in self.all_header:
self._appendtable('VORBISCOMMENT', header)
def _parseOGGS(self,file):
h = file.read(27)
if len(h) == 0:
# Regular File end
return None, None
elif len(h) < 27:
log.debug("%d Bytes of Garbage found after End." % len(h))
return None, None
if h[:4] != "OggS":
log.debug("Invalid Ogg")
raise ParseError()
version = ord(h[4])
if version != 0:
log.debug("Unsupported OGG/OGM Version %d." % version)
return None, None
head = struct.unpack('<BQIIIB', h[5:])
headertype, granulepos, serial, pageseqno, checksum, \
pageSegCount = head
self.mime = 'application/ogm'
self.type = 'OGG Media'
tab = file.read(pageSegCount)
nextlen = 0
for i in range(len(tab)):
nextlen += ord(tab[i])
else:
h = file.read(1)
packettype = ord(h[0]) & PACKET_TYPE_BITS
if packettype == PACKET_TYPE_HEADER:
h += file.read(nextlen-1)
self._parseHeader(h, granulepos)
elif packettype == PACKED_TYPE_METADATA:
h += file.read(nextlen-1)
self._parseMeta(h)
else:
file.seek(nextlen-1,1)
if len(self.all_streams) > serial:
stream = self.all_streams[serial]
if hasattr(stream, 'samplerate') and \
stream.samplerate:
stream.length = granulepos / stream.samplerate
elif hasattr(stream, 'bitrate') and \
stream.bitrate:
stream.length = granulepos / stream.bitrate
return granulepos, nextlen + 27 + pageSegCount
def _parseMeta(self,h):
flags = ord(h[0])
headerlen = len(h)
if headerlen >= 7 and h[1:7] == 'vorbis':
header = {}
nextlen, self.encoder = self._extractHeaderString(h[7:])
numItems = struct.unpack('<I',h[7+nextlen:7+nextlen+4])[0]
start = 7+4+nextlen
for i in range(numItems):
(nextlen, s) = self._extractHeaderString(h[start:])
start += nextlen
if s:
a = re.split('=',s)
header[(a[0]).upper()]=a[1]
# Put Header fields into info fields
self.type = 'OGG Vorbis'
self.subtype = ''
self.all_header.append(header)
def _parseHeader(self,header,granule):
headerlen = len(header)
flags = ord(header[0])
if headerlen >= 30 and header[1:7] == 'vorbis':
ai = core.AudioStream()
ai.version, ai.channels, ai.samplerate, bitrate_max, ai.bitrate, \
bitrate_min, blocksize, framing = \
struct.unpack('<IBIiiiBB',header[7:7+23])
ai.codec = 'Vorbis'
#ai.granule = granule
#ai.length = granule / ai.samplerate
self.audio.append(ai)
self.all_streams.append(ai)
elif headerlen >= 7 and header[1:7] == 'theora':
# Theora Header
# XXX Finish Me
vi = core.VideoStream()
vi.codec = 'theora'
self.video.append(vi)
self.all_streams.append(vi)
elif headerlen >= 142 and \
header[1:36] == 'Direct Show Samples embedded in Ogg':
# Old Directshow format
# XXX Finish Me
vi = core.VideoStream()
vi.codec = 'dshow'
self.video.append(vi)
self.all_streams.append(vi)
elif flags & PACKET_TYPE_BITS == PACKET_TYPE_HEADER and \
headerlen >= struct.calcsize(STREAM_HEADER_VIDEO)+1:
# New Directshow Format
htype = header[1:9]
if htype[:5] == 'video':
sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO)+9]
streamheader = struct.unpack(STREAM_HEADER_VIDEO, sh)
vi = core.VideoStream()
(type, ssize, timeunit, samplerate, vi.length, buffersize, \
vi.bitrate, vi.width, vi.height) = streamheader
vi.width /= 65536
vi.height /= 65536
# XXX length, bitrate are very wrong
vi.codec = type
vi.fps = 10000000 / timeunit
self.video.append(vi)
self.all_streams.append(vi)
elif htype[:5] == 'audio':
sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO)+9]
streamheader = struct.unpack(STREAM_HEADER_AUDIO, sha)
ai = core.AudioStream()
(type, ssize, timeunit, ai.samplerate, ai.length, buffersize, \
ai.bitrate, ai.channels, bloc, ai.bitrate) = streamheader
self.samplerate = ai.samplerate
log.debug("Samplerate %d" % self.samplerate)
self.audio.append(ai)
self.all_streams.append(ai)
elif htype[:4] == 'text':
subtitle = core.Subtitle()
# FIXME: add more info
self.subtitles.append(subtitle)
self.all_streams.append(subtitle)
else:
log.debug("Unknown Header")
def _extractHeaderString(self,header):
len = struct.unpack('<I', header[:4])[0]
try:
return (len+4,unicode(header[4:4+len], 'utf-8'))
except (KeyError, IndexError, UnicodeDecodeError):
return (len+4,None)
Parser = Ogm
libs/enzyme/real.py
+120
View File
@@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
import struct
import logging
from exceptions import *
import core
# http://www.pcisys.net/~melanson/codecs/rmff.htm
# http://www.pcisys.net/~melanson/codecs/
# get logging object
log = logging.getLogger(__name__)
class RealVideo(core.AVContainer):
def __init__(self,file):
core.AVContainer.__init__(self)
self.mime = 'video/real'
self.type = 'Real Video'
h = file.read(10)
try:
(object_id,object_size,object_version) = struct.unpack('>4sIH',h)
except struct.error:
# EOF.
raise ParseError()
if not object_id == '.RMF':
raise ParseError()
file_version, num_headers = struct.unpack('>II', file.read(8))
log.debug("size: %d, ver: %d, headers: %d" % \
(object_size, file_version,num_headers))
for i in range(0,num_headers):
try:
oi = struct.unpack('>4sIH',file.read(10))
except (struct.error, IOError):
# Header data we expected wasn't there. File may be
# only partially complete.
break
if object_id == 'DATA' and oi[0] != 'INDX':
log.debug('INDX chunk expected after DATA but not found -- file corrupt')
break
(object_id,object_size,object_version) = oi
if object_id == 'DATA':
# Seek over the data chunk rather than reading it in.
file.seek(object_size - 10, 1)
else:
self._read_header(object_id, file.read(object_size-10))
log.debug("%s [%d]" % (object_id,object_size-10))
# Read all the following headers
def _read_header(self,object_id,s):
if object_id == 'PROP':
prop = struct.unpack('>9IHH', s)
log.debug(prop)
if object_id == 'MDPR':
mdpr = struct.unpack('>H7I', s[:30])
log.debug(mdpr)
self.length = mdpr[7]/1000.0
(stream_name_size,) = struct.unpack('>B', s[30:31])
stream_name = s[31:31+stream_name_size]
pos = 31+stream_name_size
(mime_type_size,) = struct.unpack('>B', s[pos:pos+1])
mime = s[pos+1:pos+1+mime_type_size]
pos += mime_type_size+1
(type_specific_len,) = struct.unpack('>I', s[pos:pos+4])
type_specific = s[pos+4:pos+4+type_specific_len]
pos += 4+type_specific_len
if mime[:5] == 'audio':
ai = core.AudioStream()
ai.id = mdpr[0]
ai.bitrate = mdpr[2]
self.audio.append(ai)
elif mime[:5] == 'video':
vi = core.VideoStream()
vi.id = mdpr[0]
vi.bitrate = mdpr[2]
self.video.append(vi)
else:
log.debug("Unknown: %s" % mime)
if object_id == 'CONT':
pos = 0
(title_len,) = struct.unpack('>H', s[pos:pos+2])
self.title = s[2:title_len+2]
pos += title_len+2
(author_len,) = struct.unpack('>H', s[pos:pos+2])
self.artist = s[pos+2:pos+author_len+2]
pos += author_len+2
(copyright_len,) = struct.unpack('>H', s[pos:pos+2])
self.copyright = s[pos+2:pos+copyright_len+2]
pos += copyright_len+2
(comment_len,) = struct.unpack('>H', s[pos:pos+2])
self.comment = s[pos+2:pos+comment_len+2]
Parser = RealVideo
libs/enzyme/riff.py
+568
View File
@@ -0,0 +1,568 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2003-2006 Thomas Schueppel <stain@acm.org>
# Copyright (C) 2003-2006 Dirk Meyer <dischi@freevo.org>
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['Parser']
import os
import struct
import string
import logging
import time
from exceptions import *
import core
# get logging object
log = logging.getLogger(__name__)
# List of tags
# http://kibus1.narod.ru/frames_eng.htm?sof/abcavi/infotags.htm
# http://www.divx-digest.com/software/avitags_dll.html
# File Format: google for odmlff2.pdf
AVIINFO = {
'INAM': 'title',
'IART': 'artist',
'IPRD': 'product',
'ISFT': 'software',
'ICMT': 'comment',
'ILNG': 'language',
'IKEY': 'keywords',
'IPRT': 'trackno',
'IFRM': 'trackof',
'IPRO': 'producer',
'IWRI': 'writer',
'IGNR': 'genre',
'ICOP': 'copyright'
}
# Taken from libavcodec/mpeg4data.h (pixel_aspect struct)
PIXEL_ASPECT = {
1: (1, 1),
2: (12, 11),
3: (10, 11),
4: (16, 11),
5: (40, 33)
}
class Riff(core.AVContainer):
"""
AVI parser also parsing metadata like title, languages, etc.
"""
table_mapping = { 'AVIINFO' : AVIINFO }
def __init__(self,file):
core.AVContainer.__init__(self)
# read the header
h = file.read(12)
if h[:4] != "RIFF" and h[:4] != 'SDSS':
raise ParseError()
self.has_idx = False
self.header = {}
self.junkStart = None
self.infoStart = None
self.type = h[8:12]
if self.type == 'AVI ':
self.mime = 'video/avi'
elif self.type == 'WAVE':
self.mime = 'audio/wav'
try:
while self._parseRIFFChunk(file):
pass
except IOError:
log.exception('error in file, stop parsing')
self._find_subtitles(file.name)
if not self.has_idx and isinstance(self, core.AVContainer):
log.debug('WARNING: avi has no index')
self._set('corrupt', True)
def _find_subtitles(self, filename):
"""
Search for subtitle files. Right now only VobSub is supported
"""
base = os.path.splitext(filename)[0]
if os.path.isfile(base+'.idx') and \
(os.path.isfile(base+'.sub') or os.path.isfile(base+'.rar')):
file = open(base+'.idx')
if file.readline().find('VobSub index file') > 0:
for line in file.readlines():
if line.find('id') == 0:
sub = core.Subtitle()
sub.language = line[4:6]
sub.trackno = base + '.idx' # Maybe not?
self.subtitles.append(sub)
file.close()
def _parseAVIH(self,t):
retval = {}
v = struct.unpack('<IIIIIIIIIIIIII',t[0:56])
( retval['dwMicroSecPerFrame'],
retval['dwMaxBytesPerSec'],
retval['dwPaddingGranularity'],
retval['dwFlags'],
retval['dwTotalFrames'],
retval['dwInitialFrames'],
retval['dwStreams'],
retval['dwSuggestedBufferSize'],
retval['dwWidth'],
retval['dwHeight'],
retval['dwScale'],
retval['dwRate'],
retval['dwStart'],
retval['dwLength'] ) = v
if retval['dwMicroSecPerFrame'] == 0:
log.warning("ERROR: Corrupt AVI")
raise ParseError()
return retval
def _parseSTRH(self,t):
retval = {}
retval['fccType'] = t[0:4]
log.debug("_parseSTRH(%s) : %d bytes" % (retval['fccType'], len(t)))
if retval['fccType'] != 'auds':
retval['fccHandler'] = t[4:8]
v = struct.unpack('<IHHIIIIIIIII',t[8:52])
( retval['dwFlags'],
retval['wPriority'],
retval['wLanguage'],
retval['dwInitialFrames'],
retval['dwScale'],
retval['dwRate'],
retval['dwStart'],
retval['dwLength'],
retval['dwSuggestedBufferSize'],
retval['dwQuality'],
retval['dwSampleSize'],
retval['rcFrame'] ) = v
else:
try:
v = struct.unpack('<IHHIIIIIIIII',t[8:52])
( retval['dwFlags'],
retval['wPriority'],
retval['wLanguage'],
retval['dwInitialFrames'],
retval['dwScale'],
retval['dwRate'],
retval['dwStart'],
retval['dwLength'],
retval['dwSuggestedBufferSize'],
retval['dwQuality'],
retval['dwSampleSize'],
retval['rcFrame'] ) = v
self.delay = float(retval['dwStart']) / \
(float(retval['dwRate']) / retval['dwScale'])
except (KeyError, IndexError, ValueError, ZeroDivisionError):
pass
return retval
def _parseSTRF(self,t,strh):
fccType = strh['fccType']
retval = {}
if fccType == 'auds':
v = struct.unpack('<HHHHHH',t[0:12])
( retval['wFormatTag'],
retval['nChannels'],
retval['nSamplesPerSec'],
retval['nAvgBytesPerSec'],
retval['nBlockAlign'],
retval['nBitsPerSample'],
) = v
ai = core.AudioStream()
ai.samplerate = retval['nSamplesPerSec']
ai.channels = retval['nChannels']
# FIXME: Bitrate calculation is completely wrong.
#ai.samplebits = retval['nBitsPerSample']
#ai.bitrate = retval['nAvgBytesPerSec'] * 8
# TODO: set code if possible
# http://www.stats.uwa.edu.au/Internal/Specs/DXALL/FileSpec/\
# Languages
# ai.language = strh['wLanguage']
ai.codec = retval['wFormatTag']
self.audio.append(ai)
elif fccType == 'vids':
v = struct.unpack('<IIIHH',t[0:16])
( retval['biSize'],
retval['biWidth'],
retval['biHeight'],
retval['biPlanes'],
retval['biBitCount'] ) = v
v = struct.unpack('IIIII',t[20:40])
( retval['biSizeImage'],
retval['biXPelsPerMeter'],
retval['biYPelsPerMeter'],
retval['biClrUsed'],
retval['biClrImportant'] ) = v
vi = core.VideoStream()
vi.codec = t[16:20]
vi.width = retval['biWidth']
vi.height = retval['biHeight']
# FIXME: Bitrate calculation is completely wrong.
#vi.bitrate = strh['dwRate']
vi.fps = float(strh['dwRate']) / strh['dwScale']
vi.length = strh['dwLength'] / vi.fps
self.video.append(vi)
return retval
def _parseSTRL(self,t):
retval = {}
size = len(t)
i = 0
while i < len(t) - 8:
key = t[i:i+4]
sz = struct.unpack('<I',t[i+4:i+8])[0]
i+=8
value = t[i:]
if key == 'strh':
retval[key] = self._parseSTRH(value)
elif key == 'strf':
retval[key] = self._parseSTRF(value, retval['strh'])
else:
log.debug("_parseSTRL: unsupported stream tag '%s'", key)
i += sz
return retval, i
def _parseODML(self,t):
retval = {}
size = len(t)
i = 0
key = t[i:i+4]
sz = struct.unpack('<I',t[i+4:i+8])[0]
i += 8
value = t[i:]
if key != 'dmlh':
log.debug("_parseODML: Error")
i += sz - 8
return (retval, i)
def _parseVPRP(self,t):
retval = {}
v = struct.unpack('<IIIIIIIIII',t[:4*10])
( retval['VideoFormat'],
retval['VideoStandard'],
retval['RefreshRate'],
retval['HTotalIn'],
retval['VTotalIn'],
retval['FrameAspectRatio'],
retval['wPixel'],
retval['hPixel'] ) = v[1:-1]
# I need an avi with more informations
# enum {FORMAT_UNKNOWN, FORMAT_PAL_SQUARE, FORMAT_PAL_CCIR_601,
# FORMAT_NTSC_SQUARE, FORMAT_NTSC_CCIR_601,...} VIDEO_FORMAT;
# enum {STANDARD_UNKNOWN, STANDARD_PAL, STANDARD_NTSC, STANDARD_SECAM}
# VIDEO_STANDARD;
#
r = retval['FrameAspectRatio']
r = float(r >> 16) / (r & 0xFFFF)
retval['FrameAspectRatio'] = r
if self.video:
map(lambda v: setattr(v, 'aspect', r), self.video)
return (retval, v[0])
def _parseLISTmovi(self, size, file):
"""
Digs into movi list, looking for a Video Object Layer header in an
mpeg4 stream in order to determine aspect ratio.
"""
i = 0
n_dc = 0
done = False
# If the VOL header doesn't appear within 5MB or 5 video chunks,
# give up. The 5MB limit is not likely to apply except in
# pathological cases.
while i < min(1024*1024*5, size - 8) and n_dc < 5:
data = file.read(8)
if ord(data[0]) == 0:
# Eat leading nulls.
data = data[1:] + file.read(1)
i += 1
key, sz = struct.unpack('<4sI', data)
if key[2:] != 'dc' or sz > 1024*500:
# This chunk is not video or is unusually big (> 500KB);
# skip it.
file.seek(sz, 1)
i += 8 + sz
continue
n_dc += 1
# Read video chunk into memory
data = file.read(sz)
#for p in range(0,min(80, sz)):
# print "%02x " % ord(data[p]),
#print "\n\n"
# Look through the picture header for VOL startcode. The basic
# logic for this is taken from libavcodec, h263.c
pos = 0
startcode = 0xff
def bits(v, o, n):
# Returns n bits in v, offset o bits.
return (v & 2**n-1 << (64-n-o)) >> 64-n-o
while pos < sz:
startcode = ((startcode << 8) | ord(data[pos])) & 0xffffffff
pos += 1
if startcode & 0xFFFFFF00 != 0x100:
# No startcode found yet
continue
if startcode >= 0x120 and startcode <= 0x12F:
# We have the VOL startcode. Pull 64 bits of it and treat
# as a bitstream
v = struct.unpack(">Q", data[pos : pos+8])[0]
offset = 10
if bits(v, 9, 1):
# is_ol_id, skip over vo_ver_id and vo_priority
offset += 7
ar_info = bits(v, offset, 4)
if ar_info == 15:
# Extended aspect
num = bits(v, offset + 4, 8)
den = bits(v, offset + 12, 8)
else:
# A standard pixel aspect
num, den = PIXEL_ASPECT.get(ar_info, (0, 0))
# num/den indicates pixel aspect; convert to video aspect,
# so we need frame width and height.
if 0 not in [num, den]:
width, height = self.video[-1].width, self.video[-1].height
self.video[-1].aspect = num / float(den) * width / height
done = True
break
startcode = 0xff
i += 8 + len(data)
if done:
# We have the aspect, no need to continue parsing the movi
# list, so break out of the loop.
break
if i < size:
# Seek past whatever might be remaining of the movi list.
file.seek(size-i,1)
def _parseLIST(self,t):
retval = {}
i = 0
size = len(t)
while i < size-8:
# skip zero
if ord(t[i]) == 0: i += 1
key = t[i:i+4]
sz = 0
if key == 'LIST':
sz = struct.unpack('<I',t[i+4:i+8])[0]
i+=8
key = "LIST:"+t[i:i+4]
value = self._parseLIST(t[i:i+sz])
if key == 'strl':
for k in value.keys():
retval[k] = value[k]
else:
retval[key] = value
i+=sz
elif key == 'avih':
sz = struct.unpack('<I',t[i+4:i+8])[0]
i += 8
value = self._parseAVIH(t[i:i+sz])
i += sz
retval[key] = value
elif key == 'strl':
i += 4
(value, sz) = self._parseSTRL(t[i:])
key = value['strh']['fccType']
i += sz
retval[key] = value
elif key == 'odml':
i += 4
(value, sz) = self._parseODML(t[i:])
i += sz
elif key == 'vprp':
i += 4
(value, sz) = self._parseVPRP(t[i:])
retval[key] = value
i += sz
elif key == 'JUNK':
sz = struct.unpack('<I',t[i+4:i+8])[0]
i += sz + 8
else:
sz = struct.unpack('<I',t[i+4:i+8])[0]
i+=8
# in most cases this is some info stuff
if not key in AVIINFO.keys() and key != 'IDIT':
log.debug("Unknown Key: %s, len: %d" % (key,sz))
value = t[i:i+sz]
if key == 'ISFT':
# product information
if value.find('\0') > 0:
# works for Casio S500 camera videos
value = value[:value.find('\0')]
value = value.replace('\0', '').lstrip().rstrip()
value = value.replace('\0', '').lstrip().rstrip()
if value:
retval[key] = value
if key in ['IDIT', 'ICRD']:
# Timestamp the video was created. Spec says it
# should be a format like "Wed Jan 02 02:03:55 1990"
# Casio S500 uses "2005/12/24/ 14:11", but I've
# also seen "December 24, 2005"
specs = ('%a %b %d %H:%M:%S %Y', '%Y/%m/%d/ %H:%M', '%B %d, %Y')
for tmspec in specs:
try:
tm = time.strptime(value, tmspec)
# save timestamp as int
self.timestamp = int(time.mktime(tm))
break
except ValueError:
pass
else:
log.debug('no support for time format %s', value)
i+=sz
return retval
def _parseRIFFChunk(self,file):
h = file.read(8)
if len(h) < 8:
return False
name = h[:4]
size = struct.unpack('<I',h[4:8])[0]
if name == 'LIST':
pos = file.tell() - 8
key = file.read(4)
if key == 'movi' and self.video and not self.video[-1].aspect and \
self.video[-1].width and self.video[-1].height and \
self.video[-1].format in ['DIVX', 'XVID', 'FMP4']: # any others?
# If we don't have the aspect (i.e. it isn't in odml vprp
# header), but we do know the video's dimensions, and
# we're dealing with an mpeg4 stream, try to get the aspect
# from the VOL header in the mpeg4 stream.
self._parseLISTmovi(size-4, file)
return True
elif size > 80000:
log.debug('RIFF LIST "%s" too long to parse: %s bytes' % (key, size))
t = file.seek(size-4,1)
return True
elif size < 5:
log.debug('RIFF LIST "%s" too short: %s bytes' % (key, size))
return True
t = file.read(size-4)
log.debug('parse RIFF LIST "%s": %d bytes' % (key, size))
value = self._parseLIST(t)
self.header[key] = value
if key == 'INFO':
self.infoStart = pos
self._appendtable('AVIINFO', value)
elif key == 'MID ':
self._appendtable('AVIMID', value)
elif key == 'hdrl':
# no need to add this info to a table
pass
else:
log.debug('Skipping table info %s' % key)
elif name == 'JUNK':
self.junkStart = file.tell() - 8
self.junkSize = size
file.seek(size, 1)
elif name == 'idx1':
self.has_idx = True
log.debug('idx1: %s bytes' % size)
# no need to parse this
t = file.seek(size,1)
elif name == 'RIFF':
log.debug("New RIFF chunk, extended avi [%i]" % size)
type = file.read(4)
if type != 'AVIX':
log.debug("Second RIFF chunk is %s, not AVIX, skipping", type)
file.seek(size-4, 1)
# that's it, no new informations should be in AVIX
return False
elif name == 'fmt ' and size <= 50:
# This is a wav file.
data = file.read(size)
fmt = struct.unpack("<HHLLHH", data[:16])
self._set('codec', hex(fmt[0]))
self._set('samplerate', fmt[2])
# fmt[3] is average bytes per second, so we must divide it
# by 125 to get kbits per second
self._set('bitrate', fmt[3] / 125)
# ugly hack: remember original rate in bytes per second
# so that the length can be calculated in next elif block
self._set('byterate', fmt[3])
# Set a dummy fourcc so codec will be resolved in finalize.
self._set('fourcc', 'dummy')
elif name == 'data':
# XXX: this is naive and may not be right. For example if the
# stream is something that supports VBR like mp3, the value
# will be off. The only way to properly deal with this issue
# is to decode part of the stream based on its codec, but
# kaa.metadata doesn't have this capability (yet?)
# ugly hack: use original rate in bytes per second
self._set('length', size / float(self.byterate))
file.seek(size, 1)
elif not name.strip(string.printable + string.whitespace):
# check if name is something usefull at all, maybe it is no
# avi or broken
t = file.seek(size,1)
log.debug("Skipping %s [%i]" % (name,size))
else:
# bad avi
log.debug("Bad or broken avi")
return False
return True
Parser = Riff
libs/enzyme/strutils.py
+80
View File
@@ -0,0 +1,80 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
# Copyright (C) 2011 Antoine Bertin <diaoulael@gmail.com>
# Copyright (C) 2006-2009 Dirk Meyer <dischi@freevo.org>
# Copyright (C) 2006-2009 Jason Tackaberry
#
# This file is part of enzyme.
#
# enzyme is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# enzyme is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ['ENCODING', 'str_to_unicode', 'unicode_to_str']
import locale
# find the correct encoding
try:
ENCODING = locale.getdefaultlocale()[1]
''.encode(ENCODING)
except (UnicodeError, TypeError):
ENCODING = 'latin-1'
def str_to_unicode(s, encoding=None):
"""
Attempts to convert a string of unknown character set to a unicode
string. First it tries to decode the string based on the locale's
preferred encoding, and if that fails, fall back to UTF-8 and then
latin-1. If all fails, it will force encoding to the preferred
charset, replacing unknown characters. If the given object is no
string, this function will return the given object.
"""
if not type(s) == str:
return s
if not encoding:
encoding = ENCODING
for c in [encoding, "utf-8", "latin-1"]:
try:
return s.decode(c)
except UnicodeDecodeError:
pass
return s.decode(encoding, "replace")
def unicode_to_str(s, encoding=None):
"""
Attempts to convert a unicode string of unknown character set to a
string. First it tries to encode the string based on the locale's
preferred encoding, and if that fails, fall back to UTF-8 and then
latin-1. If all fails, it will force encoding to the preferred
charset, replacing unknown characters. If the given object is no
unicode string, this function will return the given object.
"""
if not type(s) == unicode:
return s
if not encoding:
encoding = ENCODING
for c in [encoding, "utf-8", "latin-1"]:
try:
return s.encode(c)
except UnicodeDecodeError:
pass
return s.encode(encoding, "replace")
libs/guessit/ISO-639-2_utf-8.txt
+485
View File
@@ -0,0 +1,485 @@
aar||aa|Afar|afar
abk||ab|Abkhazian|abkhaze
ace|||Achinese|aceh
ach|||Acoli|acoli
ada|||Adangme|adangme
ady|||Adyghe; Adygei|adyghé
afa|||Afro-Asiatic languages|afro-asiatiques, langues
afh|||Afrihili|afrihili
afr||af|Afrikaans|afrikaans
ain|||Ainu|aïnou
aka||ak|Akan|akan
akk|||Akkadian|akkadien
alb|sqi|sq|Albanian|albanais
ale|||Aleut|aléoute
alg|||Algonquian languages|algonquines, langues
alt|||Southern Altai|altai du Sud
amh||am|Amharic|amharique
ang|||English, Old (ca.450-1100)|anglo-saxon (ca.450-1100)
anp|||Angika|angika
apa|||Apache languages|apaches, langues
ara||ar|Arabic|arabe
arc|||Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)|araméen d'empire (700-300 BCE)
arg||an|Aragonese|aragonais
arm|hye|hy|Armenian|arménien
arn|||Mapudungun; Mapuche|mapudungun; mapuche; mapuce
arp|||Arapaho|arapaho
art|||Artificial languages|artificielles, langues
arw|||Arawak|arawak
asm||as|Assamese|assamais
ast|||Asturian; Bable; Leonese; Asturleonese|asturien; bable; léonais; asturoléonais
ath|||Athapascan languages|athapascanes, langues
aus|||Australian languages|australiennes, langues
ava||av|Avaric|avar
ave||ae|Avestan|avestique
awa|||Awadhi|awadhi
aym||ay|Aymara|aymara
aze||az|Azerbaijani|azéri
bad|||Banda languages|banda, langues
bai|||Bamileke languages|bamiléké, langues
bak||ba|Bashkir|bachkir
bal|||Baluchi|baloutchi
bam||bm|Bambara|bambara
ban|||Balinese|balinais
baq|eus|eu|Basque|basque
bas|||Basa|basa
bat|||Baltic languages|baltes, langues
bej|||Beja; Bedawiyet|bedja
bel||be|Belarusian|biélorusse
bem|||Bemba|bemba
ben||bn|Bengali|bengali
ber|||Berber languages|berbères, langues
bho|||Bhojpuri|bhojpuri
bih||bh|Bihari languages|langues biharis
bik|||Bikol|bikol
bin|||Bini; Edo|bini; edo
bis||bi|Bislama|bichlamar
bla|||Siksika|blackfoot
bnt|||Bantu (Other)|bantoues, autres langues
bos||bs|Bosnian|bosniaque
bra|||Braj|braj
bre||br|Breton|breton
btk|||Batak languages|batak, langues
bua|||Buriat|bouriate
bug|||Buginese|bugi
bul||bg|Bulgarian|bulgare
bur|mya|my|Burmese|birman
byn|||Blin; Bilin|blin; bilen
cad|||Caddo|caddo
cai|||Central American Indian languages|amérindiennes de L'Amérique centrale, langues
car|||Galibi Carib|karib; galibi; carib
cat||ca|Catalan; Valencian|catalan; valencien
cau|||Caucasian languages|caucasiennes, langues
ceb|||Cebuano|cebuano
cel|||Celtic languages|celtiques, langues; celtes, langues
cha||ch|Chamorro|chamorro
chb|||Chibcha|chibcha
che||ce|Chechen|tchétchène
chg|||Chagatai|djaghataï
chi|zho|zh|Chinese|chinois
chk|||Chuukese|chuuk
chm|||Mari|mari
chn|||Chinook jargon|chinook, jargon
cho|||Choctaw|choctaw
chp|||Chipewyan; Dene Suline|chipewyan
chr|||Cherokee|cherokee
chu||cu|Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic|slavon d'église; vieux slave; slavon liturgique; vieux bulgare
chv||cv|Chuvash|tchouvache
chy|||Cheyenne|cheyenne
cmc|||Chamic languages|chames, langues
cop|||Coptic|copte
cor||kw|Cornish|cornique
cos||co|Corsican|corse
cpe|||Creoles and pidgins, English based|créoles et pidgins basés sur l'anglais
cpf|||Creoles and pidgins, French-based |créoles et pidgins basés sur le français
cpp|||Creoles and pidgins, Portuguese-based |créoles et pidgins basés sur le portugais
cre||cr|Cree|cree
crh|||Crimean Tatar; Crimean Turkish|tatar de Crimé
crp|||Creoles and pidgins |créoles et pidgins
csb|||Kashubian|kachoube
cus|||Cushitic languages|couchitiques, langues
cze|ces|cs|Czech|tchèque
dak|||Dakota|dakota
dan||da|Danish|danois
dar|||Dargwa|dargwa
day|||Land Dayak languages|dayak, langues
del|||Delaware|delaware
den|||Slave (Athapascan)|esclave (athapascan)
dgr|||Dogrib|dogrib
din|||Dinka|dinka
div||dv|Divehi; Dhivehi; Maldivian|maldivien
doi|||Dogri|dogri
dra|||Dravidian languages|dravidiennes, langues
dsb|||Lower Sorbian|bas-sorabe
dua|||Duala|douala
dum|||Dutch, Middle (ca.1050-1350)|néerlandais moyen (ca. 1050-1350)
dut|nld|nl|Dutch; Flemish|néerlandais; flamand
dyu|||Dyula|dioula
dzo||dz|Dzongkha|dzongkha
efi|||Efik|efik
egy|||Egyptian (Ancient)|égyptien
eka|||Ekajuk|ekajuk
elx|||Elamite|élamite
eng||en|English|anglais
enm|||English, Middle (1100-1500)|anglais moyen (1100-1500)
epo||eo|Esperanto|espéranto
est||et|Estonian|estonien
ewe||ee|Ewe|éwé
ewo|||Ewondo|éwondo
fan|||Fang|fang
fao||fo|Faroese|féroïen
fat|||Fanti|fanti
fij||fj|Fijian|fidjien
fil|||Filipino; Pilipino|filipino; pilipino
fin||fi|Finnish|finnois
fiu|||Finno-Ugrian languages|finno-ougriennes, langues
fon|||Fon|fon
fre|fra|fr|French|français
frm|||French, Middle (ca.1400-1600)|français moyen (1400-1600)
fro|||French, Old (842-ca.1400)|français ancien (842-ca.1400)
frr|||Northern Frisian|frison septentrional
frs|||Eastern Frisian|frison oriental
fry||fy|Western Frisian|frison occidental
ful||ff|Fulah|peul
fur|||Friulian|frioulan
gaa|||Ga|ga
gay|||Gayo|gayo
gba|||Gbaya|gbaya
gem|||Germanic languages|germaniques, langues
geo|kat|ka|Georgian|géorgien
ger|deu|de|German|allemand
gez|||Geez|guèze
gil|||Gilbertese|kiribati
gla||gd|Gaelic; Scottish Gaelic|gaélique; gaélique écossais
gle||ga|Irish|irlandais
glg||gl|Galician|galicien
glv||gv|Manx|manx; mannois
gmh|||German, Middle High (ca.1050-1500)|allemand, moyen haut (ca. 1050-1500)
goh|||German, Old High (ca.750-1050)|allemand, vieux haut (ca. 750-1050)
gon|||Gondi|gond
gor|||Gorontalo|gorontalo
got|||Gothic|gothique
grb|||Grebo|grebo
grc|||Greek, Ancient (to 1453)|grec ancien (jusqu'à 1453)
gre|ell|el|Greek, Modern (1453-)|grec moderne (après 1453)
grn||gn|Guarani|guarani
gsw|||Swiss German; Alemannic; Alsatian|suisse alémanique; alémanique; alsacien
guj||gu|Gujarati|goudjrati
gwi|||Gwich'in|gwich'in
hai|||Haida|haida
hat||ht|Haitian; Haitian Creole|haïtien; créole haïtien
hau||ha|Hausa|haoussa
haw|||Hawaiian|hawaïen
heb||he|Hebrew|hébreu
her||hz|Herero|herero
hil|||Hiligaynon|hiligaynon
him|||Himachali languages; Western Pahari languages|langues himachalis; langues paharis occidentales
hin||hi|Hindi|hindi
hit|||Hittite|hittite
hmn|||Hmong; Mong|hmong
hmo||ho|Hiri Motu|hiri motu
hrv||hr|Croatian|croate
hsb|||Upper Sorbian|haut-sorabe
hun||hu|Hungarian|hongrois
hup|||Hupa|hupa
iba|||Iban|iban
ibo||ig|Igbo|igbo
ice|isl|is|Icelandic|islandais
ido||io|Ido|ido
iii||ii|Sichuan Yi; Nuosu|yi de Sichuan
ijo|||Ijo languages|ijo, langues
iku||iu|Inuktitut|inuktitut
ile||ie|Interlingue; Occidental|interlingue
ilo|||Iloko|ilocano
ina||ia|Interlingua (International Auxiliary Language Association)|interlingua (langue auxiliaire internationale)
inc|||Indic languages|indo-aryennes, langues
ind||id|Indonesian|indonésien
ine|||Indo-European languages|indo-européennes, langues
inh|||Ingush|ingouche
ipk||ik|Inupiaq|inupiaq
ira|||Iranian languages|iraniennes, langues
iro|||Iroquoian languages|iroquoises, langues
ita||it|Italian|italien
jav||jv|Javanese|javanais
jbo|||Lojban|lojban
jpn||ja|Japanese|japonais
jpr|||Judeo-Persian|judéo-persan
jrb|||Judeo-Arabic|judéo-arabe
kaa|||Kara-Kalpak|karakalpak
kab|||Kabyle|kabyle
kac|||Kachin; Jingpho|kachin; jingpho
kal||kl|Kalaallisut; Greenlandic|groenlandais
kam|||Kamba|kamba
kan||kn|Kannada|kannada
kar|||Karen languages|karen, langues
kas||ks|Kashmiri|kashmiri
kau||kr|Kanuri|kanouri
kaw|||Kawi|kawi
kaz||kk|Kazakh|kazakh
kbd|||Kabardian|kabardien
kha|||Khasi|khasi
khi|||Khoisan languages|khoïsan, langues
khm||km|Central Khmer|khmer central
kho|||Khotanese; Sakan|khotanais; sakan
kik||ki|Kikuyu; Gikuyu|kikuyu
kin||rw|Kinyarwanda|rwanda
kir||ky|Kirghiz; Kyrgyz|kirghiz
kmb|||Kimbundu|kimbundu
kok|||Konkani|konkani
kom||kv|Komi|kom
kon||kg|Kongo|kongo
kor||ko|Korean|coréen
kos|||Kosraean|kosrae
kpe|||Kpelle|kpellé
krc|||Karachay-Balkar|karatchai balkar
krl|||Karelian|carélien
kro|||Kru languages|krou, langues
kru|||Kurukh|kurukh
kua||kj|Kuanyama; Kwanyama|kuanyama; kwanyama
kum|||Kumyk|koumyk
kur||ku|Kurdish|kurde
kut|||Kutenai|kutenai
lad|||Ladino|judéo-espagnol
lah|||Lahnda|lahnda
lam|||Lamba|lamba
lao||lo|Lao|lao
lat||la|Latin|latin
lav||lv|Latvian|letton
lez|||Lezghian|lezghien
lim||li|Limburgan; Limburger; Limburgish|limbourgeois
lin||ln|Lingala|lingala
lit||lt|Lithuanian|lituanien
lol|||Mongo|mongo
loz|||Lozi|lozi
ltz||lb|Luxembourgish; Letzeburgesch|luxembourgeois
lua|||Luba-Lulua|luba-lulua
lub||lu|Luba-Katanga|luba-katanga
lug||lg|Ganda|ganda
lui|||Luiseno|luiseno
lun|||Lunda|lunda
luo|||Luo (Kenya and Tanzania)|luo (Kenya et Tanzanie)
lus|||Lushai|lushai
mac|mkd|mk|Macedonian|macédonien
mad|||Madurese|madourais
mag|||Magahi|magahi
mah||mh|Marshallese|marshall
mai|||Maithili|maithili
mak|||Makasar|makassar
mal||ml|Malayalam|malayalam
man|||Mandingo|mandingue
mao|mri|mi|Maori|maori
map|||Austronesian languages|austronésiennes, langues
mar||mr|Marathi|marathe
mas|||Masai|massaï
may|msa|ms|Malay|malais
mdf|||Moksha|moksa
mdr|||Mandar|mandar
men|||Mende|mendé
mga|||Irish, Middle (900-1200)|irlandais moyen (900-1200)
mic|||Mi'kmaq; Micmac|mi'kmaq; micmac
min|||Minangkabau|minangkabau
mis|||Uncoded languages|langues non codées
mkh|||Mon-Khmer languages|môn-khmer, langues
mlg||mg|Malagasy|malgache
mlt||mt|Maltese|maltais
mnc|||Manchu|mandchou
mni|||Manipuri|manipuri
mno|||Manobo languages|manobo, langues
moh|||Mohawk|mohawk
mon||mn|Mongolian|mongol
mos|||Mossi|moré
mul|||Multiple languages|multilingue
mun|||Munda languages|mounda, langues
mus|||Creek|muskogee
mwl|||Mirandese|mirandais
mwr|||Marwari|marvari
myn|||Mayan languages|maya, langues
myv|||Erzya|erza
nah|||Nahuatl languages|nahuatl, langues
nai|||North American Indian languages|nord-amérindiennes, langues
nap|||Neapolitan|napolitain
nau||na|Nauru|nauruan
nav||nv|Navajo; Navaho|navaho
nbl||nr|Ndebele, South; South Ndebele|ndébélé du Sud
nde||nd|Ndebele, North; North Ndebele|ndébélé du Nord
ndo||ng|Ndonga|ndonga
nds|||Low German; Low Saxon; German, Low; Saxon, Low|bas allemand; bas saxon; allemand, bas; saxon, bas
nep||ne|Nepali|népalais
new|||Nepal Bhasa; Newari|nepal bhasa; newari
nia|||Nias|nias
nic|||Niger-Kordofanian languages|nigéro-kordofaniennes, langues
niu|||Niuean|niué
nno||nn|Norwegian Nynorsk; Nynorsk, Norwegian|norvégien nynorsk; nynorsk, norvégien
nob||nb|Bokmål, Norwegian; Norwegian Bokmål|norvégien bokmål
nog|||Nogai|nogaï; nogay
non|||Norse, Old|norrois, vieux
nor||no|Norwegian|norvégien
nqo|||N'Ko|n'ko
nso|||Pedi; Sepedi; Northern Sotho|pedi; sepedi; sotho du Nord
nub|||Nubian languages|nubiennes, langues
nwc|||Classical Newari; Old Newari; Classical Nepal Bhasa|newari classique
nya||ny|Chichewa; Chewa; Nyanja|chichewa; chewa; nyanja
nym|||Nyamwezi|nyamwezi
nyn|||Nyankole|nyankolé
nyo|||Nyoro|nyoro
nzi|||Nzima|nzema
oci||oc|Occitan (post 1500); Provençal|occitan (après 1500); provençal
oji||oj|Ojibwa|ojibwa
ori||or|Oriya|oriya
orm||om|Oromo|galla
osa|||Osage|osage
oss||os|Ossetian; Ossetic|ossète
ota|||Turkish, Ottoman (1500-1928)|turc ottoman (1500-1928)
oto|||Otomian languages|otomi, langues
paa|||Papuan languages|papoues, langues
pag|||Pangasinan|pangasinan
pal|||Pahlavi|pahlavi
pam|||Pampanga; Kapampangan|pampangan
pan||pa|Panjabi; Punjabi|pendjabi
pap|||Papiamento|papiamento
pau|||Palauan|palau
peo|||Persian, Old (ca.600-400 B.C.)|perse, vieux (ca. 600-400 av. J.-C.)
per|fas|fa|Persian|persan
phi|||Philippine languages|philippines, langues
phn|||Phoenician|phénicien
pli||pi|Pali|pali
pol||pl|Polish|polonais
pon|||Pohnpeian|pohnpei
por||pt|Portuguese|portugais
pra|||Prakrit languages|prâkrit, langues
pro|||Provençal, Old (to 1500)|provençal ancien (jusqu'à 1500)
pus||ps|Pushto; Pashto|pachto
qaa-qtz|||Reserved for local use|réservée à l'usage local
que||qu|Quechua|quechua
raj|||Rajasthani|rajasthani
rap|||Rapanui|rapanui
rar|||Rarotongan; Cook Islands Maori|rarotonga; maori des îles Cook
roa|||Romance languages|romanes, langues
roh||rm|Romansh|romanche
rom|||Romany|tsigane
rum|ron|ro|Romanian; Moldavian; Moldovan|roumain; moldave
run||rn|Rundi|rundi
rup|||Aromanian; Arumanian; Macedo-Romanian|aroumain; macédo-roumain
rus||ru|Russian|russe
sad|||Sandawe|sandawe
sag||sg|Sango|sango
sah|||Yakut|iakoute
sai|||South American Indian (Other)|indiennes d'Amérique du Sud, autres langues
sal|||Salishan languages|salishennes, langues
sam|||Samaritan Aramaic|samaritain
san||sa|Sanskrit|sanskrit
sas|||Sasak|sasak
sat|||Santali|santal
scn|||Sicilian|sicilien
sco|||Scots|écossais
sel|||Selkup|selkoupe
sem|||Semitic languages|sémitiques, langues
sga|||Irish, Old (to 900)|irlandais ancien (jusqu'à 900)
sgn|||Sign Languages|langues des signes
shn|||Shan|chan
sid|||Sidamo|sidamo
sin||si|Sinhala; Sinhalese|singhalais
sio|||Siouan languages|sioux, langues
sit|||Sino-Tibetan languages|sino-tibétaines, langues
sla|||Slavic languages|slaves, langues
slo|slk|sk|Slovak|slovaque
slv||sl|Slovenian|slovène
sma|||Southern Sami|sami du Sud
sme||se|Northern Sami|sami du Nord
smi|||Sami languages|sames, langues
smj|||Lule Sami|sami de Lule
smn|||Inari Sami|sami d'Inari
smo||sm|Samoan|samoan
sms|||Skolt Sami|sami skolt
sna||sn|Shona|shona
snd||sd|Sindhi|sindhi
snk|||Soninke|soninké
sog|||Sogdian|sogdien
som||so|Somali|somali
son|||Songhai languages|songhai, langues
sot||st|Sotho, Southern|sotho du Sud
spa||es|Spanish; Castilian|espagnol; castillan
srd||sc|Sardinian|sarde
srn|||Sranan Tongo|sranan tongo
srp||sr|Serbian|serbe
srr|||Serer|sérère
ssa|||Nilo-Saharan languages|nilo-sahariennes, langues
ssw||ss|Swati|swati
suk|||Sukuma|sukuma
sun||su|Sundanese|soundanais
sus|||Susu|soussou
sux|||Sumerian|sumérien
swa||sw|Swahili|swahili
swe||sv|Swedish|suédois
syc|||Classical Syriac|syriaque classique
syr|||Syriac|syriaque
tah||ty|Tahitian|tahitien
tai|||Tai languages|tai, langues
tam||ta|Tamil|tamoul
tat||tt|Tatar|tatar
tel||te|Telugu|télougou
tem|||Timne|temne
ter|||Tereno|tereno
tet|||Tetum|tetum
tgk||tg|Tajik|tadjik
tgl||tl|Tagalog|tagalog
tha||th|Thai|thaï
tib|bod|bo|Tibetan|tibétain
tig|||Tigre|tigré
tir||ti|Tigrinya|tigrigna
tiv|||Tiv|tiv
tkl|||Tokelau|tokelau
tlh|||Klingon; tlhIngan-Hol|klingon
tli|||Tlingit|tlingit
tmh|||Tamashek|tamacheq
tog|||Tonga (Nyasa)|tonga (Nyasa)
ton||to|Tonga (Tonga Islands)|tongan (Îles Tonga)
tpi|||Tok Pisin|tok pisin
tsi|||Tsimshian|tsimshian
tsn||tn|Tswana|tswana
tso||ts|Tsonga|tsonga
tuk||tk|Turkmen|turkmène
tum|||Tumbuka|tumbuka
tup|||Tupi languages|tupi, langues
tur||tr|Turkish|turc
tut|||Altaic languages|altaïques, langues
tvl|||Tuvalu|tuvalu
twi||tw|Twi|twi
tyv|||Tuvinian|touva
udm|||Udmurt|oudmourte
uga|||Ugaritic|ougaritique
uig||ug|Uighur; Uyghur|ouïgour
ukr||uk|Ukrainian|ukrainien
umb|||Umbundu|umbundu
und|||Undetermined|indéterminée
urd||ur|Urdu|ourdou
uzb||uz|Uzbek|ouszbek
vai|||Vai|vaï
ven||ve|Venda|venda
vie||vi|Vietnamese|vietnamien
vol||vo|Volapük|volapük
vot|||Votic|vote
wak|||Wakashan languages|wakashanes, langues
wal|||Walamo|walamo
war|||Waray|waray
was|||Washo|washo
wel|cym|cy|Welsh|gallois
wen|||Sorbian languages|sorabes, langues
wln||wa|Walloon|wallon
wol||wo|Wolof|wolof
xal|||Kalmyk; Oirat|kalmouk; oïrat
xho||xh|Xhosa|xhosa
yao|||Yao|yao
yap|||Yapese|yapois
yid||yi|Yiddish|yiddish
yor||yo|Yoruba|yoruba
ypk|||Yupik languages|yupik, langues
zap|||Zapotec|zapotèque
zbl|||Blissymbols; Blissymbolics; Bliss|symboles Bliss; Bliss
zen|||Zenaga|zenaga
zha||za|Zhuang; Chuang|zhuang; chuang
znd|||Zande languages|zandé, langues
zul||zu|Zulu|zoulou
zun|||Zuni|zuni
zxx|||No linguistic content; Not applicable|pas de contenu linguistique; non applicable
zza|||Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki|zaza; dimili; dimli; kirdki; kirmanjki; zazaki
libs/guessit/__init__.py
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.2'
__all__ = [ 'Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info' ]
from guessit.guess import Guess, merge_all
from guessit.language import Language
from guessit.matcher import IterativeMatcher
import logging
log = logging.getLogger("guessit")
class NullHandler(logging.Handler):
def emit(self, record):
pass
# let's be a nicely behaving library
h = NullHandler()
log.addHandler(h)
def guess_file_info(filename, filetype, info = [ 'filename' ]):
"""info can contain the names of the various plugins, such as 'filename' to
detect filename info, or 'hash_md5' to get the md5 hash of the file.
>>> guess_file_info('test/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1'])
{'hash_md5': 'e781de9b94ba2753a8e2945b2c0a123d', 'hash_sha1': 'bfd18e2f4e5d59775c2bc14d80f56971891ed620'}
"""
result = []
hashers = []
for infotype in info:
if infotype == 'filename':
m = IterativeMatcher(filename, filetype = filetype)
result.append(m.matched())
elif infotype == 'hash_mpc':
import hash_mpc
try:
result.append(Guess({ 'hash_mpc': hash_mpc.hash_file(filename) },
confidence = 1.0))
except Exception, e:
log.warning('Could not compute MPC-style hash because: %s' % e)
elif infotype == 'hash_ed2k':
import hash_ed2k
try:
result.append(Guess({ 'hash_ed2k': hash_ed2k.hash_file(filename) },
confidence = 1.0))
except Exception, e:
log.warning('Could not compute ed2k hash because: %s' % e)
elif infotype.startswith('hash_'):
import hashlib
hashname = infotype[5:]
try:
hasher = getattr(hashlib, hashname)()
hashers.append((infotype, hasher))
except AttributeError:
log.warning('Could not compute %s hash because it is not available from python\'s hashlib module' % hashname)
else:
log.warning('Invalid infotype: %s' % infotype)
"""For plugins which depend on some optional library, import them like that:
if infotype == 'plugin_name':
try:
import optional_lib
except ImportError:
raise Exception, 'The plugin module cannot be loaded because the optional_lib lib is missing'
# do some stuff
"""
# do all the hashes now, but on a single pass
if hashers:
try:
blocksize = 8192
hasherobjs = dict(hashers).values()
with open(filename, 'rb') as f:
for chunk in iter(lambda: f.read(blocksize), ''):
for hasher in hasherobjs:
hasher.update(chunk)
for infotype, hasher in hashers:
result.append(Guess({ infotype: hasher.hexdigest() },
confidence = 1.0))
except Exception, e:
log.warning('Could not compute hash because: %s' % e)
return merge_all(result)
def guess_video_info(filename, info = [ 'filename' ]):
return guess_file_info(filename, 'autodetect', info)
def guess_movie_info(filename, info = [ 'filename' ]):
return guess_file_info(filename, 'movie', info)
def guess_episode_info(filename, info = [ 'filename' ]):
return guess_file_info(filename, 'episode', info)
libs/guessit/autodetect.py
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#from guessit import movie, episode
import os, os.path
import logging
log = logging.getLogger('guessit.autodetect')
def within(x, nrange):
"""Return whether a number is inside a given range, specified as a list or tuple
of the lower and upper bounds."""
low, high = nrange
return low <= x <= high
def guess_filename_info(filename):
log.debug('Trying to guess info for file: ' + filename)
# try to guess info as if it were an episode
episode_info = episode.guess_episode_filename(filename)
# 1- if we found either season/episodeNumber, then we're pretty sure it must
# be an episode
if 'season' in episode_info or 'episodeNumber' in episode_info:
log.debug('Likely an episode as it contains season and/or episodeNumber: ' + filename)
episode_info.update({ 'type': 'episode' }, confidence = 0.9)
return episode_info
# try to guess info as if it were a movie
movie_info = movie.guess_movie_filename(filename)
# 2- if the file exists, try to guess its type using its size
if os.path.exists(filename):
size = os.stat(filename).st_size / (1024 * 1024)
# if size <= 1/2 of 1CD -> episode (very unlikely a movie so small)
if size < 400:
log.debug('Likely an episode due to its small size (%dMB): %s' % (size, filename))
episode_info.update({ 'type': 'episode' }, confidence = 0.8)
return episode_info
# if size > 2G -> movie (even fullHD eps aren't that big yet)
if size > 2048:
log.debug('Likely a movie due to its big size (%dMB): %s' % (size, filename))
movie_info.update({ 'type': 'movie' }, confidence = 0.8)
return movie_info
# if size == 1CD or 2CDs -> movie
if within(size, [690, 710]) or within(size, [1380, 1420]):
log.debug('Likely a movie due to its size close to a CD size (%dMB): %s' % (size, filename))
movie_info.update({ 'type': 'movie' }, confidence = 0.8)
return movie_info
# 3- if all else fails, assume it's a movie
log.debug('Couldn\'t make an informed guess... Assuming file is a movie: %s' % filename)
movie_info.update({ 'type': 'movie' }, confidence = 0.5)
return movie_info
libs/guessit/date.py
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
import re
def search_year(string):
"""Looks for year patterns, and if found return the year and group span.
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-digit delimiting the date.
Note this only looks for valid production years, that is between 1920
and now + 5 years, so for instance 2000 would be returned as a valid
year but 1492 would not.
>>> search_year('in the year 2000...')
(2000, (12, 16))
>>> search_year('they arrived in 1492.')
(None, None)
"""
match = re.search(r'[^0-9]([0-9]{4})[^0-9]', string)
if match:
year = int(match.group(1))
if 1920 < year < datetime.date.today().year + 5:
return (year, match.span(1))
return (None, None)
def search_date(string):
"""Looks for date patterns, and if found return the date and group span.
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-digit delimiting the date.
>>> search_date('This happened on 2002-04-22.')
(datetime.date(2002, 4, 22), (17, 27))
>>> search_date('And this on 17-06-1998.')
(datetime.date(1998, 6, 17), (12, 22))
>>> search_date('no date in here')
(None, None)
"""
dsep = r'[-/ \.]'
date_rexps = [ # 20010823
r'[^0-9]' +
r'(?P<year>[0-9]{4})' +
r'(?P<month>[0-9]{2})' +
r'(?P<day>[0-9]{2})' +
r'[^0-9]',
# 2001-08-23
r'[^0-9]' +
r'(?P<year>[0-9]{4})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<day>[0-9]{2})' +
r'[^0-9]',
# 23-08-2001
r'[^0-9]' +
r'(?P<day>[0-9]{2})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<year>[0-9]{4})' +
r'[^0-9]',
# 23-08-01
r'[^0-9]' +
r'(?P<day>[0-9]{2})' + dsep +
r'(?P<month>[0-9]{2})' + dsep +
r'(?P<year>[0-9]{2})' +
r'[^0-9]',
]
for drexp in date_rexps:
match = re.search(drexp, string)
if match:
d = match.groupdict()
year, month, day = int(d['year']), int(d['month']), int(d['day'])
# years specified as 2 digits should be adjusted here
if year < 100:
if year > (datetime.date.today().year % 100)+ 5:
year = 1900 + year
else:
year = 2000 + year
date = None
try:
date = datetime.date(year, month, day)
except ValueError:
try:
date = datetime.date(year, day, month)
except ValueError:
pass
if date is None:
continue
# check date plausibility
if not 1900 < date.year < datetime.date.today().year + 5:
continue
# looks like we have a valid date
# note: span is [+1,-1] because we don't want to include the non-digit char
start, end = match.span()
return (date, (start+1, end-1))
return None, None
libs/guessit/fileutils.py
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import ntpath
import os.path
def split_path(path):
r"""Splits the given path into the list of folders and the filename (or the
last folder if you gave it a folder path.
If the given path was an absolute path, the first element will always be:
- the '/' root folder on Unix systems
- the drive letter on Windows systems (eg: r'C:\')
>>> split_path('/usr/bin/smewt')
['/', 'usr', 'bin', 'smewt']
>>> split_path('relative_path/to/my_folder/')
['relative_path', 'to', 'my_folder']
>>> split_path(r'C:\Program Files\Smewt\smewt.exe')
['C:\\', 'Program Files', 'Smewt', 'smewt.exe']
>>> split_path(r'Documents and Settings\User\config\\')
['Documents and Settings', 'User', 'config']
"""
result = []
while True:
head, tail = ntpath.split(path)
# on Unix systems, the root folder is '/'
if head == '/' and tail == '':
return [ '/' ] + result
# on Windows, the root folder is a drive letter (eg: 'C:\')
if len(head) == 3 and head[1:] == ':\\' and tail == '':
return [ head ] + result
if head == '' and tail == '':
return result
# we just split a directory ending with '/', so tail is empty
if not tail:
path = head
continue
result = [ tail ] + result
path = head
def split_path_components(filename):
"""Returns the filename split into [ dir*, basename, ext ]."""
result = split_path(filename)
basename = result.pop(-1)
return result + list(os.path.splitext(basename))
def file_in_same_dir(ref_file, desired_file):
"""Return the path for a file in the same dir as a given reference file.
>>> file_in_same_dir('~/smewt/smewt.db', 'smewt.settings')
'~/smewt/smewt.settings'
"""
return os.path.join(*(split_path(ref_file)[:-1] + [ desired_file ]))
libs/guessit/guess.py
+317
View File
@@ -0,0 +1,317 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import json
import datetime
import logging
log = logging.getLogger("guessit.guess")
class Guess(dict):
"""A Guess is a dictionary which has an associated confidence for each of its values.
As it is a subclass of dict, you can use it everywhere you expect a simple dict"""
def __init__(self, *args, **kwargs):
try:
confidence = kwargs.pop('confidence')
except KeyError:
confidence = 0
dict.__init__(self, *args, **kwargs)
self._confidence = {}
for prop in self:
self._confidence[prop] = confidence
def to_utf8_dict(self):
from guessit.language import Language
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, Language):
data[prop] = str(value)
elif isinstance(value, unicode):
data[prop] = value.encode('utf-8')
elif isinstance(value, list):
data[prop] = [ str(x) for x in value ]
return data
def nice_string(self):
data = self.to_utf8_dict()
parts = json.dumps(data, indent = 4).split('\n')
for i, p in enumerate(parts):
if p[:5] != ' "':
continue
prop = p.split('"')[1]
parts[i] = (' [%.2f] "' % (self._confidence.get(prop) or -1)) + p[5:]
return '\n'.join(parts)
def __str__(self):
return str(self.to_utf8_dict())
def confidence(self, prop):
return self._confidence[prop]
def set(self, prop, value, confidence = None):
self[prop] = value
if confidence is not None:
self._confidence[prop] = confidence
def set_confidence(self, prop, value):
self._confidence[prop] = value
def update(self, other, confidence = None):
dict.update(self, other)
if isinstance(other, Guess):
for prop in other:
self._confidence[prop] = other.confidence(prop)
if confidence is not None:
for prop in other:
self._confidence[prop] = confidence
def update_highest_confidence(self, other):
"""Update this guess with the values from the given one. In case there is
property present in both, only the one with the highest one is kept."""
if not isinstance(other, Guess):
raise ValueError, 'Can only call this function on Guess instances'
for prop in other:
if prop in self and self._confidence[prop] >= other._confidence[prop]:
continue
self[prop] = other[prop]
self._confidence[prop] = other._confidence[prop]
def choose_int(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible properties
when they are integers."""
v1, c1 = g1 # value, confidence
v2, c2 = g2
if (v1 == v2):
return (v1, 1 - (1-c1)*(1-c2))
else:
if c1 > c2:
return (v1, c1 - c2)
else:
return (v2, c2 - c1)
def choose_string(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible properties
when they are strings.
If the 2 strings are similar, or one is contained in the other, the latter is returned
with an increased confidence.
If the 2 strings are dissimilar, the one with the higher confidence is returned, with
a weaker confidence.
Note that here, 'similar' means that 2 strings are either equal, or that they
differ very little, such as one string being the other one with the 'the' word
prepended to it.
>>> choose_string(('Hello', 0.75), ('World', 0.5))
('Hello', 0.25)
>>> choose_string(('Hello', 0.5), ('hello', 0.5))
('Hello', 0.75)
>>> choose_string(('Hello', 0.4), ('Hello World', 0.4))
('Hello', 0.64000000000000001)
>>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5))
('The Simpsons', 0.75)
"""
v1, c1 = g1 # value, confidence
v2, c2 = g2
if not v1:
return g2
elif not v2:
return g1
v1, v2 = v1.strip(), v2.strip()
v1l, v2l = v1.lower(), v2.lower()
combined_prob = 1 - (1-c1)*(1-c2)
if v1l == v2l:
return (v1, combined_prob)
# check for common patterns
elif v1l == 'the ' + v2l:
return (v1, combined_prob)
elif v2l == 'the ' + v1l:
return (v2, combined_prob)
# if one string is contained in the other, return the shortest one
elif v2l in v1l:
return (v2, combined_prob)
elif v1l in v2l:
return (v1, combined_prob)
# in case of conflict, return the one with highest priority
else:
if c1 > c2:
return (v1, c1 - c2)
else:
return (v2, c2 - c1)
def _merge_similar_guesses_nocheck(guesses, prop, choose):
"""Take a list of guesses and merge those which have the same properties,
increasing or decreasing the confidence depending on whether their values
are similar.
This function assumes there are at least 2 valid guesses."""
similar = [ guess for guess in guesses if prop in guess ]
g1, g2 = similar[0], similar[1]
other_props = set(g1) & set(g2) - set([prop])
if other_props:
for prop in other_props:
if g1[prop] != g2[prop]:
log.warning('both guesses to be merged have more than one different property in common, bailing out...')
return
# merge all props of s2 into s1, updating the confidence for the considered property
v1, v2 = g1[prop], g2[prop]
c1, c2 = g1.confidence(prop), g2.confidence(prop)
new_value, new_confidence = choose((v1, c1), (v2, c2))
if new_confidence >= c1:
log.debug("Updating matching property '%s' with confidence %.2f" % (prop, new_confidence))
else:
log.debug("Updating non-matching property '%s' with confidence %.2f" % (prop, new_confidence))
g2[prop] = new_value
g2.set_confidence(prop, new_confidence)
g1.update(g2)
guesses.remove(g2)
def merge_similar_guesses(guesses, prop, choose):
"""Take a list of guesses and merge those which have the same properties,
increasing or decreasing the confidence depending on whether their values
are similar."""
similar = [ guess for guess in guesses if prop in guess ]
if len(similar) < 2:
# nothing to merge
return
if len(similar) == 2:
_merge_similar_guesses_nocheck(guesses, prop, choose)
if len(similar) > 2:
log.debug('complex merge, trying our best...')
_merge_similar_guesses_nocheck(guesses, prop, choose)
merge_similar_guesses(guesses, prop, choose)
return
def merge_append_guesses(guesses, prop):
"""Take a list of guesses and merge those which have the same properties by
appending them in a list.
DEPRECATED, remove with old guessers
"""
similar = [ guess for guess in guesses if prop in guess ]
if not similar:
return
merged = similar[0]
merged[prop] = [ merged[prop] ]
# TODO: what to do with global confidence? mean of them all?
for m in similar[1:]:
for prop2 in m:
if prop == prop2:
merged[prop].append(m[prop])
else:
if prop2 in m:
log.warning('overwriting property "%s" with value ' % (prop2, m[prop2]))
merged[prop2] = m[prop2]
# TODO: confidence also
guesses.remove(m)
def merge_all(guesses, append = []):
"""Merges all the guesses in a single result, removes very unlikely values, and returns it.
You can specify a list of properties that should be appended into a list instead of being
merged.
>>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ])
{'season': 2, 'episodeNumber': 13}
>>> merge_all([ Guess({ 'episodeNumber': 27 }, confidence = 0.02),
... Guess({ 'season': 1 }, confidence = 0.2) ])
{'season': 1}
"""
if not guesses:
return Guess()
result = guesses[0]
for g in guesses[1:]:
# first append our appendable properties
for prop in append:
if prop in g:
result.set(prop, result.get(prop, []) + [ g[prop] ],
# TODO: what to do with confidence here? maybe an arithmetic mean...
confidence = g.confidence(prop))
del g[prop]
# then merge the remaining ones
if set(result) & set(g):
log.warning('duplicate properties %s in merged result...' % (set(result) & set(g)))
result.update_highest_confidence(g)
# delete very unlikely values
for p in result.keys():
if result.confidence(p) < 0.05:
del result[p]
# make sure our appendable properties contain unique values
for prop in append:
if prop in result:
result[prop] = list(set(result[prop]))
return result
libs/guessit/hash_ed2k.py
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
import hashlib, os.path
def hash_file(filename):
"""Returns the ed2k hash of a given file.
>>> hash_file('test/dummy.srt')
'ed2k://|file|dummy.srt|44|1CA0B9DED3473B926AA93A0A546138BB|/'
"""
return 'ed2k://|file|%s|%d|%s|/' % (os.path.basename(filename),
os.path.getsize(filename),
hash_filehash(filename).upper())
def hash_filehash(filename):
"""Returns the ed2k hash of a given file.
This function is taken from:
http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
"""
md4 = hashlib.new('md4').copy
def gen(f):
while True:
x = f.read(9728000)
if x: yield x
else: return
def md4_hash(data):
m = md4()
m.update(data)
return m
with open(filename, 'rb') as f:
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return hashes[0].encode("hex")
else: return md4_hash(reduce(lambda a,d: a + d, hashes, "")).hexd
libs/guessit/hash_mpc.py
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import Guess
import struct, os
def hash_file(filename):
"""This function is taken from:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
and is licensed under the GPL."""
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(filename, "rb")
filesize = os.path.getsize(filename)
hash = filesize
if filesize < 65536 * 2:
raise Exception, "SizeError: size is %d, should be > 132K..." % filesize
for x in range(65536/bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
f.seek(max(0,filesize-65536),0)
for x in range(65536/bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = "%016x" % hash
return returnedhash
libs/guessit/language.py
+209
View File
@@ -0,0 +1,209 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import fileutils
import os.path
import re
import logging
log = logging.getLogger('guessit.language')
# downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
#
# Description of the fields:
# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
# an alpha-2 code (when given), an English name, and a French name of a language
# are all separated by pipe (|) characters."
language_matrix = [ l.strip().decode('utf-8').split('|') for l in open(fileutils.file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt')) ]
lng3 = frozenset(filter(bool, (l[0] for l in language_matrix)))
lng3term = frozenset(filter(bool, (l[1] for l in language_matrix)))
lng2 = frozenset(filter(bool, (l[2] for l in language_matrix)))
lng_en_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[3].lower().split('; '))))
lng_fr_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[4].lower().split('; '))))
lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name
lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1])
lng3term_to_lng3 = dict((l[1], l[0]) for l in language_matrix if l[1])
lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2])
lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2])
# we only return the first given english name, hoping it is the most used one
lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) for l in language_matrix if l[3])
lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) for l in language_matrix if l[3] for en_name in l[3].split('; '))
# we only return the first given french name, hoping it is the most used one
lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) for l in language_matrix if l[4])
lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) for l in language_matrix if l[4] for fr_name in l[4].split('; '))
def is_language(language):
return language.lower() in lng_all_names
class Language(object):
"""This class represents a human language.
You can initialize it with pretty much everything, as it knows conversion from
ISO-639 2-letter and 3-letter codes, English and French names.
>>> Language('fr')
Language(French)
>>> Language('eng').french_name()
u'anglais'
"""
def __init__(self, language):
lang = None
language = language.lower()
if len(language) == 2:
lang = lng2_to_lng3.get(language)
elif len(language) == 3:
lang = language if language in lng3 else lng3term_to_lng3.get(language)
else:
lang = lng_en_name_to_lng3.get(language) or lng_fr_name_to_lng3.get(language)
if lang is None:
raise ValueError, 'The given string "%s" could not be identified as a language' % language
self.lang = lang
def lng2(self):
return lng3_to_lng2[self.lang]
def lng3(self):
return self.lang
def lng3term(self):
return lng3_to_lng3term[self.lang]
def english_name(self):
return lng3_to_lng_en_name[self.lang]
def french_name(self):
return lng3_to_lng_fr_name[self.lang]
def __hash__(self):
return hash(self.lang)
def __eq__(self, other):
if isinstance(other, Language):
return self.lang == other.lang
if isinstance(other, basestring):
try:
return self == Language(other)
except ValueError:
return False
return False
def __ne__(self, other):
return not self == other
def __unicode__(self):
return lng3_to_lng_en_name[self.lang]
def __str__(self):
return unicode(self).encode('utf-8')
def __repr__(self):
return 'Language(%s)' % self
def search_language(string, lang_filter = None):
"""Looks for language patterns, and if found return the language object,
its group span and an associated confidence.
you can specify a list of allowed languages using the lang_filter argument,
as in lang_filter = [ 'fr', 'eng', 'spanish' ]
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-letter delimiting the language.
>>> search_language('movie [en].avi')
(Language(English), (7, 9), 0.80000000000000004)
>>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es'])
(None, None, None)
"""
# list of common words which could be interpreted as languages, but which
# are far too common to be able to say they represent a language in the
# middle of a string (where they most likely carry their commmon meaning)
lng_common_words = frozenset([ # english words
'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
# french words
'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
'mal', 'est', 'vol', 'or', 'mon', 'se',
# spanish words
'la', 'el', 'del', 'por', 'mar',
# other
'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san'
])
sep = r'[](){} \._-+'
if lang_filter:
lang_filter = set(Language(l) for l in lang_filter)
slow = string.lower()
confidence = 1.0 # for all of them
for lang in lng_all_names:
if lang in lng_common_words:
continue
pos = slow.find(lang)
if pos != -1:
end = pos + len(lang)
# make sure our word is always surrounded by separators
if slow[pos-1] not in sep or slow[end] not in sep:
continue
language = Language(slow[pos:end])
if lang_filter and language not in lang_filter:
continue
# only allow those languages that have a 2-letter code, those who
# don't are too esoteric and probably false matches
if language.lang not in lng3_to_lng2:
continue
# confidence depends on lng2, lng3, english name, ...
if len(lang) == 2:
confidence = 0.8
elif len(lang) == 3:
confidence = 0.9
else:
# Note: we could either be really confident that we found a language
# or assume that full language names are too common words
confidence = 0.3 # going with the low-confidence route here
return language, (pos, end), confidence
return None, None, None
libs/guessit/matcher.py
+621
View File
@@ -0,0 +1,621 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit import fileutils, textutils
from guessit.guess import Guess, merge_similar_guesses, merge_all, choose_int, choose_string
from guessit.date import search_date, search_year
from guessit.language import search_language
from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, properties, canonical_form
from guessit.matchtree import get_group, find_group, leftover_valid_groups, tree_to_string
from guessit.textutils import find_first_level_groups, split_on_groups, blank_region, clean_string, to_utf8
from guessit.fileutils import split_path_components
import datetime
import os.path
import re
import copy
import logging
log = logging.getLogger("guessit.matcher")
def split_explicit_groups(string):
"""return the string split into explicit groups, that is, those either
between parenthese, square brackets or curly braces, and those separated
by a dash."""
result = find_first_level_groups(string, '()')
result = reduce(lambda l, x: l + find_first_level_groups(x, '[]'), result, [])
result = reduce(lambda l, x: l + find_first_level_groups(x, '{}'), result, [])
# do not do this at this moment, it is not strong enough and can break other
# patterns, such as dates, etc...
#result = reduce(lambda l, x: l + x.split('-'), result, [])
return result
def format_guess(guess):
"""Format all the found values to their natural type.
For instance, a year would be stored as an int value, etc...
Note that this modifies the dictionary given as input.
"""
for prop, value in guess.items():
if prop in ('season', 'episodeNumber', 'year', 'cdNumber', 'cdNumberTotal'):
guess[prop] = int(guess[prop])
elif isinstance(value, basestring):
if prop in ('edition',):
value = clean_string(value)
guess[prop] = canonical_form(value)
return guess
def guess_groups(string, result, filetype):
# add sentinels so we can match a separator char at either end of
# our groups, even when they are at the beginning or end of the string
# we will adjust the span accordingly later
#
# filetype can either be movie, moviesubtitle, episode, episodesubtitle
current = ' ' + string + ' '
regions = [] # list of (start, end) of matched regions
def guessed(match_dict, confidence):
guess = format_guess(Guess(match_dict, confidence = confidence))
result.append(guess)
log.debug('Found with confidence %.2f: %s' % (confidence, guess))
return guess
def update_found(string, guess, span, span_adjust = (0,0)):
span = (span[0] + span_adjust[0],
span[1] + span_adjust[1])
regions.append((span, guess))
return blank_region(string, span)
# try to find dates first, as they are very specific
date, span = search_date(current)
if date:
guess = guessed({ 'date': date }, confidence = 1.0)
current = update_found(current, guess, span)
# for non episodes only, look for year information
if filetype not in ('episode', 'episodesubtitle'):
year, span = search_year(current)
if year:
guess = guessed({ 'year': year }, confidence = 1.0)
current = update_found(current, guess, span)
# specific regexps (ie: cd number, season X episode, ...)
for rexp, confidence, span_adjust in video_rexps:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
# is this the better place to put it? (maybe, as it is at least the soonest that we can catch it)
if 'cdNumberTotal' in metadata and metadata['cdNumberTotal'] is None:
del metadata['cdNumberTotal']
guess = guessed(metadata, confidence = confidence)
current = update_found(current, guess, match.span(), span_adjust)
if filetype in ('episode', 'episodesubtitle'):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
guess = guessed(metadata, confidence = confidence)
current = update_found(current, guess, match.span(), span_adjust)
# Now websites, but as exact string instead of regexps
clow = current.lower()
for site in websites:
pos = clow.find(site.lower())
if pos != -1:
guess = guessed({ 'website': site }, confidence = confidence)
current = update_found(current, guess, (pos, pos+len(site)))
clow = current.lower()
# release groups have certain constraints, cannot be included in the previous general regexps
group_names = [ r'\.(Xvid)-(?P<releaseGroup>.*?)[ \.]',
r'\.(DivX)-(?P<releaseGroup>.*?)[\. ]',
r'\.(DVDivX)-(?P<releaseGroup>.*?)[\. ]',
]
for rexp in group_names:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
metadata.update({ 'videoCodec': match.group(1) })
guess = guessed(metadata, confidence = 0.8)
current = update_found(current, guess, match.span(), span_adjust = (1, -1))
# common well-defined words and regexps
clow = current.lower()
confidence = 1.0 # for all of them
for prop, values in properties.items():
for value in values:
pos = clow.find(value.lower())
if pos != -1:
end = pos + len(value)
# make sure our word is always surrounded by separators
if clow[pos-1] not in sep or clow[end] not in sep:
# note: sep is a regexp, but in this case using it as
# a sequence achieves the same goal
continue
guess = guessed({ prop: value }, confidence = confidence)
current = update_found(current, guess, (pos, end))
clow = current.lower()
# weak guesses for episode number, only run it if we don't have an estimate already
if filetype in ('episode', 'episodesubtitle'):
if not any('episodeNumber' in match for match in result):
for rexp, _, span_adjust in weak_episode_rexps:
match = re.search(rexp, current, re.IGNORECASE)
if match:
metadata = match.groupdict()
epnum = int(metadata['episodeNumber'])
if epnum > 100:
guess = guessed({ 'season': epnum // 100,
'episodeNumber': epnum % 100 }, confidence = 0.6)
else:
guess = guessed(metadata, confidence = 0.3)
current = update_found(current, guess, match.span(), span_adjust)
# try to find languages now
language, span, confidence = search_language(current)
while language:
# is it a subtitle language?
if 'sub' in clean_string(current[:span[0]]).lower().split(' '):
guess = guessed({ 'subtitleLanguage': language }, confidence = confidence)
else:
guess = guessed({ 'language': language }, confidence = confidence)
current = update_found(current, guess, span)
language, span, confidence = search_language(current)
# remove our sentinels now and ajust spans accordingly
assert(current[0] == ' ' and current[-1] == ' ')
current = current[1:-1]
regions = [ ((start-1, end-1), guess) for (start, end), guess in regions ]
# split into '-' separated subgroups (with required separator chars
# around the dash)
didx = current.find('-')
while didx > 0:
regions.append(((didx, didx), None))
didx = current.find('-', didx+1)
# cut our final groups, and rematch the guesses to the group that created
# id, None if it is a leftover group
region_spans = [ span for span, guess in regions ]
string_groups = split_on_groups(string, region_spans)
remaining_groups = split_on_groups(current, region_spans)
guesses = []
pos = 0
for group in string_groups:
found = False
for span, guess in regions:
if span[0] == pos:
guesses.append(guess)
found = True
if not found:
guesses.append(None)
pos += len(group)
return zip(string_groups,
remaining_groups,
guesses)
def match_from_epnum_position(match_tree, epnum_pos, guessed, update_found):
"""guessed is a callback function to call with the guessed group
update_found is a callback to update the match group and returns leftover groups."""
pidx, eidx, gidx = epnum_pos
# a few helper functions to be able to filter using high-level semantics
def same_pgroup_before(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and (eeidx, ggidx) < (eidx, gidx)
def same_pgroup_after(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and (eeidx, ggidx) > (eidx, gidx)
def same_egroup_before(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and eeidx == eidx and ggidx < gidx
def same_egroup_after(group):
_, (ppidx, eeidx, ggidx) = group
return ppidx == pidx and eeidx == eidx and ggidx > gidx
leftover = leftover_valid_groups(match_tree)
# if we have at least 1 valid group before the episodeNumber, then it's probably
# the series name
series_candidates = filter(same_pgroup_before, leftover)
if len(series_candidates) >= 1:
guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.7)
leftover = update_found(leftover, series_candidates[0][1], guess)
# only 1 group after (in the same path group) and it's probably the episode title
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(same_pgroup_after, leftover))
if len(title_candidates) == 1:
guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.5)
leftover = update_found(leftover, title_candidates[0][1], guess)
else:
# try in the same explicit group, with lower confidence
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(same_egroup_after, leftover))
if len(title_candidates) == 1:
guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[0][1], guess)
# epnumber is the first group and there are only 2 after it in same path group
# -> season title - episode title
already_has_title = (find_group(match_tree, 'title') != [])
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(same_pgroup_after, leftover))
if (not already_has_title and # no title
not filter(same_pgroup_before, leftover) and # no groups before
len(title_candidates) == 2): # only 2 groups after
guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[0][1], guess)
guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[1][1], guess)
# if we only have 1 remaining valid group in the pathpart before the filename,
# then it's likely that it is the series name
series_candidates = [ group for group in leftover if group[1][0] == pidx-1 ]
if len(series_candidates) == 1:
guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.5)
leftover = update_found(leftover, series_candidates[0][1], guess)
return match_tree
class IterativeMatcher(object):
def __init__(self, filename, filetype = 'autodetect'):
"""An iterative matcher tries to match different patterns that appear
in the filename.
The 'filetype' argument indicates which type of file you want to match.
If it is 'autodetect', the matcher will try to see whether it can guess
that the file corresponds to an episode, or otherwise will assume it is
a movie.
The recognized 'filetype' values are:
[ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ]
The IterativeMatcher works mainly in 2 steps:
First, it splits the filename into a match_tree, which is a tree of groups
which have a semantic meaning, such as episode number, movie title,
etc...
The match_tree created looks like the following:
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, x264 is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter'v' in the 4th line.
(for more info, see guess.matchtree.tree_to_string)
Second, it tries to merge all this information into a single object
containing all the found properties, and does some (basic) conflict
resolution when they arise.
"""
if filetype not in ('autodetect', 'subtitle', 'movie', 'moviesubtitle',
'episode', 'episodesubtitle'):
raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')"
if not isinstance(filename, unicode):
log.debug('WARNING: given filename to matcher is not unicode...')
match_tree = []
result = [] # list of found metadata
def guessed(match_dict, confidence):
guess = format_guess(Guess(match_dict, confidence = confidence))
result.append(guess)
log.debug('Found with confidence %.2f: %s' % (confidence, guess))
return guess
def update_found(leftover, group_pos, guess):
pidx, eidx, gidx = group_pos
group = match_tree[pidx][eidx][gidx]
match_tree[pidx][eidx][gidx] = (group[0],
deleted * len(group[0]),
guess)
return [ g for g in leftover if g[1] != group_pos ]
# 1- first split our path into dirs + basename + ext
match_tree = split_path_components(filename)
fileext = match_tree.pop(-1)[1:].lower()
if fileext in subtitle_exts:
if 'movie' in filetype:
filetype = 'moviesubtitle'
elif 'episode' in filetype:
filetype = 'episodesubtitle'
else:
filetype = 'subtitle'
extguess = guessed({ 'container': fileext }, confidence = 1.0)
elif fileext in video_exts:
extguess = guessed({ 'container': fileext }, confidence = 1.0)
else:
extguess = guessed({ 'extension': fileext}, confidence = 1.0)
# TODO: depending on the extension, we could already grab some info and maybe specialized
# guessers, eg: a lang parser for idx files, an automatic detection of the language
# for srt files, a video metadata extractor for avi, mkv, ...
# if we are on autodetect, try to do it now so we can tell the
# guess_groups function what type of info it should be looking for
if filetype in ('autodetect', 'subtitle'):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, filename, re.IGNORECASE)
if match:
if filetype == 'autodetect':
filetype = 'episode'
elif filetype == 'subtitle':
filetype = 'episodesubtitle'
break
# if no episode info found, assume it's a movie
if filetype == 'autodetect':
filetype = 'movie'
elif filetype == 'subtitle':
filetype = 'moviesubtitle'
guessed({ 'type': filetype }, confidence = 1.0)
# 2- split each of those into explicit groups, if any
# note: be careful, as this might split some regexps with more confidence such as
# Alfleni-Team, or [XCT] or split a date such as (14-01-2008)
match_tree = [ split_explicit_groups(part) for part in match_tree ]
# 3- try to match information in decreasing order of confidence and
# blank the matching group in the string if we found something
for pathpart in match_tree:
for gidx, explicit_group in enumerate(pathpart):
pathpart[gidx] = guess_groups(explicit_group, result, filetype = filetype)
# 4- try to identify the remaining unknown groups by looking at their position
# relative to other known elements
if filetype in ('episode', 'episodesubtitle'):
eps = find_group(match_tree, 'episodeNumber')
if eps:
match_tree = match_from_epnum_position(match_tree, eps[0], guessed, update_found)
leftover = leftover_valid_groups(match_tree)
if not eps:
# if we don't have the episode number, but at least 2 groups in the
# last path group, then it's probably series - eptitle
title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
filter(lambda g: g[1][0] == len(match_tree)-1,
leftover_valid_groups(match_tree)))
if len(title_candidates) >= 2:
guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[0][1], guess)
guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4)
leftover = update_found(leftover, title_candidates[1][1], guess)
# if there's a path group that only contains the season info, then the previous one
# is most likely the series title (ie: .../series/season X/...)
eps = [ gpos for gpos in find_group(match_tree, 'season')
if 'episodeNumber' not in get_group(match_tree, gpos)[2] ]
if eps:
pidx, eidx, gidx = eps[0]
previous = [ group for group in leftover if group[1][0] == pidx - 1 ]
if len(previous) == 1:
guess = guessed({ 'series': previous[0][0] }, confidence = 0.5)
leftover = update_found(leftover, previous[0][1], guess)
elif filetype in ('movie', 'moviesubtitle'):
leftover_all = leftover_valid_groups(match_tree)
# specific cases:
# - movies/tttttt (yyyy)/tttttt.ccc
try:
if match_tree[-3][0][0][0].lower() == 'movies':
# Note:too generic, might solve all the unittests as they all contain 'movies'
# in their path
#
#if len(match_tree[-2][0]) == 1:
# title = match_tree[-2][0][0]
# guess = guessed({ 'title': clean_string(title[0]) }, confidence = 0.7)
# update_found(leftover_all, title, guess)
year_group = filter(lambda gpos: gpos[0] == len(match_tree)-2,
find_group(match_tree, 'year'))[0]
leftover = leftover_valid_groups(match_tree,
valid = lambda g: ((g[0] and g[0][0] not in sep) and
g[1][0] == len(match_tree) - 2))
if len(match_tree[-2]) == 2 and year_group[1] == 1:
title = leftover[0]
guess = guessed({ 'title': clean_string(title[0]) },
confidence = 0.8)
update_found(leftover_all, title[1], guess)
raise Exception # to exit the try catch now
leftover = [ g for g in leftover_all if (g[1][0] == year_group[0] and
g[1][1] < year_group[1] and
g[1][2] < year_group[2]) ]
leftover = sorted(leftover, key = lambda x:x[1])
title = leftover[0]
guess = guessed({ 'title': title[0] }, confidence = 0.8)
leftover = update_found(leftover, title[1], guess)
except:
pass
# if we have either format or videoCodec in the folder containing the file
# or one of its parents, then we should probably look for the title in
# there rather than in the basename
props = filter(lambda g: g[0] <= len(match_tree) - 2,
find_group(match_tree, 'videoCodec') +
find_group(match_tree, 'format') +
find_group(match_tree, 'language'))
leftover = None
if props and all(g[0] == props[0][0] for g in props):
leftover = [ g for g in leftover_all if g[1][0] == props[0][0] ]
if props and leftover:
guess = guessed({ 'title': leftover[0][0] }, confidence = 0.7)
leftover = update_found(leftover, leftover[0][1], guess)
else:
# first leftover group in the last path part sounds like a good candidate for title,
# except if it's only one word and that the first group before has at least 3 words in it
# (case where the filename contains an 8 chars short name and the movie title is
# actually in the parent directory name)
leftover = [ g for g in leftover_all if g[1][0] == len(match_tree)-1 ]
if leftover:
title, (pidx, eidx, gidx) = leftover[0]
previous_pgroup_leftover = filter(lambda g: g[1][0] == pidx-1, leftover_all)
if (title.count(' ') == 0 and
previous_pgroup_leftover and
previous_pgroup_leftover[0][0].count(' ') >= 2):
guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6)
leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess)
else:
guess = guessed({ 'title': title }, confidence = 0.6)
leftover = update_found(leftover, leftover[0][1], guess)
else:
# if there were no leftover groups in the last path part, look in the one before that
previous_pgroup_leftover = filter(lambda g: g[1][0] == len(match_tree)-2, leftover_all)
if previous_pgroup_leftover:
guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6)
leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess)
# 5- perform some post-processing steps
# 5.1- try to promote language to subtitle language where it makes sense
for pidx, eidx, gidx in find_group(match_tree, 'language'):
string, remaining, guess = get_group(match_tree, (pidx, eidx, gidx))
def promote_subtitle():
guess.set('subtitleLanguage', guess['language'], confidence = guess.confidence('language'))
del guess['language']
# - if we matched a language in a file with a sub extension and that the group
# is the last group of the filename, it is probably the language of the subtitle
# (eg: 'xxx.english.srt')
if (fileext in subtitle_exts and
pidx == len(match_tree) - 1 and
eidx == len(match_tree[pidx]) - 1):
promote_subtitle()
# - if a language is in an explicit group just preceded by "st", it is a subtitle
# language (eg: '...st[fr-eng]...')
if eidx > 0:
previous = get_group(match_tree, (pidx, eidx-1, -1))
if previous[0][-2:].lower() == 'st':
promote_subtitle()
# re-append the extension now
match_tree.append([[(fileext, deleted*len(fileext), extguess)]])
self.parts = result
self.match_tree = match_tree
if filename.startswith('/'):
filename = ' ' + filename
log.debug('Found match tree:\n%s\n%s' % (to_utf8(tree_to_string(match_tree)),
to_utf8(filename)))
def matched(self):
# we need to make a copy here, as the merge functions work in place and
# calling them on the match tree would modify it
parts = copy.deepcopy(self.parts)
# 1- start by doing some common preprocessing tasks
# 1.1- ", the" at the end of a series title should be prepended to it
for part in parts:
if 'series' not in part:
continue
series = part['series']
lseries = series.lower()
if lseries[-4:] == ',the':
part['series'] = 'The ' + series[:-4]
if lseries[-5:] == ', the':
part['series'] = 'The ' + series[:-5]
# 2- try to merge similar information together and give it a higher confidence
for int_part in ('year', 'season', 'episodeNumber'):
merge_similar_guesses(parts, int_part, choose_int)
for string_part in ('title', 'series', 'container', 'format', 'releaseGroup', 'website',
'audioCodec', 'videoCodec', 'screenSize', 'episodeFormat'):
merge_similar_guesses(parts, string_part, choose_string)
result = merge_all(parts, append = ['language', 'subtitleLanguage', 'other'])
# 3- some last minute post-processing
if (result['type'] == 'episode' and
'season' not in result and
result.get('episodeFormat', '') == 'Minisode'):
result['season'] = 0
log.debug('Final result: ' + result.nice_string())
return result
libs/guessit/matchtree.py
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import deleted
from guessit.textutils import clean_string
import logging
log = logging.getLogger("guessit.matchtree")
def tree_to_string(tree):
"""Return a string representation for the given tree.
The lines convey the following information:
- line 1: path idx
- line 2: explicit group idx
- line 3: group index
- line 4: remaining info
- line 5: meaning conveyed
Meaning is a letter indicating what type of info was matched by this group,
for instance 't' = title, 'f' = format, 'l' = language, etc...
An example is the following:
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
(note: the last line representing the filename is not pat of the tree representation)
"""
m_tree = [ '', # path level index
'', # explicit group index
'', # matched regexp and dash-separated
'', # groups leftover that couldn't be matched
'', # meaning conveyed: E = episodenumber, S = season, ...
]
def add_char(pidx, eidx, gidx, remaining, meaning = None):
nr = len(remaining)
def to_hex(x):
if isinstance(x, int):
return str(x) if x < 10 else chr(55+x)
return x
m_tree[0] = m_tree[0] + to_hex(pidx) * nr
m_tree[1] = m_tree[1] + to_hex(eidx) * nr
m_tree[2] = m_tree[2] + to_hex(gidx) * nr
m_tree[3] = m_tree[3] + remaining
m_tree[4] = m_tree[4] + str(meaning or ' ') * nr
def meaning(result):
mmap = { 'episodeNumber': 'E',
'season': 'S',
'extension': 'e',
'format': 'f',
'language': 'l',
'videoCodec': 'v',
'audioCodec': 'a',
'website': 'w',
'container': 'c',
'series': 'T',
'title': 't',
'date': 'd',
'year': 'y',
'releaseGroup': 'r',
'screenSize': 's'
}
if result is None:
return ' '
for prop, l in mmap.items():
if prop in result:
return l
return 'x'
for pidx, pathpart in enumerate(tree):
for eidx, explicit_group in enumerate(pathpart):
for gidx, (group, remaining, result) in enumerate(explicit_group):
add_char(pidx, eidx, gidx, remaining, meaning(result))
# special conditions for the path separator
if pidx < len(tree) - 2:
add_char(' ', ' ', ' ', '/')
elif pidx == len(tree) - 2:
add_char(' ', ' ', ' ', '.')
return '\n'.join(m_tree)
def iterate_groups(match_tree):
"""Iterate over all the groups in a match_tree and return them as pairs
of (group_pos, group) where:
- group_pos = (pidx, eidx, gidx)
- group = (string, remaining, guess)
"""
for pidx, pathpart in enumerate(match_tree):
for eidx, explicit_group in enumerate(pathpart):
for gidx, group in enumerate(explicit_group):
yield (pidx, eidx, gidx), group
def find_group(match_tree, prop):
"""Find the list of groups that resulted in a guess that contains the
asked property."""
result = []
for gpos, (string, remaining, guess) in iterate_groups(match_tree):
if guess and prop in guess:
result.append(gpos)
return result
def get_group(match_tree, gpos):
pidx, eidx, gidx = gpos
return match_tree[pidx][eidx][gidx]
def leftover_valid_groups(match_tree, valid = lambda s: len(s[0]) > 3):
"""Return the list of valid string groups (eg: len(s) > 3) that could not be
matched to anything as a list of pairs (cleaned_str, group_pos)."""
leftover = []
for gpos, (group, remaining, guess) in iterate_groups(match_tree):
if not guess:
clean_str = clean_string(remaining)
if valid((clean_str, gpos)):
leftover.append((clean_str, gpos))
return leftover
libs/guessit/patterns.py
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
subtitle_exts = [ 'srt', 'idx', 'sub' ]
video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'mov', 'ogg', 'ogm', 'ogv', 'wmv' ]
# separator character regexp
sep = r'[][)(}{+ \._-]' # regexp art, hehe :D
# character used to represent a deleted char (when matching groups)
deleted = '_'
# format: [ (regexp, confidence, span_adjust) ]
episode_rexps = [ # ... Season 2 ...
(r'season (?P<season>[0-9]+)', 1.0, (0, 0)),
(r'saison (?P<season>[0-9]+)', 1.0, (0, 0)),
# ... s02e13 ...
(r'[Ss](?P<season>[0-9]{1,2}).{,3}[EeXx](?P<episodeNumber>[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
# ... 2x13 ...
(r'[^0-9](?P<season>[0-9]{1,2})[x\.](?P<episodeNumber>[0-9]{2})[^0-9]', 0.8, (1, -1)),
# ... s02 ...
(sep + r's(?P<season>[0-9]{1,2})' + sep + '?', 0.6, (0, 0)),
# v2 or v3 for some mangas which have multiples rips
(sep + r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
]
weak_episode_rexps = [ # ... 213 or 0106 ...
(sep + r'(?P<episodeNumber>[0-9]{1,4})' + sep, 0.3, (1, -1)),
]
non_episode_title = [ 'extras' ]
video_rexps = [ # cd number
(r'cd ?(?P<cdNumber>[0-9])( ?of ?(?P<cdNumberTotal>[0-9]))?', 1.0, (0, 0)),
(r'(?P<cdNumberTotal>[1-9]) cds?', 0.9, (0, 0)),
# special editions
(r'edition' + sep + r'(?P<edition>collector)', 1.0, (0, 0)),
(r'(?P<edition>collector)' + sep + 'edition', 1.0, (0, 0)),
(r'(?P<edition>special)' + sep + 'edition', 1.0, (0, 0)),
(r'(?P<edition>criterion)' + sep + 'edition', 1.0, (0, 0)),
# director's cut
(r"(?P<edition>director'?s?" + sep + "cut)", 1.0, (0, 0)),
# video size
(r'(?P<width>[0-9]{3,4})x(?P<height>[0-9]{3,4})', 0.9, (0, 0)),
# website
(r'(?P<website>www(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0))
]
websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com', 'sharethefiles.com' ]
properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'Blu-ray', 'BDRip', 'BRRip',
'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'WEBRip', 'DVDSCR', 'Screener', 'VHS',
'VIDEO_TS' ],
'container': [ 'avi', 'mkv', 'ogv', 'ogm', 'wmv', 'mp4', 'mov' ],
'screenSize': [ '720p', '720' ],
'videoCodec': [ 'XviD', 'DivX', 'x264', 'h264', 'Rv10' ],
'audioCodec': [ 'AC3', 'DTS', 'He-AAC', 'AAC-He', 'AAC' ],
'audioChannels': [ '5.1' ],
'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', '[XCT]', 'iNT', 'PUKKA',
'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE',
'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe',
'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL' ],
'episodeFormat': [ 'Minisode', 'Minisodes' ],
'other': [ '5ch', 'PROPER', 'REPACK', 'LIMITED', 'DualAudio', 'iNTERNAL', 'Audiofixed', 'R5',
'complete', 'classic', # not so sure about these ones, could appear in a title
'ws', # widescreen
#'SE', # special edition
# TODO: director's cut
],
}
property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
'HD-DVD': [ 'HDDVD', 'HDDVDRip' ],
'BluRay': [ 'BDRip', 'BRRip', 'Blu-ray' ],
'Screener': [ 'DVDSCR' ],
'DivX': [ 'DVDivX' ],
'h264': [ 'x264' ],
'720p': [ '720' ],
'AAC': [ 'He-AAC', 'AAC-He' ],
'Special Edition': [ 'Special' ],
'Collector Edition': [ 'Collector' ],
'Criterion Edition': [ 'Criterion' ],
'Minisode': [ 'Minisodes' ]
}
reverse_synonyms = {}
for canonical, synonyms in property_synonyms.items():
for synonym in synonyms:
reverse_synonyms[synonym.lower()] = canonical
def canonical_form(string):
return reverse_synonyms.get(string.lower(), string)
libs/guessit/slogging.py
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Smewt - A smart collection manager
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# Smewt is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Smewt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import sys
GREEN_FONT = "\x1B[0;32m"
YELLOW_FONT = "\x1B[0;33m"
BLUE_FONT = "\x1B[0;34m"
RED_FONT = "\x1B[0;31m"
RESET_FONT = "\x1B[0m"
def setupLogging(colored = True):
"""Sets up a nice colored logger as the main application logger (not only smewt itself)."""
class SimpleFormatter(logging.Formatter):
def __init__(self):
self.fmt = '%(levelname)-8s %(module)s:%(funcName)s -- %(message)s'
logging.Formatter.__init__(self, self.fmt)
class ColoredFormatter(logging.Formatter):
def __init__(self):
self.fmt = '%(levelname)-8s ' + BLUE_FONT + '%(module)s:%(funcName)s' + RESET_FONT + ' -- %(message)s'
logging.Formatter.__init__(self, self.fmt)
def format(self, record):
result = logging.Formatter.format(self, record)
if record.levelno in (logging.DEBUG, logging.INFO):
return GREEN_FONT + result
elif record.levelno == logging.WARNING:
return YELLOW_FONT + result
else:
return RED_FONT + result
ch = logging.StreamHandler()
if colored and sys.platform != 'win32':
ch.setFormatter(ColoredFormatter())
else:
ch.setFormatter(SimpleFormatter())
logging.getLogger().addHandler(ch)
libs/guessit/textutils.py
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Smewt - A smart collection manager
# Copyright (c) 2008 Nicolas Wack <wackou@gmail.com>
#
# Smewt is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Smewt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import sep, deleted
import copy
# string-related functions
def strip_brackets(s):
if not s:
return s
if s[0] == '[' and s[-1] == ']': return s[1:-1]
if s[0] == '(' and s[-1] == ')': return s[1:-1]
if s[0] == '{' and s[-1] == '}': return s[1:-1]
return s
def clean_string(s):
for c in sep:
s = s.replace(c, ' ')
parts = s.split()
return ' '.join(p for p in parts if p != '')
def str_replace(string, pos, c):
return string[:pos] + c + string[pos+1:]
def blank_region(string, region, blank_sep = deleted):
start, end = region
return string[:start] + blank_sep * (end - start) + string[end:]
def between(s, left, right):
return s.split(left)[1].split(right)[0]
def to_utf8(o):
'''converts all unicode strings found in the given object to utf-8 strings'''
if isinstance(o, unicode):
return o.encode('utf-8')
elif isinstance(o, list):
return [ to_utf8(i) for i in o ]
elif isinstance(o, dict):
result = copy.deepcopy(o) # need to do it like that to handle Guess instances correctly
for key, value in o.items():
result[to_utf8(key)] = to_utf8(value)
return result
else:
return o
def levenshtein(a, b):
if not a: return len(b)
if not b: return len(a)
m = len(a)
n = len(b)
d = []
for i in range(m+1):
d.append([0] * (n+1))
for i in range(m+1):
d[i][0] = i
for j in range(n+1):
d[0][j] = j
for i in range(1, m+1):
for j in range(1, n+1):
if a[i-1] == b[j-1]:
cost = 0
else:
cost = 1
d[i][j] = min(d[i-1][j] + 1, # deletion
d[i][j-1] + 1, # insertion
d[i-1][j-1] + cost # substitution
)
return d[m][n]
# group-related functions
def find_first_level_groups_span(string, enclosing):
"""Return a list of pairs (start, end) for the groups delimited by the given
enclosing characters.
This does not return nested groups, ie: '(ab(c)(d))' will return a single group
containing the whole string.
>>> find_first_level_groups_span('abcd', '()')
[]
>>> find_first_level_groups_span('abc(de)fgh', '()')
[(3, 7)]
>>> find_first_level_groups_span('(ab(c)(d))', '()')
[(0, 10)]
>>> find_first_level_groups_span('ab[c]de[f]gh(i)', '[]')
[(2, 5), (7, 10)]
"""
opening, closing = enclosing
depth = [] # depth is a stack of indices where we opened a group
result = []
for i, c, in enumerate(string):
if c == opening:
depth.append(i)
elif c == closing:
try:
start = depth.pop()
end = i
if not depth:
# we emptied our stack, so we have a 1st level group
result.append((start, end+1))
except IndexError:
# we closed a group which was not opened before
pass
return result
def split_on_groups(string, groups):
"""Split the given string using the different known groups for boundaries.
>>> split_on_groups('0123456789', [ (2, 4) ])
['01', '23', '456789']
>>> split_on_groups('0123456789', [ (2, 4), (4, 6) ])
['01', '23', '45', '6789']
>>> split_on_groups('0123456789', [ (5, 7), (2, 4) ])
['01', '23', '4', '56', '789']
"""
if not groups:
return [ string ]
boundaries = sorted(set(reduce(lambda l, x: l + list(x), groups, [])))
if boundaries[0] != 0:
boundaries.insert(0, 0)
if boundaries[-1] != len(string):
boundaries.append(len(string))
groups = [ string[start:end] for start, end in zip(boundaries[:-1], boundaries[1:]) ]
return filter(bool, groups) # return only non-empty groups
def find_first_level_groups(string, enclosing, blank_sep = None):
"""Return a list of groups that could be split because of explicit grouping.
The groups are delimited by the given enclosing characters.
You can also specify if you want to blank the separator chars in the returned
list of groups by specifying a character for it. None means it won't be replaced.
This does not return nested groups, ie: '(ab(c)(d))' will return a single group
containing the whole string.
>>> find_first_level_groups('', '()')
['']
>>> find_first_level_groups('abcd', '()')
['abcd']
>>> find_first_level_groups('abc(de)fgh', '()')
['abc', '(de)', 'fgh']
>>> find_first_level_groups('(ab(c)(d))', '()', blank_sep = '_')
['_ab(c)(d)_']
>>> find_first_level_groups('ab[c]de[f]gh(i)', '[]')
['ab', '[c]', 'de', '[f]', 'gh(i)']
>>> find_first_level_groups('()[]()', '()', blank_sep = '-')
['--', '[]', '--']
"""
groups = find_first_level_groups_span(string, enclosing)
if blank_sep:
for start, end in groups:
string = str_replace(string, start, blank_sep)
string = str_replace(string, end-1, blank_sep)
return split_on_groups(string, groups)
libs/requests/__init__.py
+34
View File
@@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
# __
# /__) _ _ _ _ _/ _
# / ( (- (/ (/ (- _) / _)
# /
"""
requests
~~~~~~~~
:copyright: (c) 2011 by Kenneth Reitz.
:license: ISC, see LICENSE for more details.
"""
__title__ = 'requests'
__version__ = '0.8.6'
__build__ = 0x000806
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2011 Kenneth Reitz'
from . import utils
from .models import Request, Response
from .api import request, get, head, post, patch, put, delete, options
from .sessions import session, Session
from .status_codes import codes
from .exceptions import (
RequestException, Timeout, URLRequired,
TooManyRedirects, HTTPError, ConnectionError
)
libs/requests/api.py
+144
View File
@@ -0,0 +1,144 @@
# -*- coding: utf-8 -*-
"""
requests.api
~~~~~~~~~~~~
This module implements the Requests API.
:copyright: (c) 2011 by Kenneth Reitz.
:license: ISC, see LICENSE for more details.
"""
from . import sessions
def request(method, url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=False,
proxies=None,
hooks=None,
return_response=True,
prefetch=False,
session=None,
config=None):
"""Constructs and sends a :class:`Request <Request>`.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param return_response: (optional) If False, an un-sent Request object will returned.
:param session: (optional) A :class:`Session` object to be used for the request.
:param config: (optional) A configuration dictionary.
"""
s = session or sessions.session()
return s.request(
method=method,
url=url,
params=params,
data=data,
headers=headers,
cookies=cookies,
files=files,
auth=auth,
timeout=timeout,
allow_redirects=allow_redirects,
proxies=proxies,
hooks=hooks,
return_response=return_response,
config=config,
prefetch=prefetch
)
def get(url, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
return request('get', url, **kwargs)
def options(url, **kwargs):
"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
return request('options', url, **kwargs)
def head(url, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
return request('head', url, **kwargs)
def post(url, data=None, **kwargs):
"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return request('post', url, data=data, **kwargs)
def put(url, data=None, **kwargs):
"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return request('put', url, data=data, **kwargs)
def patch(url, data=None, **kwargs):
"""Sends a PATCH request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return request('patch', url, data=data, **kwargs)
def delete(url, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return request('delete', url, **kwargs)
libs/requests/async.py
+91
View File
@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""
requests.async
~~~~~~~~~~~~~~
This module contains an asynchronous replica of ``requests.api``, powered
by gevent. All API methods return a ``Request`` instance (as opposed to
``Response``). A list of requests can be sent with ``map()``.
"""
try:
import gevent
from gevent import monkey as curious_george
from gevent.pool import Pool
except ImportError:
raise RuntimeError('Gevent is required for requests.async.')
# Monkey-patch.
curious_george.patch_all(thread=False)
from . import api
__all__ = (
'map',
'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
)
def patched(f):
"""Patches a given API function to not send."""
def wrapped(*args, **kwargs):
kwargs['return_response'] = False
kwargs['prefetch'] = True
return f(*args, **kwargs)
return wrapped
def send(r, pools=None):
"""Sends a given Request object."""
if pools:
r._pools = pools
r.send()
return r.response
# Patched requests.api functions.
get = patched(api.get)
options = patched(api.options)
head = patched(api.head)
post = patched(api.post)
put = patched(api.put)
patch = patched(api.patch)
delete = patched(api.delete)
request = patched(api.request)
def map(requests, prefetch=True, size=None):
"""Concurrently converts a list of Requests to Responses.
:param requests: a collection of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
"""
requests = list(requests)
if size:
pool = Pool(size)
pool.map(send, requests)
pool.join()
else:
jobs = [gevent.spawn(send, r) for r in requests]
gevent.joinall(jobs)
if prefetch:
[r.response.content for r in requests]
return [r.response for r in requests]
libs/requests/auth.py
+136
View File
@@ -0,0 +1,136 @@
# -*- coding: utf-8 -*-
"""
requests.auth
~~~~~~~~~~~~~
This module contains the authentication handlers for Requests.
"""
import time
import hashlib
from base64 import b64encode
from urlparse import urlparse
from .utils import randombytes, parse_dict_header
def _basic_auth_str(username, password):
"""Returns a Basic Auth string."""
return 'Basic %s' % b64encode('%s:%s' % (username, password))
class AuthBase(object):
"""Base class that all auth implementations derive from"""
def __call__(self, r):
raise NotImplementedError('Auth hooks must be callable.')
class HTTPBasicAuth(AuthBase):
"""Attaches HTTP Basic Authentication to the given Request object."""
def __init__(self, username, password):
self.username = str(username)
self.password = str(password)
def __call__(self, r):
r.headers['Authorization'] = _basic_auth_str(self.username, self.password)
return r
class HTTPProxyAuth(HTTPBasicAuth):
"""Attaches HTTP Proxy Authenetication to a given Request object."""
def __call__(self, r):
r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password)
return r
class HTTPDigestAuth(AuthBase):
"""Attaches HTTP Digest Authentication to the given Request object."""
def __init__(self, username, password):
self.username = username
self.password = password
def handle_401(self, r):
"""Takes the given response and tries digest-auth, if needed."""
s_auth = r.headers.get('www-authenticate', '')
if 'digest' in s_auth.lower():
last_nonce = ''
nonce_count = 0
chal = parse_dict_header(s_auth.replace('Digest ', ''))
realm = chal['realm']
nonce = chal['nonce']
qop = chal.get('qop')
algorithm = chal.get('algorithm', 'MD5')
opaque = chal.get('opaque', None)
algorithm = algorithm.upper()
# lambdas assume digest modules are imported at the top level
if algorithm == 'MD5':
H = lambda x: hashlib.md5(x).hexdigest()
elif algorithm == 'SHA':
H = lambda x: hashlib.sha1(x).hexdigest()
# XXX MD5-sess
KD = lambda s, d: H("%s:%s" % (s, d))
if H is None:
return None
# XXX not implemented yet
entdig = None
p_parsed = urlparse(r.request.url)
path = p_parsed.path + p_parsed.query
A1 = '%s:%s:%s' % (self.username, realm, self.password)
A2 = '%s:%s' % (r.request.method, path)
if qop == 'auth':
if nonce == last_nonce:
nonce_count += 1
else:
nonce_count = 1
last_nonce = nonce
ncvalue = '%08x' % nonce_count
cnonce = (hashlib.sha1("%s:%s:%s:%s" % (
nonce_count, nonce, time.ctime(), randombytes(8)))
.hexdigest()[:16]
)
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
respdig = KD(H(A1), noncebit)
elif qop is None:
respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
else:
# XXX handle auth-int.
return None
# XXX should the partial digests be encoded too?
base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
'response="%s"' % (self.username, realm, nonce, path, respdig)
if opaque:
base += ', opaque="%s"' % opaque
if entdig:
base += ', digest="%s"' % entdig
base += ', algorithm="%s"' % algorithm
if qop:
base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
r.request.headers['Authorization'] = 'Digest %s' % (base)
r.request.send(anyway=True)
_r = r.request.response
_r.history.append(r)
return _r
return r
def __call__(self, r):
r.hooks['response'] = self.handle_401
return r
libs/requests/defaults.py
+42
View File
@@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
"""
requests.defaults
~~~~~~~~~~~~~~~~~
This module provides the Requests configuration defaults.
Configurations:
:base_headers: Default HTTP headers.
:verbose: Stream to write request logging to.
:timeout: Seconds until request timeout.
:max_redirects: Maximum number of redirects allowed within a request.
:decode_unicode: Decode unicode responses automatically?
:keep_alive: Reuse HTTP Connections?
:max_retries: The number of times a request should be retried in the event of a connection failure.
:safe_mode: If true, Requests will catch all errors.
:pool_maxsize: The maximium size of an HTTP connection pool.
:pool_connections: The number of active HTTP connection pools to use.
"""
from . import __version__
defaults = dict()
defaults['base_headers'] = {
'User-Agent': 'python-requests/%s' % __version__,
'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')),
'Accept': '*/*'
}
defaults['verbose'] = None
defaults['max_redirects'] = 30
defaults['decode_unicode'] = True
defaults['pool_connections'] = 10
defaults['pool_maxsize'] = 10
defaults['max_retries'] = 0
defaults['safe_mode'] = False
defaults['keep_alive'] = True
libs/requests/exceptions.py
+28
View File
@@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
"""
requests.exceptions
~~~~~~~~~~~~~~~~~~~
This module contains the set of Requests' exceptions.
"""
class RequestException(Exception):
"""There was an ambiguous exception that occurred while handling your
request."""
class HTTPError(RequestException):
"""An HTTP error occurred."""
class ConnectionError(RequestException):
"""A Connection error occurred."""
class Timeout(RequestException):
"""The request timed out."""
class URLRequired(RequestException):
"""A valid URL is required to make a request."""
class TooManyRedirects(RequestException):
"""Too many redirects."""
libs/requests/hooks.py
+40
View File
@@ -0,0 +1,40 @@
# -*- coding: utf-8 -*-
"""
requests.hooks
~~~~~~~~~~~~~~
This module provides the capabilities for the Requests hooks system.
Available hooks:
``args``:
A dictionary of the arguments being sent to Request().
``pre_request``:
The Request object, directly before being sent.
``post_request``:
The Request object, directly after being sent.
``response``:
The response generated from a Request.
"""
import warnings
def dispatch_hook(key, hooks, hook_data):
"""Dispatches a hook dictionary on a given piece of data."""
hooks = hooks or dict()
if key in hooks:
try:
return hooks.get(key).__call__(hook_data) or hook_data
except Exception, why:
warnings.warn(str(why))
return hook_data
libs/requests/models.py
+689
View File
@@ -0,0 +1,689 @@
# -*- coding: utf-8 -*-
"""
requests.models
~~~~~~~~~~~~~~~
This module contains the primary objects that power Requests.
"""
import urllib
import zlib
from urlparse import urlparse, urlunparse, urljoin, urlsplit
from datetime import datetime
from .hooks import dispatch_hook
from .structures import CaseInsensitiveDict
from .status_codes import codes
from .packages import oreos
from .auth import HTTPBasicAuth, HTTPProxyAuth
from .packages.urllib3.exceptions import MaxRetryError
from .packages.urllib3.exceptions import SSLError as _SSLError
from .packages.urllib3.exceptions import HTTPError as _HTTPError
from .packages.urllib3 import connectionpool, poolmanager
from .packages.urllib3.filepost import encode_multipart_formdata
from .exceptions import (
Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError)
from .utils import (
get_encoding_from_headers, stream_decode_response_unicode,
decode_gzip, stream_decode_gzip, guess_filename, requote_path)
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
class Request(object):
"""The :class:`Request <Request>` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
"""
def __init__(self,
url=None,
headers=dict(),
files=None,
method=None,
data=dict(),
params=dict(),
auth=None,
cookies=None,
timeout=None,
redirect=False,
allow_redirects=False,
proxies=None,
hooks=None,
config=None,
_poolmanager=None):
#: Float describes the timeout of the request.
# (Use socket.setdefaulttimeout() as fallback)
self.timeout = timeout
#: Request URL.
self.url = url
#: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
self.headers = dict(headers or [])
#: Dictionary of files to multipart upload (``{filename: content}``).
self.files = files
#: HTTP Method to use.
self.method = method
#: Dictionary or byte of request body data to attach to the
#: :class:`Request <Request>`.
self.data = None
#: Dictionary or byte of querystring data to attach to the
#: :class:`Request <Request>`.
self.params = None
self.params = dict(params or [])
#: True if :class:`Request <Request>` is part of a redirect chain (disables history
#: and HTTPError storage).
self.redirect = redirect
#: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``)
self.allow_redirects = allow_redirects
# Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
self.proxies = dict(proxies or [])
self.data, self._enc_data = self._encode_params(data)
self.params, self._enc_params = self._encode_params(params)
#: :class:`Response <Response>` instance, containing
#: content and metadata of HTTP Response, once :attr:`sent <send>`.
self.response = Response()
#: Authentication tuple or object to attach to :class:`Request <Request>`.
self.auth = auth
#: CookieJar to attach to :class:`Request <Request>`.
self.cookies = dict(cookies or [])
#: Dictionary of configurations for this request.
self.config = dict(config or [])
#: True if Request has been sent.
self.sent = False
#: Event-handling hooks.
self.hooks = hooks
#: Session.
self.session = None
if headers:
headers = CaseInsensitiveDict(self.headers)
else:
headers = CaseInsensitiveDict()
for (k, v) in self.config.get('base_headers', {}).items():
if k not in headers:
headers[k] = v
self.headers = headers
self._poolmanager = _poolmanager
# Pre-request hook.
r = dispatch_hook('pre_request', hooks, self)
self.__dict__.update(r.__dict__)
def __repr__(self):
return '<Request [%s]>' % (self.method)
def _build_response(self, resp, is_error=False):
"""Build internal :class:`Response <Response>` object
from given response.
"""
def build(resp):
response = Response()
# Pass settings over.
response.config = self.config
if resp:
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
# Start off with our local cookies.
cookies = self.cookies or dict()
# Add new cookies from the server.
if 'set-cookie' in response.headers:
cookie_header = response.headers['set-cookie']
cookies = oreos.dict_from_string(cookie_header)
# Save cookies in Response.
response.cookies = cookies
# Save original response for later.
response.raw = resp
if is_error:
response.error = resp
response.url = self.full_url
return response
history = []
r = build(resp)
cookies = self.cookies
self.cookies.update(r.cookies)
if r.status_code in REDIRECT_STATI and not self.redirect:
while (
('location' in r.headers) and
((r.status_code is codes.see_other) or (self.allow_redirects))
):
if not len(history) < self.config.get('max_redirects'):
raise TooManyRedirects()
history.append(r)
url = r.headers['location']
# Handle redirection without scheme (see: RFC 1808 Section 4)
if url.startswith('//'):
parsed_rurl = urlparse(r.url)
url = '%s:%s' % (parsed_rurl.scheme, url)
# Facilitate non-RFC2616-compliant 'location' headers
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
if not urlparse(url).netloc:
url = urljoin(r.url, url)
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
if r.status_code is codes.see_other:
method = 'GET'
else:
method = self.method
# Remove the cookie headers that were sent.
headers = self.headers
try:
del headers['Cookie']
except KeyError:
pass
request = Request(
url=url,
headers=headers,
files=self.files,
method=method,
params=self.session.params,
auth=self.auth,
cookies=cookies,
redirect=True,
config=self.config,
timeout=self.timeout,
_poolmanager=self._poolmanager,
proxies = self.proxies,
)
request.send()
cookies.update(request.response.cookies)
r = request.response
self.cookies.update(r.cookies)
r.history = history
self.response = r
self.response.request = self
self.response.cookies.update(self.cookies)
@staticmethod
def _encode_params(data):
"""Encode parameters in a piece of data.
If the data supplied is a dictionary, encodes each parameter in it, and
returns a list of tuples containing the encoded parameters, and a urlencoded
version of that.
Otherwise, assumes the data is already encoded appropriately, and
returns it twice.
"""
if hasattr(data, '__iter__'):
data = dict(data)
if hasattr(data, 'items'):
result = []
for k, vs in data.items():
for v in isinstance(vs, list) and vs or [vs]:
result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
v.encode('utf-8') if isinstance(v, unicode) else v))
return result, urllib.urlencode(result, doseq=True)
else:
return data, data
@property
def full_url(self):
"""Build the actual URL to use."""
if not self.url:
raise URLRequired()
# Support for unicode domain names and paths.
scheme, netloc, path, params, query, fragment = urlparse(self.url)
if not scheme:
raise ValueError()
netloc = netloc.encode('idna')
if isinstance(path, unicode):
path = path.encode('utf-8')
path = requote_path(path)
url = str(urlunparse([ scheme, netloc, path, params, query, fragment ]))
if self._enc_params:
if urlparse(url).query:
return '%s&%s' % (url, self._enc_params)
else:
return '%s?%s' % (url, self._enc_params)
else:
return url
@property
def path_url(self):
"""Build the path URL to use."""
url = []
p = urlsplit(self.full_url)
# Proxies use full URLs.
if p.scheme in self.proxies:
return self.full_url
path = p.path
if not path:
path = '/'
url.append(path)
query = p.query
if query:
url.append('?')
url.append(query)
return ''.join(url)
def send(self, anyway=False, prefetch=False):
"""Sends the request. Returns True of successful, false if not.
If there was an HTTPError during transmission,
self.response.status_code will contain the HTTPError code.
Once a request is successfully sent, `sent` will equal True.
:param anyway: If True, request will be sent, even if it has
already been sent.
"""
# Build the URL
url = self.full_url
# Logging
if self.config.get('verbose'):
self.config.get('verbose').write('%s %s %s\n' % (
datetime.now().isoformat(), self.method, url
))
# Nottin' on you.
body = None
content_type = None
# Multi-part file uploads.
if self.files:
if not isinstance(self.data, basestring):
try:
fields = self.data.copy()
except AttributeError:
fields = dict(self.data)
for (k, v) in self.files.items():
# support for explicit filename
if isinstance(v, (tuple, list)):
fn, fp = v
else:
fn = guess_filename(v) or k
fp = v
fields.update({k: (fn, fp.read())})
(body, content_type) = encode_multipart_formdata(fields)
else:
pass
# TODO: Conflict?
else:
if self.data:
body = self._enc_data
if isinstance(self.data, basestring):
content_type = None
else:
content_type = 'application/x-www-form-urlencoded'
# Add content-type if it wasn't explicitly provided.
if (content_type) and (not 'content-type' in self.headers):
self.headers['Content-Type'] = content_type
if self.auth:
if isinstance(self.auth, tuple) and len(self.auth) == 2:
# special-case basic HTTP auth
self.auth = HTTPBasicAuth(*self.auth)
# Allow auth to make its changes.
r = self.auth(self)
# Update self to reflect the auth changes.
self.__dict__.update(r.__dict__)
_p = urlparse(url)
proxy = self.proxies.get(_p.scheme)
if proxy:
conn = poolmanager.proxy_from_url(proxy)
_proxy = urlparse(proxy)
if '@' in _proxy.netloc:
auth, url = _proxy.netloc.split('@', 1)
self.proxy_auth = HTTPProxyAuth(*auth.split(':', 1))
r = self.proxy_auth(self)
self.__dict__.update(r.__dict__)
else:
# Check to see if keep_alive is allowed.
if self.config.get('keep_alive'):
conn = self._poolmanager.connection_from_url(url)
else:
conn = connectionpool.connection_from_url(url)
if not self.sent or anyway:
if self.cookies:
# Skip if 'cookie' header is explicitly set.
if 'cookie' not in self.headers:
# Simple cookie with our dict.
c = oreos.monkeys.SimpleCookie()
for (k, v) in self.cookies.items():
c[k] = v
# Turn it into a header.
cookie_header = c.output(header='', sep='; ').strip()
# Attach Cookie header to request.
self.headers['Cookie'] = cookie_header
try:
# Send the request.
r = conn.urlopen(
method=self.method,
url=self.path_url,
body=body,
headers=self.headers,
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.config.get('max_retries', 0),
timeout=self.timeout,
)
self.sent = True
except MaxRetryError, e:
if not self.config.get('safe_mode', False):
raise ConnectionError(e)
else:
r = None
except (_SSLError, _HTTPError), e:
if not self.config.get('safe_mode', False):
raise Timeout('Request timed out.')
self._build_response(r)
# Response manipulation hook.
self.response = dispatch_hook('response', self.hooks, self.response)
# Post-request hook.
r = dispatch_hook('post_request', self.hooks, self)
self.__dict__.update(r.__dict__)
# If prefetch is True, mark content as consumed.
if prefetch:
# Save the response.
self.response.content
return self.sent
class Response(object):
"""The core :class:`Response <Response>` object. All
:class:`Request <Request>` objects contain a
:class:`response <Response>` attribute, which is an instance
of this class.
"""
def __init__(self):
self._content = None
self._content_consumed = False
#: Integer Code of responded HTTP Status.
self.status_code = None
#: Case-insensitive Dictionary of Response Headers.
#: For example, ``headers['content-encoding']`` will return the
#: value of a ``'Content-Encoding'`` response header.
self.headers = CaseInsensitiveDict()
#: File-like object representation of response (for advanced usage).
self.raw = None
#: Final URL location of Response.
self.url = None
#: Resulting :class:`HTTPError` of request, if one occurred.
self.error = None
#: Encoding to decode with when accessing r.content.
self.encoding = None
#: A list of :class:`Response <Response>` objects from
#: the history of the Request. Any redirect responses will end
#: up here.
self.history = []
#: The :class:`Request <Request>` that created the Response.
self.request = None
#: A dictionary of Cookies the server sent back.
self.cookies = {}
#: Dictionary of configurations for this request.
self.config = {}
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
def __nonzero__(self):
"""Returns true if :attr:`status_code` is 'OK'."""
return self.ok
@property
def ok(self):
try:
self.raise_for_status()
except HTTPError:
return False
return True
def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
"""Iterates over the response data. This avoids reading the content
at once into memory for large responses. The chunk size is the number
of bytes it should read into memory. This is not necessarily the
length of each item returned as decoding can take place.
"""
if self._content_consumed:
raise RuntimeError(
'The content for this response was already consumed'
)
def generate():
while 1:
chunk = self.raw.read(chunk_size)
if not chunk:
break
yield chunk
self._content_consumed = True
gen = generate()
if 'gzip' in self.headers.get('content-encoding', ''):
gen = stream_decode_gzip(gen)
if decode_unicode is None:
decode_unicode = self.config.get('decode_unicode')
if decode_unicode:
gen = stream_decode_response_unicode(gen, self)
return gen
def iter_lines(self, newlines=None, decode_unicode=None):
"""Iterates over the response data, one line at a time. This
avoids reading the content at once into memory for large
responses.
:param newlines: a collection of bytes to seperate lines with.
"""
if newlines is None:
newlines = ('\r', '\n', '\r\n')
if self._content_consumed:
raise RuntimeError(
'The content for this response was already consumed'
)
def generate():
chunk = []
while 1:
c = self.raw.read(1)
if not c:
break
if c in newlines:
yield ''.join(chunk)
chunk = []
else:
chunk.append(c)
self._content_consumed = True
gen = generate()
if 'gzip' in self.headers.get('content-encoding', ''):
gen = stream_decode_gzip(gen)
if decode_unicode is None:
decode_unicode = self.config.get('decode_unicode')
if decode_unicode:
gen = stream_decode_response_unicode(gen, self)
return gen
@property
def content(self):
"""Content of the response, in bytes or unicode
(if available).
"""
if self._content is None:
# Read the contents.
try:
if self._content_consumed:
raise RuntimeError(
'The content for this response was already consumed')
self._content = self.raw.read()
except AttributeError:
self._content = None
content = self._content
# Decode GZip'd content.
if 'gzip' in self.headers.get('content-encoding', ''):
try:
content = decode_gzip(self._content)
except zlib.error:
pass
# Decode unicode content.
if self.config.get('decode_unicode'):
# Try charset from content-type
if self.encoding:
try:
content = unicode(content, self.encoding)
except UnicodeError:
pass
# Fall back:
try:
content = unicode(content, self.encoding, errors='replace')
except TypeError:
pass
self._content_consumed = True
return content
def raise_for_status(self):
"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
if self.error:
raise self.error
if (self.status_code >= 300) and (self.status_code < 400):
raise HTTPError('%s Redirection' % self.status_code)
elif (self.status_code >= 400) and (self.status_code < 500):
raise HTTPError('%s Client Error' % self.status_code)
elif (self.status_code >= 500) and (self.status_code < 600):
raise HTTPError('%s Server Error' % self.status_code)
libs/requests/packages/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from __future__ import absolute_import
from . import urllib3
libs/requests/packages/oreos/__init__.py
+3
View File
@@ -0,0 +1,3 @@
# -*- coding: utf-8 -*-
from .core import dict_from_string
libs/requests/packages/oreos/core.py
+24
View File
@@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
"""
oreos.core
~~~~~~~~~~
The creamy white center.
"""
from .monkeys import SimpleCookie
def dict_from_string(s):
"""Returns a MultiDict with Cookies."""
cookies = dict()
c = SimpleCookie()
c.load(s)
for k,v in c.items():
cookies.update({k: v.value})
return cookies
libs/requests/packages/oreos/monkeys.py
+770
View File
@@ -0,0 +1,770 @@
# -*- coding: utf-8 -*-
"""
oreos.monkeys
~~~~~~~~~~~~~
Monkeypatches.
"""
#!/usr/bin/env python
#
####
# Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software
# and its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of
# Timothy O'Malley not be used in advertising or publicity
# pertaining to distribution of the software without specific, written
# prior permission.
#
# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
#
####
#
# Id: Cookie.py,v 2.29 2000/08/23 05:28:49 timo Exp
# by Timothy O'Malley <timo@alum.mit.edu>
#
# Cookie.py is a Python module for the handling of HTTP
# cookies as a Python dictionary. See RFC 2109 for more
# information on cookies.
#
# The original idea to treat Cookies as a dictionary came from
# Dave Mitchell (davem@magnet.com) in 1995, when he released the
# first version of nscookie.py.
#
####
r"""
Here's a sample session to show how to use this module.
At the moment, this is the only documentation.
The Basics
----------
Importing is easy..
>>> import Cookie
Most of the time you start by creating a cookie. Cookies come in
three flavors, each with slightly different encoding semantics, but
more on that later.
>>> C = Cookie.SimpleCookie()
>>> C = Cookie.SerialCookie()
>>> C = Cookie.SmartCookie()
[Note: Long-time users of Cookie.py will remember using
Cookie.Cookie() to create an Cookie object. Although deprecated, it
is still supported by the code. See the Backward Compatibility notes
for more information.]
Once you've created your Cookie, you can add values just as if it were
a dictionary.
>>> C = Cookie.SmartCookie()
>>> C["fig"] = "newton"
>>> C["sugar"] = "wafer"
>>> C.output()
'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer'
Notice that the printable representation of a Cookie is the
appropriate format for a Set-Cookie: header. This is the
default behavior. You can change the header and printed
attributes by using the .output() function
>>> C = Cookie.SmartCookie()
>>> C["rocky"] = "road"
>>> C["rocky"]["path"] = "/cookie"
>>> print C.output(header="Cookie:")
Cookie: rocky=road; Path=/cookie
>>> print C.output(attrs=[], header="Cookie:")
Cookie: rocky=road
The load() method of a Cookie extracts cookies from a string. In a
CGI script, you would use this method to extract the cookies from the
HTTP_COOKIE environment variable.
>>> C = Cookie.SmartCookie()
>>> C.load("chips=ahoy; vienna=finger")
>>> C.output()
'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger'
The load() method is darn-tootin smart about identifying cookies
within a string. Escaped quotation marks, nested semicolons, and other
such trickeries do not confuse it.
>>> C = Cookie.SmartCookie()
>>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";')
>>> print C
Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;"
Each element of the Cookie also supports all of the RFC 2109
Cookie attributes. Here's an example which sets the Path
attribute.
>>> C = Cookie.SmartCookie()
>>> C["oreo"] = "doublestuff"
>>> C["oreo"]["path"] = "/"
>>> print C
Set-Cookie: oreo=doublestuff; Path=/
Each dictionary element has a 'value' attribute, which gives you
back the value associated with the key.
>>> C = Cookie.SmartCookie()
>>> C["twix"] = "none for you"
>>> C["twix"].value
'none for you'
A Bit More Advanced
-------------------
As mentioned before, there are three different flavors of Cookie
objects, each with different encoding/decoding semantics. This
section briefly discusses the differences.
SimpleCookie
The SimpleCookie expects that all values should be standard strings.
Just to be sure, SimpleCookie invokes the str() builtin to convert
the value to a string, when the values are set dictionary-style.
>>> C = Cookie.SimpleCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
'7'
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number=7\r\nSet-Cookie: string=seven'
SerialCookie
The SerialCookie expects that all values should be serialized using
cPickle (or pickle, if cPickle isn't available). As a result of
serializing, SerialCookie can save almost any Python object to a
value, and recover the exact same object when the cookie has been
returned. (SerialCookie can yield some strange-looking cookie
values, however.)
>>> C = Cookie.SerialCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
7
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number="I7\\012."\r\nSet-Cookie: string="S\'seven\'\\012p1\\012."'
Be warned, however, if SerialCookie cannot de-serialize a value (because
it isn't a valid pickle'd object), IT WILL RAISE AN EXCEPTION.
SmartCookie
The SmartCookie combines aspects of each of the other two flavors.
When setting a value in a dictionary-fashion, the SmartCookie will
serialize (ala cPickle) the value *if and only if* it isn't a
Python string. String objects are *not* serialized. Similarly,
when the load() method parses out values, it attempts to de-serialize
the value. If it fails, then it fallsback to treating the value
as a string.
>>> C = Cookie.SmartCookie()
>>> C["number"] = 7
>>> C["string"] = "seven"
>>> C["number"].value
7
>>> C["string"].value
'seven'
>>> C.output()
'Set-Cookie: number="I7\\012."\r\nSet-Cookie: string=seven'
Backwards Compatibility
-----------------------
In order to keep compatibilty with earlier versions of Cookie.py,
it is still possible to use Cookie.Cookie() to create a Cookie. In
fact, this simply returns a SmartCookie.
>>> C = Cookie.Cookie()
>>> print C.__class__.__name__
SmartCookie
Finis.
""" #"
# ^
# |----helps out font-lock
#
# Import our required modules
#
import string
try:
from cPickle import dumps, loads
except ImportError:
from pickle import dumps, loads
import re, warnings
__all__ = ["CookieError","BaseCookie","SimpleCookie","SerialCookie",
"SmartCookie","Cookie"]
_nulljoin = ''.join
_semispacejoin = '; '.join
_spacejoin = ' '.join
#
# Define an exception visible to External modules
#
class CookieError(Exception):
pass
# These quoting routines conform to the RFC2109 specification, which in
# turn references the character definitions from RFC2068. They provide
# a two-way quoting algorithm. Any non-text character is translated
# into a 4 character sequence: a forward-slash followed by the
# three-digit octal equivalent of the character. Any '\' or '"' is
# quoted with a preceeding '\' slash.
#
# These are taken from RFC2068 and RFC2109.
# _LegalChars is the list of chars which don't require "'s
# _Translator hash-table for fast quoting
#
_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~[]_"
_Translator = {
'\000' : '\\000', '\001' : '\\001', '\002' : '\\002',
'\003' : '\\003', '\004' : '\\004', '\005' : '\\005',
'\006' : '\\006', '\007' : '\\007', '\010' : '\\010',
'\011' : '\\011', '\012' : '\\012', '\013' : '\\013',
'\014' : '\\014', '\015' : '\\015', '\016' : '\\016',
'\017' : '\\017', '\020' : '\\020', '\021' : '\\021',
'\022' : '\\022', '\023' : '\\023', '\024' : '\\024',
'\025' : '\\025', '\026' : '\\026', '\027' : '\\027',
'\030' : '\\030', '\031' : '\\031', '\032' : '\\032',
'\033' : '\\033', '\034' : '\\034', '\035' : '\\035',
'\036' : '\\036', '\037' : '\\037',
# Because of the way browsers really handle cookies (as opposed
# to what the RFC says) we also encode , and ;
',' : '\\054', ';' : '\\073',
'"' : '\\"', '\\' : '\\\\',
'\177' : '\\177', '\200' : '\\200', '\201' : '\\201',
'\202' : '\\202', '\203' : '\\203', '\204' : '\\204',
'\205' : '\\205', '\206' : '\\206', '\207' : '\\207',
'\210' : '\\210', '\211' : '\\211', '\212' : '\\212',
'\213' : '\\213', '\214' : '\\214', '\215' : '\\215',
'\216' : '\\216', '\217' : '\\217', '\220' : '\\220',
'\221' : '\\221', '\222' : '\\222', '\223' : '\\223',
'\224' : '\\224', '\225' : '\\225', '\226' : '\\226',
'\227' : '\\227', '\230' : '\\230', '\231' : '\\231',
'\232' : '\\232', '\233' : '\\233', '\234' : '\\234',
'\235' : '\\235', '\236' : '\\236', '\237' : '\\237',
'\240' : '\\240', '\241' : '\\241', '\242' : '\\242',
'\243' : '\\243', '\244' : '\\244', '\245' : '\\245',
'\246' : '\\246', '\247' : '\\247', '\250' : '\\250',
'\251' : '\\251', '\252' : '\\252', '\253' : '\\253',
'\254' : '\\254', '\255' : '\\255', '\256' : '\\256',
'\257' : '\\257', '\260' : '\\260', '\261' : '\\261',
'\262' : '\\262', '\263' : '\\263', '\264' : '\\264',
'\265' : '\\265', '\266' : '\\266', '\267' : '\\267',
'\270' : '\\270', '\271' : '\\271', '\272' : '\\272',
'\273' : '\\273', '\274' : '\\274', '\275' : '\\275',
'\276' : '\\276', '\277' : '\\277', '\300' : '\\300',
'\301' : '\\301', '\302' : '\\302', '\303' : '\\303',
'\304' : '\\304', '\305' : '\\305', '\306' : '\\306',
'\307' : '\\307', '\310' : '\\310', '\311' : '\\311',
'\312' : '\\312', '\313' : '\\313', '\314' : '\\314',
'\315' : '\\315', '\316' : '\\316', '\317' : '\\317',
'\320' : '\\320', '\321' : '\\321', '\322' : '\\322',
'\323' : '\\323', '\324' : '\\324', '\325' : '\\325',
'\326' : '\\326', '\327' : '\\327', '\330' : '\\330',
'\331' : '\\331', '\332' : '\\332', '\333' : '\\333',
'\334' : '\\334', '\335' : '\\335', '\336' : '\\336',
'\337' : '\\337', '\340' : '\\340', '\341' : '\\341',
'\342' : '\\342', '\343' : '\\343', '\344' : '\\344',
'\345' : '\\345', '\346' : '\\346', '\347' : '\\347',
'\350' : '\\350', '\351' : '\\351', '\352' : '\\352',
'\353' : '\\353', '\354' : '\\354', '\355' : '\\355',
'\356' : '\\356', '\357' : '\\357', '\360' : '\\360',
'\361' : '\\361', '\362' : '\\362', '\363' : '\\363',
'\364' : '\\364', '\365' : '\\365', '\366' : '\\366',
'\367' : '\\367', '\370' : '\\370', '\371' : '\\371',
'\372' : '\\372', '\373' : '\\373', '\374' : '\\374',
'\375' : '\\375', '\376' : '\\376', '\377' : '\\377'
}
_idmap = ''.join(chr(x) for x in xrange(256))
def _quote(str, LegalChars=_LegalChars,
idmap=_idmap, translate=string.translate):
#
# If the string does not need to be double-quoted,
# then just return the string. Otherwise, surround
# the string in doublequotes and precede quote (with a \)
# special characters.
#
if "" == translate(str, idmap, LegalChars):
return str
else:
return '"' + _nulljoin( map(_Translator.get, str, str) ) + '"'
# end _quote
_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]")
_QuotePatt = re.compile(r"[\\].")
def _unquote(str):
# If there aren't any doublequotes,
# then there can't be any special characters. See RFC 2109.
if len(str) < 2:
return str
if str[0] != '"' or str[-1] != '"':
return str
# We have to assume that we must decode this string.
# Down to work.
# Remove the "s
str = str[1:-1]
# Check for special sequences. Examples:
# \012 --> \n
# \" --> "
#
i = 0
n = len(str)
res = []
while 0 <= i < n:
Omatch = _OctalPatt.search(str, i)
Qmatch = _QuotePatt.search(str, i)
if not Omatch and not Qmatch: # Neither matched
res.append(str[i:])
break
# else:
j = k = -1
if Omatch: j = Omatch.start(0)
if Qmatch: k = Qmatch.start(0)
if Qmatch and ( not Omatch or k < j ): # QuotePatt matched
res.append(str[i:k])
res.append(str[k+1])
i = k+2
else: # OctalPatt matched
res.append(str[i:j])
res.append( chr( int(str[j+1:j+4], 8) ) )
i = j+4
return _nulljoin(res)
# end _unquote
# The _getdate() routine is used to set the expiration time in
# the cookie's HTTP header. By default, _getdate() returns the
# current time in the appropriate "expires" format for a
# Set-Cookie header. The one optional argument is an offset from
# now, in seconds. For example, an offset of -3600 means "one hour ago".
# The offset may be a floating point number.
#
_weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
_monthname = [None,
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def _getdate(future=0, weekdayname=_weekdayname, monthname=_monthname):
from time import gmtime, time
now = time()
year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future)
return "%s, %02d-%3s-%4d %02d:%02d:%02d GMT" % \
(weekdayname[wd], day, monthname[month], year, hh, mm, ss)
#
# A class to hold ONE key,value pair.
# In a cookie, each such pair may have several attributes.
# so this class is used to keep the attributes associated
# with the appropriate key,value pair.
# This class also includes a coded_value attribute, which
# is used to hold the network representation of the
# value. This is most useful when Python objects are
# pickled for network transit.
#
class Morsel(dict):
# RFC 2109 lists these attributes as reserved:
# path comment domain
# max-age secure version
#
# For historical reasons, these attributes are also reserved:
# expires
#
# This is an extension from Microsoft:
# httponly
#
# This dictionary provides a mapping from the lowercase
# variant on the left to the appropriate traditional
# formatting on the right.
_reserved = { "expires" : "expires",
"path" : "Path",
"comment" : "Comment",
"domain" : "Domain",
"max-age" : "Max-Age",
"secure" : "secure",
"httponly" : "httponly",
"version" : "Version",
}
def __init__(self):
# Set defaults
self.key = self.value = self.coded_value = None
# Set default attributes
for K in self._reserved:
dict.__setitem__(self, K, "")
# end __init__
def __setitem__(self, K, V):
K = K.lower()
if not K in self._reserved:
raise CookieError("Invalid Attribute %s" % K)
dict.__setitem__(self, K, V)
# end __setitem__
def isReservedKey(self, K):
return K.lower() in self._reserved
# end isReservedKey
def set(self, key, val, coded_val,
LegalChars=_LegalChars,
idmap=_idmap, translate=string.translate):
# First we verify that the key isn't a reserved word
# Second we make sure it only contains legal characters
if key.lower() in self._reserved:
raise CookieError("Attempt to set a reserved key: %s" % key)
if "" != translate(key, idmap, LegalChars):
raise CookieError("Illegal key value: %s" % key)
# It's a good key, so save it.
self.key = key
self.value = val
self.coded_value = coded_val
# end set
def output(self, attrs=None, header = "Set-Cookie:"):
return "%s %s" % ( header, self.OutputString(attrs) )
__str__ = output
def __repr__(self):
return '<%s: %s=%s>' % (self.__class__.__name__,
self.key, repr(self.value) )
def js_output(self, attrs=None):
# Print javascript
return """
<script type="text/javascript">
<!-- begin hiding
document.cookie = \"%s\";
// end hiding -->
</script>
""" % ( self.OutputString(attrs).replace('"',r'\"'), )
# end js_output()
def OutputString(self, attrs=None):
# Build up our result
#
result = []
RA = result.append
# First, the key=value pair
RA("%s=%s" % (self.key, self.coded_value))
# Now add any defined attributes
if attrs is None:
attrs = self._reserved
items = self.items()
items.sort()
for K,V in items:
if V == "": continue
if K not in attrs: continue
if K == "expires" and type(V) == type(1):
RA("%s=%s" % (self._reserved[K], _getdate(V)))
elif K == "max-age" and type(V) == type(1):
RA("%s=%d" % (self._reserved[K], V))
elif K == "secure":
RA(str(self._reserved[K]))
elif K == "httponly":
RA(str(self._reserved[K]))
else:
RA("%s=%s" % (self._reserved[K], V))
# Return the result
return _semispacejoin(result)
# end OutputString
# end Morsel class
#
# Pattern for finding cookie
#
# This used to be strict parsing based on the RFC2109 and RFC2068
# specifications. I have since discovered that MSIE 3.0x doesn't
# follow the character rules outlined in those specs. As a
# result, the parsing rules here are less strict.
#
_LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]\_]"
_CookiePattern = re.compile(
r"(?x)" # This is a Verbose pattern
r"(?P<key>" # Start of group 'key'
""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy
r")" # End of group 'key'
r"\s*=\s*" # Equal Sign
r"(?P<val>" # Start of group 'val'
r'"(?:[^\\"]|\\.)*"' # Any doublequoted string
r"|" # or
r"\w{3},\s[\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
r"|" # or
""+ _LegalCharsPatt +"*" # Any word or empty string
r")" # End of group 'val'
r"\s*;?" # Probably ending in a semi-colon
)
# At long last, here is the cookie class.
# Using this class is almost just like using a dictionary.
# See this module's docstring for example usage.
#
class BaseCookie(dict):
# A container class for a set of Morsels
#
def value_decode(self, val):
"""real_value, coded_value = value_decode(STRING)
Called prior to setting a cookie's value from the network
representation. The VALUE is the value read from HTTP
header.
Override this function to modify the behavior of cookies.
"""
return val, val
# end value_encode
def value_encode(self, val):
"""real_value, coded_value = value_encode(VALUE)
Called prior to setting a cookie's value from the dictionary
representation. The VALUE is the value being assigned.
Override this function to modify the behavior of cookies.
"""
strval = str(val)
return strval, strval
# end value_encode
def __init__(self, input=None):
if input: self.load(input)
# end __init__
def __set(self, key, real_value, coded_value):
"""Private method for setting a cookie's value"""
M = self.get(key, Morsel())
M.set(key, real_value, coded_value)
dict.__setitem__(self, key, M)
# end __set
def __setitem__(self, key, value):
"""Dictionary style assignment."""
rval, cval = self.value_encode(value)
self.__set(key, rval, cval)
# end __setitem__
def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
"""Return a string suitable for HTTP."""
result = []
items = self.items()
items.sort()
for K,V in items:
result.append( V.output(attrs, header) )
return sep.join(result)
# end output
__str__ = output
def __repr__(self):
L = []
items = self.items()
items.sort()
for K,V in items:
L.append( '%s=%s' % (K,repr(V.value) ) )
return '<%s: %s>' % (self.__class__.__name__, _spacejoin(L))
def js_output(self, attrs=None):
"""Return a string suitable for JavaScript."""
result = []
items = self.items()
items.sort()
for K,V in items:
result.append( V.js_output(attrs) )
return _nulljoin(result)
# end js_output
def load(self, rawdata):
"""Load cookies from a string (presumably HTTP_COOKIE) or
from a dictionary. Loading cookies from a dictionary 'd'
is equivalent to calling:
map(Cookie.__setitem__, d.keys(), d.values())
"""
if type(rawdata) == type(""):
self.__ParseString(rawdata)
else:
# self.update() wouldn't call our custom __setitem__
for k, v in rawdata.items():
self[k] = v
return
# end load()
def __ParseString(self, str, patt=_CookiePattern):
i = 0 # Our starting point
n = len(str) # Length of string
M = None # current morsel
while 0 <= i < n:
# Start looking for a cookie
match = patt.search(str, i)
if not match: break # No more cookies
K,V = match.group("key"), match.group("val")
i = match.end(0)
# Parse the key, value in case it's metainfo
if K[0] == "$":
# We ignore attributes which pertain to the cookie
# mechanism as a whole. See RFC 2109.
# (Does anyone care?)
if M:
M[ K[1:] ] = V
elif K.lower() in Morsel._reserved:
if M:
M[ K ] = _unquote(V)
else:
rval, cval = self.value_decode(V)
self.__set(K, rval, cval)
M = self[K]
# end __ParseString
# end BaseCookie class
class SimpleCookie(BaseCookie):
"""SimpleCookie
SimpleCookie supports strings as cookie values. When setting
the value using the dictionary assignment notation, SimpleCookie
calls the builtin str() to convert the value to a string. Values
received from HTTP are kept as strings.
"""
def value_decode(self, val):
return _unquote( val ), val
def value_encode(self, val):
strval = str(val)
return strval, _quote( strval )
# end SimpleCookie
class SerialCookie(BaseCookie):
"""SerialCookie
SerialCookie supports arbitrary objects as cookie values. All
values are serialized (using cPickle) before being sent to the
client. All incoming values are assumed to be valid Pickle
representations. IF AN INCOMING VALUE IS NOT IN A VALID PICKLE
FORMAT, THEN AN EXCEPTION WILL BE RAISED.
Note: Large cookie values add overhead because they must be
retransmitted on every HTTP transaction.
Note: HTTP has a 2k limit on the size of a cookie. This class
does not check for this limit, so be careful!!!
"""
def __init__(self, input=None):
warnings.warn("SerialCookie class is insecure; do not use it",
DeprecationWarning)
BaseCookie.__init__(self, input)
# end __init__
def value_decode(self, val):
# This could raise an exception!
return loads( _unquote(val) ), val
def value_encode(self, val):
return val, _quote( dumps(val) )
# end SerialCookie
class SmartCookie(BaseCookie):
"""SmartCookie
SmartCookie supports arbitrary objects as cookie values. If the
object is a string, then it is quoted. If the object is not a
string, however, then SmartCookie will use cPickle to serialize
the object into a string representation.
Note: Large cookie values add overhead because they must be
retransmitted on every HTTP transaction.
Note: HTTP has a 2k limit on the size of a cookie. This class
does not check for this limit, so be careful!!!
"""
def __init__(self, input=None):
warnings.warn("Cookie/SmartCookie class is insecure; do not use it",
DeprecationWarning)
BaseCookie.__init__(self, input)
# end __init__
def value_decode(self, val):
strval = _unquote(val)
try:
return loads(strval), val
except:
return strval, val
def value_encode(self, val):
if type(val) == type(""):
return val, _quote(val)
else:
return val, _quote( dumps(val) )
# end SmartCookie
###########################################################
# Backwards Compatibility: Don't break any existing code!
# We provide Cookie() as an alias for SmartCookie()
Cookie = SmartCookie
#
###########################################################
def _test():
import doctest, Cookie
return doctest.testmod(Cookie)
if __name__ == "__main__":
_test()
#Local Variables:
#tab-width: 4
#end:
libs/requests/packages/oreos/structures.py
+399
View File
@@ -0,0 +1,399 @@
# -*- coding: utf-8 -*-
"""
oreos.sructures
~~~~~~~~~~~~~~~
The plastic blue packaging.
This is mostly directly stolen from mitsuhiko/werkzeug.
"""
__all__ = ('MultiDict',)
class _Missing(object):
def __repr__(self):
return 'no value'
def __reduce__(self):
return '_missing'
_missing = _Missing()
def iter_multi_items(mapping):
"""Iterates over the items of a mapping yielding keys and values
without dropping any from more complex structures.
"""
if isinstance(mapping, MultiDict):
for item in mapping.iteritems(multi=True):
yield item
elif isinstance(mapping, dict):
for key, value in mapping.iteritems():
if isinstance(value, (tuple, list)):
for value in value:
yield key, value
else:
yield key, value
else:
for item in mapping:
yield item
class TypeConversionDict(dict):
"""Works like a regular dict but the :meth:`get` method can perform
type conversions. :class:`MultiDict` and :class:`CombinedMultiDict`
are subclasses of this class and provide the same feature.
.. versionadded:: 0.5
"""
def get(self, key, default=None, type=None):
"""Return the default value if the requested data doesn't exist.
If `type` is provided and is a callable it should convert the value,
return it or raise a :exc:`ValueError` if that is not possible. In
this case the function will return the default as if the value was not
found:
>>> d = TypeConversionDict(foo='42', bar='blub')
>>> d.get('foo', type=int)
42
>>> d.get('bar', -1, type=int)
-1
:param key: The key to be looked up.
:param default: The default value to be returned if the key can't
be looked up. If not further specified `None` is
returned.
:param type: A callable that is used to cast the value in the
:class:`MultiDict`. If a :exc:`ValueError` is raised
by this callable the default value is returned.
"""
try:
rv = self[key]
if type is not None:
rv = type(rv)
except (KeyError, ValueError):
rv = default
return rv
class MultiDict(TypeConversionDict):
"""A :class:`MultiDict` is a dictionary subclass customized to deal with
multiple values for the same key which is for example used by the parsing
functions in the wrappers. This is necessary because some HTML form
elements pass multiple values for the same key.
:class:`MultiDict` implements all standard dictionary methods.
Internally, it saves all values for a key as a list, but the standard dict
access methods will only return the first value for a key. If you want to
gain access to the other values, too, you have to use the `list` methods as
explained below.
Basic Usage:
>>> d = MultiDict([('a', 'b'), ('a', 'c')])
>>> d
MultiDict([('a', 'b'), ('a', 'c')])
>>> d['a']
'b'
>>> d.getlist('a')
['b', 'c']
>>> 'a' in d
True
It behaves like a normal dict thus all dict functions will only return the
first value when multiple values for one key are found.
From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a
subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will
render a page for a ``400 BAD REQUEST`` if caught in a catch-all for HTTP
exceptions.
A :class:`MultiDict` can be constructed from an iterable of
``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2
onwards some keyword parameters.
:param mapping: the initial value for the :class:`MultiDict`. Either a
regular dict, an iterable of ``(key, value)`` tuples
or `None`.
"""
def __init__(self, mapping=None):
if isinstance(mapping, MultiDict):
dict.__init__(self, ((k, l[:]) for k, l in mapping.iterlists()))
elif isinstance(mapping, dict):
tmp = {}
for key, value in mapping.iteritems():
if isinstance(value, (tuple, list)):
value = list(value)
else:
value = [value]
tmp[key] = value
dict.__init__(self, tmp)
else:
tmp = {}
for key, value in mapping or ():
tmp.setdefault(key, []).append(value)
dict.__init__(self, tmp)
def __getstate__(self):
return dict(self.lists())
def __setstate__(self, value):
dict.clear(self)
dict.update(self, value)
def __iter__(self):
return self.iterkeys()
def __getitem__(self, key):
"""Return the first data value for this key;
raises KeyError if not found.
:param key: The key to be looked up.
:raise KeyError: if the key does not exist.
"""
if key in self:
return dict.__getitem__(self, key)[0]
raise KeyError(key)
def __setitem__(self, key, value):
"""Like :meth:`add` but removes an existing key first.
:param key: the key for the value.
:param value: the value to set.
"""
dict.__setitem__(self, key, [value])
def add(self, key, value):
"""Adds a new value for the key.
.. versionadded:: 0.6
:param key: the key for the value.
:param value: the value to add.
"""
dict.setdefault(self, key, []).append(value)
def getlist(self, key, type=None):
"""Return the list of items for a given key. If that key is not in the
`MultiDict`, the return value will be an empty list. Just as `get`
`getlist` accepts a `type` parameter. All items will be converted
with the callable defined there.
:param key: The key to be looked up.
:param type: A callable that is used to cast the value in the
:class:`MultiDict`. If a :exc:`ValueError` is raised
by this callable the value will be removed from the list.
:return: a :class:`list` of all the values for the key.
"""
try:
rv = dict.__getitem__(self, key)
except KeyError:
return []
if type is None:
return list(rv)
result = []
for item in rv:
try:
result.append(type(item))
except ValueError:
pass
return result
def setlist(self, key, new_list):
"""Remove the old values for a key and add new ones. Note that the list
you pass the values in will be shallow-copied before it is inserted in
the dictionary.
>>> d = MultiDict()
>>> d.setlist('foo', ['1', '2'])
>>> d['foo']
'1'
>>> d.getlist('foo')
['1', '2']
:param key: The key for which the values are set.
:param new_list: An iterable with the new values for the key. Old values
are removed first.
"""
dict.__setitem__(self, key, list(new_list))
def setdefault(self, key, default=None):
"""Returns the value for the key if it is in the dict, otherwise it
returns `default` and sets that value for `key`.
:param key: The key to be looked up.
:param default: The default value to be returned if the key is not
in the dict. If not further specified it's `None`.
"""
if key not in self:
self[key] = default
else:
default = self[key]
return default
def setlistdefault(self, key, default_list=None):
"""Like `setdefault` but sets multiple values. The list returned
is not a copy, but the list that is actually used internally. This
means that you can put new values into the dict by appending items
to the list:
>>> d = MultiDict({"foo": 1})
>>> d.setlistdefault("foo").extend([2, 3])
>>> d.getlist("foo")
[1, 2, 3]
:param key: The key to be looked up.
:param default: An iterable of default values. It is either copied
(in case it was a list) or converted into a list
before returned.
:return: a :class:`list`
"""
if key not in self:
default_list = list(default_list or ())
dict.__setitem__(self, key, default_list)
else:
default_list = dict.__getitem__(self, key)
return default_list
def items(self, multi=False):
"""Return a list of ``(key, value)`` pairs.
:param multi: If set to `True` the list returned will have a
pair for each value of each key. Otherwise it
will only contain pairs for the first value of
each key.
:return: a :class:`list`
"""
return list(self.iteritems(multi))
def lists(self):
"""Return a list of ``(key, values)`` pairs, where values is the list of
all values associated with the key.
:return: a :class:`list`
"""
return list(self.iterlists())
def values(self):
"""Returns a list of the first value on every key's value list.
:return: a :class:`list`.
"""
return [self[key] for key in self.iterkeys()]
def listvalues(self):
"""Return a list of all values associated with a key. Zipping
:meth:`keys` and this is the same as calling :meth:`lists`:
>>> d = MultiDict({"foo": [1, 2, 3]})
>>> zip(d.keys(), d.listvalues()) == d.lists()
True
:return: a :class:`list`
"""
return list(self.iterlistvalues())
def iteritems(self, multi=False):
"""Like :meth:`items` but returns an iterator."""
for key, values in dict.iteritems(self):
if multi:
for value in values:
yield key, value
else:
yield key, values[0]
def iterlists(self):
"""Like :meth:`items` but returns an iterator."""
for key, values in dict.iteritems(self):
yield key, list(values)
def itervalues(self):
"""Like :meth:`values` but returns an iterator."""
for values in dict.itervalues(self):
yield values[0]
def iterlistvalues(self):
"""Like :meth:`listvalues` but returns an iterator."""
return dict.itervalues(self)
def copy(self):
"""Return a shallow copy of this object."""
return self.__class__(self)
def to_dict(self, flat=True):
"""Return the contents as regular dict. If `flat` is `True` the
returned dict will only have the first item present, if `flat` is
`False` all values will be returned as lists.
:param flat: If set to `False` the dict returned will have lists
with all the values in it. Otherwise it will only
contain the first value for each key.
:return: a :class:`dict`
"""
if flat:
return dict(self.iteritems())
return dict(self.lists())
def update(self, other_dict):
"""update() extends rather than replaces existing key lists."""
for key, value in iter_multi_items(other_dict):
MultiDict.add(self, key, value)
def pop(self, key, default=_missing):
"""Pop the first item for a list on the dict. Afterwards the
key is removed from the dict, so additional values are discarded:
>>> d = MultiDict({"foo": [1, 2, 3]})
>>> d.pop("foo")
1
>>> "foo" in d
False
:param key: the key to pop.
:param default: if provided the value to return if the key was
not in the dictionary.
"""
try:
return dict.pop(self, key)[0]
except KeyError, e:
if default is not _missing:
return default
raise KeyError(str(e))
def popitem(self):
"""Pop an item from the dict."""
try:
item = dict.popitem(self)
return (item[0], item[1][0])
except KeyError, e:
raise KeyError(str(e))
def poplist(self, key):
"""Pop the list for a key from the dict. If the key is not in the dict
an empty list is returned.
.. versionchanged:: 0.5
If the key does no longer exist a list is returned instead of
raising an error.
"""
return dict.pop(self, key, [])
def popitemlist(self):
"""Pop a ``(key, list)`` tuple from the dict."""
try:
return dict.popitem(self)
except KeyError, e:
raise KeyError(str(e))
def __copy__(self):
return self.copy()
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self.items(multi=True))
libs/requests/packages/urllib3/__init__.py
+48
View File
@@ -0,0 +1,48 @@
# urllib3/__init__.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
"""
urllib3 - Thread-safe connection pooling and re-using.
"""
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
__version__ = '1.0.2'
from .connectionpool import (
HTTPConnectionPool,
HTTPSConnectionPool,
connection_from_url,
get_host,
make_headers)
from .exceptions import (
HTTPError,
MaxRetryError,
SSLError,
TimeoutError)
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import HTTPResponse
from .filepost import encode_multipart_formdata
# Set default logging handler to avoid "No handler found" warnings.
import logging
try:
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
def emit(self, record):
pass
logging.getLogger(__name__).addHandler(NullHandler())
# ... Clean up.
del logging
del NullHandler
libs/requests/packages/urllib3/_collections.py
+131
View File
@@ -0,0 +1,131 @@
# urllib3/_collections.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
from collections import deque
from threading import RLock
__all__ = ['RecentlyUsedContainer']
class AccessEntry(object):
__slots__ = ('key', 'is_valid')
def __init__(self, key, is_valid=True):
self.key = key
self.is_valid = is_valid
class RecentlyUsedContainer(dict):
"""
Provides a dict-like that maintains up to ``maxsize`` keys while throwing
away the least-recently-used keys beyond ``maxsize``.
"""
# If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we
# will attempt to cleanup the invalidated entries in the access_log
# datastructure during the next 'get' operation.
CLEANUP_FACTOR = 10
def __init__(self, maxsize=10):
self._maxsize = maxsize
self._container = {}
# We use a deque to to store our keys ordered by the last access.
self.access_log = deque()
self.access_log_lock = RLock()
# We look up the access log entry by the key to invalidate it so we can
# insert a new authorative entry at the head without having to dig and
# find the old entry for removal immediately.
self.access_lookup = {}
# Trigger a heap cleanup when we get past this size
self.access_log_limit = maxsize * self.CLEANUP_FACTOR
def _invalidate_entry(self, key):
"If exists: Invalidate old entry and return it."
old_entry = self.access_lookup.get(key)
if old_entry:
old_entry.is_valid = False
return old_entry
def _push_entry(self, key):
"Push entry onto our access log, invalidate the old entry if exists."
self._invalidate_entry(key)
new_entry = AccessEntry(key)
self.access_lookup[key] = new_entry
self.access_log_lock.acquire()
self.access_log.appendleft(new_entry)
self.access_log_lock.release()
def _prune_entries(self, num):
"Pop entries from our access log until we popped ``num`` valid ones."
while num > 0:
self.access_log_lock.acquire()
p = self.access_log.pop()
self.access_log_lock.release()
if not p.is_valid:
continue # Invalidated entry, skip
dict.pop(self, p.key, None)
self.access_lookup.pop(p.key, None)
num -= 1
def _prune_invalidated_entries(self):
"Rebuild our access_log without the invalidated entries."
self.access_log_lock.acquire()
self.access_log = deque(e for e in self.access_log if e.is_valid)
self.access_log_lock.release()
def _get_ordered_access_keys(self):
"Return ordered access keys for inspection. Used for testing."
self.access_log_lock.acquire()
r = [e.key for e in self.access_log if e.is_valid]
self.access_log_lock.release()
return r
def __getitem__(self, key):
item = dict.get(self, key)
if not item:
raise KeyError(key)
# Insert new entry with new high priority, also implicitly invalidates
# the old entry.
self._push_entry(key)
if len(self.access_log) > self.access_log_limit:
# Heap is getting too big, try to clean up any tailing invalidated
# entries.
self._prune_invalidated_entries()
return item
def __setitem__(self, key, item):
# Add item to our container and access log
dict.__setitem__(self, key, item)
self._push_entry(key)
# Discard invalid and excess entries
self._prune_entries(len(self) - self._maxsize)
def __delitem__(self, key):
self._invalidate_entry(key)
self.access_lookup.pop(key, None)
dict.__delitem__(self, key)
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
libs/requests/packages/urllib3/connectionpool.py
+546
View File
@@ -0,0 +1,546 @@
# urllib3/connectionpool.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import logging
import socket
from httplib import HTTPConnection, HTTPSConnection, HTTPException
from Queue import Queue, Empty, Full
from select import select
from socket import error as SocketError, timeout as SocketTimeout
try:
import ssl
BaseSSLError = ssl.SSLError
except ImportError:
ssl = None
BaseSSLError = None
from .request import RequestMethods
from .response import HTTPResponse
from .exceptions import (
SSLError,
MaxRetryError,
TimeoutError,
HostChangedError,
EmptyPoolError,
)
log = logging.getLogger(__name__)
_Default = object()
## Connection objects (extension of httplib)
class VerifiedHTTPSConnection(HTTPSConnection):
"""
Based on httplib.HTTPSConnection but wraps the socket with
SSL certification.
"""
cert_reqs = None
ca_certs = None
def set_cert(self, key_file=None, cert_file=None,
cert_reqs='CERT_NONE', ca_certs=None):
ssl_req_scheme = {
'CERT_NONE': ssl.CERT_NONE,
'CERT_OPTIONAL': ssl.CERT_OPTIONAL,
'CERT_REQUIRED': ssl.CERT_REQUIRED
}
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE
self.ca_certs = ca_certs
def connect(self):
# Add certificate verification
sock = socket.create_connection((self.host, self.port), self.timeout)
# Wrap socket using verification with the root certs in
# trusted_root_certs
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
cert_reqs=self.cert_reqs,
ca_certs=self.ca_certs)
## Pool objects
class ConnectionPool(object):
"""
Base class for all connection pools, such as
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
"""
pass
class HTTPConnectionPool(ConnectionPool, RequestMethods):
"""
Thread-safe connection pool for one host.
:param host:
Host used for this HTTP Connection (e.g. "localhost"), passed into
:class:`httplib.HTTPConnection`.
:param port:
Port used for this HTTP Connection (None is equivalent to 80), passed
into :class:`httplib.HTTPConnection`.
:param strict:
Causes BadStatusLine to be raised if the status line can't be parsed
as a valid HTTP/1.0 or 1.1 status line, passed into
:class:`httplib.HTTPConnection`.
:param timeout:
Socket timeout for each individual connection, can be a float. None
disables timeout.
:param maxsize:
Number of connections to save that can be reused. More than 1 is useful
in multithreaded situations. If ``block`` is set to false, more
connections will be created but they will not be saved once they've
been used.
:param block:
If set to True, no more than ``maxsize`` connections will be used at
a time. When no free connections are available, the call will block
until a connection has been released. This is a useful side effect for
particular multithreaded situations where one does not want to use more
than maxsize connections per host to prevent flooding.
:param headers:
Headers to include with all requests, unless other headers are given
explicitly.
"""
scheme = 'http'
def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1,
block=False, headers=None):
self.host = host
self.port = port
self.strict = strict
self.timeout = timeout
self.pool = Queue(maxsize)
self.block = block
self.headers = headers or {}
# Fill the queue up so that doing get() on it will block properly
for _ in xrange(maxsize):
self.pool.put(None)
# These are mostly for testing and debugging purposes.
self.num_connections = 0
self.num_requests = 0
def _new_conn(self):
"""
Return a fresh :class:`httplib.HTTPConnection`.
"""
self.num_connections += 1
log.info("Starting new HTTP connection (%d): %s" %
(self.num_connections, self.host))
return HTTPConnection(host=self.host, port=self.port)
def _get_conn(self, timeout=None):
"""
Get a connection. Will return a pooled connection if one is available.
If no connections are available and :prop:`.block` is ``False``, then a
fresh connection is returned.
:param timeout:
Seconds to wait before giving up and raising
:class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
:prop:`.block` is ``True``.
"""
conn = None
try:
conn = self.pool.get(block=self.block, timeout=timeout)
# If this is a persistent connection, check if it got disconnected
if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]:
# Either data is buffered (bad), or the connection is dropped.
log.info("Resetting dropped connection: %s" % self.host)
conn.close()
except Empty:
if self.block:
raise EmptyPoolError("Pool reached maximum size and no more "
"connections are allowed.")
pass # Oh well, we'll create a new connection then
return conn or self._new_conn()
def _put_conn(self, conn):
"""
Put a connection back into the pool.
:param conn:
Connection object for the current host and port as returned by
:meth:`._new_conn` or :meth:`._get_conn`.
If the pool is already full, the connection is discarded because we
exceeded maxsize. If connections are discarded frequently, then maxsize
should be increased.
"""
try:
self.pool.put(conn, block=False)
except Full:
# This should never happen if self.block == True
log.warning("HttpConnectionPool is full, discarding connection: %s"
% self.host)
def _make_request(self, conn, method, url, timeout=_Default,
**httplib_request_kw):
"""
Perform a request on a given httplib connection object taken from our
pool.
"""
self.num_requests += 1
if timeout is _Default:
timeout = self.timeout
conn.timeout = timeout # This only does anything in Py26+
conn.request(method, url, **httplib_request_kw)
conn.sock.settimeout(timeout)
httplib_response = conn.getresponse()
log.debug("\"%s %s %s\" %s %s" %
(method, url,
conn._http_vsn_str, # pylint: disable-msg=W0212
httplib_response.status, httplib_response.length))
return httplib_response
def is_same_host(self, url):
"""
Check if the given ``url`` is a member of the same host as this
conncetion pool.
"""
# TODO: Add optional support for socket.gethostbyname checking.
return (url.startswith('/') or
get_host(url) == (self.scheme, self.host, self.port))
def urlopen(self, method, url, body=None, headers=None, retries=3,
redirect=True, assert_same_host=True, timeout=_Default,
pool_timeout=None, release_conn=None, **response_kw):
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method provided
by :class:`.RequestMethods`, such as :meth:`.request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param body:
Data to send in the request body (useful for creating
POST requests, see HTTPConnectionPool.post_url for
more convenience).
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Number of retries to allow before raising a MaxRetryError exception.
:param redirect:
Automatically handle redirects (status codes 301, 302, 303, 307),
each redirect counts as a retry.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When False, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one request.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of
``response_kw.get('preload_content', True)``.
:param \**response_kw:
Additional parameters are passed to
:meth:`urllib3.response.HTTPResponse.from_httplib`
"""
if headers is None:
headers = self.headers
if retries < 0:
raise MaxRetryError("Max retries exceeded for url: %s" % url)
if timeout is _Default:
timeout = self.timeout
if release_conn is None:
release_conn = response_kw.get('preload_content', True)
# Check host
if assert_same_host and not self.is_same_host(url):
host = "%s://%s" % (self.scheme, self.host)
if self.port:
host = "%s:%d" % (host, self.port)
raise HostChangedError("Connection pool with host '%s' tried to "
"open a foreign host: %s" % (host, url))
conn = None
try:
# Request a connection from the queue
# (Could raise SocketError: Bad file descriptor)
conn = self._get_conn(timeout=pool_timeout)
# Make the request on the httplib connection object
httplib_response = self._make_request(conn, method, url,
timeout=timeout,
body=body, headers=headers)
# If we're going to release the connection in ``finally:``, then
# the request doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = not release_conn and conn
# Import httplib's response into our own wrapper object
response = HTTPResponse.from_httplib(httplib_response,
pool=self,
connection=response_conn,
**response_kw)
# else:
# The connection will be put back into the pool when
# ``response.release_conn()`` is called (implicitly by
# ``response.read()``)
except (Empty), e:
# Timed out by queue
raise TimeoutError("Request timed out. (pool_timeout=%s)" %
pool_timeout)
except (SocketTimeout), e:
# Timed out by socket
raise TimeoutError("Request timed out. (timeout=%s)" %
timeout)
except (BaseSSLError), e:
# SSL certificate error
raise SSLError(e)
except (HTTPException, SocketError), e:
# Connection broken, discard. It will be replaced next _get_conn().
conn = None
finally:
if conn and release_conn:
# Put the connection back to be reused
self._put_conn(conn)
if not conn:
log.warn("Retrying (%d attempts remain) after connection "
"broken by '%r': %s" % (retries, e, url))
return self.urlopen(method, url, body, headers, retries - 1,
redirect, assert_same_host) # Try again
# Handle redirection
if (redirect and
response.status in [301, 302, 303, 307] and
'location' in response.headers): # Redirect, retry
log.info("Redirecting %s -> %s" %
(url, response.headers.get('location')))
return self.urlopen(method, response.headers.get('location'), body,
headers, retries - 1, redirect,
assert_same_host)
return response
class HTTPSConnectionPool(HTTPConnectionPool):
"""
Same as :class:`.HTTPConnectionPool`, but HTTPS.
When Python is compiled with the :mod:`ssl` module, then
:class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
instead of :class:httplib.HTTPSConnection`.
The ``key_file``, ``cert_file``, ``cert_reqs``, and ``ca_certs`` parameters
are only used if :mod:`ssl` is available and are fed into
:meth:`ssl.wrap_socket` to upgrade the connection socket into an SSL socket.
"""
scheme = 'https'
def __init__(self, host, port=None,
strict=False, timeout=None, maxsize=1,
block=False, headers=None,
key_file=None, cert_file=None,
cert_reqs='CERT_NONE', ca_certs=None):
super(HTTPSConnectionPool, self).__init__(host, port,
strict, timeout, maxsize,
block, headers)
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = cert_reqs
self.ca_certs = ca_certs
def _new_conn(self):
"""
Return a fresh :class:`httplib.HTTPSConnection`.
"""
self.num_connections += 1
log.info("Starting new HTTPS connection (%d): %s"
% (self.num_connections, self.host))
if not ssl:
return HTTPSConnection(host=self.host, port=self.port)
connection = VerifiedHTTPSConnection(host=self.host, port=self.port)
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
return connection
## Helpers
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
basic_auth=None):
"""
Shortcuts for generating request headers.
:param keep_alive:
If ``True``, adds 'connection: keep-alive' header.
:param accept_encoding:
Can be a boolean, list, or string.
``True`` translates to 'gzip,deflate'.
List will get joined by comma.
String will be used as provided.
:param user_agent:
String representing the user-agent you want, such as
"python-urllib3/0.6"
:param basic_auth:
Colon-separated username:password string for 'authorization: basic ...'
auth header.
Example: ::
>>> make_headers(keep_alive=True, user_agent="Batman/1.0")
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
>>> make_headers(accept_encoding=True)
{'accept-encoding': 'gzip,deflate'}
"""
headers = {}
if accept_encoding:
if isinstance(accept_encoding, str):
pass
elif isinstance(accept_encoding, list):
accept_encoding = ','.join(accept_encoding)
else:
accept_encoding = 'gzip,deflate'
headers['accept-encoding'] = accept_encoding
if user_agent:
headers['user-agent'] = user_agent
if keep_alive:
headers['connection'] = 'keep-alive'
if basic_auth:
headers['authorization'] = 'Basic ' + \
basic_auth.encode('base64').strip()
return headers
def get_host(url):
"""
Given a url, return its scheme, host and port (None if it's not there).
For example: ::
>>> get_host('http://google.com/mail/')
('http', 'google.com', None)
>>> get_host('google.com:80')
('http', 'google.com', 80)
"""
# This code is actually similar to urlparse.urlsplit, but much
# simplified for our needs.
port = None
scheme = 'http'
if '://' in url:
scheme, url = url.split('://', 1)
if '/' in url:
url, _path = url.split('/', 1)
if '@' in url:
_auth, url = url.split('@', 1)
if ':' in url:
url, port = url.split(':', 1)
port = int(port)
return scheme, url, port
def connection_from_url(url, **kw):
"""
Given a url, return an :class:`.ConnectionPool` instance of its host.
This is a shortcut for not having to parse out the scheme, host, and port
of the url before creating an :class:`.ConnectionPool` instance.
:param url:
Absolute URL string that must include the scheme. Port is optional.
:param \**kw:
Passes additional parameters to the constructor of the appropriate
:class:`.ConnectionPool`. Useful for specifying things like
timeout, maxsize, headers, etc.
Example: ::
>>> conn = connection_from_url('http://google.com/')
>>> r = conn.request('GET', '/')
"""
scheme, host, port = get_host(url)
if scheme == 'https':
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
libs/requests/packages/urllib3/exceptions.py
+35
View File
@@ -0,0 +1,35 @@
# urllib3/exceptions.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
## Exceptions
class HTTPError(Exception):
"Base exception used by this module."
pass
class SSLError(Exception):
"Raised when SSL certificate fails in an HTTPS connection."
pass
class MaxRetryError(HTTPError):
"Raised when the maximum number of retries is exceeded."
pass
class TimeoutError(HTTPError):
"Raised when a socket timeout occurs."
pass
class HostChangedError(HTTPError):
"Raised when an existing pool gets a request for a foreign host."
pass
class EmptyPoolError(HTTPError):
"Raised when a pool runs out of connections and no more are allowed."
pass
libs/requests/packages/urllib3/filepost.py
+71
View File
@@ -0,0 +1,71 @@
# urllib3/filepost.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import codecs
import mimetools
import mimetypes
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO # pylint: disable-msg=W0404
writer = codecs.lookup('utf-8')[3]
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def encode_multipart_formdata(fields, boundary=None):
"""
Encode a dictionary of ``fields`` using the multipart/form-data mime format.
:param fields:
Dictionary of fields. The key is treated as the field name, and the
value as the body of the form-data. If the value is a tuple of two
elements, then the first element is treated as the filename of the
form-data section.
:param boundary:
If not specified, then a random boundary will be generated using
:func:`mimetools.choose_boundary`.
"""
body = StringIO()
if boundary is None:
boundary = mimetools.choose_boundary()
for fieldname, value in fields.iteritems():
body.write('--%s\r\n' % (boundary))
if isinstance(value, tuple):
filename, data = value
writer(body).write('Content-Disposition: form-data; name="%s"; '
'filename="%s"\r\n' % (fieldname, filename))
body.write('Content-Type: %s\r\n\r\n' %
(get_content_type(filename)))
else:
data = value
writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
% (fieldname))
body.write('Content-Type: text/plain\r\n\r\n')
if isinstance(data, int):
data = str(data) # Backwards compatibility
if isinstance(data, unicode):
writer(body).write(data)
else:
body.write(data)
body.write('\r\n')
body.write('--%s--\r\n' % (boundary))
content_type = 'multipart/form-data; boundary=%s' % boundary
return body.getvalue(), content_type
libs/requests/packages/urllib3/poolmanager.py
+128
View File
@@ -0,0 +1,128 @@
# urllib3/poolmanager.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
from ._collections import RecentlyUsedContainer
from .connectionpool import (
HTTPConnectionPool, HTTPSConnectionPool,
get_host, connection_from_url,
)
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
from .request import RequestMethods
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
pool_classes_by_scheme = {
'http': HTTPConnectionPool,
'https': HTTPSConnectionPool,
}
port_by_scheme = {
'http': 80,
'https': 443,
}
class PoolManager(RequestMethods):
"""
Allows for arbitrary requests while transparently keeping track of
necessary connection pools for you.
:param num_pools:
Number of connection pools to cache before discarding the least recently
used pool.
:param \**connection_pool_kw:
Additional parameters are used to create fresh
:class:`urllib3.connectionpool.ConnectionPool` instances.
Example: ::
>>> manager = PoolManager()
>>> r = manager.urlopen("http://google.com/")
>>> r = manager.urlopen("http://google.com/mail")
>>> r = manager.urlopen("http://yahoo.com/")
>>> len(r.pools)
2
"""
# TODO: Make sure there are no memory leaks here.
def __init__(self, num_pools=10, **connection_pool_kw):
self.connection_pool_kw = connection_pool_kw
self.pools = RecentlyUsedContainer(num_pools)
def connection_from_host(self, host, port=80, scheme='http'):
"""
Get a :class:`ConnectionPool` based on the host, port, and scheme.
Note that an appropriate ``port`` value is required here to normalize
connection pools in our container most effectively.
"""
pool_key = (scheme, host, port)
# If the scheme, host, or port doesn't match existing open connections,
# open a new ConnectionPool.
pool = self.pools.get(pool_key)
if pool:
return pool
# Make a fresh ConnectionPool of the desired type
pool_cls = pool_classes_by_scheme[scheme]
pool = pool_cls(host, port, **self.connection_pool_kw)
self.pools[pool_key] = pool
return pool
def connection_from_url(self, url):
"""
Similar to :func:`urllib3.connectionpool.connection_from_url` but
doesn't pass any additional parameters to the
:class:`urllib3.connectionpool.ConnectionPool` constructor.
Additional parameters are taken from the :class:`.PoolManager`
constructor.
"""
scheme, host, port = get_host(url)
port = port or port_by_scheme.get(scheme, 80)
return self.connection_from_host(host, port=port, scheme=scheme)
def urlopen(self, method, url, **kw):
"""
Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`.
``url`` must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
conn = self.connection_from_url(url)
return conn.urlopen(method, url, assert_same_host=False, **kw)
class ProxyManager(RequestMethods):
"""
Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method
will make requests to any url through the defined proxy.
"""
def __init__(self, proxy_pool):
self.proxy_pool = proxy_pool
def urlopen(self, method, url, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
kw['assert_same_host'] = False
return self.proxy_pool.urlopen(method, url, **kw)
def proxy_from_url(url, **pool_kw):
proxy_pool = connection_from_url(url, **pool_kw)
return ProxyManager(proxy_pool)
libs/requests/packages/urllib3/request.py
+145
View File
@@ -0,0 +1,145 @@
# urllib3/request.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
from urllib import urlencode
from .filepost import encode_multipart_formdata
__all__ = ['RequestMethods']
class RequestMethods(object):
"""
Convenience mixin for classes who implement a :meth:`urlopen` method, such
as :class:`~urllib3.connectionpool.HTTPConnectionPool` and
:class:`~urllib3.poolmanager.PoolManager`.
Provides behavior for making common types of HTTP request methods and
decides which type of request field encoding to use.
Specifically,
:meth:`.request_encode_url` is for sending requests whose fields are encoded
in the URL (such as GET, HEAD, DELETE).
:meth:`.request_encode_body` is for sending requests whose fields are
encoded in the *body* of the request using multipart or www-orm-urlencoded
(such as for POST, PUT, PATCH).
:meth:`.request` is for making any kind of request, it will look up the
appropriate encoding format and use one of the above two methods to make
the request.
"""
_encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS'])
_encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE'])
def urlopen(self, method, url, body=None, headers=None,
encode_multipart=True, multipart_boundary=None,
**kw):
raise NotImplemented("Classes extending RequestMethods must implement "
"their own ``urlopen`` method.")
def request(self, method, url, fields=None, headers=None, **urlopen_kw):
"""
Make a request using :meth:`urlopen` with the appropriate encoding of
``fields`` based on the ``method`` used.
This is a convenience method that requires the least amount of manual
effort. It can be used in most situations, while still having the option
to drop down to more specific methods when necessary, such as
:meth:`request_encode_url`, :meth:`request_encode_body`,
or even the lowest level :meth:`urlopen`.
"""
method = method.upper()
if method in self._encode_url_methods:
return self.request_encode_url(method, url, fields=fields,
headers=headers,
**urlopen_kw)
else:
return self.request_encode_body(method, url, fields=fields,
headers=headers,
**urlopen_kw)
def request_encode_url(self, method, url, fields=None, **urlopen_kw):
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
"""
if fields:
url += '?' + urlencode(fields)
return self.urlopen(method, url, **urlopen_kw)
def request_encode_body(self, method, url, fields=None, headers=None,
encode_multipart=True, multipart_boundary=None,
**urlopen_kw):
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the body. This is useful for request methods like POST, PUT, PATCH, etc.
When ``encode_multipart=True`` (default), then
:meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the
payload with the appropriate content type. Otherwise
:meth:`urllib.urlencode` is used with the
'application/x-www-form-urlencoded' content type.
Multipart encoding must be used when posting files, and it's reasonably
safe to use it in other times too. However, it may break request signing,
such as with OAuth.
Supports an optional ``fields`` parameter of key/value strings AND
key/filetuple. A filetuple is a (filename, data) tuple. For example: ::
fields = {
'foo': 'bar',
'fakefile': ('foofile.txt', 'contents of foofile'),
'realfile': ('barfile.txt', open('realfile').read()),
'nonamefile': ('contents of nonamefile field'),
}
When uploading a file, providing a filename (the first parameter of the
tuple) is optional but recommended to best mimick behavior of browsers.
Note that if ``headers`` are supplied, the 'Content-Type' header will be
overwritten because it depends on the dynamic random boundary string
which is used to compose the body of the request. The random boundary
string can be explicitly set with the ``multipart_boundary`` parameter.
"""
if encode_multipart:
body, content_type = encode_multipart_formdata(fields or {},
boundary=multipart_boundary)
else:
body, content_type = (urlencode(fields or {}),
'application/x-www-form-urlencoded')
headers = headers or {}
headers.update({'Content-Type': content_type})
return self.urlopen(method, url, body=body, headers=headers,
**urlopen_kw)
# Deprecated:
def get_url(self, url, fields=None, **urlopen_kw):
"""
.. deprecated:: 1.0
Use :meth:`request` instead.
"""
return self.request_encode_url('GET', url, fields=fields,
**urlopen_kw)
def post_url(self, url, fields=None, headers=None, **urlopen_kw):
"""
.. deprecated:: 1.0
Use :meth:`request` instead.
"""
return self.request_encode_body('POST', url, fields=fields,
headers=headers,
**urlopen_kw)
libs/requests/packages/urllib3/response.py
+181
View File
@@ -0,0 +1,181 @@
# urllib3/response.py
# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
import gzip
import logging
import zlib
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO # pylint: disable-msg=W0404
from .exceptions import HTTPError
log = logging.getLogger(__name__)
def decode_gzip(data):
gzipper = gzip.GzipFile(fileobj=StringIO(data))
return gzipper.read()
def decode_deflate(data):
try:
return zlib.decompress(data)
except zlib.error:
return zlib.decompress(data, -zlib.MAX_WBITS)
class HTTPResponse(object):
"""
HTTP Response container.
Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
loaded and decoded on-demand when the ``data`` property is accessed.
Extra parameters for behaviour not present in httplib.HTTPResponse:
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, attempts to decode specific content-encoding's based on headers
(like 'gzip' and 'deflate') will be skipped and raw data will be used
instead.
:param original_response:
When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
object, it's convenient to include the original for debug purposes. It's
otherwise unused.
"""
CONTENT_DECODERS = {
'gzip': decode_gzip,
'deflate': decode_deflate,
}
def __init__(self, body='', headers=None, status=0, version=0, reason=None,
strict=0, preload_content=True, decode_content=True,
original_response=None, pool=None, connection=None):
self.headers = headers or {}
self.status = status
self.version = version
self.reason = reason
self.strict = strict
self._decode_content = decode_content
self._body = None
self._fp = None
self._original_response = original_response
self._pool = pool
self._connection = connection
if hasattr(body, 'read'):
self._fp = body
if preload_content:
self._body = self.read(decode_content=decode_content)
def release_conn(self):
if not self._pool or not self._connection:
return
self._pool._put_conn(self._connection)
self._connection = None
@property
def data(self):
# For backwords-compat with earlier urllib3 0.4 and earlier.
if self._body:
return self._body
if self._fp:
return self.read(decode_content=self._decode_content,
cache_content=True)
def read(self, amt=None, decode_content=True, cache_content=False):
"""
Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
parameters: ``decode_content`` and ``cache_content``.
:param amt:
How much of the content to read. If specified, decoding and caching
is skipped because we can't decode partial content nor does it make
sense to cache partial content as the full response.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header. (Overridden if ``amt`` is set.)
:param cache_content:
If True, will save the returned data such that the same result is
returned despite of the state of the underlying file object. This
is useful if you want the ``.data`` property to continue working
after having ``.read()`` the file object. (Overridden if ``amt`` is
set.)
"""
content_encoding = self.headers.get('content-encoding')
decoder = self.CONTENT_DECODERS.get(content_encoding)
data = self._fp and self._fp.read(amt)
try:
if amt:
return data
if not decode_content or not decoder:
if cache_content:
self._body = data
return data
try:
data = decoder(data)
except IOError:
raise HTTPError("Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding)
if cache_content:
self._body = data
return data
finally:
if self._original_response and self._original_response.isclosed():
self.release_conn()
@staticmethod
def from_httplib(r, **response_kw):
"""
Given an :class:`httplib.HTTPResponse` instance ``r``, return a
corresponding :class:`urllib3.response.HTTPResponse` object.
Remaining parameters are passed to the HTTPResponse constructor, along
with ``original_response=r``.
"""
return HTTPResponse(body=r,
headers=dict(r.getheaders()),
status=r.status,
version=r.version,
reason=r.reason,
strict=r.strict,
original_response=r,
**response_kw)
# Backwards-compatibility methods for httplib.HTTPResponse
def getheaders(self):
return self.headers
def getheader(self, name, default=None):
return self.headers.get(name, default)
libs/requests/sessions.py
+279
View File
@@ -0,0 +1,279 @@
# -*- coding: utf-8 -*-
"""
requests.session
~~~~~~~~~~~~~~~~
This module provides a Session object to manage and persist settings across
requests (cookies, auth, proxies).
"""
from .defaults import defaults
from .models import Request
from .hooks import dispatch_hook
from .utils import header_expand
from .packages.urllib3.poolmanager import PoolManager
def merge_kwargs(local_kwarg, default_kwarg):
"""Merges kwarg dictionaries.
If a local key in the dictionary is set to None, it will be removed.
"""
if default_kwarg is None:
return local_kwarg
if isinstance(local_kwarg, basestring):
return local_kwarg
if local_kwarg is None:
return default_kwarg
# Bypass if not a dictionary (e.g. timeout)
if not hasattr(default_kwarg, 'items'):
return local_kwarg
# Update new values.
kwargs = default_kwarg.copy()
kwargs.update(local_kwarg)
# Remove keys that are set to None.
for (k,v) in local_kwarg.items():
if v is None:
del kwargs[k]
return kwargs
class Session(object):
"""A Requests session."""
__attrs__ = [
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
'params', 'config']
def __init__(self,
headers=None,
cookies=None,
auth=None,
timeout=None,
proxies=None,
hooks=None,
params=None,
config=None):
self.headers = headers or {}
self.cookies = cookies or {}
self.auth = auth
self.timeout = timeout
self.proxies = proxies or {}
self.hooks = hooks or {}
self.params = params or {}
self.config = config or {}
for (k, v) in defaults.items():
self.config.setdefault(k, v)
self.poolmanager = PoolManager(
num_pools=self.config.get('pool_connections'),
maxsize=self.config.get('pool_maxsize')
)
# Set up a CookieJar to be used by default
self.cookies = {}
# Add passed cookies in.
if cookies is not None:
self.cookies.update(cookies)
def __repr__(self):
return '<requests-client at 0x%x>' % (id(self))
def __enter__(self):
return self
def __exit__(self, *args):
pass
def request(self, method, url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=False,
proxies=None,
hooks=None,
return_response=True,
config=None,
prefetch=False):
"""Constructs and sends a :class:`Request <Request>`.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param return_response: (optional) If False, an un-sent Request object will returned.
:param config: (optional) A configuration dictionary.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
"""
method = str(method).upper()
# Default empty dicts for dict params.
cookies = {} if cookies is None else cookies
data = {} if data is None else data
files = {} if files is None else files
headers = {} if headers is None else headers
params = {} if params is None else params
hooks = {} if hooks is None else hooks
# use session's hooks as defaults
for key, cb in self.hooks.iteritems():
hooks.setdefault(key, cb)
# Expand header values.
if headers:
for k, v in headers.items() or {}:
headers[k] = header_expand(v)
args = dict(
method=method,
url=url,
data=data,
params=params,
headers=headers,
cookies=cookies,
files=files,
auth=auth,
hooks=hooks,
timeout=timeout,
allow_redirects=allow_redirects,
proxies=proxies,
config=config,
_poolmanager=self.poolmanager
)
# Merge local kwargs with session kwargs.
for attr in self.__attrs__:
session_val = getattr(self, attr, None)
local_val = args.get(attr)
args[attr] = merge_kwargs(local_val, session_val)
# Arguments manipulation hook.
args = dispatch_hook('args', args['hooks'], args)
# Create the (empty) response.
r = Request(**args)
# Give the response some context.
r.session = self
# Don't send if asked nicely.
if not return_response:
return r
# Send the HTTP Request.
r.send(prefetch=prefetch)
# Send any cookies back up the to the session.
self.cookies.update(r.response.cookies)
# Return the response.
return r.response
def get(self, url, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
return self.request('get', url, **kwargs)
def options(self, url, **kwargs):
"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
return self.request('options', url, **kwargs)
def head(self, url, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
return self.request('head', url, **kwargs)
def post(self, url, data=None, **kwargs):
"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return self.request('post', url, data=data, **kwargs)
def put(self, url, data=None, **kwargs):
"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return self.request('put', url, data=data, **kwargs)
def patch(self, url, data=None, **kwargs):
"""Sends a PATCH request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return self.request('patch', url, data=data, **kwargs)
def delete(self, url, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param **kwargs: Optional arguments that ``request`` takes.
"""
return self.request('delete', url, **kwargs)
def session(**kwargs):
"""Returns a :class:`Session` for context-management."""
return Session(**kwargs)
libs/requests/status_codes.py
+86
View File
@@ -0,0 +1,86 @@
# -*- coding: utf-8 -*-
from .structures import LookupDict
_codes = {
# Informational.
100: ('continue',),
101: ('switching_protocols',),
102: ('processing',),
103: ('checkpoint',),
122: ('uri_too_long', 'request_uri_too_long'),
200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/'),
201: ('created',),
202: ('accepted',),
203: ('non_authoritative_info', 'non_authoritative_information'),
204: ('no_content',),
205: ('reset_content', 'reset'),
206: ('partial_content', 'partial'),
207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'),
208: ('im_used',),
# Redirection.
300: ('multiple_choices',),
301: ('moved_permanently', 'moved', '\\o-'),
302: ('found',),
303: ('see_other', 'other'),
304: ('not_modified',),
305: ('use_proxy',),
306: ('switch_proxy',),
307: ('temporary_redirect', 'temporary_moved', 'temporary'),
308: ('resume_incomplete', 'resume'),
# Client Error.
400: ('bad_request', 'bad'),
401: ('unauthorized',),
402: ('payment_required', 'payment'),
403: ('forbidden',),
404: ('not_found', '-o-'),
405: ('method_not_allowed', 'not_allowed'),
406: ('not_acceptable',),
407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'),
408: ('request_timeout', 'timeout'),
409: ('conflict',),
410: ('gone',),
411: ('length_required',),
412: ('precondition_failed', 'precondition'),
413: ('request_entity_too_large',),
414: ('request_uri_too_large',),
415: ('unsupported_media_type', 'unsupported_media', 'media_type'),
416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'),
417: ('expectation_failed',),
418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'),
422: ('unprocessable_entity', 'unprocessable'),
423: ('locked',),
424: ('failed_dependency', 'dependency'),
425: ('unordered_collection', 'unordered'),
426: ('upgrade_required', 'upgrade'),
428: ('precondition_required', 'precondition'),
429: ('too_many_requests', 'too_many'),
431: ('header_fields_too_large', 'fields_too_large'),
444: ('no_response', 'none'),
449: ('retry_with', 'retry'),
450: ('blocked_by_windows_parental_controls', 'parental_controls'),
499: ('client_closed_request',),
# Server Error.
500: ('internal_server_error', 'server_error', '/o\\'),
501: ('not_implemented',),
502: ('bad_gateway',),
503: ('service_unavailable', 'unavailable'),
504: ('gateway_timeout',),
505: ('http_version_not_supported', 'http_version'),
506: ('variant_also_negotiates',),
507: ('insufficient_storage',),
509: ('bandwidth_limit_exceeded', 'bandwidth'),
510: ('not_extended',),
}
codes = LookupDict(name='status_codes')
for (code, titles) in _codes.items():
for title in titles:
setattr(codes, title, code)
if not title.startswith('\\'):
setattr(codes, title.upper(), code)
libs/requests/structures.py
+66
View File
@@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
"""
requests.structures
~~~~~~~~~~~~~~~~~~~
Data structures that power Requests.
"""
class CaseInsensitiveDict(dict):
"""Case-insensitive Dictionary
For example, ``headers['content-encoding']`` will return the
value of a ``'Content-Encoding'`` response header."""
@property
def lower_keys(self):
if not hasattr(self, '_lower_keys') or not self._lower_keys:
self._lower_keys = dict((k.lower(), k) for k in self.iterkeys())
return self._lower_keys
def _clear_lower_keys(self):
if hasattr(self, '_lower_keys'):
self._lower_keys.clear()
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
self._clear_lower_keys()
def __delitem__(self, key):
dict.__delitem__(self, key)
self._lower_keys.clear()
def __contains__(self, key):
return key.lower() in self.lower_keys
def __getitem__(self, key):
# We allow fall-through here, so values default to None
if key in self:
return dict.__getitem__(self, self.lower_keys[key.lower()])
def get(self, key, default=None):
if key in self:
return self[key]
else:
return default
class LookupDict(dict):
"""Dictionary lookup object."""
def __init__(self, name=None):
self.name = name
super(LookupDict, self).__init__()
def __repr__(self):
return '<lookup \'%s\'>' % (self.name)
def __getitem__(self, key):
# We allow fall-through here, so values default to None
return self.__dict__.get(key, None)
def get(self, key, default=None):
return self.__dict__.get(key, default)
libs/requests/utils.py
+381
View File
@@ -0,0 +1,381 @@
# -*- coding: utf-8 -*-
"""
requests.utils
~~~~~~~~~~~~~~
This module provides utility functions that are used within Requests
that are also useful for external consumption.
"""
import cgi
import codecs
import cookielib
import os
import random
import re
import zlib
import urllib
from urllib2 import parse_http_list as _parse_list_header
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
if name and name[0] != '<' and name[-1] != '>':
return name
# From mitsuhiko/werkzeug (used with permission).
def parse_list_header(value):
"""Parse lists as described by RFC 2068 Section 2.
In particular, parse comma-separated lists where the elements of
the list may include quoted-strings. A quoted-string could
contain a comma. A non-quoted string could have quotes in the
middle. Quotes are removed automatically after parsing.
It basically works like :func:`parse_set_header` just that items
may appear multiple times and case sensitivity is preserved.
The return value is a standard :class:`list`:
>>> parse_list_header('token, "quoted value"')
['token', 'quoted value']
To create a header from the :class:`list` again, use the
:func:`dump_header` function.
:param value: a string with a list header.
:return: :class:`list`
"""
result = []
for item in _parse_list_header(value):
if item[:1] == item[-1:] == '"':
item = unquote_header_value(item[1:-1])
result.append(item)
return result
# From mitsuhiko/werkzeug (used with permission).
def parse_dict_header(value):
"""Parse lists of key, value pairs as described by RFC 2068 Section 2 and
convert them into a python dict:
>>> d = parse_dict_header('foo="is a fish", bar="as well"')
>>> type(d) is dict
True
>>> sorted(d.items())
[('bar', 'as well'), ('foo', 'is a fish')]
If there is no value for a key it will be `None`:
>>> parse_dict_header('key_without_value')
{'key_without_value': None}
To create a header from the :class:`dict` again, use the
:func:`dump_header` function.
:param value: a string with a dict header.
:return: :class:`dict`
"""
result = {}
for item in _parse_list_header(value):
if '=' not in item:
result[item] = None
continue
name, value = item.split('=', 1)
if value[:1] == value[-1:] == '"':
value = unquote_header_value(value[1:-1])
result[name] = value
return result
# From mitsuhiko/werkzeug (used with permission).
def unquote_header_value(value, is_filename=False):
r"""Unquotes a header value. (Reversal of :func:`quote_header_value`).
This does not use the real unquoting but what browsers are actually
using for quoting.
:param value: the header value to unquote.
"""
if value and value[0] == value[-1] == '"':
# this is not the real unquoting, but fixing this so that the
# RFC is met will result in bugs with internet explorer and
# probably some other browsers as well. IE for example is
# uploading files with "C:\foo\bar.txt" as filename
value = value[1:-1]
# if this is a filename and the starting characters look like
# a UNC path, then just return the value without quotes. Using the
# replace sequence below on a UNC path has the effect of turning
# the leading double slash into a single slash and then
# _fix_ie_filename() doesn't work correctly. See #458.
if not is_filename or value[:2] != '\\\\':
return value.replace('\\\\', '\\').replace('\\"', '"')
return value
def header_expand(headers):
"""Returns an HTTP Header value string from a dictionary.
Example expansion::
{'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}}
# Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
(('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {}))
# Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
"""
collector = []
if isinstance(headers, dict):
headers = headers.items()
elif isinstance(headers, basestring):
return headers
for i, (value, params) in enumerate(headers):
_params = []
for (p_k, p_v) in params.items():
_params.append('%s=%s' % (p_k, p_v))
collector.append(value)
collector.append('; ')
if len(params):
collector.append('; '.join(_params))
if not len(headers) == i+1:
collector.append(', ')
# Remove trailing separators.
if collector[-1] in (', ', '; '):
del collector[-1]
return ''.join(collector)
def randombytes(n):
"""Return n random bytes."""
# Use /dev/urandom if it is available. Fall back to random module
# if not. It might be worthwhile to extend this function to use
# other platform-specific mechanisms for getting random bytes.
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
s = f.read(n)
f.close()
return s
else:
L = [chr(random.randrange(0, 256)) for i in range(n)]
return "".join(L)
def dict_from_cookiejar(cj):
"""Returns a key/value dictionary from a CookieJar.
:param cj: CookieJar object to extract cookies from.
"""
cookie_dict = {}
for _, cookies in cj._cookies.items():
for _, cookies in cookies.items():
for cookie in cookies.values():
# print cookie
cookie_dict[cookie.name] = cookie.value
return cookie_dict
def cookiejar_from_dict(cookie_dict):
"""Returns a CookieJar from a key/value dictionary.
:param cookie_dict: Dict of key/values to insert into CookieJar.
"""
# return cookiejar if one was passed in
if isinstance(cookie_dict, cookielib.CookieJar):
return cookie_dict
# create cookiejar
cj = cookielib.CookieJar()
cj = add_dict_to_cookiejar(cj, cookie_dict)
return cj
def add_dict_to_cookiejar(cj, cookie_dict):
"""Returns a CookieJar from a key/value dictionary.
:param cj: CookieJar to insert cookies into.
:param cookie_dict: Dict of key/values to insert into CookieJar.
"""
for k, v in cookie_dict.items():
cookie = cookielib.Cookie(
version=0,
name=k,
value=v,
port=None,
port_specified=False,
domain='',
domain_specified=False,
domain_initial_dot=False,
path='/',
path_specified=True,
secure=False,
expires=None,
discard=True,
comment=None,
comment_url=None,
rest={'HttpOnly': None},
rfc2109=False
)
# add cookie to cookiejar
cj.set_cookie(cookie)
return cj
def get_encodings_from_content(content):
"""Returns encodings from given content string.
:param content: bytestring to extract encodings from.
"""
charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
return charset_re.findall(content)
def get_encoding_from_headers(headers):
"""Returns encodings from given HTTP Header Dict.
:param headers: dictionary to extract encoding from.
"""
content_type = headers.get('content-type')
if not content_type:
return None
content_type, params = cgi.parse_header(content_type)
if 'charset' in params:
return params['charset'].strip("'\"")
def unicode_from_html(content):
"""Attempts to decode an HTML string into unicode.
If unsuccessful, the original content is returned.
"""
encodings = get_encodings_from_content(content)
for encoding in encodings:
try:
return unicode(content, encoding)
except (UnicodeError, TypeError):
pass
return content
def stream_decode_response_unicode(iterator, r):
"""Stream decodes a iterator."""
if r.encoding is None:
for item in iterator:
yield item
return
decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace')
for chunk in iterator:
rv = decoder.decode(chunk)
if rv:
yield rv
rv = decoder.decode('', final=True)
if rv:
yield rv
def get_unicode_from_response(r):
"""Returns the requested content back in unicode.
:param r: Response object to get unicode content from.
Tried:
1. charset from content-type
2. every encodings from ``<meta ... charset=XXX>``
3. fall back and replace all unicode characters
"""
tried_encodings = []
# Try charset from content-type
encoding = get_encoding_from_headers(r.headers)
if encoding:
try:
return unicode(r.content, encoding)
except UnicodeError:
tried_encodings.append(encoding)
# Fall back:
try:
return unicode(r.content, encoding, errors='replace')
except TypeError:
return r.content
def decode_gzip(content):
"""Return gzip-decoded string.
:param content: bytestring to gzip-decode.
"""
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
def stream_decode_gzip(iterator):
"""Stream decodes a gzip-encoded iterator"""
try:
dec = zlib.decompressobj(16 + zlib.MAX_WBITS)
for chunk in iterator:
rv = dec.decompress(chunk)
if rv:
yield rv
buf = dec.decompress('')
rv = buf + dec.flush()
if rv:
yield rv
except zlib.error:
pass
def requote_path(path):
"""Re-quote the given URL path component.
This function passes the given path through an unquote/quote cycle to
ensure that it is fully and consistently quoted.
"""
parts = path.split("/")
parts = (urllib.quote(urllib.unquote(part), safe="") for part in parts)
return "/".join(parts)
libs/suds/__init__.py
+154
View File
@@ -0,0 +1,154 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Suds is a lightweight SOAP python client that provides a
service proxy for Web Services.
"""
import os
import sys
#
# Project properties
#
__version__ = '0.4'
__build__="GA R699-20100913"
#
# Exceptions
#
class MethodNotFound(Exception):
def __init__(self, name):
Exception.__init__(self, "Method not found: '%s'" % name)
class PortNotFound(Exception):
def __init__(self, name):
Exception.__init__(self, "Port not found: '%s'" % name)
class ServiceNotFound(Exception):
def __init__(self, name):
Exception.__init__(self, "Service not found: '%s'" % name)
class TypeNotFound(Exception):
def __init__(self, name):
Exception.__init__(self, "Type not found: '%s'" % tostr(name))
class BuildError(Exception):
msg = \
"""
An error occured while building a instance of (%s). As a result
the object you requested could not be constructed. It is recommended
that you construct the type manually using a Suds object.
Please open a ticket with a description of this error.
Reason: %s
"""
def __init__(self, name, exception):
Exception.__init__(self, BuildError.msg % (name, exception))
class SoapHeadersNotPermitted(Exception):
msg = \
"""
Method (%s) was invoked with SOAP headers. The WSDL does not
define SOAP headers for this method. Retry without the soapheaders
keyword argument.
"""
def __init__(self, name):
Exception.__init__(self, self.msg % name)
class WebFault(Exception):
def __init__(self, fault, document):
if hasattr(fault, 'faultstring'):
Exception.__init__(self, "Server raised fault: '%s'" % fault.faultstring)
self.fault = fault
self.document = document
#
# Logging
#
class Repr:
def __init__(self, x):
self.x = x
def __str__(self):
return repr(self.x)
#
# Utility
#
def tostr(object, encoding=None):
""" get a unicode safe string representation of an object """
if isinstance(object, basestring):
if encoding is None:
return object
else:
return object.encode(encoding)
if isinstance(object, tuple):
s = ['(']
for item in object:
if isinstance(item, basestring):
s.append(item)
else:
s.append(tostr(item))
s.append(', ')
s.append(')')
return ''.join(s)
if isinstance(object, list):
s = ['[']
for item in object:
if isinstance(item, basestring):
s.append(item)
else:
s.append(tostr(item))
s.append(', ')
s.append(']')
return ''.join(s)
if isinstance(object, dict):
s = ['{']
for item in object.items():
if isinstance(item[0], basestring):
s.append(item[0])
else:
s.append(tostr(item[0]))
s.append(' = ')
if isinstance(item[1], basestring):
s.append(item[1])
else:
s.append(tostr(item[1]))
s.append(', ')
s.append('}')
return ''.join(s)
try:
return unicode(object)
except:
return str(object)
class null:
"""
The I{null} object.
Used to pass NULL for optional XML nodes.
"""
pass
def objid(obj):
return obj.__class__.__name__\
+':'+hex(id(obj))
import client
libs/suds/bindings/__init__.py
+20
View File
@@ -0,0 +1,20 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides modules containing classes to support Web Services (SOAP)
bindings.
"""
libs/suds/bindings/binding.py
+538
View File
@@ -0,0 +1,538 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides classes for (WS) SOAP bindings.
"""
from logging import getLogger
from suds import *
from suds.sax import Namespace
from suds.sax.parser import Parser
from suds.sax.document import Document
from suds.sax.element import Element
from suds.sudsobject import Factory, Object
from suds.mx import Content
from suds.mx.literal import Literal as MxLiteral
from suds.umx.basic import Basic as UmxBasic
from suds.umx.typed import Typed as UmxTyped
from suds.bindings.multiref import MultiRef
from suds.xsd.query import TypeQuery, ElementQuery
from suds.xsd.sxbasic import Element as SchemaElement
from suds.options import Options
from suds.plugin import PluginContainer
from copy import deepcopy
log = getLogger(__name__)
envns = ('SOAP-ENV', 'http://schemas.xmlsoap.org/soap/envelope/')
class Binding:
"""
The soap binding class used to process outgoing and imcoming
soap messages per the WSDL port binding.
@cvar replyfilter: The reply filter function.
@type replyfilter: (lambda s,r: r)
@ivar wsdl: The wsdl.
@type wsdl: L{suds.wsdl.Definitions}
@ivar schema: The collective schema contained within the wsdl.
@type schema: L{xsd.schema.Schema}
@ivar options: A dictionary options.
@type options: L{Options}
"""
replyfilter = (lambda s,r: r)
def __init__(self, wsdl):
"""
@param wsdl: A wsdl.
@type wsdl: L{wsdl.Definitions}
"""
self.wsdl = wsdl
self.multiref = MultiRef()
def schema(self):
return self.wsdl.schema
def options(self):
return self.wsdl.options
def unmarshaller(self, typed=True):
"""
Get the appropriate XML decoder.
@return: Either the (basic|typed) unmarshaller.
@rtype: L{UmxTyped}
"""
if typed:
return UmxTyped(self.schema())
else:
return UmxBasic()
def marshaller(self):
"""
Get the appropriate XML encoder.
@return: An L{MxLiteral} marshaller.
@rtype: L{MxLiteral}
"""
return MxLiteral(self.schema(), self.options().xstq)
def param_defs(self, method):
"""
Get parameter definitions.
Each I{pdef} is a tuple (I{name}, L{xsd.sxbase.SchemaObject})
@param method: A servic emethod.
@type method: I{service.Method}
@return: A collection of parameter definitions
@rtype: [I{pdef},..]
"""
raise Exception, 'not implemented'
def get_message(self, method, args, kwargs):
"""
Get the soap message for the specified method, args and soapheaders.
This is the entry point for creating the outbound soap message.
@param method: The method being invoked.
@type method: I{service.Method}
@param args: A list of args for the method invoked.
@type args: list
@param kwargs: Named (keyword) args for the method invoked.
@type kwargs: dict
@return: The soap envelope.
@rtype: L{Document}
"""
content = self.headercontent(method)
header = self.header(content)
content = self.bodycontent(method, args, kwargs)
body = self.body(content)
env = self.envelope(header, body)
if self.options().prefixes:
body.normalizePrefixes()
env.promotePrefixes()
else:
env.refitPrefixes()
return Document(env)
def get_reply(self, method, reply):
"""
Process the I{reply} for the specified I{method} by sax parsing the I{reply}
and then unmarshalling into python object(s).
@param method: The name of the invoked method.
@type method: str
@param reply: The reply XML received after invoking the specified method.
@type reply: str
@return: The unmarshalled reply. The returned value is an L{Object} for a
I{list} depending on whether the service returns a single object or a
collection.
@rtype: tuple ( L{Element}, L{Object} )
"""
reply = self.replyfilter(reply)
sax = Parser()
replyroot = sax.parse(string=reply)
plugins = PluginContainer(self.options().plugins)
plugins.message.parsed(reply=replyroot)
soapenv = replyroot.getChild('Envelope')
soapenv.promotePrefixes()
soapbody = soapenv.getChild('Body')
self.detect_fault(soapbody)
soapbody = self.multiref.process(soapbody)
nodes = self.replycontent(method, soapbody)
rtypes = self.returned_types(method)
if len(rtypes) > 1:
result = self.replycomposite(rtypes, nodes)
return (replyroot, result)
if len(rtypes) == 1:
if rtypes[0].unbounded():
result = self.replylist(rtypes[0], nodes)
return (replyroot, result)
if len(nodes):
unmarshaller = self.unmarshaller()
resolved = rtypes[0].resolve(nobuiltin=True)
result = unmarshaller.process(nodes[0], resolved)
return (replyroot, result)
return (replyroot, None)
def detect_fault(self, body):
"""
Detect I{hidden} soapenv:Fault element in the soap body.
@param body: The soap envelope body.
@type body: L{Element}
@raise WebFault: When found.
"""
fault = body.getChild('Fault', envns)
if fault is None:
return
unmarshaller = self.unmarshaller(False)
p = unmarshaller.process(fault)
if self.options().faults:
raise WebFault(p, fault)
return self
def replylist(self, rt, nodes):
"""
Construct a I{list} reply. This mehod is called when it has been detected
that the reply is a list.
@param rt: The return I{type}.
@type rt: L{suds.xsd.sxbase.SchemaObject}
@param nodes: A collection of XML nodes.
@type nodes: [L{Element},...]
@return: A list of I{unmarshalled} objects.
@rtype: [L{Object},...]
"""
result = []
resolved = rt.resolve(nobuiltin=True)
unmarshaller = self.unmarshaller()
for node in nodes:
sobject = unmarshaller.process(node, resolved)
result.append(sobject)
return result
def replycomposite(self, rtypes, nodes):
"""
Construct a I{composite} reply. This method is called when it has been
detected that the reply has multiple root nodes.
@param rtypes: A list of known return I{types}.
@type rtypes: [L{suds.xsd.sxbase.SchemaObject},...]
@param nodes: A collection of XML nodes.
@type nodes: [L{Element},...]
@return: The I{unmarshalled} composite object.
@rtype: L{Object},...
"""
dictionary = {}
for rt in rtypes:
dictionary[rt.name] = rt
unmarshaller = self.unmarshaller()
composite = Factory.object('reply')
for node in nodes:
tag = node.name
rt = dictionary.get(tag, None)
if rt is None:
if node.get('id') is None:
raise Exception('<%s/> not mapped to message part' % tag)
else:
continue
resolved = rt.resolve(nobuiltin=True)
sobject = unmarshaller.process(node, resolved)
value = getattr(composite, tag, None)
if value is None:
if rt.unbounded():
value = []
setattr(composite, tag, value)
value.append(sobject)
else:
setattr(composite, tag, sobject)
else:
if not isinstance(value, list):
value = [value,]
setattr(composite, tag, value)
value.append(sobject)
return composite
def get_fault(self, reply):
"""
Extract the fault from the specified soap reply. If I{faults} is True, an
exception is raised. Otherwise, the I{unmarshalled} fault L{Object} is
returned. This method is called when the server raises a I{web fault}.
@param reply: A soap reply message.
@type reply: str
@return: A fault object.
@rtype: tuple ( L{Element}, L{Object} )
"""
reply = self.replyfilter(reply)
sax = Parser()
faultroot = sax.parse(string=reply)
soapenv = faultroot.getChild('Envelope')
soapbody = soapenv.getChild('Body')
fault = soapbody.getChild('Fault')
unmarshaller = self.unmarshaller(False)
p = unmarshaller.process(fault)
if self.options().faults:
raise WebFault(p, faultroot)
return (faultroot, p.detail)
def mkparam(self, method, pdef, object):
"""
Builds a parameter for the specified I{method} using the parameter
definition (pdef) and the specified value (object).
@param method: A method name.
@type method: str
@param pdef: A parameter definition.
@type pdef: tuple: (I{name}, L{xsd.sxbase.SchemaObject})
@param object: The parameter value.
@type object: any
@return: The parameter fragment.
@rtype: L{Element}
"""
marshaller = self.marshaller()
content = \
Content(tag=pdef[0],
value=object,
type=pdef[1],
real=pdef[1].resolve())
return marshaller.process(content)
def mkheader(self, method, hdef, object):
"""
Builds a soapheader for the specified I{method} using the header
definition (hdef) and the specified value (object).
@param method: A method name.
@type method: str
@param hdef: A header definition.
@type hdef: tuple: (I{name}, L{xsd.sxbase.SchemaObject})
@param object: The header value.
@type object: any
@return: The parameter fragment.
@rtype: L{Element}
"""
marshaller = self.marshaller()
if isinstance(object, (list, tuple)):
tags = []
for item in object:
tags.append(self.mkheader(method, hdef, item))
return tags
content = Content(tag=hdef[0], value=object, type=hdef[1])
return marshaller.process(content)
def envelope(self, header, body):
"""
Build the B{<Envelope/>} for an soap outbound message.
@param header: The soap message B{header}.
@type header: L{Element}
@param body: The soap message B{body}.
@type body: L{Element}
@return: The soap envelope containing the body and header.
@rtype: L{Element}
"""
env = Element('Envelope', ns=envns)
env.addPrefix(Namespace.xsins[0], Namespace.xsins[1])
env.append(header)
env.append(body)
return env
def header(self, content):
"""
Build the B{<Body/>} for an soap outbound message.
@param content: The header content.
@type content: L{Element}
@return: the soap body fragment.
@rtype: L{Element}
"""
header = Element('Header', ns=envns)
header.append(content)
return header
def bodycontent(self, method, args, kwargs):
"""
Get the content for the soap I{body} node.
@param method: A service method.
@type method: I{service.Method}
@param args: method parameter values
@type args: list
@param kwargs: Named (keyword) args for the method invoked.
@type kwargs: dict
@return: The xml content for the <body/>
@rtype: [L{Element},..]
"""
raise Exception, 'not implemented'
def headercontent(self, method):
"""
Get the content for the soap I{Header} node.
@param method: A service method.
@type method: I{service.Method}
@return: The xml content for the <body/>
@rtype: [L{Element},..]
"""
n = 0
content = []
wsse = self.options().wsse
if wsse is not None:
content.append(wsse.xml())
headers = self.options().soapheaders
if not isinstance(headers, (tuple,list,dict)):
headers = (headers,)
if len(headers) == 0:
return content
pts = self.headpart_types(method)
if isinstance(headers, (tuple,list)):
for header in headers:
if isinstance(header, Element):
content.append(deepcopy(header))
continue
if len(pts) == n: break
h = self.mkheader(method, pts[n], header)
ns = pts[n][1].namespace('ns0')
h.setPrefix(ns[0], ns[1])
content.append(h)
n += 1
else:
for pt in pts:
header = headers.get(pt[0])
if header is None:
continue
h = self.mkheader(method, pt, header)
ns = pt[1].namespace('ns0')
h.setPrefix(ns[0], ns[1])
content.append(h)
return content
def replycontent(self, method, body):
"""
Get the reply body content.
@param method: A service method.
@type method: I{service.Method}
@param body: The soap body
@type body: L{Element}
@return: the body content
@rtype: [L{Element},...]
"""
raise Exception, 'not implemented'
def body(self, content):
"""
Build the B{<Body/>} for an soap outbound message.
@param content: The body content.
@type content: L{Element}
@return: the soap body fragment.
@rtype: L{Element}
"""
body = Element('Body', ns=envns)
body.append(content)
return body
def bodypart_types(self, method, input=True):
"""
Get a list of I{parameter definitions} (pdef) defined for the specified method.
Each I{pdef} is a tuple (I{name}, L{xsd.sxbase.SchemaObject})
@param method: A service method.
@type method: I{service.Method}
@param input: Defines input/output message.
@type input: boolean
@return: A list of parameter definitions
@rtype: [I{pdef},]
"""
result = []
if input:
parts = method.soap.input.body.parts
else:
parts = method.soap.output.body.parts
for p in parts:
if p.element is not None:
query = ElementQuery(p.element)
else:
query = TypeQuery(p.type)
pt = query.execute(self.schema())
if pt is None:
raise TypeNotFound(query.ref)
if p.type is not None:
pt = PartElement(p.name, pt)
if input:
if pt.name is None:
result.append((p.name, pt))
else:
result.append((pt.name, pt))
else:
result.append(pt)
return result
def headpart_types(self, method, input=True):
"""
Get a list of I{parameter definitions} (pdef) defined for the specified method.
Each I{pdef} is a tuple (I{name}, L{xsd.sxbase.SchemaObject})
@param method: A service method.
@type method: I{service.Method}
@param input: Defines input/output message.
@type input: boolean
@return: A list of parameter definitions
@rtype: [I{pdef},]
"""
result = []
if input:
headers = method.soap.input.headers
else:
headers = method.soap.output.headers
for header in headers:
part = header.part
if part.element is not None:
query = ElementQuery(part.element)
else:
query = TypeQuery(part.type)
pt = query.execute(self.schema())
if pt is None:
raise TypeNotFound(query.ref)
if part.type is not None:
pt = PartElement(part.name, pt)
if input:
if pt.name is None:
result.append((part.name, pt))
else:
result.append((pt.name, pt))
else:
result.append(pt)
return result
def returned_types(self, method):
"""
Get the L{xsd.sxbase.SchemaObject} returned by the I{method}.
@param method: A service method.
@type method: I{service.Method}
@return: The name of the type return by the method.
@rtype: [I{rtype},..]
"""
result = []
for rt in self.bodypart_types(method, input=False):
result.append(rt)
return result
class PartElement(SchemaElement):
"""
A part used to represent a message part when the part
references a schema type and thus assumes to be an element.
@ivar resolved: The part type.
@type resolved: L{suds.xsd.sxbase.SchemaObject}
"""
def __init__(self, name, resolved):
"""
@param name: The part name.
@type name: str
@param resolved: The part type.
@type resolved: L{suds.xsd.sxbase.SchemaObject}
"""
root = Element('element', ns=Namespace.xsdns)
SchemaElement.__init__(self, resolved.schema, root)
self.__resolved = resolved
self.name = name
self.form_qualified = False
def implany(self):
return self
def optional(self):
return True
def namespace(self, prefix=None):
return Namespace.default
def resolve(self, nobuiltin=False):
if nobuiltin and self.__resolved.builtin():
return self
else:
return self.__resolved
libs/suds/bindings/document.py
+160
View File
@@ -0,0 +1,160 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides classes for the (WS) SOAP I{document/literal}.
"""
from logging import getLogger
from suds import *
from suds.bindings.binding import Binding
from suds.sax.element import Element
log = getLogger(__name__)
class Document(Binding):
"""
The document/literal style. Literal is the only (@use) supported
since document/encoded is pretty much dead.
Although the soap specification supports multiple documents within the soap
<body/>, it is very uncommon. As such, suds presents an I{RPC} view of
service methods defined with a single document parameter. This is done so
that the user can pass individual parameters instead of one, single document.
To support the complete specification, service methods defined with multiple documents
(multiple message parts), must present a I{document} view for that method.
"""
def bodycontent(self, method, args, kwargs):
#
# The I{wrapped} vs I{bare} style is detected in 2 ways.
# If there is 2+ parts in the message then it is I{bare}.
# If there is only (1) part and that part resolves to a builtin then
# it is I{bare}. Otherwise, it is I{wrapped}.
#
if not len(method.soap.input.body.parts):
return ()
wrapped = method.soap.input.body.wrapped
if wrapped:
pts = self.bodypart_types(method)
root = self.document(pts[0])
else:
root = []
n = 0
for pd in self.param_defs(method):
if n < len(args):
value = args[n]
else:
value = kwargs.get(pd[0])
n += 1
p = self.mkparam(method, pd, value)
if p is None:
continue
if not wrapped:
ns = pd[1].namespace('ns0')
p.setPrefix(ns[0], ns[1])
root.append(p)
return root
def replycontent(self, method, body):
wrapped = method.soap.output.body.wrapped
if wrapped:
return body[0].children
else:
return body.children
def document(self, wrapper):
"""
Get the document root. For I{document/literal}, this is the
name of the wrapper element qualifed by the schema tns.
@param wrapper: The method name.
@type wrapper: L{xsd.sxbase.SchemaObject}
@return: A root element.
@rtype: L{Element}
"""
tag = wrapper[1].name
ns = wrapper[1].namespace('ns0')
d = Element(tag, ns=ns)
return d
def mkparam(self, method, pdef, object):
#
# Expand list parameters into individual parameters
# each with the type information. This is because in document
# arrays are simply unbounded elements.
#
if isinstance(object, (list, tuple)):
tags = []
for item in object:
tags.append(self.mkparam(method, pdef, item))
return tags
else:
return Binding.mkparam(self, method, pdef, object)
def param_defs(self, method):
#
# Get parameter definitions for document literal.
# The I{wrapped} vs I{bare} style is detected in 2 ways.
# If there is 2+ parts in the message then it is I{bare}.
# If there is only (1) part and that part resolves to a builtin then
# it is I{bare}. Otherwise, it is I{wrapped}.
#
pts = self.bodypart_types(method)
wrapped = method.soap.input.body.wrapped
if not wrapped:
return pts
result = []
# wrapped
for p in pts:
resolved = p[1].resolve()
for child, ancestry in resolved:
if child.isattr():
continue
if self.bychoice(ancestry):
log.debug(
'%s\ncontained by <choice/>, excluded as param for %s()',
child,
method.name)
continue
result.append((child.name, child))
return result
def returned_types(self, method):
result = []
wrapped = method.soap.output.body.wrapped
rts = self.bodypart_types(method, input=False)
if wrapped:
for pt in rts:
resolved = pt.resolve(nobuiltin=True)
for child, ancestry in resolved:
result.append(child)
break
else:
result += rts
return result
def bychoice(self, ancestry):
"""
The ancestry contains a <choice/>
@param ancestry: A list of ancestors.
@type ancestry: list
@return: True if contains <choice/>
@rtype: boolean
"""
for x in ancestry:
if x.choice():
return True
return False
libs/suds/bindings/multiref.py
+126
View File
@@ -0,0 +1,126 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides classes for handling soap multirefs.
"""
from logging import getLogger
from suds import *
from suds.sax.element import Element
log = getLogger(__name__)
soapenc = (None, 'http://schemas.xmlsoap.org/soap/encoding/')
class MultiRef:
"""
Resolves and replaces multirefs.
@ivar nodes: A list of non-multiref nodes.
@type nodes: list
@ivar catalog: A dictionary of multiref nodes by id.
@type catalog: dict
"""
def __init__(self):
self.nodes = []
self.catalog = {}
def process(self, body):
"""
Process the specified soap envelope body and replace I{multiref} node
references with the contents of the referenced node.
@param body: A soap envelope body node.
@type body: L{Element}
@return: The processed I{body}
@rtype: L{Element}
"""
self.nodes = []
self.catalog = {}
self.build_catalog(body)
self.update(body)
body.children = self.nodes
return body
def update(self, node):
"""
Update the specified I{node} by replacing the I{multiref} references with
the contents of the referenced nodes and remove the I{href} attribute.
@param node: A node to update.
@type node: L{Element}
@return: The updated node
@rtype: L{Element}
"""
self.replace_references(node)
for c in node.children:
self.update(c)
return node
def replace_references(self, node):
"""
Replacing the I{multiref} references with the contents of the
referenced nodes and remove the I{href} attribute. Warning: since
the I{ref} is not cloned,
@param node: A node to update.
@type node: L{Element}
"""
href = node.getAttribute('href')
if href is None:
return
id = href.getValue()
ref = self.catalog.get(id)
if ref is None:
log.error('soap multiref: %s, not-resolved', id)
return
node.append(ref.children)
node.setText(ref.getText())
for a in ref.attributes:
if a.name != 'id':
node.append(a)
node.remove(href)
def build_catalog(self, body):
"""
Create the I{catalog} of multiref nodes by id and the list of
non-multiref nodes.
@param body: A soap envelope body node.
@type body: L{Element}
"""
for child in body.children:
if self.soaproot(child):
self.nodes.append(child)
id = child.get('id')
if id is None: continue
key = '#%s' % id
self.catalog[key] = child
def soaproot(self, node):
"""
Get whether the specified I{node} is a soap encoded root.
This is determined by examining @soapenc:root='1'.
The node is considered to be a root when the attribute
is not specified.
@param node: A node to evaluate.
@type node: L{Element}
@return: True if a soap encoded root.
@rtype: bool
"""
root = node.getAttribute('root', ns=soapenc)
if root is None:
return True
else:
return ( root.value == '1' )
libs/suds/bindings/rpc.py
+98
View File
@@ -0,0 +1,98 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides classes for the (WS) SOAP I{rpc/literal} and I{rpc/encoded} bindings.
"""
from logging import getLogger
from suds import *
from suds.mx.encoded import Encoded as MxEncoded
from suds.umx.encoded import Encoded as UmxEncoded
from suds.bindings.binding import Binding, envns
from suds.sax.element import Element
log = getLogger(__name__)
encns = ('SOAP-ENC', 'http://schemas.xmlsoap.org/soap/encoding/')
class RPC(Binding):
"""
RPC/Literal binding style.
"""
def param_defs(self, method):
return self.bodypart_types(method)
def envelope(self, header, body):
env = Binding.envelope(self, header, body)
env.addPrefix(encns[0], encns[1])
env.set('%s:encodingStyle' % envns[0],
'http://schemas.xmlsoap.org/soap/encoding/')
return env
def bodycontent(self, method, args, kwargs):
n = 0
root = self.method(method)
for pd in self.param_defs(method):
if n < len(args):
value = args[n]
else:
value = kwargs.get(pd[0])
p = self.mkparam(method, pd, value)
if p is not None:
root.append(p)
n += 1
return root
def replycontent(self, method, body):
return body[0].children
def method(self, method):
"""
Get the document root. For I{rpc/(literal|encoded)}, this is the
name of the method qualifed by the schema tns.
@param method: A service method.
@type method: I{service.Method}
@return: A root element.
@rtype: L{Element}
"""
ns = method.soap.input.body.namespace
if ns[0] is None:
ns = ('ns0', ns[1])
method = Element(method.name, ns=ns)
return method
class Encoded(RPC):
"""
RPC/Encoded (section 5) binding style.
"""
def marshaller(self):
return MxEncoded(self.schema())
def unmarshaller(self, typed=True):
"""
Get the appropriate XML decoder.
@return: Either the (basic|typed) unmarshaller.
@rtype: L{UmxTyped}
"""
if typed:
return UmxEncoded(self.schema())
else:
return RPC.unmarshaller(self, typed)
libs/suds/builder.py
+121
View File
@@ -0,0 +1,121 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{builder} module provides an wsdl/xsd defined types factory
"""
from logging import getLogger
from suds import *
from suds.sudsobject import Factory
log = getLogger(__name__)
class Builder:
""" Builder used to construct an object for types defined in the schema """
def __init__(self, resolver):
"""
@param resolver: A schema object name resolver.
@type resolver: L{resolver.Resolver}
"""
self.resolver = resolver
def build(self, name):
""" build a an object for the specified typename as defined in the schema """
if isinstance(name, basestring):
type = self.resolver.find(name)
if type is None:
raise TypeNotFound(name)
else:
type = name
cls = type.name
if type.mixed():
data = Factory.property(cls)
else:
data = Factory.object(cls)
resolved = type.resolve()
md = data.__metadata__
md.sxtype = resolved
md.ordering = self.ordering(resolved)
history = []
self.add_attributes(data, resolved)
for child, ancestry in type.children():
if self.skip_child(child, ancestry):
continue
self.process(data, child, history[:])
return data
def process(self, data, type, history):
""" process the specified type then process its children """
if type in history:
return
if type.enum():
return
history.append(type)
resolved = type.resolve()
value = None
if type.unbounded():
value = []
else:
if len(resolved) > 0:
if resolved.mixed():
value = Factory.property(resolved.name)
md = value.__metadata__
md.sxtype = resolved
else:
value = Factory.object(resolved.name)
md = value.__metadata__
md.sxtype = resolved
md.ordering = self.ordering(resolved)
setattr(data, type.name, value)
if value is not None:
data = value
if not isinstance(data, list):
self.add_attributes(data, resolved)
for child, ancestry in resolved.children():
if self.skip_child(child, ancestry):
continue
self.process(data, child, history[:])
def add_attributes(self, data, type):
""" add required attributes """
for attr, ancestry in type.attributes():
name = '_%s' % attr.name
value = attr.get_default()
setattr(data, name, value)
def skip_child(self, child, ancestry):
""" get whether or not to skip the specified child """
if child.any(): return True
for x in ancestry:
if x.choice():
return True
return False
def ordering(self, type):
""" get the ordering """
result = []
for child, ancestry in type.resolve():
name = child.name
if child.name is None:
continue
if child.isattr():
name = '_%s' % child.name
result.append(name)
return result
libs/suds/cache.py
+337
View File
@@ -0,0 +1,337 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains basic caching classes.
"""
import os
import suds
from tempfile import gettempdir as tmp
from suds.transport import *
from suds.sax.parser import Parser
from suds.sax.element import Element
from datetime import datetime as dt
from datetime import timedelta
from cStringIO import StringIO
from logging import getLogger
try:
import cPickle as pickle
except:
import pickle
log = getLogger(__name__)
class Cache:
"""
An object object cache.
"""
def get(self, id):
"""
Get a object from the cache by ID.
@param id: The object ID.
@type id: str
@return: The object, else None
@rtype: any
"""
raise Exception('not-implemented')
def getf(self, id):
"""
Get a object from the cache by ID.
@param id: The object ID.
@type id: str
@return: The object, else None
@rtype: any
"""
raise Exception('not-implemented')
def put(self, id, object):
"""
Put a object into the cache.
@param id: The object ID.
@type id: str
@param object: The object to add.
@type object: any
"""
raise Exception('not-implemented')
def putf(self, id, fp):
"""
Write a fp into the cache.
@param id: The object ID.
@type id: str
@param fp: File pointer.
@type fp: file-like object.
"""
raise Exception('not-implemented')
def purge(self, id):
"""
Purge a object from the cache by id.
@param id: A object ID.
@type id: str
"""
raise Exception('not-implemented')
def clear(self):
"""
Clear all objects from the cache.
"""
raise Exception('not-implemented')
class NoCache(Cache):
"""
The passthru object cache.
"""
def get(self, id):
return None
def getf(self, id):
return None
def put(self, id, object):
pass
def putf(self, id, fp):
pass
class FileCache(Cache):
"""
A file-based URL cache.
@cvar fnprefix: The file name prefix.
@type fnsuffix: str
@ivar duration: The cached file duration which defines how
long the file will be cached.
@type duration: (unit, value)
@ivar location: The directory for the cached files.
@type location: str
"""
fnprefix = 'suds'
units = ('months', 'weeks', 'days', 'hours', 'minutes', 'seconds')
def __init__(self, location=None, **duration):
"""
@param location: The directory for the cached files.
@type location: str
@param duration: The cached file duration which defines how
long the file will be cached. A duration=0 means forever.
The duration may be: (months|weeks|days|hours|minutes|seconds).
@type duration: {unit:value}
"""
if location is None:
location = os.path.join(tmp(), 'suds')
self.location = location
self.duration = (None, 0)
self.setduration(**duration)
self.checkversion()
def fnsuffix(self):
"""
Get the file name suffix
@return: The suffix
@rtype: str
"""
return 'gcf'
def setduration(self, **duration):
"""
Set the caching duration which defines how long the
file will be cached.
@param duration: The cached file duration which defines how
long the file will be cached. A duration=0 means forever.
The duration may be: (months|weeks|days|hours|minutes|seconds).
@type duration: {unit:value}
"""
if len(duration) == 1:
arg = duration.items()[0]
if not arg[0] in self.units:
raise Exception('must be: %s' % str(self.units))
self.duration = arg
return self
def setlocation(self, location):
"""
Set the location (directory) for the cached files.
@param location: The directory for the cached files.
@type location: str
"""
self.location = location
def mktmp(self):
"""
Make the I{location} directory if it doesn't already exits.
"""
try:
if not os.path.isdir(self.location):
os.makedirs(self.location)
except:
log.debug(self.location, exc_info=1)
return self
def put(self, id, bfr):
try:
fn = self.__fn(id)
f = self.open(fn, 'w')
f.write(bfr)
f.close()
return bfr
except:
log.debug(id, exc_info=1)
return bfr
def putf(self, id, fp):
try:
fn = self.__fn(id)
f = self.open(fn, 'w')
f.write(fp.read())
fp.close()
f.close()
return open(fn)
except:
log.debug(id, exc_info=1)
return fp
def get(self, id):
try:
f = self.getf(id)
bfr = f.read()
f.close()
return bfr
except:
pass
def getf(self, id):
try:
fn = self.__fn(id)
self.validate(fn)
return self.open(fn)
except:
pass
def validate(self, fn):
"""
Validate that the file has not expired based on the I{duration}.
@param fn: The file name.
@type fn: str
"""
if self.duration[1] < 1:
return
created = dt.fromtimestamp(os.path.getctime(fn))
d = { self.duration[0]:self.duration[1] }
expired = created+timedelta(**d)
if expired < dt.now():
log.debug('%s expired, deleted', fn)
os.remove(fn)
def clear(self):
for fn in os.listdir(self.location):
if os.path.isdir(fn):
continue
if fn.startswith(self.fnprefix):
log.debug('deleted: %s', fn)
os.remove(os.path.join(self.location, fn))
def purge(self, id):
fn = self.__fn(id)
try:
os.remove(fn)
except:
pass
def open(self, fn, *args):
"""
Open the cache file making sure the directory is created.
"""
self.mktmp()
return open(fn, *args)
def checkversion(self):
path = os.path.join(self.location, 'version')
try:
f = self.open(path)
version = f.read()
f.close()
if version != suds.__version__:
raise Exception()
except:
self.clear()
f = self.open(path, 'w')
f.write(suds.__version__)
f.close()
def __fn(self, id):
name = id
suffix = self.fnsuffix()
fn = '%s-%s.%s' % (self.fnprefix, name, suffix)
return os.path.join(self.location, fn)
class DocumentCache(FileCache):
"""
Provides xml document caching.
"""
def fnsuffix(self):
return 'xml'
def get(self, id):
try:
fp = FileCache.getf(self, id)
if fp is None:
return None
p = Parser()
return p.parse(fp)
except:
FileCache.purge(self, id)
def put(self, id, object):
if isinstance(object, Element):
FileCache.put(self, id, str(object))
return object
class ObjectCache(FileCache):
"""
Provides pickled object caching.
@cvar protocol: The pickling protocol.
@type protocol: int
"""
protocol = 2
def fnsuffix(self):
return 'px'
def get(self, id):
try:
fp = FileCache.getf(self, id)
if fp is None:
return None
else:
return pickle.load(fp)
except:
FileCache.purge(self, id)
def put(self, id, object):
bfr = pickle.dumps(object, self.protocol)
FileCache.put(self, id, bfr)
return object
libs/suds/client.py
+785
View File
@@ -0,0 +1,785 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{2nd generation} service proxy provides access to web services.
See I{README.txt}
"""
import suds
import suds.metrics as metrics
from cookielib import CookieJar
from suds import *
from suds.reader import DefinitionsReader
from suds.transport import TransportError, Request
from suds.transport.https import HttpAuthenticated
from suds.servicedefinition import ServiceDefinition
from suds import sudsobject
from sudsobject import Factory as InstFactory
from sudsobject import Object
from suds.resolver import PathResolver
from suds.builder import Builder
from suds.wsdl import Definitions
from suds.cache import ObjectCache
from suds.sax.document import Document
from suds.sax.parser import Parser
from suds.options import Options
from suds.properties import Unskin
from urlparse import urlparse
from copy import deepcopy
from suds.plugin import PluginContainer
from logging import getLogger
log = getLogger(__name__)
class Client(object):
"""
A lightweight web services client.
I{(2nd generation)} API.
@ivar wsdl: The WSDL object.
@type wsdl:L{Definitions}
@ivar service: The service proxy used to invoke operations.
@type service: L{Service}
@ivar factory: The factory used to create objects.
@type factory: L{Factory}
@ivar sd: The service definition
@type sd: L{ServiceDefinition}
@ivar messages: The last sent/received messages.
@type messages: str[2]
"""
@classmethod
def items(cls, sobject):
"""
Extract the I{items} from a suds object much like the
items() method works on I{dict}.
@param sobject: A suds object
@type sobject: L{Object}
@return: A list of items contained in I{sobject}.
@rtype: [(key, value),...]
"""
return sudsobject.items(sobject)
@classmethod
def dict(cls, sobject):
"""
Convert a sudsobject into a dictionary.
@param sobject: A suds object
@type sobject: L{Object}
@return: A python dictionary containing the
items contained in I{sobject}.
@rtype: dict
"""
return sudsobject.asdict(sobject)
@classmethod
def metadata(cls, sobject):
"""
Extract the metadata from a suds object.
@param sobject: A suds object
@type sobject: L{Object}
@return: The object's metadata
@rtype: L{sudsobject.Metadata}
"""
return sobject.__metadata__
def __init__(self, url, **kwargs):
"""
@param url: The URL for the WSDL.
@type url: str
@param kwargs: keyword arguments.
@see: L{Options}
"""
options = Options()
options.transport = HttpAuthenticated()
self.options = options
options.cache = ObjectCache(days=1)
self.set_options(**kwargs)
reader = DefinitionsReader(options, Definitions)
self.wsdl = reader.open(url)
plugins = PluginContainer(options.plugins)
plugins.init.initialized(wsdl=self.wsdl)
self.factory = Factory(self.wsdl)
self.service = ServiceSelector(self, self.wsdl.services)
self.sd = []
for s in self.wsdl.services:
sd = ServiceDefinition(self.wsdl, s)
self.sd.append(sd)
self.messages = dict(tx=None, rx=None)
def set_options(self, **kwargs):
"""
Set options.
@param kwargs: keyword arguments.
@see: L{Options}
"""
p = Unskin(self.options)
p.update(kwargs)
def add_prefix(self, prefix, uri):
"""
Add I{static} mapping of an XML namespace prefix to a namespace.
This is useful for cases when a wsdl and referenced schemas make heavy
use of namespaces and those namespaces are subject to changed.
@param prefix: An XML namespace prefix.
@type prefix: str
@param uri: An XML namespace URI.
@type uri: str
@raise Exception: when prefix is already mapped.
"""
root = self.wsdl.root
mapped = root.resolvePrefix(prefix, None)
if mapped is None:
root.addPrefix(prefix, uri)
return
if mapped[1] != uri:
raise Exception('"%s" already mapped as "%s"' % (prefix, mapped))
def last_sent(self):
"""
Get last sent I{soap} message.
@return: The last sent I{soap} message.
@rtype: L{Document}
"""
return self.messages.get('tx')
def last_received(self):
"""
Get last received I{soap} message.
@return: The last received I{soap} message.
@rtype: L{Document}
"""
return self.messages.get('rx')
def clone(self):
"""
Get a shallow clone of this object.
The clone only shares the WSDL. All other attributes are
unique to the cloned object including options.
@return: A shallow clone.
@rtype: L{Client}
"""
class Uninitialized(Client):
def __init__(self):
pass
clone = Uninitialized()
clone.options = Options()
cp = Unskin(clone.options)
mp = Unskin(self.options)
cp.update(deepcopy(mp))
clone.wsdl = self.wsdl
clone.factory = self.factory
clone.service = ServiceSelector(clone, self.wsdl.services)
clone.sd = self.sd
clone.messages = dict(tx=None, rx=None)
return clone
def __str__(self):
return unicode(self)
def __unicode__(self):
s = ['\n']
build = suds.__build__.split()
s.append('Suds ( https://fedorahosted.org/suds/ )')
s.append(' version: %s' % suds.__version__)
s.append(' %s build: %s' % (build[0], build[1]))
for sd in self.sd:
s.append('\n\n%s' % unicode(sd))
return ''.join(s)
class Factory:
"""
A factory for instantiating types defined in the wsdl
@ivar resolver: A schema type resolver.
@type resolver: L{PathResolver}
@ivar builder: A schema object builder.
@type builder: L{Builder}
"""
def __init__(self, wsdl):
"""
@param wsdl: A schema object.
@type wsdl: L{wsdl.Definitions}
"""
self.wsdl = wsdl
self.resolver = PathResolver(wsdl)
self.builder = Builder(self.resolver)
def create(self, name):
"""
create a WSDL type by name
@param name: The name of a type defined in the WSDL.
@type name: str
@return: The requested object.
@rtype: L{Object}
"""
timer = metrics.Timer()
timer.start()
type = self.resolver.find(name)
if type is None:
raise TypeNotFound(name)
if type.enum():
result = InstFactory.object(name)
for e, a in type.children():
setattr(result, e.name, e.name)
else:
try:
result = self.builder.build(type)
except Exception, e:
log.error("create '%s' failed", name, exc_info=True)
raise BuildError(name, e)
timer.stop()
metrics.log.debug('%s created: %s', name, timer)
return result
def separator(self, ps):
"""
Set the path separator.
@param ps: The new path separator.
@type ps: char
"""
self.resolver = PathResolver(self.wsdl, ps)
class ServiceSelector:
"""
The B{service} selector is used to select a web service.
In most cases, the wsdl only defines (1) service in which access
by subscript is passed through to a L{PortSelector}. This is also the
behavior when a I{default} service has been specified. In cases
where multiple services have been defined and no default has been
specified, the service is found by name (or index) and a L{PortSelector}
for the service is returned. In all cases, attribute access is
forwarded to the L{PortSelector} for either the I{first} service or the
I{default} service (when specified).
@ivar __client: A suds client.
@type __client: L{Client}
@ivar __services: A list of I{wsdl} services.
@type __services: list
"""
def __init__(self, client, services):
"""
@param client: A suds client.
@type client: L{Client}
@param services: A list of I{wsdl} services.
@type services: list
"""
self.__client = client
self.__services = services
def __getattr__(self, name):
"""
Request to access an attribute is forwarded to the
L{PortSelector} for either the I{first} service or the
I{default} service (when specified).
@param name: The name of a method.
@type name: str
@return: A L{PortSelector}.
@rtype: L{PortSelector}.
"""
default = self.__ds()
if default is None:
port = self.__find(0)
else:
port = default
return getattr(port, name)
def __getitem__(self, name):
"""
Provides selection of the I{service} by name (string) or
index (integer). In cases where only (1) service is defined
or a I{default} has been specified, the request is forwarded
to the L{PortSelector}.
@param name: The name (or index) of a service.
@type name: (int|str)
@return: A L{PortSelector} for the specified service.
@rtype: L{PortSelector}.
"""
if len(self.__services) == 1:
port = self.__find(0)
return port[name]
default = self.__ds()
if default is not None:
port = default
return port[name]
return self.__find(name)
def __find(self, name):
"""
Find a I{service} by name (string) or index (integer).
@param name: The name (or index) of a service.
@type name: (int|str)
@return: A L{PortSelector} for the found service.
@rtype: L{PortSelector}.
"""
service = None
if not len(self.__services):
raise Exception, 'No services defined'
if isinstance(name, int):
try:
service = self.__services[name]
name = service.name
except IndexError:
raise ServiceNotFound, 'at [%d]' % name
else:
for s in self.__services:
if name == s.name:
service = s
break
if service is None:
raise ServiceNotFound, name
return PortSelector(self.__client, service.ports, name)
def __ds(self):
"""
Get the I{default} service if defined in the I{options}.
@return: A L{PortSelector} for the I{default} service.
@rtype: L{PortSelector}.
"""
ds = self.__client.options.service
if ds is None:
return None
else:
return self.__find(ds)
class PortSelector:
"""
The B{port} selector is used to select a I{web service} B{port}.
In cases where multiple ports have been defined and no default has been
specified, the port is found by name (or index) and a L{MethodSelector}
for the port is returned. In all cases, attribute access is
forwarded to the L{MethodSelector} for either the I{first} port or the
I{default} port (when specified).
@ivar __client: A suds client.
@type __client: L{Client}
@ivar __ports: A list of I{service} ports.
@type __ports: list
@ivar __qn: The I{qualified} name of the port (used for logging).
@type __qn: str
"""
def __init__(self, client, ports, qn):
"""
@param client: A suds client.
@type client: L{Client}
@param ports: A list of I{service} ports.
@type ports: list
@param qn: The name of the service.
@type qn: str
"""
self.__client = client
self.__ports = ports
self.__qn = qn
def __getattr__(self, name):
"""
Request to access an attribute is forwarded to the
L{MethodSelector} for either the I{first} port or the
I{default} port (when specified).
@param name: The name of a method.
@type name: str
@return: A L{MethodSelector}.
@rtype: L{MethodSelector}.
"""
default = self.__dp()
if default is None:
m = self.__find(0)
else:
m = default
return getattr(m, name)
def __getitem__(self, name):
"""
Provides selection of the I{port} by name (string) or
index (integer). In cases where only (1) port is defined
or a I{default} has been specified, the request is forwarded
to the L{MethodSelector}.
@param name: The name (or index) of a port.
@type name: (int|str)
@return: A L{MethodSelector} for the specified port.
@rtype: L{MethodSelector}.
"""
default = self.__dp()
if default is None:
return self.__find(name)
else:
return default
def __find(self, name):
"""
Find a I{port} by name (string) or index (integer).
@param name: The name (or index) of a port.
@type name: (int|str)
@return: A L{MethodSelector} for the found port.
@rtype: L{MethodSelector}.
"""
port = None
if not len(self.__ports):
raise Exception, 'No ports defined: %s' % self.__qn
if isinstance(name, int):
qn = '%s[%d]' % (self.__qn, name)
try:
port = self.__ports[name]
except IndexError:
raise PortNotFound, qn
else:
qn = '.'.join((self.__qn, name))
for p in self.__ports:
if name == p.name:
port = p
break
if port is None:
raise PortNotFound, qn
qn = '.'.join((self.__qn, port.name))
return MethodSelector(self.__client, port.methods, qn)
def __dp(self):
"""
Get the I{default} port if defined in the I{options}.
@return: A L{MethodSelector} for the I{default} port.
@rtype: L{MethodSelector}.
"""
dp = self.__client.options.port
if dp is None:
return None
else:
return self.__find(dp)
class MethodSelector:
"""
The B{method} selector is used to select a B{method} by name.
@ivar __client: A suds client.
@type __client: L{Client}
@ivar __methods: A dictionary of methods.
@type __methods: dict
@ivar __qn: The I{qualified} name of the method (used for logging).
@type __qn: str
"""
def __init__(self, client, methods, qn):
"""
@param client: A suds client.
@type client: L{Client}
@param methods: A dictionary of methods.
@type methods: dict
@param qn: The I{qualified} name of the port.
@type qn: str
"""
self.__client = client
self.__methods = methods
self.__qn = qn
def __getattr__(self, name):
"""
Get a method by name and return it in an I{execution wrapper}.
@param name: The name of a method.
@type name: str
@return: An I{execution wrapper} for the specified method name.
@rtype: L{Method}
"""
return self[name]
def __getitem__(self, name):
"""
Get a method by name and return it in an I{execution wrapper}.
@param name: The name of a method.
@type name: str
@return: An I{execution wrapper} for the specified method name.
@rtype: L{Method}
"""
m = self.__methods.get(name)
if m is None:
qn = '.'.join((self.__qn, name))
raise MethodNotFound, qn
return Method(self.__client, m)
class Method:
"""
The I{method} (namespace) object.
@ivar client: A client object.
@type client: L{Client}
@ivar method: A I{wsdl} method.
@type I{wsdl} Method.
"""
def __init__(self, client, method):
"""
@param client: A client object.
@type client: L{Client}
@param method: A I{raw} method.
@type I{raw} Method.
"""
self.client = client
self.method = method
def __call__(self, *args, **kwargs):
"""
Invoke the method.
"""
clientclass = self.clientclass(kwargs)
client = clientclass(self.client, self.method)
if not self.faults():
try:
return client.invoke(args, kwargs)
except WebFault, e:
return (500, e)
else:
return client.invoke(args, kwargs)
def faults(self):
""" get faults option """
return self.client.options.faults
def clientclass(self, kwargs):
""" get soap client class """
if SimClient.simulation(kwargs):
return SimClient
else:
return SoapClient
class SoapClient:
"""
A lightweight soap based web client B{**not intended for external use}
@ivar service: The target method.
@type service: L{Service}
@ivar method: A target method.
@type method: L{Method}
@ivar options: A dictonary of options.
@type options: dict
@ivar cookiejar: A cookie jar.
@type cookiejar: libcookie.CookieJar
"""
def __init__(self, client, method):
"""
@param client: A suds client.
@type client: L{Client}
@param method: A target method.
@type method: L{Method}
"""
self.client = client
self.method = method
self.options = client.options
self.cookiejar = CookieJar()
def invoke(self, args, kwargs):
"""
Send the required soap message to invoke the specified method
@param args: A list of args for the method invoked.
@type args: list
@param kwargs: Named (keyword) args for the method invoked.
@type kwargs: dict
@return: The result of the method invocation.
@rtype: I{builtin}|I{subclass of} L{Object}
"""
timer = metrics.Timer()
timer.start()
result = None
binding = self.method.binding.input
soapenv = binding.get_message(self.method, args, kwargs)
timer.stop()
metrics.log.debug(
"message for '%s' created: %s",
self.method.name,
timer)
timer.start()
result = self.send(soapenv)
timer.stop()
metrics.log.debug(
"method '%s' invoked: %s",
self.method.name,
timer)
return result
def send(self, soapenv):
"""
Send soap message.
@param soapenv: A soap envelope to send.
@type soapenv: L{Document}
@return: The reply to the sent message.
@rtype: I{builtin} or I{subclass of} L{Object}
"""
result = None
location = self.location()
binding = self.method.binding.input
transport = self.options.transport
retxml = self.options.retxml
prettyxml = self.options.prettyxml
log.debug('sending to (%s)\nmessage:\n%s', location, soapenv)
try:
self.last_sent(soapenv)
plugins = PluginContainer(self.options.plugins)
plugins.message.marshalled(envelope=soapenv.root())
if prettyxml:
soapenv = soapenv.str()
else:
soapenv = soapenv.plain()
soapenv = soapenv.encode('utf-8')
plugins.message.sending(envelope=soapenv)
request = Request(location, soapenv)
request.headers = self.headers()
reply = transport.send(request)
ctx = plugins.message.received(reply=reply.message)
reply.message = ctx.reply
if retxml:
result = reply.message
else:
result = self.succeeded(binding, reply.message)
except TransportError, e:
if e.httpcode in (202,204):
result = None
else:
log.error(self.last_sent())
result = self.failed(binding, e)
return result
def headers(self):
"""
Get http headers or the http/https request.
@return: A dictionary of header/values.
@rtype: dict
"""
action = self.method.soap.action
stock = { 'Content-Type' : 'text/xml; charset=utf-8', 'SOAPAction': action }
result = dict(stock, **self.options.headers)
log.debug('headers = %s', result)
return result
def succeeded(self, binding, reply):
"""
Request succeeded, process the reply
@param binding: The binding to be used to process the reply.
@type binding: L{bindings.binding.Binding}
@param reply: The raw reply text.
@type reply: str
@return: The method result.
@rtype: I{builtin}, L{Object}
@raise WebFault: On server.
"""
log.debug('http succeeded:\n%s', reply)
plugins = PluginContainer(self.options.plugins)
if len(reply) > 0:
reply, result = binding.get_reply(self.method, reply)
self.last_received(reply)
else:
result = None
ctx = plugins.message.unmarshalled(reply=result)
result = ctx.reply
if self.options.faults:
return result
else:
return (200, result)
def failed(self, binding, error):
"""
Request failed, process reply based on reason
@param binding: The binding to be used to process the reply.
@type binding: L{suds.bindings.binding.Binding}
@param error: The http error message
@type error: L{transport.TransportError}
"""
status, reason = (error.httpcode, tostr(error))
reply = error.fp.read()
log.debug('http failed:\n%s', reply)
if status == 500:
if len(reply) > 0:
r, p = binding.get_fault(reply)
self.last_received(r)
return (status, p)
else:
return (status, None)
if self.options.faults:
raise Exception((status, reason))
else:
return (status, None)
def location(self):
p = Unskin(self.options)
return p.get('location', self.method.location)
def last_sent(self, d=None):
key = 'tx'
messages = self.client.messages
if d is None:
return messages.get(key)
else:
messages[key] = d
def last_received(self, d=None):
key = 'rx'
messages = self.client.messages
if d is None:
return messages.get(key)
else:
messages[key] = d
class SimClient(SoapClient):
"""
Loopback client used for message/reply simulation.
"""
injkey = '__inject'
@classmethod
def simulation(cls, kwargs):
""" get whether loopback has been specified in the I{kwargs}. """
return kwargs.has_key(SimClient.injkey)
def invoke(self, args, kwargs):
"""
Send the required soap message to invoke the specified method
@param args: A list of args for the method invoked.
@type args: list
@param kwargs: Named (keyword) args for the method invoked.
@type kwargs: dict
@return: The result of the method invocation.
@rtype: I{builtin} or I{subclass of} L{Object}
"""
simulation = kwargs[self.injkey]
msg = simulation.get('msg')
reply = simulation.get('reply')
fault = simulation.get('fault')
if msg is None:
if reply is not None:
return self.__reply(reply, args, kwargs)
if fault is not None:
return self.__fault(fault)
raise Exception('(reply|fault) expected when msg=None')
sax = Parser()
msg = sax.parse(string=msg)
return self.send(msg)
def __reply(self, reply, args, kwargs):
""" simulate the reply """
binding = self.method.binding.input
msg = binding.get_message(self.method, args, kwargs)
log.debug('inject (simulated) send message:\n%s', msg)
binding = self.method.binding.output
return self.succeeded(binding, reply)
def __fault(self, reply):
""" simulate the (fault) reply """
binding = self.method.binding.output
if self.options.faults:
r, p = binding.get_fault(reply)
self.last_received(r)
return (500, p)
else:
return (500, None)
libs/suds/metrics.py
+62
View File
@@ -0,0 +1,62 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{metrics} module defines classes and other resources
designed for collecting and reporting performance metrics.
"""
import time
from logging import getLogger
from suds import *
from math import modf
log = getLogger(__name__)
class Timer:
def __init__(self):
self.started = 0
self.stopped = 0
def start(self):
self.started = time.time()
self.stopped = 0
return self
def stop(self):
if self.started > 0:
self.stopped = time.time()
return self
def duration(self):
return ( self.stopped - self.started )
def __str__(self):
if self.started == 0:
return 'not-running'
if self.started > 0 and self.stopped == 0:
return 'started: %d (running)' % self.started
duration = self.duration()
jmod = ( lambda m : (m[1], m[0]*1000) )
if duration < 1:
ms = (duration*1000)
return '%d (ms)' % ms
if duration < 60:
m = modf(duration)
return '%d.%.3d (seconds)' % jmod(m)
m = modf(duration/60)
return '%d.%.3d (minutes)' % jmod(m)
libs/suds/mx/__init__.py
+59
View File
@@ -0,0 +1,59 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides modules containing classes to support
marshalling (XML).
"""
from suds.sudsobject import Object
class Content(Object):
"""
Marshaller Content.
@ivar tag: The content tag.
@type tag: str
@ivar value: The content's value.
@type value: I{any}
"""
extensions = []
def __init__(self, tag=None, value=None, **kwargs):
"""
@param tag: The content tag.
@type tag: str
@param value: The content's value.
@type value: I{any}
"""
Object.__init__(self)
self.tag = tag
self.value = value
for k,v in kwargs.items():
setattr(self, k, v)
def __getattr__(self, name):
if name not in self.__dict__:
if name in self.extensions:
v = None
setattr(self, name, v)
else:
raise AttributeError, \
'Content has no attribute %s' % name
else:
v = self.__dict__[name]
return v
libs/suds/mx/appender.py
+316
View File
@@ -0,0 +1,316 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides appender classes for I{marshalling}.
"""
from logging import getLogger
from suds import *
from suds.mx import *
from suds.sudsobject import footprint
from suds.sudsobject import Object, Property
from suds.sax.element import Element
from suds.sax.text import Text
from copy import deepcopy
log = getLogger(__name__)
class Matcher:
"""
Appender matcher.
@ivar cls: A class object.
@type cls: I{classobj}
"""
def __init__(self, cls):
"""
@param cls: A class object.
@type cls: I{classobj}
"""
self.cls = cls
def __eq__(self, x):
if self.cls is None:
return ( x is None )
else:
return isinstance(x, self.cls)
class ContentAppender:
"""
Appender used to add content to marshalled objects.
@ivar default: The default appender.
@type default: L{Appender}
@ivar appenders: A I{table} of appenders mapped by class.
@type appenders: I{table}
"""
def __init__(self, marshaller):
"""
@param marshaller: A marshaller.
@type marshaller: L{suds.mx.core.Core}
"""
self.default = PrimativeAppender(marshaller)
self.appenders = (
(Matcher(None),
NoneAppender(marshaller)),
(Matcher(null),
NoneAppender(marshaller)),
(Matcher(Property),
PropertyAppender(marshaller)),
(Matcher(Object),
ObjectAppender(marshaller)),
(Matcher(Element),
ElementAppender(marshaller)),
(Matcher(Text),
TextAppender(marshaller)),
(Matcher(list),
ListAppender(marshaller)),
(Matcher(tuple),
ListAppender(marshaller)),
(Matcher(dict),
DictAppender(marshaller)),
)
def append(self, parent, content):
"""
Select an appender and append the content to parent.
@param parent: A parent node.
@type parent: L{Element}
@param content: The content to append.
@type content: L{Content}
"""
appender = self.default
for a in self.appenders:
if a[0] == content.value:
appender = a[1]
break
appender.append(parent, content)
class Appender:
"""
An appender used by the marshaller to append content.
@ivar marshaller: A marshaller.
@type marshaller: L{suds.mx.core.Core}
"""
def __init__(self, marshaller):
"""
@param marshaller: A marshaller.
@type marshaller: L{suds.mx.core.Core}
"""
self.marshaller = marshaller
def node(self, content):
"""
Create and return an XML node that is qualified
using the I{type}. Also, make sure all referenced namespace
prefixes are declared.
@param content: The content for which proccessing has ended.
@type content: L{Object}
@return: A new node.
@rtype: L{Element}
"""
return self.marshaller.node(content)
def setnil(self, node, content):
"""
Set the value of the I{node} to nill.
@param node: A I{nil} node.
@type node: L{Element}
@param content: The content for which proccessing has ended.
@type content: L{Object}
"""
self.marshaller.setnil(node, content)
def setdefault(self, node, content):
"""
Set the value of the I{node} to a default value.
@param node: A I{nil} node.
@type node: L{Element}
@param content: The content for which proccessing has ended.
@type content: L{Object}
@return: The default.
"""
return self.marshaller.setdefault(node, content)
def optional(self, content):
"""
Get whether the specified content is optional.
@param content: The content which to check.
@type content: L{Content}
"""
return self.marshaller.optional(content)
def suspend(self, content):
"""
Notify I{marshaller} that appending this content has suspended.
@param content: The content for which proccessing has been suspended.
@type content: L{Object}
"""
self.marshaller.suspend(content)
def resume(self, content):
"""
Notify I{marshaller} that appending this content has resumed.
@param content: The content for which proccessing has been resumed.
@type content: L{Object}
"""
self.marshaller.resume(content)
def append(self, parent, content):
"""
Append the specified L{content} to the I{parent}.
@param content: The content to append.
@type content: L{Object}
"""
self.marshaller.append(parent, content)
class PrimativeAppender(Appender):
"""
An appender for python I{primative} types.
"""
def append(self, parent, content):
if content.tag.startswith('_'):
attr = content.tag[1:]
value = tostr(content.value)
if value:
parent.set(attr, value)
else:
child = self.node(content)
child.setText(tostr(content.value))
parent.append(child)
class NoneAppender(Appender):
"""
An appender for I{None} values.
"""
def append(self, parent, content):
child = self.node(content)
default = self.setdefault(child, content)
if default is None:
self.setnil(child, content)
parent.append(child)
class PropertyAppender(Appender):
"""
A L{Property} appender.
"""
def append(self, parent, content):
p = content.value
child = self.node(content)
child.setText(p.get())
parent.append(child)
for item in p.items():
cont = Content(tag=item[0], value=item[1])
Appender.append(self, child, cont)
class ObjectAppender(Appender):
"""
An L{Object} appender.
"""
def append(self, parent, content):
object = content.value
if self.optional(content) and footprint(object) == 0:
return
child = self.node(content)
parent.append(child)
for item in object:
cont = Content(tag=item[0], value=item[1])
Appender.append(self, child, cont)
class DictAppender(Appender):
"""
An python I{dict} appender.
"""
def append(self, parent, content):
d = content.value
if self.optional(content) and len(d) == 0:
return
child = self.node(content)
parent.append(child)
for item in d.items():
cont = Content(tag=item[0], value=item[1])
Appender.append(self, child, cont)
class ElementWrapper(Element):
"""
Element wrapper.
"""
def __init__(self, content):
Element.__init__(self, content.name, content.parent)
self.__content = content
def str(self, indent=0):
return self.__content.str(indent)
class ElementAppender(Appender):
"""
An appender for I{Element} types.
"""
def append(self, parent, content):
if content.tag.startswith('_'):
raise Exception('raw XML not valid as attribute value')
child = ElementWrapper(content.value)
parent.append(child)
class ListAppender(Appender):
"""
A list/tuple appender.
"""
def append(self, parent, content):
collection = content.value
if len(collection):
self.suspend(content)
for item in collection:
cont = Content(tag=content.tag, value=item)
Appender.append(self, parent, cont)
self.resume(content)
class TextAppender(Appender):
"""
An appender for I{Text} values.
"""
def append(self, parent, content):
if content.tag.startswith('_'):
attr = content.tag[1:]
value = tostr(content.value)
if value:
parent.set(attr, value)
else:
child = self.node(content)
child.setText(content.value)
parent.append(child)
libs/suds/mx/basic.py
+48
View File
@@ -0,0 +1,48 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides basic I{marshaller} classes.
"""
from logging import getLogger
from suds import *
from suds.mx import *
from suds.mx.core import Core
log = getLogger(__name__)
class Basic(Core):
"""
A I{basic} (untyped) marshaller.
"""
def process(self, value, tag=None):
"""
Process (marshal) the tag with the specified value using the
optional type information.
@param value: The value (content) of the XML node.
@type value: (L{Object}|any)
@param tag: The (optional) tag name for the value. The default is
value.__class__.__name__
@type tag: str
@return: An xml node.
@rtype: L{Element}
"""
content = Content(tag=tag, value=value)
result = Core.process(self, content)
return result
libs/suds/mx/core.py
+158
View File
@@ -0,0 +1,158 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides I{marshaller} core classes.
"""
from logging import getLogger
from suds import *
from suds.mx import *
from suds.mx.appender import ContentAppender
from suds.sax.element import Element
from suds.sax.document import Document
from suds.sudsobject import Property
log = getLogger(__name__)
class Core:
"""
An I{abstract} marshaller. This class implement the core
functionality of the marshaller.
@ivar appender: A content appender.
@type appender: L{ContentAppender}
"""
def __init__(self):
"""
"""
self.appender = ContentAppender(self)
def process(self, content):
"""
Process (marshal) the tag with the specified value using the
optional type information.
@param content: The content to process.
@type content: L{Object}
"""
log.debug('processing:\n%s', content)
self.reset()
if content.tag is None:
content.tag = content.value.__class__.__name__
document = Document()
if isinstance(content.value, Property):
root = self.node(content)
self.append(document, content)
else:
self.append(document, content)
return document.root()
def append(self, parent, content):
"""
Append the specified L{content} to the I{parent}.
@param parent: The parent node to append to.
@type parent: L{Element}
@param content: The content to append.
@type content: L{Object}
"""
log.debug('appending parent:\n%s\ncontent:\n%s', parent, content)
if self.start(content):
self.appender.append(parent, content)
self.end(parent, content)
def reset(self):
"""
Reset the marshaller.
"""
pass
def node(self, content):
"""
Create and return an XML node.
@param content: The content for which proccessing has been suspended.
@type content: L{Object}
@return: An element.
@rtype: L{Element}
"""
return Element(content.tag)
def start(self, content):
"""
Appending this content has started.
@param content: The content for which proccessing has started.
@type content: L{Content}
@return: True to continue appending
@rtype: boolean
"""
return True
def suspend(self, content):
"""
Appending this content has suspended.
@param content: The content for which proccessing has been suspended.
@type content: L{Content}
"""
pass
def resume(self, content):
"""
Appending this content has resumed.
@param content: The content for which proccessing has been resumed.
@type content: L{Content}
"""
pass
def end(self, parent, content):
"""
Appending this content has ended.
@param parent: The parent node ending.
@type parent: L{Element}
@param content: The content for which proccessing has ended.
@type content: L{Content}
"""
pass
def setnil(self, node, content):
"""
Set the value of the I{node} to nill.
@param node: A I{nil} node.
@type node: L{Element}
@param content: The content to set nil.
@type content: L{Content}
"""
pass
def setdefault(self, node, content):
"""
Set the value of the I{node} to a default value.
@param node: A I{nil} node.
@type node: L{Element}
@param content: The content to set the default value.
@type content: L{Content}
@return: The default.
"""
pass
def optional(self, content):
"""
Get whether the specified content is optional.
@param content: The content which to check.
@type content: L{Content}
"""
return False
libs/suds/mx/encoded.py
+133
View File
@@ -0,0 +1,133 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides encoded I{marshaller} classes.
"""
from logging import getLogger
from suds import *
from suds.mx import *
from suds.mx.literal import Literal
from suds.mx.typer import Typer
from suds.sudsobject import Factory, Object
from suds.xsd.query import TypeQuery
log = getLogger(__name__)
#
# Add encoded extensions
# aty = The soap (section 5) encoded array type.
#
Content.extensions.append('aty')
class Encoded(Literal):
"""
A SOAP section (5) encoding marshaller.
This marshaller supports rpc/encoded soap styles.
"""
def start(self, content):
#
# For soap encoded arrays, the 'aty' (array type) information
# is extracted and added to the 'content'. Then, the content.value
# is replaced with an object containing an 'item=[]' attribute
# containing values that are 'typed' suds objects.
#
start = Literal.start(self, content)
if start and isinstance(content.value, (list,tuple)):
resolved = content.type.resolve()
for c in resolved:
if hasattr(c[0], 'aty'):
content.aty = (content.tag, c[0].aty)
self.cast(content)
break
return start
def end(self, parent, content):
#
# For soap encoded arrays, the soapenc:arrayType attribute is
# added with proper type and size information.
# Eg: soapenc:arrayType="xs:int[3]"
#
Literal.end(self, parent, content)
if content.aty is None:
return
tag, aty = content.aty
ns0 = ('at0', aty[1])
ns1 = ('at1', 'http://schemas.xmlsoap.org/soap/encoding/')
array = content.value.item
child = parent.getChild(tag)
child.addPrefix(ns0[0], ns0[1])
child.addPrefix(ns1[0], ns1[1])
name = '%s:arrayType' % ns1[0]
value = '%s:%s[%d]' % (ns0[0], aty[0], len(array))
child.set(name, value)
def encode(self, node, content):
if content.type.any():
Typer.auto(node, content.value)
return
if content.real.any():
Typer.auto(node, content.value)
return
ns = None
name = content.real.name
if self.xstq:
ns = content.real.namespace()
Typer.manual(node, name, ns)
def cast(self, content):
"""
Cast the I{untyped} list items found in content I{value}.
Each items contained in the list is checked for XSD type information.
Items (values) that are I{untyped}, are replaced with suds objects and
type I{metadata} is added.
@param content: The content holding the collection.
@type content: L{Content}
@return: self
@rtype: L{Encoded}
"""
aty = content.aty[1]
resolved = content.type.resolve()
array = Factory.object(resolved.name)
array.item = []
query = TypeQuery(aty)
ref = query.execute(self.schema)
if ref is None:
raise TypeNotFound(qref)
for x in content.value:
if isinstance(x, (list, tuple)):
array.item.append(x)
continue
if isinstance(x, Object):
md = x.__metadata__
md.sxtype = ref
array.item.append(x)
continue
if isinstance(x, dict):
x = Factory.object(ref.name, x)
md = x.__metadata__
md.sxtype = ref
array.item.append(x)
continue
x = Factory.property(ref.name, x)
md = x.__metadata__
md.sxtype = ref
array.item.append(x)
content.value = array
return self
libs/suds/mx/literal.py
+291
View File
@@ -0,0 +1,291 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides literal I{marshaller} classes.
"""
from logging import getLogger
from suds import *
from suds.mx import *
from suds.mx.core import Core
from suds.mx.typer import Typer
from suds.resolver import GraphResolver, Frame
from suds.sax.element import Element
from suds.sudsobject import Factory
log = getLogger(__name__)
#
# Add typed extensions
# type = The expected xsd type
# real = The 'true' XSD type
# ancestry = The 'type' ancestry
#
Content.extensions.append('type')
Content.extensions.append('real')
Content.extensions.append('ancestry')
class Typed(Core):
"""
A I{typed} marshaller.
This marshaller is semi-typed as needed to support both
I{document/literal} and I{rpc/literal} soap message styles.
@ivar schema: An xsd schema.
@type schema: L{xsd.schema.Schema}
@ivar resolver: A schema type resolver.
@type resolver: L{GraphResolver}
"""
def __init__(self, schema, xstq=True):
"""
@param schema: A schema object
@type schema: L{xsd.schema.Schema}
@param xstq: The B{x}ml B{s}chema B{t}ype B{q}ualified flag indicates
that the I{xsi:type} attribute values should be qualified by namespace.
@type xstq: bool
"""
Core.__init__(self)
self.schema = schema
self.xstq = xstq
self.resolver = GraphResolver(self.schema)
def reset(self):
self.resolver.reset()
def start(self, content):
#
# Start marshalling the 'content' by ensuring that both the
# 'content' _and_ the resolver are primed with the XSD type
# information. The 'content' value is both translated and
# sorted based on the XSD type. Only values that are objects
# have their attributes sorted.
#
log.debug('starting content:\n%s', content)
if content.type is None:
name = content.tag
if name.startswith('_'):
name = '@'+name[1:]
content.type = self.resolver.find(name, content.value)
if content.type is None:
raise TypeNotFound(content.tag)
else:
known = None
if isinstance(content.value, Object):
known = self.resolver.known(content.value)
if known is None:
log.debug('object has no type information', content.value)
known = content.type
frame = Frame(content.type, resolved=known)
self.resolver.push(frame)
frame = self.resolver.top()
content.real = frame.resolved
content.ancestry = frame.ancestry
self.translate(content)
self.sort(content)
if self.skip(content):
log.debug('skipping (optional) content:\n%s', content)
self.resolver.pop()
return False
else:
return True
def suspend(self, content):
#
# Suspend to process a list content. Primarily, this
# involves popping the 'list' content off the resolver's
# stack so the list items can be marshalled.
#
self.resolver.pop()
def resume(self, content):
#
# Resume processing a list content. To do this, we
# really need to simply push the 'list' content
# back onto the resolver stack.
#
self.resolver.push(Frame(content.type))
def end(self, parent, content):
#
# End processing the content. Make sure the content
# ending matches the top of the resolver stack since for
# list processing we play games with the resolver stack.
#
log.debug('ending content:\n%s', content)
current = self.resolver.top().type
if current == content.type:
self.resolver.pop()
else:
raise Exception, \
'content (end) mismatch: top=(%s) cont=(%s)' % \
(current, content)
def node(self, content):
#
# Create an XML node and namespace qualify as defined
# by the schema (elementFormDefault).
#
ns = content.type.namespace()
if content.type.form_qualified:
node = Element(content.tag, ns=ns)
node.addPrefix(ns[0], ns[1])
else:
node = Element(content.tag)
self.encode(node, content)
log.debug('created - node:\n%s', node)
return node
def setnil(self, node, content):
#
# Set the 'node' nil only if the XSD type
# specifies that it is permitted.
#
if content.type.nillable:
node.setnil()
def setdefault(self, node, content):
#
# Set the node to the default value specified
# by the XSD type.
#
default = content.type.default
if default is None:
pass
else:
node.setText(default)
return default
def optional(self, content):
if content.type.optional():
return True
for a in content.ancestry:
if a.optional():
return True
return False
def encode(self, node, content):
# Add (soap) encoding information only if the resolved
# type is derived by extension. Further, the xsi:type values
# is qualified by namespace only if the content (tag) and
# referenced type are in different namespaces.
if content.type.any():
return
if not content.real.extension():
return
if content.type.resolve() == content.real:
return
ns = None
name = content.real.name
if self.xstq:
ns = content.real.namespace('ns1')
Typer.manual(node, name, ns)
def skip(self, content):
"""
Get whether to skip this I{content}.
Should be skipped when the content is optional
and either the value=None or the value is an empty list.
@param content: The content to skip.
@type content: L{Object}
@return: True if content is to be skipped.
@rtype: bool
"""
if self.optional(content):
v = content.value
if v is None:
return True
if isinstance(v, (list,tuple)) and len(v) == 0:
return True
return False
def optional(self, content):
if content.type.optional():
return True
for a in content.ancestry:
if a.optional():
return True
return False
def translate(self, content):
"""
Translate using the XSD type information.
Python I{dict} is translated to a suds object. Most
importantly, primative values are translated from python
types to XML types using the XSD type.
@param content: The content to translate.
@type content: L{Object}
@return: self
@rtype: L{Typed}
"""
v = content.value
if v is None:
return
if isinstance(v, dict):
cls = content.real.name
content.value = Factory.object(cls, v)
md = content.value.__metadata__
md.sxtype = content.type
return
v = content.real.translate(v, False)
content.value = v
return self
def sort(self, content):
"""
Sort suds object attributes based on ordering defined
in the XSD type information.
@param content: The content to sort.
@type content: L{Object}
@return: self
@rtype: L{Typed}
"""
v = content.value
if isinstance(v, Object):
md = v.__metadata__
md.ordering = self.ordering(content.real)
return self
def ordering(self, type):
"""
Get the attribute ordering defined in the specified
XSD type information.
@param type: An XSD type object.
@type type: SchemaObject
@return: An ordered list of attribute names.
@rtype: list
"""
result = []
for child, ancestry in type.resolve():
name = child.name
if child.name is None:
continue
if child.isattr():
name = '_%s' % child.name
result.append(name)
return result
class Literal(Typed):
"""
A I{literal} marshaller.
This marshaller is semi-typed as needed to support both
I{document/literal} and I{rpc/literal} soap message styles.
"""
pass
libs/suds/mx/typer.py
+123
View File
@@ -0,0 +1,123 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides sx typing classes.
"""
from logging import getLogger
from suds import *
from suds.mx import *
from suds.sax import Namespace as NS
from suds.sax.text import Text
log = getLogger(__name__)
class Typer:
"""
Provides XML node typing as either automatic or manual.
@cvar types: A dict of class to xs type mapping.
@type types: dict
"""
types = {
int : ('int', NS.xsdns),
long : ('long', NS.xsdns),
float : ('float', NS.xsdns),
str : ('string', NS.xsdns),
unicode : ('string', NS.xsdns),
Text : ('string', NS.xsdns),
bool : ('boolean', NS.xsdns),
}
@classmethod
def auto(cls, node, value=None):
"""
Automatically set the node's xsi:type attribute based on either I{value}'s
class or the class of the node's text. When I{value} is an unmapped class,
the default type (xs:any) is set.
@param node: An XML node
@type node: L{sax.element.Element}
@param value: An object that is or would be the node's text.
@type value: I{any}
@return: The specified node.
@rtype: L{sax.element.Element}
"""
if value is None:
value = node.getText()
if isinstance(value, Object):
known = cls.known(value)
if known.name is None:
return node
tm = (known.name, known.namespace())
else:
tm = cls.types.get(value.__class__, cls.types.get(str))
cls.manual(node, *tm)
return node
@classmethod
def manual(cls, node, tval, ns=None):
"""
Set the node's xsi:type attribute based on either I{value}'s
class or the class of the node's text. Then adds the referenced
prefix(s) to the node's prefix mapping.
@param node: An XML node
@type node: L{sax.element.Element}
@param tval: The name of the schema type.
@type tval: str
@param ns: The XML namespace of I{tval}.
@type ns: (prefix, uri)
@return: The specified node.
@rtype: L{sax.element.Element}
"""
xta = ':'.join((NS.xsins[0], 'type'))
node.addPrefix(NS.xsins[0], NS.xsins[1])
if ns is None:
node.set(xta, tval)
else:
ns = cls.genprefix(node, ns)
qname = ':'.join((ns[0], tval))
node.set(xta, qname)
node.addPrefix(ns[0], ns[1])
return node
@classmethod
def genprefix(cls, node, ns):
"""
Generate a prefix.
@param node: An XML node on which the prefix will be used.
@type node: L{sax.element.Element}
@param ns: A namespace needing an unique prefix.
@type ns: (prefix, uri)
@return: The I{ns} with a new prefix.
"""
for n in range(1, 1024):
p = 'ns%d' % n
u = node.resolvePrefix(p, default=None)
if u is None or u == ns[1]:
return (p, ns[1])
raise Exception('auto prefix, exhausted')
@classmethod
def known(cls, object):
try:
md = object.__metadata__
known = md.sxtype
return known
except:
pass
libs/suds/options.py
+123
View File
@@ -0,0 +1,123 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Suds basic options classes.
"""
from suds.properties import *
from suds.wsse import Security
from suds.xsd.doctor import Doctor
from suds.transport import Transport
from suds.cache import Cache, NoCache
class TpLinker(AutoLinker):
"""
Transport (auto) linker used to manage linkage between
transport objects Properties and those Properties that contain them.
"""
def updated(self, properties, prev, next):
if isinstance(prev, Transport):
tp = Unskin(prev.options)
properties.unlink(tp)
if isinstance(next, Transport):
tp = Unskin(next.options)
properties.link(tp)
class Options(Skin):
"""
Options:
- B{cache} - The XML document cache. May be set (None) for no caching.
- type: L{Cache}
- default: L{NoCache}
- B{faults} - Raise faults raised by server,
else return tuple from service method invocation as (httpcode, object).
- type: I{bool}
- default: True
- B{service} - The default service name.
- type: I{str}
- default: None
- B{port} - The default service port name, not tcp port.
- type: I{str}
- default: None
- B{location} - This overrides the service port address I{URL} defined
in the WSDL.
- type: I{str}
- default: None
- B{transport} - The message transport.
- type: L{Transport}
- default: None
- B{soapheaders} - The soap headers to be included in the soap message.
- type: I{any}
- default: None
- B{wsse} - The web services I{security} provider object.
- type: L{Security}
- default: None
- B{doctor} - A schema I{doctor} object.
- type: L{Doctor}
- default: None
- B{xstq} - The B{x}ml B{s}chema B{t}ype B{q}ualified flag indicates
that the I{xsi:type} attribute values should be qualified by namespace.
- type: I{bool}
- default: True
- B{prefixes} - Elements of the soap message should be qualified (when needed)
using XML prefixes as opposed to xmlns="" syntax.
- type: I{bool}
- default: True
- B{retxml} - Flag that causes the I{raw} soap envelope to be returned instead
of the python object graph.
- type: I{bool}
- default: False
- B{prettyxml} - Flag that causes I{pretty} xml to be rendered when generating
the outbound soap envelope.
- type: I{bool}
- default: False
- B{autoblend} - Flag that ensures that the schema(s) defined within the
WSDL import each other.
- type: I{bool}
- default: False
- B{cachingpolicy} - The caching policy.
- type: I{int}
- 0 = Cache XML documents.
- 1 = Cache WSDL (pickled) object.
- default: 0
- B{plugins} - A plugin container.
- type: I{list}
"""
def __init__(self, **kwargs):
domain = __name__
definitions = [
Definition('cache', Cache, NoCache()),
Definition('faults', bool, True),
Definition('transport', Transport, None, TpLinker()),
Definition('service', (int, basestring), None),
Definition('port', (int, basestring), None),
Definition('location', basestring, None),
Definition('soapheaders', (), ()),
Definition('wsse', Security, None),
Definition('doctor', Doctor, None),
Definition('xstq', bool, True),
Definition('prefixes', bool, True),
Definition('retxml', bool, False),
Definition('prettyxml', bool, False),
Definition('autoblend', bool, False),
Definition('cachingpolicy', int, 0),
Definition('plugins', (list, tuple), []),
]
Skin.__init__(self, domain, definitions, kwargs)
libs/suds/plugin.py
+257
View File
@@ -0,0 +1,257 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The plugin module provides classes for implementation
of suds plugins.
"""
from suds import *
from logging import getLogger
log = getLogger(__name__)
class Context(object):
"""
Plugin context.
"""
pass
class InitContext(Context):
"""
Init Context.
@ivar wsdl: The wsdl.
@type wsdl: L{wsdl.Definitions}
"""
pass
class DocumentContext(Context):
"""
The XML document load context.
@ivar url: The URL.
@type url: str
@ivar document: Either the XML text or the B{parsed} document root.
@type document: (str|L{sax.element.Element})
"""
pass
class MessageContext(Context):
"""
The context for sending the soap envelope.
@ivar envelope: The soap envelope to be sent.
@type envelope: (str|L{sax.element.Element})
@ivar reply: The reply.
@type reply: (str|L{sax.element.Element}|object)
"""
pass
class Plugin:
"""
Plugin base.
"""
pass
class InitPlugin(Plugin):
"""
The base class for suds I{init} plugins.
"""
def initialized(self, context):
"""
Suds client initialization.
Called after wsdl the has been loaded. Provides the plugin
with the opportunity to inspect/modify the WSDL.
@param context: The init context.
@type context: L{InitContext}
"""
pass
class DocumentPlugin(Plugin):
"""
The base class for suds I{document} plugins.
"""
def loaded(self, context):
"""
Suds has loaded a WSDL/XSD document. Provides the plugin
with an opportunity to inspect/modify the unparsed document.
Called after each WSDL/XSD document is loaded.
@param context: The document context.
@type context: L{DocumentContext}
"""
pass
def parsed(self, context):
"""
Suds has parsed a WSDL/XSD document. Provides the plugin
with an opportunity to inspect/modify the parsed document.
Called after each WSDL/XSD document is parsed.
@param context: The document context.
@type context: L{DocumentContext}
"""
pass
class MessagePlugin(Plugin):
"""
The base class for suds I{soap message} plugins.
"""
def marshalled(self, context):
"""
Suds will send the specified soap envelope.
Provides the plugin with the opportunity to inspect/modify
the envelope Document before it is sent.
@param context: The send context.
The I{envelope} is the envelope docuemnt.
@type context: L{MessageContext}
"""
pass
def sending(self, context):
"""
Suds will send the specified soap envelope.
Provides the plugin with the opportunity to inspect/modify
the message text it is sent.
@param context: The send context.
The I{envelope} is the envelope text.
@type context: L{MessageContext}
"""
pass
def received(self, context):
"""
Suds has received the specified reply.
Provides the plugin with the opportunity to inspect/modify
the received XML text before it is SAX parsed.
@param context: The reply context.
The I{reply} is the raw text.
@type context: L{MessageContext}
"""
pass
def parsed(self, context):
"""
Suds has sax parsed the received reply.
Provides the plugin with the opportunity to inspect/modify
the sax parsed DOM tree for the reply before it is unmarshalled.
@param context: The reply context.
The I{reply} is DOM tree.
@type context: L{MessageContext}
"""
pass
def unmarshalled(self, context):
"""
Suds has unmarshalled the received reply.
Provides the plugin with the opportunity to inspect/modify
the unmarshalled reply object before it is returned.
@param context: The reply context.
The I{reply} is unmarshalled suds object.
@type context: L{MessageContext}
"""
pass
class PluginContainer:
"""
Plugin container provides easy method invocation.
@ivar plugins: A list of plugin objects.
@type plugins: [L{Plugin},]
@cvar ctxclass: A dict of plugin method / context classes.
@type ctxclass: dict
"""
domains = {\
'init': (InitContext, InitPlugin),
'document': (DocumentContext, DocumentPlugin),
'message': (MessageContext, MessagePlugin ),
}
def __init__(self, plugins):
"""
@param plugins: A list of plugin objects.
@type plugins: [L{Plugin},]
"""
self.plugins = plugins
def __getattr__(self, name):
domain = self.domains.get(name)
if domain:
plugins = []
ctx, pclass = domain
for p in self.plugins:
if isinstance(p, pclass):
plugins.append(p)
return PluginDomain(ctx, plugins)
else:
raise Exception, 'plugin domain (%s), invalid' % name
class PluginDomain:
"""
The plugin domain.
@ivar ctx: A context.
@type ctx: L{Context}
@ivar plugins: A list of plugins (targets).
@type plugins: list
"""
def __init__(self, ctx, plugins):
self.ctx = ctx
self.plugins = plugins
def __getattr__(self, name):
return Method(name, self)
class Method:
"""
Plugin method.
@ivar name: The method name.
@type name: str
@ivar domain: The plugin domain.
@type domain: L{PluginDomain}
"""
def __init__(self, name, domain):
"""
@param name: The method name.
@type name: str
@param domain: A plugin domain.
@type domain: L{PluginDomain}
"""
self.name = name
self.domain = domain
def __call__(self, **kwargs):
ctx = self.domain.ctx()
ctx.__dict__.update(kwargs)
for plugin in self.domain.plugins:
try:
method = getattr(plugin, self.name, None)
if method and callable(method):
method(ctx)
except Exception, pe:
log.exception(pe)
return ctx
libs/suds/properties.py
+543
View File
@@ -0,0 +1,543 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Properties classes.
"""
from logging import getLogger
log = getLogger(__name__)
class AutoLinker(object):
"""
Base class, provides interface for I{automatic} link
management between a L{Properties} object and the L{Properties}
contained within I{values}.
"""
def updated(self, properties, prev, next):
"""
Notification that a values was updated and the linkage
between the I{properties} contained with I{prev} need to
be relinked to the L{Properties} contained within the
I{next} value.
"""
pass
class Link(object):
"""
Property link object.
@ivar endpoints: A tuple of the (2) endpoints of the link.
@type endpoints: tuple(2)
"""
def __init__(self, a, b):
"""
@param a: Property (A) to link.
@type a: L{Property}
@param b: Property (B) to link.
@type b: L{Property}
"""
pA = Endpoint(self, a)
pB = Endpoint(self, b)
self.endpoints = (pA, pB)
self.validate(a, b)
a.links.append(pB)
b.links.append(pA)
def validate(self, pA, pB):
"""
Validate that the two properties may be linked.
@param pA: Endpoint (A) to link.
@type pA: L{Endpoint}
@param pB: Endpoint (B) to link.
@type pB: L{Endpoint}
@return: self
@rtype: L{Link}
"""
if pA in pB.links or \
pB in pA.links:
raise Exception, 'Already linked'
dA = pA.domains()
dB = pB.domains()
for d in dA:
if d in dB:
raise Exception, 'Duplicate domain "%s" found' % d
for d in dB:
if d in dA:
raise Exception, 'Duplicate domain "%s" found' % d
kA = pA.keys()
kB = pB.keys()
for k in kA:
if k in kB:
raise Exception, 'Duplicate key %s found' % k
for k in kB:
if k in kA:
raise Exception, 'Duplicate key %s found' % k
return self
def teardown(self):
"""
Teardown the link.
Removes endpoints from properties I{links} collection.
@return: self
@rtype: L{Link}
"""
pA, pB = self.endpoints
if pA in pB.links:
pB.links.remove(pA)
if pB in pA.links:
pA.links.remove(pB)
return self
class Endpoint(object):
"""
Link endpoint (wrapper).
@ivar link: The associated link.
@type link: L{Link}
@ivar target: The properties object.
@type target: L{Property}
"""
def __init__(self, link, target):
self.link = link
self.target = target
def teardown(self):
return self.link.teardown()
def __eq__(self, rhs):
return ( self.target == rhs )
def __hash__(self):
return hash(self.target)
def __getattr__(self, name):
return getattr(self.target, name)
class Definition:
"""
Property definition.
@ivar name: The property name.
@type name: str
@ivar classes: The (class) list of permitted values
@type classes: tuple
@ivar default: The default value.
@ivar type: any
"""
def __init__(self, name, classes, default, linker=AutoLinker()):
"""
@param name: The property name.
@type name: str
@param classes: The (class) list of permitted values
@type classes: tuple
@param default: The default value.
@type default: any
"""
if not isinstance(classes, (list, tuple)):
classes = (classes,)
self.name = name
self.classes = classes
self.default = default
self.linker = linker
def nvl(self, value=None):
"""
Convert the I{value} into the default when I{None}.
@param value: The proposed value.
@type value: any
@return: The I{default} when I{value} is I{None}, else I{value}.
@rtype: any
"""
if value is None:
return self.default
else:
return value
def validate(self, value):
"""
Validate the I{value} is of the correct class.
@param value: The value to validate.
@type value: any
@raise AttributeError: When I{value} is invalid.
"""
if value is None:
return
if len(self.classes) and \
not isinstance(value, self.classes):
msg = '"%s" must be: %s' % (self.name, self.classes)
raise AttributeError,msg
def __repr__(self):
return '%s: %s' % (self.name, str(self))
def __str__(self):
s = []
if len(self.classes):
s.append('classes=%s' % str(self.classes))
else:
s.append('classes=*')
s.append("default=%s" % str(self.default))
return ', '.join(s)
class Properties:
"""
Represents basic application properties.
Provides basic type validation, default values and
link/synchronization behavior.
@ivar domain: The domain name.
@type domain: str
@ivar definitions: A table of property definitions.
@type definitions: {name: L{Definition}}
@ivar links: A list of linked property objects used to create
a network of properties.
@type links: [L{Property},..]
@ivar defined: A dict of property values.
@type defined: dict
"""
def __init__(self, domain, definitions, kwargs):
"""
@param domain: The property domain name.
@type domain: str
@param definitions: A table of property definitions.
@type definitions: {name: L{Definition}}
@param kwargs: A list of property name/values to set.
@type kwargs: dict
"""
self.definitions = {}
for d in definitions:
self.definitions[d.name] = d
self.domain = domain
self.links = []
self.defined = {}
self.modified = set()
self.prime()
self.update(kwargs)
def definition(self, name):
"""
Get the definition for the property I{name}.
@param name: The property I{name} to find the definition for.
@type name: str
@return: The property definition
@rtype: L{Definition}
@raise AttributeError: On not found.
"""
d = self.definitions.get(name)
if d is None:
raise AttributeError(name)
return d
def update(self, other):
"""
Update the property values as specified by keyword/value.
@param other: An object to update from.
@type other: (dict|L{Properties})
@return: self
@rtype: L{Properties}
"""
if isinstance(other, Properties):
other = other.defined
for n,v in other.items():
self.set(n, v)
return self
def notset(self, name):
"""
Get whether a property has never been set by I{name}.
@param name: A property name.
@type name: str
@return: True if never been set.
@rtype: bool
"""
self.provider(name).__notset(name)
def set(self, name, value):
"""
Set the I{value} of a property by I{name}.
The value is validated against the definition and set
to the default when I{value} is None.
@param name: The property name.
@type name: str
@param value: The new property value.
@type value: any
@return: self
@rtype: L{Properties}
"""
self.provider(name).__set(name, value)
return self
def unset(self, name):
"""
Unset a property by I{name}.
@param name: A property name.
@type name: str
@return: self
@rtype: L{Properties}
"""
self.provider(name).__set(name, None)
return self
def get(self, name, *df):
"""
Get the value of a property by I{name}.
@param name: The property name.
@type name: str
@param df: An optional value to be returned when the value
is not set
@type df: [1].
@return: The stored value, or I{df[0]} if not set.
@rtype: any
"""
return self.provider(name).__get(name, *df)
def link(self, other):
"""
Link (associate) this object with anI{other} properties object
to create a network of properties. Links are bidirectional.
@param other: The object to link.
@type other: L{Properties}
@return: self
@rtype: L{Properties}
"""
Link(self, other)
return self
def unlink(self, *others):
"""
Unlink (disassociate) the specified properties object.
@param others: The list object to unlink. Unspecified means unlink all.
@type others: [L{Properties},..]
@return: self
@rtype: L{Properties}
"""
if not len(others):
others = self.links[:]
for p in self.links[:]:
if p in others:
p.teardown()
return self
def provider(self, name, history=None):
"""
Find the provider of the property by I{name}.
@param name: The property name.
@type name: str
@param history: A history of nodes checked to prevent
circular hunting.
@type history: [L{Properties},..]
@return: The provider when found. Otherwise, None (when nested)
and I{self} when not nested.
@rtype: L{Properties}
"""
if history is None:
history = []
history.append(self)
if name in self.definitions:
return self
for x in self.links:
if x in history:
continue
provider = x.provider(name, history)
if provider is not None:
return provider
history.remove(self)
if len(history):
return None
return self
def keys(self, history=None):
"""
Get the set of I{all} property names.
@param history: A history of nodes checked to prevent
circular hunting.
@type history: [L{Properties},..]
@return: A set of property names.
@rtype: list
"""
if history is None:
history = []
history.append(self)
keys = set()
keys.update(self.definitions.keys())
for x in self.links:
if x in history:
continue
keys.update(x.keys(history))
history.remove(self)
return keys
def domains(self, history=None):
"""
Get the set of I{all} domain names.
@param history: A history of nodes checked to prevent
circular hunting.
@type history: [L{Properties},..]
@return: A set of domain names.
@rtype: list
"""
if history is None:
history = []
history.append(self)
domains = set()
domains.add(self.domain)
for x in self.links:
if x in history:
continue
domains.update(x.domains(history))
history.remove(self)
return domains
def prime(self):
"""
Prime the stored values based on default values
found in property definitions.
@return: self
@rtype: L{Properties}
"""
for d in self.definitions.values():
self.defined[d.name] = d.default
return self
def __notset(self, name):
return not (name in self.modified)
def __set(self, name, value):
d = self.definition(name)
d.validate(value)
value = d.nvl(value)
prev = self.defined[name]
self.defined[name] = value
self.modified.add(name)
d.linker.updated(self, prev, value)
def __get(self, name, *df):
d = self.definition(name)
value = self.defined.get(name)
if value == d.default and len(df):
value = df[0]
return value
def str(self, history):
s = []
s.append('Definitions:')
for d in self.definitions.values():
s.append('\t%s' % repr(d))
s.append('Content:')
for d in self.defined.items():
s.append('\t%s' % str(d))
if self not in history:
history.append(self)
s.append('Linked:')
for x in self.links:
s.append(x.str(history))
history.remove(self)
return '\n'.join(s)
def __repr__(self):
return str(self)
def __str__(self):
return self.str([])
class Skin(object):
"""
The meta-programming I{skin} around the L{Properties} object.
@ivar __pts__: The wrapped object.
@type __pts__: L{Properties}.
"""
def __init__(self, domain, definitions, kwargs):
self.__pts__ = Properties(domain, definitions, kwargs)
def __setattr__(self, name, value):
builtin = name.startswith('__') and name.endswith('__')
if builtin:
self.__dict__[name] = value
return
self.__pts__.set(name, value)
def __getattr__(self, name):
return self.__pts__.get(name)
def __repr__(self):
return str(self)
def __str__(self):
return str(self.__pts__)
class Unskin(object):
def __new__(self, *args, **kwargs):
return args[0].__pts__
class Inspector:
"""
Wrapper inspector.
"""
def __init__(self, options):
self.properties = options.__pts__
def get(self, name, *df):
"""
Get the value of a property by I{name}.
@param name: The property name.
@type name: str
@param df: An optional value to be returned when the value
is not set
@type df: [1].
@return: The stored value, or I{df[0]} if not set.
@rtype: any
"""
return self.properties.get(name, *df)
def update(self, **kwargs):
"""
Update the property values as specified by keyword/value.
@param kwargs: A list of property name/values to set.
@type kwargs: dict
@return: self
@rtype: L{Properties}
"""
return self.properties.update(**kwargs)
def link(self, other):
"""
Link (associate) this object with anI{other} properties object
to create a network of properties. Links are bidirectional.
@param other: The object to link.
@type other: L{Properties}
@return: self
@rtype: L{Properties}
"""
p = other.__pts__
return self.properties.link(p)
def unlink(self, other):
"""
Unlink (disassociate) the specified properties object.
@param other: The object to unlink.
@type other: L{Properties}
@return: self
@rtype: L{Properties}
"""
p = other.__pts__
return self.properties.unlink(p)
libs/suds/reader.py
+169
View File
@@ -0,0 +1,169 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains xml document reader classes.
"""
from suds.sax.parser import Parser
from suds.transport import Request
from suds.cache import Cache, NoCache
from suds.store import DocumentStore
from suds.plugin import PluginContainer
from logging import getLogger
log = getLogger(__name__)
class Reader:
"""
The reader provides integration with cache.
@ivar options: An options object.
@type options: I{Options}
"""
def __init__(self, options):
"""
@param options: An options object.
@type options: I{Options}
"""
self.options = options
self.plugins = PluginContainer(options.plugins)
def mangle(self, name, x):
"""
Mangle the name by hashing the I{name} and appending I{x}.
@return: the mangled name.
"""
h = abs(hash(name))
return '%s-%s' % (h, x)
class DocumentReader(Reader):
"""
The XML document reader provides an integration
between the SAX L{Parser} and the document cache.
"""
def open(self, url):
"""
Open an XML document at the specified I{url}.
First, the document attempted to be retrieved from
the I{object cache}. If not found, it is downloaded and
parsed using the SAX parser. The result is added to the
cache for the next open().
@param url: A document url.
@type url: str.
@return: The specified XML document.
@rtype: I{Document}
"""
cache = self.cache()
id = self.mangle(url, 'document')
d = cache.get(id)
if d is None:
d = self.download(url)
cache.put(id, d)
self.plugins.document.parsed(url=url, document=d.root())
return d
def download(self, url):
"""
Download the docuemnt.
@param url: A document url.
@type url: str.
@return: A file pointer to the docuemnt.
@rtype: file-like
"""
store = DocumentStore()
fp = store.open(url)
if fp is None:
fp = self.options.transport.open(Request(url))
content = fp.read()
fp.close()
ctx = self.plugins.document.loaded(url=url, document=content)
content = ctx.document
sax = Parser()
return sax.parse(string=content)
def cache(self):
"""
Get the cache.
@return: The I{options} when I{cachingpolicy} = B{0}.
@rtype: L{Cache}
"""
if self.options.cachingpolicy == 0:
return self.options.cache
else:
return NoCache()
class DefinitionsReader(Reader):
"""
The WSDL definitions reader provides an integration
between the Definitions and the object cache.
@ivar fn: A factory function (constructor) used to
create the object not found in the cache.
@type fn: I{Constructor}
"""
def __init__(self, options, fn):
"""
@param options: An options object.
@type options: I{Options}
@param fn: A factory function (constructor) used to
create the object not found in the cache.
@type fn: I{Constructor}
"""
Reader.__init__(self, options)
self.fn = fn
def open(self, url):
"""
Open a WSDL at the specified I{url}.
First, the WSDL attempted to be retrieved from
the I{object cache}. After unpickled from the cache, the
I{options} attribute is restored.
If not found, it is downloaded and instantiated using the
I{fn} constructor and added to the cache for the next open().
@param url: A WSDL url.
@type url: str.
@return: The WSDL object.
@rtype: I{Definitions}
"""
cache = self.cache()
id = self.mangle(url, 'wsdl')
d = cache.get(id)
if d is None:
d = self.fn(url, self.options)
cache.put(id, d)
else:
d.options = self.options
for imp in d.imports:
imp.imported.options = self.options
return d
def cache(self):
"""
Get the cache.
@return: The I{options} when I{cachingpolicy} = B{1}.
@rtype: L{Cache}
"""
if self.options.cachingpolicy == 1:
return self.options.cache
else:
return NoCache()
libs/suds/resolver.py
+496
View File
@@ -0,0 +1,496 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{resolver} module provides a collection of classes that
provide wsdl/xsd named type resolution.
"""
import re
from logging import getLogger
from suds import *
from suds.sax import splitPrefix, Namespace
from suds.sudsobject import Object
from suds.xsd.query import BlindQuery, TypeQuery, qualify
log = getLogger(__name__)
class Resolver:
"""
An I{abstract} schema-type resolver.
@ivar schema: A schema object.
@type schema: L{xsd.schema.Schema}
"""
def __init__(self, schema):
"""
@param schema: A schema object.
@type schema: L{xsd.schema.Schema}
"""
self.schema = schema
def find(self, name, resolved=True):
"""
Get the definition object for the schema object by name.
@param name: The name of a schema object.
@type name: basestring
@param resolved: A flag indicating that the fully resolved type
should be returned.
@type resolved: boolean
@return: The found schema I{type}
@rtype: L{xsd.sxbase.SchemaObject}
"""
log.debug('searching schema for (%s)', name)
qref = qualify(name, self.schema.root, self.schema.tns)
query = BlindQuery(qref)
result = query.execute(self.schema)
if result is None:
log.error('(%s) not-found', name)
return None
log.debug('found (%s) as (%s)', name, Repr(result))
if resolved:
result = result.resolve()
return result
class PathResolver(Resolver):
"""
Resolveds the definition object for the schema type located at the specified path.
The path may contain (.) dot notation to specify nested types.
@ivar wsdl: A wsdl object.
@type wsdl: L{wsdl.Definitions}
"""
def __init__(self, wsdl, ps='.'):
"""
@param wsdl: A schema object.
@type wsdl: L{wsdl.Definitions}
@param ps: The path separator character
@type ps: char
"""
Resolver.__init__(self, wsdl.schema)
self.wsdl = wsdl
self.altp = re.compile('({)(.+)(})(.+)')
self.splitp = re.compile('({.+})*[^\%s]+' % ps[0])
def find(self, path, resolved=True):
"""
Get the definition object for the schema type located at the specified path.
The path may contain (.) dot notation to specify nested types.
Actually, the path separator is usually a (.) but can be redefined
during contruction.
@param path: A (.) separated path to a schema type.
@type path: basestring
@param resolved: A flag indicating that the fully resolved type
should be returned.
@type resolved: boolean
@return: The found schema I{type}
@rtype: L{xsd.sxbase.SchemaObject}
"""
result = None
parts = self.split(path)
try:
result = self.root(parts)
if len(parts) > 1:
result = result.resolve(nobuiltin=True)
result = self.branch(result, parts)
result = self.leaf(result, parts)
if resolved:
result = result.resolve(nobuiltin=True)
except PathResolver.BadPath:
log.error('path: "%s", not-found' % path)
return result
def root(self, parts):
"""
Find the path root.
@param parts: A list of path parts.
@type parts: [str,..]
@return: The root.
@rtype: L{xsd.sxbase.SchemaObject}
"""
result = None
name = parts[0]
log.debug('searching schema for (%s)', name)
qref = self.qualify(parts[0])
query = BlindQuery(qref)
result = query.execute(self.schema)
if result is None:
log.error('(%s) not-found', name)
raise PathResolver.BadPath(name)
else:
log.debug('found (%s) as (%s)', name, Repr(result))
return result
def branch(self, root, parts):
"""
Traverse the path until the leaf is reached.
@param parts: A list of path parts.
@type parts: [str,..]
@param root: The root.
@type root: L{xsd.sxbase.SchemaObject}
@return: The end of the branch.
@rtype: L{xsd.sxbase.SchemaObject}
"""
result = root
for part in parts[1:-1]:
name = splitPrefix(part)[1]
log.debug('searching parent (%s) for (%s)', Repr(result), name)
result, ancestry = result.get_child(name)
if result is None:
log.error('(%s) not-found', name)
raise PathResolver.BadPath(name)
else:
result = result.resolve(nobuiltin=True)
log.debug('found (%s) as (%s)', name, Repr(result))
return result
def leaf(self, parent, parts):
"""
Find the leaf.
@param parts: A list of path parts.
@type parts: [str,..]
@param parent: The leaf's parent.
@type parent: L{xsd.sxbase.SchemaObject}
@return: The leaf.
@rtype: L{xsd.sxbase.SchemaObject}
"""
name = splitPrefix(parts[-1])[1]
if name.startswith('@'):
result, path = parent.get_attribute(name[1:])
else:
result, ancestry = parent.get_child(name)
if result is None:
raise PathResolver.BadPath(name)
return result
def qualify(self, name):
"""
Qualify the name as either:
- plain name
- ns prefixed name (eg: ns0:Person)
- fully ns qualified name (eg: {http://myns-uri}Person)
@param name: The name of an object in the schema.
@type name: str
@return: A qualifed name.
@rtype: qname
"""
m = self.altp.match(name)
if m is None:
return qualify(name, self.wsdl.root, self.wsdl.tns)
else:
return (m.group(4), m.group(2))
def split(self, s):
"""
Split the string on (.) while preserving any (.) inside the
'{}' alternalte syntax for full ns qualification.
@param s: A plain or qualifed name.
@type s: str
@return: A list of the name's parts.
@rtype: [str,..]
"""
parts = []
b = 0
while 1:
m = self.splitp.match(s, b)
if m is None:
break
b,e = m.span()
parts.append(s[b:e])
b = e+1
return parts
class BadPath(Exception): pass
class TreeResolver(Resolver):
"""
The tree resolver is a I{stateful} tree resolver
used to resolve each node in a tree. As such, it mirrors
the tree structure to ensure that nodes are resolved in
context.
@ivar stack: The context stack.
@type stack: list
"""
def __init__(self, schema):
"""
@param schema: A schema object.
@type schema: L{xsd.schema.Schema}
"""
Resolver.__init__(self, schema)
self.stack = Stack()
def reset(self):
"""
Reset the resolver's state.
"""
self.stack = Stack()
def push(self, x):
"""
Push an I{object} onto the stack.
@param x: An object to push.
@type x: L{Frame}
@return: The pushed frame.
@rtype: L{Frame}
"""
if isinstance(x, Frame):
frame = x
else:
frame = Frame(x)
self.stack.append(frame)
log.debug('push: (%s)\n%s', Repr(frame), Repr(self.stack))
return frame
def top(self):
"""
Get the I{frame} at the top of the stack.
@return: The top I{frame}, else None.
@rtype: L{Frame}
"""
if len(self.stack):
return self.stack[-1]
else:
return Frame.Empty()
def pop(self):
"""
Pop the frame at the top of the stack.
@return: The popped frame, else None.
@rtype: L{Frame}
"""
if len(self.stack):
popped = self.stack.pop()
log.debug('pop: (%s)\n%s', Repr(popped), Repr(self.stack))
return popped
else:
log.debug('stack empty, not-popped')
return None
def depth(self):
"""
Get the current stack depth.
@return: The current stack depth.
@rtype: int
"""
return len(self.stack)
def getchild(self, name, parent):
""" get a child by name """
log.debug('searching parent (%s) for (%s)', Repr(parent), name)
if name.startswith('@'):
return parent.get_attribute(name[1:])
else:
return parent.get_child(name)
class NodeResolver(TreeResolver):
"""
The node resolver is a I{stateful} XML document resolver
used to resolve each node in a tree. As such, it mirrors
the tree structure to ensure that nodes are resolved in
context.
"""
def __init__(self, schema):
"""
@param schema: A schema object.
@type schema: L{xsd.schema.Schema}
"""
TreeResolver.__init__(self, schema)
def find(self, node, resolved=False, push=True):
"""
@param node: An xml node to be resolved.
@type node: L{sax.element.Element}
@param resolved: A flag indicating that the fully resolved type should be
returned.
@type resolved: boolean
@param push: Indicates that the resolved type should be
pushed onto the stack.
@type push: boolean
@return: The found schema I{type}
@rtype: L{xsd.sxbase.SchemaObject}
"""
name = node.name
parent = self.top().resolved
if parent is None:
result, ancestry = self.query(name, node)
else:
result, ancestry = self.getchild(name, parent)
known = self.known(node)
if result is None:
return result
if push:
frame = Frame(result, resolved=known, ancestry=ancestry)
pushed = self.push(frame)
if resolved:
result = result.resolve()
return result
def findattr(self, name, resolved=True):
"""
Find an attribute type definition.
@param name: An attribute name.
@type name: basestring
@param resolved: A flag indicating that the fully resolved type should be
returned.
@type resolved: boolean
@return: The found schema I{type}
@rtype: L{xsd.sxbase.SchemaObject}
"""
name = '@%s'%name
parent = self.top().resolved
if parent is None:
result, ancestry = self.query(name, node)
else:
result, ancestry = self.getchild(name, parent)
if result is None:
return result
if resolved:
result = result.resolve()
return result
def query(self, name, node):
""" blindly query the schema by name """
log.debug('searching schema for (%s)', name)
qref = qualify(name, node, node.namespace())
query = BlindQuery(qref)
result = query.execute(self.schema)
return (result, [])
def known(self, node):
""" resolve type referenced by @xsi:type """
ref = node.get('type', Namespace.xsins)
if ref is None:
return None
qref = qualify(ref, node, node.namespace())
query = BlindQuery(qref)
return query.execute(self.schema)
class GraphResolver(TreeResolver):
"""
The graph resolver is a I{stateful} L{Object} graph resolver
used to resolve each node in a tree. As such, it mirrors
the tree structure to ensure that nodes are resolved in
context.
"""
def __init__(self, schema):
"""
@param schema: A schema object.
@type schema: L{xsd.schema.Schema}
"""
TreeResolver.__init__(self, schema)
def find(self, name, object, resolved=False, push=True):
"""
@param name: The name of the object to be resolved.
@type name: basestring
@param object: The name's value.
@type object: (any|L{Object})
@param resolved: A flag indicating that the fully resolved type
should be returned.
@type resolved: boolean
@param push: Indicates that the resolved type should be
pushed onto the stack.
@type push: boolean
@return: The found schema I{type}
@rtype: L{xsd.sxbase.SchemaObject}
"""
known = None
parent = self.top().resolved
if parent is None:
result, ancestry = self.query(name)
else:
result, ancestry = self.getchild(name, parent)
if result is None:
return None
if isinstance(object, Object):
known = self.known(object)
if push:
frame = Frame(result, resolved=known, ancestry=ancestry)
pushed = self.push(frame)
if resolved:
if known is None:
result = result.resolve()
else:
result = known
return result
def query(self, name):
""" blindly query the schema by name """
log.debug('searching schema for (%s)', name)
schema = self.schema
wsdl = self.wsdl()
if wsdl is None:
qref = qualify(name, schema.root, schema.tns)
else:
qref = qualify(name, wsdl.root, wsdl.tns)
query = BlindQuery(qref)
result = query.execute(schema)
return (result, [])
def wsdl(self):
""" get the wsdl """
container = self.schema.container
if container is None:
return None
else:
return container.wsdl
def known(self, object):
""" get the type specified in the object's metadata """
try:
md = object.__metadata__
known = md.sxtype
return known
except:
pass
class Frame:
def __init__(self, type, resolved=None, ancestry=()):
self.type = type
if resolved is None:
resolved = type.resolve()
self.resolved = resolved.resolve()
self.ancestry = ancestry
def __str__(self):
return '%s\n%s\n%s' % \
(Repr(self.type),
Repr(self.resolved),
[Repr(t) for t in self.ancestry])
class Empty:
def __getattr__(self, name):
if name == 'ancestry':
return ()
else:
return None
class Stack(list):
def __repr__(self):
result = []
for item in self:
result.append(repr(item))
return '\n'.join(result)
libs/suds/sax/__init__.py
+109
View File
@@ -0,0 +1,109 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The sax module contains a collection of classes that provide a
(D)ocument (O)bject (M)odel representation of an XML document.
The goal is to provide an easy, intuative interface for managing XML
documents. Although, the term, DOM, is used above, this model is
B{far} better.
XML namespaces in suds are represented using a (2) element tuple
containing the prefix and the URI. Eg: I{('tns', 'http://myns')}
@var encoder: A I{pluggable} XML special character processor used to
encode/decode strings.
@type encoder: L{Encoder}
"""
from suds.sax.enc import Encoder
#
# pluggable XML special character encoder.
#
encoder = Encoder()
def splitPrefix(name):
"""
Split the name into a tuple (I{prefix}, I{name}). The first element in
the tuple is I{None} when the name does't have a prefix.
@param name: A node name containing an optional prefix.
@type name: basestring
@return: A tuple containing the (2) parts of I{name}
@rtype: (I{prefix}, I{name})
"""
if isinstance(name, basestring) \
and ':' in name:
return tuple(name.split(':', 1))
else:
return (None, name)
class Namespace:
"""
The namespace class represents XML namespaces.
"""
default = (None, None)
xmlns = ('xml', 'http://www.w3.org/XML/1998/namespace')
xsdns = ('xs', 'http://www.w3.org/2001/XMLSchema')
xsins = ('xsi', 'http://www.w3.org/2001/XMLSchema-instance')
all = (xsdns, xsins)
@classmethod
def create(cls, p=None, u=None):
return (p, u)
@classmethod
def none(cls, ns):
return ( ns == cls.default )
@classmethod
def xsd(cls, ns):
try:
return cls.w3(ns) and ns[1].endswith('XMLSchema')
except:
pass
return False
@classmethod
def xsi(cls, ns):
try:
return cls.w3(ns) and ns[1].endswith('XMLSchema-instance')
except:
pass
return False
@classmethod
def xs(cls, ns):
return ( cls.xsd(ns) or cls.xsi(ns) )
@classmethod
def w3(cls, ns):
try:
return ns[1].startswith('http://www.w3.org')
except:
pass
return False
@classmethod
def isns(cls, ns):
try:
return isinstance(ns, tuple) and len(ns) == len(cls.default)
except:
pass
return False
libs/suds/sax/attribute.py
+181
View File
@@ -0,0 +1,181 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides XML I{attribute} classes.
"""
import suds.sax
from logging import getLogger
from suds import *
from suds.sax import *
from suds.sax.text import Text
log = getLogger(__name__)
class Attribute:
"""
An XML attribute object.
@ivar parent: The node containing this attribute
@type parent: L{element.Element}
@ivar prefix: The I{optional} namespace prefix.
@type prefix: basestring
@ivar name: The I{unqualified} name of the attribute
@type name: basestring
@ivar value: The attribute's value
@type value: basestring
"""
def __init__(self, name, value=None):
"""
@param name: The attribute's name with I{optional} namespace prefix.
@type name: basestring
@param value: The attribute's value
@type value: basestring
"""
self.parent = None
self.prefix, self.name = splitPrefix(name)
self.setValue(value)
def clone(self, parent=None):
"""
Clone this object.
@param parent: The parent for the clone.
@type parent: L{element.Element}
@return: A copy of this object assigned to the new parent.
@rtype: L{Attribute}
"""
a = Attribute(self.qname(), self.value)
a.parent = parent
return a
def qname(self):
"""
Get the B{fully} qualified name of this attribute
@return: The fully qualified name.
@rtype: basestring
"""
if self.prefix is None:
return self.name
else:
return ':'.join((self.prefix, self.name))
def setValue(self, value):
"""
Set the attributes value
@param value: The new value (may be None)
@type value: basestring
@return: self
@rtype: L{Attribute}
"""
if isinstance(value, Text):
self.value = value
else:
self.value = Text(value)
return self
def getValue(self, default=Text('')):
"""
Get the attributes value with optional default.
@param default: An optional value to be return when the
attribute's has not been set.
@type default: basestring
@return: The attribute's value, or I{default}
@rtype: L{Text}
"""
if self.hasText():
return self.value
else:
return default
def hasText(self):
"""
Get whether the attribute has I{text} and that it is not an empty
(zero length) string.
@return: True when has I{text}.
@rtype: boolean
"""
return ( self.value is not None and len(self.value) )
def namespace(self):
"""
Get the attributes namespace. This may either be the namespace
defined by an optional prefix, or its parent's namespace.
@return: The attribute's namespace
@rtype: (I{prefix}, I{name})
"""
if self.prefix is None:
return Namespace.default
else:
return self.resolvePrefix(self.prefix)
def resolvePrefix(self, prefix):
"""
Resolve the specified prefix to a known namespace.
@param prefix: A declared prefix
@type prefix: basestring
@return: The namespace that has been mapped to I{prefix}
@rtype: (I{prefix}, I{name})
"""
ns = Namespace.default
if self.parent is not None:
ns = self.parent.resolvePrefix(prefix)
return ns
def match(self, name=None, ns=None):
"""
Match by (optional) name and/or (optional) namespace.
@param name: The optional attribute tag name.
@type name: str
@param ns: An optional namespace.
@type ns: (I{prefix}, I{name})
@return: True if matched.
@rtype: boolean
"""
if name is None:
byname = True
else:
byname = ( self.name == name )
if ns is None:
byns = True
else:
byns = ( self.namespace()[1] == ns[1] )
return ( byname and byns )
def __eq__(self, rhs):
""" equals operator """
return rhs is not None and \
isinstance(rhs, Attribute) and \
self.prefix == rhs.name and \
self.name == rhs.name
def __repr__(self):
""" get a string representation """
return \
'attr (prefix=%s, name=%s, value=(%s))' %\
(self.prefix, self.name, self.value)
def __str__(self):
""" get an xml string representation """
return unicode(self).encode('utf-8')
def __unicode__(self):
""" get an xml string representation """
n = self.qname()
if self.hasText():
v = self.value.escape()
else:
v = self.value
return u'%s="%s"' % (n, v)
libs/suds/sax/date.py
+378
View File
@@ -0,0 +1,378 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Nathan Van Gheem (vangheem@gmail.com)
"""
The I{xdate} module provides classes for converstion
between XML dates and python objects.
"""
from logging import getLogger
from suds import *
from suds.xsd import *
import time
import datetime as dt
import re
log = getLogger(__name__)
class Date:
"""
An XML date object.
Supported formats:
- YYYY-MM-DD
- YYYY-MM-DD(z|Z)
- YYYY-MM-DD+06:00
- YYYY-MM-DD-06:00
@ivar date: The object value.
@type date: B{datetime}.I{date}
"""
def __init__(self, date):
"""
@param date: The value of the object.
@type date: (date|str)
@raise ValueError: When I{date} is invalid.
"""
if isinstance(date, dt.date):
self.date = date
return
if isinstance(date, basestring):
self.date = self.__parse(date)
return
raise ValueError, type(date)
def year(self):
"""
Get the I{year} component.
@return: The year.
@rtype: int
"""
return self.date.year
def month(self):
"""
Get the I{month} component.
@return: The month.
@rtype: int
"""
return self.date.month
def day(self):
"""
Get the I{day} component.
@return: The day.
@rtype: int
"""
return self.date.day
def __parse(self, s):
"""
Parse the string date.
Supported formats:
- YYYY-MM-DD
- YYYY-MM-DD(z|Z)
- YYYY-MM-DD+06:00
- YYYY-MM-DD-06:00
Although, the TZ is ignored because it's meaningless
without the time, right?
@param s: A date string.
@type s: str
@return: A date object.
@rtype: I{date}
"""
try:
year, month, day = s[:10].split('-', 2)
year = int(year)
month = int(month)
day = int(day)
return dt.date(year, month, day)
except:
log.debug(s, exec_info=True)
raise ValueError, 'Invalid format "%s"' % s
def __str__(self):
return unicode(self)
def __unicode__(self):
return self.date.isoformat()
class Time:
"""
An XML time object.
Supported formats:
- HH:MI:SS
- HH:MI:SS(z|Z)
- HH:MI:SS.ms
- HH:MI:SS.ms(z|Z)
- HH:MI:SS(+|-)06:00
- HH:MI:SS.ms(+|-)06:00
@ivar tz: The timezone
@type tz: L{Timezone}
@ivar date: The object value.
@type date: B{datetime}.I{time}
"""
def __init__(self, time, adjusted=True):
"""
@param time: The value of the object.
@type time: (time|str)
@param adjusted: Adjust for I{local} Timezone.
@type adjusted: boolean
@raise ValueError: When I{time} is invalid.
"""
self.tz = Timezone()
if isinstance(time, dt.time):
self.time = time
return
if isinstance(time, basestring):
self.time = self.__parse(time)
if adjusted:
self.__adjust()
return
raise ValueError, type(time)
def hour(self):
"""
Get the I{hour} component.
@return: The hour.
@rtype: int
"""
return self.time.hour
def minute(self):
"""
Get the I{minute} component.
@return: The minute.
@rtype: int
"""
return self.time.minute
def second(self):
"""
Get the I{seconds} component.
@return: The seconds.
@rtype: int
"""
return self.time.second
def microsecond(self):
"""
Get the I{microsecond} component.
@return: The microsecond.
@rtype: int
"""
return self.time.microsecond
def __adjust(self):
"""
Adjust for TZ offset.
"""
if hasattr(self, 'offset'):
today = dt.date.today()
delta = self.tz.adjustment(self.offset)
d = dt.datetime.combine(today, self.time)
d = ( d + delta )
self.time = d.time()
def __parse(self, s):
"""
Parse the string date.
Patterns:
- HH:MI:SS
- HH:MI:SS(z|Z)
- HH:MI:SS.ms
- HH:MI:SS.ms(z|Z)
- HH:MI:SS(+|-)06:00
- HH:MI:SS.ms(+|-)06:00
@param s: A time string.
@type s: str
@return: A time object.
@rtype: B{datetime}.I{time}
"""
try:
offset = None
part = Timezone.split(s)
hour, minute, second = part[0].split(':', 2)
hour = int(hour)
minute = int(minute)
second, ms = self.__second(second)
if len(part) == 2:
self.offset = self.__offset(part[1])
if ms is None:
return dt.time(hour, minute, second)
else:
return dt.time(hour, minute, second, ms)
except:
log.debug(s, exec_info=True)
raise ValueError, 'Invalid format "%s"' % s
def __second(self, s):
"""
Parse the seconds and microseconds.
The microseconds are truncated to 999999 due to a restriction in
the python datetime.datetime object.
@param s: A string representation of the seconds.
@type s: str
@return: Tuple of (sec,ms)
@rtype: tuple.
"""
part = s.split('.')
if len(part) > 1:
return (int(part[0]), int(part[1][:6]))
else:
return (int(part[0]), None)
def __offset(self, s):
"""
Parse the TZ offset.
@param s: A string representation of the TZ offset.
@type s: str
@return: The signed offset in hours.
@rtype: str
"""
if len(s) == len('-00:00'):
return int(s[:3])
if len(s) == 0:
return self.tz.local
if len(s) == 1:
return 0
raise Exception()
def __str__(self):
return unicode(self)
def __unicode__(self):
time = self.time.isoformat()
if self.tz.local:
return '%s%+.2d:00' % (time, self.tz.local)
else:
return '%sZ' % time
class DateTime(Date,Time):
"""
An XML time object.
Supported formats:
- YYYY-MM-DDB{T}HH:MI:SS
- YYYY-MM-DDB{T}HH:MI:SS(z|Z)
- YYYY-MM-DDB{T}HH:MI:SS.ms
- YYYY-MM-DDB{T}HH:MI:SS.ms(z|Z)
- YYYY-MM-DDB{T}HH:MI:SS(+|-)06:00
- YYYY-MM-DDB{T}HH:MI:SS.ms(+|-)06:00
@ivar datetime: The object value.
@type datetime: B{datetime}.I{datedate}
"""
def __init__(self, date):
"""
@param date: The value of the object.
@type date: (datetime|str)
@raise ValueError: When I{tm} is invalid.
"""
if isinstance(date, dt.datetime):
Date.__init__(self, date.date())
Time.__init__(self, date.time())
self.datetime = \
dt.datetime.combine(self.date, self.time)
return
if isinstance(date, basestring):
part = date.split('T')
Date.__init__(self, part[0])
Time.__init__(self, part[1], 0)
self.datetime = \
dt.datetime.combine(self.date, self.time)
self.__adjust()
return
raise ValueError, type(date)
def __adjust(self):
"""
Adjust for TZ offset.
"""
if not hasattr(self, 'offset'):
return
delta = self.tz.adjustment(self.offset)
try:
d = ( self.datetime + delta )
self.datetime = d
self.date = d.date()
self.time = d.time()
except OverflowError:
log.warn('"%s" caused overflow, not-adjusted', self.datetime)
def __str__(self):
return unicode(self)
def __unicode__(self):
s = []
s.append(Date.__unicode__(self))
s.append(Time.__unicode__(self))
return 'T'.join(s)
class UTC(DateTime):
"""
Represents current UTC time.
"""
def __init__(self, date=None):
if date is None:
date = dt.datetime.utcnow()
DateTime.__init__(self, date)
self.tz.local = 0
class Timezone:
"""
Timezone object used to do TZ conversions
@cvar local: The (A) local TZ offset.
@type local: int
@cvar patten: The regex patten to match TZ.
@type patten: re.Pattern
"""
pattern = re.compile('([zZ])|([\-\+][0-9]{2}:[0-9]{2})')
LOCAL = ( 0-time.timezone/60/60 )
def __init__(self, offset=None):
if offset is None:
offset = self.LOCAL
self.local = offset
@classmethod
def split(cls, s):
"""
Split the TZ from string.
@param s: A string containing a timezone
@type s: basestring
@return: The split parts.
@rtype: tuple
"""
m = cls.pattern.search(s)
if m is None:
return (s,)
x = m.start(0)
return (s[:x], s[x:])
def adjustment(self, offset):
"""
Get the adjustment to the I{local} TZ.
@return: The delta between I{offset} and local TZ.
@rtype: B{datetime}.I{timedelta}
"""
delta = ( self.local - offset )
return dt.timedelta(hours=delta)
libs/suds/sax/document.py
+61
View File
@@ -0,0 +1,61 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides XML I{document} classes.
"""
from logging import getLogger
from suds import *
from suds.sax import *
from suds.sax.element import Element
log = getLogger(__name__)
class Document(Element):
""" simple document """
DECL = '<?xml version="1.0" encoding="UTF-8"?>'
def __init__(self, root=None):
Element.__init__(self, 'document')
if root is not None:
self.append(root)
def root(self):
if len(self.children):
return self.children[0]
else:
return None
def str(self):
s = []
s.append(self.DECL)
s.append('\n')
s.append(self.root().str())
return ''.join(s)
def plain(self):
s = []
s.append(self.DECL)
s.append(self.root().plain())
return ''.join(s)
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
return self.str()
libs/suds/sax/element.py
+1147
View File
File diff suppressed because it is too large Load Diff
libs/suds/sax/enc.py
+79
View File
@@ -0,0 +1,79 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides XML I{special character} encoder classes.
"""
import re
class Encoder:
"""
An XML special character encoder/decoder.
@cvar encodings: A mapping of special characters encoding.
@type encodings: [(str,str)]
@cvar decodings: A mapping of special characters decoding.
@type decodings: [(str,str)]
@cvar special: A list of special characters
@type special: [char]
"""
encodings = \
(( '&(?!(amp|lt|gt|quot|apos);)', '&amp;' ),( '<', '&lt;' ),( '>', '&gt;' ),( '"', '&quot;' ),("'", '&apos;' ))
decodings = \
(( '&lt;', '<' ),( '&gt;', '>' ),( '&quot;', '"' ),( '&apos;', "'" ),( '&amp;', '&' ))
special = \
('&', '<', '>', '"', "'")
def needsEncoding(self, s):
"""
Get whether string I{s} contains special characters.
@param s: A string to check.
@type s: str
@return: True if needs encoding.
@rtype: boolean
"""
if isinstance(s, basestring):
for c in self.special:
if c in s:
return True
return False
def encode(self, s):
"""
Encode special characters found in string I{s}.
@param s: A string to encode.
@type s: str
@return: The encoded string.
@rtype: str
"""
if isinstance(s, basestring) and self.needsEncoding(s):
for x in self.encodings:
s = re.sub(x[0], x[1], s)
return s
def decode(self, s):
"""
Decode special characters encodings found in string I{s}.
@param s: A string to decode.
@type s: str
@return: The decoded string.
@rtype: str
"""
if isinstance(s, basestring) and '&' in s:
for x in self.decodings:
s = s.replace(x[0], x[1])
return s
libs/suds/sax/parser.py
+139
View File
@@ -0,0 +1,139 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The sax module contains a collection of classes that provide a
(D)ocument (O)bject (M)odel representation of an XML document.
The goal is to provide an easy, intuative interface for managing XML
documents. Although, the term, DOM, is used above, this model is
B{far} better.
XML namespaces in suds are represented using a (2) element tuple
containing the prefix and the URI. Eg: I{('tns', 'http://myns')}
"""
from logging import getLogger
import suds.metrics
from suds import *
from suds.sax import *
from suds.sax.document import Document
from suds.sax.element import Element
from suds.sax.text import Text
from suds.sax.attribute import Attribute
from xml.sax import make_parser, InputSource, ContentHandler
from xml.sax.handler import feature_external_ges
from cStringIO import StringIO
log = getLogger(__name__)
class Handler(ContentHandler):
""" sax hanlder """
def __init__(self):
self.nodes = [Document()]
def startElement(self, name, attrs):
top = self.top()
node = Element(unicode(name), parent=top)
for a in attrs.getNames():
n = unicode(a)
v = unicode(attrs.getValue(a))
attribute = Attribute(n,v)
if self.mapPrefix(node, attribute):
continue
node.append(attribute)
node.charbuffer = []
top.append(node)
self.push(node)
def mapPrefix(self, node, attribute):
skip = False
if attribute.name == 'xmlns':
if len(attribute.value):
node.expns = unicode(attribute.value)
skip = True
elif attribute.prefix == 'xmlns':
prefix = attribute.name
node.nsprefixes[prefix] = unicode(attribute.value)
skip = True
return skip
def endElement(self, name):
name = unicode(name)
current = self.top()
if len(current.charbuffer):
current.text = Text(u''.join(current.charbuffer))
del current.charbuffer
if len(current):
current.trim()
currentqname = current.qname()
if name == currentqname:
self.pop()
else:
raise Exception('malformed document')
def characters(self, content):
text = unicode(content)
node = self.top()
node.charbuffer.append(text)
def push(self, node):
self.nodes.append(node)
return node
def pop(self):
return self.nodes.pop()
def top(self):
return self.nodes[len(self.nodes)-1]
class Parser:
""" SAX Parser """
@classmethod
def saxparser(cls):
p = make_parser()
p.setFeature(feature_external_ges, 0)
h = Handler()
p.setContentHandler(h)
return (p, h)
def parse(self, file=None, string=None):
"""
SAX parse XML text.
@param file: Parse a python I{file-like} object.
@type file: I{file-like} object.
@param string: Parse string XML.
@type string: str
"""
timer = metrics.Timer()
timer.start()
sax, handler = self.saxparser()
if file is not None:
sax.parse(file)
timer.stop()
metrics.log.debug('sax (%s) duration: %s', file, timer)
return handler.nodes[0]
if string is not None:
source = InputSource(None)
source.setByteStream(StringIO(string))
sax.parse(source)
timer.stop()
metrics.log.debug('%s\nsax duration: %s', string, timer)
return handler.nodes[0]
libs/suds/sax/text.py
+116
View File
@@ -0,0 +1,116 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains XML text classes.
"""
from suds import *
from suds.sax import *
class Text(unicode):
"""
An XML text object used to represent text content.
@ivar lang: The (optional) language flag.
@type lang: bool
@ivar escaped: The (optional) XML special character escaped flag.
@type escaped: bool
"""
__slots__ = ('lang', 'escaped',)
@classmethod
def __valid(cls, *args):
return ( len(args) and args[0] is not None )
def __new__(cls, *args, **kwargs):
if cls.__valid(*args):
lang = kwargs.pop('lang', None)
escaped = kwargs.pop('escaped', False)
result = super(Text, cls).__new__(cls, *args, **kwargs)
result.lang = lang
result.escaped = escaped
else:
result = None
return result
def escape(self):
"""
Encode (escape) special XML characters.
@return: The text with XML special characters escaped.
@rtype: L{Text}
"""
if not self.escaped:
post = sax.encoder.encode(self)
escaped = ( post != self )
return Text(post, lang=self.lang, escaped=escaped)
return self
def unescape(self):
"""
Decode (unescape) special XML characters.
@return: The text with escaped XML special characters decoded.
@rtype: L{Text}
"""
if self.escaped:
post = sax.encoder.decode(self)
return Text(post, lang=self.lang)
return self
def trim(self):
post = self.strip()
return Text(post, lang=self.lang, escaped=self.escaped)
def __add__(self, other):
joined = u''.join((self, other))
result = Text(joined, lang=self.lang, escaped=self.escaped)
if isinstance(other, Text):
result.escaped = ( self.escaped or other.escaped )
return result
def __repr__(self):
s = [self]
if self.lang is not None:
s.append(' [%s]' % self.lang)
if self.escaped:
s.append(' <escaped>')
return ''.join(s)
def __getstate__(self):
state = {}
for k in self.__slots__:
state[k] = getattr(self, k)
return state
def __setstate__(self, state):
for k in self.__slots__:
setattr(self, k, state[k])
class Raw(Text):
"""
Raw text which is not XML escaped.
This may include I{string} XML.
"""
def escape(self):
return self
def unescape(self):
return self
def __add__(self, other):
joined = u''.join((self, other))
return Raw(joined, lang=self.lang)
libs/suds/servicedefinition.py
+248
View File
@@ -0,0 +1,248 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{service definition} provides a textual representation of a service.
"""
from logging import getLogger
from suds import *
import suds.metrics as metrics
from suds.sax import Namespace
log = getLogger(__name__)
class ServiceDefinition:
"""
A service definition provides an object used to generate a textual description
of a service.
@ivar wsdl: A wsdl.
@type wsdl: L{wsdl.Definitions}
@ivar service: The service object.
@type service: L{suds.wsdl.Service}
@ivar ports: A list of port-tuple: (port, [(method-name, pdef)])
@type ports: [port-tuple,..]
@ivar prefixes: A list of remapped prefixes.
@type prefixes: [(prefix,uri),..]
@ivar types: A list of type definitions
@type types: [I{Type},..]
"""
def __init__(self, wsdl, service):
"""
@param wsdl: A wsdl object
@type wsdl: L{Definitions}
@param service: A service B{name}.
@type service: str
"""
self.wsdl = wsdl
self.service = service
self.ports = []
self.params = []
self.types = []
self.prefixes = []
self.addports()
self.paramtypes()
self.publictypes()
self.getprefixes()
self.pushprefixes()
def pushprefixes(self):
"""
Add our prefixes to the wsdl so that when users invoke methods
and reference the prefixes, the will resolve properly.
"""
for ns in self.prefixes:
self.wsdl.root.addPrefix(ns[0], ns[1])
def addports(self):
"""
Look through the list of service ports and construct a list of tuples where
each tuple is used to describe a port and it's list of methods as:
(port, [method]). Each method is tuple: (name, [pdef,..] where each pdef is
a tuple: (param-name, type).
"""
timer = metrics.Timer()
timer.start()
for port in self.service.ports:
p = self.findport(port)
for op in port.binding.operations.values():
m = p[0].method(op.name)
binding = m.binding.input
method = (m.name, binding.param_defs(m))
p[1].append(method)
metrics.log.debug("method '%s' created: %s", m.name, timer)
p[1].sort()
timer.stop()
def findport(self, port):
"""
Find and return a port tuple for the specified port.
Created and added when not found.
@param port: A port.
@type port: I{service.Port}
@return: A port tuple.
@rtype: (port, [method])
"""
for p in self.ports:
if p[0] == p: return p
p = (port, [])
self.ports.append(p)
return p
def getprefixes(self):
"""
Add prefixes foreach namespace referenced by parameter types.
"""
namespaces = []
for l in (self.params, self.types):
for t,r in l:
ns = r.namespace()
if ns[1] is None: continue
if ns[1] in namespaces: continue
if Namespace.xs(ns) or Namespace.xsd(ns):
continue
namespaces.append(ns[1])
if t == r: continue
ns = t.namespace()
if ns[1] is None: continue
if ns[1] in namespaces: continue
namespaces.append(ns[1])
i = 0
namespaces.sort()
for u in namespaces:
p = self.nextprefix()
ns = (p, u)
self.prefixes.append(ns)
def paramtypes(self):
""" get all parameter types """
for m in [p[1] for p in self.ports]:
for p in [p[1] for p in m]:
for pd in p:
if pd[1] in self.params: continue
item = (pd[1], pd[1].resolve())
self.params.append(item)
def publictypes(self):
""" get all public types """
for t in self.wsdl.schema.types.values():
if t in self.params: continue
if t in self.types: continue
item = (t, t)
self.types.append(item)
tc = lambda x,y: cmp(x[0].name, y[0].name)
self.types.sort(cmp=tc)
def nextprefix(self):
"""
Get the next available prefix. This means a prefix starting with 'ns' with
a number appended as (ns0, ns1, ..) that is not already defined on the
wsdl document.
"""
used = [ns[0] for ns in self.prefixes]
used += [ns[0] for ns in self.wsdl.root.nsprefixes.items()]
for n in range(0,1024):
p = 'ns%d'%n
if p not in used:
return p
raise Exception('prefixes exhausted')
def getprefix(self, u):
"""
Get the prefix for the specified namespace (uri)
@param u: A namespace uri.
@type u: str
@return: The namspace.
@rtype: (prefix, uri).
"""
for ns in Namespace.all:
if u == ns[1]: return ns[0]
for ns in self.prefixes:
if u == ns[1]: return ns[0]
raise Exception('ns (%s) not mapped' % u)
def xlate(self, type):
"""
Get a (namespace) translated I{qualified} name for specified type.
@param type: A schema type.
@type type: I{suds.xsd.sxbasic.SchemaObject}
@return: A translated I{qualified} name.
@rtype: str
"""
resolved = type.resolve()
name = resolved.name
if type.unbounded():
name += '[]'
ns = resolved.namespace()
if ns[1] == self.wsdl.tns[1]:
return name
prefix = self.getprefix(ns[1])
return ':'.join((prefix, name))
def description(self):
"""
Get a textual description of the service for which this object represents.
@return: A textual description.
@rtype: str
"""
s = []
indent = (lambda n : '\n%*s'%(n*3,' '))
s.append('Service ( %s ) tns="%s"' % (self.service.name, self.wsdl.tns[1]))
s.append(indent(1))
s.append('Prefixes (%d)' % len(self.prefixes))
for p in self.prefixes:
s.append(indent(2))
s.append('%s = "%s"' % p)
s.append(indent(1))
s.append('Ports (%d):' % len(self.ports))
for p in self.ports:
s.append(indent(2))
s.append('(%s)' % p[0].name)
s.append(indent(3))
s.append('Methods (%d):' % len(p[1]))
for m in p[1]:
sig = []
s.append(indent(4))
sig.append(m[0])
sig.append('(')
for p in m[1]:
sig.append(self.xlate(p[1]))
sig.append(' ')
sig.append(p[0])
sig.append(', ')
sig.append(')')
try:
s.append(''.join(sig))
except:
pass
s.append(indent(3))
s.append('Types (%d):' % len(self.types))
for t in self.types:
s.append(indent(4))
s.append(self.xlate(t[0]))
s.append('\n\n')
return ''.join(s)
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
try:
return self.description()
except Exception, e:
log.exception(e)
return tostr(e)
libs/suds/serviceproxy.py
+86
View File
@@ -0,0 +1,86 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The service proxy provides access to web services.
Replaced by: L{client.Client}
"""
from logging import getLogger
from suds import *
from suds.client import Client
log = getLogger(__name__)
class ServiceProxy(object):
"""
A lightweight soap based web service proxy.
@ivar __client__: A client.
Everything is delegated to the 2nd generation API.
@type __client__: L{Client}
@note: Deprecated, replaced by L{Client}.
"""
def __init__(self, url, **kwargs):
"""
@param url: The URL for the WSDL.
@type url: str
@param kwargs: keyword arguments.
@keyword faults: Raise faults raised by server (default:True),
else return tuple from service method invocation as (http code, object).
@type faults: boolean
@keyword proxy: An http proxy to be specified on requests (default:{}).
The proxy is defined as {protocol:proxy,}
@type proxy: dict
"""
client = Client(url, **kwargs)
self.__client__ = client
def get_instance(self, name):
"""
Get an instance of a WSDL type by name
@param name: The name of a type defined in the WSDL.
@type name: str
@return: An instance on success, else None
@rtype: L{sudsobject.Object}
"""
return self.__client__.factory.create(name)
def get_enum(self, name):
"""
Get an instance of an enumeration defined in the WSDL by name.
@param name: The name of a enumeration defined in the WSDL.
@type name: str
@return: An instance on success, else None
@rtype: L{sudsobject.Object}
"""
return self.__client__.factory.create(name)
def __str__(self):
return str(self.__client__)
def __unicode__(self):
return unicode(self.__client__)
def __getattr__(self, name):
builtin = name.startswith('__') and name.endswith('__')
if builtin:
return self.__dict__[name]
else:
return getattr(self.__client__.service, name)
libs/suds/soaparray.py
+72
View File
@@ -0,0 +1,72 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{soaparray} module provides XSD extensions for handling
soap (section 5) encoded arrays.
"""
from suds import *
from logging import getLogger
from suds.xsd.sxbasic import Factory as SXFactory
from suds.xsd.sxbasic import Attribute as SXAttribute
class Attribute(SXAttribute):
"""
Represents an XSD <attribute/> that handles special
attributes that are extensions for WSDLs.
@ivar aty: Array type information.
@type aty: The value of wsdl:arrayType.
"""
def __init__(self, schema, root, aty):
"""
@param aty: Array type information.
@type aty: The value of wsdl:arrayType.
"""
SXAttribute.__init__(self, schema, root)
if aty.endswith('[]'):
self.aty = aty[:-2]
else:
self.aty = aty
def autoqualified(self):
aqs = SXAttribute.autoqualified(self)
aqs.append('aty')
return aqs
def description(self):
d = SXAttribute.description(self)
d = d+('aty',)
return d
#
# Builder function, only builds Attribute when arrayType
# attribute is defined on root.
#
def __fn(x, y):
ns = (None, "http://schemas.xmlsoap.org/wsdl/")
aty = y.get('arrayType', ns=ns)
if aty is None:
return SXAttribute(x, y)
else:
return Attribute(x, y, aty)
#
# Remap <xs:attrbute/> tags to __fn() builder.
#
SXFactory.maptag('attribute', __fn)
libs/suds/store.py
+594
View File
@@ -0,0 +1,594 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains XML text for documents to be distributed
with the suds lib. Also, contains classes for accessing
these documents.
"""
from StringIO import StringIO
from logging import getLogger
log = getLogger(__name__)
#
# Soap section 5 encoding schema.
#
encoding = \
"""<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:tns="http://schemas.xmlsoap.org/soap/encoding/" targetNamespace="http://schemas.xmlsoap.org/soap/encoding/">
<xs:attribute name="root">
<xs:annotation>
<xs:documentation>
'root' can be used to distinguish serialization roots from other
elements that are present in a serialization but are not roots of
a serialized value graph
</xs:documentation>
</xs:annotation>
<xs:simpleType>
<xs:restriction base="xs:boolean">
<xs:pattern value="0|1"/>
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attributeGroup name="commonAttributes">
<xs:annotation>
<xs:documentation>
Attributes common to all elements that function as accessors or
represent independent (multi-ref) values. The href attribute is
intended to be used in a manner like CONREF. That is, the element
content should be empty iff the href attribute appears
</xs:documentation>
</xs:annotation>
<xs:attribute name="id" type="xs:ID"/>
<xs:attribute name="href" type="xs:anyURI"/>
<xs:anyAttribute namespace="##other" processContents="lax"/>
</xs:attributeGroup>
<!-- Global Attributes. The following attributes are intended to be usable via qualified attribute names on any complex type referencing them. -->
<!-- Array attributes. Needed to give the type and dimensions of an array's contents, and the offset for partially-transmitted arrays. -->
<xs:simpleType name="arrayCoordinate">
<xs:restriction base="xs:string"/>
</xs:simpleType>
<xs:attribute name="arrayType" type="xs:string"/>
<xs:attribute name="offset" type="tns:arrayCoordinate"/>
<xs:attributeGroup name="arrayAttributes">
<xs:attribute ref="tns:arrayType"/>
<xs:attribute ref="tns:offset"/>
</xs:attributeGroup>
<xs:attribute name="position" type="tns:arrayCoordinate"/>
<xs:attributeGroup name="arrayMemberAttributes">
<xs:attribute ref="tns:position"/>
</xs:attributeGroup>
<xs:group name="Array">
<xs:sequence>
<xs:any namespace="##any" minOccurs="0" maxOccurs="unbounded" processContents="lax"/>
</xs:sequence>
</xs:group>
<xs:element name="Array" type="tns:Array"/>
<xs:complexType name="Array">
<xs:annotation>
<xs:documentation>
'Array' is a complex type for accessors identified by position
</xs:documentation>
</xs:annotation>
<xs:group ref="tns:Array" minOccurs="0"/>
<xs:attributeGroup ref="tns:arrayAttributes"/>
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:complexType>
<!-- 'Struct' is a complex type for accessors identified by name.
Constraint: No element may be have the same name as any other,
nor may any element have a maxOccurs > 1. -->
<xs:element name="Struct" type="tns:Struct"/>
<xs:group name="Struct">
<xs:sequence>
<xs:any namespace="##any" minOccurs="0" maxOccurs="unbounded" processContents="lax"/>
</xs:sequence>
</xs:group>
<xs:complexType name="Struct">
<xs:group ref="tns:Struct" minOccurs="0"/>
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:complexType>
<!-- 'Base64' can be used to serialize binary data using base64 encoding
as defined in RFC2045 but without the MIME line length limitation. -->
<xs:simpleType name="base64">
<xs:restriction base="xs:base64Binary"/>
</xs:simpleType>
<!-- Element declarations corresponding to each of the simple types in the
XML Schemas Specification. -->
<xs:element name="duration" type="tns:duration"/>
<xs:complexType name="duration">
<xs:simpleContent>
<xs:extension base="xs:duration">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="dateTime" type="tns:dateTime"/>
<xs:complexType name="dateTime">
<xs:simpleContent>
<xs:extension base="xs:dateTime">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="NOTATION" type="tns:NOTATION"/>
<xs:complexType name="NOTATION">
<xs:simpleContent>
<xs:extension base="xs:QName">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="time" type="tns:time"/>
<xs:complexType name="time">
<xs:simpleContent>
<xs:extension base="xs:time">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="date" type="tns:date"/>
<xs:complexType name="date">
<xs:simpleContent>
<xs:extension base="xs:date">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="gYearMonth" type="tns:gYearMonth"/>
<xs:complexType name="gYearMonth">
<xs:simpleContent>
<xs:extension base="xs:gYearMonth">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="gYear" type="tns:gYear"/>
<xs:complexType name="gYear">
<xs:simpleContent>
<xs:extension base="xs:gYear">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="gMonthDay" type="tns:gMonthDay"/>
<xs:complexType name="gMonthDay">
<xs:simpleContent>
<xs:extension base="xs:gMonthDay">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="gDay" type="tns:gDay"/>
<xs:complexType name="gDay">
<xs:simpleContent>
<xs:extension base="xs:gDay">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="gMonth" type="tns:gMonth"/>
<xs:complexType name="gMonth">
<xs:simpleContent>
<xs:extension base="xs:gMonth">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="boolean" type="tns:boolean"/>
<xs:complexType name="boolean">
<xs:simpleContent>
<xs:extension base="xs:boolean">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="base64Binary" type="tns:base64Binary"/>
<xs:complexType name="base64Binary">
<xs:simpleContent>
<xs:extension base="xs:base64Binary">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="hexBinary" type="tns:hexBinary"/>
<xs:complexType name="hexBinary">
<xs:simpleContent>
<xs:extension base="xs:hexBinary">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="float" type="tns:float"/>
<xs:complexType name="float">
<xs:simpleContent>
<xs:extension base="xs:float">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="double" type="tns:double"/>
<xs:complexType name="double">
<xs:simpleContent>
<xs:extension base="xs:double">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="anyURI" type="tns:anyURI"/>
<xs:complexType name="anyURI">
<xs:simpleContent>
<xs:extension base="xs:anyURI">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="QName" type="tns:QName"/>
<xs:complexType name="QName">
<xs:simpleContent>
<xs:extension base="xs:QName">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="string" type="tns:string"/>
<xs:complexType name="string">
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="normalizedString" type="tns:normalizedString"/>
<xs:complexType name="normalizedString">
<xs:simpleContent>
<xs:extension base="xs:normalizedString">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="token" type="tns:token"/>
<xs:complexType name="token">
<xs:simpleContent>
<xs:extension base="xs:token">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="language" type="tns:language"/>
<xs:complexType name="language">
<xs:simpleContent>
<xs:extension base="xs:language">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="Name" type="tns:Name"/>
<xs:complexType name="Name">
<xs:simpleContent>
<xs:extension base="xs:Name">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="NMTOKEN" type="tns:NMTOKEN"/>
<xs:complexType name="NMTOKEN">
<xs:simpleContent>
<xs:extension base="xs:NMTOKEN">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="NCName" type="tns:NCName"/>
<xs:complexType name="NCName">
<xs:simpleContent>
<xs:extension base="xs:NCName">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="NMTOKENS" type="tns:NMTOKENS"/>
<xs:complexType name="NMTOKENS">
<xs:simpleContent>
<xs:extension base="xs:NMTOKENS">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="ID" type="tns:ID"/>
<xs:complexType name="ID">
<xs:simpleContent>
<xs:extension base="xs:ID">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="IDREF" type="tns:IDREF"/>
<xs:complexType name="IDREF">
<xs:simpleContent>
<xs:extension base="xs:IDREF">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="ENTITY" type="tns:ENTITY"/>
<xs:complexType name="ENTITY">
<xs:simpleContent>
<xs:extension base="xs:ENTITY">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="IDREFS" type="tns:IDREFS"/>
<xs:complexType name="IDREFS">
<xs:simpleContent>
<xs:extension base="xs:IDREFS">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="ENTITIES" type="tns:ENTITIES"/>
<xs:complexType name="ENTITIES">
<xs:simpleContent>
<xs:extension base="xs:ENTITIES">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="decimal" type="tns:decimal"/>
<xs:complexType name="decimal">
<xs:simpleContent>
<xs:extension base="xs:decimal">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="integer" type="tns:integer"/>
<xs:complexType name="integer">
<xs:simpleContent>
<xs:extension base="xs:integer">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="nonPositiveInteger" type="tns:nonPositiveInteger"/>
<xs:complexType name="nonPositiveInteger">
<xs:simpleContent>
<xs:extension base="xs:nonPositiveInteger">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="negativeInteger" type="tns:negativeInteger"/>
<xs:complexType name="negativeInteger">
<xs:simpleContent>
<xs:extension base="xs:negativeInteger">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="long" type="tns:long"/>
<xs:complexType name="long">
<xs:simpleContent>
<xs:extension base="xs:long">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="int" type="tns:int"/>
<xs:complexType name="int">
<xs:simpleContent>
<xs:extension base="xs:int">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="short" type="tns:short"/>
<xs:complexType name="short">
<xs:simpleContent>
<xs:extension base="xs:short">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="byte" type="tns:byte"/>
<xs:complexType name="byte">
<xs:simpleContent>
<xs:extension base="xs:byte">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="nonNegativeInteger" type="tns:nonNegativeInteger"/>
<xs:complexType name="nonNegativeInteger">
<xs:simpleContent>
<xs:extension base="xs:nonNegativeInteger">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="unsignedLong" type="tns:unsignedLong"/>
<xs:complexType name="unsignedLong">
<xs:simpleContent>
<xs:extension base="xs:unsignedLong">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="unsignedInt" type="tns:unsignedInt"/>
<xs:complexType name="unsignedInt">
<xs:simpleContent>
<xs:extension base="xs:unsignedInt">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="unsignedShort" type="tns:unsignedShort"/>
<xs:complexType name="unsignedShort">
<xs:simpleContent>
<xs:extension base="xs:unsignedShort">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="unsignedByte" type="tns:unsignedByte"/>
<xs:complexType name="unsignedByte">
<xs:simpleContent>
<xs:extension base="xs:unsignedByte">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="positiveInteger" type="tns:positiveInteger"/>
<xs:complexType name="positiveInteger">
<xs:simpleContent>
<xs:extension base="xs:positiveInteger">
<xs:attributeGroup ref="tns:commonAttributes"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="anyType"/>
</xs:schema>
"""
class DocumentStore:
"""
The I{suds} document store provides a local repository
for xml documnts.
@cvar protocol: The URL protocol for the store.
@type protocol: str
@cvar store: The mapping of URL location to documents.
@type store: dict
"""
protocol = 'suds'
store = {
'schemas.xmlsoap.org/soap/encoding/' : encoding
}
def open(self, url):
"""
Open a document at the specified url.
@param url: A document URL.
@type url: str
@return: A file pointer to the document.
@rtype: StringIO
"""
protocol, location = self.split(url)
if protocol == self.protocol:
return self.find(location)
else:
return None
def find(self, location):
"""
Find the specified location in the store.
@param location: The I{location} part of a URL.
@type location: str
@return: An input stream to the document.
@rtype: StringIO
"""
try:
content = self.store[location]
return StringIO(content)
except:
reason = 'location "%s" not in document store' % location
raise Exception, reason
def split(self, url):
"""
Split the url into I{protocol} and I{location}
@param url: A URL.
@param url: str
@return: (I{url}, I{location})
@rtype: tuple
"""
parts = url.split('://', 1)
if len(parts) == 2:
return parts
else:
return (None, url)
libs/suds/sudsobject.py
+390
View File
@@ -0,0 +1,390 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{sudsobject} module provides a collection of suds objects
that are primarily used for the highly dynamic interactions with
wsdl/xsd defined types.
"""
from logging import getLogger
from suds import *
from new import classobj
log = getLogger(__name__)
def items(sobject):
"""
Extract the I{items} from a suds object much like the
items() method works on I{dict}.
@param sobject: A suds object
@type sobject: L{Object}
@return: A list of items contained in I{sobject}.
@rtype: [(key, value),...]
"""
for item in sobject:
yield item
def asdict(sobject):
"""
Convert a sudsobject into a dictionary.
@param sobject: A suds object
@type sobject: L{Object}
@return: A python dictionary containing the
items contained in I{sobject}.
@rtype: dict
"""
return dict(items(sobject))
def merge(a, b):
"""
Merge all attributes and metadata from I{a} to I{b}.
@param a: A I{source} object
@type a: L{Object}
@param b: A I{destination} object
@type b: L{Object}
"""
for item in a:
setattr(b, item[0], item[1])
b.__metadata__ = b.__metadata__
return b
def footprint(sobject):
"""
Get the I{virtual footprint} of the object.
This is really a count of the attributes in the branch with a significant value.
@param sobject: A suds object.
@type sobject: L{Object}
@return: The branch footprint.
@rtype: int
"""
n = 0
for a in sobject.__keylist__:
v = getattr(sobject, a)
if v is None: continue
if isinstance(v, Object):
n += footprint(v)
continue
if hasattr(v, '__len__'):
if len(v): n += 1
continue
n +=1
return n
class Factory:
cache = {}
@classmethod
def subclass(cls, name, bases, dict={}):
if not isinstance(bases, tuple):
bases = (bases,)
name = name.encode('utf-8')
key = '.'.join((name, str(bases)))
subclass = cls.cache.get(key)
if subclass is None:
subclass = classobj(name, bases, dict)
cls.cache[key] = subclass
return subclass
@classmethod
def object(cls, classname=None, dict={}):
if classname is not None:
subclass = cls.subclass(classname, Object)
inst = subclass()
else:
inst = Object()
for a in dict.items():
setattr(inst, a[0], a[1])
return inst
@classmethod
def metadata(cls):
return Metadata()
@classmethod
def property(cls, name, value=None):
subclass = cls.subclass(name, Property)
return subclass(value)
class Object:
def __init__(self):
self.__keylist__ = []
self.__printer__ = Printer()
self.__metadata__ = Metadata()
def __setattr__(self, name, value):
builtin = name.startswith('__') and name.endswith('__')
if not builtin and \
name not in self.__keylist__:
self.__keylist__.append(name)
self.__dict__[name] = value
def __delattr__(self, name):
try:
del self.__dict__[name]
builtin = name.startswith('__') and name.endswith('__')
if not builtin:
self.__keylist__.remove(name)
except:
cls = self.__class__.__name__
raise AttributeError, "%s has no attribute '%s'" % (cls, name)
def __getitem__(self, name):
if isinstance(name, int):
name = self.__keylist__[int(name)]
return getattr(self, name)
def __setitem__(self, name, value):
setattr(self, name, value)
def __iter__(self):
return Iter(self)
def __len__(self):
return len(self.__keylist__)
def __contains__(self, name):
return name in self.__keylist__
def __repr__(self):
return str(self)
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
return self.__printer__.tostr(self)
class Iter:
def __init__(self, sobject):
self.sobject = sobject
self.keylist = self.__keylist(sobject)
self.index = 0
def next(self):
keylist = self.keylist
nkeys = len(self.keylist)
while self.index < nkeys:
k = keylist[self.index]
self.index += 1
if hasattr(self.sobject, k):
v = getattr(self.sobject, k)
return (k, v)
raise StopIteration()
def __keylist(self, sobject):
keylist = sobject.__keylist__
try:
keyset = set(keylist)
ordering = sobject.__metadata__.ordering
ordered = set(ordering)
if not ordered.issuperset(keyset):
log.debug(
'%s must be superset of %s, ordering ignored',
keylist,
ordering)
raise KeyError()
return ordering
except:
return keylist
def __iter__(self):
return self
class Metadata(Object):
def __init__(self):
self.__keylist__ = []
self.__printer__ = Printer()
class Facade(Object):
def __init__(self, name):
Object.__init__(self)
md = self.__metadata__
md.facade = name
class Property(Object):
def __init__(self, value):
Object.__init__(self)
self.value = value
def items(self):
for item in self:
if item[0] != 'value':
yield item
def get(self):
return self.value
def set(self, value):
self.value = value
return self
class Printer:
"""
Pretty printing of a Object object.
"""
@classmethod
def indent(cls, n): return '%*s'%(n*3,' ')
def tostr(self, object, indent=-2):
""" get s string representation of object """
history = []
return self.process(object, history, indent)
def process(self, object, h, n=0, nl=False):
""" print object using the specified indent (n) and newline (nl). """
if object is None:
return 'None'
if isinstance(object, Object):
if len(object) == 0:
return '<empty>'
else:
return self.print_object(object, h, n+2, nl)
if isinstance(object, dict):
if len(object) == 0:
return '<empty>'
else:
return self.print_dictionary(object, h, n+2, nl)
if isinstance(object, (list,tuple)):
if len(object) == 0:
return '<empty>'
else:
return self.print_collection(object, h, n+2)
if isinstance(object, basestring):
return '"%s"' % tostr(object)
return '%s' % tostr(object)
def print_object(self, d, h, n, nl=False):
""" print complex using the specified indent (n) and newline (nl). """
s = []
cls = d.__class__
md = d.__metadata__
if d in h:
s.append('(')
s.append(cls.__name__)
s.append(')')
s.append('...')
return ''.join(s)
h.append(d)
if nl:
s.append('\n')
s.append(self.indent(n))
if cls != Object:
s.append('(')
if isinstance(d, Facade):
s.append(md.facade)
else:
s.append(cls.__name__)
s.append(')')
s.append('{')
for item in d:
if self.exclude(d, item):
continue
item = self.unwrap(d, item)
s.append('\n')
s.append(self.indent(n+1))
if isinstance(item[1], (list,tuple)):
s.append(item[0])
s.append('[]')
else:
s.append(item[0])
s.append(' = ')
s.append(self.process(item[1], h, n, True))
s.append('\n')
s.append(self.indent(n))
s.append('}')
h.pop()
return ''.join(s)
def print_dictionary(self, d, h, n, nl=False):
""" print complex using the specified indent (n) and newline (nl). """
if d in h: return '{}...'
h.append(d)
s = []
if nl:
s.append('\n')
s.append(self.indent(n))
s.append('{')
for item in d.items():
s.append('\n')
s.append(self.indent(n+1))
if isinstance(item[1], (list,tuple)):
s.append(tostr(item[0]))
s.append('[]')
else:
s.append(tostr(item[0]))
s.append(' = ')
s.append(self.process(item[1], h, n, True))
s.append('\n')
s.append(self.indent(n))
s.append('}')
h.pop()
return ''.join(s)
def print_collection(self, c, h, n):
""" print collection using the specified indent (n) and newline (nl). """
if c in h: return '[]...'
h.append(c)
s = []
for item in c:
s.append('\n')
s.append(self.indent(n))
s.append(self.process(item, h, n-2))
s.append(',')
h.pop()
return ''.join(s)
def unwrap(self, d, item):
""" translate (unwrap) using an optional wrapper function """
nopt = ( lambda x: x )
try:
md = d.__metadata__
pmd = getattr(md, '__print__', None)
if pmd is None:
return item
wrappers = getattr(pmd, 'wrappers', {})
fn = wrappers.get(item[0], nopt)
return (item[0], fn(item[1]))
except:
pass
return item
def exclude(self, d, item):
""" check metadata for excluded items """
try:
md = d.__metadata__
pmd = getattr(md, '__print__', None)
if pmd is None:
return False
excludes = getattr(pmd, 'excludes', [])
return ( item[0] in excludes )
except:
pass
return False
libs/suds/transport/__init__.py
+130
View File
@@ -0,0 +1,130 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains transport interface (classes).
"""
class TransportError(Exception):
def __init__(self, reason, httpcode, fp=None):
Exception.__init__(self, reason)
self.httpcode = httpcode
self.fp = fp
class Request:
"""
A transport request
@ivar url: The url for the request.
@type url: str
@ivar message: The message to be sent in a POST request.
@type message: str
@ivar headers: The http headers to be used for the request.
@type headers: dict
"""
def __init__(self, url, message=None):
"""
@param url: The url for the request.
@type url: str
@param message: The (optional) message to be send in the request.
@type message: str
"""
self.url = url
self.headers = {}
self.message = message
def __str__(self):
s = []
s.append('URL:%s' % self.url)
s.append('HEADERS: %s' % self.headers)
s.append('MESSAGE:')
s.append(self.message)
return '\n'.join(s)
class Reply:
"""
A transport reply
@ivar code: The http code returned.
@type code: int
@ivar message: The message to be sent in a POST request.
@type message: str
@ivar headers: The http headers to be used for the request.
@type headers: dict
"""
def __init__(self, code, headers, message):
"""
@param code: The http code returned.
@type code: int
@param headers: The http returned headers.
@type headers: dict
@param message: The (optional) reply message received.
@type message: str
"""
self.code = code
self.headers = headers
self.message = message
def __str__(self):
s = []
s.append('CODE: %s' % self.code)
s.append('HEADERS: %s' % self.headers)
s.append('MESSAGE:')
s.append(self.message)
return '\n'.join(s)
class Transport:
"""
The transport I{interface}.
"""
def __init__(self):
"""
Constructor.
"""
from suds.transport.options import Options
self.options = Options()
del Options
def open(self, request):
"""
Open the url in the specified request.
@param request: A transport request.
@type request: L{Request}
@return: An input stream.
@rtype: stream
@raise TransportError: On all transport errors.
"""
raise Exception('not-implemented')
def send(self, request):
"""
Send soap message. Implementations are expected to handle:
- proxies
- I{http} headers
- cookies
- sending message
- brokering exceptions into L{TransportError}
@param request: A transport request.
@type request: L{Request}
@return: The reply
@rtype: L{Reply}
@raise TransportError: On all transport errors.
"""
raise Exception('not-implemented')
libs/suds/transport/http.py
+187
View File
@@ -0,0 +1,187 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains classes for basic HTTP transport implementations.
"""
import urllib2 as u2
import base64
import socket
from suds.transport import *
from suds.properties import Unskin
from urlparse import urlparse
from cookielib import CookieJar
from logging import getLogger
log = getLogger(__name__)
class HttpTransport(Transport):
"""
HTTP transport using urllib2. Provided basic http transport
that provides for cookies, proxies but no authentication.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
"""
Transport.__init__(self)
Unskin(self.options).update(kwargs)
self.cookiejar = CookieJar()
self.proxy = {}
self.urlopener = None
def open(self, request):
try:
url = request.url
log.debug('opening (%s)', url)
u2request = u2.Request(url)
self.proxy = self.options.proxy
return self.u2open(u2request)
except u2.HTTPError, e:
raise TransportError(str(e), e.code, e.fp)
def send(self, request):
result = None
url = request.url
msg = request.message
headers = request.headers
try:
u2request = u2.Request(url, msg, headers)
self.addcookies(u2request)
self.proxy = self.options.proxy
request.headers.update(u2request.headers)
log.debug('sending:\n%s', request)
fp = self.u2open(u2request)
self.getcookies(fp, u2request)
result = Reply(200, fp.headers.dict, fp.read())
log.debug('received:\n%s', result)
except u2.HTTPError, e:
if e.code in (202,204):
result = None
else:
raise TransportError(e.msg, e.code, e.fp)
return result
def addcookies(self, u2request):
"""
Add cookies in the cookiejar to the request.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Requet.
"""
self.cookiejar.add_cookie_header(u2request)
def getcookies(self, fp, u2request):
"""
Add cookies in the request to the cookiejar.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Requet.
"""
self.cookiejar.extract_cookies(fp, u2request)
def u2open(self, u2request):
"""
Open a connection.
@param u2request: A urllib2 request.
@type u2request: urllib2.Requet.
@return: The opened file-like urllib2 object.
@rtype: fp
"""
tm = self.options.timeout
url = self.u2opener()
if self.u2ver() < 2.6:
socket.setdefaulttimeout(tm)
return url.open(u2request)
else:
return url.open(u2request, timeout=tm)
def u2opener(self):
"""
Create a urllib opener.
@return: An opener.
@rtype: I{OpenerDirector}
"""
if self.urlopener is None:
return u2.build_opener(*self.u2handlers())
else:
return self.urlopener
def u2handlers(self):
"""
Get a collection of urllib handlers.
@return: A list of handlers to be installed in the opener.
@rtype: [Handler,...]
"""
handlers = []
handlers.append(u2.ProxyHandler(self.proxy))
return handlers
def u2ver(self):
"""
Get the major/minor version of the urllib2 lib.
@return: The urllib2 version.
@rtype: float
"""
try:
part = u2.__version__.split('.', 1)
n = float('.'.join(part))
return n
except Exception, e:
log.exception(e)
return 0
def __deepcopy__(self, memo={}):
clone = self.__class__()
p = Unskin(self.options)
cp = Unskin(clone.options)
cp.update(p)
return clone
class HttpAuthenticated(HttpTransport):
"""
Provides basic http authentication for servers that don't follow
the specified challenge / response model. This implementation
appends the I{Authorization} http header with base64 encoded
credentials on every http request.
"""
def open(self, request):
self.addcredentials(request)
return HttpTransport.open(self, request)
def send(self, request):
self.addcredentials(request)
return HttpTransport.send(self, request)
def addcredentials(self, request):
credentials = self.credentials()
if not (None in credentials):
encoded = base64.encodestring(':'.join(credentials))
basic = 'Basic %s' % encoded[:-1]
request.headers['Authorization'] = basic
def credentials(self):
return (self.options.username, self.options.password)
libs/suds/transport/https.py
+98
View File
@@ -0,0 +1,98 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains classes for basic HTTP (authenticated) transport implementations.
"""
import urllib2 as u2
from suds.transport import *
from suds.transport.http import HttpTransport
from logging import getLogger
log = getLogger(__name__)
class HttpAuthenticated(HttpTransport):
"""
Provides basic http authentication that follows the RFC-2617 specification.
As defined by specifications, credentials are provided to the server
upon request (HTTP/1.0 401 Authorization Required) by the server only.
@ivar pm: The password manager.
@ivar handler: The authentication handler.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
"""
HttpTransport.__init__(self, **kwargs)
self.pm = u2.HTTPPasswordMgrWithDefaultRealm()
def open(self, request):
self.addcredentials(request)
return HttpTransport.open(self, request)
def send(self, request):
self.addcredentials(request)
return HttpTransport.send(self, request)
def addcredentials(self, request):
credentials = self.credentials()
if not (None in credentials):
u = credentials[0]
p = credentials[1]
self.pm.add_password(None, request.url, u, p)
def credentials(self):
return (self.options.username, self.options.password)
def u2handlers(self):
handlers = HttpTransport.u2handlers(self)
handlers.append(u2.HTTPBasicAuthHandler(self.pm))
return handlers
class WindowsHttpAuthenticated(HttpAuthenticated):
"""
Provides Windows (NTLM) http authentication.
@ivar pm: The password manager.
@ivar handler: The authentication handler.
@author: Christopher Bess
"""
def u2handlers(self):
# try to import ntlm support
try:
from ntlm import HTTPNtlmAuthHandler
except ImportError:
raise Exception("Cannot import python-ntlm module")
handlers = HttpTransport.u2handlers(self)
handlers.append(HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(self.pm))
return handlers
libs/suds/transport/options.py
+57
View File
@@ -0,0 +1,57 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Contains classes for transport options.
"""
from suds.transport import *
from suds.properties import *
class Options(Skin):
"""
Options:
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
- B{headers} - Extra HTTP headers.
- type: I{dict}
- I{str} B{http} - The I{http} protocol proxy URL.
- I{str} B{https} - The I{https} protocol proxy URL.
- default: {}
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
"""
def __init__(self, **kwargs):
domain = __name__
definitions = [
Definition('proxy', dict, {}),
Definition('timeout', (int,float), 90),
Definition('headers', dict, {}),
Definition('username', basestring, None),
Definition('password', basestring, None),
]
Skin.__init__(self, domain, definitions, kwargs)
libs/suds/umx/__init__.py
+56
View File
@@ -0,0 +1,56 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides modules containing classes to support
unmarshalling (XML).
"""
from suds.sudsobject import Object
class Content(Object):
"""
@ivar node: The content source node.
@type node: L{sax.element.Element}
@ivar data: The (optional) content data.
@type data: L{Object}
@ivar text: The (optional) content (xml) text.
@type text: basestring
"""
extensions = []
def __init__(self, node, **kwargs):
Object.__init__(self)
self.node = node
self.data = None
self.text = None
for k,v in kwargs.items():
setattr(self, k, v)
def __getattr__(self, name):
if name not in self.__dict__:
if name in self.extensions:
v = None
setattr(self, name, v)
else:
raise AttributeError, \
'Content has no attribute %s' % name
else:
v = self.__dict__[name]
return v
libs/suds/umx/attrlist.py
+88
View File
@@ -0,0 +1,88 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides filtered attribute list classes.
"""
from suds import *
from suds.umx import *
from suds.sax import Namespace
class AttrList:
"""
A filtered attribute list.
Items are included during iteration if they are in either the (xs) or
(xml) namespaces.
@ivar raw: The I{raw} attribute list.
@type raw: list
"""
def __init__(self, attributes):
"""
@param attributes: A list of attributes
@type attributes: list
"""
self.raw = attributes
def real(self):
"""
Get list of I{real} attributes which exclude xs and xml attributes.
@return: A list of I{real} attributes.
@rtype: I{generator}
"""
for a in self.raw:
if self.skip(a): continue
yield a
def rlen(self):
"""
Get the number of I{real} attributes which exclude xs and xml attributes.
@return: A count of I{real} attributes.
@rtype: L{int}
"""
n = 0
for a in self.real():
n += 1
return n
def lang(self):
"""
Get list of I{filtered} attributes which exclude xs.
@return: A list of I{filtered} attributes.
@rtype: I{generator}
"""
for a in self.raw:
if a.qname() == 'xml:lang':
return a.value
return None
def skip(self, attr):
"""
Get whether to skip (filter-out) the specified attribute.
@param attr: An attribute.
@type attr: I{Attribute}
@return: True if should be skipped.
@rtype: bool
"""
ns = attr.namespace()
skip = (
Namespace.xmlns[1],
'http://schemas.xmlsoap.org/soap/encoding/',
'http://schemas.xmlsoap.org/soap/envelope/',
'http://www.w3.org/2003/05/soap-envelope',
)
return ( Namespace.xs(ns) or ns[1] in skip )
libs/suds/umx/basic.py
+41
View File
@@ -0,0 +1,41 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides basic unmarshaller classes.
"""
from logging import getLogger
from suds import *
from suds.umx import *
from suds.umx.core import Core
class Basic(Core):
"""
A object builder (unmarshaller).
"""
def process(self, node):
"""
Process an object graph representation of the xml I{node}.
@param node: An XML tree.
@type node: L{sax.element.Element}
@return: A suds object.
@rtype: L{Object}
"""
content = Content(node)
return Core.process(self, content)
libs/suds/umx/core.py
+216
View File
@@ -0,0 +1,216 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides base classes for XML->object I{unmarshalling}.
"""
from logging import getLogger
from suds import *
from suds.umx import *
from suds.umx.attrlist import AttrList
from suds.sax.text import Text
from suds.sudsobject import Factory, merge
log = getLogger(__name__)
reserved = { 'class':'cls', 'def':'dfn', }
class Core:
"""
The abstract XML I{node} unmarshaller. This class provides the
I{core} unmarshalling functionality.
"""
def process(self, content):
"""
Process an object graph representation of the xml I{node}.
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: A suds object.
@rtype: L{Object}
"""
self.reset()
return self.append(content)
def append(self, content):
"""
Process the specified node and convert the XML document into
a I{suds} L{object}.
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: A I{append-result} tuple as: (L{Object}, I{value})
@rtype: I{append-result}
@note: This is not the proper entry point.
@see: L{process()}
"""
self.start(content)
self.append_attributes(content)
self.append_children(content)
self.append_text(content)
self.end(content)
return self.postprocess(content)
def postprocess(self, content):
"""
Perform final processing of the resulting data structure as follows:
- Mixed values (children and text) will have a result of the I{content.node}.
- Simi-simple values (attributes, no-children and text) will have a result of a
property object.
- Simple values (no-attributes, no-children with text nodes) will have a string
result equal to the value of the content.node.getText().
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: The post-processed result.
@rtype: I{any}
"""
node = content.node
if len(node.children) and node.hasText():
return node
attributes = AttrList(node.attributes)
if attributes.rlen() and \
not len(node.children) and \
node.hasText():
p = Factory.property(node.name, node.getText())
return merge(content.data, p)
if len(content.data):
return content.data
lang = attributes.lang()
if content.node.isnil():
return None
if not len(node.children) and content.text is None:
if self.nillable(content):
return None
else:
return Text('', lang=lang)
if isinstance(content.text, basestring):
return Text(content.text, lang=lang)
else:
return content.text
def append_attributes(self, content):
"""
Append attribute nodes into L{Content.data}.
Attributes in the I{schema} or I{xml} namespaces are skipped.
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
attributes = AttrList(content.node.attributes)
for attr in attributes.real():
name = attr.name
value = attr.value
self.append_attribute(name, value, content)
def append_attribute(self, name, value, content):
"""
Append an attribute name/value into L{Content.data}.
@param name: The attribute name
@type name: basestring
@param value: The attribute's value
@type value: basestring
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
key = name
key = '_%s' % reserved.get(key, key)
setattr(content.data, key, value)
def append_children(self, content):
"""
Append child nodes into L{Content.data}
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
for child in content.node:
cont = Content(child)
cval = self.append(cont)
key = reserved.get(child.name, child.name)
if key in content.data:
v = getattr(content.data, key)
if isinstance(v, list):
v.append(cval)
else:
setattr(content.data, key, [v, cval])
continue
if self.unbounded(cont):
if cval is None:
setattr(content.data, key, [])
else:
setattr(content.data, key, [cval,])
else:
setattr(content.data, key, cval)
def append_text(self, content):
"""
Append text nodes into L{Content.data}
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
if content.node.hasText():
content.text = content.node.getText()
def reset(self):
pass
def start(self, content):
"""
Processing on I{node} has started. Build and return
the proper object.
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: A subclass of Object.
@rtype: L{Object}
"""
content.data = Factory.object(content.node.name)
def end(self, content):
"""
Processing on I{node} has ended.
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
pass
def bounded(self, content):
"""
Get whether the content is bounded (not a list).
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: True if bounded, else False
@rtype: boolean
'"""
return ( not self.unbounded(content) )
def unbounded(self, content):
"""
Get whether the object is unbounded (a list).
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: True if unbounded, else False
@rtype: boolean
'"""
return False
def nillable(self, content):
"""
Get whether the object is nillable.
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: True if nillable, else False
@rtype: boolean
'"""
return False
libs/suds/umx/encoded.py
+128
View File
@@ -0,0 +1,128 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides soap encoded unmarshaller classes.
"""
from logging import getLogger
from suds import *
from suds.umx import *
from suds.umx.typed import Typed
from suds.sax import splitPrefix, Namespace
log = getLogger(__name__)
#
# Add encoded extensions
# aty = The soap (section 5) encoded array type.
#
Content.extensions.append('aty')
class Encoded(Typed):
"""
A SOAP section (5) encoding unmarshaller.
This marshaller supports rpc/encoded soap styles.
"""
def start(self, content):
#
# Grab the array type and continue
#
self.setaty(content)
Typed.start(self, content)
def end(self, content):
#
# Squash soap encoded arrays into python lists. This is
# also where we insure that empty arrays are represented
# as empty python lists.
#
aty = content.aty
if aty is not None:
self.promote(content)
return Typed.end(self, content)
def postprocess(self, content):
#
# Ensure proper rendering of empty arrays.
#
if content.aty is None:
return Typed.postprocess(self, content)
else:
return content.data
def setaty(self, content):
"""
Grab the (aty) soap-enc:arrayType and attach it to the
content for proper array processing later in end().
@param content: The current content being unmarshalled.
@type content: L{Content}
@return: self
@rtype: L{Encoded}
"""
name = 'arrayType'
ns = (None, 'http://schemas.xmlsoap.org/soap/encoding/')
aty = content.node.get(name, ns)
if aty is not None:
content.aty = aty
parts = aty.split('[')
ref = parts[0]
if len(parts) == 2:
self.applyaty(content, ref)
else:
pass # (2) dimensional array
return self
def applyaty(self, content, xty):
"""
Apply the type referenced in the I{arrayType} to the content
(child nodes) of the array. Each element (node) in the array
that does not have an explicit xsi:type attribute is given one
based on the I{arrayType}.
@param content: An array content.
@type content: L{Content}
@param xty: The XSI type reference.
@type xty: str
@return: self
@rtype: L{Encoded}
"""
name = 'type'
ns = Namespace.xsins
parent = content.node
for child in parent.getChildren():
ref = child.get(name, ns)
if ref is None:
parent.addPrefix(ns[0], ns[1])
attr = ':'.join((ns[0], name))
child.set(attr, xty)
return self
def promote(self, content):
"""
Promote (replace) the content.data with the first attribute
of the current content.data that is a I{list}. Note: the
content.data may be empty or contain only _x attributes.
In either case, the content.data is assigned an empty list.
@param content: An array content.
@type content: L{Content}
"""
for n,v in content.data:
if isinstance(v, list):
content.data = v
return
content.data = []
libs/suds/umx/typed.py
+141
View File
@@ -0,0 +1,141 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
Provides typed unmarshaller classes.
"""
from logging import getLogger
from suds import *
from suds.umx import *
from suds.umx.core import Core
from suds.resolver import NodeResolver, Frame
from suds.sudsobject import Factory
log = getLogger(__name__)
#
# Add typed extensions
# type = The expected xsd type
# real = The 'true' XSD type
#
Content.extensions.append('type')
Content.extensions.append('real')
class Typed(Core):
"""
A I{typed} XML unmarshaller
@ivar resolver: A schema type resolver.
@type resolver: L{NodeResolver}
"""
def __init__(self, schema):
"""
@param schema: A schema object.
@type schema: L{xsd.schema.Schema}
"""
self.resolver = NodeResolver(schema)
def process(self, node, type):
"""
Process an object graph representation of the xml L{node}.
@param node: An XML tree.
@type node: L{sax.element.Element}
@param type: The I{optional} schema type.
@type type: L{xsd.sxbase.SchemaObject}
@return: A suds object.
@rtype: L{Object}
"""
content = Content(node)
content.type = type
return Core.process(self, content)
def reset(self):
log.debug('reset')
self.resolver.reset()
def start(self, content):
#
# Resolve to the schema type; build an object and setup metadata.
#
if content.type is None:
found = self.resolver.find(content.node)
if found is None:
log.error(self.resolver.schema)
raise TypeNotFound(content.node.qname())
content.type = found
else:
known = self.resolver.known(content.node)
frame = Frame(content.type, resolved=known)
self.resolver.push(frame)
real = self.resolver.top().resolved
content.real = real
cls_name = real.name
if cls_name is None:
cls_name = content.node.name
content.data = Factory.object(cls_name)
md = content.data.__metadata__
md.sxtype = real
def end(self, content):
self.resolver.pop()
def unbounded(self, content):
return content.type.unbounded()
def nillable(self, content):
resolved = content.type.resolve()
return ( content.type.nillable or \
(resolved.builtin() and resolved.nillable ) )
def append_attribute(self, name, value, content):
"""
Append an attribute name/value into L{Content.data}.
@param name: The attribute name
@type name: basestring
@param value: The attribute's value
@type value: basestring
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
type = self.resolver.findattr(name)
if type is None:
log.warn('attribute (%s) type, not-found', name)
else:
value = self.translated(value, type)
Core.append_attribute(self, name, value, content)
def append_text(self, content):
"""
Append text nodes into L{Content.data}
Here is where the I{true} type is used to translate the value
into the proper python type.
@param content: The current content being unmarshalled.
@type content: L{Content}
"""
Core.append_text(self, content)
known = self.resolver.top().resolved
content.text = self.translated(content.text, known)
def translated(self, value, type):
""" translate using the schema type """
if value is not None:
resolved = type.resolve()
return resolved.translate(value)
else:
return value
libs/suds/wsdl.py
+922
View File
@@ -0,0 +1,922 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the (LGPL) GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Lesser General Public License for more details at
# ( http://www.gnu.org/licenses/lgpl.html ).
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# written by: Jeff Ortel ( jortel@redhat.com )
"""
The I{wsdl} module provides an objectification of the WSDL.
The primary class is I{Definitions} as it represends the root element
found in the document.
"""
from logging import getLogger
from suds import *
from suds.sax import splitPrefix
from suds.sax.element import Element
from suds.bindings.document import Document
from suds.bindings.rpc import RPC, Encoded
from suds.xsd import qualify, Namespace
from suds.xsd.schema import Schema, SchemaCollection
from suds.xsd.query import ElementQuery
from suds.sudsobject import Object, Facade, Metadata
from suds.reader import DocumentReader, DefinitionsReader
from urlparse import urljoin
import re, soaparray
log = getLogger(__name__)
wsdlns = (None, "http://schemas.xmlsoap.org/wsdl/")
soapns = (None, 'http://schemas.xmlsoap.org/wsdl/soap/')
soap12ns = (None, 'http://schemas.xmlsoap.org/wsdl/soap12/')
class WObject(Object):
"""
Base object for wsdl types.
@ivar root: The XML I{root} element.
@type root: L{Element}
"""
def __init__(self, root, definitions=None):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
Object.__init__(self)
self.root = root
pmd = Metadata()
pmd.excludes = ['root']
pmd.wrappers = dict(qname=repr)
self.__metadata__.__print__ = pmd
def resolve(self, definitions):
"""
Resolve named references to other WSDL objects.
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
pass
class NamedObject(WObject):
"""
A B{named} WSDL object.
@ivar name: The name of the object.
@type name: str
@ivar qname: The I{qualified} name of the object.
@type qname: (name, I{namespace-uri}).
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
WObject.__init__(self, root, definitions)
self.name = root.get('name')
self.qname = (self.name, definitions.tns[1])
pmd = self.__metadata__.__print__
pmd.wrappers['qname'] = repr
class Definitions(WObject):
"""
Represents the I{root} container of the WSDL objects as defined
by <wsdl:definitions/>
@ivar id: The object id.
@type id: str
@ivar options: An options dictionary.
@type options: L{options.Options}
@ivar url: The URL used to load the object.
@type url: str
@ivar tns: The target namespace for the WSDL.
@type tns: str
@ivar schema: The collective WSDL schema object.
@type schema: L{SchemaCollection}
@ivar children: The raw list of child objects.
@type children: [L{WObject},...]
@ivar imports: The list of L{Import} children.
@type imports: [L{Import},...]
@ivar messages: The dictionary of L{Message} children key'd by I{qname}
@type messages: [L{Message},...]
@ivar port_types: The dictionary of L{PortType} children key'd by I{qname}
@type port_types: [L{PortType},...]
@ivar bindings: The dictionary of L{Binding} children key'd by I{qname}
@type bindings: [L{Binding},...]
@ivar service: The service object.
@type service: L{Service}
"""
Tag = 'definitions'
def __init__(self, url, options):
"""
@param url: A URL to the WSDL.
@type url: str
@param options: An options dictionary.
@type options: L{options.Options}
"""
log.debug('reading wsdl at: %s ...', url)
reader = DocumentReader(options)
d = reader.open(url)
root = d.root()
WObject.__init__(self, root)
self.id = objid(self)
self.options = options
self.url = url
self.tns = self.mktns(root)
self.types = []
self.schema = None
self.children = []
self.imports = []
self.messages = {}
self.port_types = {}
self.bindings = {}
self.services = []
self.add_children(self.root)
self.children.sort()
pmd = self.__metadata__.__print__
pmd.excludes.append('children')
pmd.excludes.append('wsdl')
pmd.wrappers['schema'] = repr
self.open_imports()
self.resolve()
self.build_schema()
self.set_wrapped()
for s in self.services:
self.add_methods(s)
log.debug("wsdl at '%s' loaded:\n%s", url, self)
def mktns(self, root):
""" Get/create the target namespace """
tns = root.get('targetNamespace')
prefix = root.findPrefix(tns)
if prefix is None:
log.debug('warning: tns (%s), not mapped to prefix', tns)
prefix = 'tns'
return (prefix, tns)
def add_children(self, root):
""" Add child objects using the factory """
for c in root.getChildren(ns=wsdlns):
child = Factory.create(c, self)
if child is None: continue
self.children.append(child)
if isinstance(child, Import):
self.imports.append(child)
continue
if isinstance(child, Types):
self.types.append(child)
continue
if isinstance(child, Message):
self.messages[child.qname] = child
continue
if isinstance(child, PortType):
self.port_types[child.qname] = child
continue
if isinstance(child, Binding):
self.bindings[child.qname] = child
continue
if isinstance(child, Service):
self.services.append(child)
continue
def open_imports(self):
""" Import the I{imported} WSDLs. """
for imp in self.imports:
imp.load(self)
def resolve(self):
""" Tell all children to resolve themselves """
for c in self.children:
c.resolve(self)
def build_schema(self):
""" Process L{Types} objects and create the schema collection """
container = SchemaCollection(self)
for t in [t for t in self.types if t.local()]:
for root in t.contents():
schema = Schema(root, self.url, self.options, container)
container.add(schema)
if not len(container): # empty
root = Element.buildPath(self.root, 'types/schema')
schema = Schema(root, self.url, self.options, container)
container.add(schema)
self.schema = container.load(self.options)
for s in [t.schema() for t in self.types if t.imported()]:
self.schema.merge(s)
return self.schema
def add_methods(self, service):
""" Build method view for service """
bindings = {
'document/literal' : Document(self),
'rpc/literal' : RPC(self),
'rpc/encoded' : Encoded(self)
}
for p in service.ports:
binding = p.binding
ptype = p.binding.type
operations = p.binding.type.operations.values()
for name in [op.name for op in operations]:
m = Facade('Method')
m.name = name
m.location = p.location
m.binding = Facade('binding')
op = binding.operation(name)
m.soap = op.soap
key = '/'.join((op.soap.style, op.soap.input.body.use))
m.binding.input = bindings.get(key)
key = '/'.join((op.soap.style, op.soap.output.body.use))
m.binding.output = bindings.get(key)
op = ptype.operation(name)
p.methods[name] = m
def set_wrapped(self):
""" set (wrapped|bare) flag on messages """
for b in self.bindings.values():
for op in b.operations.values():
for body in (op.soap.input.body, op.soap.output.body):
body.wrapped = False
if len(body.parts) != 1:
continue
for p in body.parts:
if p.element is None:
continue
query = ElementQuery(p.element)
pt = query.execute(self.schema)
if pt is None:
raise TypeNotFound(query.ref)
resolved = pt.resolve()
if resolved.builtin():
continue
body.wrapped = True
def __getstate__(self):
nopickle = ('options',)
state = self.__dict__.copy()
for k in nopickle:
if k in state:
del state[k]
return state
def __repr__(self):
return 'Definitions (id=%s)' % self.id
class Import(WObject):
"""
Represents the <wsdl:import/>.
@ivar location: The value of the I{location} attribute.
@type location: str
@ivar ns: The value of the I{namespace} attribute.
@type ns: str
@ivar imported: The imported object.
@type imported: L{Definitions}
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
WObject.__init__(self, root, definitions)
self.location = root.get('location')
self.ns = root.get('namespace')
self.imported = None
pmd = self.__metadata__.__print__
pmd.wrappers['imported'] = repr
def load(self, definitions):
""" Load the object by opening the URL """
url = self.location
log.debug('importing (%s)', url)
if '://' not in url:
url = urljoin(definitions.url, url)
options = definitions.options
d = Definitions(url, options)
if d.root.match(Definitions.Tag, wsdlns):
self.import_definitions(definitions, d)
return
if d.root.match(Schema.Tag, Namespace.xsdns):
self.import_schema(definitions, d)
return
raise Exception('document at "%s" is unknown' % url)
def import_definitions(self, definitions, d):
""" import/merge wsdl definitions """
definitions.types += d.types
definitions.messages.update(d.messages)
definitions.port_types.update(d.port_types)
definitions.bindings.update(d.bindings)
self.imported = d
log.debug('imported (WSDL):\n%s', d)
def import_schema(self, definitions, d):
""" import schema as <types/> content """
if not len(definitions.types):
types = Types.create(definitions)
definitions.types.append(types)
else:
types = definitions.types[-1]
types.root.append(d.root)
log.debug('imported (XSD):\n%s', d.root)
def __gt__(self, other):
return False
class Types(WObject):
"""
Represents <types><schema/></types>.
"""
@classmethod
def create(cls, definitions):
root = Element('types', ns=wsdlns)
definitions.root.insert(root)
return Types(root, definitions)
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
WObject.__init__(self, root, definitions)
self.definitions = definitions
def contents(self):
return self.root.getChildren('schema', Namespace.xsdns)
def schema(self):
return self.definitions.schema
def local(self):
return ( self.definitions.schema is None )
def imported(self):
return ( not self.local() )
def __gt__(self, other):
return isinstance(other, Import)
class Part(NamedObject):
"""
Represents <message><part/></message>.
@ivar element: The value of the {element} attribute.
Stored as a I{qref} as converted by L{suds.xsd.qualify}.
@type element: str
@ivar type: The value of the {type} attribute.
Stored as a I{qref} as converted by L{suds.xsd.qualify}.
@type type: str
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
NamedObject.__init__(self, root, definitions)
pmd = Metadata()
pmd.wrappers = dict(element=repr, type=repr)
self.__metadata__.__print__ = pmd
tns = definitions.tns
self.element = self.__getref('element', tns)
self.type = self.__getref('type', tns)
def __getref(self, a, tns):
""" Get the qualified value of attribute named 'a'."""
s = self.root.get(a)
if s is None:
return s
else:
return qualify(s, self.root, tns)
class Message(NamedObject):
"""
Represents <message/>.
@ivar parts: A list of message parts.
@type parts: [I{Part},...]
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
NamedObject.__init__(self, root, definitions)
self.parts = []
for p in root.getChildren('part'):
part = Part(p, definitions)
self.parts.append(part)
def __gt__(self, other):
return isinstance(other, (Import, Types))
class PortType(NamedObject):
"""
Represents <portType/>.
@ivar operations: A list of contained operations.
@type operations: list
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
NamedObject.__init__(self, root, definitions)
self.operations = {}
for c in root.getChildren('operation'):
op = Facade('Operation')
op.name = c.get('name')
op.tns = definitions.tns
input = c.getChild('input')
if input is None:
op.input = None
else:
op.input = input.get('message')
output = c.getChild('output')
if output is None:
op.output = None
else:
op.output = output.get('message')
faults = []
for fault in c.getChildren('fault'):
f = Facade('Fault')
f.name = fault.get('name')
f.message = fault.get('message')
faults.append(f)
op.faults = faults
self.operations[op.name] = op
def resolve(self, definitions):
"""
Resolve named references to other WSDL objects.
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
for op in self.operations.values():
if op.input is None:
op.input = Message(Element('no-input'), definitions)
else:
qref = qualify(op.input, self.root, definitions.tns)
msg = definitions.messages.get(qref)
if msg is None:
raise Exception("msg '%s', not-found" % op.input)
else:
op.input = msg
if op.output is None:
op.output = Message(Element('no-output'), definitions)
else:
qref = qualify(op.output, self.root, definitions.tns)
msg = definitions.messages.get(qref)
if msg is None:
raise Exception("msg '%s', not-found" % op.output)
else:
op.output = msg
for f in op.faults:
qref = qualify(f.message, self.root, definitions.tns)
msg = definitions.messages.get(qref)
if msg is None:
raise Exception, "msg '%s', not-found" % f.message
f.message = msg
def operation(self, name):
"""
Shortcut used to get a contained operation by name.
@param name: An operation name.
@type name: str
@return: The named operation.
@rtype: Operation
@raise L{MethodNotFound}: When not found.
"""
try:
return self.operations[name]
except Exception, e:
raise MethodNotFound(name)
def __gt__(self, other):
return isinstance(other, (Import, Types, Message))
class Binding(NamedObject):
"""
Represents <binding/>
@ivar operations: A list of contained operations.
@type operations: list
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
NamedObject.__init__(self, root, definitions)
self.operations = {}
self.type = root.get('type')
sr = self.soaproot()
if sr is None:
self.soap = None
log.debug('binding: "%s" not a soap binding', self.name)
return
soap = Facade('soap')
self.soap = soap
self.soap.style = sr.get('style', default='document')
self.add_operations(self.root, definitions)
def soaproot(self):
""" get the soap:binding """
for ns in (soapns, soap12ns):
sr = self.root.getChild('binding', ns=ns)
if sr is not None:
return sr
return None
def add_operations(self, root, definitions):
""" Add <operation/> children """
dsop = Element('operation', ns=soapns)
for c in root.getChildren('operation'):
op = Facade('Operation')
op.name = c.get('name')
sop = c.getChild('operation', default=dsop)
soap = Facade('soap')
soap.action = '"%s"' % sop.get('soapAction', default='')
soap.style = sop.get('style', default=self.soap.style)
soap.input = Facade('Input')
soap.input.body = Facade('Body')
soap.input.headers = []
soap.output = Facade('Output')
soap.output.body = Facade('Body')
soap.output.headers = []
op.soap = soap
input = c.getChild('input')
if input is None:
input = Element('input', ns=wsdlns)
body = input.getChild('body')
self.body(definitions, soap.input.body, body)
for header in input.getChildren('header'):
self.header(definitions, soap.input, header)
output = c.getChild('output')
if output is None:
output = Element('output', ns=wsdlns)
body = output.getChild('body')
self.body(definitions, soap.output.body, body)
for header in output.getChildren('header'):
self.header(definitions, soap.output, header)
faults = []
for fault in c.getChildren('fault'):
sf = fault.getChild('fault')
if sf is None:
continue
fn = fault.get('name')
f = Facade('Fault')
f.name = sf.get('name', default=fn)
f.use = sf.get('use', default='literal')
faults.append(f)
soap.faults = faults
self.operations[op.name] = op
def body(self, definitions, body, root):
""" add the input/output body properties """
if root is None:
body.use = 'literal'
body.namespace = definitions.tns
body.parts = ()
return
parts = root.get('parts')
if parts is None:
body.parts = ()
else:
body.parts = re.split('[\s,]', parts)
body.use = root.get('use', default='literal')
ns = root.get('namespace')
if ns is None:
body.namespace = definitions.tns
else:
prefix = root.findPrefix(ns, 'b0')
body.namespace = (prefix, ns)
def header(self, definitions, parent, root):
""" add the input/output header properties """
if root is None:
return
header = Facade('Header')
parent.headers.append(header)
header.use = root.get('use', default='literal')
ns = root.get('namespace')
if ns is None:
header.namespace = definitions.tns
else:
prefix = root.findPrefix(ns, 'h0')
header.namespace = (prefix, ns)
msg = root.get('message')
if msg is not None:
header.message = msg
part = root.get('part')
if part is not None:
header.part = part
def resolve(self, definitions):
"""
Resolve named references to other WSDL objects. This includes
cross-linking information (from) the portType (to) the I{soap}
protocol information on the binding for each operation.
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
self.resolveport(definitions)
for op in self.operations.values():
self.resolvesoapbody(definitions, op)
self.resolveheaders(definitions, op)
self.resolvefaults(definitions, op)
def resolveport(self, definitions):
"""
Resolve port_type reference.
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
ref = qualify(self.type, self.root, definitions.tns)
port_type = definitions.port_types.get(ref)
if port_type is None:
raise Exception("portType '%s', not-found" % self.type)
else:
self.type = port_type
def resolvesoapbody(self, definitions, op):
"""
Resolve soap body I{message} parts by
cross-referencing with operation defined in port type.
@param definitions: A definitions object.
@type definitions: L{Definitions}
@param op: An I{operation} object.
@type op: I{operation}
"""
ptop = self.type.operation(op.name)
if ptop is None:
raise Exception, \
"operation '%s' not defined in portType" % op.name
soap = op.soap
parts = soap.input.body.parts
if len(parts):
pts = []
for p in ptop.input.parts:
if p.name in parts:
pts.append(p)
soap.input.body.parts = pts
else:
soap.input.body.parts = ptop.input.parts
parts = soap.output.body.parts
if len(parts):
pts = []
for p in ptop.output.parts:
if p.name in parts:
pts.append(p)
soap.output.body.parts = pts
else:
soap.output.body.parts = ptop.output.parts
def resolveheaders(self, definitions, op):
"""
Resolve soap header I{message} references.
@param definitions: A definitions object.
@type definitions: L{Definitions}
@param op: An I{operation} object.
@type op: I{operation}
"""
soap = op.soap
headers = soap.input.headers + soap.output.headers
for header in headers:
mn = header.message
ref = qualify(mn, self.root, definitions.tns)
message = definitions.messages.get(ref)
if message is None:
raise Exception, "message'%s', not-found" % mn
pn = header.part
for p in message.parts:
if p.name == pn:
header.part = p
break
if pn == header.part:
raise Exception, \
"message '%s' has not part named '%s'" % (ref, pn)
def resolvefaults(self, definitions, op):
"""
Resolve soap fault I{message} references by
cross-referencing with operation defined in port type.
@param definitions: A definitions object.
@type definitions: L{Definitions}
@param op: An I{operation} object.
@type op: I{operation}
"""
ptop = self.type.operation(op.name)
if ptop is None:
raise Exception, \
"operation '%s' not defined in portType" % op.name
soap = op.soap
for fault in soap.faults:
for f in ptop.faults:
if f.name == fault.name:
fault.parts = f.message.parts
continue
if hasattr(fault, 'parts'):
continue
raise Exception, \
"fault '%s' not defined in portType '%s'" % (fault.name, self.type.name)
def operation(self, name):
"""
Shortcut used to get a contained operation by name.
@param name: An operation name.
@type name: str
@return: The named operation.
@rtype: Operation
@raise L{MethodNotFound}: When not found.
"""
try:
return self.operations[name]
except:
raise MethodNotFound(name)
def __gt__(self, other):
return ( not isinstance(other, Service) )
class Port(NamedObject):
"""
Represents a service port.
@ivar service: A service.
@type service: L{Service}
@ivar binding: A binding name.
@type binding: str
@ivar location: The service location (url).
@type location: str
"""
def __init__(self, root, definitions, service):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
@param service: A service object.
@type service: L{Service}
"""
NamedObject.__init__(self, root, definitions)
self.__service = service
self.binding = root.get('binding')
address = root.getChild('address')
if address is None:
self.location = None
else:
self.location = address.get('location').encode('utf-8')
self.methods = {}
def method(self, name):
"""
Get a method defined in this portType by name.
@param name: A method name.
@type name: str
@return: The requested method object.
@rtype: I{Method}
"""
return self.methods.get(name)
class Service(NamedObject):
"""
Represents <service/>.
@ivar port: The contained ports.
@type port: [Port,..]
@ivar methods: The contained methods for all ports.
@type methods: [Method,..]
"""
def __init__(self, root, definitions):
"""
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
NamedObject.__init__(self, root, definitions)
self.ports = []
for p in root.getChildren('port'):
port = Port(p, definitions, self)
self.ports.append(port)
def port(self, name):
"""
Locate a port by name.
@param name: A port name.
@type name: str
@return: The port object.
@rtype: L{Port}
"""
for p in self.ports:
if p.name == name:
return p
return None
def setlocation(self, url, names=None):
"""
Override the invocation location (url) for service method.
@param url: A url location.
@type url: A url.
@param names: A list of method names. None=ALL
@type names: [str,..]
"""
for p in self.ports:
for m in p.methods.values():
if names is None or m.name in names:
m.location = url
def resolve(self, definitions):
"""
Resolve named references to other WSDL objects.
Ports without soap bindings are discarded.
@param definitions: A definitions object.
@type definitions: L{Definitions}
"""
filtered = []
for p in self.ports:
ref = qualify(p.binding, self.root, definitions.tns)
binding = definitions.bindings.get(ref)
if binding is None:
raise Exception("binding '%s', not-found" % p.binding)
if binding.soap is None:
log.debug('binding "%s" - not a soap, discarded', binding.name)
continue
p.binding = binding
filtered.append(p)
self.ports = filtered
def __gt__(self, other):
return True
class Factory:
"""
Simple WSDL object factory.
@cvar tags: Dictionary of tag->constructor mappings.
@type tags: dict
"""
tags =\
{
'import' : Import,
'types' : Types,
'message' : Message,
'portType' : PortType,
'binding' : Binding,
'service' : Service,
}
@classmethod
def create(cls, root, definitions):
"""
Create an object based on the root tag name.
@param root: An XML root element.
@type root: L{Element}
@param definitions: A definitions object.
@type definitions: L{Definitions}
@return: The created object.
@rtype: L{WObject}
"""
fn = cls.tags.get(root.name)
if fn is not None:
return fn(root, definitions)
else:
return None

Some files were not shown because too many files have changed in this diff Show More